From b6be3b43de5c6dd845dda4f601f7ebe179b6c8a9 Mon Sep 17 00:00:00 2001
From: sylvain <slaizet@workstation>
Date: Wed, 16 Nov 2022 13:05:30 +0000
Subject: [PATCH 001/436] just removing some printing to screen when doing io
 tests

---
 examples/io_test/io_plane_test.f90 | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 31f3d366..42a6d221 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -76,11 +76,13 @@ program io_plane_test
            m=m+1
         end do
      end do
-     write(*,*) ' '
-     write(*,'(15I5)') int(work)
+!     write(*,*) ' '
+!     write(*,'(15I5)') int(work)
      close(10)
      deallocate(work)
 
+     write(*,*) 'passed self test x-plane'
+
      ! Y-plane
      allocate(work(nx,1,nz))
      open(10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
@@ -92,11 +94,13 @@ program io_plane_test
            m=m+1
         end do
      end do
-     write(*,*) ' '
-     write(*,'(15I5)') int(work)
+!     write(*,*) ' '
+!     write(*,'(15I5)') int(work)
      close(10)
      deallocate(work)
 
+     write(*,*) 'passed self test y-plane'
+
      ! Z-plane
      allocate(work(nx,ny,1))
      open(10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
@@ -108,11 +112,13 @@ program io_plane_test
            m=m+1
         end do
      end do
-     write(*,*) ' '
-     write(*,'(15I5)') int(work)
+!     write(*,*) ' '
+!     write(*,'(15I5)') int(work)
      close(10)
      deallocate(work)
 
+     write(*,*) 'passed self test z-plane'
+
   end if
 
   call decomp_2d_finalize 

From 566326b01b50ec44eedcb1d44616cf03e9f18f00 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 16 Nov 2022 16:32:45 +0100
Subject: [PATCH 002/436] Preliminary fix to avoid bug when the null() pointer
 is provided to decomp_info_init

---
 src/fft_common.f90    | 4 +++-
 src/fft_fftw3_f03.f90 | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 62fef974..95cb528e 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -91,6 +91,7 @@ subroutine fft_init_general(pencil, nx, ny, nz)
 integer, intent(IN) :: pencil
 integer, intent(IN) :: nx, ny, nz
 
+type(decomp_info), target :: tmp
 integer :: status, errorcode
 
 #ifdef PROFILER
@@ -120,7 +121,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
 if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
 ph=>decomp_main
 else
-call decomp_info_init(nx, ny, nz, ph)
+call decomp_info_init(nx, ny, nz, tmp)
+ph=>tmp
 endif
 if (format==PHYSICAL_IN_X) then
 call decomp_info_init(nx/2+1, ny, nz, sp)
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 6dab0f44..3b724aee 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -116,6 +116,7 @@ subroutine fft_init_general(pencil, nx, ny, nz)
     integer, intent(IN) :: pencil
     integer, intent(IN) :: nx, ny, nz
 
+    type(decomp_info), target :: tmp
     integer :: errorcode
     integer(C_SIZE_T) :: sz
 
@@ -146,7 +147,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
     if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
        ph => decomp_main
     else
-       call decomp_info_init(nx, ny, nz, ph)
+       call decomp_info_init(nx, ny, nz, tmp)
+       ph => tmp
     endif
     if (format==PHYSICAL_IN_X) then
        call decomp_info_init(nx/2+1, ny, nz, sp)

From cfe45f1408916dfbd514391282b62fdc6453f74b Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 16 Nov 2022 16:43:24 +0100
Subject: [PATCH 003/436] Fix 566326b01b50ec44eedcb1d44616cf03e9f18f00

---
 src/fft_common.f90    | 11 +++++------
 src/fft_fftw3_f03.f90 | 11 +++++------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 95cb528e..f85266e3 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -32,6 +32,7 @@
 
 ! Decomposition objects
 TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
+TYPE(DECOMP_INFO), target, save :: ph_opt       ! needed when fft grid /= main grid
 TYPE(DECOMP_INFO), save :: sp  ! spectral space
 
 ! Workspace to store the intermediate Y-pencil data
@@ -91,7 +92,6 @@ subroutine fft_init_general(pencil, nx, ny, nz)
 integer, intent(IN) :: pencil
 integer, intent(IN) :: nx, ny, nz
 
-type(decomp_info), target :: tmp
 integer :: status, errorcode
 
 #ifdef PROFILER
@@ -121,8 +121,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
 if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
 ph=>decomp_main
 else
-call decomp_info_init(nx, ny, nz, tmp)
-ph=>tmp
+call decomp_info_init(nx, ny, nz, ph_opt)
+ph=>ph_opt
 endif
 if (format==PHYSICAL_IN_X) then
 call decomp_info_init(nx/2+1, ny, nz, sp)
@@ -166,11 +166,10 @@ subroutine decomp_2d_fft_finalize
 if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
-nullify(ph)
-else
+if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
 call decomp_info_finalize(ph)
 endif
+nullify(ph)
 call decomp_info_finalize(sp)
 
 if (allocated(wk2_c2c)) deallocate(wk2_c2c)
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 3b724aee..0c20e469 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -57,6 +57,7 @@ module decomp_2d_fft
 
   ! Decomposition objects
   TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
+  TYPE(DECOMP_INFO), target, save :: ph_opt       ! needed when fft grid /= main grid
   TYPE(DECOMP_INFO), save :: sp  ! spectral space
 
   ! Workspace to store the intermediate Y-pencil data
@@ -116,7 +117,6 @@ subroutine fft_init_general(pencil, nx, ny, nz)
     integer, intent(IN) :: pencil
     integer, intent(IN) :: nx, ny, nz
 
-    type(decomp_info), target :: tmp
     integer :: errorcode
     integer(C_SIZE_T) :: sz
 
@@ -147,8 +147,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
     if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
        ph => decomp_main
     else
-       call decomp_info_init(nx, ny, nz, tmp)
-       ph => tmp
+       call decomp_info_init(nx, ny, nz, ph_opt)
+       ph => ph_opt
     endif
     if (format==PHYSICAL_IN_X) then
        call decomp_info_init(nx/2+1, ny, nz, sp)
@@ -198,11 +198,10 @@ subroutine decomp_2d_fft_finalize
     if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-    if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
-      nullify(ph)
-    else
+    if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
       call decomp_info_finalize(ph)
     endif
+    nullify(ph)
     call decomp_info_finalize(sp)
 
     call fftw_free(wk2_c2c_p)

From d9e62cb780da69554e00c253b8d7bd5fb9c0351f Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 16 Nov 2022 19:34:36 +0100
Subject: [PATCH 004/436] No need for extra target object ph_opt

---
 src/fft_common.f90    | 6 +++---
 src/fft_fftw3_f03.f90 | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index f85266e3..0a461d98 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -32,7 +32,6 @@
 
 ! Decomposition objects
 TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
-TYPE(DECOMP_INFO), target, save :: ph_opt       ! needed when fft grid /= main grid
 TYPE(DECOMP_INFO), save :: sp  ! spectral space
 
 ! Workspace to store the intermediate Y-pencil data
@@ -121,8 +120,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
 if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
 ph=>decomp_main
 else
-call decomp_info_init(nx, ny, nz, ph_opt)
-ph=>ph_opt
+if (.not.associated(ph)) allocate(ph)
+call decomp_info_init(nx, ny, nz, ph)
 endif
 if (format==PHYSICAL_IN_X) then
 call decomp_info_init(nx/2+1, ny, nz, sp)
@@ -168,6 +167,7 @@ subroutine decomp_2d_fft_finalize
 
 if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
 call decomp_info_finalize(ph)
+deallocate(ph)
 endif
 nullify(ph)
 call decomp_info_finalize(sp)
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 0c20e469..ace2803f 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -57,7 +57,6 @@ module decomp_2d_fft
 
   ! Decomposition objects
   TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
-  TYPE(DECOMP_INFO), target, save :: ph_opt       ! needed when fft grid /= main grid
   TYPE(DECOMP_INFO), save :: sp  ! spectral space
 
   ! Workspace to store the intermediate Y-pencil data
@@ -147,8 +146,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
     if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
        ph => decomp_main
     else
-       call decomp_info_init(nx, ny, nz, ph_opt)
-       ph => ph_opt
+       if (.not.associated(ph)) allocate(ph)
+       call decomp_info_init(nx, ny, nz, ph)
     endif
     if (format==PHYSICAL_IN_X) then
        call decomp_info_init(nx/2+1, ny, nz, sp)
@@ -200,6 +199,7 @@ subroutine decomp_2d_fft_finalize
 
     if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
       call decomp_info_finalize(ph)
+      deallocate(ph)
     endif
     nullify(ph)
     call decomp_info_finalize(sp)

From 3bf5727d51bf1bbbc73ff5321088c278a63998f3 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 20 Nov 2022 12:35:13 +0100
Subject: [PATCH 005/436] Add a test with FFT grid /= main decomp_2d grid

---
 examples/Makefile                      |   2 +-
 examples/fft_physical_x/.gitignore     |   1 +
 examples/fft_physical_x/Makefile       |  15 ++-
 examples/fft_physical_x/fft_grid_x.f90 | 130 +++++++++++++++++++++++++
 src/fft_common.f90                     |  40 +++++++-
 src/fft_fftw3_f03.f90                  |  38 +++++++-
 6 files changed, 215 insertions(+), 11 deletions(-)
 create mode 100644 examples/fft_physical_x/fft_grid_x.f90

diff --git a/examples/Makefile b/examples/Makefile
index 195a6e4a..40cbd0f7 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -9,7 +9,7 @@ test2d:
 fft_physical_z:
 	$(MAKE) -C $@ $@
 fft_physical_x:
-	$(MAKE) -C $@ $@
+	$(MAKE) -C $@ all
 halo_test:
 	$(MAKE) -C $@ $@
 io_test:
diff --git a/examples/fft_physical_x/.gitignore b/examples/fft_physical_x/.gitignore
index d69230b2..d32650f6 100644
--- a/examples/fft_physical_x/.gitignore
+++ b/examples/fft_physical_x/.gitignore
@@ -1 +1,2 @@
 fft_physical_x
+fft_grid_x
diff --git a/examples/fft_physical_x/Makefile b/examples/fft_physical_x/Makefile
index 746c1c63..ad417efc 100644
--- a/examples/fft_physical_x/Makefile
+++ b/examples/fft_physical_x/Makefile
@@ -5,27 +5,32 @@ FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
 FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
 LIBS = -L../../ -l$(LIBDECOMP) $(LIBFFT) $(LFLAGS)
 
-OBJ = fft_physical_x.o
-
 NP ?= 1
 MPIRUN ?= mpirun
 
-fft_physical_x: $(OBJ)
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $(OBJ) $(LIBS)
+all: fft_physical_x fft_grid_x
+
+fft_physical_x: fft_physical_x.o
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
+
+fft_grid_x: fft_grid_x.o
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
 
 ifeq ($(PARAMOD),gpu)
 check:
 	$(MPIRUN) -n $(NP) ./bind.sh ./fft_physical_x
+	$(MPIRUN) -n $(NP) ./bind.sh ./fft_grid_x
 else
 check:
 	$(MPIRUN) -n $(NP) ./fft_physical_x
+	$(MPIRUN) -n $(NP) ./fft_grid_x
 endif
 
 mem_leak:
 	valgrind --leak-check=full --show-leak-kinds=all $(MPIRUN) -n 1 ./fft_physical_x 1 1
 
 clean:
-	rm -f *.o fft_physical_x *.log
+	rm -f *.o fft_physical_x fft_grid_x *.log
 
 %.o : %.f90
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
new file mode 100644
index 00000000..5b424138
--- /dev/null
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -0,0 +1,130 @@
+program fft_physical_x
+
+  use decomp_2d
+  use decomp_2d_fft
+  use MPI
+#if defined(_GPU) 
+  use cudafor
+  use cufft
+  use openacc 
+#endif
+
+  implicit none
+  
+  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
+  integer :: nx, ny, nz
+  integer :: p_row=0, p_col=0
+  integer :: resize_domain
+  integer :: nranks_tot
+  
+  integer, parameter :: ntest = 10  ! repeat test this times
+ 
+  type(decomp_info), pointer :: ph=>null()
+  complex(mytype), allocatable, dimension(:,:,:) :: in, out
+  
+  real(mytype) :: dr,di, error, err_all, n1,flops
+  integer :: ierror, i,j,k,m
+  real(mytype) :: t1, t2, t3 ,t4
+  
+  call MPI_INIT(ierror)
+  ! To resize the domain we need to know global number of ranks
+  ! This operation is also done as part of decomp_2d_init
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+  resize_domain = int(nranks_tot/4)+1
+  nx = nx_base*resize_domain
+  ny = ny_base*resize_domain
+  nz = nz_base*resize_domain
+  call decomp_2d_init(nx+1,ny+1,nz+1,p_row,p_col)
+
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  ! Test the c2c interface
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+  call decomp_2d_fft_init(PHYSICAL_IN_X, nx, ny, nz) ! force the default x pencil
+  ph => decomp_2d_fft_get_ph()
+  !  input is X-pencil data
+  ! output is Z-pencil data
+  call alloc_x(in, ph, .true.)
+  call alloc_z(out, ph, .true.)
+  ! initilise input
+  do k=ph%xst(3),ph%xen(3)
+     do j=ph%xst(2),ph%xen(2)
+        do i=ph%xst(1),ph%xen(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           di = dr
+           in(i,j,k) = cmplx(dr,di,mytype)
+        end do
+     end do
+  end do
+  
+  t2 = 0._mytype
+  t4 = 0._mytype
+  do m=1,ntest
+     
+     ! forward FFT
+     t1 = MPI_WTIME()
+     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+     t2 = t2 + MPI_WTIME() - t1
+
+     ! inverse FFT
+     t3 = MPI_WTIME()
+     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+     t4 = t4 + MPI_WTIME() - t3
+  
+     ! normalisation - note 2DECOMP&FFT doesn't normalise
+     !$acc kernels
+     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
+     !$acc end kernels
+
+  end do
+#if defined(_GPU)
+  ierror = cudaDeviceSynchronize()
+#endif
+  
+  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t1 = t1 / real(nproc,mytype)
+  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t3 = t3 / real(nproc,mytype)
+  
+  ! checking accuracy
+  error = 0._mytype
+  do k=ph%xst(3),ph%xen(3) 
+     do j=ph%xst(2),ph%xen(2)
+        do i=ph%xst(1),ph%xen(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           di = dr
+           dr = dr - real(in(i,j,k),mytype)
+           di = di - aimag(in(i,j,k))
+           error = error + sqrt(dr*dr + di*di)
+        end do
+     end do
+  end do
+  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
+  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
+
+  if (nrank==0) then
+     write(*,*) '===== c2c interface ====='
+     write(*,*) 'error / mesh point: ', err_all 
+     write(*,*) 'time (sec): ', t1,t3
+     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
+     n1 = n1 ** (1._mytype/3._mytype)
+     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
+     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+     flops = flops * 3._mytype * n1**2  
+     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
+     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
+  end if
+  
+  deallocate(in,out)
+  nullify(ph)
+  call decomp_2d_fft_finalize
+  call decomp_2d_finalize
+  call MPI_FINALIZE(ierror)
+
+end program fft_physical_x
+
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 62fef974..bcd1bd8c 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -32,7 +32,7 @@
 
 ! Decomposition objects
 TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
-TYPE(DECOMP_INFO), save :: sp  ! spectral space
+TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
 
 ! Workspace to store the intermediate Y-pencil data
 ! *** TODO: investigate how to use only one workspace array
@@ -40,7 +40,8 @@
 complex(mytype), allocatable, dimension(:,:,:) :: wk13
 
 public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-decomp_2d_fft_finalize, decomp_2d_fft_get_size
+decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
 
 ! Declare generic interfaces to handle different inputs
 
@@ -56,7 +57,6 @@
 module procedure fft_3d_c2r
 end interface
 
-
 contains
 
 
@@ -209,3 +209,37 @@ subroutine decomp_2d_fft_get_size(istart, iend, isize)
 
 return
 end subroutine decomp_2d_fft_get_size
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+! Return a pointer to the decomp_info object ph
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+function decomp_2d_fft_get_ph()
+
+implicit none
+
+type(decomp_info), pointer :: decomp_2d_fft_get_ph
+
+if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
+                                              __LINE__, &
+                                             -1, &
+                                             'FFT library must be initialised first')
+decomp_2d_fft_get_ph => ph
+
+end function decomp_2d_fft_get_ph
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+! Return a pointer to the decomp_info object sp
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+function decomp_2d_fft_get_sp()
+
+implicit none
+
+type(decomp_info), pointer :: decomp_2d_fft_get_sp
+
+if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
+                                              __LINE__, &
+                                             -1, &
+                                             'FFT library must be initialised first')
+decomp_2d_fft_get_sp => sp
+
+end function decomp_2d_fft_get_sp
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 6dab0f44..a0bdb3a6 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -57,7 +57,7 @@ module decomp_2d_fft
 
   ! Decomposition objects
   TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
-  TYPE(DECOMP_INFO), save :: sp  ! spectral space
+  TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
 
   ! Workspace to store the intermediate Y-pencil data
   ! *** TODO: investigate how to use only one workspace array
@@ -65,7 +65,8 @@ module decomp_2d_fft
   type(C_PTR) :: wk2_c2c_p, wk2_r2c_p, wk13_p
 
   public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-       decomp_2d_fft_finalize, decomp_2d_fft_get_size
+       decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+       decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
   
   ! Declare generic interfaces to handle different inputs
   
@@ -242,6 +243,39 @@ subroutine decomp_2d_fft_get_size(istart, iend, isize)
     return
   end subroutine decomp_2d_fft_get_size
 
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  ! Return a pointer to the decomp_info object ph
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  function decomp_2d_fft_get_ph()
+
+    implicit none
+
+    type(decomp_info), pointer :: decomp_2d_fft_get_ph
+
+    if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  -1, &
+                                                  'FFT library must be initialised first')
+    decomp_2d_fft_get_ph => ph
+
+  end function decomp_2d_fft_get_ph
+
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  ! Return a pointer to the decomp_info object sp
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  function decomp_2d_fft_get_sp()
+
+    implicit none
+
+    type(decomp_info), pointer :: decomp_2d_fft_get_sp
+
+    if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  -1, &
+                                                  'FFT library must be initialised first')
+    decomp_2d_fft_get_sp => sp
+
+  end function decomp_2d_fft_get_sp
 
   ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
   subroutine c2c_1m_x_plan(plan1, decomp, isign)

From 2ec060d17e6025b42085e45d75ee4310322bafb9 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Sun, 20 Nov 2022 19:18:19 +0100
Subject: [PATCH 006/436] CI : fix mpich build

---
 .github/workflows/Build_all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index ba4915ef..0f4a509d 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -47,7 +47,7 @@ jobs:
 
     # Runs a set of commands using the runners shell
     - name: Compile
-      run: make BUILD=dev && make examples
+      run: make BUILD=debug && make examples
 
 ###  bld_nvidia:
 ###    # The type of runner that the job will run on

From 7200ace531f488b0ebb70362b44ad1176339c04a Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Tue, 22 Nov 2022 13:20:11 +0100
Subject: [PATCH 007/436] Update comments

---
 src/fft_common.f90    | 4 ++++
 src/fft_fftw3_f03.f90 | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index bcd1bd8c..b70bc295 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -212,6 +212,8 @@ end subroutine decomp_2d_fft_get_size
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Return a pointer to the decomp_info object ph
+!
+! The caller should not apply decomp_info_finalize on the pointer
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 function decomp_2d_fft_get_ph()
 
@@ -229,6 +231,8 @@ end function decomp_2d_fft_get_ph
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Return a pointer to the decomp_info object sp
+!
+! The caller should not apply decomp_info_finalize on the pointer
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 function decomp_2d_fft_get_sp()
 
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index a0bdb3a6..e95e3620 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -245,6 +245,8 @@ end subroutine decomp_2d_fft_get_size
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   ! Return a pointer to the decomp_info object ph
+  !
+  ! The caller should not apply decomp_info_finalize on the pointer
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   function decomp_2d_fft_get_ph()
 
@@ -262,6 +264,8 @@ end function decomp_2d_fft_get_ph
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   ! Return a pointer to the decomp_info object sp
+  !
+  ! The caller should not apply decomp_info_finalize on the pointer
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   function decomp_2d_fft_get_sp()
 

From 1bb90565fc23a3f123ee9ce3111fa219e25d4889 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 29 Nov 2022 09:22:09 +0000
Subject: [PATCH 008/436] Explain parallel configuration of examples

---
 README.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 43a9115a..80facf1c 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,13 @@ make check
 ```
 which will (re)build 2decomp&fft as necessary.
 
-**TODO** Convert examples to tests and automate running them
+By default this will run the tests/examples on a single rank, launched with ``mpirun``.
+The number of ranks can be controlled by setting the ``NP`` variable and the launcher is overridden by
+setting ``MPIRUN``, for example to run tests on 4 ranks using an ``mpiexec`` launcher installed at
+``${HOME}/bin/mpiexec`` the command would be
+```
+make NP=4 MPIRUN=${HOME}/bin/mpiexec check
+```
 
 ## GPU compilation
 

From 30aeb0f9fd15a396b3457b954c4a3b3b31f4d5ec Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 29 Nov 2022 17:36:01 +0100
Subject: [PATCH 009/436] More options for the log output (#102)

* More options for the log output

* Remove the enum

* Apply suggestions from code review

Co-authored-by: Pedro Costa <p.simoes.costa@gmail.com>

* Update README with D2D_LOG_X values

Co-authored-by: Pedro Costa <p.simoes.costa@gmail.com>
Co-authored-by: Paul <ptb0890@gmail.com>
---
 README.md                  | 19 +++++++++++++++++++
 src/decomp_2d.f90          | 20 ++++++++++++++++++++
 src/decomp_2d_init_fin.f90 | 12 ++++--------
 src/log.f90                | 12 ++++--------
 4 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 43a9115a..23ffed11 100644
--- a/README.md
+++ b/README.md
@@ -64,6 +64,25 @@ Profiling can be activated in the Makefile. Set the variable `PROFILER` to one o
 3. Profile FFT operations (default : true)
 4. Profile decomp_2d init / fin subroutines (default : true)
 
+## Miscellaneous
+
+### Print the log to a file or to stdout
+
+Before calling `decomp_2d_init`, the external code can modify the variable `decomp_log` to change the output for the log. The expected value is defined by the integer constants
+```
+integer, parameter, public :: D2D_LOG_QUIET = 0       ! No logging output
+integer, parameter, public :: D2D_LOG_STDOUT = 1      ! Root rank logs output to stdout
+integer, parameter, public :: D2D_LOG_TOFILE = 2      ! Root rank logs output to the file "decomp_2d_setup.log"
+integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3 ! All ranks log output to a dedicated file
+```
+exported by the `decomp_2d` module.
+Although their values are shown here, users should not rely on these and are recommended to prefer to use the named variables `D2D_LOG_QUIET`, etc. instead.
+The default value used is `D2D_LOG_TOFILE` for the default build and `D2D_LOG_TOFILE_FULL` for a debug build.
+
+### Change the debug level for debug builds
+
+Before calling `decomp_2d_init`, the external code can modify the variable `decomp_debug` to change the debug level. The user can also modify this variable using the environment variable `DECOMP_2D_DEBUG`. Please note that the environment variable is processed only for debug builds. The expected value for the variable `decomp_debug` is some integer between 0 and 6, bounds included.
+
 ## Optional dependencies
 
 ### FFTW
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index e1205aab..11953900 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -74,6 +74,26 @@ module decomp_2d
   ! flags for periodic condition in three dimensions
   logical, save :: periodic_x, periodic_y, periodic_z
 
+  !
+  ! Output for the log can be changed by the external code before calling decomp_2d_init
+  !
+  !    0 => No log output
+  !    1 => Master rank log output to stdout
+  !    2 => Master rank log output to the file "decomp_2d_setup.log"
+  !    3 => All ranks log output to a dedicated file
+  !
+  ! The default value is 2 (3 for debug builds)
+  !
+  integer, parameter, public :: D2D_LOG_QUIET = 0
+  integer, parameter, public :: D2D_LOG_STDOUT = 1
+  integer, parameter, public :: D2D_LOG_TOFILE = 2
+  integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
+#ifdef DEBUG
+  integer, public, save :: decomp_log = D2D_LOG_TOFILE_FULL
+#else
+  integer, public, save :: decomp_log = D2D_LOG_TOFILE
+#endif
+
   !
   ! Debug level can be changed by the external code before calling decomp_2d_init
   !
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index fda17fd0..9bae1008 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -37,9 +37,7 @@ subroutine decomp_2d_init_ref(nx,ny,nz,p_row,p_col,periodic_bc,comm)
     integer :: cuda_stat
     type(ncclResult) :: nccl_stat
 #endif
-#ifdef DEBUG
     character(len=7) fname ! Sufficient for up to O(1M) ranks
-#endif
 
 #ifdef PROFILER
     ! Prepare the profiler if it was not already prepared
@@ -209,17 +207,15 @@ subroutine decomp_2d_init_ref(nx,ny,nz,p_row,p_col,periodic_bc,comm)
     !
     ! Select the IO unit for decomp_2d setup
     !
-#ifdef DEBUG
-    write(fname, "(I0)") nrank ! Adapt to magnitude of nrank
-    open(newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
-#else
-    if (nrank == 0) then
+    if (decomp_log == D2D_LOG_TOFILE_FULL) then
+       write(fname, "(I0)") nrank ! Adapt to magnitude of nrank
+       open(newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
+    elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
        open(newunit=iounit, file="decomp_2d_setup.log", iostat=ierror)
     else
        iounit = output_unit
        ierror = 0
     endif
-#endif
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
     !
     ! Print the decomp_2d setup
diff --git a/src/log.f90 b/src/log.f90
index 3581ea7c..866cd97e 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -35,14 +35,10 @@ module subroutine d2d_listing(given_io_unit)
     character(len=64) :: fname
 #endif
 
-    !
-    ! Default : only rank 0 will print a listing
-    !
-    ! In DEBUG mode, all ranks will print a listing
-    !
-#ifndef DEBUG
-    if (nrank /= 0) return
-#endif
+    ! Output log if needed
+    if (decomp_log == D2D_LOG_QUIET) return
+    if (decomp_log == D2D_LOG_STDOUT .and. nrank /= 0) return
+    if (decomp_log == D2D_LOG_TOFILE .and. nrank /= 0) return
 
     ! If no IO unit provided, use stdout
     if (present(given_io_unit)) then

From f28ef102d9994cee96d779569ab39cb87aecf276 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Wed, 30 Nov 2022 10:25:31 +0100
Subject: [PATCH 010/436] Merge main (#104)

* Preliminary fix to avoid bug when the null() pointer is provided to decomp_info_init

* Fix 566326b01b50ec44eedcb1d44616cf03e9f18f00

* No need for extra target object ph_opt

* CI : fix mpich build

* Explain parallel configuration of examples

* More options for the log output (#102)

* More options for the log output

* Remove the enum

* Apply suggestions from code review

Co-authored-by: Pedro Costa <p.simoes.costa@gmail.com>

* Update README with D2D_LOG_X values

Co-authored-by: Pedro Costa <p.simoes.costa@gmail.com>
Co-authored-by: Paul <ptb0890@gmail.com>

Co-authored-by: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Co-authored-by: Pedro Costa <p.simoes.costa@gmail.com>
Co-authored-by: Paul <ptb0890@gmail.com>
Co-authored-by: slaizet <45965875+slaizet@users.noreply.github.com>
---
 .github/workflows/Build_all.yml |  2 +-
 README.md                       | 27 ++++++++++++++++++++++++++-
 src/decomp_2d.f90               | 20 ++++++++++++++++++++
 src/decomp_2d_init_fin.f90      | 12 ++++--------
 src/fft_common.f90              |  7 ++++---
 src/fft_fftw3_f03.f90           |  7 ++++---
 src/log.f90                     | 12 ++++--------
 7 files changed, 63 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index ba4915ef..0f4a509d 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -47,7 +47,7 @@ jobs:
 
     # Runs a set of commands using the runners shell
     - name: Compile
-      run: make BUILD=dev && make examples
+      run: make BUILD=debug && make examples
 
 ###  bld_nvidia:
 ###    # The type of runner that the job will run on
diff --git a/README.md b/README.md
index 43a9115a..f4b3e949 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,13 @@ make check
 ```
 which will (re)build 2decomp&fft as necessary.
 
-**TODO** Convert examples to tests and automate running them
+By default this will run the tests/examples on a single rank, launched with ``mpirun``.
+The number of ranks can be controlled by setting the ``NP`` variable and the launcher is overridden by
+setting ``MPIRUN``, for example to run tests on 4 ranks using an ``mpiexec`` launcher installed at
+``${HOME}/bin/mpiexec`` the command would be
+```
+make NP=4 MPIRUN=${HOME}/bin/mpiexec check
+```
 
 ## GPU compilation
 
@@ -64,6 +70,25 @@ Profiling can be activated in the Makefile. Set the variable `PROFILER` to one o
 3. Profile FFT operations (default : true)
 4. Profile decomp_2d init / fin subroutines (default : true)
 
+## Miscellaneous
+
+### Print the log to a file or to stdout
+
+Before calling `decomp_2d_init`, the external code can modify the variable `decomp_log` to change the output for the log. The expected value is defined by the integer constants
+```
+integer, parameter, public :: D2D_LOG_QUIET = 0       ! No logging output
+integer, parameter, public :: D2D_LOG_STDOUT = 1      ! Root rank logs output to stdout
+integer, parameter, public :: D2D_LOG_TOFILE = 2      ! Root rank logs output to the file "decomp_2d_setup.log"
+integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3 ! All ranks log output to a dedicated file
+```
+exported by the `decomp_2d` module.
+Although their values are shown here, users should not rely on these and are recommended to prefer to use the named variables `D2D_LOG_QUIET`, etc. instead.
+The default value used is `D2D_LOG_TOFILE` for the default build and `D2D_LOG_TOFILE_FULL` for a debug build.
+
+### Change the debug level for debug builds
+
+Before calling `decomp_2d_init`, the external code can modify the variable `decomp_debug` to change the debug level. The user can also modify this variable using the environment variable `DECOMP_2D_DEBUG`. Please note that the environment variable is processed only for debug builds. The expected value for the variable `decomp_debug` is some integer between 0 and 6, bounds included.
+
 ## Optional dependencies
 
 ### FFTW
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index e1205aab..11953900 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -74,6 +74,26 @@ module decomp_2d
   ! flags for periodic condition in three dimensions
   logical, save :: periodic_x, periodic_y, periodic_z
 
+  !
+  ! Output for the log can be changed by the external code before calling decomp_2d_init
+  !
+  !    0 => No log output
+  !    1 => Master rank log output to stdout
+  !    2 => Master rank log output to the file "decomp_2d_setup.log"
+  !    3 => All ranks log output to a dedicated file
+  !
+  ! The default value is 2 (3 for debug builds)
+  !
+  integer, parameter, public :: D2D_LOG_QUIET = 0
+  integer, parameter, public :: D2D_LOG_STDOUT = 1
+  integer, parameter, public :: D2D_LOG_TOFILE = 2
+  integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
+#ifdef DEBUG
+  integer, public, save :: decomp_log = D2D_LOG_TOFILE_FULL
+#else
+  integer, public, save :: decomp_log = D2D_LOG_TOFILE
+#endif
+
   !
   ! Debug level can be changed by the external code before calling decomp_2d_init
   !
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index fda17fd0..9bae1008 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -37,9 +37,7 @@ subroutine decomp_2d_init_ref(nx,ny,nz,p_row,p_col,periodic_bc,comm)
     integer :: cuda_stat
     type(ncclResult) :: nccl_stat
 #endif
-#ifdef DEBUG
     character(len=7) fname ! Sufficient for up to O(1M) ranks
-#endif
 
 #ifdef PROFILER
     ! Prepare the profiler if it was not already prepared
@@ -209,17 +207,15 @@ subroutine decomp_2d_init_ref(nx,ny,nz,p_row,p_col,periodic_bc,comm)
     !
     ! Select the IO unit for decomp_2d setup
     !
-#ifdef DEBUG
-    write(fname, "(I0)") nrank ! Adapt to magnitude of nrank
-    open(newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
-#else
-    if (nrank == 0) then
+    if (decomp_log == D2D_LOG_TOFILE_FULL) then
+       write(fname, "(I0)") nrank ! Adapt to magnitude of nrank
+       open(newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
+    elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
        open(newunit=iounit, file="decomp_2d_setup.log", iostat=ierror)
     else
        iounit = output_unit
        ierror = 0
     endif
-#endif
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
     !
     ! Print the decomp_2d setup
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index b70bc295..8c17df27 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -120,6 +120,7 @@ subroutine fft_init_general(pencil, nx, ny, nz)
 if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
 ph=>decomp_main
 else
+if (.not.associated(ph)) allocate(ph)
 call decomp_info_init(nx, ny, nz, ph)
 endif
 if (format==PHYSICAL_IN_X) then
@@ -164,11 +165,11 @@ subroutine decomp_2d_fft_finalize
 if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
-nullify(ph)
-else
+if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
 call decomp_info_finalize(ph)
+deallocate(ph)
 endif
+nullify(ph)
 call decomp_info_finalize(sp)
 
 if (allocated(wk2_c2c)) deallocate(wk2_c2c)
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index e95e3620..5765ab66 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -147,6 +147,7 @@ subroutine fft_init_general(pencil, nx, ny, nz)
     if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
        ph => decomp_main
     else
+       if (.not.associated(ph)) allocate(ph)
        call decomp_info_init(nx, ny, nz, ph)
     endif
     if (format==PHYSICAL_IN_X) then
@@ -197,11 +198,11 @@ subroutine decomp_2d_fft_finalize
     if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-    if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
-      nullify(ph)
-    else
+    if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
       call decomp_info_finalize(ph)
+      deallocate(ph)
     endif
+    nullify(ph)
     call decomp_info_finalize(sp)
 
     call fftw_free(wk2_c2c_p)
diff --git a/src/log.f90 b/src/log.f90
index 3581ea7c..866cd97e 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -35,14 +35,10 @@ module subroutine d2d_listing(given_io_unit)
     character(len=64) :: fname
 #endif
 
-    !
-    ! Default : only rank 0 will print a listing
-    !
-    ! In DEBUG mode, all ranks will print a listing
-    !
-#ifndef DEBUG
-    if (nrank /= 0) return
-#endif
+    ! Output log if needed
+    if (decomp_log == D2D_LOG_QUIET) return
+    if (decomp_log == D2D_LOG_STDOUT .and. nrank /= 0) return
+    if (decomp_log == D2D_LOG_TOFILE .and. nrank /= 0) return
 
     ! If no IO unit provided, use stdout
     if (present(given_io_unit)) then

From 5a66283bd3da7a98238c0cf76815d7bca8e78a71 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Wed, 30 Nov 2022 23:28:00 +0100
Subject: [PATCH 011/436] Update caliper version

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index f4b3e949..2dd259cd 100644
--- a/README.md
+++ b/README.md
@@ -107,14 +107,14 @@ make install
 
 ### Caliper
 
-The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.8.0 was tested and is supported. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
+The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.9.0 was tested and is supported. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
 
 ```
 git clone https://github.com/LLNL/Caliper.git caliper_github
 cd caliper_github
-git checkout v2.8.0
+git checkout v2.9.0
 mkdir build && cd build
-cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../caliper_build_2.8.0 -DWITH_FORTRAN=yes -DWITH_MPI=yes -DBUILD_TESTING=yes ../
+cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../caliper_build_2.9.0 -DWITH_FORTRAN=yes -DWITH_MPI=yes -DBUILD_TESTING=yes ../
 make -j
 make test
 make install

From 5f9655fb6fa583cd9d52ab24c00e93a7eecaaf9c Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 1 Dec 2022 11:47:23 +0100
Subject: [PATCH 012/436] Make sure MPI_SUCCESS is 0

---
 src/decomp_2d_init_fin.f90 | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index 9bae1008..df0c4668 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -55,6 +55,12 @@ subroutine decomp_2d_init_ref(nx,ny,nz,p_row,p_col,periodic_bc,comm)
        decomp_2d_comm = MPI_COMM_WORLD
     endif
 
+    ! Safety check
+    if (MPI_SUCCESS /= 0) call decomp_2d_abort(__FILE__, &
+                                               __LINE__, &
+                                               MPI_SUCCESS, &
+                                               "MPI error check is broken")
+
     ! If the external code has not set nrank and nproc
     if (nrank == -1) then
        call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)

From f5ca7f3c303db23d166c97f016922cd6e49ea675 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 2 Dec 2022 14:44:38 +0000
Subject: [PATCH 013/436] Initialisation test cases

Starting from a very simple case of testing that we can initialise
the library and that the distribution of nodes is correct
---
 examples/Makefile                |  6 ++-
 examples/init_test/.gitignore    |  1 +
 examples/init_test/Makefile      | 25 +++++++++++
 examples/init_test/init_test.f90 | 74 ++++++++++++++++++++++++++++++++
 4 files changed, 104 insertions(+), 2 deletions(-)
 create mode 100644 examples/init_test/.gitignore
 create mode 100644 examples/init_test/Makefile
 create mode 100644 examples/init_test/init_test.f90

diff --git a/examples/Makefile b/examples/Makefile
index 195a6e4a..f9e197e5 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,7 +1,7 @@
-.PHONY: test2d fft_physical_z fft_physical_x halo_test io_test 
+.PHONY: test2d fft_physical_z fft_physical_x halo_test init_test io_test 
 
 # Just build the examples
-examples: test2d fft_physical_z fft_physical_x halo_test io_test
+examples: test2d fft_physical_z fft_physical_x halo_test init_test io_test
 	@echo "Built the examples"
 
 test2d:
@@ -12,6 +12,8 @@ fft_physical_x:
 	$(MAKE) -C $@ $@
 halo_test:
 	$(MAKE) -C $@ $@
+init_test:
+	$(MAKE) -C $@ $@
 io_test:
 	$(MAKE) -C $@ all
 
diff --git a/examples/init_test/.gitignore b/examples/init_test/.gitignore
new file mode 100644
index 00000000..a00b8f97
--- /dev/null
+++ b/examples/init_test/.gitignore
@@ -0,0 +1 @@
+init_test
\ No newline at end of file
diff --git a/examples/init_test/Makefile b/examples/init_test/Makefile
new file mode 100644
index 00000000..4f1fe54b
--- /dev/null
+++ b/examples/init_test/Makefile
@@ -0,0 +1,25 @@
+FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
+FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
+LIBS = -L../../ -l$(LIBDECOMP) $(LFLAGS)
+
+OBJ = init_test.o
+
+NP ?= 1
+MPIRUN ?= mpirun
+
+init_test: $(OBJ)
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $(OBJ) $(LIBS)
+
+ifeq ($(PARAMOD),gpu)
+check:
+	$(MPIRUN) -n $(NP) ./bind.sh ./init_test
+else
+check:
+	$(MPIRUN) -n $(NP) ./init_test
+endif
+
+clean:
+	rm -f *.o init_test *.log
+
+%.o : %.f90
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
new file mode 100644
index 00000000..5f9c5dd1
--- /dev/null
+++ b/examples/init_test/init_test.f90
@@ -0,0 +1,74 @@
+!!!! init_test.f90
+!!!  Tests initialising the 2decomp&fft library.
+!!!=====================================================
+
+program init_test
+
+  use MPI
+  use decomp_2d
+  
+  implicit none
+
+  integer, parameter :: nx = 5
+  integer, parameter :: ny = 6
+  integer, parameter :: nz = 7
+  integer, parameter :: nexpect = nx * ny * nz
+
+  integer :: p_row, p_col
+
+  integer :: ierr
+
+  call MPI_Init(ierr)
+  
+  p_row = 0; p_col = 0
+  call run(p_row, p_col)
+
+  call MPI_Finalize(ierr)
+  
+contains
+
+  subroutine run(p_row, p_col)
+
+    integer, intent(inout) :: p_row, p_col
+
+    call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+    call check_axis("X")
+    call check_axis("Y")
+    call check_axis("Z")
+    
+    call decomp_2d_finalize()
+    
+  end subroutine run
+
+  subroutine check_axis(axis)
+
+    character(len=*), intent(in) :: axis
+
+    integer :: suml
+    integer :: sumg
+    integer, dimension(3) :: sizes
+
+    if (axis == "X") then
+       sizes = xsize
+    else if (axis == "Y") then
+       sizes = ysize
+    else if (axis == "Z") then
+       sizes = zsize
+    else
+       sizes = 0
+       print *, "ERROR: unknown axis requested!"
+       stop 1
+    end if
+
+    suml = product(sizes)
+    call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierr)
+    
+    if (sumg /= nexpect) then
+       print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
+       stop 1
+    end if
+    
+  end subroutine check_axis
+  
+end program init_test

From 1877560e83ed335c1bcd93dae252b775b685b20a Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 2 Dec 2022 16:38:24 +0000
Subject: [PATCH 014/436] Add init_test to main examples Makefile

---
 examples/Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/examples/Makefile b/examples/Makefile
index f9e197e5..eccdad3d 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -13,7 +13,7 @@ fft_physical_x:
 halo_test:
 	$(MAKE) -C $@ $@
 init_test:
-	$(MAKE) -C $@ $@
+	$(MAKE) -C $@ all
 io_test:
 	$(MAKE) -C $@ all
 
@@ -22,6 +22,7 @@ check:
 	cd fft_physical_z; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd halo_test; $(MAKE) $@
+	cd init_test; $(MAKE) $@
 	cd io_test; $(MAKE) $@
 
 clean:
@@ -29,6 +30,7 @@ clean:
 	cd fft_physical_z; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd halo_test; $(MAKE) $@
+	cd init_test; $(MAKE) $@
 	cd io_test; $(MAKE) $@
 
 export

From 1cbc5ded51023176aff8df60bff34cc7c8e5a49c Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 2 Dec 2022 18:11:55 +0000
Subject: [PATCH 015/436] Forgot to push the all target

---
 examples/init_test/Makefile | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/examples/init_test/Makefile b/examples/init_test/Makefile
index 4f1fe54b..5adedd9c 100644
--- a/examples/init_test/Makefile
+++ b/examples/init_test/Makefile
@@ -1,4 +1,3 @@
-FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
 FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
 LIBS = -L../../ -l$(LIBDECOMP) $(LFLAGS)
 
@@ -6,6 +5,12 @@ OBJ = init_test.o
 
 NP ?= 1
 MPIRUN ?= mpirun
+MODDIR ?= $(DECOMPINC)
+
+all: $(MODDIR) init_test
+
+$(MODDIR):
+	mkdir -p $@
 
 init_test: $(OBJ)
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $(OBJ) $(LIBS)
@@ -19,7 +24,7 @@ check:
 endif
 
 clean:
-	rm -f *.o init_test *.log
+	rm -f *.o init_test *.log *.mod *.smod $(MODDIR)/*.mod $(MODDIR)/*.smod
 
 %.o : %.f90
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@

From 047debc010f5ed1dbf1cf8d92c9199002c80cf57 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 5 Dec 2022 08:27:30 +0100
Subject: [PATCH 016/436] IO : Assumed shape arrays should be contiguous

---
 src/io.f90 | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/io.f90 b/src/io.f90
index 160b4257..99f09473 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -184,7 +184,7 @@ subroutine write_one_real(ipencil,var,filename,opt_decomp)
     implicit none
 
     integer, intent(IN) :: ipencil
-    real(mytype), dimension(:,:,:), intent(IN) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     character(len=*), intent(IN) :: filename
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
@@ -213,7 +213,7 @@ subroutine write_one_complex(ipencil,var,filename,opt_decomp)
     implicit none
 
     integer, intent(IN) :: ipencil
-    complex(mytype), dimension(:,:,:), intent(IN) :: var
+    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     character(len=*), intent(IN) :: filename
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
@@ -246,7 +246,7 @@ subroutine read_one_real(ipencil,var,dirname,varname,io_name,opt_decomp,reduce_p
     implicit none
 
     integer, intent(IN) :: ipencil
-    real(mytype), dimension(:,:,:), intent(INOUT) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
     character(len=*), intent(IN) :: varname, dirname, io_name
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
     logical, intent(in), optional :: reduce_prec
@@ -393,7 +393,7 @@ subroutine read_one_complex(ipencil,var,filename,opt_decomp)
     implicit none
 
     integer, intent(IN) :: ipencil
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: var
+    complex(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
     character(len=*), intent(IN) :: filename
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
@@ -424,7 +424,7 @@ subroutine adios2_read_one_real(var,engine_name,varname,io_name)
     character(len=*), intent(in) :: engine_name
     character(len=*), intent(in) :: io_name
     character*(*), intent(in) :: varname
-    real(mytype), dimension(:,:,:), intent(out) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(out) :: var
 
     integer :: ierror
     type(adios2_io) :: io_handle
@@ -482,7 +482,7 @@ subroutine write_var_real(fh,disp,ipencil,var,opt_decomp)
     integer, intent(IN) :: fh
     integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
     integer, intent(IN) :: ipencil
-    real(mytype), dimension(:,:,:), intent(IN) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
     TYPE(DECOMP_INFO) :: decomp
@@ -503,7 +503,7 @@ subroutine write_var_complex(fh,disp,ipencil,var,opt_decomp)
     integer, intent(IN) :: fh
     integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
     integer, intent(IN) :: ipencil
-    complex(mytype), dimension(:,:,:), intent(IN) :: var
+    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
     TYPE(DECOMP_INFO) :: decomp
@@ -524,7 +524,7 @@ subroutine write_outflow(dirname,varname,ntimesteps,var,io_name,opt_decomp)
 
     character(len=*), intent(in) :: dirname, varname, io_name
     integer, intent(IN) :: ntimesteps 
-    real(mytype), dimension(:,:,:), intent(IN) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
     TYPE(DECOMP_INFO) :: decomp
@@ -565,7 +565,7 @@ subroutine read_var_real(fh,disp,ipencil,var,opt_decomp)
     integer, intent(IN) :: fh
     integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
     integer, intent(IN) :: ipencil
-    real(mytype), dimension(:,:,:), intent(INOUT) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
     TYPE(DECOMP_INFO) :: decomp
@@ -587,7 +587,7 @@ subroutine read_var_complex(fh,disp,ipencil,var,opt_decomp)
     integer, intent(IN) :: fh
     integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
     integer, intent(IN) :: ipencil
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: var
+    complex(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
     TYPE(DECOMP_INFO) :: decomp
@@ -608,7 +608,7 @@ subroutine read_inflow(dirname,varname,ntimesteps,var,io_name,opt_decomp)
 
     character(len=*), intent(in) :: dirname, varname, io_name
     integer, intent(IN) :: ntimesteps
-    real(mytype), dimension(:,:,:), intent(INOUT) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
     TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
     TYPE(DECOMP_INFO) :: decomp
@@ -931,7 +931,7 @@ subroutine write_plane_3d_real(ipencil,var,iplane,n,dirname,varname,io_name, &
     implicit none
 
     integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    real(mytype), dimension(:,:,:), intent(IN) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
     integer, intent(IN) :: n ! which plane to write (global coordinate)
     character(len=*), intent(IN) :: dirname,varname,io_name
@@ -975,7 +975,7 @@ subroutine write_plane_3d_complex(ipencil,var,iplane,n, &
     implicit none
 
     integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    complex(mytype), dimension(:,:,:), intent(IN) :: var
+    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
     integer, intent(IN) :: n ! which plane to write (global coordinate)
     character(len=*), intent(IN) :: dirname,varname,io_name
@@ -1044,7 +1044,7 @@ subroutine write_every_real(ipencil,var,iskip,jskip,kskip, &
     implicit none
 
     integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    real(mytype), dimension(:,:,:), intent(IN) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     integer, intent(IN) :: iskip,jskip,kskip 
     character(len=*), intent(IN) :: filename
     logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
@@ -1077,7 +1077,7 @@ subroutine write_every_complex(ipencil,var,iskip,jskip,kskip, &
     implicit none
 
     integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    complex(mytype), dimension(:,:,:), intent(IN) :: var
+    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     integer, intent(IN) :: iskip,jskip,kskip 
     character(len=*), intent(IN) :: filename
     logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
@@ -1211,7 +1211,7 @@ subroutine mpiio_write_real_coarse(ipencil,var,dirname,varname,icoarse,io_name,o
 
     integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
     integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-    real(mytype), dimension(:,:,:), intent(IN) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     character(len=*), intent(in) :: dirname, varname, io_name
     type(decomp_info), intent(in), optional :: opt_decomp
     logical, intent(in), optional :: reduce_prec
@@ -1464,7 +1464,7 @@ subroutine mpiio_write_real_probe(ipencil,var,filename,nlength)
 
     integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
     integer, intent(in) :: nlength
-    real(mytype), dimension(:,:,:,:), intent(IN) :: var
+    real(mytype), contiguous, dimension(:,:,:,:), intent(IN) :: var
 
     character(len=*) :: filename
 
@@ -1544,7 +1544,7 @@ subroutine write_subdomain(ipencil,var,is,ie,js,je,ks,ke,filename)
     implicit none
 
     integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    real(mytype), dimension(:,:,:), intent(IN) :: var
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
     integer, intent(IN) :: is, ie, js, je, ks, ke
     character(len=*), intent(IN) :: filename
 

From 799b61a26d0a573816c4518dd15c9d9bb8045ed2 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 5 Dec 2022 08:39:13 +0100
Subject: [PATCH 017/436] IO : Try to avoid integer overflow when the
 displacement offset is updated

---
 src/io.f90               | 36 ++++++++++++++++++++++++++----------
 src/io_read_inflow.f90   | 12 ++++++++----
 src/io_read_var.inc      | 12 ++++++++----
 src/io_write_outflow.f90 | 12 ++++++++----
 src/io_write_var.inc     | 12 ++++++++----
 5 files changed, 58 insertions(+), 26 deletions(-)

diff --git a/src/io.f90 b/src/io.f90
index 99f09473..feba3fd9 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -367,7 +367,10 @@ subroutine read_one_real(ipencil,var,dirname,varname,io_name,opt_decomp,reduce_p
       call MPI_TYPE_FREE(newtype,ierror)
       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-      disp = disp + sizes(1) * sizes(2) * sizes(3) * disp_bytes
+      disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                  * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                  * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                  * int(disp_bytes, kind=MPI_OFFSET_KIND)
     end associate
 
     if (opened_new) then
@@ -665,7 +668,8 @@ subroutine write_scalar_real(fh,disp,n,var)
     call MPI_FILE_WRITE_ALL(fh, var, m, real_type, &
          MPI_STATUS_IGNORE, ierror)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    disp = disp + n*mytype_bytes
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 
     return
   end subroutine write_scalar_real
@@ -693,7 +697,9 @@ subroutine write_scalar_complex(fh,disp,n,var)
     call MPI_FILE_WRITE_ALL(fh, var, m, complex_type, &
          MPI_STATUS_IGNORE, ierror)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    disp = disp + n*mytype_bytes*2
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+                * 2_MPI_OFFSET_KIND
 
     return
   end subroutine write_scalar_complex
@@ -723,7 +729,8 @@ subroutine write_scalar_integer(fh,disp,n,var)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
     call MPI_TYPE_SIZE(MPI_INTEGER,m,ierror)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + n*m
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(m, kind=MPI_OFFSET_KIND)
 
     return
   end subroutine write_scalar_integer
@@ -753,7 +760,8 @@ subroutine write_scalar_logical(fh,disp,n,var)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
     call MPI_TYPE_SIZE(MPI_LOGICAL,m,ierror)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + n*m
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(m, kind=MPI_OFFSET_KIND)
 
     return
   end subroutine write_scalar_logical
@@ -783,7 +791,8 @@ subroutine read_scalar_real(fh,disp,n,var)
     call MPI_FILE_READ_ALL(fh, var, n, real_type, &
          MPI_STATUS_IGNORE, ierror)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    disp = disp + n*mytype_bytes
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 
     return
   end subroutine read_scalar_real
@@ -806,7 +815,9 @@ subroutine read_scalar_complex(fh,disp,n,var)
     call MPI_FILE_READ_ALL(fh, var, n, complex_type, &
          MPI_STATUS_IGNORE, ierror)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    disp = disp + n*mytype_bytes*2
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+                * 2_MPI_OFFSET_KIND
 
     return
   end subroutine read_scalar_complex
@@ -831,7 +842,8 @@ subroutine read_scalar_integer(fh,disp,n,var)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
     call MPI_TYPE_SIZE(MPI_INTEGER,m,ierror)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + n*m
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(m, kind=MPI_OFFSET_KIND)
 
     return
   end subroutine read_scalar_integer
@@ -856,7 +868,8 @@ subroutine read_scalar_logical(fh,disp,n,var)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
     call MPI_TYPE_SIZE(MPI_LOGICAL,m,ierror)
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + n*m
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(m, kind=MPI_OFFSET_KIND)
 
     return
   end subroutine read_scalar_logical
@@ -1318,7 +1331,10 @@ subroutine mpiio_write_real_coarse(ipencil,var,dirname,varname,icoarse,io_name,o
     end if
     if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
     
-    fh_disp(idx) = fh_disp(idx) + sizes(1) * sizes(2) * sizes(3) * disp_bytes
+    fh_disp(idx) = fh_disp(idx) + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                * int(disp_bytes, kind=MPI_OFFSET_KIND)
     
     if (opened_new) then
        call decomp_2d_close_io(io_name, full_io_name)
diff --git a/src/io_read_inflow.f90 b/src/io_read_inflow.f90
index 8acec522..3e5a9fb1 100644
--- a/src/io_read_inflow.f90
+++ b/src/io_read_inflow.f90
@@ -57,10 +57,14 @@
   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
   ! update displacement for the next read operation
-  disp = disp + sizes(1)*sizes(2)*sizes(3)*mytype_bytes
-  if (data_type == complex_type) then
-     disp = disp + sizes(1)*sizes(2)*sizes(3)*mytype_bytes
-  end if
+  disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+              * int(sizes(2), kind=MPI_OFFSET_KIND) &
+              * int(sizes(3), kind=MPI_OFFSET_KIND) &
+              * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+  if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                             * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                             * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
 associate(vnm => varname) ! Silence unused argument
diff --git a/src/io_read_var.inc b/src/io_read_var.inc
index 0bc0bf1d..489247cb 100644
--- a/src/io_read_var.inc
+++ b/src/io_read_var.inc
@@ -68,7 +68,11 @@ call MPI_TYPE_FREE(newtype,ierror)
 if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 ! update displacement for the next read operation
-disp = disp + sizes(1)*sizes(2)*sizes(3)*mytype_bytes
-if (data_type == complex_type) then
-   disp = disp + sizes(1)*sizes(2)*sizes(3)*mytype_bytes
-end if
+disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+            * int(sizes(2), kind=MPI_OFFSET_KIND) &
+            * int(sizes(3), kind=MPI_OFFSET_KIND) &
+            * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                           * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                           * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                           * int(mytype_bytes, kind=MPI_OFFSET_KIND)
diff --git a/src/io_write_outflow.f90 b/src/io_write_outflow.f90
index fbd845a2..ccbd0314 100644
--- a/src/io_write_outflow.f90
+++ b/src/io_write_outflow.f90
@@ -57,10 +57,14 @@
   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
   ! update displacement for the next write operation
-  disp = disp + sizes(1)*sizes(2)*sizes(3)*mytype_bytes
-  if (data_type == complex_type) then
-     disp = disp + sizes(1)*sizes(2)*sizes(3)*mytype_bytes
-  end if
+  disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+              * int(sizes(2), kind=MPI_OFFSET_KIND) &
+              * int(sizes(3), kind=MPI_OFFSET_KIND) &
+              * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+  if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                             * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                             * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
 associate(vnm => varname) ! Silence unused dummy argument
diff --git a/src/io_write_var.inc b/src/io_write_var.inc
index b497e589..23c8b617 100644
--- a/src/io_write_var.inc
+++ b/src/io_write_var.inc
@@ -68,7 +68,11 @@ call MPI_TYPE_FREE(newtype,ierror)
 if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 ! update displacement for the next write operation
-disp = disp + sizes(1)*sizes(2)*sizes(3)*mytype_bytes
-if (data_type == complex_type) then
-   disp = disp + sizes(1)*sizes(2)*sizes(3)*mytype_bytes
-end if
+disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+            * int(sizes(2), kind=MPI_OFFSET_KIND) &
+            * int(sizes(3), kind=MPI_OFFSET_KIND) &
+            * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                           * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                           * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                           * int(mytype_bytes, kind=MPI_OFFSET_KIND)

From 7c56549a25d4eabf60655c570cc5d61a070bd7b2 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 5 Dec 2022 09:51:20 +0100
Subject: [PATCH 018/436] IO : Minor fix. Use trim when defining the variable
 full_io_name

---
 src/io.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/io.f90 b/src/io.f90
index feba3fd9..7b2e4c4a 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -277,7 +277,7 @@ subroutine read_one_real(ipencil,var,dirname,varname,io_name,opt_decomp,reduce_p
     opened_new = .false.
     if (idx .lt. 1) then
        ! Check file exists
-       full_io_name = dirname//"/"//varname
+       full_io_name = trim(dirname)//"/"//trim(varname)
        if (nrank==0) then
           inquire(file=full_io_name, exist=dir_exists)
           if (.not.dir_exists) then
@@ -1306,7 +1306,7 @@ subroutine mpiio_write_real_coarse(ipencil,var,dirname,varname,icoarse,io_name,o
              call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
           end if
        end if
-       full_io_name = dirname//"/"//varname
+       full_io_name = trim(dirname)//"/"//trim(varname)
        call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
        idx = get_io_idx(io_name, full_io_name)
        opened_new = .true.

From c1bb93a1f19fde33d7f86198eae6dd6230466a33 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 5 Dec 2022 09:51:48 +0100
Subject: [PATCH 019/436] IO : Make sure ipencil is 1, 2 or 3.

---
 src/io.f90 | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/io.f90 b/src/io.f90
index 7b2e4c4a..2869145d 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -338,6 +338,8 @@ subroutine read_one_real(ipencil,var,dirname,varname,io_name,opt_decomp,reduce_p
        starts(1) = decomp%zst(1)-1
        starts(2) = decomp%zst(2)-1
        starts(3) = decomp%zst(3)-1
+    else
+       call decomp_2d_abort(-1, "IO/read_one_real : Wrong value for ipencil")
     endif
 
     associate(fh => fh_registry(idx), &
@@ -1156,6 +1158,8 @@ subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
        elseif (ipencil == 3) then
           subsizes(1:3) = decomp%zsz(1:3)
           starts(1:3) = decomp%zst(1:3) - 1
+       else
+          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
        endif
     elseif (icoarse==1) then
        sizes(1) = xszS(1)
@@ -1183,6 +1187,8 @@ subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
           starts(1) = zstS(1)-1
           starts(2) = zstS(2)-1
           starts(3) = zstS(3)-1
+       else
+          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
        endif
     elseif (icoarse==2) then
        sizes(1) = xszV(1)
@@ -1210,6 +1216,8 @@ subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
           starts(1) = zstV(1)-1
           starts(2) = zstV(2)-1
           starts(3) = zstV(3)-1
+       else
+          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
        endif
     endif
     

From d9350d8e547490f4adafa06697341a12033e0f5d Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 5 Dec 2022 10:03:07 +0100
Subject: [PATCH 020/436] Remove T3PIO-related code

---
 src/io.f90           |  7 ++-----
 src/io_write_one.inc | 21 ---------------------
 2 files changed, 2 insertions(+), 26 deletions(-)

diff --git a/src/io.f90 b/src/io.f90
index 160b4257..04e282db 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -17,9 +17,6 @@ module decomp_2d_io
 
   use decomp_2d
   use MPI
-#ifdef T3PIO
-  use t3pio
-#endif
 
 #ifdef ADIOS2
   use adios2
@@ -191,7 +188,7 @@ subroutine write_one_real(ipencil,var,filename,opt_decomp)
     TYPE(DECOMP_INFO) :: decomp
     integer(kind=MPI_OFFSET_KIND) :: filesize, disp
     integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh, data_type, info, gs
+    integer :: ierror, newtype, fh, data_type
 
 #ifdef PROFILER
   if (decomp_profiler_io) call decomp_profiler_start("io_write_one_real")
@@ -220,7 +217,7 @@ subroutine write_one_complex(ipencil,var,filename,opt_decomp)
     TYPE(DECOMP_INFO) :: decomp
     integer(kind=MPI_OFFSET_KIND) :: filesize, disp
     integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh, data_type, info, gs
+    integer :: ierror, newtype, fh, data_type
 
 #ifdef PROFILER
   if (decomp_profiler_io) call decomp_profiler_start("io_write_one_cplx")
diff --git a/src/io_write_one.inc b/src/io_write_one.inc
index 29facac5..8492be5a 100644
--- a/src/io_write_one.inc
+++ b/src/io_write_one.inc
@@ -50,31 +50,14 @@ else if (ipencil == 3) then
    starts(3) = decomp%zst(3)-1
 endif
 
-#ifdef T3PIO
-call MPI_INFO_CREATE(info, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_CREATE")
-gs = ceiling(real(sizes(1),mytype)*real(sizes(2),mytype)* &
-     real(sizes(3),mytype)/1024./1024.)
-call t3pio_set_info(decomp_2d_comm, info, "./", ierror, &
-     GLOBAL_SIZE=gs, factor=1)
-#else
-gs = 1 ! Silence unused variable
-info = 1 ! Silence unused variable
-#endif
-
 call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
      MPI_ORDER_FORTRAN, data_type, newtype, ierror)
 if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
 call MPI_TYPE_COMMIT(newtype,ierror)
 if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-#ifdef T3PIO
-call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-     MPI_MODE_CREATE+MPI_MODE_WRONLY, info, fh, ierror)
-#else
 call MPI_FILE_OPEN(decomp_2d_comm, filename, &
      MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
      fh, ierror)
-#endif
 if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
 filesize = 0_MPI_OFFSET_KIND
 call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
@@ -91,7 +74,3 @@ call MPI_FILE_CLOSE(fh,ierror)
 if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
 call MPI_TYPE_FREE(newtype,ierror)
 if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-#ifdef T3PIO
-call MPI_INFO_FREE(info,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_FREE")
-#endif

From 3184c7d944c4b16093a2f4ac1923015eeb43045b Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Mon, 5 Dec 2022 17:37:21 +0100
Subject: [PATCH 021/436] CI : Restore nvhpc CPU

---
 .github/workflows/Build_all.yml | 53 +++++++++++++++++----------------
 1 file changed, 27 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 0f4a509d..4f1970a6 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -49,33 +49,34 @@ jobs:
     - name: Compile
       run: make BUILD=debug && make examples
 
-###  bld_nvidia:
-###    # The type of runner that the job will run on
-###    runs-on: ubuntu-latest
-###    env:
-###      TMP: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/mpi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin
-###
-###    # Steps represent a sequence of tasks that will be executed as part of the job
-###    steps:
-###    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-###    - uses: actions/checkout@v3
-###
-###    # Add Nvidia HPC SDK
-###    - name: setup repo
-###      run: |
-###        echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list
-###    # Install dependencies
-###    - name: install
-###      run: |
-###        sudo apt-get update  -y
+  bld_nvidia:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    env:
+      TMP: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/mpi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - uses: actions/checkout@v3
+
+    # Add Nvidia HPC SDK
+    - name: setup repo
+      run: |
+        echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list
+    # Install dependencies
+    - name: install
+      run: |
+        sudo apt-get update  -y
+        sudo apt-get install -y nvhpc-22-7
 ###        sudo apt-get install -y nvhpc-22-7-cuda-multi
-###
-###    # Runs a set of commands using the runners shell
-###    - name: CPU version
-###      run: |
-###        PATH=$TMP:$PATH make CMP=nvhpc BUILD=debug
-###        PATH=$TMP:$PATH make examples
-###
+
+    # Runs a set of commands using the runners shell
+    - name: CPU version
+      run: |
+        PATH=$TMP:$PATH make CMP=nvhpc BUILD=debug
+        PATH=$TMP:$PATH make examples
+
 ###    - name: GPU version
 ###      run: |
 ###        PATH=$TMP:$PATH make clean

From d80d7946d5ff0b31a76c878588d0e960eedf247a Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Mon, 5 Dec 2022 20:03:30 +0000
Subject: [PATCH 022/436] Add Caliper v2.8.0 still expected to work

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2dd259cd..a4ccea56 100644
--- a/README.md
+++ b/README.md
@@ -107,7 +107,7 @@ make install
 
 ### Caliper
 
-The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.9.0 was tested and is supported. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
+The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.9.0 was tested and is supported, version 2.8.0 has also been tested and is still expected to work. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
 
 ```
 git clone https://github.com/LLNL/Caliper.git caliper_github

From 1cb8c36725f80b8c5ac9cd42dedc8b067815ae40 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 6 Dec 2022 14:28:56 +0000
Subject: [PATCH 023/436] Add binding for GPU run

---
 examples/init_test/bind.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100755 examples/init_test/bind.sh

diff --git a/examples/init_test/bind.sh b/examples/init_test/bind.sh
new file mode 100755
index 00000000..38d4fedf
--- /dev/null
+++ b/examples/init_test/bind.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export LOCAL_RANK=${OMPI_COMM_WORLD_LOCAL_RANK}
+export CUDA_VISIBLE_DEVICES=${LOCAL_RANK}
+
+echo "[LOG] local rank $LOCAL_RANK: bind to $CUDA_VISIBLE_DEVICES"
+echo ""
+
+$* 
+

From e8f2cc6a268aec5db308a6ab9a59eada5e14ecd6 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 7 Dec 2022 06:16:40 +0000
Subject: [PATCH 024/436] Split FFT physical X into C2C and R2C tests

---
 examples/fft_physical_x/Makefile              |  17 ++-
 .../{fft_physical_x.f90 => fft_c2c_x.f90}     |  80 +-----------
 examples/fft_physical_x/fft_r2c_x.f90         | 119 ++++++++++++++++++
 3 files changed, 132 insertions(+), 84 deletions(-)
 rename examples/fft_physical_x/{fft_physical_x.f90 => fft_c2c_x.f90} (63%)
 create mode 100644 examples/fft_physical_x/fft_r2c_x.f90

diff --git a/examples/fft_physical_x/Makefile b/examples/fft_physical_x/Makefile
index ad417efc..0ccb3311 100644
--- a/examples/fft_physical_x/Makefile
+++ b/examples/fft_physical_x/Makefile
@@ -8,9 +8,12 @@ LIBS = -L../../ -l$(LIBDECOMP) $(LIBFFT) $(LFLAGS)
 NP ?= 1
 MPIRUN ?= mpirun
 
-all: fft_physical_x fft_grid_x
+all: fft_c2c_x fft_r2c_x fft_grid_x
 
-fft_physical_x: fft_physical_x.o
+fft_c2c_x: fft_c2c_x.o
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
+
+fft_r2c_x: fft_r2c_x.o
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
 
 fft_grid_x: fft_grid_x.o
@@ -18,19 +21,21 @@ fft_grid_x: fft_grid_x.o
 
 ifeq ($(PARAMOD),gpu)
 check:
-	$(MPIRUN) -n $(NP) ./bind.sh ./fft_physical_x
+	$(MPIRUN) -n $(NP) ./bind.sh ./fft_c2c_x
+	$(MPIRUN) -n $(NP) ./bind.sh ./fft_r2c_x
 	$(MPIRUN) -n $(NP) ./bind.sh ./fft_grid_x
 else
 check:
-	$(MPIRUN) -n $(NP) ./fft_physical_x
+	$(MPIRUN) -n $(NP) ./fft_c2c_x
+	$(MPIRUN) -n $(NP) ./fft_r2c_x
 	$(MPIRUN) -n $(NP) ./fft_grid_x
 endif
 
 mem_leak:
-	valgrind --leak-check=full --show-leak-kinds=all $(MPIRUN) -n 1 ./fft_physical_x 1 1
+	valgrind --leak-check=full --show-leak-kinds=all $(MPIRUN) -n 1 ./fft_c2c_x 1 1
 
 clean:
-	rm -f *.o fft_physical_x fft_grid_x *.log
+	rm -f *.o fft_c2c_x fft_r2c_x fft_grid_x *.log
 
 %.o : %.f90
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/fft_physical_x/fft_physical_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
similarity index 63%
rename from examples/fft_physical_x/fft_physical_x.f90
rename to examples/fft_physical_x/fft_c2c_x.f90
index 9a999e95..94bf1d4d 100644
--- a/examples/fft_physical_x/fft_physical_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -1,4 +1,4 @@
-program fft_physical_x
+program fft_c2c_x
 
   use decomp_2d
   use decomp_2d_fft
@@ -124,84 +124,8 @@ program fft_physical_x
   
   deallocate(in,out)
   call decomp_2d_fft_finalize
-
-
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the r2c/c2r interface
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  call decomp_2d_fft_init(PHYSICAL_IN_X)
-  
-  allocate (in_r(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
-  allocate (out(fft_start(1):fft_end(1), &
-                fft_start(2):fft_end(2), &
-                fft_start(3):fft_end(3)))
-  
-  ! initilise input
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-  
-     ! 3D r2c FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in_r, out)
-     t2 = t2 + MPI_WTIME() - t1
-  
-     ! 3D inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in_r)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     ! normalisation - note 2DECOMP&FFT doesn't normalise
-     !$acc kernels
-     in_r = in_r / (real(nx,mytype)*real(ny,mytype)*real(nz,mytype))
-     !$acc end kernels
-
-  end do
-#if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
-#endif 
-
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           error = error + abs(in_r(i,j,k)-dr)
-        end do
-     end do
-  end do
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-  
-  if (nrank==0) then
-     write(*,*) '===== r2c/c2r interface ====='
-     write(*,*) 'error / mesh point: ', err_all
-     write(*,*) 'time (sec): ', t1,t3
-  end if
-  
-  deallocate(in_r,out)
-  call decomp_2d_fft_finalize
   call decomp_2d_finalize
   call MPI_FINALIZE(ierror)
 
-end program fft_physical_x
+end program fft_c2c_x
 
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
new file mode 100644
index 00000000..207e275e
--- /dev/null
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -0,0 +1,119 @@
+program fft_r2c_x
+
+  use decomp_2d
+  use decomp_2d_fft
+  use MPI
+#if defined(_GPU) 
+  use cudafor
+  use cufft
+  use openacc 
+#endif
+
+  implicit none
+  
+  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
+  integer :: nx, ny, nz
+  integer :: p_row=0, p_col=0
+  integer :: resize_domain
+  integer :: nranks_tot
+  
+  integer, parameter :: ntest = 10  ! repeat test this times
+  
+  complex(mytype), allocatable, dimension(:,:,:) :: in, out
+  real(mytype), allocatable, dimension(:,:,:) :: in_r
+  
+  integer, dimension(3) :: fft_start, fft_end, fft_size
+  
+  real(mytype) :: dr,di, error, err_all, n1,flops
+  integer :: ierror, i,j,k,m
+  real(mytype) :: t1, t2, t3 ,t4
+  
+  call MPI_INIT(ierror)
+  ! To resize the domain we need to know global number of ranks
+  ! This operation is also done as part of decomp_2d_init
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+  resize_domain = int(nranks_tot/4)+1
+  nx = nx_base*resize_domain
+  ny = ny_base*resize_domain
+  nz = nz_base*resize_domain
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  ! Test the r2c/c2r interface
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  call decomp_2d_fft_init(PHYSICAL_IN_X)
+  
+  allocate (in_r(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
+  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
+  allocate (out(fft_start(1):fft_end(1), &
+                fft_start(2):fft_end(2), &
+                fft_start(3):fft_end(3)))
+  
+  ! initilise input
+  do k=xstart(3),xend(3)
+     do j=xstart(2),xend(2)
+        do i=xstart(1),xend(1)
+           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+        end do
+     end do
+  end do
+  
+  t2 = 0._mytype
+  t4 = 0._mytype
+  do m=1,ntest
+  
+     ! 3D r2c FFT
+     t1 = MPI_WTIME()
+     call decomp_2d_fft_3d(in_r, out)
+     t2 = t2 + MPI_WTIME() - t1
+  
+     ! 3D inverse FFT
+     t3 = MPI_WTIME()
+     call decomp_2d_fft_3d(out, in_r)
+     t4 = t4 + MPI_WTIME() - t3
+  
+     ! normalisation - note 2DECOMP&FFT doesn't normalise
+     !$acc kernels
+     in_r = in_r / (real(nx,mytype)*real(ny,mytype)*real(nz,mytype))
+     !$acc end kernels
+
+  end do
+#if defined(_GPU)
+  ierror = cudaDeviceSynchronize()
+#endif 
+
+  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t1 = t1 / real(nproc,mytype)
+  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t3 = t3 / real(nproc,mytype)
+  
+  ! checking accuracy
+  error = 0._mytype
+  do k=xstart(3),xend(3)
+     do j=xstart(2),xend(2)
+        do i=xstart(1),xend(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           error = error + abs(in_r(i,j,k)-dr)
+        end do
+     end do
+  end do
+  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
+  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
+  
+  if (nrank==0) then
+     write(*,*) '===== r2c/c2r interface ====='
+     write(*,*) 'error / mesh point: ', err_all
+     write(*,*) 'time (sec): ', t1,t3
+  end if
+  
+  deallocate(in_r,out)
+  call decomp_2d_fft_finalize
+  call decomp_2d_finalize
+  call MPI_FINALIZE(ierror)
+
+end program fft_r2c_x
+

From fc5250bc6a5f3ff5eba23d8a0f1025920b2b6cb7 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 7 Dec 2022 06:58:00 +0000
Subject: [PATCH 025/436] Split FFT physical Z into C2C and R2C tests

---
 examples/Makefile                             |  12 +-
 examples/fft_physical_z/Makefile              |  21 +--
 .../{fft_physical_z.f90 => fft_c2c_z.f90}     |  82 +-----------
 examples/fft_physical_z/fft_r2c_z.f90         | 121 ++++++++++++++++++
 4 files changed, 142 insertions(+), 94 deletions(-)
 rename examples/fft_physical_z/{fft_physical_z.f90 => fft_c2c_z.f90} (62%)
 create mode 100644 examples/fft_physical_z/fft_r2c_z.f90

diff --git a/examples/Makefile b/examples/Makefile
index 7ddce446..5934165d 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,15 +1,15 @@
-.PHONY: test2d fft_physical_z fft_physical_x halo_test init_test io_test 
+.PHONY: test2d fft_physical_x fft_physical_z halo_test init_test io_test 
 
 # Just build the examples
-examples: test2d fft_physical_z fft_physical_x halo_test init_test io_test
+examples: test2d fft_physical_x fft_physical_z halo_test init_test io_test
 	@echo "Built the examples"
 
 test2d:
 	$(MAKE) -C $@ $@
-fft_physical_z:
-	$(MAKE) -C $@ $@
 fft_physical_x:
 	$(MAKE) -C $@ all
+fft_physical_z:
+	$(MAKE) -C $@ all
 halo_test:
 	$(MAKE) -C $@ $@
 init_test:
@@ -19,16 +19,16 @@ io_test:
 
 check:
 	cd test2d; $(MAKE) $@
-	cd fft_physical_z; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
+	cd fft_physical_z; $(MAKE) $@
 	cd halo_test; $(MAKE) $@
 	cd init_test; $(MAKE) $@
 	cd io_test; $(MAKE) $@
 
 clean:
 	cd test2d; $(MAKE) $@
-	cd fft_physical_z; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
+	cd fft_physical_z; $(MAKE) $@
 	cd halo_test; $(MAKE) $@
 	cd init_test; $(MAKE) $@
 	cd io_test; $(MAKE) $@
diff --git a/examples/fft_physical_z/Makefile b/examples/fft_physical_z/Makefile
index 396ff848..ce3e2aea 100644
--- a/examples/fft_physical_z/Makefile
+++ b/examples/fft_physical_z/Makefile
@@ -5,27 +5,32 @@ FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
 FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
 LIBS = -L../../ -l$(LIBDECOMP) $(LIBFFT) $(LFLAGS)
 
-OBJ = fft_physical_z.o
-
 NP ?= 1
 MPIRUN ?= mpirun
 
-fft_physical_z: $(OBJ)
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $(OBJ) $(LIBS)
+all: fft_c2c_z fft_r2c_z 
+
+fft_c2c_z: fft_c2c_z.o
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
+
+fft_r2c_z: fft_r2c_z.o
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
 
 ifeq ($(PARAMOD),gpu)
 check:
-	$(MPIRUN) -n $(NP) ./bind.sh ./fft_physical_z
+	$(MPIRUN) -n $(NP) ./bind.sh ./fft_c2c_z
+	$(MPIRUN) -n $(NP) ./bind.sh ./fft_r2c_z
 else
 check:
-	$(MPIRUN) -n $(NP) ./fft_physical_z
+	$(MPIRUN) -n $(NP) ./fft_c2c_z
+	$(MPIRUN) -n $(NP) ./fft_r2c_z
 endif
 
 mem_leak:
-	valgrind --leak-check=full --show-leak-kinds=all $(MPIRUN) -n 1 ./fft_physical_z 1 1
+	valgrind --leak-check=full --show-leak-kinds=all $(MPIRUN) -n 1 ./fft_c2c_z 1 1
 
 clean:
-	rm -f *.o fft_physical_z *.log
+	rm -f *.o fft_c2c_z fft_r2c_z *.log
 
 %.o : %.f90
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/fft_physical_z/fft_physical_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
similarity index 62%
rename from examples/fft_physical_z/fft_physical_z.f90
rename to examples/fft_physical_z/fft_c2c_z.f90
index 5610b8e9..7e354dc3 100644
--- a/examples/fft_physical_z/fft_physical_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -1,4 +1,4 @@
-program fft_physical_z
+program fft_c2c_z
 
   use decomp_2d
   use decomp_2d_fft
@@ -124,85 +124,7 @@ program fft_physical_z
   
   deallocate(in,out)
   call decomp_2d_fft_finalize
-
-
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the r2c/c2r interface
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
-  
-  allocate (in_r(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
-  allocate (out(fft_start(1):fft_end(1), &
-                fft_start(2):fft_end(2), &
-                fft_start(3):fft_end(3)))
-  
-  ! initilise input
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-     
-     ! 3D r2c FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in_r, out)
-     t2 = t2 + MPI_WTIME() - t1
-  
-     ! 3D inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in_r)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     !$acc kernels
-     in_r = in_r / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
-     !$acc end kernels     
-
-  end do
-#if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
-#endif 
-
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           error = error + abs(in_r(i,j,k)-dr)
-           !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
-        end do
-     end do
-  end do
-!10 format('in_r final ', I2,1x,I2,1x,I2,1x,I2,1x,F12.6,1x,F12.6)
-
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-  
-  if (nrank==0) then
-     write(*,*) '===== r2c/c2r interface ====='
-     write(*,*) 'error / mesh point: ', err_all
-     write(*,*) 'time (sec): ', t1,t3
-  end if
-  
-  deallocate(in_r,out)
-  call decomp_2d_fft_finalize
   call decomp_2d_finalize
   call MPI_FINALIZE(ierror)
 
-end program fft_physical_z
+end program fft_c2c_z
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
new file mode 100644
index 00000000..52d214b6
--- /dev/null
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -0,0 +1,121 @@
+program fft_r2c_z
+
+  use decomp_2d
+  use decomp_2d_fft
+  use MPI
+#if defined(_GPU) 
+  use cudafor
+  use cufft
+  use openacc 
+#endif
+
+  implicit none
+  
+  !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
+  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
+  integer :: nx, ny, nz
+  integer :: p_row=0, p_col=0
+  integer :: resize_domain
+  integer :: nranks_tot
+  
+  integer, parameter :: ntest = 10  ! repeat test this times
+  
+  complex(mytype), allocatable, dimension(:,:,:) :: in, out
+  real(mytype), allocatable, dimension(:,:,:) :: in_r
+
+  integer, dimension(3) :: fft_start, fft_end, fft_size
+  
+  real(mytype) :: dr,di, error, err_all, n1,flops
+  integer :: ierror, i,j,k,m
+  real(mytype) :: t1, t2, t3 ,t4
+  
+  call MPI_INIT(ierror)
+  ! To resize the domain we need to know global number of ranks
+  ! This operation is also done as part of decomp_2d_init
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+  resize_domain = int(nranks_tot/4)+1
+  nx = nx_base*resize_domain
+  ny = ny_base*resize_domain
+  nz = nz_base*resize_domain
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  ! Test the r2c/c2r interface
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
+  
+  allocate (in_r(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
+  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
+  allocate (out(fft_start(1):fft_end(1), &
+                fft_start(2):fft_end(2), &
+                fft_start(3):fft_end(3)))
+  
+  ! initilise input
+  do k=zstart(3),zend(3)
+     do j=zstart(2),zend(2)
+        do i=zstart(1),zend(1)
+           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+        end do
+     end do
+  end do
+  
+  t2 = 0._mytype
+  t4 = 0._mytype
+  do m=1,ntest
+     
+     ! 3D r2c FFT
+     t1 = MPI_WTIME()
+     call decomp_2d_fft_3d(in_r, out)
+     t2 = t2 + MPI_WTIME() - t1
+  
+     ! 3D inverse FFT
+     t3 = MPI_WTIME()
+     call decomp_2d_fft_3d(out, in_r)
+     t4 = t4 + MPI_WTIME() - t3
+  
+     !$acc kernels
+     in_r = in_r / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
+     !$acc end kernels     
+
+  end do
+#if defined(_GPU)
+  ierror = cudaDeviceSynchronize()
+#endif 
+
+  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t1 = t1 / real(nproc,mytype)
+  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t3 = t3 / real(nproc,mytype)
+  
+  ! checking accuracy
+  error = 0._mytype
+  do k=zstart(3),zend(3)
+     do j=zstart(2),zend(2)
+        do i=zstart(1),zend(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           error = error + abs(in_r(i,j,k)-dr)
+           !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
+        end do
+     end do
+  end do
+!10 format('in_r final ', I2,1x,I2,1x,I2,1x,I2,1x,F12.6,1x,F12.6)
+
+  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
+  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
+  
+  if (nrank==0) then
+     write(*,*) '===== r2c/c2r interface ====='
+     write(*,*) 'error / mesh point: ', err_all
+     write(*,*) 'time (sec): ', t1,t3
+  end if
+  
+  deallocate(in_r,out)
+  call decomp_2d_fft_finalize
+  call decomp_2d_finalize
+  call MPI_FINALIZE(ierror)
+
+end program fft_r2c_z

From 0dfb63d1a8ee11da33447afc78bc74c0b90ebfb4 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 7 Dec 2022 07:20:28 +0000
Subject: [PATCH 026/436] Update to comply with BUILD=dev by removing unused
 variables

---
 examples/fft_physical_x/fft_c2c_x.f90 | 3 ---
 examples/fft_physical_x/fft_r2c_x.f90 | 4 ++--
 examples/fft_physical_z/fft_c2c_z.f90 | 3 ---
 examples/fft_physical_z/fft_r2c_z.f90 | 4 ++--
 4 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 94bf1d4d..2c48ddab 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -20,9 +20,6 @@ program fft_c2c_x
   integer, parameter :: ntest = 10  ! repeat test this times
   
   complex(mytype), allocatable, dimension(:,:,:) :: in, out
-  real(mytype), allocatable, dimension(:,:,:) :: in_r
-  
-  integer, dimension(3) :: fft_start, fft_end, fft_size
   
   real(mytype) :: dr,di, error, err_all, n1,flops
   integer :: ierror, i,j,k,m
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 207e275e..045b8264 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -19,12 +19,12 @@ program fft_r2c_x
   
   integer, parameter :: ntest = 10  ! repeat test this times
   
-  complex(mytype), allocatable, dimension(:,:,:) :: in, out
+  complex(mytype), allocatable, dimension(:,:,:) :: out
   real(mytype), allocatable, dimension(:,:,:) :: in_r
   
   integer, dimension(3) :: fft_start, fft_end, fft_size
   
-  real(mytype) :: dr,di, error, err_all, n1,flops
+  real(mytype) :: dr, error, err_all
   integer :: ierror, i,j,k,m
   real(mytype) :: t1, t2, t3 ,t4
   
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 7e354dc3..ed346cc4 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -21,10 +21,7 @@ program fft_c2c_z
   integer, parameter :: ntest = 10  ! repeat test this times
   
   complex(mytype), allocatable, dimension(:,:,:) :: in, out
-  real(mytype), allocatable, dimension(:,:,:) :: in_r
 
-  integer, dimension(3) :: fft_start, fft_end, fft_size
-  
   real(mytype) :: dr,di, error, err_all, n1,flops
   integer :: ierror, i,j,k,m
   real(mytype) :: t1, t2, t3 ,t4
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 52d214b6..86fbbd76 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -20,12 +20,12 @@ program fft_r2c_z
   
   integer, parameter :: ntest = 10  ! repeat test this times
   
-  complex(mytype), allocatable, dimension(:,:,:) :: in, out
+  complex(mytype), allocatable, dimension(:,:,:) :: out
   real(mytype), allocatable, dimension(:,:,:) :: in_r
 
   integer, dimension(3) :: fft_start, fft_end, fft_size
   
-  real(mytype) :: dr,di, error, err_all, n1,flops
+  real(mytype) :: dr, error, err_all
   integer :: ierror, i,j,k,m
   real(mytype) :: t1, t2, t3 ,t4
   

From 13814db122af04f72d1243204c9b7e7a6b44925f Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 8 Dec 2022 09:44:29 +0100
Subject: [PATCH 027/436] Apply fprettify --enable-replacements --c-relations

---
 examples/fft_physical_x/fft_grid_x.f90     |  234 +-
 examples/fft_physical_x/fft_physical_x.f90 |  377 ++-
 examples/fft_physical_z/fft_physical_z.f90 |  378 ++-
 examples/halo_test/halo_test.f90           |  772 ++---
 examples/init_test/init_test.f90           |  120 +-
 examples/io_test/io_bench.f90              |   50 +-
 examples/io_test/io_plane_test.f90         |  230 +-
 examples/io_test/io_read.f90               |  128 +-
 examples/io_test/io_test.f90               |  210 +-
 examples/io_test/io_var_test.f90           |  552 ++--
 examples/test2d/test2d.f90                 |  260 +-
 src/alloc.f90                              |  424 +--
 src/decomp_2d.f90                          | 3077 ++++++++++---------
 src/decomp_2d_init_fin.f90                 |  591 ++--
 src/factor.f90                             |  150 +-
 src/fft_common.f90                         |  221 +-
 src/fft_common_3d.f90                      |  284 +-
 src/fft_cufft.f90                          | 1558 +++++-----
 src/fft_fftw3.f90                          |  937 +++---
 src/fft_fftw3_f03.f90                      | 1507 +++++----
 src/fft_generic.f90                        |  536 ++--
 src/fft_mkl.f90                            | 1048 ++++---
 src/glassman.f90                           |  329 +-
 src/halo.f90                               |  120 +-
 src/halo_common.f90                        |  366 +--
 src/io.f90                                 | 3219 ++++++++++----------
 src/io_read_inflow.f90                     |   80 +-
 src/io_read_one.inc                        |   64 +-
 src/io_read_var.inc                        |   72 +-
 src/io_write_every.inc                     |  218 +-
 src/io_write_one.inc                       |   84 +-
 src/io_write_outflow.f90                   |   80 +-
 src/io_write_plane.inc                     |  172 +-
 src/io_write_var.inc                       |   70 +-
 src/log.f90                                |  454 +--
 src/transpose_x_to_y.f90                   |  590 ++--
 src/transpose_y_to_x.f90                   |  589 ++--
 src/transpose_y_to_z.f90                   |  610 ++--
 src/transpose_z_to_y.f90                   |  605 ++--
 39 files changed, 10630 insertions(+), 10736 deletions(-)

diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 5b424138..7955a81e 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -1,130 +1,130 @@
 program fft_physical_x
 
-  use decomp_2d
-  use decomp_2d_fft
-  use MPI
-#if defined(_GPU) 
-  use cudafor
-  use cufft
-  use openacc 
+   use decomp_2d
+   use decomp_2d_fft
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use cufft
+   use openacc
 #endif
 
-  implicit none
-  
-  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
-  integer :: nx, ny, nz
-  integer :: p_row=0, p_col=0
-  integer :: resize_domain
-  integer :: nranks_tot
-  
-  integer, parameter :: ntest = 10  ! repeat test this times
- 
-  type(decomp_info), pointer :: ph=>null()
-  complex(mytype), allocatable, dimension(:,:,:) :: in, out
-  
-  real(mytype) :: dr,di, error, err_all, n1,flops
-  integer :: ierror, i,j,k,m
-  real(mytype) :: t1, t2, t3 ,t4
-  
-  call MPI_INIT(ierror)
-  ! To resize the domain we need to know global number of ranks
-  ! This operation is also done as part of decomp_2d_init
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-  resize_domain = int(nranks_tot/4)+1
-  nx = nx_base*resize_domain
-  ny = ny_base*resize_domain
-  nz = nz_base*resize_domain
-  call decomp_2d_init(nx+1,ny+1,nz+1,p_row,p_col)
+   implicit none
+
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+
+   integer, parameter :: ntest = 10  ! repeat test this times
+
+   type(decomp_info), pointer :: ph => null()
+   complex(mytype), allocatable, dimension(:, :, :) :: in, out
+
+   real(mytype) :: dr, di, error, err_all, n1, flops
+   integer :: ierror, i, j, k, m
+   real(mytype) :: t1, t2, t3, t4
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   call decomp_2d_init(nx + 1, ny + 1, nz + 1, p_row, p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the c2c interface
+   ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  call decomp_2d_fft_init(PHYSICAL_IN_X, nx, ny, nz) ! force the default x pencil
-  ph => decomp_2d_fft_get_ph()
-  !  input is X-pencil data
-  ! output is Z-pencil data
-  call alloc_x(in, ph, .true.)
-  call alloc_z(out, ph, .true.)
-  ! initilise input
-  do k=ph%xst(3),ph%xen(3)
-     do j=ph%xst(2),ph%xen(2)
-        do i=ph%xst(1),ph%xen(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           in(i,j,k) = cmplx(dr,di,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-     
-     ! forward FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
-     t2 = t2 + MPI_WTIME() - t1
-
-     ! inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     ! normalisation - note 2DECOMP&FFT doesn't normalise
-     !$acc kernels
-     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
-     !$acc end kernels
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_X, nx, ny, nz) ! force the default x pencil
+   ph => decomp_2d_fft_get_ph()
+   !  input is X-pencil data
+   ! output is Z-pencil data
+   call alloc_x(in, ph, .true.)
+   call alloc_z(out, ph, .true.)
+   ! initilise input
+   do k = ph%xst(3), ph%xen(3)
+      do j = ph%xst(2), ph%xen(2)
+         do i = ph%xst(1), ph%xen(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            in(i, j, k) = cmplx(dr, di, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! forward FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+      t4 = t4 + MPI_WTIME() - t3
+
+      ! normalisation - note 2DECOMP&FFT doesn't normalise
+      !$acc kernels
+      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
+   ierror = cudaDeviceSynchronize()
 #endif
-  
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=ph%xst(3),ph%xen(3) 
-     do j=ph%xst(2),ph%xen(2)
-        do i=ph%xst(1),ph%xen(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           dr = dr - real(in(i,j,k),mytype)
-           di = di - aimag(in(i,j,k))
-           error = error + sqrt(dr*dr + di*di)
-        end do
-     end do
-  end do
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-
-  if (nrank==0) then
-     write(*,*) '===== c2c interface ====='
-     write(*,*) 'error / mesh point: ', err_all 
-     write(*,*) 'time (sec): ', t1,t3
-     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
-     n1 = n1 ** (1._mytype/3._mytype)
-     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
-     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-     flops = flops * 3._mytype * n1**2  
-     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
-     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
-  end if
-  
-  deallocate(in,out)
-  nullify(ph)
-  call decomp_2d_fft_finalize
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = ph%xst(3), ph%xen(3)
+      do j = ph%xst(2), ph%xen(2)
+         do i = ph%xst(1), ph%xen(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            dr = dr - real(in(i, j, k), mytype)
+            di = di - aimag(in(i, j, k))
+            error = error + sqrt(dr*dr + di*di)
+         end do
+      end do
+   end do
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== c2c interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
+      n1 = n1**(1._mytype/3._mytype)
+      ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+      flops = flops*3._mytype*n1**2
+      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+   end if
+
+   deallocate (in, out)
+   nullify (ph)
+   call decomp_2d_fft_finalize
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
 end program fft_physical_x
 
diff --git a/examples/fft_physical_x/fft_physical_x.f90 b/examples/fft_physical_x/fft_physical_x.f90
index 9a999e95..07daf7c9 100644
--- a/examples/fft_physical_x/fft_physical_x.f90
+++ b/examples/fft_physical_x/fft_physical_x.f90
@@ -1,207 +1,206 @@
 program fft_physical_x
 
-  use decomp_2d
-  use decomp_2d_fft
-  use MPI
-#if defined(_GPU) 
-  use cudafor
-  use cufft
-  use openacc 
+   use decomp_2d
+   use decomp_2d_fft
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use cufft
+   use openacc
 #endif
 
-  implicit none
-  
-  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
-  integer :: nx, ny, nz
-  integer :: p_row=0, p_col=0
-  integer :: resize_domain
-  integer :: nranks_tot
-  
-  integer, parameter :: ntest = 10  ! repeat test this times
-  
-  complex(mytype), allocatable, dimension(:,:,:) :: in, out
-  real(mytype), allocatable, dimension(:,:,:) :: in_r
-  
-  integer, dimension(3) :: fft_start, fft_end, fft_size
-  
-  real(mytype) :: dr,di, error, err_all, n1,flops
-  integer :: ierror, i,j,k,m
-  real(mytype) :: t1, t2, t3 ,t4
-  
-  call MPI_INIT(ierror)
-  ! To resize the domain we need to know global number of ranks
-  ! This operation is also done as part of decomp_2d_init
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-  resize_domain = int(nranks_tot/4)+1
-  nx = nx_base*resize_domain
-  ny = ny_base*resize_domain
-  nz = nz_base*resize_domain
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   implicit none
+
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+
+   integer, parameter :: ntest = 10  ! repeat test this times
+
+   complex(mytype), allocatable, dimension(:, :, :) :: in, out
+   real(mytype), allocatable, dimension(:, :, :) :: in_r
+
+   integer, dimension(3) :: fft_start, fft_end, fft_size
+
+   real(mytype) :: dr, di, error, err_all, n1, flops
+   integer :: ierror, i, j, k, m
+   real(mytype) :: t1, t2, t3, t4
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the c2c interface
+   ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  call decomp_2d_fft_init(PHYSICAL_IN_X) ! force the default x pencil
-
-  !  input is X-pencil data
-  ! output is Z-pencil data
-  allocate (in(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-  allocate (out(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-  ! initilise input
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           in(i,j,k) = cmplx(dr,di,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-     
-     ! forward FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
-     t2 = t2 + MPI_WTIME() - t1
-
-     ! inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     ! normalisation - note 2DECOMP&FFT doesn't normalise
-     !$acc kernels
-     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
-     !$acc end kernels
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_X) ! force the default x pencil
+
+   !  input is X-pencil data
+   ! output is Z-pencil data
+   allocate (in(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (out(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   ! initilise input
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            in(i, j, k) = cmplx(dr, di, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! forward FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+      t4 = t4 + MPI_WTIME() - t3
+
+      ! normalisation - note 2DECOMP&FFT doesn't normalise
+      !$acc kernels
+      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
+   ierror = cudaDeviceSynchronize()
 #endif
-  
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           dr = dr - real(in(i,j,k),mytype)
-           di = di - aimag(in(i,j,k))
-           error = error + sqrt(dr*dr + di*di)
-        end do
-     end do
-  end do
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-
-  if (nrank==0) then
-     write(*,*) '===== c2c interface ====='
-     write(*,*) 'error / mesh point: ', err_all 
-     write(*,*) 'time (sec): ', t1,t3
-     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
-     n1 = n1 ** (1._mytype/3._mytype)
-     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
-     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-     flops = flops * 3._mytype * n1**2  
-     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
-     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
-  end if
-  
-  deallocate(in,out)
-  call decomp_2d_fft_finalize
 
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            dr = dr - real(in(i, j, k), mytype)
+            di = di - aimag(in(i, j, k))
+            error = error + sqrt(dr*dr + di*di)
+         end do
+      end do
+   end do
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== c2c interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
+      n1 = n1**(1._mytype/3._mytype)
+      ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+      flops = flops*3._mytype*n1**2
+      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+   end if
+
+   deallocate (in, out)
+   call decomp_2d_fft_finalize
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the r2c/c2r interface
+   ! Test the r2c/c2r interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  call decomp_2d_fft_init(PHYSICAL_IN_X)
-  
-  allocate (in_r(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
-  allocate (out(fft_start(1):fft_end(1), &
-                fft_start(2):fft_end(2), &
-                fft_start(3):fft_end(3)))
-  
-  ! initilise input
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-  
-     ! 3D r2c FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in_r, out)
-     t2 = t2 + MPI_WTIME() - t1
-  
-     ! 3D inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in_r)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     ! normalisation - note 2DECOMP&FFT doesn't normalise
-     !$acc kernels
-     in_r = in_r / (real(nx,mytype)*real(ny,mytype)*real(nz,mytype))
-     !$acc end kernels
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_X)
+
+   allocate (in_r(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   call decomp_2d_fft_get_size(fft_start, fft_end, fft_size)
+   allocate (out(fft_start(1):fft_end(1), &
+                 fft_start(2):fft_end(2), &
+                 fft_start(3):fft_end(3)))
+
+   ! initilise input
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! 3D r2c FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in_r, out)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! 3D inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in_r)
+      t4 = t4 + MPI_WTIME() - t3
+
+      ! normalisation - note 2DECOMP&FFT doesn't normalise
+      !$acc kernels
+      in_r = in_r/(real(nx, mytype)*real(ny, mytype)*real(nz, mytype))
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
-#endif 
-
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           error = error + abs(in_r(i,j,k)-dr)
-        end do
-     end do
-  end do
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-  
-  if (nrank==0) then
-     write(*,*) '===== r2c/c2r interface ====='
-     write(*,*) 'error / mesh point: ', err_all
-     write(*,*) 'time (sec): ', t1,t3
-  end if
-  
-  deallocate(in_r,out)
-  call decomp_2d_fft_finalize
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
+   ierror = cudaDeviceSynchronize()
+#endif
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            error = error + abs(in_r(i, j, k) - dr)
+         end do
+      end do
+   end do
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== r2c/c2r interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+   end if
+
+   deallocate (in_r, out)
+   call decomp_2d_fft_finalize
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
 end program fft_physical_x
 
diff --git a/examples/fft_physical_z/fft_physical_z.f90 b/examples/fft_physical_z/fft_physical_z.f90
index 5610b8e9..fd8e45e5 100644
--- a/examples/fft_physical_z/fft_physical_z.f90
+++ b/examples/fft_physical_z/fft_physical_z.f90
@@ -1,208 +1,206 @@
 program fft_physical_z
 
-  use decomp_2d
-  use decomp_2d_fft
-  use MPI
-#if defined(_GPU) 
-  use cudafor
-  use cufft
-  use openacc 
+   use decomp_2d
+   use decomp_2d_fft
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use cufft
+   use openacc
 #endif
 
-  implicit none
-  
-  !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
-  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
-  integer :: nx, ny, nz
-  integer :: p_row=0, p_col=0
-  integer :: resize_domain
-  integer :: nranks_tot
-  
-  integer, parameter :: ntest = 10  ! repeat test this times
-  
-  complex(mytype), allocatable, dimension(:,:,:) :: in, out
-  real(mytype), allocatable, dimension(:,:,:) :: in_r
-
-  integer, dimension(3) :: fft_start, fft_end, fft_size
-  
-  real(mytype) :: dr,di, error, err_all, n1,flops
-  integer :: ierror, i,j,k,m
-  real(mytype) :: t1, t2, t3 ,t4
-  
-  call MPI_INIT(ierror)
-  ! To resize the domain we need to know global number of ranks
-  ! This operation is also done as part of decomp_2d_init
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-  resize_domain = int(nranks_tot/4)+1
-  nx = nx_base*resize_domain
-  ny = ny_base*resize_domain
-  nz = nz_base*resize_domain
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   implicit none
+
+   !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+
+   integer, parameter :: ntest = 10  ! repeat test this times
+
+   complex(mytype), allocatable, dimension(:, :, :) :: in, out
+   real(mytype), allocatable, dimension(:, :, :) :: in_r
+
+   integer, dimension(3) :: fft_start, fft_end, fft_size
+
+   real(mytype) :: dr, di, error, err_all, n1, flops
+   integer :: ierror, i, j, k, m
+   real(mytype) :: t1, t2, t3, t4
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the c2c interface
+   ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
-  !  input is Z-pencil data
-  ! output is X-pencil data
-  allocate (in(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-  allocate (out(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-  ! initilise input
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           in(i,j,k) = cmplx(dr,di,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-     
-     ! forward FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
-     t2 = t2 + MPI_WTIME() - t1
-
-     ! inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     ! normalisation - note 2DECOMP&FFT doesn't normalise
-     !$acc kernels
-     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
-     !$acc end kernels
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
+   !  input is Z-pencil data
+   ! output is X-pencil data
+   allocate (in(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   allocate (out(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   ! initilise input
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            in(i, j, k) = cmplx(dr, di, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! forward FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+      t4 = t4 + MPI_WTIME() - t3
+
+      ! normalisation - note 2DECOMP&FFT doesn't normalise
+      !$acc kernels
+      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
+   ierror = cudaDeviceSynchronize()
 #endif
-  
-
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           dr = dr - real(in(i,j,k),mytype)
-           di = di - aimag(in(i,j,k))
-           error = error + sqrt(dr*dr + di*di)
-        end do
-     end do
-  end do
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-
-  if (nrank==0) then
-     write(*,*) '===== c2c interface ====='
-     write(*,*) 'error / mesh point: ', err_all 
-     write(*,*) 'time (sec): ', t1,t3
-     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
-     n1 = n1 ** (1._mytype/3._mytype)
-     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
-     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-     flops = flops * 3._mytype * n1**2  
-     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
-     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
-  end if
-  
-  deallocate(in,out)
-  call decomp_2d_fft_finalize
 
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            dr = dr - real(in(i, j, k), mytype)
+            di = di - aimag(in(i, j, k))
+            error = error + sqrt(dr*dr + di*di)
+         end do
+      end do
+   end do
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== c2c interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
+      n1 = n1**(1._mytype/3._mytype)
+      ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+      flops = flops*3._mytype*n1**2
+      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+   end if
+
+   deallocate (in, out)
+   call decomp_2d_fft_finalize
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the r2c/c2r interface
+   ! Test the r2c/c2r interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
-  
-  allocate (in_r(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
-  allocate (out(fft_start(1):fft_end(1), &
-                fft_start(2):fft_end(2), &
-                fft_start(3):fft_end(3)))
-  
-  ! initilise input
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-     
-     ! 3D r2c FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in_r, out)
-     t2 = t2 + MPI_WTIME() - t1
-  
-     ! 3D inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in_r)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     !$acc kernels
-     in_r = in_r / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
-     !$acc end kernels     
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
+
+   allocate (in_r(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   call decomp_2d_fft_get_size(fft_start, fft_end, fft_size)
+   allocate (out(fft_start(1):fft_end(1), &
+                 fft_start(2):fft_end(2), &
+                 fft_start(3):fft_end(3)))
+
+   ! initilise input
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! 3D r2c FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in_r, out)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! 3D inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in_r)
+      t4 = t4 + MPI_WTIME() - t3
+
+      !$acc kernels
+      in_r = in_r/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
-#endif 
-
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           error = error + abs(in_r(i,j,k)-dr)
-           !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
-        end do
-     end do
-  end do
+   ierror = cudaDeviceSynchronize()
+#endif
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            error = error + abs(in_r(i, j, k) - dr)
+            !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
+         end do
+      end do
+   end do
 !10 format('in_r final ', I2,1x,I2,1x,I2,1x,I2,1x,F12.6,1x,F12.6)
 
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-  
-  if (nrank==0) then
-     write(*,*) '===== r2c/c2r interface ====='
-     write(*,*) 'error / mesh point: ', err_all
-     write(*,*) 'time (sec): ', t1,t3
-  end if
-  
-  deallocate(in_r,out)
-  call decomp_2d_fft_finalize
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== r2c/c2r interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+   end if
+
+   deallocate (in_r, out)
+   call decomp_2d_fft_finalize
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
 end program fft_physical_z
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 9f52a2bd..2222d3c4 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -6,495 +6,495 @@
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 program halo_test
 
-  use mpi
+   use mpi
 
-  use decomp_2d
+   use decomp_2d
 
-  implicit none
+   implicit none
 
-  integer, parameter :: nx=171, ny=132, nz=113
-  integer :: p_row=0, p_col=0
+   integer, parameter :: nx = 171, ny = 132, nz = 113
+   integer :: p_row = 0, p_col = 0
 
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  real(mytype), allocatable, dimension(:,:,:) :: v1, v2, v3
-  real(mytype), allocatable, dimension(:,:,:) :: w1, w2, w3
-  real(mytype), allocatable, dimension(:,:,:) :: wk2, wk3
-  real(mytype), allocatable, dimension(:,:,:) :: uh, vh, wh
-  real(mytype), allocatable, dimension(:,:,:) :: div1, div2, div3, div4
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   real(mytype), allocatable, dimension(:, :, :) :: v1, v2, v3
+   real(mytype), allocatable, dimension(:, :, :) :: w1, w2, w3
+   real(mytype), allocatable, dimension(:, :, :) :: wk2, wk3
+   real(mytype), allocatable, dimension(:, :, :) :: uh, vh, wh
+   real(mytype), allocatable, dimension(:, :, :) :: div1, div2, div3, div4
 
-  integer :: i,j,k, ierror, n
+   integer :: i, j, k, ierror, n
 
-  integer, allocatable, dimension(:) :: seed
+   integer, allocatable, dimension(:) :: seed
 
-  real(mytype) :: err
-  integer :: xlast, ylast, zlast
+   real(mytype) :: err
+   integer :: xlast, ylast, zlast
 
-  integer :: nx_expected, ny_expected, nz_expected
-  
-  logical :: passing, all_pass
+   integer :: nx_expected, ny_expected, nz_expected
 
-  call MPI_INIT(ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   logical :: passing, all_pass
 
-  xlast = xsize(1) - 1
-  if (xend(2) == ny) then
-     ylast = xsize(2) - 1
-  else
-     ylast = xsize(2)
-  end if
-  if (xend(3) == nz) then
-     zlast = xsize(3) - 1
-  else
-     zlast = xsize(3)
-  end if
+   call MPI_INIT(ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-  call initialise()
-  call test_div_transpose()
-  call test_div_haloX()
-  call test_div_haloY()
-  call test_div_haloZ()
+   xlast = xsize(1) - 1
+   if (xend(2) == ny) then
+      ylast = xsize(2) - 1
+   else
+      ylast = xsize(2)
+   end if
+   if (xend(3) == nz) then
+      zlast = xsize(3) - 1
+   else
+      zlast = xsize(3)
+   end if
 
-  if (nrank == 0) then
-     write(*,*) '-----------------------------------------------'
-     write(*,*) "All pass: ", all_pass
-     write(*,*) '==============================================='
-  end if
+   call initialise()
+   call test_div_transpose()
+   call test_div_haloX()
+   call test_div_haloY()
+   call test_div_haloZ()
 
-  deallocate(u1,v1,w1,u2,v2,w2,u3,v3,w3)
-  deallocate(div1,div2,div3,div4)
+   if (nrank == 0) then
+      write (*, *) '-----------------------------------------------'
+      write (*, *) "All pass: ", all_pass
+      write (*, *) '==============================================='
+   end if
 
-  call decomp_2d_finalize 
+   deallocate (u1, v1, w1, u2, v2, w2, u3, v3, w3)
+   deallocate (div1, div2, div3, div4)
 
-  if (.not. all_pass) call decomp_2d_abort(1, "Error in halo_test")
+   call decomp_2d_finalize
 
-  call MPI_FINALIZE(ierror)
+   if (.not. all_pass) call decomp_2d_abort(1, "Error in halo_test")
+
+   call MPI_FINALIZE(ierror)
 
 contains
 
-  subroutine initialise()
+   subroutine initialise()
 
-    ! initialise u,v,w with random numbers in X-pencil
+      ! initialise u,v,w with random numbers in X-pencil
 #ifdef HALO_GLOBAL
-    allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    allocate(v1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    allocate(w1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (v1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (w1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
 #else
-    allocate(u1(xsize(1), xsize(2), xsize(3)))
-    allocate(v1(xsize(1), xsize(2), xsize(3)))
-    allocate(w1(xsize(1), xsize(2), xsize(3)))
+      allocate (u1(xsize(1), xsize(2), xsize(3)))
+      allocate (v1(xsize(1), xsize(2), xsize(3)))
+      allocate (w1(xsize(1), xsize(2), xsize(3)))
 #endif
 
-    call random_seed(size = n)
-    allocate(seed(n))
-    seed = nrank+1
-    call random_seed(put=seed)
-    call random_number(u1)
-    call random_number(v1)
-    call random_number(w1)
+      call random_seed(size=n)
+      allocate (seed(n))
+      seed = nrank + 1
+      call random_seed(put=seed)
+      call random_number(u1)
+      call random_number(v1)
+      call random_number(w1)
 
-    all_pass = .true.
+      all_pass = .true.
 
-  end subroutine initialise
+   end subroutine initialise
 
-  !=====================================================================
-  ! Calculate divergence using global transposition
-  !=====================================================================
-  subroutine test_div_transpose()
+   !=====================================================================
+   ! Calculate divergence using global transposition
+   !=====================================================================
+   subroutine test_div_transpose()
 
-    integer :: i1, in ! I loop start/end
-    integer :: j1, jn ! J loop start/end
-    integer :: k1, kn ! K loop start/end
+      integer :: i1, in ! I loop start/end
+      integer :: j1, jn ! J loop start/end
+      integer :: k1, kn ! K loop start/end
 
-    ! du/dx calculated on X-pencil
+      ! du/dx calculated on X-pencil
 #ifdef HALO_GLOBAL
-    allocate(div1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    k1 = xstart(3); kn = xend(3)
-    j1 = xstart(2); jn = xend(2)
+      allocate (div1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      k1 = xstart(3); kn = xend(3)
+      j1 = xstart(2); jn = xend(2)
 #else
-    allocate(div1(xsize(1), xsize(2), xsize(3)))
-    k1 = 1; kn = xsize(3)
-    j1 = 1; jn = xsize(2)
+      allocate (div1(xsize(1), xsize(2), xsize(3)))
+      k1 = 1; kn = xsize(3)
+      j1 = 1; jn = xsize(2)
 #endif
-    i1 = 2; in = xsize(1) - 1
-
-    div1 = 0.0_mytype
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             div1(i,j,k) = u1(i+1,j,k)-u1(i-1,j,k)
-          end do
-       end do
-    end do
-
-    ! dv/dy calculated on Y-pencil
+      i1 = 2; in = xsize(1) - 1
+
+      div1 = 0.0_mytype
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               div1(i, j, k) = u1(i + 1, j, k) - u1(i - 1, j, k)
+            end do
+         end do
+      end do
+
+      ! dv/dy calculated on Y-pencil
 #ifdef HALO_GLOBAL
-    allocate(v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    k1 = ystart(3); kn = yend(3)
-    i1 = ystart(1); in = yend(1)
+      allocate (v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      k1 = ystart(3); kn = yend(3)
+      i1 = ystart(1); in = yend(1)
 #else
-    allocate(v2(ysize(1), ysize(2), ysize(3)))
-    allocate(wk2(ysize(1), ysize(2), ysize(3)))
-    k1 = 1; kn = ysize(3)
-    i1 = 1; in = ysize(1)
+      allocate (v2(ysize(1), ysize(2), ysize(3)))
+      allocate (wk2(ysize(1), ysize(2), ysize(3)))
+      k1 = 1; kn = ysize(3)
+      i1 = 1; in = ysize(1)
 #endif
-    j1 = 2; jn = ysize(2) - 1
+      j1 = 2; jn = ysize(2) - 1
 
-    call transpose_x_to_y(v1,v2)
-    call transpose_x_to_y(div1,wk2)
+      call transpose_x_to_y(v1, v2)
+      call transpose_x_to_y(div1, wk2)
 
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             wk2(i,j,k) = wk2(i,j,k) + v2(i,j+1,k)-v2(i,j-1,k)
-          end do
-       end do
-    end do
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               wk2(i, j, k) = wk2(i, j, k) + v2(i, j + 1, k) - v2(i, j - 1, k)
+            end do
+         end do
+      end do
 
-    ! dw/dz calculated on Z-pencil
+      ! dw/dz calculated on Z-pencil
 #ifdef HALO_GLOBAL
-    allocate(w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-    allocate(wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-    j1 = zstart(2); jn = zend(2)
-    i1 = zstart(1); in = zend(1)
+      allocate (w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      j1 = zstart(2); jn = zend(2)
+      i1 = zstart(1); in = zend(1)
 #else
-    allocate(w2(ysize(1), ysize(2), ysize(3)))
-    allocate(w3(zsize(1), zsize(2), zsize(3)))
-    allocate(wk3(zsize(1), zsize(2), zsize(3)))
-    j1 = 1; jn = zsize(2)
-    i1 = 1; in = zsize(1)
+      allocate (w2(ysize(1), ysize(2), ysize(3)))
+      allocate (w3(zsize(1), zsize(2), zsize(3)))
+      allocate (wk3(zsize(1), zsize(2), zsize(3)))
+      j1 = 1; jn = zsize(2)
+      i1 = 1; in = zsize(1)
 #endif
-    k1 = 2; kn = zsize(3) - 1
-    
-    call transpose_x_to_y(w1,w2)
-    call transpose_y_to_z(w2,w3)
-    call transpose_y_to_z(wk2,wk3)
-
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             wk3(i,j,k) = wk3(i,j,k) + w3(i,j,k+1)-w3(i,j,k-1)
-          end do
-       end do
-    end do
-
-    ! result in X-pencil
-    call transpose_z_to_y(wk3,wk2)
-    call transpose_y_to_x(wk2,div1)
-
-    if (nrank==0) then
-       write(*,*) 'Calculated via global transposition'
+      k1 = 2; kn = zsize(3) - 1
+
+      call transpose_x_to_y(w1, w2)
+      call transpose_y_to_z(w2, w3)
+      call transpose_y_to_z(wk2, wk3)
+
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
+            end do
+         end do
+      end do
+
+      ! result in X-pencil
+      call transpose_z_to_y(wk3, wk2)
+      call transpose_y_to_x(wk2, div1)
+
+      if (nrank == 0) then
+         write (*, *) 'Calculated via global transposition'
 #ifdef DEBUG
-       write(*,*) (div1(i,i,i), i=2,13)
+         write (*, *) (div1(i, i, i), i=2, 13)
 #endif
-    end if
-
-    deallocate(v2,w2,w3,wk2,wk3)
-
-  end subroutine test_div_transpose
-
-  !=====================================================================
-  ! Calculate divergence using halo-cell exchange (data in X-pencil)
-  !=====================================================================
-  subroutine test_div_haloX()
-
-    integer :: i1, in ! I loop start/end
-    integer :: j1, jn ! J loop start/end
-    integer :: k1, kn ! K loop start/end
-    
-    ! Expected sizes
-    nx_expected = nx
-    ny_expected = xsize(2) + 2
-    nz_expected = xsize(3) + 2
-    
+      end if
+
+      deallocate (v2, w2, w3, wk2, wk3)
+
+   end subroutine test_div_transpose
+
+   !=====================================================================
+   ! Calculate divergence using halo-cell exchange (data in X-pencil)
+   !=====================================================================
+   subroutine test_div_haloX()
+
+      integer :: i1, in ! I loop start/end
+      integer :: j1, jn ! J loop start/end
+      integer :: k1, kn ! K loop start/end
+
+      ! Expected sizes
+      nx_expected = nx
+      ny_expected = xsize(2) + 2
+      nz_expected = xsize(3) + 2
+
 #ifdef HALO_GLOBAL
-    allocate(div2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    call update_halo(v1,vh,1,opt_global=.true.,opt_pencil=1)
-    call update_halo(w1,wh,1,opt_global=.true.,opt_pencil=1)
-    
-    k1 = xstart(3); kn = xend(3)
-    j1 = xstart(2); jn = xend(2)
+      allocate (div2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      call update_halo(v1, vh, 1, opt_global=.true., opt_pencil=1)
+      call update_halo(w1, wh, 1, opt_global=.true., opt_pencil=1)
+
+      k1 = xstart(3); kn = xend(3)
+      j1 = xstart(2); jn = xend(2)
 #else
-    allocate(div2(xsize(1), xsize(2), xsize(3)))
-    call update_halo(v1,vh,1,opt_pencil=1)
-    call update_halo(w1,wh,1,opt_pencil=1)
+      allocate (div2(xsize(1), xsize(2), xsize(3)))
+      call update_halo(v1, vh, 1, opt_pencil=1)
+      call update_halo(w1, wh, 1, opt_pencil=1)
 
-    k1 = 1; kn = xsize(3)
-    j1 = 1; jn = xsize(2)
+      k1 = 1; kn = xsize(3)
+      j1 = 1; jn = xsize(2)
 #endif
-    i1 = 2; in = xsize(1) - 1
-
-    call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "X:v")
-    call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
-    
-    div2 = 0.0_mytype
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             div2(i,j,k) = (u1(i+1,j,k)-u1(i-1,j,k)) &
-                  + (vh(i,j+1,k)-vh(i,j-1,k)) &
-                  + (wh(i,j,k+1)-wh(i,j,k-1))
-          end do
-       end do
-    end do
-
-    ! Compute error
-    call check_err(div2, "X")
-           
-    deallocate(vh,wh)
-
-  end subroutine test_div_haloX
-
-  !=====================================================================
-  ! Calculate divergence using halo-cell exchange (data in Y-pencil)
-  !=====================================================================
-  subroutine test_div_haloY()
-
-    integer :: i1, in ! I loop start/end
-    integer :: j1, jn ! J loop start/end
-    integer :: k1, kn ! K loop start/end
-
-    ! Expected sizes
-    nx_expected = ysize(1) + 2
-    ny_expected = ny
-    nz_expected = ysize(3) + 2
-  
+      i1 = 2; in = xsize(1) - 1
+
+      call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "X:v")
+      call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
+
+      div2 = 0.0_mytype
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               div2(i, j, k) = (u1(i + 1, j, k) - u1(i - 1, j, k)) &
+                               + (vh(i, j + 1, k) - vh(i, j - 1, k)) &
+                               + (wh(i, j, k + 1) - wh(i, j, k - 1))
+            end do
+         end do
+      end do
+
+      ! Compute error
+      call check_err(div2, "X")
+
+      deallocate (vh, wh)
+
+   end subroutine test_div_haloX
+
+   !=====================================================================
+   ! Calculate divergence using halo-cell exchange (data in Y-pencil)
+   !=====================================================================
+   subroutine test_div_haloY()
+
+      integer :: i1, in ! I loop start/end
+      integer :: j1, jn ! J loop start/end
+      integer :: k1, kn ! K loop start/end
+
+      ! Expected sizes
+      nx_expected = ysize(1) + 2
+      ny_expected = ny
+      nz_expected = ysize(3) + 2
+
 #ifdef HALO_GLOBAL
-    allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(div3(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    allocate(wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (div3(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
 #else
-    allocate(u2(ysize(1), ysize(2), ysize(3)))
-    allocate(v2(ysize(1), ysize(2), ysize(3)))
-    allocate(w2(ysize(1), ysize(2), ysize(3)))
-    allocate(div3(xsize(1), xsize(2), xsize(3)))
-    allocate(wk2(ysize(1), ysize(2), ysize(3)))
+      allocate (u2(ysize(1), ysize(2), ysize(3)))
+      allocate (v2(ysize(1), ysize(2), ysize(3)))
+      allocate (w2(ysize(1), ysize(2), ysize(3)))
+      allocate (div3(xsize(1), xsize(2), xsize(3)))
+      allocate (wk2(ysize(1), ysize(2), ysize(3)))
 #endif
-    call transpose_x_to_y(u1,u2)
-    call transpose_x_to_y(v1,v2)
-    call transpose_x_to_y(w1,w2)
+      call transpose_x_to_y(u1, u2)
+      call transpose_x_to_y(v1, v2)
+      call transpose_x_to_y(w1, w2)
 
-    ! du/dx
+      ! du/dx
 #ifdef HALO_GLOBAL
-    call update_halo(u2,uh,1,opt_global=.true.,opt_pencil=2)
-    call update_halo(w2,wh,1,opt_global=.true.,opt_pencil=2)
-    k1 = ystart(3); kn = yend(3)
-    i1 = ystart(1); in = yend(1)
+      call update_halo(u2, uh, 1, opt_global=.true., opt_pencil=2)
+      call update_halo(w2, wh, 1, opt_global=.true., opt_pencil=2)
+      k1 = ystart(3); kn = yend(3)
+      i1 = ystart(1); in = yend(1)
 #else
-    call update_halo(u2,uh,1,opt_pencil=2)
-    call update_halo(w2,wh,1,opt_pencil=2)
-    k1 = 1; kn = ysize(3)
-    i1 = 1; in = ysize(1)
+      call update_halo(u2, uh, 1, opt_pencil=2)
+      call update_halo(w2, wh, 1, opt_pencil=2)
+      k1 = 1; kn = ysize(3)
+      i1 = 1; in = ysize(1)
 #endif
-    j1 = 2; jn = ysize(2) - 1
+      j1 = 2; jn = ysize(2) - 1
 
-    call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Y:u")
-    call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "Y:w")
+      call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Y:u")
+      call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "Y:w")
 
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             wk2(i,j,k) = (uh(i+1,j,k)-uh(i-1,j,k)) &
-                  + (v2(i,j+1,k)-v2(i,j-1,k)) &
-                  + (wh(i,j,k+1)-wh(i,j,k-1))
-          end do
-       end do
-    end do
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               wk2(i, j, k) = (uh(i + 1, j, k) - uh(i - 1, j, k)) &
+                              + (v2(i, j + 1, k) - v2(i, j - 1, k)) &
+                              + (wh(i, j, k + 1) - wh(i, j, k - 1))
+            end do
+         end do
+      end do
 
-    call transpose_y_to_x(wk2,div3)
+      call transpose_y_to_x(wk2, div3)
 
-    ! Compute error
-    call check_err(div3, "Y")
+      ! Compute error
+      call check_err(div3, "Y")
 
-    deallocate(uh,wh,wk2)
+      deallocate (uh, wh, wk2)
 
-  end subroutine test_div_haloY
+   end subroutine test_div_haloY
 
-  !=====================================================================
-  ! Calculate divergence using halo-cell exchange (data in Z-pencil)
-  !=====================================================================
-  subroutine test_div_haloZ()
+   !=====================================================================
+   ! Calculate divergence using halo-cell exchange (data in Z-pencil)
+   !=====================================================================
+   subroutine test_div_haloZ()
 
-  ! Expected sizes
-  nx_expected = zsize(1) + 2
-  ny_expected = zsize(2) + 2
-  nz_expected = nz
+      ! Expected sizes
+      nx_expected = zsize(1) + 2
+      ny_expected = zsize(2) + 2
+      nz_expected = nz
 
 #ifdef HALO_GLOBAL
-  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  allocate(v3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  allocate(w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (v3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
 #else
-  allocate(u3(zsize(1), zsize(2), zsize(3)))
-  allocate(v3(zsize(1), zsize(2), zsize(3)))
-  allocate(w3(zsize(1), zsize(2), zsize(3)))
+      allocate (u3(zsize(1), zsize(2), zsize(3)))
+      allocate (v3(zsize(1), zsize(2), zsize(3)))
+      allocate (w3(zsize(1), zsize(2), zsize(3)))
 #endif
-  call transpose_y_to_z(u2,u3)
-  call transpose_y_to_z(v2,v3)
-  call transpose_y_to_z(w2,w3)
+      call transpose_y_to_z(u2, u3)
+      call transpose_y_to_z(v2, v3)
+      call transpose_y_to_z(w2, w3)
 
 #ifdef HALO_GLOBAL
-  allocate(div4(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (div4(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
 #else
-  allocate(div4(xsize(1), xsize(2), xsize(3)))
-  allocate(wk2(ysize(1), ysize(2), ysize(3)))
-  allocate(wk3(zsize(1), zsize(2), zsize(3)))
+      allocate (div4(xsize(1), xsize(2), xsize(3)))
+      allocate (wk2(ysize(1), ysize(2), ysize(3)))
+      allocate (wk3(zsize(1), zsize(2), zsize(3)))
 #endif
 
-  ! du/dx
+      ! du/dx
 #ifdef HALO_GLOBAL
-  call update_halo(u3,uh,1,opt_global=.true.,opt_pencil=3)
+      call update_halo(u3, uh, 1, opt_global=.true., opt_pencil=3)
 #else
-  call update_halo(u3,uh,1,opt_pencil=3)
+      call update_halo(u3, uh, 1, opt_pencil=3)
 #endif
 
-  call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
-  
+      call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
+
 #ifdef HALO_GLOBAL
-  do j=zstart(2),zend(2)
-     do i=zstart(1),zend(1)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
 #else
-  do j=1,zsize(2)
-     do i=1,zsize(1)
+            do j = 1, zsize(2)
+               do i = 1, zsize(1)
 #endif
-        do k=2,zsize(3)-1
-           wk3(i,j,k) = uh(i+1,j,k)-uh(i-1,j,k)
-        end do
-     end do
-  end do
+                  do k = 2, zsize(3) - 1
+                     wk3(i, j, k) = uh(i + 1, j, k) - uh(i - 1, j, k)
+                  end do
+               end do
+            end do
 
-  ! dv/dy
+            ! dv/dy
 #ifdef HALO_GLOBAL
-  call update_halo(v3,vh,1,opt_global=.true.,opt_pencil=3)
+            call update_halo(v3, vh, 1, opt_global=.true., opt_pencil=3)
 #else
-  call update_halo(v3,vh,1,opt_pencil=3)
+            call update_halo(v3, vh, 1, opt_pencil=3)
 #endif
 
-  call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
-  
+            call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
+
 #ifdef HALO_GLOBAL
-  do j=zstart(2),zend(2)
-     do i=zstart(1),zend(1)
+            do j = zstart(2), zend(2)
+               do i = zstart(1), zend(1)
 #else
-  do j=1,zsize(2)
-     do i=1,zsize(1)
+                  do j = 1, zsize(2)
+                     do i = 1, zsize(1)
 #endif
-        do k=2,zsize(3)-1
-           wk3(i,j,k) = wk3(i,j,k) + vh(i,j+1,k)-vh(i,j-1,k)
-        end do
-     end do
-  end do
+                        do k = 2, zsize(3) - 1
+                           wk3(i, j, k) = wk3(i, j, k) + vh(i, j + 1, k) - vh(i, j - 1, k)
+                        end do
+                     end do
+                  end do
 
-  ! dw/dz
+                  ! dw/dz
 #ifdef HALO_GLOBAL
-  do j=zstart(2),zend(2)
-     do i=zstart(1),zend(1)
+                  do j = zstart(2), zend(2)
+                     do i = zstart(1), zend(1)
 #else
-  do j=1,zsize(2)
-     do i=1,zsize(1)
+                        do j = 1, zsize(2)
+                           do i = 1, zsize(1)
 #endif
-        do k=2,zsize(3)-1
-           wk3(i,j,k) = wk3(i,j,k) + w3(i,j,k+1)-w3(i,j,k-1)
-        end do
-     end do
-  end do
-
-  call transpose_z_to_y(wk3,wk2)
-  call transpose_y_to_x(wk2,div4)
-
-  ! Compute error
-  call check_err(div4, "Z")
-
-  deallocate(uh,vh,wk2,wk3)
-  end subroutine test_div_haloZ
-
-  subroutine check_err(divh, pencil)
-
-    real(mytype), dimension(:,:,:), intent(in) :: divh
-    character(len=*), intent(in) :: pencil
-
-    real(mytype), dimension(:,:,:), allocatable :: tmp
-
-    real(mytype) :: divmag
-
-    ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
-    !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
-    !      causes a SEGFAULT. Explicitly computing the difference in a temporary
-    !      array seems to be OK.
-    allocate(tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
-    tmp(2:xlast,2:ylast,2:zlast) = divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast)
-    err = mag(tmp(2:xlast,2:ylast,2:zlast))
-    deallocate(tmp)
-    divmag = mag(div1(2:xlast,2:ylast,2:zlast))
-    if (err < epsilon(divmag) * divmag) then
-       passing = .true.
-    else
-       passing = .false.
-    end if
-    all_pass = all_pass .and. passing
-
-    if (nrank==0) then
-       write(*,*) '-----------------------------------------------'
-       write(*,*) 'Calculated via halo exchange (data in '//pencil//'-pencil)'
+                              do k = 2, zsize(3) - 1
+                                 wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
+                              end do
+                           end do
+                        end do
+
+                        call transpose_z_to_y(wk3, wk2)
+                        call transpose_y_to_x(wk2, div4)
+
+                        ! Compute error
+                        call check_err(div4, "Z")
+
+                        deallocate (uh, vh, wk2, wk3)
+                        end subroutine test_div_haloZ
+
+                        subroutine check_err(divh, pencil)
+
+                           real(mytype), dimension(:, :, :), intent(in) :: divh
+                           character(len=*), intent(in) :: pencil
+
+                           real(mytype), dimension(:, :, :), allocatable :: tmp
+
+                           real(mytype) :: divmag
+
+                           ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
+                           !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
+                           !      causes a SEGFAULT. Explicitly computing the difference in a temporary
+                           !      array seems to be OK.
+                           allocate (tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
+                           tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - div1(2:xlast, 2:ylast, 2:zlast)
+                           err = mag(tmp(2:xlast, 2:ylast, 2:zlast))
+                           deallocate (tmp)
+                           divmag = mag(div1(2:xlast, 2:ylast, 2:zlast))
+                           if (err < epsilon(divmag)*divmag) then
+                              passing = .true.
+                           else
+                              passing = .false.
+                           end if
+                           all_pass = all_pass .and. passing
+
+                           if (nrank == 0) then
+                              write (*, *) '-----------------------------------------------'
+                              write (*, *) 'Calculated via halo exchange (data in '//pencil//'-pencil)'
 #ifdef DEBUG
-       write(*,*) (divh(i,i,i), i=2,13)
+                              write (*, *) (divh(i, i, i), i=2, 13)
 #endif
-       write(*,*) 'Error: ', err, '; Relative: ', err / divmag
-       write(*,*) 'Pass: ', passing
-    end if
+                              write (*, *) 'Error: ', err, '; Relative: ', err/divmag
+                              write (*, *) 'Pass: ', passing
+                           end if
+
+                        end subroutine check_err
+
+                        real(mytype) function mag(a)
+
+                           real(mytype), dimension(:, :, :), intent(in) :: a
 
-  end subroutine check_err
+                           real(mytype) :: lmag, gmag
 
-  real(mytype) function mag(a)
+                           lmag = sum(a(:, :, :)**2)
+                           call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+                           if (ierror /= 0) then
+                              call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                                                   "halo_test::mag::MPI_Allreduce")
+                           end if
 
-    real(mytype), dimension(:,:,:), intent(in) :: a
-    
-    real(mytype) :: lmag, gmag
+                           mag = sqrt(gmag/(nx - 2)/(ny - 2)/(nz - 2))
 
-    lmag = sum(a(:,:,:)**2)
-    call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-    if (ierror /= 0) then
-       call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-            "halo_test::mag::MPI_Allreduce")
-    endif
+                        end function mag
 
-    mag = sqrt(gmag / (nx - 2) / (ny - 2) / (nz - 2))
+                        subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
 
-  end function mag
+                           real(mytype), dimension(:, :, :), intent(in) :: arrh
+                           integer, intent(in) :: nx_expected, ny_expected, nz_expected
+                           character(len=*), intent(in) :: tag
 
-  subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
+                           integer :: nx, ny, nz
 
-    real(mytype), dimension(:,:,:), intent(in) :: arrh
-    integer, intent(in) :: nx_expected, ny_expected, nz_expected
-    character(len=*), intent(in) :: tag
+                           character(len=128) :: rank_lbl
 
-    integer :: nx, ny, nz
+                           nx = size(arrh, 1)
+                           ny = size(arrh, 2)
+                           nz = size(arrh, 3)
 
-    character(len=128) :: rank_lbl
+                           write (rank_lbl, "(A,I0,A)") "Rank", nrank, ":"
 
-    nx = size(arrh, 1)
-    ny = size(arrh, 2)
-    nz = size(arrh, 3)
+                           if ((nx /= nx_expected) .or. &
+                               (ny /= ny_expected) .or. &
+                               (nz /= nz_expected)) then
+                              write (*, *) trim(rank_lbl), " ", tag, ":ERROR: halo size"
+                              write (*, *) trim(rank_lbl), " ", "+ Expected: ", nx_expected, " ", ny_expected, " ", nz_expected, " "
+                              write (*, *) trim(rank_lbl), " ", "+ Got:      ", nx, " ", ny, " ", nz, " "
 
-    write(rank_lbl, "(A,I0,A)") "Rank", nrank, ":"
+                              all_pass = .false.
+                           else
+                              write (*, *) trim(rank_lbl), " ", tag, ":PASS"
+                           end if
 
-    if ((nx /= nx_expected) .or. &
-         (ny /= ny_expected) .or. &
-         (nz /= nz_expected)) then
-       write(*,*) trim(rank_lbl), " ", tag, ":ERROR: halo size"
-       write(*,*) trim(rank_lbl), " ", "+ Expected: ", nx_expected, " ", ny_expected, " ", nz_expected, " "
-       write(*,*) trim(rank_lbl), " ", "+ Got:      ", nx, " ", ny, " ", nz, " "
+                        end subroutine test_halo_size
 
-       all_pass = .false.
-    else
-       write(*,*) trim(rank_lbl), " ", tag, ":PASS"
-    end if
-    
-  end subroutine test_halo_size
-  
-end program halo_test
+                        end program halo_test
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 5f9c5dd1..25953391 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -4,71 +4,71 @@
 
 program init_test
 
-  use MPI
-  use decomp_2d
-  
-  implicit none
+   use MPI
+   use decomp_2d
 
-  integer, parameter :: nx = 5
-  integer, parameter :: ny = 6
-  integer, parameter :: nz = 7
-  integer, parameter :: nexpect = nx * ny * nz
+   implicit none
 
-  integer :: p_row, p_col
+   integer, parameter :: nx = 5
+   integer, parameter :: ny = 6
+   integer, parameter :: nz = 7
+   integer, parameter :: nexpect = nx*ny*nz
 
-  integer :: ierr
+   integer :: p_row, p_col
 
-  call MPI_Init(ierr)
-  
-  p_row = 0; p_col = 0
-  call run(p_row, p_col)
+   integer :: ierr
+
+   call MPI_Init(ierr)
+
+   p_row = 0; p_col = 0
+   call run(p_row, p_col)
+
+   call MPI_Finalize(ierr)
 
-  call MPI_Finalize(ierr)
-  
 contains
 
-  subroutine run(p_row, p_col)
-
-    integer, intent(inout) :: p_row, p_col
-
-    call decomp_2d_init(nx, ny, nz, p_row, p_col)
-
-    call check_axis("X")
-    call check_axis("Y")
-    call check_axis("Z")
-    
-    call decomp_2d_finalize()
-    
-  end subroutine run
-
-  subroutine check_axis(axis)
-
-    character(len=*), intent(in) :: axis
-
-    integer :: suml
-    integer :: sumg
-    integer, dimension(3) :: sizes
-
-    if (axis == "X") then
-       sizes = xsize
-    else if (axis == "Y") then
-       sizes = ysize
-    else if (axis == "Z") then
-       sizes = zsize
-    else
-       sizes = 0
-       print *, "ERROR: unknown axis requested!"
-       stop 1
-    end if
-
-    suml = product(sizes)
-    call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierr)
-    
-    if (sumg /= nexpect) then
-       print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
-       stop 1
-    end if
-    
-  end subroutine check_axis
-  
+   subroutine run(p_row, p_col)
+
+      integer, intent(inout) :: p_row, p_col
+
+      call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+      call check_axis("X")
+      call check_axis("Y")
+      call check_axis("Z")
+
+      call decomp_2d_finalize()
+
+   end subroutine run
+
+   subroutine check_axis(axis)
+
+      character(len=*), intent(in) :: axis
+
+      integer :: suml
+      integer :: sumg
+      integer, dimension(3) :: sizes
+
+      if (axis == "X") then
+         sizes = xsize
+      else if (axis == "Y") then
+         sizes = ysize
+      else if (axis == "Z") then
+         sizes = zsize
+      else
+         sizes = 0
+         print *, "ERROR: unknown axis requested!"
+         stop 1
+      end if
+
+      suml = product(sizes)
+      call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierr)
+
+      if (sumg /= nexpect) then
+         print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
+         stop 1
+      end if
+
+   end subroutine check_axis
+
 end program init_test
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index a8e05390..4e3200ca 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -1,36 +1,36 @@
 program io_bench
 
-  use decomp_2d
-  use decomp_2d_io
-  use MPI
+   use decomp_2d
+   use decomp_2d_io
+   use MPI
 
-  implicit none
-  
-  integer, parameter :: nx=100, ny=100, nz=100
-  integer :: p_row=0, p_col=0
+   implicit none
 
-  real(mytype), allocatable, dimension(:,:,:) :: u1
-  
-  double precision :: t1, t2
-  integer :: ierror
+   integer, parameter :: nx = 100, ny = 100, nz = 100
+   integer :: p_row = 0, p_col = 0
 
-  call MPI_INIT(ierror)
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   real(mytype), allocatable, dimension(:, :, :) :: u1
 
-  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  call random_number(u1)
+   double precision :: t1, t2
+   integer :: ierror
 
-  t1 = MPI_WTIME()
-  call decomp_2d_write_one(1,u1,'io.dat')
-  t2 = MPI_WTIME()
+   call MPI_INIT(ierror)
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-  if (nrank==0) write(*,*) 'I/O time: ', t2-t1
+   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   call random_number(u1)
 
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1)
+   t1 = MPI_WTIME()
+   call decomp_2d_write_one(1, u1, 'io.dat')
+   t2 = MPI_WTIME()
+
+   if (nrank == 0) write (*, *) 'I/O time: ', t2 - t1
+
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1)
 
 end program io_bench
-  
+
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 42a6d221..7bebfa03 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -1,128 +1,128 @@
 program io_plane_test
 
-  use mpi
-
-  use decomp_2d
-  use decomp_2d_io
-
-  implicit none
-
-  integer, parameter :: nx=17, ny=13, nz=11
-  integer :: p_row=0, p_col=0
-
-  real(mytype), dimension(nx,ny,nz) :: data1
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-
-  real(mytype), allocatable, dimension(:,:,:) :: work
-  
-  integer :: i,j,k, m, ierror, iol
-
-  call MPI_INIT(ierror)
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-  
-  ! ***** global data *****
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
-           data1(i,j,k) = real(m,mytype)
-           m = m+1
-        end do
-     end do
-  end do
-
-  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  ! original X-pensil based data 
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        u1(i,j,k) = data1(i,j,k)
+   use mpi
+
+   use decomp_2d
+   use decomp_2d_io
+
+   implicit none
+
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer :: p_row = 0, p_col = 0
+
+   real(mytype), dimension(nx, ny, nz) :: data1
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+
+   real(mytype), allocatable, dimension(:, :, :) :: work
+
+   integer :: i, j, k, m, ierror, iol
+
+   call MPI_INIT(ierror)
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
+            data1(i, j, k) = real(m, mytype)
+            m = m + 1
+         end do
+      end do
+   end do
+
+   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   ! original X-pensil based data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            u1(i, j, k) = data1(i, j, k)
+         end do
+      end do
+   end do
+   call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
+
+   ! Y-pencil data
+   call transpose_x_to_y(u1, u2)
+   call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 2, ny/2, '.', 'y_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 3, nz/2, '.', 'y_pencil-z_plane.dat', 'test')
+
+   ! Z-pencil data
+   call transpose_y_to_z(u2, u3)
+   call decomp_2d_write_plane(3, u3, 1, nx/2, '.', 'z_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 2, ny/2, '.', 'z_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 3, nz/2, '.', 'z_pencil-z_plane.dat', 'test')
+
+   ! Attemp to read the files
+   if (nrank == 0) then
+      inquire (iolength=iol) data1(1, 1, 1)
+
+      ! X-plane
+      allocate (work(1, ny, nz))
+      open (10, FILE='x_pencil-x_plane.dat', FORM='unformatted', &
+            ACCESS='DIRECT', RECL=iol)
+      m = 1
+      do k = 1, nz
+         do j = 1, ny
+            read (10, rec=m) work(1, j, k)
+            m = m + 1
+         end do
       end do
-    end do
-  end do
-  call decomp_2d_write_plane(1,u1,1,nx/2,'.','x_pencil-x_plane.dat','test')
-  call decomp_2d_write_plane(1,u1,2,ny/2,'.','x_pencil-y_plane.dat','test')
-  call decomp_2d_write_plane(1,u1,3,nz/2,'.','x_pencil-z_plane.dat','test')
-
-  ! Y-pencil data
-  call transpose_x_to_y(u1,u2)
-  call decomp_2d_write_plane(2,u2,1,nx/2,'.','y_pencil-x_plane.dat','test')
-  call decomp_2d_write_plane(2,u2,2,ny/2,'.','y_pencil-y_plane.dat','test')
-  call decomp_2d_write_plane(2,u2,3,nz/2,'.','y_pencil-z_plane.dat','test')
-
-  ! Z-pencil data
-  call transpose_y_to_z(u2,u3)
-  call decomp_2d_write_plane(3,u3,1,nx/2,'.','z_pencil-x_plane.dat','test')
-  call decomp_2d_write_plane(3,u3,2,ny/2,'.','z_pencil-y_plane.dat','test')
-  call decomp_2d_write_plane(3,u3,3,nz/2,'.','z_pencil-z_plane.dat','test')
-
-  ! Attemp to read the files
-  if (nrank==0) then
-     inquire(iolength=iol) data1(1,1,1)
-
-     ! X-plane
-     allocate(work(1,ny,nz))
-     open(10, FILE='x_pencil-x_plane.dat', FORM='unformatted', &
-          ACCESS='DIRECT', RECL=iol)
-     m=1
-     do k=1,nz
-        do j=1,ny
-           read(10,rec=m) work(1,j,k)
-           m=m+1
-        end do
-     end do
 !     write(*,*) ' '
 !     write(*,'(15I5)') int(work)
-     close(10)
-     deallocate(work)
-
-     write(*,*) 'passed self test x-plane'
-
-     ! Y-plane
-     allocate(work(nx,1,nz))
-     open(10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
-          ACCESS='DIRECT', RECL=iol)
-     m=1
-     do k=1,nz
-        do i=1,nx
-           read(10,rec=m) work(i,1,k)
-           m=m+1
-        end do
-     end do
+      close (10)
+      deallocate (work)
+
+      write (*, *) 'passed self test x-plane'
+
+      ! Y-plane
+      allocate (work(nx, 1, nz))
+      open (10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
+            ACCESS='DIRECT', RECL=iol)
+      m = 1
+      do k = 1, nz
+         do i = 1, nx
+            read (10, rec=m) work(i, 1, k)
+            m = m + 1
+         end do
+      end do
 !     write(*,*) ' '
 !     write(*,'(15I5)') int(work)
-     close(10)
-     deallocate(work)
-
-     write(*,*) 'passed self test y-plane'
-
-     ! Z-plane
-     allocate(work(nx,ny,1))
-     open(10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
-          ACCESS='DIRECT', RECL=iol)
-     m=1
-     do j=1,ny
-        do i=1,nx
-           read(10,rec=m) work(i,j,1)
-           m=m+1
-        end do
-     end do
+      close (10)
+      deallocate (work)
+
+      write (*, *) 'passed self test y-plane'
+
+      ! Z-plane
+      allocate (work(nx, ny, 1))
+      open (10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
+            ACCESS='DIRECT', RECL=iol)
+      m = 1
+      do j = 1, ny
+         do i = 1, nx
+            read (10, rec=m) work(i, j, 1)
+            m = m + 1
+         end do
+      end do
 !     write(*,*) ' '
 !     write(*,'(15I5)') int(work)
-     close(10)
-     deallocate(work)
+      close (10)
+      deallocate (work)
+
+      write (*, *) 'passed self test z-plane'
 
-     write(*,*) 'passed self test z-plane'
+   end if
 
-  end if
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1, u2, u3)
 
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1,u2,u3)
-  
 end program io_plane_test
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 38c9cb85..694f1f21 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -1,86 +1,86 @@
 program io_read
 
-  use mpi
+   use mpi
 
-  use decomp_2d
-  use decomp_2d_io
+   use decomp_2d
+   use decomp_2d_io
 
-  implicit none
+   implicit none
 
-  integer, parameter :: nx=17, ny=13, nz=11
-  ! use different number of processes
-  integer :: p_row=0, p_col=0
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   ! use different number of processes
+   integer :: p_row = 0, p_col = 0
 
 #ifdef COMPLEX_TEST
-  complex(mytype), dimension(nx,ny,nz) :: data1
+   complex(mytype), dimension(nx, ny, nz) :: data1
 
-  complex(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
+   complex(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #else
-  real(mytype), dimension(nx,ny,nz) :: data1
+   real(mytype), dimension(nx, ny, nz) :: data1
 
-  real(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
+   real(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #endif
 
-  real(mytype), parameter :: eps = 1.0E-7_mytype
- 
-  integer :: i,j,k, m, ierror
-  
-  call MPI_INIT(ierror)
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-
-  ! ***** global data *****
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
+   real(mytype), parameter :: eps = 1.0E-7_mytype
+
+   integer :: i, j, k, m, ierror
+
+   call MPI_INIT(ierror)
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
 #ifdef COMPLEX_TEST
-           data1(i,j,k) = cmplx(real(m,mytype), real(nx*ny*nz-m,mytype)) 
+            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
 #else
-           data1(i,j,k) = real(m,mytype)
+            data1(i, j, k) = real(m, mytype)
 #endif
-           m = m+1
-        end do
-     end do
-  end do
-
-  allocate(u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  ! read back to different arrays
-  call decomp_2d_read_one(1,u1b,'.','u1.dat','test',reduce_prec=.false.)
-  call decomp_2d_read_one(2,u2b,'.','u2.dat','test',reduce_prec=.false.)
-  call decomp_2d_read_one(3,u3b,'.','u3.dat','test',reduce_prec=.false.)
-
-  ! Check against the global data array
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        if (abs((data1(i,j,k)-u1b(i,j,k))) > eps) stop 4
+            m = m + 1
+         end do
+      end do
+   end do
+
+   allocate (u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   ! read back to different arrays
+   call decomp_2d_read_one(1, u1b, '.', 'u1.dat', 'test', reduce_prec=.false.)
+   call decomp_2d_read_one(2, u2b, '.', 'u2.dat', 'test', reduce_prec=.false.)
+   call decomp_2d_read_one(3, u3b, '.', 'u3.dat', 'test', reduce_prec=.false.)
+
+   ! Check against the global data array
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs((data1(i, j, k) - u1b(i, j, k))) > eps) stop 4
+         end do
       end do
-    end do
-  end do
+   end do
 
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs((data1(i,j,k)-u2b(i,j,k))) > eps) stop 5
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
+         end do
       end do
-    end do
-  end do
-  
-  do k=zstart(3),zend(3)
-    do j=zstart(2),zend(2)
-      do i=zstart(1),zend(1)
-        if (abs((data1(i,j,k)-u3b(i,j,k))) > eps) stop 6
+   end do
+
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
+         end do
       end do
-    end do
-  end do
+   end do
 
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1b,u2b,u3b)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1b, u2b, u3b)
 
 end program io_read
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index b38aa2a0..eecdcfeb 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -1,133 +1,133 @@
 program io_test
 
-  use mpi
+   use mpi
 
-  use decomp_2d
-  use decomp_2d_io
+   use decomp_2d
+   use decomp_2d_io
 
-  implicit none
+   implicit none
 
-  integer, parameter :: nx=17, ny=13, nz=11
-  integer :: p_row=0, p_col=0
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer :: p_row = 0, p_col = 0
 
 #ifdef COMPLEX_TEST
-  complex(mytype), dimension(nx,ny,nz) :: data1
+   complex(mytype), dimension(nx, ny, nz) :: data1
 
-  complex(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  complex(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
+   complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   complex(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #else
-  real(mytype), dimension(nx,ny,nz) :: data1
+   real(mytype), dimension(nx, ny, nz) :: data1
 
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  real(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   real(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #endif
 
-  real(mytype), parameter :: eps = 1.0E-7_mytype
- 
-  integer :: i,j,k, m, ierror
-  
-  call MPI_INIT(ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-
-  ! ***** global data *****
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
+   real(mytype), parameter :: eps = 1.0E-7_mytype
+
+   integer :: i, j, k, m, ierror
+
+   call MPI_INIT(ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
 #ifdef COMPLEX_TEST
-           data1(i,j,k) = cmplx(real(m,mytype), real(nx*ny*nz-m,mytype)) 
+            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
 #else
-           data1(i,j,k) = real(m,mytype)
+            data1(i, j, k) = real(m, mytype)
 #endif
-           m = m+1
-        end do
-     end do
-  end do
-
-  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  allocate(u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  ! original x-pencil based data 
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        u1(i,j,k) = data1(i,j,k)
+            m = m + 1
+         end do
       end do
-    end do
-  end do
-
-  ! transpose
-  call transpose_x_to_y(u1,u2)
-  call transpose_y_to_z(u2,u3)
-
-  ! write to disk
-  call decomp_2d_write_one(1,u1,'.','u1.dat',0,'test')
-  call decomp_2d_write_one(2,u2,'.','u2.dat',0,'test')
-  call decomp_2d_write_one(3,u3,'.','u3.dat',0,'test')
-
-  ! read back to different arrays
-  call decomp_2d_read_one(1,u1b,'.','u1.dat','test',reduce_prec=.false.)
-  call decomp_2d_read_one(2,u2b,'.','u2.dat','test',reduce_prec=.false.)
-  call decomp_2d_read_one(3,u3b,'.','u3.dat','test',reduce_prec=.false.)
-
-  ! compare  
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        if (abs((u1(i,j,k)-u1b(i,j,k))) > eps) stop 1
+   end do
+
+   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   allocate (u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   ! original x-pencil based data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            u1(i, j, k) = data1(i, j, k)
+         end do
       end do
-    end do
-  end do
+   end do
+
+   ! transpose
+   call transpose_x_to_y(u1, u2)
+   call transpose_y_to_z(u2, u3)
+
+   ! write to disk
+   call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
+   call decomp_2d_write_one(2, u2, '.', 'u2.dat', 0, 'test')
+   call decomp_2d_write_one(3, u3, '.', 'u3.dat', 0, 'test')
+
+   ! read back to different arrays
+   call decomp_2d_read_one(1, u1b, '.', 'u1.dat', 'test', reduce_prec=.false.)
+   call decomp_2d_read_one(2, u2b, '.', 'u2.dat', 'test', reduce_prec=.false.)
+   call decomp_2d_read_one(3, u3b, '.', 'u3.dat', 'test', reduce_prec=.false.)
+
+   ! compare
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) stop 1
+         end do
+      end do
+   end do
 
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs((u2(i,j,k)-u2b(i,j,k))) > eps) stop 2
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 2
+         end do
       end do
-    end do
-  end do
+   end do
 
-  do k=zstart(3),zend(3)
-    do j=zstart(2),zend(2)
-      do i=zstart(1),zend(1)
-        if (abs((u3(i,j,k)-u3b(i,j,k))) > eps) stop 3
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs((u3(i, j, k) - u3b(i, j, k))) > eps) stop 3
+         end do
       end do
-    end do
-  end do
-
-  ! Also check against the global data array
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        if (abs(data1(i,j,k)-u1b(i,j,k)) > eps) stop 4
+   end do
+
+   ! Also check against the global data array
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs(data1(i, j, k) - u1b(i, j, k)) > eps) stop 4
+         end do
       end do
-    end do
-  end do
+   end do
 
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs((data1(i,j,k)-u2b(i,j,k))) > eps) stop 5
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
+         end do
       end do
-    end do
-  end do
-  
-  do k=zstart(3),zend(3)
-    do j=zstart(2),zend(2)
-      do i=zstart(1),zend(1)
-        if (abs((data1(i,j,k)-u3b(i,j,k))) > eps) stop 6
+   end do
+
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
+         end do
       end do
-    end do
-  end do
+   end do
 
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1,u2,u3)
-  deallocate(u1b,u2b,u3b)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1, u2, u3)
+   deallocate (u1b, u2b, u3b)
 
 end program io_test
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 2d25dcf9..7f91e01d 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -3,283 +3,283 @@
 
 program io_var_test
 
-  use decomp_2d
-  use decomp_2d_io
-  use MPI
-
-  implicit none
-
-  integer, parameter :: nx=17, ny=13, nz=11
-  integer :: p_row, p_col
-
-  real(mytype), parameter :: eps = 1.0E-7
-
-  ! for global data
-  real(mytype), dimension(nx,ny,nz) :: data1
-  real(mytype), allocatable, dimension(:,:,:) :: data1_large
-  complex(mytype), dimension(nx,ny,nz) :: cdata1
-
-  ! for distributed data
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  real(mytype), allocatable, dimension(:,:,:) :: u1l, u2l, u3l
-  complex(mytype), allocatable, dimension(:,:,:) :: cu1, cu2, cu3
-
-  ! another copy
-  real(mytype), allocatable, dimension(:,:,:) :: u1_b, u2_b, u3_b
-  real(mytype), allocatable, dimension(:,:,:) :: u1l_b, u2l_b, u3l_b
-  complex(mytype), allocatable, dimension(:,:,:) :: cu1_b, cu2_b, cu3_b
-
-  real(mytype), allocatable, dimension(:) :: tmp
-  complex(mytype), allocatable, dimension(:) :: ctmp
-  integer, allocatable, dimension(:) :: itmp
-
-  TYPE(DECOMP_INFO) :: large
-
-  integer :: i,j,k, m, ierror, fh
-  character(len=15) :: filename, arg
-  integer (kind=MPI_OFFSET_KIND) :: filesize, disp
-
-  allocate(data1_large(nx*2,ny*2,nz*2))
-
-  call MPI_INIT(ierror)
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-
-  ! Defaults
-  p_row = 0
-  p_col = 0
-
-  ! Read commandline input
-  i = command_argument_count()
-  if (i/=2) then
-     call MPI_ABORT(MPI_COMM_WORLD, 1, ierror)
-  else
-     call get_command_argument(1, arg)
-     read(arg, '(I10)') i
-     p_row = i
-     call get_command_argument(2, arg)
-     read(arg, '(I10)') i
-     p_col = i
-  end if
-  
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-
-  ! also create a data set over a large domain
-  call decomp_info_init(nx*2, ny*2, nz*2, large)
-
-  ! initialise global data
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
-           data1(i,j,k) = real(m,mytype)
-           cdata1(i,j,k) = cmplx(real(m,mytype),real(m,mytype), kind=mytype)
-           m = m+1
-        end do
-     end do
-  end do
-
-  m = 1
-  do k=1,nz*2
-     do j=1,ny*2
-        do i=1,nx*2
-           data1_large(i,j,k) = real(m,mytype)
-           m = m+1
-        end do
-     end do
-  end do
-
-  ! allocate memory
-  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  allocate(u1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  allocate(cu1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(cu2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(cu3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  allocate(cu1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(cu2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(cu3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  allocate(u1l(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
-       large%xst(3):large%xen(3)))
-  allocate(u2l(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
-       large%yst(3):large%yen(3)))
-  allocate(u3l(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
-       large%zst(3):large%zen(3)))
-  allocate(u1l_b(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
-       large%xst(3):large%xen(3)))
-  allocate(u2l_b(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
-       large%yst(3):large%yen(3)))
-  allocate(u3l_b(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
-       large%zst(3):large%zen(3)))
-
-  ! distribute the data
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           u1(i,j,k) = data1(i,j,k)
-           cu1(i,j,k) = cdata1(i,j,k)
-        end do
-     end do
-  end do
-  do k=large%xst(3),large%xen(3)
-     do j=large%xst(2),large%xen(2)
-        do i=large%xst(1),large%xen(1)
-           u1l(i,j,k) = data1_large(i,j,k)
-        end do
-     end do
-  end do
-
-  ! transpose
-  call transpose_x_to_y(u1,u2)
-  call transpose_y_to_z(u2,u3)
-  call transpose_x_to_y(u1l,u2l,large)
-  call transpose_y_to_z(u2l,u3l,large)
-  call transpose_x_to_y(cu1,cu2)
-  call transpose_y_to_z(cu2,cu3)
-
-  ! open file for IO
-  write(filename,'(A,I3.3)') 'io_var_data.', nproc
-  call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
-       MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-       fh, ierror)
-  filesize = 0_MPI_OFFSET_KIND
-  call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-  disp = 0_MPI_OFFSET_KIND
-
-  ! test writing scalar data
-  allocate(tmp(2))
-  tmp(1) = 1._mytype
-  tmp(2) = 2._mytype
-  allocate(ctmp(3))
-  ctmp(1) = cmplx(1.0,1.0,mytype)
-  ctmp(2) = cmplx(2.0,2.0,mytype)
-  ctmp(3) = cmplx(3.0,3.0,mytype)
-  allocate(itmp(3))
-  call decomp_2d_write_scalar(fh,disp,2,tmp)
-  call decomp_2d_write_scalar(fh,disp,3,ctmp)
-  call decomp_2d_write_scalar(fh,disp,3,(/nx,ny,nz/))
-  
-  ! test the IO routines by writing all data to disk
-  call decomp_2d_write_var(fh,disp,1,u1)
-  call decomp_2d_write_var(fh,disp,2,u2)
-  call decomp_2d_write_var(fh,disp,3,u3)
-  call decomp_2d_write_var(fh,disp,1,u1l,large)
-  call decomp_2d_write_var(fh,disp,2,u2l,large)
-  call decomp_2d_write_var(fh,disp,3,u3l,large)
-  call decomp_2d_write_var(fh,disp,1,cu1)
-  call decomp_2d_write_var(fh,disp,2,cu2)
-  call decomp_2d_write_var(fh,disp,3,cu3)
-
-  call MPI_FILE_CLOSE(fh,ierror)
-
-  if (nrank==0) write(*,*) 'disp=',disp
-
-  ! read data back in from file
-  call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
-       MPI_MODE_RDONLY, MPI_INFO_NULL, &
-       fh, ierror)
-  ! skip the scalars (2 real, 3 cmplx, 3 int)
+   use decomp_2d
+   use decomp_2d_io
+   use MPI
+
+   implicit none
+
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer :: p_row, p_col
+
+   real(mytype), parameter :: eps = 1.0E-7
+
+   ! for global data
+   real(mytype), dimension(nx, ny, nz) :: data1
+   real(mytype), allocatable, dimension(:, :, :) :: data1_large
+   complex(mytype), dimension(nx, ny, nz) :: cdata1
+
+   ! for distributed data
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   real(mytype), allocatable, dimension(:, :, :) :: u1l, u2l, u3l
+   complex(mytype), allocatable, dimension(:, :, :) :: cu1, cu2, cu3
+
+   ! another copy
+   real(mytype), allocatable, dimension(:, :, :) :: u1_b, u2_b, u3_b
+   real(mytype), allocatable, dimension(:, :, :) :: u1l_b, u2l_b, u3l_b
+   complex(mytype), allocatable, dimension(:, :, :) :: cu1_b, cu2_b, cu3_b
+
+   real(mytype), allocatable, dimension(:) :: tmp
+   complex(mytype), allocatable, dimension(:) :: ctmp
+   integer, allocatable, dimension(:) :: itmp
+
+   TYPE(DECOMP_INFO) :: large
+
+   integer :: i, j, k, m, ierror, fh
+   character(len=15) :: filename, arg
+   integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+
+   allocate (data1_large(nx*2, ny*2, nz*2))
+
+   call MPI_INIT(ierror)
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+
+   ! Defaults
+   p_row = 0
+   p_col = 0
+
+   ! Read commandline input
+   i = command_argument_count()
+   if (i /= 2) then
+      call MPI_ABORT(MPI_COMM_WORLD, 1, ierror)
+   else
+      call get_command_argument(1, arg)
+      read (arg, '(I10)') i
+      p_row = i
+      call get_command_argument(2, arg)
+      read (arg, '(I10)') i
+      p_col = i
+   end if
+
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! also create a data set over a large domain
+   call decomp_info_init(nx*2, ny*2, nz*2, large)
+
+   ! initialise global data
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
+            data1(i, j, k) = real(m, mytype)
+            cdata1(i, j, k) = cmplx(real(m, mytype), real(m, mytype), kind=mytype)
+            m = m + 1
+         end do
+      end do
+   end do
+
+   m = 1
+   do k = 1, nz*2
+      do j = 1, ny*2
+         do i = 1, nx*2
+            data1_large(i, j, k) = real(m, mytype)
+            m = m + 1
+         end do
+      end do
+   end do
+
+   ! allocate memory
+   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   allocate (u1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   allocate (cu1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (cu2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (cu3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   allocate (cu1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (cu2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (cu3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   allocate (u1l(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
+                 large%xst(3):large%xen(3)))
+   allocate (u2l(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
+                 large%yst(3):large%yen(3)))
+   allocate (u3l(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
+                 large%zst(3):large%zen(3)))
+   allocate (u1l_b(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
+                   large%xst(3):large%xen(3)))
+   allocate (u2l_b(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
+                   large%yst(3):large%yen(3)))
+   allocate (u3l_b(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
+                   large%zst(3):large%zen(3)))
+
+   ! distribute the data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            u1(i, j, k) = data1(i, j, k)
+            cu1(i, j, k) = cdata1(i, j, k)
+         end do
+      end do
+   end do
+   do k = large%xst(3), large%xen(3)
+      do j = large%xst(2), large%xen(2)
+         do i = large%xst(1), large%xen(1)
+            u1l(i, j, k) = data1_large(i, j, k)
+         end do
+      end do
+   end do
+
+   ! transpose
+   call transpose_x_to_y(u1, u2)
+   call transpose_y_to_z(u2, u3)
+   call transpose_x_to_y(u1l, u2l, large)
+   call transpose_y_to_z(u2l, u3l, large)
+   call transpose_x_to_y(cu1, cu2)
+   call transpose_y_to_z(cu2, cu3)
+
+   ! open file for IO
+   write (filename, '(A,I3.3)') 'io_var_data.', nproc
+   call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
+                      MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                      fh, ierror)
+   filesize = 0_MPI_OFFSET_KIND
+   call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+   disp = 0_MPI_OFFSET_KIND
+
+   ! test writing scalar data
+   allocate (tmp(2))
+   tmp(1) = 1._mytype
+   tmp(2) = 2._mytype
+   allocate (ctmp(3))
+   ctmp(1) = cmplx(1.0, 1.0, mytype)
+   ctmp(2) = cmplx(2.0, 2.0, mytype)
+   ctmp(3) = cmplx(3.0, 3.0, mytype)
+   allocate (itmp(3))
+   call decomp_2d_write_scalar(fh, disp, 2, tmp)
+   call decomp_2d_write_scalar(fh, disp, 3, ctmp)
+   call decomp_2d_write_scalar(fh, disp, 3, (/nx, ny, nz/))
+
+   ! test the IO routines by writing all data to disk
+   call decomp_2d_write_var(fh, disp, 1, u1)
+   call decomp_2d_write_var(fh, disp, 2, u2)
+   call decomp_2d_write_var(fh, disp, 3, u3)
+   call decomp_2d_write_var(fh, disp, 1, u1l, large)
+   call decomp_2d_write_var(fh, disp, 2, u2l, large)
+   call decomp_2d_write_var(fh, disp, 3, u3l, large)
+   call decomp_2d_write_var(fh, disp, 1, cu1)
+   call decomp_2d_write_var(fh, disp, 2, cu2)
+   call decomp_2d_write_var(fh, disp, 3, cu3)
+
+   call MPI_FILE_CLOSE(fh, ierror)
+
+   if (nrank == 0) write (*, *) 'disp=', disp
+
+   ! read data back in from file
+   call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
+                      MPI_MODE_RDONLY, MPI_INFO_NULL, &
+                      fh, ierror)
+   ! skip the scalars (2 real, 3 cmplx, 3 int)
 #ifdef DOUBLE_PREC
-  ! if double precision: 2*8+3*8*2+3*4
-  disp = 76_MPI_OFFSET_KIND
+   ! if double precision: 2*8+3*8*2+3*4
+   disp = 76_MPI_OFFSET_KIND
 #else
-  ! if single precision: 2*4+3*4*2+3*4
-  disp = 44_MPI_OFFSET_KIND
+   ! if single precision: 2*4+3*4*2+3*4
+   disp = 44_MPI_OFFSET_KIND
 #endif
-  
-  call decomp_2d_read_var(fh,disp,1,u1_b)
-  call decomp_2d_read_var(fh,disp,2,u2_b)
-  call decomp_2d_read_var(fh,disp,3,u3_b)
-  call decomp_2d_read_var(fh,disp,1,u1l_b,large)
-  call decomp_2d_read_var(fh,disp,2,u2l_b,large)
-  call decomp_2d_read_var(fh,disp,3,u3l_b,large)
-  call decomp_2d_read_var(fh,disp,1,cu1_b)
-  call decomp_2d_read_var(fh,disp,2,cu2_b)
-  call decomp_2d_read_var(fh,disp,3,cu3_b)
-
-  disp = 0_MPI_OFFSET_KIND
-  call decomp_2d_read_scalar(fh,disp,2,tmp)
-  call decomp_2d_read_scalar(fh,disp,3,ctmp)
-  call decomp_2d_read_scalar(fh,disp,3,itmp)
-  if (nrank==0) then
-     write(*,'(2F8.3)') tmp
-     write(*,20) ctmp
-20   format(3(:,'(',F5.2,',',F5.2,')'))
-     write(*,'(A,3I5)') 'nx,ny,nz', itmp
-  end if
-
-  call MPI_FILE_CLOSE(fh,ierror)
-  deallocate(tmp, ctmp, itmp)
-
-  ! validate the data
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           if (abs(u1(i,j,k)-u1_b(i,j,k)) > eps) stop 1
-           if (abs(cu1(i,j,k)-cu1_b(i,j,k)) > eps) stop 2
-        end do
-     end do
-  end do
-
-  do k=ystart(3),yend(3)
-     do j=ystart(2),yend(2)
-        do i=ystart(1),yend(1)
-           if (abs(u2(i,j,k)-u2_b(i,j,k)) > eps) stop 3
-           if (abs(cu2(i,j,k)-cu2_b(i,j,k)) > eps) stop 4
-        end do
-     end do
-  end do
-
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           if (abs(u3(i,j,k)-u3_b(i,j,k)) > eps) stop 5
-           if (abs(cu3(i,j,k)-cu3_b(i,j,k)) > eps) stop 6
-        end do
-     end do
-  end do
-
-  do k=large%xst(3),large%xen(3)
-     do j=large%xst(2),large%xen(2)
-        do i=large%xst(1),large%xen(1)
-           if (abs(u1l(i,j,k)-u1l_b(i,j,k)) > eps) stop 7
-        end do
-     end do
-  end do
-
-  do k=large%yst(3),large%yen(3)
-     do j=large%yst(2),large%yen(2)
-        do i=large%yst(1),large%yen(1)
-           if (abs(u2l(i,j,k)-u2l_b(i,j,k)) > eps) stop 8
-        end do
-     end do
-  end do
-
-  do k=large%zst(3),large%zen(3)
-     do j=large%zst(2),large%zen(2)
-        do i=large%zst(1),large%zen(1)
-           if (abs(u3l(i,j,k)-u3l_b(i,j,k)) > eps) stop 9
-        end do
-     end do
-  end do
-
-  if (nrank==0) write(*,*) 'passed self test'
-
-  ! clean up
-  call decomp_info_finalize(large)
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
-  deallocate(u1,u2,u3)
-  deallocate(u1l,u2l,u3l)
-  deallocate(cu1,cu2,cu3)
-  deallocate(u1_b,u2_b,u3_b)
-  deallocate(u1l_b,u2l_b,u3l_b)
-  deallocate(cu1_b,cu2_b,cu3_b)
-  deallocate(data1_large)
-  
+
+   call decomp_2d_read_var(fh, disp, 1, u1_b)
+   call decomp_2d_read_var(fh, disp, 2, u2_b)
+   call decomp_2d_read_var(fh, disp, 3, u3_b)
+   call decomp_2d_read_var(fh, disp, 1, u1l_b, large)
+   call decomp_2d_read_var(fh, disp, 2, u2l_b, large)
+   call decomp_2d_read_var(fh, disp, 3, u3l_b, large)
+   call decomp_2d_read_var(fh, disp, 1, cu1_b)
+   call decomp_2d_read_var(fh, disp, 2, cu2_b)
+   call decomp_2d_read_var(fh, disp, 3, cu3_b)
+
+   disp = 0_MPI_OFFSET_KIND
+   call decomp_2d_read_scalar(fh, disp, 2, tmp)
+   call decomp_2d_read_scalar(fh, disp, 3, ctmp)
+   call decomp_2d_read_scalar(fh, disp, 3, itmp)
+   if (nrank == 0) then
+      write (*, '(2F8.3)') tmp
+      write (*, 20) ctmp
+20    format(3(:, '(', F5.2, ',', F5.2, ')'))
+      write (*, '(A,3I5)') 'nx,ny,nz', itmp
+   end if
+
+   call MPI_FILE_CLOSE(fh, ierror)
+   deallocate (tmp, ctmp, itmp)
+
+   ! validate the data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs(u1(i, j, k) - u1_b(i, j, k)) > eps) stop 1
+            if (abs(cu1(i, j, k) - cu1_b(i, j, k)) > eps) stop 2
+         end do
+      end do
+   end do
+
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs(u2(i, j, k) - u2_b(i, j, k)) > eps) stop 3
+            if (abs(cu2(i, j, k) - cu2_b(i, j, k)) > eps) stop 4
+         end do
+      end do
+   end do
+
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs(u3(i, j, k) - u3_b(i, j, k)) > eps) stop 5
+            if (abs(cu3(i, j, k) - cu3_b(i, j, k)) > eps) stop 6
+         end do
+      end do
+   end do
+
+   do k = large%xst(3), large%xen(3)
+      do j = large%xst(2), large%xen(2)
+         do i = large%xst(1), large%xen(1)
+            if (abs(u1l(i, j, k) - u1l_b(i, j, k)) > eps) stop 7
+         end do
+      end do
+   end do
+
+   do k = large%yst(3), large%yen(3)
+      do j = large%yst(2), large%yen(2)
+         do i = large%yst(1), large%yen(1)
+            if (abs(u2l(i, j, k) - u2l_b(i, j, k)) > eps) stop 8
+         end do
+      end do
+   end do
+
+   do k = large%zst(3), large%zen(3)
+      do j = large%zst(2), large%zen(2)
+         do i = large%zst(1), large%zen(1)
+            if (abs(u3l(i, j, k) - u3l_b(i, j, k)) > eps) stop 9
+         end do
+      end do
+   end do
+
+   if (nrank == 0) write (*, *) 'passed self test'
+
+   ! clean up
+   call decomp_info_finalize(large)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1, u2, u3)
+   deallocate (u1l, u2l, u3l)
+   deallocate (cu1, cu2, cu3)
+   deallocate (u1_b, u2_b, u3_b)
+   deallocate (u1l_b, u2l_b, u3l_b)
+   deallocate (cu1_b, cu2_b, cu3_b)
+   deallocate (data1_large)
+
 end program io_var_test
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 9c19841a..1723d013 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -1,166 +1,166 @@
 program test2d
- 
-  use mpi
-
-  use decomp_2d
-  ! use decomp_2d_io
-
-  implicit none
-
-  integer, parameter :: nx=17, ny=13, nz=11
-  integer :: p_row=0, p_col=0
-
-  real(mytype), dimension(nx,ny,nz) :: data1
-  
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  
-  integer :: i,j,k, m, ierror
-  logical :: error_flag
-
-  ! Init
-  error_flag = .false.
-  call MPI_INIT(ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_INIT")
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-
-  ! ***** global data *****
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
-           data1(i,j,k) = float(m)
-           m = m+1
-        end do
-     end do
-  end do
+
+   use mpi
+
+   use decomp_2d
+   ! use decomp_2d_io
+
+   implicit none
+
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer :: p_row = 0, p_col = 0
+
+   real(mytype), dimension(nx, ny, nz) :: data1
+
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+
+   integer :: i, j, k, m, ierror
+   logical :: error_flag
+
+   ! Init
+   error_flag = .false.
+   call MPI_INIT(ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_INIT")
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
+            data1(i, j, k) = float(m)
+            m = m + 1
+         end do
+      end do
+   end do
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Testing the swap routines
+   ! Testing the swap routines
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  
-  !allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  !allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  !allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  call alloc_x(u1, opt_global=.true.)
-  call alloc_y(u2, opt_global=.true.)
-  call alloc_z(u3, opt_global=.true.)
- 
-  ! original x-pensil based data 
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        u1(i,j,k) = data1(i,j,k)
+
+   !allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   !allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   !allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   call alloc_x(u1, opt_global=.true.)
+   call alloc_y(u2, opt_global=.true.)
+   call alloc_z(u3, opt_global=.true.)
+
+   ! original x-pensil based data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            u1(i, j, k) = data1(i, j, k)
+         end do
       end do
-    end do
-  end do
+   end do
 
 10 format(15I5)
 
 #ifdef DEBUG
-  if (nrank==0) then 
-     write(*,*) 'Numbers held on Rank 0'
-     write(*,*) ' '
-     write(*,*) 'X-pencil'
-     write(*,10) int(u1)
-  end if
+   if (nrank == 0) then
+      write (*, *) 'Numbers held on Rank 0'
+      write (*, *) ' '
+      write (*, *) 'X-pencil'
+      write (*, 10) int(u1)
+   end if
 #endif
 
-  ! call decomp_2d_write_one(1,u1,'u1.dat')
+   ! call decomp_2d_write_one(1,u1,'u1.dat')
 
   !!!!!!!!!!!!!!!!!!!!!!!
-  ! x-pensil ==> y-pensil
-  call transpose_x_to_y(u1,u2)
+   ! x-pensil ==> y-pensil
+   call transpose_x_to_y(u1, u2)
 
 #ifdef DEBUG
-  if (nrank==0) then
-     write(*,*) ' '
-     write(*,*) 'Y-pencil'
-     write(*,10) int(u2)
-  end if
+   if (nrank == 0) then
+      write (*, *) ' '
+      write (*, *) 'Y-pencil'
+      write (*, 10) int(u2)
+   end if
 #endif
 
-  ! call decomp_2d_write_one(2,u2,'u2.dat')
-  ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte 
+   ! call decomp_2d_write_one(2,u2,'u2.dat')
+   ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
 
-  ! also check the transposition this way
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs(u2(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
+   ! also check the transposition this way
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs(u2(i, j, k) - data1(i, j, k)) .gt. 0) error_flag = .true.
+         end do
       end do
-    end do
-  end do
-  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-  if (error_flag) call decomp_2d_abort(1, "error swaping x->y")
+   end do
+   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+   if (error_flag) call decomp_2d_abort(1, "error swaping x->y")
 
   !!!!!!!!!!!!!!!!!!!!!!!
-  ! y-pensil ==> z-pensil
-  call transpose_y_to_z(u2,u3)
+   ! y-pensil ==> z-pensil
+   call transpose_y_to_z(u2, u3)
 
 #ifdef DEBUG
-  if (nrank==0) then
-     write(*,*) ' '
-     write(*,*) 'Z-pencil'
-     write(*,10) int(u3)
-  end if
+   if (nrank == 0) then
+      write (*, *) ' '
+      write (*, *) 'Z-pencil'
+      write (*, 10) int(u3)
+   end if
 #endif
 
-  ! call decomp_2d_write_one(3,u3,'u3.dat')
-  ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
+   ! call decomp_2d_write_one(3,u3,'u3.dat')
+   ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
-  do k=zstart(3),zend(3)
-    do j=zstart(2),zend(2)
-      do i=zstart(1),zend(1)
-        if (abs(u3(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs(u3(i, j, k) - data1(i, j, k)) .gt. 0) error_flag = .true.
+         end do
       end do
-    end do
-  end do
-  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-  if (error_flag) call decomp_2d_abort(2, "error swaping y->z")
+   end do
+   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+   if (error_flag) call decomp_2d_abort(2, "error swaping y->z")
 
   !!!!!!!!!!!!!!!!!!!!!!!
-  ! z-pensil ==> y-pensil
-  call transpose_z_to_y(u3,u2)
-  ! call decomp_2d_write_one(2,u2,'u2b.dat')
-
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs(u2(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
+   ! z-pensil ==> y-pensil
+   call transpose_z_to_y(u3, u2)
+   ! call decomp_2d_write_one(2,u2,'u2b.dat')
+
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs(u2(i, j, k) - data1(i, j, k)) .gt. 0) error_flag = .true.
+         end do
       end do
-    end do
-  end do
-  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-  if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
+   end do
+   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+   if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
 
   !!!!!!!!!!!!!!!!!!!!!!!
-  ! y-pensil ==> x-pensil
-  call transpose_y_to_x(u2,u1)
-  ! call decomp_2d_write_one(1,u1,'u1b.dat')
-
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        if (abs(u1(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
+   ! y-pensil ==> x-pensil
+   call transpose_y_to_x(u2, u1)
+   ! call decomp_2d_write_one(1,u1,'u1b.dat')
+
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs(u1(i, j, k) - data1(i, j, k)) .gt. 0) error_flag = .true.
+         end do
       end do
-    end do
-  end do
-  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-  if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
-
-  if (nrank == 0) then
-    write(*,*) " "
-    write(*,*) "test2d completed"
-    write(*,*) " "
-  endif
-
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1,u2,u3)
+   end do
+   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+   if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
+
+   if (nrank == 0) then
+      write (*, *) " "
+      write (*, *) "test2d completed"
+      write (*, *) " "
+   end if
+
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1, u2, u3)
 
 end program test2d
 
diff --git a/src/alloc.f90 b/src/alloc.f90
index ffcbfa14..51c893ef 100644
--- a/src/alloc.f90
+++ b/src/alloc.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -16,281 +16,281 @@
   ! X-pencil real arrays
   subroutine alloc_x_real_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_x(var, decomp_main, opt_global)
+     call alloc_x(var, decomp_main, opt_global)
 
   end subroutine alloc_x_real_short
 
   subroutine alloc_x_real(var, decomp, opt_global)
 
-    implicit none
-
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%xst(1):decomp%xen(1), &
-            decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%xst(1):decomp%xen(1), &
+                      decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_x_real
 
   ! X-pencil complex arrays
   subroutine alloc_x_complex_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_x(var, decomp_main, opt_global)
+     call alloc_x(var, decomp_main, opt_global)
 
   end subroutine alloc_x_complex_short
 
   subroutine alloc_x_complex(var, decomp, opt_global)
 
-    implicit none
-
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%xst(1):decomp%xen(1), &
-            decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%xst(1):decomp%xen(1), &
+                      decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_x_complex
 
   ! Y-pencil real arrays
   subroutine alloc_y_real_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_y(var, decomp_main, opt_global)
+     call alloc_y(var, decomp_main, opt_global)
 
   end subroutine alloc_y_real_short
 
   subroutine alloc_y_real(var, decomp, opt_global)
 
-    implicit none
-
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%yst(1):decomp%yen(1), &
-            decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%yst(1):decomp%yen(1), &
+                      decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_y_real
 
   ! Y-pencil complex arrays
   subroutine alloc_y_complex_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_y(var, decomp_main, opt_global)
+     call alloc_y(var, decomp_main, opt_global)
 
   end subroutine alloc_y_complex_short
 
   subroutine alloc_y_complex(var, decomp, opt_global)
 
-    implicit none
-
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%yst(1):decomp%yen(1), &
-            decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%yst(1):decomp%yen(1), &
+                      decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_y_complex
 
   ! Z-pencil real arrays
   subroutine alloc_z_real_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_z(var, decomp_main, opt_global)
+     call alloc_z(var, decomp_main, opt_global)
 
   end subroutine alloc_z_real_short
 
   subroutine alloc_z_real(var, decomp, opt_global)
 
-    implicit none
-
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%zst(1):decomp%zen(1), &
-            decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%zst(1):decomp%zen(1), &
+                      decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_z_real
 
   ! Z-pencil complex arrays
   subroutine alloc_z_complex_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_z(var, decomp_main, opt_global)
+     call alloc_z(var, decomp_main, opt_global)
 
   end subroutine alloc_z_complex_short
 
   subroutine alloc_z_complex(var, decomp, opt_global)
 
-    implicit none
-
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%zst(1):decomp%zen(1), &
-            decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%zst(1):decomp%zen(1), &
+                      decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_z_complex
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 11953900..155f20d8 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -14,362 +14,362 @@
 
 module decomp_2d
 
-  use MPI
-  use, intrinsic :: iso_fortran_env, only : real32, real64
-  use factor
+   use MPI
+   use, intrinsic :: iso_fortran_env, only: real32, real64
+   use factor
 #if defined(_GPU)
-  use cudafor
+   use cudafor
 #if defined(_NCCL)
-  use nccl
+   use nccl
 #endif
 #endif
 
-  implicit none
+   implicit none
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
 #ifdef DOUBLE_PREC
-  integer, parameter, public :: mytype = KIND(0._real64)
-  integer, parameter, public :: real_type = MPI_DOUBLE_PRECISION
-  integer, parameter, public :: real2_type = MPI_2DOUBLE_PRECISION
-  integer, parameter, public :: complex_type = MPI_DOUBLE_COMPLEX
+   integer, parameter, public :: mytype = KIND(0._real64)
+   integer, parameter, public :: real_type = MPI_DOUBLE_PRECISION
+   integer, parameter, public :: real2_type = MPI_2DOUBLE_PRECISION
+   integer, parameter, public :: complex_type = MPI_DOUBLE_COMPLEX
 #ifdef SAVE_SINGLE
-  integer, parameter, public :: mytype_single = KIND(0._real32)
-  integer, parameter, public :: real_type_single = MPI_REAL
+   integer, parameter, public :: mytype_single = KIND(0._real32)
+   integer, parameter, public :: real_type_single = MPI_REAL
 #else
-  integer, parameter, public :: mytype_single = KIND(0._real64)
-  integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
+   integer, parameter, public :: mytype_single = KIND(0._real64)
+   integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
 #endif
 #else
-  integer, parameter, public :: mytype = KIND(0._real32)
-  integer, parameter, public :: real_type = MPI_REAL
-  integer, parameter, public :: real2_type = MPI_2REAL
-  integer, parameter, public :: complex_type = MPI_COMPLEX
-  integer, parameter, public :: mytype_single = KIND(0._real32)
-  integer, parameter, public :: real_type_single = MPI_REAL
+   integer, parameter, public :: mytype = KIND(0._real32)
+   integer, parameter, public :: real_type = MPI_REAL
+   integer, parameter, public :: real2_type = MPI_2REAL
+   integer, parameter, public :: complex_type = MPI_COMPLEX
+   integer, parameter, public :: mytype_single = KIND(0._real32)
+   integer, parameter, public :: real_type_single = MPI_REAL
 #endif
 
-  integer, save, public :: mytype_bytes
-
-  ! some key global variables
-  integer, save, public :: nx_global, ny_global, nz_global  ! global size
-
-  integer, save, public :: nrank = -1 ! local MPI rank 
-  integer, save, public :: nproc = -1 ! total number of processors
-  integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
-
-  ! parameters for 2D Cartesian topology 
-  integer, save, dimension(2) :: dims, coord
-  integer, save, public :: DECOMP_2D_COMM_CART_X = MPI_COMM_NULL
-  integer, save, public :: DECOMP_2D_COMM_CART_Y = MPI_COMM_NULL
-  integer, save, public :: DECOMP_2D_COMM_CART_Z = MPI_COMM_NULL
-  integer, save :: DECOMP_2D_COMM_ROW = MPI_COMM_NULL
-  integer, save :: DECOMP_2D_COMM_COL = MPI_COMM_NULL
-
-  ! define neighboring blocks (to be used in halo-cell support)
-  !  first dimension 1=X-pencil, 2=Y-pencil, 3=Z-pencil
-  ! second dimension 1=east, 2=west, 3=north, 4=south, 5=top, 6=bottom 
-  integer, save, dimension(3,6) :: neighbour 
-
-  ! flags for periodic condition in three dimensions
-  logical, save :: periodic_x, periodic_y, periodic_z
-
-  !
-  ! Output for the log can be changed by the external code before calling decomp_2d_init
-  !
-  !    0 => No log output
-  !    1 => Master rank log output to stdout
-  !    2 => Master rank log output to the file "decomp_2d_setup.log"
-  !    3 => All ranks log output to a dedicated file
-  !
-  ! The default value is 2 (3 for debug builds)
-  !
-  integer, parameter, public :: D2D_LOG_QUIET = 0
-  integer, parameter, public :: D2D_LOG_STDOUT = 1
-  integer, parameter, public :: D2D_LOG_TOFILE = 2
-  integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
+   integer, save, public :: mytype_bytes
+
+   ! some key global variables
+   integer, save, public :: nx_global, ny_global, nz_global  ! global size
+
+   integer, save, public :: nrank = -1 ! local MPI rank
+   integer, save, public :: nproc = -1 ! total number of processors
+   integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
+
+   ! parameters for 2D Cartesian topology
+   integer, save, dimension(2) :: dims, coord
+   integer, save, public :: DECOMP_2D_COMM_CART_X = MPI_COMM_NULL
+   integer, save, public :: DECOMP_2D_COMM_CART_Y = MPI_COMM_NULL
+   integer, save, public :: DECOMP_2D_COMM_CART_Z = MPI_COMM_NULL
+   integer, save :: DECOMP_2D_COMM_ROW = MPI_COMM_NULL
+   integer, save :: DECOMP_2D_COMM_COL = MPI_COMM_NULL
+
+   ! define neighboring blocks (to be used in halo-cell support)
+   !  first dimension 1=X-pencil, 2=Y-pencil, 3=Z-pencil
+   ! second dimension 1=east, 2=west, 3=north, 4=south, 5=top, 6=bottom
+   integer, save, dimension(3, 6) :: neighbour
+
+   ! flags for periodic condition in three dimensions
+   logical, save :: periodic_x, periodic_y, periodic_z
+
+   !
+   ! Output for the log can be changed by the external code before calling decomp_2d_init
+   !
+   !    0 => No log output
+   !    1 => Master rank log output to stdout
+   !    2 => Master rank log output to the file "decomp_2d_setup.log"
+   !    3 => All ranks log output to a dedicated file
+   !
+   ! The default value is 2 (3 for debug builds)
+   !
+   integer, parameter, public :: D2D_LOG_QUIET = 0
+   integer, parameter, public :: D2D_LOG_STDOUT = 1
+   integer, parameter, public :: D2D_LOG_TOFILE = 2
+   integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
 #ifdef DEBUG
-  integer, public, save :: decomp_log = D2D_LOG_TOFILE_FULL
+   integer, public, save :: decomp_log = D2D_LOG_TOFILE_FULL
 #else
-  integer, public, save :: decomp_log = D2D_LOG_TOFILE
+   integer, public, save :: decomp_log = D2D_LOG_TOFILE
 #endif
 
-  !
-  ! Debug level can be changed by the external code before calling decomp_2d_init
-  !
-  ! The environment variable "DECOMP_2D_DEBUG" can be used to change the debug level
-  !
-  ! Debug checks are performed only when the preprocessor variable DEBUG is defined
-  !
-  enum, bind(c)
-     enumerator :: D2D_DEBUG_LEVEL_OFF = 0
-     enumerator :: D2D_DEBUG_LEVEL_CRITICAL = 1
-     enumerator :: D2D_DEBUG_LEVEL_ERROR = 2
-     enumerator :: D2D_DEBUG_LEVEL_WARN = 3
-     enumerator :: D2D_DEBUG_LEVEL_INFO = 4
-     enumerator :: D2D_DEBUG_LEVEL_DEBUG = 5
-     enumerator :: D2D_DEBUG_LEVEL_TRACE = 6
-  end enum
+   !
+   ! Debug level can be changed by the external code before calling decomp_2d_init
+   !
+   ! The environment variable "DECOMP_2D_DEBUG" can be used to change the debug level
+   !
+   ! Debug checks are performed only when the preprocessor variable DEBUG is defined
+   !
+   enum, bind(c)
+      enumerator :: D2D_DEBUG_LEVEL_OFF = 0
+      enumerator :: D2D_DEBUG_LEVEL_CRITICAL = 1
+      enumerator :: D2D_DEBUG_LEVEL_ERROR = 2
+      enumerator :: D2D_DEBUG_LEVEL_WARN = 3
+      enumerator :: D2D_DEBUG_LEVEL_INFO = 4
+      enumerator :: D2D_DEBUG_LEVEL_DEBUG = 5
+      enumerator :: D2D_DEBUG_LEVEL_TRACE = 6
+   end enum
 #ifdef DEBUG
-  integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_INFO
+   integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_INFO
 #else
-  integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_OFF
+   integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_OFF
 #endif
 
 #if defined(_GPU)
 #if defined(_NCCL)
-  integer, save :: row_rank, col_rank
+   integer, save :: row_rank, col_rank
 #endif
 #endif
 
 #ifdef SHM
-  ! derived type to store shared-memory info
-  TYPE, public :: SMP_INFO
-     integer MPI_COMM          ! SMP associated with this communicator
-     integer NODE_ME           ! rank in this communicator
-     integer NCPU              ! size of this communicator
-     integer SMP_COMM          ! communicator for SMP-node masters
-     integer CORE_COMM         ! communicator for cores on SMP-node
-     integer SMP_ME            ! SMP-node id starting from 1 ... NSMP
-     integer NSMP              ! number of SMP-nodes in this communicator
-     integer CORE_ME           ! core id starting from 1 ... NCORE
-     integer NCORE             ! number of cores on this SMP-node
-     integer MAXCORE           ! maximum no. cores on any SMP-node
-     integer N_SND             ! size of SMP shared memory buffer
-     integer N_RCV             ! size of SMP shared memory buffer
-     integer(8) SND_P          ! SNDBUF address (cray pointer), for real 
-     integer(8) RCV_P          ! RCVBUF address (cray pointer), for real
-     integer(8) SND_P_c        ! for complex
-     integer(8) RCV_P_c        ! for complex
-  END TYPE SMP_INFO
+   ! derived type to store shared-memory info
+   TYPE, public :: SMP_INFO
+      integer MPI_COMM          ! SMP associated with this communicator
+      integer NODE_ME           ! rank in this communicator
+      integer NCPU              ! size of this communicator
+      integer SMP_COMM          ! communicator for SMP-node masters
+      integer CORE_COMM         ! communicator for cores on SMP-node
+      integer SMP_ME            ! SMP-node id starting from 1 ... NSMP
+      integer NSMP              ! number of SMP-nodes in this communicator
+      integer CORE_ME           ! core id starting from 1 ... NCORE
+      integer NCORE             ! number of cores on this SMP-node
+      integer MAXCORE           ! maximum no. cores on any SMP-node
+      integer N_SND             ! size of SMP shared memory buffer
+      integer N_RCV             ! size of SMP shared memory buffer
+      integer(8) SND_P          ! SNDBUF address (cray pointer), for real
+      integer(8) RCV_P          ! RCVBUF address (cray pointer), for real
+      integer(8) SND_P_c        ! for complex
+      integer(8) RCV_P_c        ! for complex
+   END TYPE SMP_INFO
 #endif
 
-  ! derived type to store decomposition info for a given global data size
-  TYPE, public :: DECOMP_INFO
-     ! staring/ending index and size of data held by current processor
-     integer, dimension(3) :: xst, xen, xsz  ! x-pencil
-     integer, dimension(3) :: yst, yen, ysz  ! y-pencil
-     integer, dimension(3) :: zst, zen, zsz  ! z-pencil
+   ! derived type to store decomposition info for a given global data size
+   TYPE, public :: DECOMP_INFO
+      ! staring/ending index and size of data held by current processor
+      integer, dimension(3) :: xst, xen, xsz  ! x-pencil
+      integer, dimension(3) :: yst, yen, ysz  ! y-pencil
+      integer, dimension(3) :: zst, zen, zsz  ! z-pencil
 
-     ! in addition to local information, processors also need to know 
-     ! some global information for global communications to work 
+      ! in addition to local information, processors also need to know
+      ! some global information for global communications to work
 
-     ! how each dimension is distributed along pencils
-     integer, allocatable, dimension(:) :: &
-          x1dist, y1dist, y2dist, z2dist
+      ! how each dimension is distributed along pencils
+      integer, allocatable, dimension(:) :: &
+         x1dist, y1dist, y2dist, z2dist
 
-     ! send/receive buffer counts and displacements for MPI_ALLTOALLV
-     integer, allocatable, dimension(:) :: &
-          x1cnts, y1cnts, y2cnts, z2cnts
-     integer, allocatable, dimension(:) :: &
-          x1disp, y1disp, y2disp, z2disp
+      ! send/receive buffer counts and displacements for MPI_ALLTOALLV
+      integer, allocatable, dimension(:) :: &
+         x1cnts, y1cnts, y2cnts, z2cnts
+      integer, allocatable, dimension(:) :: &
+         x1disp, y1disp, y2disp, z2disp
 
-     ! buffer counts for MPI_ALLTOALL: either for evenly distributed data
-     ! or for padded-alltoall
-     integer :: x1count, y1count, y2count, z2count
+      ! buffer counts for MPI_ALLTOALL: either for evenly distributed data
+      ! or for padded-alltoall
+      integer :: x1count, y1count, y2count, z2count
 
-     ! evenly distributed data
-     logical :: even
+      ! evenly distributed data
+      logical :: even
 
 #ifdef SHM
-     ! For shared-memory implementation
-
-     ! one instance of this derived type for each communicator
-     ! shared moemory info, such as which MPI rank belongs to which node
-     TYPE(SMP_INFO) :: ROW_INFO, COL_INFO
-
-     ! shared send/recv buffers for ALLTOALLV
-     integer, allocatable, dimension(:) :: x1cnts_s, y1cnts_s, &
-          y2cnts_s, z2cnts_s
-     integer, allocatable, dimension(:) :: x1disp_s, y1disp_s, &
-          y2disp_s, z2disp_s
-     ! A copy of original buffer displacement (will be overwriten)
-     integer, allocatable, dimension(:) :: x1disp_o, y1disp_o, &
-          y2disp_o, z2disp_o
+      ! For shared-memory implementation
+
+      ! one instance of this derived type for each communicator
+      ! shared moemory info, such as which MPI rank belongs to which node
+      TYPE(SMP_INFO) :: ROW_INFO, COL_INFO
+
+      ! shared send/recv buffers for ALLTOALLV
+      integer, allocatable, dimension(:) :: x1cnts_s, y1cnts_s, &
+                                            y2cnts_s, z2cnts_s
+      integer, allocatable, dimension(:) :: x1disp_s, y1disp_s, &
+                                            y2disp_s, z2disp_s
+      ! A copy of original buffer displacement (will be overwriten)
+      integer, allocatable, dimension(:) :: x1disp_o, y1disp_o, &
+                                            y2disp_o, z2disp_o
 #endif
-  END TYPE DECOMP_INFO
+   END TYPE DECOMP_INFO
 
-  ! main (default) decomposition information for global size nx*ny*nz
-  TYPE(DECOMP_INFO), target, save, public :: decomp_main
-  ! FIXME The extra decomp_info objects should be defined in the external code, not here
-  !       Currently keeping them to avoid breaking external codes
-  TYPE(DECOMP_INFO), save, public :: phG,ph1,ph2,ph3,ph4
+   ! main (default) decomposition information for global size nx*ny*nz
+   TYPE(DECOMP_INFO), target, save, public :: decomp_main
+   ! FIXME The extra decomp_info objects should be defined in the external code, not here
+   !       Currently keeping them to avoid breaking external codes
+   TYPE(DECOMP_INFO), save, public :: phG, ph1, ph2, ph3, ph4
 
-  ! staring/ending index and size of data held by current processor
-  ! duplicate 'decomp_main', needed by apps to define data structure 
-  integer, save, dimension(3), public :: xstart, xend, xsize  ! x-pencil
-  integer, save, dimension(3), public :: ystart, yend, ysize  ! y-pencil
-  integer, save, dimension(3), public :: zstart, zend, zsize  ! z-pencil
+   ! staring/ending index and size of data held by current processor
+   ! duplicate 'decomp_main', needed by apps to define data structure
+   integer, save, dimension(3), public :: xstart, xend, xsize  ! x-pencil
+   integer, save, dimension(3), public :: ystart, yend, ysize  ! y-pencil
+   integer, save, dimension(3), public :: zstart, zend, zsize  ! z-pencil
 
-  ! These are the buffers used by MPI_ALLTOALL(V) calls
-  integer, save :: decomp_buf_size = 0
-  real(mytype),    allocatable, dimension(:) :: work1_r, work2_r
-  complex(mytype), allocatable, dimension(:) :: work1_c, work2_c
+   ! These are the buffers used by MPI_ALLTOALL(V) calls
+   integer, save :: decomp_buf_size = 0
+   real(mytype), allocatable, dimension(:) :: work1_r, work2_r
+   complex(mytype), allocatable, dimension(:) :: work1_c, work2_c
 
 #if defined(_GPU)
-  real(mytype), allocatable, dimension(:), device :: work1_r_d, work2_r_d
-  complex(mytype), allocatable, dimension(:), device :: work1_c_d, work2_c_d
+   real(mytype), allocatable, dimension(:), device :: work1_r_d, work2_r_d
+   complex(mytype), allocatable, dimension(:), device :: work1_c_d, work2_c_d
 
 #if defined(_NCCL)
-  integer col_comm_size, row_comm_size
-  integer, allocatable, dimension(:) :: local_to_global_col, local_to_global_row
-  type(ncclUniqueId) :: nccl_uid_2decomp
-  type(ncclComm) :: nccl_comm_2decomp
-  integer(kind=cuda_stream_kind) :: cuda_stream_2decomp
+   integer col_comm_size, row_comm_size
+   integer, allocatable, dimension(:) :: local_to_global_col, local_to_global_row
+   type(ncclUniqueId) :: nccl_uid_2decomp
+   type(ncclComm) :: nccl_comm_2decomp
+   integer(kind=cuda_stream_kind) :: cuda_stream_2decomp
 #endif
 #endif
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! To define smaller arrays using every several mesh points
-  integer, save, dimension(3), public :: xszS,yszS,zszS,xstS,ystS,zstS,xenS,yenS,zenS
-  integer, save, dimension(3), public :: xszV,yszV,zszV,xstV,ystV,zstV,xenV,yenV,zenV
-  integer, save, dimension(3), public :: xszP,yszP,zszP,xstP,ystP,zstP,xenP,yenP,zenP
-  logical, save :: coarse_mesh_starts_from_1
-  integer, save :: iskipS, jskipS, kskipS
-  integer, save :: iskipV, jskipV, kskipV
-  integer, save :: iskipP, jskipP, kskipP
-
-  !
-  ! Profiler section
-  !
-  ! Integer to select the profiling tool
-  !    0 => no profiling, default
-  !    1 => Caliper (https://github.com/LLNL/Caliper)
-  !
-  enum, bind(c)
-    enumerator :: decomp_profiler_none = 0
-    enumerator :: decomp_profiler_caliper = 1
-  end enum
-  integer(kind(decomp_profiler_none)), save, public :: decomp_profiler = decomp_profiler_none
-  ! Default : profile everything
-  logical, save, public :: decomp_profiler_transpose = .true.
-  logical, save, public :: decomp_profiler_io = .true.
-  logical, save, public :: decomp_profiler_fft = .true.
-  logical, save, public :: decomp_profiler_d2d = .true.
-
-  ! public user routines
-  public :: decomp_2d_init, decomp_2d_finalize, &
-       transpose_x_to_y, transpose_y_to_z, &
-       transpose_z_to_y, transpose_y_to_x, &
-       decomp_info_init, decomp_info_finalize, partition, &
-       decomp_info_print, decomp_profiler_prep, &
-       decomp_profiler_start, decomp_profiler_end, &
-       init_coarser_mesh_statS,fine_to_coarseS,&
-       init_coarser_mesh_statV,fine_to_coarseV,&
-       init_coarser_mesh_statP,fine_to_coarseP,&
-       alloc_x, alloc_y, alloc_z, &
-       update_halo, decomp_2d_abort, &
-       decomp_2d_warning, get_decomp_info, &
-       decomp_mpi_comm_free, get_decomp_dims
+   ! To define smaller arrays using every several mesh points
+   integer, save, dimension(3), public :: xszS, yszS, zszS, xstS, ystS, zstS, xenS, yenS, zenS
+   integer, save, dimension(3), public :: xszV, yszV, zszV, xstV, ystV, zstV, xenV, yenV, zenV
+   integer, save, dimension(3), public :: xszP, yszP, zszP, xstP, ystP, zstP, xenP, yenP, zenP
+   logical, save :: coarse_mesh_starts_from_1
+   integer, save :: iskipS, jskipS, kskipS
+   integer, save :: iskipV, jskipV, kskipV
+   integer, save :: iskipP, jskipP, kskipP
+
+   !
+   ! Profiler section
+   !
+   ! Integer to select the profiling tool
+   !    0 => no profiling, default
+   !    1 => Caliper (https://github.com/LLNL/Caliper)
+   !
+   enum, bind(c)
+      enumerator :: decomp_profiler_none = 0
+      enumerator :: decomp_profiler_caliper = 1
+   end enum
+   integer(kind(decomp_profiler_none)), save, public :: decomp_profiler = decomp_profiler_none
+   ! Default : profile everything
+   logical, save, public :: decomp_profiler_transpose = .true.
+   logical, save, public :: decomp_profiler_io = .true.
+   logical, save, public :: decomp_profiler_fft = .true.
+   logical, save, public :: decomp_profiler_d2d = .true.
+
+   ! public user routines
+   public :: decomp_2d_init, decomp_2d_finalize, &
+             transpose_x_to_y, transpose_y_to_z, &
+             transpose_z_to_y, transpose_y_to_x, &
+             decomp_info_init, decomp_info_finalize, partition, &
+             decomp_info_print, decomp_profiler_prep, &
+             decomp_profiler_start, decomp_profiler_end, &
+             init_coarser_mesh_statS, fine_to_coarseS, &
+             init_coarser_mesh_statV, fine_to_coarseV, &
+             init_coarser_mesh_statP, fine_to_coarseP, &
+             alloc_x, alloc_y, alloc_z, &
+             update_halo, decomp_2d_abort, &
+             decomp_2d_warning, get_decomp_info, &
+             decomp_mpi_comm_free, get_decomp_dims
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! These are routines to perform global data transpositions
-  ! 
-  !   Four combinations are available, enough to cover all situations
-  !    - transpose_x_to_y (X-pencil --> Y-pencil)
-  !    - transpose_y_to_z (Y-pencil --> Z-pencil)
-  !    - transpose_z_to_y (Z-pencil --> Y-pencil)
-  !    - transpose_y_to_x (Y-pencil --> X-pencil)
-  !
-  !   Generic interface provided here to support multiple data types
-  !    - real and complex types supported through generic interface
-  !    - single/double precision supported through pre-processing
-  !       * see 'mytype' variable at the beginning
-  !    - an optional argument can be supplied to transpose data whose 
-  !      global size is not the default nx*ny*nz 
-  !       * as the case in fft r2c/c2r interface 
+   ! These are routines to perform global data transpositions
+   !
+   !   Four combinations are available, enough to cover all situations
+   !    - transpose_x_to_y (X-pencil --> Y-pencil)
+   !    - transpose_y_to_z (Y-pencil --> Z-pencil)
+   !    - transpose_z_to_y (Z-pencil --> Y-pencil)
+   !    - transpose_y_to_x (Y-pencil --> X-pencil)
+   !
+   !   Generic interface provided here to support multiple data types
+   !    - real and complex types supported through generic interface
+   !    - single/double precision supported through pre-processing
+   !       * see 'mytype' variable at the beginning
+   !    - an optional argument can be supplied to transpose data whose
+   !      global size is not the default nx*ny*nz
+   !       * as the case in fft r2c/c2r interface
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  interface decomp_2d_init
-     module procedure decomp_2d_init_ref
-  end interface decomp_2d_init
-
-  interface decomp_2d_finalize
-     module procedure decomp_2d_finalize_ref
-  end interface decomp_2d_finalize
-  
-  interface transpose_x_to_y
-     module procedure transpose_x_to_y_real
-     module procedure transpose_x_to_y_real_short
-     module procedure transpose_x_to_y_complex
-     module procedure transpose_x_to_y_complex_short
-  end interface transpose_x_to_y
-
-  interface transpose_y_to_z
-     module procedure transpose_y_to_z_real
-     module procedure transpose_y_to_z_real_short
-     module procedure transpose_y_to_z_complex
-     module procedure transpose_y_to_z_complex_short
-  end interface transpose_y_to_z
-
-  interface transpose_z_to_y
-     module procedure transpose_z_to_y_real
-     module procedure transpose_z_to_y_real_short
-     module procedure transpose_z_to_y_complex
-     module procedure transpose_z_to_y_complex_short
-  end interface transpose_z_to_y
-
-  interface transpose_y_to_x
-     module procedure transpose_y_to_x_real
-     module procedure transpose_y_to_x_real_short
-     module procedure transpose_y_to_x_complex
-     module procedure transpose_y_to_x_complex_short
-  end interface transpose_y_to_x
-
-  interface update_halo
-     module procedure update_halo_real
-     module procedure update_halo_real_short
-     module procedure update_halo_complex
-     module procedure update_halo_complex_short
-  end interface update_halo
-
-  interface alloc_x
-     module procedure alloc_x_real
-     module procedure alloc_x_real_short
-     module procedure alloc_x_complex
-     module procedure alloc_x_complex_short
-  end interface alloc_x
-
-  interface alloc_y
-     module procedure alloc_y_real
-     module procedure alloc_y_real_short
-     module procedure alloc_y_complex
-     module procedure alloc_y_complex_short
-  end interface alloc_y
-
-  interface alloc_z
-     module procedure alloc_z_real
-     module procedure alloc_z_real_short
-     module procedure alloc_z_complex
-     module procedure alloc_z_complex_short
-  end interface alloc_z
-
-  interface decomp_2d_abort
-     module procedure decomp_2d_abort_basic
-     module procedure decomp_2d_abort_file_line
+   interface decomp_2d_init
+      module procedure decomp_2d_init_ref
+   end interface decomp_2d_init
+
+   interface decomp_2d_finalize
+      module procedure decomp_2d_finalize_ref
+   end interface decomp_2d_finalize
+
+   interface transpose_x_to_y
+      module procedure transpose_x_to_y_real
+      module procedure transpose_x_to_y_real_short
+      module procedure transpose_x_to_y_complex
+      module procedure transpose_x_to_y_complex_short
+   end interface transpose_x_to_y
+
+   interface transpose_y_to_z
+      module procedure transpose_y_to_z_real
+      module procedure transpose_y_to_z_real_short
+      module procedure transpose_y_to_z_complex
+      module procedure transpose_y_to_z_complex_short
+   end interface transpose_y_to_z
+
+   interface transpose_z_to_y
+      module procedure transpose_z_to_y_real
+      module procedure transpose_z_to_y_real_short
+      module procedure transpose_z_to_y_complex
+      module procedure transpose_z_to_y_complex_short
+   end interface transpose_z_to_y
+
+   interface transpose_y_to_x
+      module procedure transpose_y_to_x_real
+      module procedure transpose_y_to_x_real_short
+      module procedure transpose_y_to_x_complex
+      module procedure transpose_y_to_x_complex_short
+   end interface transpose_y_to_x
+
+   interface update_halo
+      module procedure update_halo_real
+      module procedure update_halo_real_short
+      module procedure update_halo_complex
+      module procedure update_halo_complex_short
+   end interface update_halo
+
+   interface alloc_x
+      module procedure alloc_x_real
+      module procedure alloc_x_real_short
+      module procedure alloc_x_complex
+      module procedure alloc_x_complex_short
+   end interface alloc_x
+
+   interface alloc_y
+      module procedure alloc_y_real
+      module procedure alloc_y_real_short
+      module procedure alloc_y_complex
+      module procedure alloc_y_complex_short
+   end interface alloc_y
+
+   interface alloc_z
+      module procedure alloc_z_real
+      module procedure alloc_z_real_short
+      module procedure alloc_z_complex
+      module procedure alloc_z_complex_short
+   end interface alloc_z
+
+   interface decomp_2d_abort
+      module procedure decomp_2d_abort_basic
+      module procedure decomp_2d_abort_file_line
 #if defined(_GPU) && defined(_NCCL)
-     module procedure decomp_2d_abort_nccl_basic
-     module procedure decomp_2d_abort_nccl_file_line
+      module procedure decomp_2d_abort_nccl_basic
+      module procedure decomp_2d_abort_nccl_file_line
 #endif
-  end interface decomp_2d_abort
+   end interface decomp_2d_abort
 
-  interface decomp_2d_warning
-     module procedure decomp_2d_warning_basic
-     module procedure decomp_2d_warning_file_line
-  end interface decomp_2d_warning
+   interface decomp_2d_warning
+      module procedure decomp_2d_warning_basic
+      module procedure decomp_2d_warning_file_line
+   end interface decomp_2d_warning
 
-  interface
+   interface
 
-     module subroutine d2d_listing(given_io_unit)
-        integer, intent(in), optional :: given_io_unit
-     end subroutine d2d_listing
+      module subroutine d2d_listing(given_io_unit)
+         integer, intent(in), optional :: given_io_unit
+      end subroutine d2d_listing
 
-     module subroutine decomp_info_print(d2d, io_unit, d2dname)
-        type(decomp_info), intent(in) :: d2d
-        integer, intent(in) :: io_unit
-        character(len=*), intent(in) :: d2dname
-     end subroutine decomp_info_print
+      module subroutine decomp_info_print(d2d, io_unit, d2dname)
+         type(decomp_info), intent(in) :: d2d
+         integer, intent(in) :: io_unit
+         character(len=*), intent(in) :: d2dname
+      end subroutine decomp_info_print
 
-  end interface
+   end interface
 
    ! Generic interface to initialize the profiler
    interface decomp_profiler_init
@@ -412,1228 +412,1220 @@ end subroutine decomp_profiler_end_char
    end interface decomp_profiler_end
 
 contains
-        
+
 #include "decomp_2d_init_fin.f90"
 
-  !
-  ! Small wrapper to free a MPI communicator
-  !
-  subroutine decomp_mpi_comm_free(mpi_comm)
+   !
+   ! Small wrapper to free a MPI communicator
+   !
+   subroutine decomp_mpi_comm_free(mpi_comm)
 
-    implicit none
+      implicit none
 
-    integer, intent(inout) :: mpi_comm
-    integer :: ierror
+      integer, intent(inout) :: mpi_comm
+      integer :: ierror
 
-    ! Return if no MPI comm to free
-    if (mpi_comm == MPI_COMM_NULL) return
+      ! Return if no MPI comm to free
+      if (mpi_comm == MPI_COMM_NULL) return
 
-    ! Free the provided MPI communicator
-    call MPI_COMM_FREE(mpi_comm, ierror)
-    if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
-    mpi_comm = MPI_COMM_NULL
+      ! Free the provided MPI communicator
+      call MPI_COMM_FREE(mpi_comm, ierror)
+      if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
+      mpi_comm = MPI_COMM_NULL
 
-  end subroutine decomp_mpi_comm_free
+   end subroutine decomp_mpi_comm_free
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Return the default decomposition object
+   ! Return the default decomposition object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  ! FIXME avoid a copy and return a pointer to decomp_main
-  ! TODO list the external codes using this subroutine
-  subroutine get_decomp_info(decomp)
+   ! FIXME avoid a copy and return a pointer to decomp_main
+   ! TODO list the external codes using this subroutine
+   subroutine get_decomp_info(decomp)
 
-    implicit none
+      implicit none
 
-    ! FIXME TYPE(DECOMP_INFO), pointer :: decomp
-    TYPE(DECOMP_INFO), intent(OUT) :: decomp
+      ! FIXME TYPE(DECOMP_INFO), pointer :: decomp
+      TYPE(DECOMP_INFO), intent(OUT) :: decomp
 
-    ! FIXME decomp => decomp_main
-    decomp = decomp_main
+      ! FIXME decomp => decomp_main
+      decomp = decomp_main
 
-    return
-  end subroutine get_decomp_info
+      return
+   end subroutine get_decomp_info
 
-  !
-  ! Return the 2D processor grid
-  !
-  function get_decomp_dims()
+   !
+   ! Return the 2D processor grid
+   !
+   function get_decomp_dims()
 
-    implicit none
+      implicit none
 
-    integer, dimension(2) :: get_decomp_dims
+      integer, dimension(2) :: get_decomp_dims
 
-    get_decomp_dims = dims
+      get_decomp_dims = dims
 
-  end function get_decomp_dims
+   end function get_decomp_dims
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Advanced Interface allowing applications to define globle domain of
-  ! any size, distribute it, and then transpose data among pencils.
-  !  - generate 2D decomposition details as defined in DECOMP_INFO
-  !  - the default global data size is nx*ny*nz
-  !  - a different global size nx/2+1,ny,nz is used in FFT r2c/c2r
-  !  - multiple global sizes can co-exist in one application, each
-  !    using its own DECOMP_INFO object
+   ! Advanced Interface allowing applications to define globle domain of
+   ! any size, distribute it, and then transpose data among pencils.
+   !  - generate 2D decomposition details as defined in DECOMP_INFO
+   !  - the default global data size is nx*ny*nz
+   !  - a different global size nx/2+1,ny,nz is used in FFT r2c/c2r
+   !  - multiple global sizes can co-exist in one application, each
+   !    using its own DECOMP_INFO object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_info_init(nx,ny,nz,decomp)
-
-    implicit none
-
-    integer, intent(IN) :: nx,ny,nz
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-
-    integer :: buf_size, status, errorcode
-
-    ! verify the global size can actually be distributed as pencils
-    if (nx_global<dims(1) .or. ny_global<dims(1) .or. ny_global<dims(2) .or. nz_global<dims(2)) then
-       errorcode = 6
-       call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-            'Invalid 2D processor grid. ' // &
-            'Make sure that min(nx,ny) >= p_row and ' // &
-            'min(ny,nz) >= p_col')
-    end if
-
-    if (mod(nx,dims(1))==0 .and. mod(ny,dims(1))==0 .and. &
-         mod(ny,dims(2))==0 .and. mod(nz,dims(2))==0) then
-       decomp%even = .true.
-    else
-       decomp%even = .false.
-    end if
-
-    ! distribute mesh points
-    allocate(decomp%x1dist(0:dims(1)-1),decomp%y1dist(0:dims(1)-1), &
-         decomp%y2dist(0:dims(2)-1),decomp%z2dist(0:dims(2)-1))
-    call get_dist(nx,ny,nz,decomp)
-
-    ! generate partition information - starting/ending index etc.
-    call partition(nx, ny, nz, (/ 1,2,3 /), &
-         decomp%xst, decomp%xen, decomp%xsz)
-    call partition(nx, ny, nz, (/ 2,1,3 /), &
-         decomp%yst, decomp%yen, decomp%ysz)
-    call partition(nx, ny, nz, (/ 2,3,1 /), &
-         decomp%zst, decomp%zen, decomp%zsz)
-
-    ! prepare send/receive buffer displacement and count for ALLTOALL(V)
-    allocate(decomp%x1cnts(0:dims(1)-1),decomp%y1cnts(0:dims(1)-1), &
-         decomp%y2cnts(0:dims(2)-1),decomp%z2cnts(0:dims(2)-1))
-    allocate(decomp%x1disp(0:dims(1)-1),decomp%y1disp(0:dims(1)-1), &
-         decomp%y2disp(0:dims(2)-1),decomp%z2disp(0:dims(2)-1))
-    call prepare_buffer(decomp)
+   subroutine decomp_info_init(nx, ny, nz, decomp)
+
+      implicit none
+
+      integer, intent(IN) :: nx, ny, nz
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+
+      integer :: buf_size, status, errorcode
+
+      ! verify the global size can actually be distributed as pencils
+      if (nx_global < dims(1) .or. ny_global < dims(1) .or. ny_global < dims(2) .or. nz_global < dims(2)) then
+         errorcode = 6
+         call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                              'Invalid 2D processor grid. '// &
+                              'Make sure that min(nx,ny) >= p_row and '// &
+                              'min(ny,nz) >= p_col')
+      end if
+
+      if (mod(nx, dims(1)) == 0 .and. mod(ny, dims(1)) == 0 .and. &
+          mod(ny, dims(2)) == 0 .and. mod(nz, dims(2)) == 0) then
+         decomp%even = .true.
+      else
+         decomp%even = .false.
+      end if
+
+      ! distribute mesh points
+      allocate (decomp%x1dist(0:dims(1) - 1), decomp%y1dist(0:dims(1) - 1), &
+                decomp%y2dist(0:dims(2) - 1), decomp%z2dist(0:dims(2) - 1))
+      call get_dist(nx, ny, nz, decomp)
+
+      ! generate partition information - starting/ending index etc.
+      call partition(nx, ny, nz, (/1, 2, 3/), &
+                     decomp%xst, decomp%xen, decomp%xsz)
+      call partition(nx, ny, nz, (/2, 1, 3/), &
+                     decomp%yst, decomp%yen, decomp%ysz)
+      call partition(nx, ny, nz, (/2, 3, 1/), &
+                     decomp%zst, decomp%zen, decomp%zsz)
+
+      ! prepare send/receive buffer displacement and count for ALLTOALL(V)
+      allocate (decomp%x1cnts(0:dims(1) - 1), decomp%y1cnts(0:dims(1) - 1), &
+                decomp%y2cnts(0:dims(2) - 1), decomp%z2cnts(0:dims(2) - 1))
+      allocate (decomp%x1disp(0:dims(1) - 1), decomp%y1disp(0:dims(1) - 1), &
+                decomp%y2disp(0:dims(2) - 1), decomp%z2disp(0:dims(2) - 1))
+      call prepare_buffer(decomp)
 
 #ifdef SHM
-    ! prepare shared-memory information if required
-    call decomp_info_init_shm(decomp)
+      ! prepare shared-memory information if required
+      call decomp_info_init_shm(decomp)
 #endif
 
-    ! allocate memory for the MPI_ALLTOALL(V) buffers
-    ! define the buffers globally for performance reason
+      ! allocate memory for the MPI_ALLTOALL(V) buffers
+      ! define the buffers globally for performance reason
 
-    buf_size = max(decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3), &
-         max(decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3), &
-         decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)) )
+      buf_size = max(decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3), &
+                     max(decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3), &
+                         decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)))
 #ifdef EVEN
-    ! padded alltoall optimisation may need larger buffer space
-    buf_size = max(buf_size, &
-         max(decomp%x1count*dims(1),decomp%y2count*dims(2)) ) 
+      ! padded alltoall optimisation may need larger buffer space
+      buf_size = max(buf_size, &
+                     max(decomp%x1count*dims(1), decomp%y2count*dims(2)))
 #endif
 
-    ! check if additional memory is required
-    ! *** TODO: consider how to share the real/complex buffers 
-    if (buf_size > decomp_buf_size) then
-       decomp_buf_size = buf_size
+      ! check if additional memory is required
+      ! *** TODO: consider how to share the real/complex buffers
+      if (buf_size > decomp_buf_size) then
+         decomp_buf_size = buf_size
 #if defined(_GPU)
-       if (allocated(work1_r_d)) deallocate(work1_r_d)
-       if (allocated(work2_r_d)) deallocate(work2_r_d)
-       if (allocated(work1_c_d)) deallocate(work1_c_d)
-       if (allocated(work2_c_d)) deallocate(work2_c_d)
-       allocate(work1_r_d(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work1_c_d(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work2_r_d(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work2_c_d(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
+         if (allocated(work1_r_d)) deallocate (work1_r_d)
+         if (allocated(work2_r_d)) deallocate (work2_r_d)
+         if (allocated(work1_c_d)) deallocate (work1_c_d)
+         if (allocated(work2_c_d)) deallocate (work2_c_d)
+         allocate (work1_r_d(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work1_c_d(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work2_r_d(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work2_c_d(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
 #endif
-       if (allocated(work1_r)) deallocate(work1_r)
-       if (allocated(work2_r)) deallocate(work2_r)
-       if (allocated(work1_c)) deallocate(work1_c)
-       if (allocated(work2_c)) deallocate(work2_c)
-       allocate(work1_r(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work2_r(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work1_c(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work2_c(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-    end if
-
-    return
-  end subroutine decomp_info_init
-
+         if (allocated(work1_r)) deallocate (work1_r)
+         if (allocated(work2_r)) deallocate (work2_r)
+         if (allocated(work1_c)) deallocate (work1_c)
+         if (allocated(work2_c)) deallocate (work2_c)
+         allocate (work1_r(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work2_r(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work1_c(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work2_c(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+      end if
+
+      return
+   end subroutine decomp_info_init
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Release memory associated with a DECOMP_INFO object
+   ! Release memory associated with a DECOMP_INFO object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_info_finalize(decomp)
+   subroutine decomp_info_finalize(decomp)
 
-    implicit none
+      implicit none
 
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
 
-    if (allocated(decomp%x1dist)) deallocate(decomp%x1dist)
-    if (allocated(decomp%y1dist)) deallocate(decomp%y1dist)
-    if (allocated(decomp%y2dist)) deallocate(decomp%y2dist)
-    if (allocated(decomp%z2dist)) deallocate(decomp%z2dist)
-    if (allocated(decomp%x1cnts)) deallocate(decomp%x1cnts)
-    if (allocated(decomp%y1cnts)) deallocate(decomp%y1cnts)
-    if (allocated(decomp%y2cnts)) deallocate(decomp%y2cnts)
-    if (allocated(decomp%z2cnts)) deallocate(decomp%z2cnts)
-    if (allocated(decomp%x1disp)) deallocate(decomp%x1disp)
-    if (allocated(decomp%y1disp)) deallocate(decomp%y1disp)
-    if (allocated(decomp%y2disp)) deallocate(decomp%y2disp)
-    if (allocated(decomp%z2disp)) deallocate(decomp%z2disp)
+      if (allocated(decomp%x1dist)) deallocate (decomp%x1dist)
+      if (allocated(decomp%y1dist)) deallocate (decomp%y1dist)
+      if (allocated(decomp%y2dist)) deallocate (decomp%y2dist)
+      if (allocated(decomp%z2dist)) deallocate (decomp%z2dist)
+      if (allocated(decomp%x1cnts)) deallocate (decomp%x1cnts)
+      if (allocated(decomp%y1cnts)) deallocate (decomp%y1cnts)
+      if (allocated(decomp%y2cnts)) deallocate (decomp%y2cnts)
+      if (allocated(decomp%z2cnts)) deallocate (decomp%z2cnts)
+      if (allocated(decomp%x1disp)) deallocate (decomp%x1disp)
+      if (allocated(decomp%y1disp)) deallocate (decomp%y1disp)
+      if (allocated(decomp%y2disp)) deallocate (decomp%y2disp)
+      if (allocated(decomp%z2disp)) deallocate (decomp%z2disp)
 
 #ifdef SHM
-    if (allocated(decomp%x1disp_o)) deallocate(decomp%x1disp_o)
-    if (allocated(decomp%y1disp_o)) deallocate(decomp%y1disp_o)
-    if (allocated(decomp%y2disp_o)) deallocate(decomp%y2disp_o)
-    if (allocated(decomp%z2disp_o)) deallocate(decomp%z2disp_o)
-    if (allocated(decomp%x1cnts_s)) deallocate(decomp%x1cnts_s)
-    if (allocated(decomp%y1cnts_s)) deallocate(decomp%y1cnts_s)
-    if (allocated(decomp%y2cnts_s)) deallocate(decomp%y2cnts_s)
-    if (allocated(decomp%z2cnts_s)) deallocate(decomp%z2cnts_s)
-    if (allocated(decomp%x1disp_s)) deallocate(decomp%x1disp_s)
-    if (allocated(decomp%y1disp_s)) deallocate(decomp%y1disp_s)
-    if (allocated(decomp%y2disp_s)) deallocate(decomp%y2disp_s)
-    if (allocated(decomp%z2disp_s)) deallocate(decomp%z2disp_s)
+      if (allocated(decomp%x1disp_o)) deallocate (decomp%x1disp_o)
+      if (allocated(decomp%y1disp_o)) deallocate (decomp%y1disp_o)
+      if (allocated(decomp%y2disp_o)) deallocate (decomp%y2disp_o)
+      if (allocated(decomp%z2disp_o)) deallocate (decomp%z2disp_o)
+      if (allocated(decomp%x1cnts_s)) deallocate (decomp%x1cnts_s)
+      if (allocated(decomp%y1cnts_s)) deallocate (decomp%y1cnts_s)
+      if (allocated(decomp%y2cnts_s)) deallocate (decomp%y2cnts_s)
+      if (allocated(decomp%z2cnts_s)) deallocate (decomp%z2cnts_s)
+      if (allocated(decomp%x1disp_s)) deallocate (decomp%x1disp_s)
+      if (allocated(decomp%y1disp_s)) deallocate (decomp%y1disp_s)
+      if (allocated(decomp%y2disp_s)) deallocate (decomp%y2disp_s)
+      if (allocated(decomp%z2disp_s)) deallocate (decomp%z2disp_s)
 #endif
 
-    return
-  end subroutine decomp_info_finalize
-
+      return
+   end subroutine decomp_info_finalize
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Coarser mesh support for statistic
+   ! Coarser mesh support for statistic
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_coarser_mesh_statS(i_skip,j_skip,k_skip,from1)
-
-    implicit none
-
-    integer, intent(IN) :: i_skip,j_skip,k_skip
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
-
-    integer, dimension(3) :: skip
-    integer :: i
-
-    coarse_mesh_starts_from_1 = from1
-    iskipS = i_skip
-    jskipS = j_skip
-    kskipS = k_skip
-
-    skip(1)=iskipS
-    skip(2)=jskipS
-    skip(3)=kskipS
-
-    do i=1,3
-       if (from1) then
-          xstS(i) = (xstart(i)+skip(i)-1)/skip(i)
-          if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xstS(i)=xstS(i)+1
-          xenS(i) = (xend(i)+skip(i)-1)/skip(i)
-       else
-          xstS(i) = xstart(i)/skip(i)
-          if (mod(xstart(i),skip(i))/=0) xstS(i)=xstS(i)+1
-          xenS(i) = xend(i)/skip(i)
-       end if
-       xszS(i) = xenS(i)-xstS(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          ystS(i) = (ystart(i)+skip(i)-1)/skip(i)
-          if (mod(ystart(i)+skip(i)-1,skip(i))/=0) ystS(i)=ystS(i)+1
-          yenS(i) = (yend(i)+skip(i)-1)/skip(i)
-       else
-          ystS(i) = ystart(i)/skip(i)
-          if (mod(ystart(i),skip(i))/=0) ystS(i)=ystS(i)+1
-          yenS(i) = yend(i)/skip(i)
-       end if
-       yszS(i) = yenS(i)-ystS(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          zstS(i) = (zstart(i)+skip(i)-1)/skip(i)
-          if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zstS(i)=zstS(i)+1
-          zenS(i) = (zend(i)+skip(i)-1)/skip(i)
-       else
-          zstS(i) = zstart(i)/skip(i)
-          if (mod(zstart(i),skip(i))/=0) zstS(i)=zstS(i)+1
-          zenS(i) = zend(i)/skip(i)
-       end if
-       zszS(i) = zenS(i)-zstS(i)+1
-    end do
-
-    return
-  end subroutine init_coarser_mesh_statS
+   subroutine init_coarser_mesh_statS(i_skip, j_skip, k_skip, from1)
+
+      implicit none
+
+      integer, intent(IN) :: i_skip, j_skip, k_skip
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
+
+      integer, dimension(3) :: skip
+      integer :: i
+
+      coarse_mesh_starts_from_1 = from1
+      iskipS = i_skip
+      jskipS = j_skip
+      kskipS = k_skip
+
+      skip(1) = iskipS
+      skip(2) = jskipS
+      skip(3) = kskipS
+
+      do i = 1, 3
+         if (from1) then
+            xstS(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstS(i) = xstS(i) + 1
+            xenS(i) = (xend(i) + skip(i) - 1)/skip(i)
+         else
+            xstS(i) = xstart(i)/skip(i)
+            if (mod(xstart(i), skip(i)) /= 0) xstS(i) = xstS(i) + 1
+            xenS(i) = xend(i)/skip(i)
+         end if
+         xszS(i) = xenS(i) - xstS(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            ystS(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystS(i) = ystS(i) + 1
+            yenS(i) = (yend(i) + skip(i) - 1)/skip(i)
+         else
+            ystS(i) = ystart(i)/skip(i)
+            if (mod(ystart(i), skip(i)) /= 0) ystS(i) = ystS(i) + 1
+            yenS(i) = yend(i)/skip(i)
+         end if
+         yszS(i) = yenS(i) - ystS(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            zstS(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstS(i) = zstS(i) + 1
+            zenS(i) = (zend(i) + skip(i) - 1)/skip(i)
+         else
+            zstS(i) = zstart(i)/skip(i)
+            if (mod(zstart(i), skip(i)) /= 0) zstS(i) = zstS(i) + 1
+            zenS(i) = zend(i)/skip(i)
+         end if
+         zszS(i) = zenS(i) - zstS(i) + 1
+      end do
+
+      return
+   end subroutine init_coarser_mesh_statS
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Coarser mesh support for visualization
+   ! Coarser mesh support for visualization
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_coarser_mesh_statV(i_skip,j_skip,k_skip,from1)
-
-    implicit none
-
-    integer, intent(IN) :: i_skip,j_skip,k_skip
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
-
-    integer, dimension(3) :: skip
-    integer :: i
-
-    coarse_mesh_starts_from_1 = from1
-    iskipV = i_skip
-    jskipV = j_skip
-    kskipV = k_skip
-
-    skip(1)=iskipV
-    skip(2)=jskipV
-    skip(3)=kskipV
-
-    do i=1,3
-       if (from1) then
-          xstV(i) = (xstart(i)+skip(i)-1)/skip(i)
-          if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xstV(i)=xstV(i)+1
-          xenV(i) = (xend(i)+skip(i)-1)/skip(i)
-       else
-          xstV(i) = xstart(i)/skip(i)
-          if (mod(xstart(i),skip(i))/=0) xstV(i)=xstV(i)+1
-          xenV(i) = xend(i)/skip(i)
-       end if
-       xszV(i) = xenV(i)-xstV(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          ystV(i) = (ystart(i)+skip(i)-1)/skip(i)
-          if (mod(ystart(i)+skip(i)-1,skip(i))/=0) ystV(i)=ystV(i)+1
-          yenV(i) = (yend(i)+skip(i)-1)/skip(i)
-       else
-          ystV(i) = ystart(i)/skip(i)
-          if (mod(ystart(i),skip(i))/=0) ystV(i)=ystV(i)+1
-          yenV(i) = yend(i)/skip(i)
-       end if
-       yszV(i) = yenV(i)-ystV(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          zstV(i) = (zstart(i)+skip(i)-1)/skip(i)
-          if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zstV(i)=zstV(i)+1
-          zenV(i) = (zend(i)+skip(i)-1)/skip(i)
-       else
-          zstV(i) = zstart(i)/skip(i)
-          if (mod(zstart(i),skip(i))/=0) zstV(i)=zstV(i)+1
-          zenV(i) = zend(i)/skip(i)
-       end if
-       zszV(i) = zenV(i)-zstV(i)+1
-    end do
-
-    return
-  end subroutine init_coarser_mesh_statV
+   subroutine init_coarser_mesh_statV(i_skip, j_skip, k_skip, from1)
+
+      implicit none
+
+      integer, intent(IN) :: i_skip, j_skip, k_skip
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
+
+      integer, dimension(3) :: skip
+      integer :: i
+
+      coarse_mesh_starts_from_1 = from1
+      iskipV = i_skip
+      jskipV = j_skip
+      kskipV = k_skip
+
+      skip(1) = iskipV
+      skip(2) = jskipV
+      skip(3) = kskipV
+
+      do i = 1, 3
+         if (from1) then
+            xstV(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstV(i) = xstV(i) + 1
+            xenV(i) = (xend(i) + skip(i) - 1)/skip(i)
+         else
+            xstV(i) = xstart(i)/skip(i)
+            if (mod(xstart(i), skip(i)) /= 0) xstV(i) = xstV(i) + 1
+            xenV(i) = xend(i)/skip(i)
+         end if
+         xszV(i) = xenV(i) - xstV(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            ystV(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystV(i) = ystV(i) + 1
+            yenV(i) = (yend(i) + skip(i) - 1)/skip(i)
+         else
+            ystV(i) = ystart(i)/skip(i)
+            if (mod(ystart(i), skip(i)) /= 0) ystV(i) = ystV(i) + 1
+            yenV(i) = yend(i)/skip(i)
+         end if
+         yszV(i) = yenV(i) - ystV(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            zstV(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstV(i) = zstV(i) + 1
+            zenV(i) = (zend(i) + skip(i) - 1)/skip(i)
+         else
+            zstV(i) = zstart(i)/skip(i)
+            if (mod(zstart(i), skip(i)) /= 0) zstV(i) = zstV(i) + 1
+            zenV(i) = zend(i)/skip(i)
+         end if
+         zszV(i) = zenV(i) - zstV(i) + 1
+      end do
+
+      return
+   end subroutine init_coarser_mesh_statV
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Coarser mesh support for probe
+   ! Coarser mesh support for probe
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_coarser_mesh_statP(i_skip,j_skip,k_skip,from1)
-
-    implicit none
-
-    integer, intent(IN) :: i_skip,j_skip,k_skip
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
-
-    integer, dimension(3) :: skip
-    integer :: i
-
-    coarse_mesh_starts_from_1 = from1
-    iskipP = i_skip
-    jskipP = j_skip
-    kskipP = k_skip
-
-    skip(1)=iskipP
-    skip(2)=jskipP
-    skip(3)=kskipP
-
-    do i=1,3
-       if (from1) then
-          xstP(i) = (xstart(i)+skip(i)-1)/skip(i)
-          if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xstP(i)=xstP(i)+1
-          xenP(i) = (xend(i)+skip(i)-1)/skip(i)
-       else
-          xstP(i) = xstart(i)/skip(i)
-          if (mod(xstart(i),skip(i))/=0) xstP(i)=xstP(i)+1
-          xenP(i) = xend(i)/skip(i)
-       end if
-       xszP(i) = xenP(i)-xstP(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          ystP(i) = (ystart(i)+skip(i)-1)/skip(i)
-          if (mod(ystart(i)+skip(i)-1,skip(i))/=0) ystP(i)=ystP(i)+1
-          yenP(i) = (yend(i)+skip(i)-1)/skip(i)
-       else
-          ystP(i) = ystart(i)/skip(i)
-          if (mod(ystart(i),skip(i))/=0) ystP(i)=ystP(i)+1
-          yenP(i) = yend(i)/skip(i)
-       end if
-       yszP(i) = yenP(i)-ystP(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          zstP(i) = (zstart(i)+skip(i)-1)/skip(i)
-          if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zstP(i)=zstP(i)+1
-          zenP(i) = (zend(i)+skip(i)-1)/skip(i)
-       else
-          zstP(i) = zstart(i)/skip(i)
-          if (mod(zstart(i),skip(i))/=0) zstP(i)=zstP(i)+1
-          zenP(i) = zend(i)/skip(i)
-       end if
-       zszP(i) = zenP(i)-zstP(i)+1
-    end do
-
-    return
-  end subroutine init_coarser_mesh_statP
-
-  ! Copy data from a fine-resolution array to a coarse one for statistic
-  subroutine fine_to_coarseS(ipencil,var_fine,var_coarse)
-
-    implicit none
-
-    real(mytype), dimension(:,:,:) :: var_fine
-    real(mytype), dimension(:,:,:) :: var_coarse
-    integer, intent(IN) :: ipencil
-
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer :: i,j,k
-
-    if (ipencil==1) then
-       allocate(wk(xstS(1):xenS(1),xstS(2):xenS(2),xstS(3):xenS(3)))
-       allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=xstS(3),xenS(3)
-             do j=xstS(2),xenS(2)
-                do i=xstS(1),xenS(1)
-                   wk(i,j,k) = wk2((i-1)*iskipS+1,(j-1)*jskipS+1,(k-1)*kskipS+1)
-                end do
-             end do
-          end do
-       else
-          do k=xstS(3),xenS(3)
-             do j=xstS(2),xenS(2)
-                do i=xstS(1),xenS(1)
-                   wk(i,j,k) = wk2(i*iskipS,j*jskipS,k*kskipS)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==2) then
-       allocate(wk(ystS(1):yenS(1),ystS(2):yenS(2),ystS(3):yenS(3)))
-       allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=ystS(3),yenS(3)
-             do j=ystS(2),yenS(2)
-                do i=ystS(1),yenS(1)
-                   wk(i,j,k) = wk2((i-1)*iskipS+1,(j-1)*jskipS+1,(k-1)*kskipS+1)
-                end do
-             end do
-          end do
-       else
-          do k=ystS(3),yenS(3)
-             do j=ystS(2),yenS(2)
-                do i=ystS(1),yenS(1)
-                   wk(i,j,k) = wk2(i*iskipS,j*jskipS,k*kskipS)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==3) then
-       allocate(wk(zstS(1):zenS(1),zstS(2):zenS(2),zstS(3):zenS(3)))
-       allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=zstS(3),zenS(3)
-             do j=zstS(2),zenS(2)
-                do i=zstS(1),zenS(1)
-                   wk(i,j,k) = wk2((i-1)*iskipS+1,(j-1)*jskipS+1,(k-1)*kskipS+1)
-                end do
-             end do
-          end do
-       else
-          do k=zstS(3),zenS(3)
-             do j=zstS(2),zenS(2)
-                do i=zstS(1),zenS(1)
-                   wk(i,j,k) = wk2(i*iskipS,j*jskipS,k*kskipS)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    end if
-
-    deallocate(wk,wk2)
-
-    return
-  end subroutine fine_to_coarseS
-
-  ! Copy data from a fine-resolution array to a coarse one for visualization
-  subroutine fine_to_coarseV(ipencil,var_fine,var_coarse)
-
-    implicit none
-
-    real(mytype), dimension(:,:,:) :: var_fine
-    real(mytype), dimension(:,:,:) :: var_coarse
-    integer, intent(IN) :: ipencil
-
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer :: i,j,k
-
-    if (ipencil==1) then
-       allocate(wk(xstV(1):xenV(1),xstV(2):xenV(2),xstV(3):xenV(3)))
-       allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=xstV(3),xenV(3)
-             do j=xstV(2),xenV(2)
-                do i=xstV(1),xenV(1)
-                   wk(i,j,k) = wk2((i-1)*iskipV+1,(j-1)*jskipV+1,(k-1)*kskipV+1)
-                end do
-             end do
-          end do
-       else
-          do k=xstV(3),xenV(3)
-             do j=xstV(2),xenV(2)
-                do i=xstV(1),xenV(1)
-                   wk(i,j,k) = wk2(i*iskipV,j*jskipV,k*kskipV)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==2) then
-       allocate(wk(ystV(1):yenV(1),ystV(2):yenV(2),ystV(3):yenV(3)))
-       allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=ystV(3),yenV(3)
-             do j=ystV(2),yenV(2)
-                do i=ystV(1),yenV(1)
-                   wk(i,j,k) = wk2((i-1)*iskipV+1,(j-1)*jskipV+1,(k-1)*kskipV+1)
-                end do
-             end do
-          end do
-       else
-          do k=ystV(3),yenV(3)
-             do j=ystV(2),yenV(2)
-                do i=ystV(1),yenV(1)
-                   wk(i,j,k) = wk2(i*iskipV,j*jskipV,k*kskipV)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==3) then
-       allocate(wk(zstV(1):zenV(1),zstV(2):zenV(2),zstV(3):zenV(3)))
-       allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=zstV(3),zenV(3)
-             do j=zstV(2),zenV(2)
-                do i=zstV(1),zenV(1)
-                   wk(i,j,k) = wk2((i-1)*iskipV+1,(j-1)*jskipV+1,(k-1)*kskipV+1)
-                end do
-             end do
-          end do
-       else
-          do k=zstV(3),zenV(3)
-             do j=zstV(2),zenV(2)
-                do i=zstV(1),zenV(1)
-                   wk(i,j,k) = wk2(i*iskipV,j*jskipV,k*kskipV)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    end if
-
-    deallocate(wk,wk2)
-
-    return
-  end subroutine fine_to_coarseV
-
-  ! Copy data from a fine-resolution array to a coarse one for probe
-  subroutine fine_to_coarseP(ipencil,var_fine,var_coarse)
-
-    implicit none
-
-    real(mytype), dimension(:,:,:) :: var_fine
-    real(mytype), dimension(:,:,:) :: var_coarse
-    integer, intent(IN) :: ipencil
-
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer :: i,j,k
-
-    if (ipencil==1) then
-       allocate(wk(xstP(1):xenP(1),xstP(2):xenP(2),xstP(3):xenP(3)))
-       allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=xstP(3),xenP(3)
-             do j=xstP(2),xenP(2)
-                do i=xstP(1),xenP(1)
-                   wk(i,j,k) = wk2((i-1)*iskipP+1,(j-1)*jskipP+1,(k-1)*kskipP+1)
-                end do
-             end do
-          end do
-       else
-          do k=xstP(3),xenP(3)
-             do j=xstP(2),xenP(2)
-                do i=xstP(1),xenP(1)
-                   wk(i,j,k) = wk2(i*iskipP,j*jskipP,k*kskipP)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==2) then
-       allocate(wk(ystP(1):yenP(1),ystP(2):yenP(2),ystP(3):yenP(3)))
-       allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=ystP(3),yenP(3)
-             do j=ystP(2),yenP(2)
-                do i=ystP(1),yenP(1)
-                   wk(i,j,k) = wk2((i-1)*iskipP+1,(j-1)*jskipP+1,(k-1)*kskipP+1)
-                end do
-             end do
-          end do
-       else
-          do k=ystP(3),yenP(3)
-             do j=ystP(2),yenP(2)
-                do i=ystP(1),yenP(1)
-                   wk(i,j,k) = wk2(i*iskipP,j*jskipP,k*kskipP)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==3) then
-       allocate(wk(zstP(1):zenP(1),zstP(2):zenP(2),zstP(3):zenP(3)))
-       allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=zstP(3),zenP(3)
-             do j=zstP(2),zenP(2)
-                do i=zstP(1),zenP(1)
-                   wk(i,j,k) = wk2((i-1)*iskipP+1,(j-1)*jskipP+1,(k-1)*kskipP+1)
-                end do
-             end do
-          end do
-       else
-          do k=zstP(3),zenP(3)
-             do j=zstP(2),zenP(2)
-                do i=zstP(1),zenP(1)
-                   wk(i,j,k) = wk2(i*iskipP,j*jskipP,k*kskipP)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    end if
-
-    deallocate(wk,wk2)
-
-    return
-  end subroutine fine_to_coarseP
-
+   subroutine init_coarser_mesh_statP(i_skip, j_skip, k_skip, from1)
+
+      implicit none
+
+      integer, intent(IN) :: i_skip, j_skip, k_skip
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
+
+      integer, dimension(3) :: skip
+      integer :: i
+
+      coarse_mesh_starts_from_1 = from1
+      iskipP = i_skip
+      jskipP = j_skip
+      kskipP = k_skip
+
+      skip(1) = iskipP
+      skip(2) = jskipP
+      skip(3) = kskipP
+
+      do i = 1, 3
+         if (from1) then
+            xstP(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstP(i) = xstP(i) + 1
+            xenP(i) = (xend(i) + skip(i) - 1)/skip(i)
+         else
+            xstP(i) = xstart(i)/skip(i)
+            if (mod(xstart(i), skip(i)) /= 0) xstP(i) = xstP(i) + 1
+            xenP(i) = xend(i)/skip(i)
+         end if
+         xszP(i) = xenP(i) - xstP(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            ystP(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystP(i) = ystP(i) + 1
+            yenP(i) = (yend(i) + skip(i) - 1)/skip(i)
+         else
+            ystP(i) = ystart(i)/skip(i)
+            if (mod(ystart(i), skip(i)) /= 0) ystP(i) = ystP(i) + 1
+            yenP(i) = yend(i)/skip(i)
+         end if
+         yszP(i) = yenP(i) - ystP(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            zstP(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstP(i) = zstP(i) + 1
+            zenP(i) = (zend(i) + skip(i) - 1)/skip(i)
+         else
+            zstP(i) = zstart(i)/skip(i)
+            if (mod(zstart(i), skip(i)) /= 0) zstP(i) = zstP(i) + 1
+            zenP(i) = zend(i)/skip(i)
+         end if
+         zszP(i) = zenP(i) - zstP(i) + 1
+      end do
+
+      return
+   end subroutine init_coarser_mesh_statP
+
+   ! Copy data from a fine-resolution array to a coarse one for statistic
+   subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
+
+      implicit none
+
+      real(mytype), dimension(:, :, :) :: var_fine
+      real(mytype), dimension(:, :, :) :: var_coarse
+      integer, intent(IN) :: ipencil
+
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer :: i, j, k
+
+      if (ipencil == 1) then
+         allocate (wk(xstS(1):xenS(1), xstS(2):xenS(2), xstS(3):xenS(3)))
+         allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = xstS(3), xenS(3)
+               do j = xstS(2), xenS(2)
+                  do i = xstS(1), xenS(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = xstS(3), xenS(3)
+               do j = xstS(2), xenS(2)
+                  do i = xstS(1), xenS(1)
+                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 2) then
+         allocate (wk(ystS(1):yenS(1), ystS(2):yenS(2), ystS(3):yenS(3)))
+         allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = ystS(3), yenS(3)
+               do j = ystS(2), yenS(2)
+                  do i = ystS(1), yenS(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = ystS(3), yenS(3)
+               do j = ystS(2), yenS(2)
+                  do i = ystS(1), yenS(1)
+                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 3) then
+         allocate (wk(zstS(1):zenS(1), zstS(2):zenS(2), zstS(3):zenS(3)))
+         allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = zstS(3), zenS(3)
+               do j = zstS(2), zenS(2)
+                  do i = zstS(1), zenS(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = zstS(3), zenS(3)
+               do j = zstS(2), zenS(2)
+                  do i = zstS(1), zenS(1)
+                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      end if
+
+      deallocate (wk, wk2)
+
+      return
+   end subroutine fine_to_coarseS
+
+   ! Copy data from a fine-resolution array to a coarse one for visualization
+   subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
+
+      implicit none
+
+      real(mytype), dimension(:, :, :) :: var_fine
+      real(mytype), dimension(:, :, :) :: var_coarse
+      integer, intent(IN) :: ipencil
+
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer :: i, j, k
+
+      if (ipencil == 1) then
+         allocate (wk(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
+         allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = xstV(3), xenV(3)
+               do j = xstV(2), xenV(2)
+                  do i = xstV(1), xenV(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = xstV(3), xenV(3)
+               do j = xstV(2), xenV(2)
+                  do i = xstV(1), xenV(1)
+                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 2) then
+         allocate (wk(ystV(1):yenV(1), ystV(2):yenV(2), ystV(3):yenV(3)))
+         allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = ystV(3), yenV(3)
+               do j = ystV(2), yenV(2)
+                  do i = ystV(1), yenV(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = ystV(3), yenV(3)
+               do j = ystV(2), yenV(2)
+                  do i = ystV(1), yenV(1)
+                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 3) then
+         allocate (wk(zstV(1):zenV(1), zstV(2):zenV(2), zstV(3):zenV(3)))
+         allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = zstV(3), zenV(3)
+               do j = zstV(2), zenV(2)
+                  do i = zstV(1), zenV(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = zstV(3), zenV(3)
+               do j = zstV(2), zenV(2)
+                  do i = zstV(1), zenV(1)
+                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      end if
+
+      deallocate (wk, wk2)
+
+      return
+   end subroutine fine_to_coarseV
+
+   ! Copy data from a fine-resolution array to a coarse one for probe
+   subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
+
+      implicit none
+
+      real(mytype), dimension(:, :, :) :: var_fine
+      real(mytype), dimension(:, :, :) :: var_coarse
+      integer, intent(IN) :: ipencil
+
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer :: i, j, k
+
+      if (ipencil == 1) then
+         allocate (wk(xstP(1):xenP(1), xstP(2):xenP(2), xstP(3):xenP(3)))
+         allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = xstP(3), xenP(3)
+               do j = xstP(2), xenP(2)
+                  do i = xstP(1), xenP(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = xstP(3), xenP(3)
+               do j = xstP(2), xenP(2)
+                  do i = xstP(1), xenP(1)
+                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 2) then
+         allocate (wk(ystP(1):yenP(1), ystP(2):yenP(2), ystP(3):yenP(3)))
+         allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = ystP(3), yenP(3)
+               do j = ystP(2), yenP(2)
+                  do i = ystP(1), yenP(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = ystP(3), yenP(3)
+               do j = ystP(2), yenP(2)
+                  do i = ystP(1), yenP(1)
+                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 3) then
+         allocate (wk(zstP(1):zenP(1), zstP(2):zenP(2), zstP(3):zenP(3)))
+         allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = zstP(3), zenP(3)
+               do j = zstP(2), zenP(2)
+                  do i = zstP(1), zenP(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = zstP(3), zenP(3)
+               do j = zstP(2), zenP(2)
+                  do i = zstP(1), zenP(1)
+                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      end if
+
+      deallocate (wk, wk2)
+
+      return
+   end subroutine fine_to_coarseP
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Find sub-domain information held by current processor
-  !   INPUT: 
-  !     nx, ny, nz - global data dimension
-  !     pdim(3)    - number of processor grid in each dimension, 
-  !                  valid values: 1 - distibute locally; 
-  !                                2 - distribute across p_row; 
-  !                                3 - distribute across p_col
-  !   OUTPUT:
-  !     lstart(3)  - starting index
-  !     lend(3)    - ending index
-  !     lsize(3)   - size of the sub-block (redundant) 
+   ! Find sub-domain information held by current processor
+   !   INPUT:
+   !     nx, ny, nz - global data dimension
+   !     pdim(3)    - number of processor grid in each dimension,
+   !                  valid values: 1 - distibute locally;
+   !                                2 - distribute across p_row;
+   !                                3 - distribute across p_col
+   !   OUTPUT:
+   !     lstart(3)  - starting index
+   !     lend(3)    - ending index
+   !     lsize(3)   - size of the sub-block (redundant)
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine partition(nx, ny, nz, pdim, lstart, lend, lsize)
-
-    implicit none
-
-    integer, intent(IN) :: nx, ny, nz
-    integer, dimension(3), intent(IN) :: pdim
-    integer, dimension(3), intent(OUT) :: lstart, lend, lsize
-
-    integer, allocatable, dimension(:) :: st,en,sz
-    integer :: i, gsize
-
-    do i = 1, 3
-
-       if (i==1) then
-          gsize = nx
-       else if (i==2) then
-          gsize = ny
-       else if (i==3) then
-          gsize = nz
-       end if
-
-       if (pdim(i) == 1) then        ! all local
-          lstart(i) = 1
-          lend(i)   = gsize
-          lsize(i)  = gsize
-       elseif (pdim(i) == 2) then    ! distribute across dims(1)
-          allocate(st(0:dims(1)-1))
-          allocate(en(0:dims(1)-1))
-          allocate(sz(0:dims(1)-1))
-          call distribute(gsize,dims(1),st,en,sz)
-          lstart(i) = st(coord(1))
-          lend(i)   = en(coord(1))
-          lsize(i)  = sz(coord(1))
-          deallocate(st,en,sz)
-       elseif (pdim(i) == 3) then    ! distribute across dims(2)
-          allocate(st(0:dims(2)-1))
-          allocate(en(0:dims(2)-1))
-          allocate(sz(0:dims(2)-1))
-          call distribute(gsize,dims(2),st,en,sz)
-          lstart(i) = st(coord(2))
-          lend(i)   = en(coord(2))
-          lsize(i)  = sz(coord(2))
-          deallocate(st,en,sz)
-       end if
-
-    end do
-    return   
-
-  end subroutine partition
+   subroutine partition(nx, ny, nz, pdim, lstart, lend, lsize)
+
+      implicit none
+
+      integer, intent(IN) :: nx, ny, nz
+      integer, dimension(3), intent(IN) :: pdim
+      integer, dimension(3), intent(OUT) :: lstart, lend, lsize
+
+      integer, allocatable, dimension(:) :: st, en, sz
+      integer :: i, gsize
+
+      do i = 1, 3
+
+         if (i == 1) then
+            gsize = nx
+         else if (i == 2) then
+            gsize = ny
+         else if (i == 3) then
+            gsize = nz
+         end if
+
+         if (pdim(i) == 1) then        ! all local
+            lstart(i) = 1
+            lend(i) = gsize
+            lsize(i) = gsize
+         elseif (pdim(i) == 2) then    ! distribute across dims(1)
+            allocate (st(0:dims(1) - 1))
+            allocate (en(0:dims(1) - 1))
+            allocate (sz(0:dims(1) - 1))
+            call distribute(gsize, dims(1), st, en, sz)
+            lstart(i) = st(coord(1))
+            lend(i) = en(coord(1))
+            lsize(i) = sz(coord(1))
+            deallocate (st, en, sz)
+         elseif (pdim(i) == 3) then    ! distribute across dims(2)
+            allocate (st(0:dims(2) - 1))
+            allocate (en(0:dims(2) - 1))
+            allocate (sz(0:dims(2) - 1))
+            call distribute(gsize, dims(2), st, en, sz)
+            lstart(i) = st(coord(2))
+            lend(i) = en(coord(2))
+            lsize(i) = sz(coord(2))
+            deallocate (st, en, sz)
+         end if
+
+      end do
+      return
+
+   end subroutine partition
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !   - distibutes grid points in one dimension
-  !   - handles uneven distribution properly 
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 
-  subroutine distribute(data1,proc,st,en,sz)
-
-    implicit none
-    ! data1 -- data size in any dimension to be partitioned
-    ! proc  -- number of processors in that dimension
-    ! st    -- array of starting index
-    ! en    -- array of ending index
-    ! sz    -- array of local size  (redundent)
-    integer data1,proc,st(0:proc-1),en(0:proc-1),sz(0:proc-1)
-    integer i,size1,nl,nu
-
-    size1=data1/proc
-    nu = data1 - size1 * proc
-    nl = proc - nu
-    st(0) = 1
-    sz(0) = size1
-    en(0) = size1
-    do i=1,nl-1
-       st(i) = st(i-1) + size1
-       sz(i) = size1
-       en(i) = en(i-1) + size1
-    end do
-    size1 = size1 + 1
-    do i=nl,proc-1
-       st(i) = en(i-1) + 1
-       sz(i) = size1
-       en(i) = en(i-1) + size1
-    end do
-    en(proc-1)= data1 
-    sz(proc-1)= data1-st(proc-1)+1
-
-    return
-  end subroutine distribute
+   !   - distibutes grid points in one dimension
+   !   - handles uneven distribution properly
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   subroutine distribute(data1, proc, st, en, sz)
+
+      implicit none
+      ! data1 -- data size in any dimension to be partitioned
+      ! proc  -- number of processors in that dimension
+      ! st    -- array of starting index
+      ! en    -- array of ending index
+      ! sz    -- array of local size  (redundent)
+      integer data1, proc, st(0:proc - 1), en(0:proc - 1), sz(0:proc - 1)
+      integer i, size1, nl, nu
+
+      size1 = data1/proc
+      nu = data1 - size1*proc
+      nl = proc - nu
+      st(0) = 1
+      sz(0) = size1
+      en(0) = size1
+      do i = 1, nl - 1
+         st(i) = st(i - 1) + size1
+         sz(i) = size1
+         en(i) = en(i - 1) + size1
+      end do
+      size1 = size1 + 1
+      do i = nl, proc - 1
+         st(i) = en(i - 1) + 1
+         sz(i) = size1
+         en(i) = en(i - 1) + size1
+      end do
+      en(proc - 1) = data1
+      sz(proc - 1) = data1 - st(proc - 1) + 1
+
+      return
+   end subroutine distribute
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  Define how each dimension is distributed across processors
-  !    e.g. 17 meshes across 4 processor would be distibuted as (4,4,4,5)
-  !    such global information is required locally at MPI_ALLTOALLV time
+   !  Define how each dimension is distributed across processors
+   !    e.g. 17 meshes across 4 processor would be distibuted as (4,4,4,5)
+   !    such global information is required locally at MPI_ALLTOALLV time
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine get_dist(nx,ny,nz,decomp)
+   subroutine get_dist(nx, ny, nz, decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: nx, ny, nz
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-    integer, allocatable, dimension(:) :: st,en
+      integer, intent(IN) :: nx, ny, nz
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+      integer, allocatable, dimension(:) :: st, en
 
-    allocate(st(0:dims(1)-1))
-    allocate(en(0:dims(1)-1))
-    call distribute(nx,dims(1),st,en,decomp%x1dist)
-    call distribute(ny,dims(1),st,en,decomp%y1dist)
-    deallocate(st,en)
+      allocate (st(0:dims(1) - 1))
+      allocate (en(0:dims(1) - 1))
+      call distribute(nx, dims(1), st, en, decomp%x1dist)
+      call distribute(ny, dims(1), st, en, decomp%y1dist)
+      deallocate (st, en)
 
-    allocate(st(0:dims(2)-1))
-    allocate(en(0:dims(2)-1))
-    call distribute(ny,dims(2),st,en,decomp%y2dist)
-    call distribute(nz,dims(2),st,en,decomp%z2dist)
-    deallocate(st,en)
+      allocate (st(0:dims(2) - 1))
+      allocate (en(0:dims(2) - 1))
+      call distribute(ny, dims(2), st, en, decomp%y2dist)
+      call distribute(nz, dims(2), st, en, decomp%z2dist)
+      deallocate (st, en)
 
-    return
-  end subroutine get_dist
+      return
+   end subroutine get_dist
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Prepare the send / receive buffers for MPI_ALLTOALLV communications
+   ! Prepare the send / receive buffers for MPI_ALLTOALLV communications
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine prepare_buffer(decomp)
-
-    implicit none
-
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-
-    integer :: i
-
-    !LG : AJOUTS "bidons" pour eviter un plantage en -O3 avec gcc9.3
-    !       * la fonction sortait des valeurs 'aleatoires'
-    !         et le calcul plantait dans MPI_ALLTOALLV
-    !       * pas de plantage en O2
-    
-    if (nrank==0) then
-       open(newunit=i, file='temp.dat', form='unformatted')
-       write(i) decomp%x1dist,decomp%y1dist,decomp%y2dist,decomp%z2dist, &
-                decomp%xsz,decomp%ysz,decomp%zsz
-       close(i, status='delete')
-    endif
-
-    ! MPI_ALLTOALLV buffer information
-
-    do i=0, dims(1)-1
-       decomp%x1cnts(i) = decomp%x1dist(i)*decomp%xsz(2)*decomp%xsz(3)
-       decomp%y1cnts(i) = decomp%ysz(1)*decomp%y1dist(i)*decomp%ysz(3)
-       if (i==0) then
-          decomp%x1disp(i) = 0  ! displacement is 0-based index
-          decomp%y1disp(i) = 0
-       else
-          decomp%x1disp(i) = decomp%x1disp(i-1) + decomp%x1cnts(i-1)
-          decomp%y1disp(i) = decomp%y1disp(i-1) + decomp%y1cnts(i-1)
-       end if
-    end do
-
-    do i=0, dims(2)-1
-       decomp%y2cnts(i) = decomp%ysz(1)*decomp%y2dist(i)*decomp%ysz(3)
-       decomp%z2cnts(i) = decomp%zsz(1)*decomp%zsz(2)*decomp%z2dist(i)
-       if (i==0) then
-          decomp%y2disp(i) = 0  ! displacement is 0-based index
-          decomp%z2disp(i) = 0
-       else
-          decomp%y2disp(i) = decomp%y2disp(i-1) + decomp%y2cnts(i-1)
-          decomp%z2disp(i) = decomp%z2disp(i-1) + decomp%z2cnts(i-1)
-       end if
-    end do
-
-    ! MPI_ALLTOALL buffer information
-
-    ! For evenly distributed data, following is an easier implementation.
-    ! But it should be covered by the more general formulation below.
-    !decomp%x1count = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)/dims(1)
-    !decomp%y1count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(1) 
-    !decomp%y2count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(2)
-    !decomp%z2count = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)/dims(2)
-
-    ! For unevenly distributed data, pad smaller messages. Note the 
-    ! last blocks along pencils always get assigned more mesh points
-    ! for X <=> Y transposes
-    decomp%x1count = decomp%x1dist(dims(1)-1) * &
-         decomp%y1dist(dims(1)-1) * decomp%xsz(3)
-    decomp%y1count = decomp%x1count
-    ! for Y <=> Z transposes
-    decomp%y2count = decomp%y2dist(dims(2)-1) * &
-         decomp%z2dist(dims(2)-1) * decomp%zsz(1)
-    decomp%z2count = decomp%y2count
-
-    return
-  end subroutine prepare_buffer
+   subroutine prepare_buffer(decomp)
+
+      implicit none
+
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+
+      integer :: i
+
+      !LG : AJOUTS "bidons" pour eviter un plantage en -O3 avec gcc9.3
+      !       * la fonction sortait des valeurs 'aleatoires'
+      !         et le calcul plantait dans MPI_ALLTOALLV
+      !       * pas de plantage en O2
+
+      if (nrank == 0) then
+         open (newunit=i, file='temp.dat', form='unformatted')
+         write (i) decomp%x1dist, decomp%y1dist, decomp%y2dist, decomp%z2dist, &
+            decomp%xsz, decomp%ysz, decomp%zsz
+         close (i, status='delete')
+      end if
+
+      ! MPI_ALLTOALLV buffer information
+
+      do i = 0, dims(1) - 1
+         decomp%x1cnts(i) = decomp%x1dist(i)*decomp%xsz(2)*decomp%xsz(3)
+         decomp%y1cnts(i) = decomp%ysz(1)*decomp%y1dist(i)*decomp%ysz(3)
+         if (i == 0) then
+            decomp%x1disp(i) = 0  ! displacement is 0-based index
+            decomp%y1disp(i) = 0
+         else
+            decomp%x1disp(i) = decomp%x1disp(i - 1) + decomp%x1cnts(i - 1)
+            decomp%y1disp(i) = decomp%y1disp(i - 1) + decomp%y1cnts(i - 1)
+         end if
+      end do
+
+      do i = 0, dims(2) - 1
+         decomp%y2cnts(i) = decomp%ysz(1)*decomp%y2dist(i)*decomp%ysz(3)
+         decomp%z2cnts(i) = decomp%zsz(1)*decomp%zsz(2)*decomp%z2dist(i)
+         if (i == 0) then
+            decomp%y2disp(i) = 0  ! displacement is 0-based index
+            decomp%z2disp(i) = 0
+         else
+            decomp%y2disp(i) = decomp%y2disp(i - 1) + decomp%y2cnts(i - 1)
+            decomp%z2disp(i) = decomp%z2disp(i - 1) + decomp%z2cnts(i - 1)
+         end if
+      end do
+
+      ! MPI_ALLTOALL buffer information
+
+      ! For evenly distributed data, following is an easier implementation.
+      ! But it should be covered by the more general formulation below.
+      !decomp%x1count = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)/dims(1)
+      !decomp%y1count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(1)
+      !decomp%y2count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(2)
+      !decomp%z2count = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)/dims(2)
+
+      ! For unevenly distributed data, pad smaller messages. Note the
+      ! last blocks along pencils always get assigned more mesh points
+      ! for X <=> Y transposes
+      decomp%x1count = decomp%x1dist(dims(1) - 1)* &
+                       decomp%y1dist(dims(1) - 1)*decomp%xsz(3)
+      decomp%y1count = decomp%x1count
+      ! for Y <=> Z transposes
+      decomp%y2count = decomp%y2dist(dims(2) - 1)* &
+                       decomp%z2dist(dims(2) - 1)*decomp%zsz(1)
+      decomp%z2count = decomp%y2count
+
+      return
+   end subroutine prepare_buffer
 
 #ifdef SHM
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  Generate shared-memory information 
+   !  Generate shared-memory information
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_info_init_shm(decomp)
+   subroutine decomp_info_init_shm(decomp)
 
-    implicit none
+      implicit none
 
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
 
-    ! a copy of old displacement array (will be overwritten by shm code)
-    allocate(decomp%x1disp_o(0:dims(1)-1),decomp%y1disp_o(0:dims(1)-1), &
-         decomp%y2disp_o(0:dims(2)-1),decomp%z2disp_o(0:dims(2)-1))
-    decomp%x1disp_o = decomp%x1disp
-    decomp%y1disp_o = decomp%y1disp
-    decomp%y2disp_o = decomp%y2disp
-    decomp%z2disp_o = decomp%z2disp
+      ! a copy of old displacement array (will be overwritten by shm code)
+      allocate (decomp%x1disp_o(0:dims(1) - 1), decomp%y1disp_o(0:dims(1) - 1), &
+                decomp%y2disp_o(0:dims(2) - 1), decomp%z2disp_o(0:dims(2) - 1))
+      decomp%x1disp_o = decomp%x1disp
+      decomp%y1disp_o = decomp%y1disp
+      decomp%y2disp_o = decomp%y2disp
+      decomp%z2disp_o = decomp%z2disp
 
-    call prepare_shared_buffer(decomp%ROW_INFO,DECOMP_2D_COMM_ROW,decomp)
-    call prepare_shared_buffer(decomp%COL_INFO,DECOMP_2D_COMM_COL,decomp)
-
-    return
-  end subroutine decomp_info_init_shm
+      call prepare_shared_buffer(decomp%ROW_INFO, DECOMP_2D_COMM_ROW, decomp)
+      call prepare_shared_buffer(decomp%COL_INFO, DECOMP_2D_COMM_COL, decomp)
 
+      return
+   end subroutine decomp_info_init_shm
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! For shared-memory implementation, prepare send/recv shared buffer
+   ! For shared-memory implementation, prepare send/recv shared buffer
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine prepare_shared_buffer(C,MPI_COMM,decomp)
-
-    implicit none
-
-    TYPE(SMP_INFO) :: C
-    INTEGER :: MPI_COMM
-    TYPE(DECOMP_INFO) :: decomp
-
-    INTEGER, ALLOCATABLE :: KTBL(:,:),NARY(:,:),KTBLALL(:,:)
-    INTEGER MYSMP, MYCORE, COLOR
-
-    integer :: ierror
-
-    C%MPI_COMM = MPI_COMM
-    CALL MPI_COMM_SIZE(MPI_COMM,C%NCPU,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-    CALL MPI_COMM_RANK(MPI_COMM,C%NODE_ME,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_RANK")
-    C%SMP_COMM  = MPI_COMM_NULL
-    C%CORE_COMM = MPI_COMM_NULL
-    C%SMP_ME= 0
-    C%NCORE = 0
-    C%CORE_ME = 0
-    C%MAXCORE = 0
-    C%NSMP  = 0
-    C%N_SND = 0
-    C%N_RCV = 0
-    C%SND_P = 0
-    C%RCV_P = 0
-    C%SND_P_c = 0
-    C%RCV_P_c = 0
-
-    ! get smp-node map for this communicator and set up smp communicators
-    CALL GET_SMP_MAP(C%MPI_COMM, C%NSMP, MYSMP, &
-         C%NCORE, MYCORE, C%MAXCORE)
-    C%SMP_ME = MYSMP + 1
-    C%CORE_ME = MYCORE + 1
-    ! - set up inter/intra smp-node communicators
-    COLOR = MYCORE
-    IF (COLOR.GT.0) COLOR = MPI_UNDEFINED
-    CALL MPI_Comm_split(C%MPI_COMM, COLOR, MYSMP, C%SMP_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-    CALL MPI_Comm_split(C%MPI_COMM, MYSMP, MYCORE, C%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-    ! - allocate work space
-    ALLOCATE(KTBL(C%MAXCORE,C%NSMP),NARY(C%NCPU,C%NCORE))
-    ALLOCATE(KTBLALL(C%MAXCORE,C%NSMP))
-    ! - set up smp-node/core to node_me lookup table
-    KTBL = 0
-    KTBL(C%CORE_ME,C%SMP_ME) = C%NODE_ME + 1
-    CALL MPI_ALLREDUCE(KTBL,KTBLALL,C%NSMP*C%MAXCORE,MPI_INTEGER, &
-         MPI_SUM,MPI_COMM,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-    KTBL=KTBLALL
-    !  IF (SUM(KTBL) /= C%NCPU*(C%NCPU+1)/2) &
-    !       CALL MPI_ABORT(...
-
-    ! compute offsets in shared SNDBUF and RCVBUF
-    CALL MAPSET_SMPSHM(C, KTBL, NARY, decomp)
-
-    DEALLOCATE(KTBL,NARY)
-
-    return
-  end subroutine prepare_shared_buffer
+   subroutine prepare_shared_buffer(C, MPI_COMM, decomp)
+
+      implicit none
+
+      TYPE(SMP_INFO) :: C
+      INTEGER :: MPI_COMM
+      TYPE(DECOMP_INFO) :: decomp
+
+      INTEGER, ALLOCATABLE :: KTBL(:, :), NARY(:, :), KTBLALL(:, :)
+      INTEGER MYSMP, MYCORE, COLOR
+
+      integer :: ierror
+
+      C%MPI_COMM = MPI_COMM
+      CALL MPI_COMM_SIZE(MPI_COMM, C%NCPU, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+      CALL MPI_COMM_RANK(MPI_COMM, C%NODE_ME, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_RANK")
+      C%SMP_COMM = MPI_COMM_NULL
+      C%CORE_COMM = MPI_COMM_NULL
+      C%SMP_ME = 0
+      C%NCORE = 0
+      C%CORE_ME = 0
+      C%MAXCORE = 0
+      C%NSMP = 0
+      C%N_SND = 0
+      C%N_RCV = 0
+      C%SND_P = 0
+      C%RCV_P = 0
+      C%SND_P_c = 0
+      C%RCV_P_c = 0
+
+      ! get smp-node map for this communicator and set up smp communicators
+      CALL GET_SMP_MAP(C%MPI_COMM, C%NSMP, MYSMP, &
+                       C%NCORE, MYCORE, C%MAXCORE)
+      C%SMP_ME = MYSMP + 1
+      C%CORE_ME = MYCORE + 1
+      ! - set up inter/intra smp-node communicators
+      COLOR = MYCORE
+      IF (COLOR .GT. 0) COLOR = MPI_UNDEFINED
+      CALL MPI_Comm_split(C%MPI_COMM, COLOR, MYSMP, C%SMP_COMM, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+      CALL MPI_Comm_split(C%MPI_COMM, MYSMP, MYCORE, C%CORE_COMM, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+      ! - allocate work space
+      ALLOCATE (KTBL(C%MAXCORE, C%NSMP), NARY(C%NCPU, C%NCORE))
+      ALLOCATE (KTBLALL(C%MAXCORE, C%NSMP))
+      ! - set up smp-node/core to node_me lookup table
+      KTBL = 0
+      KTBL(C%CORE_ME, C%SMP_ME) = C%NODE_ME + 1
+      CALL MPI_ALLREDUCE(KTBL, KTBLALL, C%NSMP*C%MAXCORE, MPI_INTEGER, &
+                         MPI_SUM, MPI_COMM, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+      KTBL = KTBLALL
+      !  IF (SUM(KTBL) /= C%NCPU*(C%NCPU+1)/2) &
+      !       CALL MPI_ABORT(...
+
+      ! compute offsets in shared SNDBUF and RCVBUF
+      CALL MAPSET_SMPSHM(C, KTBL, NARY, decomp)
+
+      DEALLOCATE (KTBL, NARY)
+
+      return
+   end subroutine prepare_shared_buffer
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Use Ian Bush's FreeIPC to generate shared-memory information
-  !  - system independent solution
-  !  - replacing David Tanqueray's implementation in alloc_shm.c
-  !    (old C code renamed to get_smp_map2)
+   ! Use Ian Bush's FreeIPC to generate shared-memory information
+   !  - system independent solution
+   !  - replacing David Tanqueray's implementation in alloc_shm.c
+   !    (old C code renamed to get_smp_map2)
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine get_smp_map(comm, nnodes, my_node, ncores, my_core, maxcor)
-
-    use FIPC_module
+   subroutine get_smp_map(comm, nnodes, my_node, ncores, my_core, maxcor)
 
-    implicit none
+      use FIPC_module
 
-    integer, intent(IN) :: comm
-    integer, intent(OUT) :: nnodes, my_node, ncores, my_core, maxcor
+      implicit none
 
-    integer :: intra_comm, extra_comm
-    integer :: ierror
+      integer, intent(IN) :: comm
+      integer, intent(OUT) :: nnodes, my_node, ncores, my_core, maxcor
 
-    call FIPC_init(comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_init")
+      integer :: intra_comm, extra_comm
+      integer :: ierror
 
-    ! intra_comm: communicator for processes on this shared memory node
-    ! extra_comm: communicator for all rank 0 on each shared memory node
-    call FIPC_ctxt_intra_comm(FIPC_ctxt_world, intra_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_intra_comm")
-    call FIPC_ctxt_extra_comm(FIPC_ctxt_world, extra_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_extra_comm")
+      call FIPC_init(comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_init")
 
-    call MPI_COMM_SIZE(intra_comm,  ncores, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-    call MPI_COMM_RANK(intra_comm, my_core, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+      ! intra_comm: communicator for processes on this shared memory node
+      ! extra_comm: communicator for all rank 0 on each shared memory node
+      call FIPC_ctxt_intra_comm(FIPC_ctxt_world, intra_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_intra_comm")
+      call FIPC_ctxt_extra_comm(FIPC_ctxt_world, extra_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_extra_comm")
 
-    ! only rank 0 on each shared memory node member of extra_comm
-    ! for others extra_comm = MPI_COMM_NULL
-    if (extra_comm /= MPI_COMM_NULL) then
-       call MPI_COMM_SIZE(extra_comm,  nnodes, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-       call MPI_COMM_RANK(extra_comm, my_node, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-    end if
+      call MPI_COMM_SIZE(intra_comm, ncores, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+      call MPI_COMM_RANK(intra_comm, my_core, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
 
-    ! other ranks share the same information as their leaders
-    call MPI_BCAST( nnodes, 1, MPI_INTEGER, 0, intra_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
-    call MPI_BCAST(my_node, 1, MPI_INTEGER, 0, intra_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+      ! only rank 0 on each shared memory node member of extra_comm
+      ! for others extra_comm = MPI_COMM_NULL
+      if (extra_comm /= MPI_COMM_NULL) then
+         call MPI_COMM_SIZE(extra_comm, nnodes, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+         call MPI_COMM_RANK(extra_comm, my_node, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+      end if
 
-    ! maxcor
-    call MPI_ALLREDUCE(ncores, maxcor, 1, MPI_INTEGER, MPI_MAX, &
-         decomp_2d_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+      ! other ranks share the same information as their leaders
+      call MPI_BCAST(nnodes, 1, MPI_INTEGER, 0, intra_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+      call MPI_BCAST(my_node, 1, MPI_INTEGER, 0, intra_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
 
-    call FIPC_finalize(ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_finalize")
+      ! maxcor
+      call MPI_ALLREDUCE(ncores, maxcor, 1, MPI_INTEGER, MPI_MAX, &
+                         decomp_2d_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
 
-    return
+      call FIPC_finalize(ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_finalize")
 
-  end subroutine get_smp_map
+      return
 
+   end subroutine get_smp_map
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Set up smp-node based shared memory maps
+   ! Set up smp-node based shared memory maps
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  SUBROUTINE MAPSET_SMPSHM(C, KTBL, NARY, decomp)
-
-    IMPLICIT NONE
-
-    TYPE (SMP_INFO) C
-    INTEGER KTBL(C%MAXCORE,C%NSMP)
-    INTEGER NARY(C%NCPU,C%NCORE)
-    TYPE (DECOMP_INFO) :: decomp
-
-    INTEGER i, j, k, l, N, PTR, BSIZ, ierror, status, seed
-    character*16 s
-
-    BSIZ = C%N_SND
-
-    ! a - SNDBUF
-    IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
-       ALLOCATE(decomp%x1cnts_s(C%NSMP),decomp%x1disp_s(C%NSMP+1), &
-            stat=status)
-       CALL MPI_Allgather(decomp%x1cnts, C%NCPU, MPI_INTEGER, &
-            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-       PTR = 0
-       DO i=1,C%NSMP
-          decomp%x1disp_s(i) = PTR
-          N = 0
-          DO j=1,C%MAXCORE
-             k = KTBL(j,i)
-             IF (k > 0) then
-                DO l=1,C%NCORE
-                   IF (l == C%CORE_ME) decomp%x1disp_o(k-1) = PTR
-                   N = N + NARY(k,l)
-                   PTR = PTR + NARY(k,l)
-                END DO
-             END IF
-          END DO
-          decomp%x1cnts_s(i) = N
-       END DO
-       decomp%x1disp_s(C%NSMP+1) = PTR
-       IF (PTR > BSIZ) BSIZ = PTR
-
-    ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
-       ALLOCATE(decomp%y2cnts_s(C%NSMP),decomp%y2disp_s(C%NSMP+1), &
-            stat=status)
-       CALL MPI_Allgather(decomp%y2cnts, C%NCPU, MPI_INTEGER, &
-            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-       PTR = 0
-       DO i=1,C%NSMP
-          decomp%y2disp_s(i) = PTR
-          N = 0
-          DO j=1,C%MAXCORE
-             k = KTBL(j,i)
-             IF (k > 0) then
-                DO l=1,C%NCORE
-                   IF (l == C%CORE_ME) decomp%y2disp_o(k-1) = PTR
-                   N = N + NARY(k,l)
-                   PTR = PTR + NARY(k,l)
-                END DO
-             END IF
-          END DO
-          decomp%y2cnts_s(i) = N
-       END DO
-       decomp%y2disp_s(C%NSMP+1) = PTR
-       IF (PTR > BSIZ) BSIZ = PTR
-    END IF
-
-    ! b - RCVBUF
-
-    IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
-       ALLOCATE(decomp%y1cnts_s(C%NSMP),decomp%y1disp_s(C%NSMP+1), &
-            stat=status)
-       CALL MPI_Allgather(decomp%y1cnts, C%NCPU, MPI_INTEGER, &
-            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-       PTR = 0
-       DO i=1,C%NSMP
-          decomp%y1disp_s(i) = PTR
-          N=0
-          DO j=1,C%NCORE
-             DO l=1,C%MAXCORE
-                k = KTBL(l,i)
-                IF (k > 0) then
-                   IF (j == C%CORE_ME) decomp%y1disp_o(k-1) = PTR
-                   N = N + NARY(k,j)
-                   PTR = PTR + NARY(k,j)
-                END IF
-             END DO
-          END DO
-          decomp%y1cnts_s(i) = N
-       END DO
-       decomp%y1disp_s(C%NSMP+1) = PTR
-       IF (PTR > BSIZ) BSIZ = PTR
-
-    ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
-       ALLOCATE(decomp%z2cnts_s(C%NSMP),decomp%z2disp_s(C%NSMP+1), &
-            stat=status)
-       CALL MPI_Allgather(decomp%z2cnts, C%NCPU, MPI_INTEGER, &
-            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-       PTR = 0
-       DO i=1,C%NSMP
-          decomp%z2disp_s(i) = PTR
-          N=0
-          DO j=1,C%NCORE
-             DO l=1,C%MAXCORE
-                k = KTBL(l,i)
-                IF (k > 0) then
-                   IF (j == C%CORE_ME) decomp%z2disp_o(k-1) = PTR
-                   N = N + NARY(k,j)
-                   PTR = PTR + NARY(k,j)
-                END IF
-             END DO
-          END DO
-          decomp%z2cnts_s(i) = N
-       END DO
-       decomp%z2disp_s(C%NSMP+1) = PTR
-       IF (PTR > BSIZ) BSIZ = PTR
-
-    END IF
-
-    ! check buffer size and (re)-allocate buffer space if necessary
-    IF (BSIZ > C%N_SND) then
-       IF (C%SND_P /= 0) CALL DEALLOC_SHM(C%SND_P, C%CORE_COMM)
-       ! make sure each rank has unique keys to get shared memory
-       !IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
-       !   seed = nrank+nproc*0+1 ! has to be non-zero
-       !ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
-       !   seed = nrank+nproc*1+1
-       !END IF
-       status = 1
-       !CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status, &
-       !     seed)
-       CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status)
-       C%N_SND = BSIZ
-
-       IF (C%RCV_P /= 0) CALL DEALLOC_SHM(C%RCV_P, C%CORE_COMM)
-       status = 1
-       CALL ALLOC_SHM(C%RCV_P, BSIZ, real_type, C%CORE_COMM, status)
-       C%N_RCV = BSIZ
-
-       IF (C%SND_P_c /= 0) CALL DEALLOC_SHM(C%SND_P_c, C%CORE_COMM)
-       status = 1
-       CALL ALLOC_SHM(C%SND_P_c, BSIZ, complex_type, C%CORE_COMM, status)
-       C%N_SND = BSIZ
-
-       IF (C%RCV_P_c /= 0) CALL DEALLOC_SHM(C%RCV_P_c, C%CORE_COMM)
-       status = 1
-       CALL ALLOC_SHM(C%RCV_P_c, BSIZ, complex_type, C%CORE_COMM, status)
-       C%N_RCV = BSIZ
-
-
-    END IF
-
-    RETURN
-  END SUBROUTINE MAPSET_SMPSHM
+   SUBROUTINE MAPSET_SMPSHM(C, KTBL, NARY, decomp)
+
+      IMPLICIT NONE
+
+      TYPE(SMP_INFO) C
+      INTEGER KTBL(C%MAXCORE, C%NSMP)
+      INTEGER NARY(C%NCPU, C%NCORE)
+      TYPE(DECOMP_INFO) :: decomp
+
+      INTEGER i, j, k, l, N, PTR, BSIZ, ierror, status, seed
+      character*16 s
+
+      BSIZ = C%N_SND
+
+      ! a - SNDBUF
+      IF (C%MPI_COMM == DECOMP_2D_COMM_COL) THEN
+         ALLOCATE (decomp%x1cnts_s(C%NSMP), decomp%x1disp_s(C%NSMP + 1), &
+                   stat=status)
+         CALL MPI_Allgather(decomp%x1cnts, C%NCPU, MPI_INTEGER, &
+                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+         PTR = 0
+         DO i = 1, C%NSMP
+            decomp%x1disp_s(i) = PTR
+            N = 0
+            DO j = 1, C%MAXCORE
+               k = KTBL(j, i)
+               IF (k > 0) then
+                  DO l = 1, C%NCORE
+                     IF (l == C%CORE_ME) decomp%x1disp_o(k - 1) = PTR
+                     N = N + NARY(k, l)
+                     PTR = PTR + NARY(k, l)
+                  END DO
+               END IF
+            END DO
+            decomp%x1cnts_s(i) = N
+         END DO
+         decomp%x1disp_s(C%NSMP + 1) = PTR
+         IF (PTR > BSIZ) BSIZ = PTR
+
+      ELSE IF (C%MPI_COMM == DECOMP_2D_COMM_ROW) THEN
+         ALLOCATE (decomp%y2cnts_s(C%NSMP), decomp%y2disp_s(C%NSMP + 1), &
+                   stat=status)
+         CALL MPI_Allgather(decomp%y2cnts, C%NCPU, MPI_INTEGER, &
+                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+         PTR = 0
+         DO i = 1, C%NSMP
+            decomp%y2disp_s(i) = PTR
+            N = 0
+            DO j = 1, C%MAXCORE
+               k = KTBL(j, i)
+               IF (k > 0) then
+                  DO l = 1, C%NCORE
+                     IF (l == C%CORE_ME) decomp%y2disp_o(k - 1) = PTR
+                     N = N + NARY(k, l)
+                     PTR = PTR + NARY(k, l)
+                  END DO
+               END IF
+            END DO
+            decomp%y2cnts_s(i) = N
+         END DO
+         decomp%y2disp_s(C%NSMP + 1) = PTR
+         IF (PTR > BSIZ) BSIZ = PTR
+      END IF
+
+      ! b - RCVBUF
+
+      IF (C%MPI_COMM == DECOMP_2D_COMM_COL) THEN
+         ALLOCATE (decomp%y1cnts_s(C%NSMP), decomp%y1disp_s(C%NSMP + 1), &
+                   stat=status)
+         CALL MPI_Allgather(decomp%y1cnts, C%NCPU, MPI_INTEGER, &
+                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+         PTR = 0
+         DO i = 1, C%NSMP
+            decomp%y1disp_s(i) = PTR
+            N = 0
+            DO j = 1, C%NCORE
+               DO l = 1, C%MAXCORE
+                  k = KTBL(l, i)
+                  IF (k > 0) then
+                     IF (j == C%CORE_ME) decomp%y1disp_o(k - 1) = PTR
+                     N = N + NARY(k, j)
+                     PTR = PTR + NARY(k, j)
+                  END IF
+               END DO
+            END DO
+            decomp%y1cnts_s(i) = N
+         END DO
+         decomp%y1disp_s(C%NSMP + 1) = PTR
+         IF (PTR > BSIZ) BSIZ = PTR
+
+      ELSE IF (C%MPI_COMM == DECOMP_2D_COMM_ROW) THEN
+         ALLOCATE (decomp%z2cnts_s(C%NSMP), decomp%z2disp_s(C%NSMP + 1), &
+                   stat=status)
+         CALL MPI_Allgather(decomp%z2cnts, C%NCPU, MPI_INTEGER, &
+                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+         PTR = 0
+         DO i = 1, C%NSMP
+            decomp%z2disp_s(i) = PTR
+            N = 0
+            DO j = 1, C%NCORE
+               DO l = 1, C%MAXCORE
+                  k = KTBL(l, i)
+                  IF (k > 0) then
+                     IF (j == C%CORE_ME) decomp%z2disp_o(k - 1) = PTR
+                     N = N + NARY(k, j)
+                     PTR = PTR + NARY(k, j)
+                  END IF
+               END DO
+            END DO
+            decomp%z2cnts_s(i) = N
+         END DO
+         decomp%z2disp_s(C%NSMP + 1) = PTR
+         IF (PTR > BSIZ) BSIZ = PTR
+
+      END IF
+
+      ! check buffer size and (re)-allocate buffer space if necessary
+      IF (BSIZ > C%N_SND) then
+         IF (C%SND_P /= 0) CALL DEALLOC_SHM(C%SND_P, C%CORE_COMM)
+         ! make sure each rank has unique keys to get shared memory
+         !IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
+         !   seed = nrank+nproc*0+1 ! has to be non-zero
+         !ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
+         !   seed = nrank+nproc*1+1
+         !END IF
+         status = 1
+         !CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status, &
+         !     seed)
+         CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status)
+         C%N_SND = BSIZ
+
+         IF (C%RCV_P /= 0) CALL DEALLOC_SHM(C%RCV_P, C%CORE_COMM)
+         status = 1
+         CALL ALLOC_SHM(C%RCV_P, BSIZ, real_type, C%CORE_COMM, status)
+         C%N_RCV = BSIZ
+
+         IF (C%SND_P_c /= 0) CALL DEALLOC_SHM(C%SND_P_c, C%CORE_COMM)
+         status = 1
+         CALL ALLOC_SHM(C%SND_P_c, BSIZ, complex_type, C%CORE_COMM, status)
+         C%N_SND = BSIZ
+
+         IF (C%RCV_P_c /= 0) CALL DEALLOC_SHM(C%RCV_P_c, C%CORE_COMM)
+         status = 1
+         CALL ALLOC_SHM(C%RCV_P_c, BSIZ, complex_type, C%CORE_COMM, status)
+         C%N_RCV = BSIZ
+
+      END IF
+
+      RETURN
+   END SUBROUTINE MAPSET_SMPSHM
 
 #endif
 
-
 #ifdef OCC
-  ! For non-blocking communication code, progress the comminication stack
-  subroutine transpose_test(handle)
+   ! For non-blocking communication code, progress the comminication stack
+   subroutine transpose_test(handle)
 
-    implicit none
+      implicit none
 
-    integer :: handle, ierror
+      integer :: handle, ierror
 
-    call NBC_TEST(handle,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "NBC_TEST")
+      call NBC_TEST(handle, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "NBC_TEST")
 
-    return
-  end subroutine transpose_test
+      return
+   end subroutine transpose_test
 #endif
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Transposition routines 
+   ! Transposition routines
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "transpose_x_to_y.f90"
 #include "transpose_y_to_z.f90"
@@ -1641,175 +1633,174 @@ end subroutine transpose_test
 #include "transpose_y_to_x.f90"
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Halo cell support
+   ! Halo cell support
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "halo.f90"
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Error handling
+   ! Error handling
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_2d_abort_basic(errorcode, msg)
+   subroutine decomp_2d_abort_basic(errorcode, msg)
 
-    use iso_fortran_env, only : error_unit
+      use iso_fortran_env, only: error_unit
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: errorcode
-    character(len=*), intent(IN) :: msg
+      integer, intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
 
-    integer :: ierror
+      integer :: ierror
 
-    if (nrank==0) then
-       write(*,*) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-       write(*,*) 'ERROR MESSAGE: ' // msg
-       write(error_unit,*) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-       write(error_unit,*) 'ERROR MESSAGE: ' // msg
-    end if
-    call MPI_ABORT(decomp_2d_comm,errorcode,ierror)
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+         write (*, *) 'ERROR MESSAGE: '//msg
+         write (error_unit, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+         write (error_unit, *) 'ERROR MESSAGE: '//msg
+      end if
+      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
 
-  end subroutine decomp_2d_abort_basic
+   end subroutine decomp_2d_abort_basic
 
-  subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
+   subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
 
-    use iso_fortran_env, only : error_unit
+      use iso_fortran_env, only: error_unit
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: errorcode, line
-    character(len=*), intent(IN) :: msg, file
+      integer, intent(IN) :: errorcode, line
+      character(len=*), intent(IN) :: msg, file
 
-    integer :: ierror
+      integer :: ierror
 
-    if (nrank==0) then
-       write(*,*) '2DECOMP&FFT ERROR'
-       write(*,*) '  errorcode:     ', errorcode
-       write(*,*) '  error in file  ' // file
-       write(*,*) '           line  ', line
-       write(*,*) '  error message: ' // msg
-       write(error_unit,*) '2DECOMP&FFT ERROR'
-       write(error_unit,*) '  errorcode:     ', errorcode
-       write(error_unit,*) '  error in file  ' // file
-       write(error_unit,*) '           line  ', line
-       write(error_unit,*) '  error message: ' // msg
-    end if
-    call MPI_ABORT(decomp_2d_comm,errorcode,ierror)
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT ERROR'
+         write (*, *) '  errorcode:     ', errorcode
+         write (*, *) '  error in file  '//file
+         write (*, *) '           line  ', line
+         write (*, *) '  error message: '//msg
+         write (error_unit, *) '2DECOMP&FFT ERROR'
+         write (error_unit, *) '  errorcode:     ', errorcode
+         write (error_unit, *) '  error in file  '//file
+         write (error_unit, *) '           line  ', line
+         write (error_unit, *) '  error message: '//msg
+      end if
+      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
 
-  end subroutine decomp_2d_abort_file_line
+   end subroutine decomp_2d_abort_file_line
 
 #if defined(_GPU) && defined(_NCCL)
-  !
-  ! This is based on the file "nccl.h" in nvhpc 22.1
-  !
-  function _ncclresult_to_integer(errorcode)
-
-    implicit none
-
-    type(ncclresult), intent(IN) :: errorcode
-    integer :: _ncclresult_to_integer
-
-    if (errorcode == ncclSuccess) then
-        _ncclresult_to_integer = 0
-    elseif (errorcode == ncclUnhandledCudaError) then
-        _ncclresult_to_integer = 1
-    elseif (errorcode == ncclSystemError) then
-        _ncclresult_to_integer = 2
-    elseif (errorcode == ncclInternalError) then
-        _ncclresult_to_integer = 3
-    elseif (errorcode == ncclInvalidArgument) then
-        _ncclresult_to_integer = 4
-    elseif (errorcode == ncclInvalidUsage) then
-        _ncclresult_to_integer = 5
-    elseif (errorcode == ncclNumResults) then
-        _ncclresult_to_integer = 6
-    else
-      _ncclresult_to_integer = -1
-      call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
-                             "NCCL error handling needs some update")
-    end if
-
-  end function _ncclresult_to_integer
-
-  !
-  ! Small wrapper for basic NCCL errors
-  !
-  subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
-
-    implicit none
-
-    type(ncclresult), intent(IN) :: errorcode
-    character(len=*), intent(IN) :: msg
-
-    call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
-                         msg // " " // ncclGetErrorString(errorcode))
-
-  end subroutine decomp_2d_abort_nccl_basic
-
-  !
-  ! Small wrapper for NCCL errors
-  !
-  subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
-
-    implicit none
-
-    type(ncclresult), intent(IN) :: errorcode
-    integer, intent(in) :: line
-    character(len=*), intent(IN) :: msg, file
-
-    call decomp_2d_abort(file, &
-                         line, &
-                         _ncclresult_to_integer(errorcode), &
-                         msg // " " // ncclGetErrorString(errorcode))
-
-  end subroutine decomp_2d_abort_nccl_file_line
+   !
+   ! This is based on the file "nccl.h" in nvhpc 22.1
+   !
+   function _ncclresult_to_integer(errorcode)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      integer :: _ncclresult_to_integer
+
+      if (errorcode == ncclSuccess) then
+         _ncclresult_to_integer = 0
+      elseif (errorcode == ncclUnhandledCudaError) then
+         _ncclresult_to_integer = 1
+      elseif (errorcode == ncclSystemError) then
+         _ncclresult_to_integer = 2
+      elseif (errorcode == ncclInternalError) then
+         _ncclresult_to_integer = 3
+      elseif (errorcode == ncclInvalidArgument) then
+         _ncclresult_to_integer = 4
+      elseif (errorcode == ncclInvalidUsage) then
+         _ncclresult_to_integer = 5
+      elseif (errorcode == ncclNumResults) then
+         _ncclresult_to_integer = 6
+      else
+         _ncclresult_to_integer = -1
+         call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
+                                "NCCL error handling needs some update")
+      end if
+
+   end function _ncclresult_to_integer
+
+   !
+   ! Small wrapper for basic NCCL errors
+   !
+   subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
+
+      call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
+                           msg//" "//ncclGetErrorString(errorcode))
+
+   end subroutine decomp_2d_abort_nccl_basic
+
+   !
+   ! Small wrapper for NCCL errors
+   !
+   subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      integer, intent(in) :: line
+      character(len=*), intent(IN) :: msg, file
+
+      call decomp_2d_abort(file, &
+                           line, &
+                           _ncclresult_to_integer(errorcode), &
+                           msg//" "//ncclGetErrorString(errorcode))
+
+   end subroutine decomp_2d_abort_nccl_file_line
 #endif
 
-  subroutine decomp_2d_warning_basic(errorcode, msg)
+   subroutine decomp_2d_warning_basic(errorcode, msg)
 
-    use iso_fortran_env, only : error_unit
+      use iso_fortran_env, only: error_unit
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: errorcode
-    character(len=*), intent(IN) :: msg
+      integer, intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
 
-    if (nrank==0) then
-       write(*,*) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-       write(*,*) 'ERROR MESSAGE: ' // msg
-       write(error_unit,*) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-       write(error_unit,*) 'ERROR MESSAGE: ' // msg
-    end if
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+         write (*, *) 'ERROR MESSAGE: '//msg
+         write (error_unit, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+         write (error_unit, *) 'ERROR MESSAGE: '//msg
+      end if
 
-  end subroutine decomp_2d_warning_basic
+   end subroutine decomp_2d_warning_basic
 
-  subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
+   subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
 
-    use iso_fortran_env, only : error_unit
+      use iso_fortran_env, only: error_unit
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: errorcode, line
-    character(len=*), intent(IN) :: msg, file
+      integer, intent(IN) :: errorcode, line
+      character(len=*), intent(IN) :: msg, file
 
-    if (nrank==0) then
-       write(*,*) '2DECOMP&FFT WARNING'
-       write(*,*) '  errorcode:     ', errorcode
-       write(*,*) '  error in file  ' // file
-       write(*,*) '           line  ', line
-       write(*,*) '  error message: ' // msg
-       write(error_unit,*) '2DECOMP&FFT WARNING'
-       write(error_unit,*) '  errorcode:     ', errorcode
-       write(error_unit,*) '  error in file  ' // file
-       write(error_unit,*) '           line  ', line
-       write(error_unit,*) '  error message: ' // msg
-    end if
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT WARNING'
+         write (*, *) '  errorcode:     ', errorcode
+         write (*, *) '  error in file  '//file
+         write (*, *) '           line  ', line
+         write (*, *) '  error message: '//msg
+         write (error_unit, *) '2DECOMP&FFT WARNING'
+         write (error_unit, *) '  errorcode:     ', errorcode
+         write (error_unit, *) '  error in file  '//file
+         write (error_unit, *) '           line  ', line
+         write (error_unit, *) '  error message: '//msg
+      end if
 
-  end subroutine decomp_2d_warning_file_line
+   end subroutine decomp_2d_warning_file_line
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Utility routines to help allocate 3D arrays
+   ! Utility routines to help allocate 3D arrays
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "alloc.f90"
-  
+
 end module decomp_2d
 
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index df0c4668..e4545543 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -1,7 +1,7 @@
 !!!=======================================================================
 !!! This is part of the 2DECOMP&FFT library
-!!! 
-!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!!!
+!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 !!! decomposition. It also implements a highly scalable distributed
 !!! three-dimensional Fast Fourier Transform (FFT).
 !!!
@@ -19,273 +19,272 @@
   !     all internal data structures initialised properly
   !     library ready to use
   !======================================================================
-  subroutine decomp_2d_init_ref(nx,ny,nz,p_row,p_col,periodic_bc,comm)
+  subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
 
-    use mpi
-    use iso_fortran_env, only : output_unit
+     use mpi
+     use iso_fortran_env, only: output_unit
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: nx,ny,nz
-    integer, intent(INOUT) :: p_row,p_col
-    logical, dimension(3), intent(IN), optional :: periodic_bc
-    integer, intent(in), optional :: comm
+     integer, intent(IN) :: nx, ny, nz
+     integer, intent(INOUT) :: p_row, p_col
+     logical, dimension(3), intent(IN), optional :: periodic_bc
+     integer, intent(in), optional :: comm
 
-    integer :: errorcode, ierror, row, col, iounit
-    logical, dimension(2) :: periodic
+     integer :: errorcode, ierror, row, col, iounit
+     logical, dimension(2) :: periodic
 #if defined(_GPU) && defined(_NCCL)
-    integer :: cuda_stat
-    type(ncclResult) :: nccl_stat
+     integer :: cuda_stat
+     type(ncclResult) :: nccl_stat
 #endif
-    character(len=7) fname ! Sufficient for up to O(1M) ranks
+     character(len=7) fname ! Sufficient for up to O(1M) ranks
 
 #ifdef PROFILER
-    ! Prepare the profiler if it was not already prepared
-    if (decomp_profiler.eq.decomp_profiler_none) call decomp_profiler_prep()
-    ! Start the profiler
-    call decomp_profiler_init()
-    ! Start the timer for decomp_2d_init
-    if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_init")
+     ! Prepare the profiler if it was not already prepared
+     if (decomp_profiler .eq. decomp_profiler_none) call decomp_profiler_prep()
+     ! Start the profiler
+     call decomp_profiler_init()
+     ! Start the timer for decomp_2d_init
+     if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_init")
 #endif
 
-    ! Use the provided MPI communicator if present
-    if (present(comm)) then
-       decomp_2d_comm = comm
-    else
-       decomp_2d_comm = MPI_COMM_WORLD
-    endif
-
-    ! Safety check
-    if (MPI_SUCCESS /= 0) call decomp_2d_abort(__FILE__, &
-                                               __LINE__, &
-                                               MPI_SUCCESS, &
-                                               "MPI error check is broken")
-
-    ! If the external code has not set nrank and nproc
-    if (nrank == -1) then
-       call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                             __LINE__, &
-                                             ierror, &
-                                             "MPI_COMM_RANK")
-    endif
-    if (nproc == -1) then
-       call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                             __LINE__, &
-                                             ierror, &
-                                             "MPI_COMM_SIZE")
-    endif
+     ! Use the provided MPI communicator if present
+     if (present(comm)) then
+        decomp_2d_comm = comm
+     else
+        decomp_2d_comm = MPI_COMM_WORLD
+     end if
+
+     ! Safety check
+     if (MPI_SUCCESS /= 0) call decomp_2d_abort(__FILE__, &
+                                                __LINE__, &
+                                                MPI_SUCCESS, &
+                                                "MPI error check is broken")
+
+     ! If the external code has not set nrank and nproc
+     if (nrank == -1) then
+        call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, &
+                                              __LINE__, &
+                                              ierror, &
+                                              "MPI_COMM_RANK")
+     end if
+     if (nproc == -1) then
+        call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, &
+                                              __LINE__, &
+                                              ierror, &
+                                              "MPI_COMM_SIZE")
+     end if
 #ifdef DEBUG
-    ! Check if a modification of the debug level is needed
-    call decomp_2d_debug()
+     ! Check if a modification of the debug level is needed
+     call decomp_2d_debug()
 #endif
 
-    nx_global = nx
-    ny_global = ny
-    nz_global = nz
-
-    if (present(periodic_bc)) then
-       periodic_x = periodic_bc(1)
-       periodic_y = periodic_bc(2)
-       periodic_z = periodic_bc(3)
-    else
-       periodic_x = .false.
-       periodic_y = .false.
-       periodic_z = .false.
-    end if
-
-    if (p_row==0 .and. p_col==0) then
-       ! determine the best 2D processor grid
-       call best_2d_grid(nproc, row, col)
-       p_row = row
-       p_col = col
-    else
-       if (nproc /= p_row*p_col) then
-          errorcode = 1
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Invalid 2D processor grid - nproc /= p_row*p_col')
-       else
-          row = p_row
-          col = p_col
-       end if
-    end if
-
-    ! Create 2D Catersian topology
-    ! Note that in order to support periodic B.C. in the halo-cell code,
-    ! need to create multiple topology objects: DECOMP_2D_COMM_CART_?,
-    ! corresponding to three pencil orientations. They contain almost
-    ! identical topological information but allow different combinations
-    ! of periodic conditions.
-    dims(1) = row
-    dims(2) = col
-    periodic(1) = periodic_y
-    periodic(2) = periodic_z
-    call MPI_CART_CREATE(decomp_2d_comm,2,dims,periodic, &
-         .false., &  ! do not reorder rank
-         DECOMP_2D_COMM_CART_X, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
-    periodic(1) = periodic_x
-    periodic(2) = periodic_z
-    call MPI_CART_CREATE(decomp_2d_comm,2,dims,periodic, &
-         .false., DECOMP_2D_COMM_CART_Y, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
-    periodic(1) = periodic_x
-    periodic(2) = periodic_y
-    call MPI_CART_CREATE(decomp_2d_comm,2,dims,periodic, &
-         .false., DECOMP_2D_COMM_CART_Z, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
-
-    call MPI_CART_COORDS(DECOMP_2D_COMM_CART_X,nrank,2,coord,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_COORDS")
-
-    ! derive communicators defining sub-groups for ALLTOALL(V)
-    call MPI_CART_SUB(DECOMP_2D_COMM_CART_X,(/.true.,.false./), &
-         DECOMP_2D_COMM_COL,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
-    call MPI_CART_SUB(DECOMP_2D_COMM_CART_X,(/.false.,.true./), &
-         DECOMP_2D_COMM_ROW,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
-
-    ! gather information for halo-cell support code
-    call init_neighbour
-
-    ! actually generate all 2D decomposition information
-    call decomp_info_init(nx,ny,nz,decomp_main)
-
-    ! make a copy of the decomposition information associated with the
-    ! default global size in these global variables so applications can
-    ! use them to create data structures 
-    xstart = decomp_main%xst
-    ystart = decomp_main%yst
-    zstart = decomp_main%zst
-    xend   = decomp_main%xen
-    yend   = decomp_main%yen
-    zend   = decomp_main%zen
-    xsize  = decomp_main%xsz
-    ysize  = decomp_main%ysz
-    zsize  = decomp_main%zsz
-
-    ! determine the number of bytes per float number
-    ! do not use 'mytype' which is compiler dependent
-    ! also possible to use inquire(iolength=...) 
-    call MPI_TYPE_SIZE(real_type,mytype_bytes,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+     nx_global = nx
+     ny_global = ny
+     nz_global = nz
+
+     if (present(periodic_bc)) then
+        periodic_x = periodic_bc(1)
+        periodic_y = periodic_bc(2)
+        periodic_z = periodic_bc(3)
+     else
+        periodic_x = .false.
+        periodic_y = .false.
+        periodic_z = .false.
+     end if
+
+     if (p_row == 0 .and. p_col == 0) then
+        ! determine the best 2D processor grid
+        call best_2d_grid(nproc, row, col)
+        p_row = row
+        p_col = col
+     else
+        if (nproc /= p_row*p_col) then
+           errorcode = 1
+           call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                'Invalid 2D processor grid - nproc /= p_row*p_col')
+        else
+           row = p_row
+           col = p_col
+        end if
+     end if
+
+     ! Create 2D Catersian topology
+     ! Note that in order to support periodic B.C. in the halo-cell code,
+     ! need to create multiple topology objects: DECOMP_2D_COMM_CART_?,
+     ! corresponding to three pencil orientations. They contain almost
+     ! identical topological information but allow different combinations
+     ! of periodic conditions.
+     dims(1) = row
+     dims(2) = col
+     periodic(1) = periodic_y
+     periodic(2) = periodic_z
+     call MPI_CART_CREATE(decomp_2d_comm, 2, dims, periodic, &
+                          .false., &  ! do not reorder rank
+                          DECOMP_2D_COMM_CART_X, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
+     periodic(1) = periodic_x
+     periodic(2) = periodic_z
+     call MPI_CART_CREATE(decomp_2d_comm, 2, dims, periodic, &
+                          .false., DECOMP_2D_COMM_CART_Y, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
+     periodic(1) = periodic_x
+     periodic(2) = periodic_y
+     call MPI_CART_CREATE(decomp_2d_comm, 2, dims, periodic, &
+                          .false., DECOMP_2D_COMM_CART_Z, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
+
+     call MPI_CART_COORDS(DECOMP_2D_COMM_CART_X, nrank, 2, coord, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_COORDS")
+
+     ! derive communicators defining sub-groups for ALLTOALL(V)
+     call MPI_CART_SUB(DECOMP_2D_COMM_CART_X, (/.true., .false./), &
+                       DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
+     call MPI_CART_SUB(DECOMP_2D_COMM_CART_X, (/.false., .true./), &
+                       DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
+
+     ! gather information for halo-cell support code
+     call init_neighbour
+
+     ! actually generate all 2D decomposition information
+     call decomp_info_init(nx, ny, nz, decomp_main)
+
+     ! make a copy of the decomposition information associated with the
+     ! default global size in these global variables so applications can
+     ! use them to create data structures
+     xstart = decomp_main%xst
+     ystart = decomp_main%yst
+     zstart = decomp_main%zst
+     xend = decomp_main%xen
+     yend = decomp_main%yen
+     zend = decomp_main%zen
+     xsize = decomp_main%xsz
+     ysize = decomp_main%ysz
+     zsize = decomp_main%zsz
+
+     ! determine the number of bytes per float number
+     ! do not use 'mytype' which is compiler dependent
+     ! also possible to use inquire(iolength=...)
+     call MPI_TYPE_SIZE(real_type, mytype_bytes, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
 
 #ifdef EVEN
-    if (nrank==0) write(*,*) 'Padded ALLTOALL optimisation on'
-#endif 
+     if (nrank == 0) write (*, *) 'Padded ALLTOALL optimisation on'
+#endif
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    call MPI_COMM_RANK(DECOMP_2D_COMM_COL,col_rank,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-    call MPI_COMM_RANK(DECOMP_2D_COMM_ROW,row_rank,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-    call MPI_COMM_SIZE(DECOMP_2D_COMM_COL,col_comm_size,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-    call MPI_COMM_SIZE(DECOMP_2D_COMM_ROW,row_comm_size,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-
-    allocate(local_to_global_col(col_comm_size), local_to_global_row(row_comm_size))
-    
-    local_to_global_col(:) = 0
-    local_to_global_row(:) = 0
-    local_to_global_col(col_rank+1) = nrank
-    local_to_global_row(row_rank+1) = nrank
-    
-    call mpi_allreduce(MPI_IN_PLACE,local_to_global_col,col_comm_size,MPI_INTEGER,MPI_SUM,DECOMP_2D_COMM_COL,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-    call mpi_allreduce(MPI_IN_PLACE,local_to_global_row,row_comm_size,MPI_INTEGER,MPI_SUM,DECOMP_2D_COMM_ROW,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-
-    if (nrank .eq. 0) then
-       nccl_stat = ncclGetUniqueId(nccl_uid_2decomp)
-    end if
-    call MPI_Bcast(nccl_uid_2decomp, int(sizeof(ncclUniqueId)), MPI_BYTE, 0, decomp_2d_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
-
-    nccl_stat = ncclCommInitRank(nccl_comm_2decomp, nproc, nccl_uid_2decomp, nrank)
-    cuda_stat = cudaStreamCreate(cuda_stream_2decomp)
+     call MPI_COMM_RANK(DECOMP_2D_COMM_COL, col_rank, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+     call MPI_COMM_RANK(DECOMP_2D_COMM_ROW, row_rank, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+     call MPI_COMM_SIZE(DECOMP_2D_COMM_COL, col_comm_size, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+     call MPI_COMM_SIZE(DECOMP_2D_COMM_ROW, row_comm_size, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+
+     allocate (local_to_global_col(col_comm_size), local_to_global_row(row_comm_size))
+
+     local_to_global_col(:) = 0
+     local_to_global_row(:) = 0
+     local_to_global_col(col_rank + 1) = nrank
+     local_to_global_row(row_rank + 1) = nrank
+
+     call mpi_allreduce(MPI_IN_PLACE, local_to_global_col, col_comm_size, MPI_INTEGER, MPI_SUM, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+     call mpi_allreduce(MPI_IN_PLACE, local_to_global_row, row_comm_size, MPI_INTEGER, MPI_SUM, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+
+     if (nrank .eq. 0) then
+        nccl_stat = ncclGetUniqueId(nccl_uid_2decomp)
+     end if
+     call MPI_Bcast(nccl_uid_2decomp, int(sizeof(ncclUniqueId)), MPI_BYTE, 0, decomp_2d_comm, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+
+     nccl_stat = ncclCommInitRank(nccl_comm_2decomp, nproc, nccl_uid_2decomp, nrank)
+     cuda_stat = cudaStreamCreate(cuda_stream_2decomp)
 #endif
 #endif
 
-    !
-    ! Select the IO unit for decomp_2d setup
-    !
-    if (decomp_log == D2D_LOG_TOFILE_FULL) then
-       write(fname, "(I0)") nrank ! Adapt to magnitude of nrank
-       open(newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
-    elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
-       open(newunit=iounit, file="decomp_2d_setup.log", iostat=ierror)
-    else
-       iounit = output_unit
-       ierror = 0
-    endif
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
-    !
-    ! Print the decomp_2d setup
-    !
-    call d2d_listing(iounit)
-    !
-    ! Close the IO unit if it was not stdout
-    !
-    if (iounit /= output_unit) then
-       close(iounit, iostat=ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-    endif
+     !
+     ! Select the IO unit for decomp_2d setup
+     !
+     if (decomp_log == D2D_LOG_TOFILE_FULL) then
+        write (fname, "(I0)") nrank ! Adapt to magnitude of nrank
+        open (newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
+     elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
+        open (newunit=iounit, file="decomp_2d_setup.log", iostat=ierror)
+     else
+        iounit = output_unit
+        ierror = 0
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
+     !
+     ! Print the decomp_2d setup
+     !
+     call d2d_listing(iounit)
+     !
+     ! Close the IO unit if it was not stdout
+     !
+     if (iounit /= output_unit) then
+        close (iounit, iostat=ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+     end if
 
 #ifdef PROFILER
-    ! Stop the timer for decomp_2d_init
-    if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_init")
+     ! Stop the timer for decomp_2d_init
+     if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_init")
 #endif
 
-    return
+     return
   end subroutine decomp_2d_init_ref
 
   !======================================================================
   ! Routine to be called by applications to clean things up
   !======================================================================
   subroutine decomp_2d_finalize_ref
-    
-    implicit none
+
+     implicit none
 #if defined(_GPU) && defined(_NCCL)
-    type(ncclResult) :: nccl_stat
+     type(ncclResult) :: nccl_stat
 #endif
-    
- 
+
 #ifdef PROFILER
-    if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_fin")
+     if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_fin")
 #endif
 
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_ROW)
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_COL)
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_X)
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Y)
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Z)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_ROW)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_COL)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_X)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Y)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Z)
 
-    call decomp_info_finalize(decomp_main)
+     call decomp_info_finalize(decomp_main)
 
-    decomp_buf_size = 0
-    deallocate(work1_r, work2_r, work1_c, work2_c)
+     decomp_buf_size = 0
+     deallocate (work1_r, work2_r, work1_c, work2_c)
 #if defined(_GPU)
-    deallocate(work1_r_d, work2_r_d, work1_c_d, work2_c_d)
+     deallocate (work1_r_d, work2_r_d, work1_c_d, work2_c_d)
 
 #if defined(_NCCL)
-    nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
+     nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
 #endif
 #endif
 
-    nrank = -1
-    nproc = -1
+     nrank = -1
+     nproc = -1
 
 #ifdef PROFILER
-    if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_fin")
-    ! Finalize the profiler
-    call decomp_profiler_fin()
+     if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_fin")
+     ! Finalize the profiler
+     call decomp_profiler_fin()
 #endif
 
-    return
+     return
   end subroutine decomp_2d_finalize_ref
 
 #ifdef DEBUG
@@ -296,33 +295,33 @@ end subroutine decomp_2d_finalize_ref
   !
   subroutine decomp_2d_debug
 
-    implicit none
+     implicit none
 
-    integer :: ierror
-    character(len=4) :: val
-    character(len=*), parameter :: varname = "DECOMP_2D_DEBUG"
+     integer :: ierror
+     character(len=4) :: val
+     character(len=*), parameter :: varname = "DECOMP_2D_DEBUG"
 
-    ! Read the variable
-    call get_environment_variable(varname, value=val, status=ierror)
+     ! Read the variable
+     call get_environment_variable(varname, value=val, status=ierror)
 
-    ! Return if no variable, or no support for env. variable
-    if (ierror >= 1) return
+     ! Return if no variable, or no support for env. variable
+     if (ierror >= 1) return
 
-    ! Minor error, print warning and return
-    if (ierror /= 0) then
-       call decomp_2d_warning(__FILE__, &
-                              __LINE__, &
-                              ierror, &
-                              "Error when reading DECOMP_2D_DEBUG : "//val)
-       return
-    endif
+     ! Minor error, print warning and return
+     if (ierror /= 0) then
+        call decomp_2d_warning(__FILE__, &
+                               __LINE__, &
+                               ierror, &
+                               "Error when reading DECOMP_2D_DEBUG : "//val)
+        return
+     end if
 
-    ! Conversion to integer if possible
-    read(val, '(i4)', iostat=ierror) decomp_debug
-    if (ierror /= 0) call decomp_2d_warning(__FILE__, &
-                                            __LINE__, &
-                                            ierror, &
-                                            "Error when reading DECOMP_2D_DEBUG : "//val)
+     ! Conversion to integer if possible
+     read (val, '(i4)', iostat=ierror) decomp_debug
+     if (ierror /= 0) call decomp_2d_warning(__FILE__, &
+                                             __LINE__, &
+                                             ierror, &
+                                             "Error when reading DECOMP_2D_DEBUG : "//val)
 
   end subroutine decomp_2d_debug
 #endif
@@ -332,34 +331,34 @@ end subroutine decomp_2d_debug
   !---------------------------------------------------------------------
   subroutine best_2d_grid(iproc, best_p_row, best_p_col)
 
-    implicit none
+     implicit none
+
+     integer, intent(IN) :: iproc
+     integer, intent(OUT) :: best_p_row, best_p_col
 
-    integer, intent(IN) :: iproc
-    integer, intent(OUT) :: best_p_row, best_p_col
+     integer, allocatable, dimension(:) :: factors
+     integer :: nfact, i, col, i_best
 
-    integer, allocatable, dimension(:) :: factors
-    integer :: nfact, i, col, i_best
+     if (nrank == 0) write (*, *) 'In auto-tuning mode......'
 
-    if (nrank==0) write(*,*) 'In auto-tuning mode......'
+     i = int(sqrt(real(iproc))) + 10  ! enough space to save all factors
+     allocate (factors(i))
+     call findfactor(iproc, factors, nfact)
+     if (nrank == 0) write (*, *) 'factors: ', (factors(i), i=1, nfact)
 
-    i = int(sqrt(real(iproc))) + 10  ! enough space to save all factors 
-    allocate(factors(i))
-    call findfactor(iproc, factors, nfact)
-    if (nrank==0) write(*,*) 'factors: ', (factors(i), i=1,nfact)
+     i_best = nfact/2 + 1
+     col = factors(i_best)
 
-    i_best=nfact/2+1
-    col=factors(i_best)
+     best_p_col = col
+     best_p_row = iproc/col
+     if (nrank == 0) print *, 'p_row x p_col', best_p_row, best_p_col
+     if ((best_p_col == 1) .and. (nrank == 0)) then
+        print *, 'WARNING: current 2D DECOMP set-up might not work'
+     end if
 
-    best_p_col = col
-    best_p_row=iproc/col
-    if (nrank==0) print *,'p_row x p_col', best_p_row, best_p_col
-    if ((best_p_col==1).and.(nrank==0)) then
-       print *,'WARNING: current 2D DECOMP set-up might not work'
-    endif
-    
-    deallocate(factors)
+     deallocate (factors)
 
-    return
+     return
   end subroutine best_2d_grid
 
   !---------------------------------------------------------------------
@@ -369,36 +368,36 @@ end subroutine best_2d_grid
   !---------------------------------------------------------------------
   subroutine init_neighbour
 
-    integer :: ierror
-
-    ! For X-pencil
-    neighbour(1,1) = MPI_PROC_NULL               ! east
-    neighbour(1,2) = MPI_PROC_NULL               ! west
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 0, 1, &
-         neighbour(1,4), neighbour(1,3), ierror) ! north & south
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 1, 1, &
-         neighbour(1,6), neighbour(1,5), ierror) ! top & bottom
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-
-    ! For Y-pencil
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 0, 1, &
-         neighbour(2,2), neighbour(2,1), ierror) ! east & west
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-    neighbour(2,3) = MPI_PROC_NULL               ! north
-    neighbour(2,4) = MPI_PROC_NULL               ! south
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 1, 1, &
-         neighbour(2,6), neighbour(2,5), ierror) ! top & bottom
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-
-    ! For Z-pencil
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 0, 1, &
-         neighbour(3,2), neighbour(3,1), ierror) ! east & west
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 1, 1, &
-         neighbour(3,4), neighbour(3,3), ierror) ! north & south
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-    neighbour(3,5) = MPI_PROC_NULL               ! top
-    neighbour(3,6) = MPI_PROC_NULL               ! bottom
-    return
+     integer :: ierror
+
+     ! For X-pencil
+     neighbour(1, 1) = MPI_PROC_NULL               ! east
+     neighbour(1, 2) = MPI_PROC_NULL               ! west
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 0, 1, &
+                         neighbour(1, 4), neighbour(1, 3), ierror) ! north & south
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 1, 1, &
+                         neighbour(1, 6), neighbour(1, 5), ierror) ! top & bottom
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+
+     ! For Y-pencil
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 0, 1, &
+                         neighbour(2, 2), neighbour(2, 1), ierror) ! east & west
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+     neighbour(2, 3) = MPI_PROC_NULL               ! north
+     neighbour(2, 4) = MPI_PROC_NULL               ! south
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 1, 1, &
+                         neighbour(2, 6), neighbour(2, 5), ierror) ! top & bottom
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+
+     ! For Z-pencil
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 0, 1, &
+                         neighbour(3, 2), neighbour(3, 1), ierror) ! east & west
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 1, 1, &
+                         neighbour(3, 4), neighbour(3, 3), ierror) ! north & south
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+     neighbour(3, 5) = MPI_PROC_NULL               ! top
+     neighbour(3, 6) = MPI_PROC_NULL               ! bottom
+     return
   end subroutine init_neighbour
diff --git a/src/factor.f90 b/src/factor.f90
index 73d96e15..5f7e530a 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -1,7 +1,7 @@
 !!!=======================================================================
 !!! This is part of the 2DECOMP&FFT library
-!!! 
-!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!!!
+!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 !!! decomposition. It also implements a highly scalable distributed
 !!! three-dimensional Fast Fourier Transform (FFT).
 !!!
@@ -13,80 +13,80 @@
 !!! A few utility routines to find factors of integer numbers
 module factor
 
-  implicit none
+   implicit none
+
+   private
 
-  private
+   public :: findfactor
+   public :: primefactors
 
-  public :: findfactor
-  public :: primefactors
-  
 contains
-  
-  subroutine findfactor(num, factors, nfact)
-    
-    implicit none
-
-    integer, intent(IN) :: num
-    integer, intent(OUT), dimension(*) :: factors
-    integer, intent(OUT) :: nfact
-    integer :: i, m
-
-    ! find the factors <= sqrt(num)
-    m = int(sqrt(real(num)))
-    nfact = 1
-    do i=1,m
-       if (num/i*i == num) then
-          factors(nfact) = i
-          nfact = nfact + 1
-       end if
-    end do
-    nfact = nfact - 1
-
-    ! derive those > sqrt(num)
-    if (factors(nfact)**2/=num) then
-       do i=nfact+1, 2*nfact
-          factors(i) = num / factors(2*nfact-i+1)
-       end do
-       nfact = nfact * 2
-    else
-       do i=nfact+1, 2*nfact-1
-          factors(i) = num / factors(2*nfact-i)
-       end do
-       nfact = nfact * 2 - 1
-    endif
-       
-    return
-
-  end subroutine findfactor
-
-  subroutine primefactors(num, factors, nfact)
-
-    implicit none
-  
-    integer, intent(IN) :: num
-    integer, intent(OUT), dimension(*) :: factors
-    integer, intent(INOUT) :: nfact
-
-    integer :: i, n
-    
-    i = 2  
-    nfact = 1
-    n = num 
-    do
-       if (mod(n,i) == 0) then
-          factors(nfact) = i
-          nfact = nfact + 1
-          n = n / i
-       else
-          i = i + 1
-       end if
-       if (n == 1) then
-          nfact = nfact - 1
-          exit
-       end if
-    end do
-    
-    return
-
-  end subroutine primefactors
+
+   subroutine findfactor(num, factors, nfact)
+
+      implicit none
+
+      integer, intent(IN) :: num
+      integer, intent(OUT), dimension(*) :: factors
+      integer, intent(OUT) :: nfact
+      integer :: i, m
+
+      ! find the factors <= sqrt(num)
+      m = int(sqrt(real(num)))
+      nfact = 1
+      do i = 1, m
+         if (num/i*i == num) then
+            factors(nfact) = i
+            nfact = nfact + 1
+         end if
+      end do
+      nfact = nfact - 1
+
+      ! derive those > sqrt(num)
+      if (factors(nfact)**2 /= num) then
+         do i = nfact + 1, 2*nfact
+            factors(i) = num/factors(2*nfact - i + 1)
+         end do
+         nfact = nfact*2
+      else
+         do i = nfact + 1, 2*nfact - 1
+            factors(i) = num/factors(2*nfact - i)
+         end do
+         nfact = nfact*2 - 1
+      end if
+
+      return
+
+   end subroutine findfactor
+
+   subroutine primefactors(num, factors, nfact)
+
+      implicit none
+
+      integer, intent(IN) :: num
+      integer, intent(OUT), dimension(*) :: factors
+      integer, intent(INOUT) :: nfact
+
+      integer :: i, n
+
+      i = 2
+      nfact = 1
+      n = num
+      do
+         if (mod(n, i) == 0) then
+            factors(nfact) = i
+            nfact = nfact + 1
+            n = n/i
+         else
+            i = i + 1
+         end if
+         if (n == 1) then
+            nfact = nfact - 1
+            exit
+         end if
+      end do
+
+      return
+
+   end subroutine primefactors
 end module
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 8c17df27..e0cd9cac 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -16,12 +16,12 @@
 
 ! Physical space data can be stored in either X-pencil or Z-pencil
 integer, parameter, public :: PHYSICAL_IN_X = 1
-integer, parameter, public :: PHYSICAL_IN_Z = 3 
+integer, parameter, public :: PHYSICAL_IN_Z = 3
 
 integer, save :: format                 ! input X-pencil or Z-pencil
 
 ! The libary can only be initialised once
-logical, save :: initialised = .false. 
+logical, save :: initialised = .false.
 
 ! Global size of the FFT
 integer, save :: nx_fft, ny_fft, nz_fft
@@ -31,85 +31,84 @@
 integer, save, dimension(2) :: dims
 
 ! Decomposition objects
-TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
+TYPE(DECOMP_INFO), pointer, save :: ph => null()  ! physical space
 TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
 
 ! Workspace to store the intermediate Y-pencil data
 ! *** TODO: investigate how to use only one workspace array
-complex(mytype), allocatable, dimension(:,:,:) :: wk2_c2c, wk2_r2c
-complex(mytype), allocatable, dimension(:,:,:) :: wk13
+complex(mytype), allocatable, dimension(:, :, :) :: wk2_c2c, wk2_r2c
+complex(mytype), allocatable, dimension(:, :, :) :: wk13
 
 public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
-decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
+          decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+          decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
 
 ! Declare generic interfaces to handle different inputs
 
 interface decomp_2d_fft_init
-module procedure fft_init_noarg
-module procedure fft_init_arg
-module procedure fft_init_general
+   module procedure fft_init_noarg
+   module procedure fft_init_arg
+   module procedure fft_init_general
 end interface
 
 interface decomp_2d_fft_3d
-module procedure fft_3d_c2c
-module procedure fft_3d_r2c
-module procedure fft_3d_c2r
+   module procedure fft_3d_c2c
+   module procedure fft_3d_r2c
+   module procedure fft_3d_c2r
 end interface
 
 contains
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Initialise the FFT module
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_init_noarg
 
-implicit none
+   implicit none
 
-call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
+   call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
 
-return
+   return
 end subroutine fft_init_noarg
 
 subroutine fft_init_arg(pencil)     ! allow to handle Z-pencil input
 
-implicit none
+   implicit none
 
-integer, intent(IN) :: pencil
+   integer, intent(IN) :: pencil
 
-call fft_init_general(pencil, nx_global, ny_global, nz_global)
+   call fft_init_general(pencil, nx_global, ny_global, nz_global)
 
-return
+   return
 end subroutine fft_init_arg
 
 ! Initialise the FFT library to perform arbitrary size transforms
 subroutine fft_init_general(pencil, nx, ny, nz)
 
-implicit none
+   implicit none
 
-integer, intent(IN) :: pencil
-integer, intent(IN) :: nx, ny, nz
+   integer, intent(IN) :: pencil
+   integer, intent(IN) :: nx, ny, nz
 
-integer :: status, errorcode
+   integer :: status, errorcode
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
 #endif
 
-if (initialised) then
-errorcode = 4
-call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-'FFT library should only be initialised once')
-end if
+   if (initialised) then
+      errorcode = 4
+      call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                           'FFT library should only be initialised once')
+   end if
 
-format = pencil
-nx_fft = nx
-ny_fft = ny
-nz_fft = nz
+   format = pencil
+   nx_fft = nx
+   ny_fft = ny
+   nz_fft = nz
 
 ! determine the processor grid in use
-dims = get_decomp_dims()
+   dims = get_decomp_dims()
 
 ! for c2r/r2c interface:
 ! if in physical space, a real array is of size: nx*ny*nz
@@ -117,98 +116,96 @@ subroutine fft_init_general(pencil, nx, ny, nz)
 !         (nx/2+1)*ny*nz, if PHYSICAL_IN_X
 !      or nx*ny*(nz/2+1), if PHYSICAL_IN_Z
 
-if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
-ph=>decomp_main
-else
-if (.not.associated(ph)) allocate(ph)
-call decomp_info_init(nx, ny, nz, ph)
-endif
-if (format==PHYSICAL_IN_X) then
-call decomp_info_init(nx/2+1, ny, nz, sp)
-else if (format==PHYSICAL_IN_Z) then
-call decomp_info_init(nx, ny, nz/2+1, sp)
-end if
-
-allocate(wk2_c2c(ph%ysz(1),ph%ysz(2),ph%ysz(3)), STAT=status)
-allocate(wk2_r2c(sp%ysz(1),sp%ysz(2),sp%ysz(3)), STAT=status)
-if (format==PHYSICAL_IN_X) then
-allocate(wk13(sp%xsz(1),sp%xsz(2),sp%xsz(3)), STAT=status)
-else if (format==PHYSICAL_IN_Z) then
-allocate(wk13(sp%zsz(1),sp%zsz(2),sp%zsz(3)), STAT=status)
-end if
-if (status /= 0) then
-errorcode = 3
-call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-'Out of memory when initialising FFT')
-end if
-
-call init_fft_engine
-
-initialised = .true.
+   if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
+      ph => decomp_main
+   else
+      if (.not. associated(ph)) allocate (ph)
+      call decomp_info_init(nx, ny, nz, ph)
+   end if
+   if (format == PHYSICAL_IN_X) then
+      call decomp_info_init(nx/2 + 1, ny, nz, sp)
+   else if (format == PHYSICAL_IN_Z) then
+      call decomp_info_init(nx, ny, nz/2 + 1, sp)
+   end if
+
+   allocate (wk2_c2c(ph%ysz(1), ph%ysz(2), ph%ysz(3)), STAT=status)
+   allocate (wk2_r2c(sp%ysz(1), sp%ysz(2), sp%ysz(3)), STAT=status)
+   if (format == PHYSICAL_IN_X) then
+      allocate (wk13(sp%xsz(1), sp%xsz(2), sp%xsz(3)), STAT=status)
+   else if (format == PHYSICAL_IN_Z) then
+      allocate (wk13(sp%zsz(1), sp%zsz(2), sp%zsz(3)), STAT=status)
+   end if
+   if (status /= 0) then
+      errorcode = 3
+      call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                           'Out of memory when initialising FFT')
+   end if
+
+   call init_fft_engine
+
+   initialised = .true.
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
 #endif
 
-return
+   return
 end subroutine fft_init_general
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Final clean up
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine decomp_2d_fft_finalize
 
-implicit none
+   implicit none
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
-call decomp_info_finalize(ph)
-deallocate(ph)
-endif
-nullify(ph)
-call decomp_info_finalize(sp)
+   if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
+      call decomp_info_finalize(ph)
+      deallocate (ph)
+   end if
+   nullify (ph)
+   call decomp_info_finalize(sp)
 
-if (allocated(wk2_c2c)) deallocate(wk2_c2c)
-if (allocated(wk2_r2c)) deallocate(wk2_r2c)
-if (allocated(wk13)) deallocate(wk13)
+   if (allocated(wk2_c2c)) deallocate (wk2_c2c)
+   if (allocated(wk2_r2c)) deallocate (wk2_r2c)
+   if (allocated(wk13)) deallocate (wk13)
 
-call finalize_fft_engine
+   call finalize_fft_engine
 
-initialised = .false.
+   initialised = .false.
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
 #endif
 
-return
+   return
 end subroutine decomp_2d_fft_finalize
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-! Return the size, starting/ending index of the distributed array 
+! Return the size, starting/ending index of the distributed array
 !  whose global size is (nx/2+1)*ny*nz, for defining data structures
 !  in r2c and c2r interfaces
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine decomp_2d_fft_get_size(istart, iend, isize)
 
-implicit none
-integer, dimension(3), intent(OUT) :: istart, iend, isize
+   implicit none
+   integer, dimension(3), intent(OUT) :: istart, iend, isize
 
-if (format==PHYSICAL_IN_X) then
-istart = sp%zst
-iend   = sp%zen
-isize  = sp%zsz
-else if (format==PHYSICAL_IN_Z) then
-istart = sp%xst
-iend   = sp%xen
-isize  = sp%xsz
-end if
+   if (format == PHYSICAL_IN_X) then
+      istart = sp%zst
+      iend = sp%zen
+      isize = sp%zsz
+   else if (format == PHYSICAL_IN_Z) then
+      istart = sp%xst
+      iend = sp%xen
+      isize = sp%xsz
+   end if
 
-return
+   return
 end subroutine decomp_2d_fft_get_size
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -218,15 +215,15 @@ end subroutine decomp_2d_fft_get_size
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 function decomp_2d_fft_get_ph()
 
-implicit none
+   implicit none
 
-type(decomp_info), pointer :: decomp_2d_fft_get_ph
+   type(decomp_info), pointer :: decomp_2d_fft_get_ph
 
-if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
-                                              __LINE__, &
-                                             -1, &
-                                             'FFT library must be initialised first')
-decomp_2d_fft_get_ph => ph
+   if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  -1, &
+                                                  'FFT library must be initialised first')
+   decomp_2d_fft_get_ph => ph
 
 end function decomp_2d_fft_get_ph
 
@@ -237,14 +234,14 @@ end function decomp_2d_fft_get_ph
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 function decomp_2d_fft_get_sp()
 
-implicit none
+   implicit none
 
-type(decomp_info), pointer :: decomp_2d_fft_get_sp
+   type(decomp_info), pointer :: decomp_2d_fft_get_sp
 
-if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
-                                              __LINE__, &
-                                             -1, &
-                                             'FFT library must be initialised first')
-decomp_2d_fft_get_sp => sp
+   if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  -1, &
+                                                  'FFT library must be initialised first')
+   decomp_2d_fft_get_sp => sp
 
 end function decomp_2d_fft_get_sp
diff --git a/src/fft_common_3d.f90 b/src/fft_common_3d.f90
index 3c0abc1b..eb809b37 100644
--- a/src/fft_common_3d.f90
+++ b/src/fft_common_3d.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,290 +10,288 @@
 !=======================================================================
 
 ! This file contains 3D c2c/r2c/c2r transform subroutines which are
-! identical for several FFT engines 
+! identical for several FFT engines
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_3d_c2c(in, out, isign)
 
-implicit none
+   implicit none
 
-complex(mytype), dimension(:,:,:), intent(INOUT) :: in
-complex(mytype), dimension(:,:,:), intent(OUT) :: out
-integer, intent(IN) :: isign
+   complex(mytype), dimension(:, :, :), intent(INOUT) :: in
+   complex(mytype), dimension(:, :, :), intent(OUT) :: out
+   integer, intent(IN) :: isign
 
-integer :: i, j, k
+   integer :: i, j, k
 
 #ifndef OVERWRITE
-complex(mytype), allocatable, dimension(:,:,:) :: wk1
+   complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
+   if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+       format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
 
 ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-call c2c_1m_x(in,isign,ph)
+      call c2c_1m_x(in, isign, ph)
 #else
-allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-do concurrent (k=1:ph%xsz(3), j=1:ph%xsz(2), i=1:ph%xsz(1))
-wk1(i,j,k) = in(i,j,k)
-end do
-call c2c_1m_x(wk1,isign,ph)
+      allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+      do concurrent(k=1:ph%xsz(3), j=1:ph%xsz(2), i=1:ph%xsz(1))
+         wk1(i, j, k) = in(i, j, k)
+      end do
+      call c2c_1m_x(wk1, isign, ph)
 #endif
 
 ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-if (dims(1)>1) then
+      if (dims(1) > 1) then
 #ifdef OVERWRITE
-call transpose_x_to_y(in,wk2_c2c,ph)
+         call transpose_x_to_y(in, wk2_c2c, ph)
 #else
-call transpose_x_to_y(wk1,wk2_c2c,ph)
+         call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-call c2c_1m_y(wk2_c2c,isign,ph)
-else
+         call c2c_1m_y(wk2_c2c, isign, ph)
+      else
 #ifdef OVERWRITE
-call c2c_1m_y(in,isign,ph)
+         call c2c_1m_y(in, isign, ph)
 #else
-call c2c_1m_y(wk1,isign,ph)
+         call c2c_1m_y(wk1, isign, ph)
 #endif
-end if
+      end if
 
 ! ===== Swap Y --> Z; 1D FFTs in Z =====
-if (dims(1)>1) then
-call transpose_y_to_z(wk2_c2c,out,ph)
-else
+      if (dims(1) > 1) then
+         call transpose_y_to_z(wk2_c2c, out, ph)
+      else
 #ifdef OVERWRITE
-call transpose_y_to_z(in,out,ph)
+         call transpose_y_to_z(in, out, ph)
 #else
-call transpose_y_to_z(wk1,out,ph)
+         call transpose_y_to_z(wk1, out, ph)
 #endif
-end if
-call c2c_1m_z(out,isign,ph)
+      end if
+      call c2c_1m_z(out, isign, ph)
 
-else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-.OR. & 
-format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
+   else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+            .OR. &
+            format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
 
 ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-call c2c_1m_z(in,isign,ph)
+      call c2c_1m_z(in, isign, ph)
 #else
-allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-do concurrent (k=1:ph%zsz(3), j=1:ph%zsz(2), i=1:ph%zsz(1))
-wk1(i,j,k) = in(i,j,k)
-end do
-call c2c_1m_z(wk1,isign,ph)
+      allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+      do concurrent(k=1:ph%zsz(3), j=1:ph%zsz(2), i=1:ph%zsz(1))
+         wk1(i, j, k) = in(i, j, k)
+      end do
+      call c2c_1m_z(wk1, isign, ph)
 #endif
 
 ! ===== Swap Z --> Y; 1D FFTs in Y =====
-if (dims(1)>1) then
+      if (dims(1) > 1) then
 #ifdef OVERWRITE
-call transpose_z_to_y(in,wk2_c2c,ph)
+         call transpose_z_to_y(in, wk2_c2c, ph)
 #else
-call transpose_z_to_y(wk1,wk2_c2c,ph)
+         call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-call c2c_1m_y(wk2_c2c,isign,ph)
-else  ! out==wk2_c2c if 1D decomposition
+         call c2c_1m_y(wk2_c2c, isign, ph)
+      else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-call transpose_z_to_y(in,out,ph)
+         call transpose_z_to_y(in, out, ph)
 #else
-call transpose_z_to_y(wk1,out,ph)
+         call transpose_z_to_y(wk1, out, ph)
 #endif
-call c2c_1m_y(out,isign,ph)
-end if
+         call c2c_1m_y(out, isign, ph)
+      end if
 
 ! ===== Swap Y --> X; 1D FFTs in X =====
-if (dims(1)>1) then
-call transpose_y_to_x(wk2_c2c,out,ph)
-end if
-call c2c_1m_x(out,isign,ph)
+      if (dims(1) > 1) then
+         call transpose_y_to_x(wk2_c2c, out, ph)
+      end if
+      call c2c_1m_x(out, isign, ph)
 
-end if
+   end if
 
 #ifndef OVERWRITE
 ! Free memory
-if (allocated(wk1)) deallocate(wk1)
+   if (allocated(wk1)) deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-return
+   return
 end subroutine fft_3d_c2c
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_3d_r2c(in_r, out_c)
 
-implicit none
+   implicit none
 
-real(mytype), dimension(:,:,:), intent(IN) :: in_r
-complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
+   real(mytype), dimension(:, :, :), intent(IN) :: in_r
+   complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-if (format==PHYSICAL_IN_X) then
+   if (format == PHYSICAL_IN_X) then
 
 ! ===== 1D FFTs in X =====
-call r2c_1m_x(in_r,wk13)
+      call r2c_1m_x(in_r, wk13)
 
 ! ===== Swap X --> Y; 1D FFTs in Y =====
-if (dims(1)>1) then
-call transpose_x_to_y(wk13,wk2_r2c,sp)
-call c2c_1m_y(wk2_r2c,-1,sp)
-else
-call c2c_1m_y(wk13,-1,sp)
-end if
+      if (dims(1) > 1) then
+         call transpose_x_to_y(wk13, wk2_r2c, sp)
+         call c2c_1m_y(wk2_r2c, -1, sp)
+      else
+         call c2c_1m_y(wk13, -1, sp)
+      end if
 
 ! ===== Swap Y --> Z; 1D FFTs in Z =====
-if (dims(1)>1) then
-call transpose_y_to_z(wk2_r2c,out_c,sp)
-else
-call transpose_y_to_z(wk13,out_c,sp)
-end if
-call c2c_1m_z(out_c,-1,sp)
+      if (dims(1) > 1) then
+         call transpose_y_to_z(wk2_r2c, out_c, sp)
+      else
+         call transpose_y_to_z(wk13, out_c, sp)
+      end if
+      call c2c_1m_z(out_c, -1, sp)
 
-else if (format==PHYSICAL_IN_Z) then
+   else if (format == PHYSICAL_IN_Z) then
 
 ! ===== 1D FFTs in Z =====
-call r2c_1m_z(in_r,wk13)
+      call r2c_1m_z(in_r, wk13)
 
 ! ===== Swap Z --> Y; 1D FFTs in Y =====
-if (dims(1)>1) then
-call transpose_z_to_y(wk13,wk2_r2c,sp)
-call c2c_1m_y(wk2_r2c,-1,sp)
-else  ! out_c==wk2_r2c if 1D decomposition
-call transpose_z_to_y(wk13,out_c,sp)
-call c2c_1m_y(out_c,-1,sp)
-end if
+      if (dims(1) > 1) then
+         call transpose_z_to_y(wk13, wk2_r2c, sp)
+         call c2c_1m_y(wk2_r2c, -1, sp)
+      else  ! out_c==wk2_r2c if 1D decomposition
+         call transpose_z_to_y(wk13, out_c, sp)
+         call c2c_1m_y(out_c, -1, sp)
+      end if
 
 ! ===== Swap Y --> X; 1D FFTs in X =====
-if (dims(1)>1) then
-call transpose_y_to_x(wk2_r2c,out_c,sp)
-end if
-call c2c_1m_x(out_c,-1,sp)
+      if (dims(1) > 1) then
+         call transpose_y_to_x(wk2_r2c, out_c, sp)
+      end if
+      call c2c_1m_x(out_c, -1, sp)
 
-end if
+   end if
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-return
+   return
 end subroutine fft_3d_r2c
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_3d_c2r(in_c, out_r)
 
-implicit none
+   implicit none
 
-complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
-real(mytype), dimension(:,:,:), intent(OUT) :: out_r
+   complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
+   real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
-integer :: i, j, k
+   integer :: i, j, k
 
 #ifndef OVERWRITE
-complex(mytype), allocatable, dimension(:,:,:) :: wk1
+   complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-if (format==PHYSICAL_IN_X) then
+   if (format == PHYSICAL_IN_X) then
 
 ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-call c2c_1m_z(in_c,1,sp)       
+      call c2c_1m_z(in_c, 1, sp)
 #else
-allocate(wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-do concurrent (k=1:sp%zsz(3), j=1:sp%zsz(2), i=1:sp%zsz(1))
-wk1(i,j,k) = in_c(i,j,k)
-end do
-call c2c_1m_z(wk1,1,sp)
+      allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+      do concurrent(k=1:sp%zsz(3), j=1:sp%zsz(2), i=1:sp%zsz(1))
+         wk1(i, j, k) = in_c(i, j, k)
+      end do
+      call c2c_1m_z(wk1, 1, sp)
 #endif
 
 ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-call transpose_z_to_y(in_c,wk2_r2c,sp)
+      call transpose_z_to_y(in_c, wk2_r2c, sp)
 #else
-call transpose_z_to_y(wk1,wk2_r2c,sp)
+      call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-call c2c_1m_y(wk2_r2c,1,sp)
+      call c2c_1m_y(wk2_r2c, 1, sp)
 
 ! ===== Swap Y --> X; 1D FFTs in X =====
-if (dims(1)>1) then
-call transpose_y_to_x(wk2_r2c,wk13,sp)
-call c2r_1m_x(wk13,out_r)
-else
-call c2r_1m_x(wk2_r2c,out_r)
-end if
+      if (dims(1) > 1) then
+         call transpose_y_to_x(wk2_r2c, wk13, sp)
+         call c2r_1m_x(wk13, out_r)
+      else
+         call c2r_1m_x(wk2_r2c, out_r)
+      end if
 
-else if (format==PHYSICAL_IN_Z) then
+   else if (format == PHYSICAL_IN_Z) then
 
 ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-call c2c_1m_x(in_c,1,sp)
+      call c2c_1m_x(in_c, 1, sp)
 #else
-allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-do concurrent (k=1:sp%xsz(3), j=1:sp%xsz(2), i=1:sp%xsz(1))
-wk1(i,j,k) = in_c(i,j,k)
-end do
-call c2c_1m_x(wk1,1,sp)
+      allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+      do concurrent(k=1:sp%xsz(3), j=1:sp%xsz(2), i=1:sp%xsz(1))
+         wk1(i, j, k) = in_c(i, j, k)
+      end do
+      call c2c_1m_x(wk1, 1, sp)
 #endif
 
 ! ===== Swap X --> Y; 1D FFTs in Y =====
-if (dims(1)>1) then
+      if (dims(1) > 1) then
 #ifdef OVERWRITE
-call transpose_x_to_y(in_c,wk2_r2c,sp)
+         call transpose_x_to_y(in_c, wk2_r2c, sp)
 #else
-call transpose_x_to_y(wk1,wk2_r2c,sp)
+         call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-call c2c_1m_y(wk2_r2c,1,sp)
-else  ! in_c==wk2_r2c if 1D decomposition
+         call c2c_1m_y(wk2_r2c, 1, sp)
+      else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-call c2c_1m_y(in_c,1,sp)
+         call c2c_1m_y(in_c, 1, sp)
 #else
-call c2c_1m_y(wk1,1,sp)
+         call c2c_1m_y(wk1, 1, sp)
 #endif
-end if
+      end if
 
 ! ===== Swap Y --> Z; 1D FFTs in Z =====
-if (dims(1)>1) then
-call transpose_y_to_z(wk2_r2c,wk13,sp)
-else
+      if (dims(1) > 1) then
+         call transpose_y_to_z(wk2_r2c, wk13, sp)
+      else
 #ifdef OVERWRITE
-call transpose_y_to_z(in_c,wk13,sp)
+         call transpose_y_to_z(in_c, wk13, sp)
 #else
-call transpose_y_to_z(wk1,wk13,sp)
+         call transpose_y_to_z(wk1, wk13, sp)
 #endif
-end if
-call c2r_1m_z(wk13,out_r)
+      end if
+      call c2r_1m_z(wk13, out_r)
 
-end if
+   end if
 
 #ifndef OVERWRITE
 ! Free memory
-if (allocated(wk1)) deallocate(wk1)
+   if (allocated(wk1)) deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-return
+   return
 end subroutine fft_3d_c2r
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 0622f958..5f7575df 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,1032 +13,1022 @@
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use iso_c_binding
-  use cudafor
-  use cufft
+   use decomp_2d  ! 2D decomposition module
+   use iso_c_binding
+   use cudafor
+   use cufft
 
-  implicit none
+   implicit none
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
-  ! engine-specific global variables
-  ! integer, save :: plan_type = FFTW_MEASURE
+   ! engine-specific global variables
+   ! integer, save :: plan_type = FFTW_MEASURE
 
-  ! FFTW plans
-  ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
-  ! For c2c transforms: 
-  !     use plan(-1,j) for  forward transform; 
-  !     use plan( 1,j) for backward transform;
-  ! For r2c/c2r transforms:
-  !     use plan(0,j) for r2c transforms;
-  !     use plan(2,j) for c2r transforms;
-  integer*4, save :: plan(-1:2,3)
-  complex*8, device, allocatable, dimension(:) :: cufft_workspace
+   ! FFTW plans
+   ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
+   ! For c2c transforms:
+   !     use plan(-1,j) for  forward transform;
+   !     use plan( 1,j) for backward transform;
+   ! For r2c/c2r transforms:
+   !     use plan(0,j) for r2c transforms;
+   !     use plan(2,j) for c2r transforms;
+   integer*4, save :: plan(-1:2, 3)
+   complex*8, device, allocatable, dimension(:) :: cufft_workspace
 
-  ! common code used for all engines, including global variables, 
-  ! generic interface definitions and several subroutines
+   ! common code used for all engines, including global variables,
+   ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2C case
-  subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
-    
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-   
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp%xsz(1),     &
-                                      decomp%xsz(1),1,decomp%xsz(1),     &
-                                      decomp%xsz(1),1,decomp%xsz(1),     &
-                                      cufft_type,decomp%xsz(2)*decomp%xsz(3),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2c_1m_x_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in Y direction: C2C case
-  subroutine c2c_1m_y_plan(plan1, decomp, cufft_type,worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: cufft_type
-
-    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-    ! done one Z-plane at a time. So plan for 2D data sets here.
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-   
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp%ysz(2),     &
-                                      decomp%ysz(2),decomp%ysz(1),1,     &
-                                      decomp%ysz(2),decomp%ysz(1),1,     &
-                                      cufft_type,decomp%ysz(1),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2c_1m_y_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in Z direction: C2C case
-  subroutine c2c_1m_z_plan(plan1, decomp, cufft_type,worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp%zsz(3),                           &
-                                      decomp%zsz(3),decomp%zsz(1)*decomp%zsz(2),1,     &
-                                      decomp%zsz(3),decomp%zsz(1)*decomp%zsz(2),1,     &
-                                      cufft_type,decomp%zsz(1)*decomp%zsz(2),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2c_1m_z_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
-  subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp_ph%xsz(1),     &
-                                      decomp_ph%xsz(1),1,decomp_ph%xsz(1),     &
-                                      decomp_sp%xsz(1),1,decomp_sp%xsz(1),     &
-                                      cufft_type,decomp_ph%xsz(2)*decomp_ph%xsz(3),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine r2c_1m_x_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
-  subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp_ph%xsz(1),     &
-                                      decomp_sp%xsz(1),1,decomp_sp%xsz(1),     &
-                                      decomp_ph%xsz(1),1,decomp_ph%xsz(1),     &
-                                      cufft_type,decomp_ph%xsz(2)*decomp_ph%xsz(3),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2r_1m_x_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
-  subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp_ph%zsz(3),                           &
-                                      decomp_ph%zsz(3),decomp_ph%zsz(1)*decomp_ph%zsz(2),1,     &
-                                      decomp_sp%zsz(3),decomp_sp%zsz(1)*decomp_sp%zsz(2),1,     &
-                                      cufft_type,decomp_ph%zsz(1)*decomp_ph%zsz(2),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine r2c_1m_z_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
-  subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp_ph%zsz(3),                           &
-                                      decomp_sp%zsz(3),decomp_sp%zsz(1)*decomp_sp%zsz(2),1,     &
-                                      decomp_ph%zsz(3),decomp_ph%zsz(1)*decomp_ph%zsz(2),1,     &
-                                      cufft_type,decomp_ph%zsz(1)*decomp_ph%zsz(2),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2r_1m_z_plan
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2C case
+   subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp%xsz(1), &
+                                decomp%xsz(1), 1, decomp%xsz(1), &
+                                decomp%xsz(1), 1, decomp%xsz(1), &
+                                cufft_type, decomp%xsz(2)*decomp%xsz(3), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2c_1m_x_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in Y direction: C2C case
+   subroutine c2c_1m_y_plan(plan1, decomp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: cufft_type
+
+      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+      ! done one Z-plane at a time. So plan for 2D data sets here.
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp%ysz(2), &
+                                decomp%ysz(2), decomp%ysz(1), 1, &
+                                decomp%ysz(2), decomp%ysz(1), 1, &
+                                cufft_type, decomp%ysz(1), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2c_1m_y_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in Z direction: C2C case
+   subroutine c2c_1m_z_plan(plan1, decomp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp%zsz(3), &
+                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
+                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
+                                cufft_type, decomp%zsz(1)*decomp%zsz(2), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2c_1m_z_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
+   subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
+                                decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine r2c_1m_x_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
+   subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
+                                decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2r_1m_x_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
+   subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
+                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
+                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
+                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine r2c_1m_z_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
+   subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
+                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
+                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
+                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2r_1m_z_plan
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
+   subroutine init_fft_engine
+
+      implicit none
 
-    implicit none
+      !integer*4 :: cufft_ws, ws
+      integer(int_ptr_kind()) :: cufft_ws, ws
+      integer   :: i, j, istat
 
-    !integer*4 :: cufft_ws, ws
-    integer(int_ptr_kind()) :: cufft_ws, ws
-    integer   :: i, j, istat
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the New cuFFT engine *****'
+         write (*, *) ' '
+      end if
 
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the New cuFFT engine *****'
-       write(*,*) ' '
-    end if
-    
-     cufft_ws = 0 
+      cufft_ws = 0
 #ifdef DOUBLE_PREC
-    if (format == PHYSICAL_IN_X) then
-       ! For C2C transforms
-       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       ! For R2C/C2R tranforms
-       call r2c_1m_x_plan(plan(0,1), ph, sp, CUFFT_D2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(0,3), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(2,3), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2r_1m_x_plan(plan(2,1), sp, ph, CUFFT_Z2D,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-    else if (format == PHYSICAL_IN_Z) then
-
-       ! For C2C transforms
-       write(*,*) 'Create the plans'
-       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-       ! For R2C/C2R tranforms
-       call r2c_1m_z_plan(plan(0,3), ph, sp, CUFFT_D2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(0,1), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(2,1), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2r_1m_z_plan(plan(2,3), sp, ph, CUFFT_Z2D,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-    end if
+      if (format == PHYSICAL_IN_X) then
+         ! For C2C transforms
+         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         ! For R2C/C2R tranforms
+         call r2c_1m_x_plan(plan(0, 1), ph, sp, CUFFT_D2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(0, 3), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(2, 3), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2r_1m_x_plan(plan(2, 1), sp, ph, CUFFT_Z2D, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! For C2C transforms
+         write (*, *) 'Create the plans'
+         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_z_plan(plan(0, 3), ph, sp, CUFFT_D2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(0, 1), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(2, 1), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2r_1m_z_plan(plan(2, 3), sp, ph, CUFFT_Z2D, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+      end if
 #else
-    if (format == PHYSICAL_IN_X) then
-       ! For C2C transforms
-       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       ! For R2C/C2R tranforms
-       call r2c_1m_x_plan(plan(0,1), ph, sp, CUFFT_R2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(0,3), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(2,3), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2r_1m_x_plan(plan(2,1), sp, ph, CUFFT_C2R,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-    else if (format == PHYSICAL_IN_Z) then
-
-       ! For C2C transforms
-       write(*,*) 'Create the plans on rank ', nproc
-       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-       ! For R2C/C2R tranforms
-       call r2c_1m_z_plan(plan(0,3), ph, sp, CUFFT_R2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(0,1), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(2,1), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2r_1m_z_plan(plan(2,3), sp, ph, CUFFT_C2R,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-    end if
+      if (format == PHYSICAL_IN_X) then
+         ! For C2C transforms
+         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         ! For R2C/C2R tranforms
+         call r2c_1m_x_plan(plan(0, 1), ph, sp, CUFFT_R2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(0, 3), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(2, 3), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2r_1m_x_plan(plan(2, 1), sp, ph, CUFFT_C2R, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! For C2C transforms
+         write (*, *) 'Create the plans on rank ', nproc
+         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_z_plan(plan(0, 3), ph, sp, CUFFT_R2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(0, 1), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(2, 1), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2r_1m_z_plan(plan(2, 3), sp, ph, CUFFT_C2R, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+      end if
 #endif
-    allocate(cufft_workspace(cufft_ws))
-    do j=1,3
-       do i=-1,2
-         istat = cufftSetWorkArea(plan(i,j),cufft_workspace)
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetWorkArea")
-      enddo
-    enddo
-
-  end subroutine init_fft_engine
+      allocate (cufft_workspace(cufft_ws))
+      do j = 1, 3
+         do i = -1, 2
+            istat = cufftSetWorkArea(plan(i, j), cufft_workspace)
+            if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetWorkArea")
+         end do
+      end do
 
+   end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
-
-    implicit none
+   subroutine finalize_fft_engine
 
-    integer :: i,j, istat
+      implicit none
 
-    do j=1,3
-       do i=-1,2
-          istat = cufftDestroy(plan(i,j))
-          if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftDestroy")
-       end do
-    end do
+      integer :: i, j, istat
 
-  end subroutine finalize_fft_engine
+      do j = 1, 3
+         do i = -1, 2
+            istat = cufftDestroy(plan(i, j))
+            if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftDestroy")
+         end do
+      end do
 
+   end subroutine finalize_fft_engine
 
-  ! Following routines calculate multiple one-dimensional FFTs to form 
-  ! the basis of three-dimensional FFTs.
+   ! Following routines calculate multiple one-dimensional FFTs to form
+   ! the basis of three-dimensional FFTs.
 
-  ! c2c transform, multiple 1D FFTs in x direction
-  subroutine c2c_1m_x(inout, isign, plan1)
+   ! c2c transform, multiple 1D FFTs in x direction
+   subroutine c2c_1m_x(inout, isign, plan1)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    integer*4, intent(IN) :: plan1
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      integer*4, intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), allocatable :: output
-    integer :: istat
+      complex(mytype), dimension(:, :, :), allocatable :: output
+      integer :: istat
 
-    allocate(output,mold=inout)
+      allocate (output, mold=inout)
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(inout,output)
-    istat = cufftExecZ2Z(plan1, inout, output,isign)
-    !$acc end host_data
+      !$acc host_data use_device(inout,output)
+      istat = cufftExecZ2Z(plan1, inout, output, isign)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(inout,output)
-    istat = cufftExecC2C(plan1, inout, output,isign)
-    !$acc end host_data
+      !$acc host_data use_device(inout,output)
+      istat = cufftExecC2C(plan1, inout, output, isign)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-    !$acc kernels
-    inout = output
-    !$acc end kernels 
-    deallocate(output)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
+      !$acc kernels
+      inout = output
+      !$acc end kernels
+      deallocate (output)
 
-  end subroutine c2c_1m_x
+   end subroutine c2c_1m_x
 
+   ! c2c transform, multiple 1D FFTs in y direction
+   subroutine c2c_1m_y(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in y direction
-  subroutine c2c_1m_y(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      integer*4, intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    integer*4, intent(IN) :: plan1
+      complex(mytype), dimension(:, :, :), allocatable :: output
+      integer :: s3, k, istat
 
-    complex(mytype), dimension(:,:,:), allocatable :: output
-    integer :: s3, k, istat
-    
-    allocate(output,mold=inout)
-    ! transform on one Z-plane at a time
-    s3 = size(inout,3)
-    do k=1,s3
+      allocate (output, mold=inout)
+      ! transform on one Z-plane at a time
+      s3 = size(inout, 3)
+      do k = 1, s3
 #ifdef DOUBLE_PREC
-       !$acc host_data use_device(inout,output)
-       istat = cufftExecZ2Z(plan1, inout(:,:,k), output(:,:,k),isign)
-       !$acc end host_data
+         !$acc host_data use_device(inout,output)
+         istat = cufftExecZ2Z(plan1, inout(:, :, k), output(:, :, k), isign)
+         !$acc end host_data
 #else
-       !$acc host_data use_device(inout,output)
-       istat = cufftExecC2C(plan1, inout(:,:,k), output(:,:,k),isign)
-       !$acc end host_data
+         !$acc host_data use_device(inout,output)
+         istat = cufftExecC2C(plan1, inout(:, :, k), output(:, :, k), isign)
+         !$acc end host_data
 #endif
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-    enddo
-    !$acc kernels
-    inout = output
-    !$acc end kernels 
-    deallocate(output)
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
+      end do
+      !$acc kernels
+      inout = output
+      !$acc end kernels
+      deallocate (output)
 
-  end subroutine c2c_1m_y
+   end subroutine c2c_1m_y
 
-  ! c2c transform, multiple 1D FFTs in z direction
-  subroutine c2c_1m_z(inout, isign, plan1)
+   ! c2c transform, multiple 1D FFTs in z direction
+   subroutine c2c_1m_z(inout, isign, plan1)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    integer*4, intent(IN) :: plan1
-    
-    complex(mytype), dimension(:,:,:), allocatable :: output
-    integer :: istat
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      integer*4, intent(IN) :: plan1
 
-    allocate(output,mold=inout)
+      complex(mytype), dimension(:, :, :), allocatable :: output
+      integer :: istat
 
+      allocate (output, mold=inout)
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(inout,output)
-    istat = cufftExecZ2Z(plan1, inout, output,isign)
-    !$acc end host_data
+      !$acc host_data use_device(inout,output)
+      istat = cufftExecZ2Z(plan1, inout, output, isign)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(inout,output)
-    istat = cufftExecC2C(plan1, inout, output,isign)
-    !$acc end host_data
+      !$acc host_data use_device(inout,output)
+      istat = cufftExecC2C(plan1, inout, output, isign)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-    !$acc kernels
-    inout = output
-    !$acc end kernels
-    deallocate(output)
-
-  end subroutine c2c_1m_z
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
+      !$acc kernels
+      inout = output
+      !$acc end kernels
+      deallocate (output)
 
-  ! r2c transform, multiple 1D FFTs in x direction
-  subroutine r2c_1m_x(input, output)
+   end subroutine c2c_1m_z
 
-    implicit none
+   ! r2c transform, multiple 1D FFTs in x direction
+   subroutine r2c_1m_x(input, output)
 
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
-    integer :: istat
+      implicit none
 
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+      integer :: istat
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(input,output)
-    istat =  cufftExecD2Z(plan(0,1), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecD2Z(plan(0, 1), input, output)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(input,output)
-    istat =  cufftExecR2C(plan(0,1), input, output)
-    !$acc end host_data
-#endif    
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
+      !$acc host_data use_device(input,output)
+      istat = cufftExecR2C(plan(0, 1), input, output)
+      !$acc end host_data
+#endif
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
 
-  end subroutine r2c_1m_x
+   end subroutine r2c_1m_x
 
-  ! r2c transform, multiple 1D FFTs in z direction
-  subroutine r2c_1m_z(input, output)
+   ! r2c transform, multiple 1D FFTs in z direction
+   subroutine r2c_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN)     ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(IN)     ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
-    integer :: istat
+      integer :: istat
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(input,output)
-    istat = cufftExecD2Z(plan(0,3), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecD2Z(plan(0, 3), input, output)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(input,output)
-    istat = cufftExecR2C(plan(0,3), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecR2C(plan(0, 3), input, output)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
 
-  end subroutine r2c_1m_z
+   end subroutine r2c_1m_z
 
-  ! c2r transform, multiple 1D FFTs in x direction
-  subroutine c2r_1m_x(input, output)
+   ! c2r transform, multiple 1D FFTs in x direction
+   subroutine c2r_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT)    :: output
+      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT)    :: output
 
-    integer :: istat
+      integer :: istat
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(input,output)
-    istat = cufftExecZ2D(plan(2,1), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecZ2D(plan(2, 1), input, output)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(input,output)
-    istat = cufftExecC2R(plan(2,1), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecC2R(plan(2, 1), input, output)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
 
-  end subroutine c2r_1m_x
+   end subroutine c2r_1m_x
 
-  ! c2r transform, multiple 1D FFTs in z direction
-  subroutine c2r_1m_z(input, output)
+   ! c2r transform, multiple 1D FFTs in z direction
+   subroutine c2r_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(IN) :: input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
-    integer :: istat
+      integer :: istat
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(input,output)
-    istat = cufftExecZ2D(plan(2,3), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecZ2D(plan(2, 3), input, output)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(input,output)
-    istat = cufftExecC2R(plan(2,3), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecC2R(plan(2, 3), input, output)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
-
-  end subroutine c2r_1m_z
-
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
 
+   end subroutine c2r_1m_z
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D FFT - complex to complex
+   ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2c(in, out, isign)
+   subroutine fft_3d_c2c(in, out, isign)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out
-    integer, intent(IN) :: isign
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out
+      integer, intent(IN) :: isign
 
 #ifndef OVERWRITE
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
+      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
 
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in,isign,plan(isign,1))
+         call c2c_1m_x(in, isign, plan(isign, 1))
 #else
-       allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-       wk1 = in
-       call c2c_1m_x(wk1,isign,plan(isign,1))
+         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+         wk1 = in
+         call c2c_1m_x(wk1, isign, plan(isign, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-       if (dims(1)>1) then
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in,wk2_c2c,ph)
+            call transpose_x_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_x_to_y(wk1,wk2_c2c,ph)
+            call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else
 #ifdef OVERWRITE
-          call c2c_1m_y(in,isign,plan(isign,2))
+            call c2c_1m_y(in, isign, plan(isign, 2))
 #else
-          call c2c_1m_y(wk1,isign,plan(isign,2))
+            call c2c_1m_y(wk1, isign, plan(isign, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_c2c,out,ph)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_c2c, out, ph)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in,out,ph)
+            call transpose_y_to_z(in, out, ph)
 #else
-          call transpose_y_to_z(wk1,out,ph)
+            call transpose_y_to_z(wk1, out, ph)
 #endif
-       end if
-       call c2c_1m_z(out,isign,plan(isign,3))
+         end if
+         call c2c_1m_z(out, isign, plan(isign, 3))
 
-    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-         .OR. & 
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
+      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+               .OR. &
+               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in,isign,plan(isign,3))
+         call c2c_1m_z(in, isign, plan(isign, 3))
 #else
-       allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-       wk1 = in
-       call c2c_1m_z(wk1,isign,plan(isign,3))
+         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+         wk1 = in
+         call c2c_1m_z(wk1, isign, plan(isign, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,wk2_c2c,ph)
+            call transpose_z_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_z_to_y(wk1,wk2_c2c,ph)
+            call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else  ! out==wk2_c2c if 1D decomposition
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,out,ph)
+            call transpose_z_to_y(in, out, ph)
 #else
-          call transpose_z_to_y(wk1,out,ph)
+            call transpose_z_to_y(wk1, out, ph)
 #endif
-          call c2c_1m_y(out,isign,plan(isign,2))
-       end if
+            call c2c_1m_y(out, isign, plan(isign, 2))
+         end if
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_c2c,out,ph)
-       end if
-       call c2c_1m_x(out,isign,plan(isign,1))
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_c2c, out, ph)
+         end if
+         call c2c_1m_x(out, isign, plan(isign, 1))
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    deallocate (wk1)
+      deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-  end subroutine fft_3d_c2c
-
+   end subroutine fft_3d_c2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D forward FFT - real to complex
+   ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_r2c(in_r, out_c)
-    !use nvtx
-    implicit none
+   subroutine fft_3d_r2c(in_r, out_c)
+      !use nvtx
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: in_r
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
-    integer :: i, j ,k
-    integer, dimension(3) :: dim3d
+      real(mytype), dimension(:, :, :), intent(IN) :: in_r
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
+      integer :: i, j, k
+      integer, dimension(3) :: dim3d
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in X =====
-       call r2c_1m_x(in_r,wk13)
+         ! ===== 1D FFTs in X =====
+         call r2c_1m_x(in_r, wk13)
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_x_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else
-          call c2c_1m_y(wk13,-1,plan(0,2))
-       end if
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_x_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else
+            call c2c_1m_y(wk13, -1, plan(0, 2))
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,out_c,sp)
-       else
-          call transpose_y_to_z(wk13,out_c,sp)
-       end if
-       call c2c_1m_z(out_c,-1,plan(0,3))
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, out_c, sp)
+         else
+            call transpose_y_to_z(wk13, out_c, sp)
+         end if
+         call c2c_1m_z(out_c, -1, plan(0, 3))
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in Z =====
-       !call nvtxStartRange("Z r2c_1m_z")
+         ! ===== 1D FFTs in Z =====
+         !call nvtxStartRange("Z r2c_1m_z")
 #ifdef DEBUG
-       dim3d = shape(in_r)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_r(i,j,k))
+         dim3d = shape(in_r)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_r(i, j, k))
+               end do
             end do
          end do
-       end do
 #endif
-       call r2c_1m_z(in_r,wk13)
-
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          !call nvtxStartRange("Z1 transpose_z_to_y")
-          call transpose_z_to_y(wk13,wk2_r2c,sp)
-          !call nvtxEndRange
-          !call nvtxStartRange("Z1 c2c_1m_y")
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-          !call nvtxEndRange
+         call r2c_1m_z(in_r, wk13)
+
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            !call nvtxStartRange("Z1 transpose_z_to_y")
+            call transpose_z_to_y(wk13, wk2_r2c, sp)
+            !call nvtxEndRange
+            !call nvtxStartRange("Z1 c2c_1m_y")
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+            !call nvtxEndRange
 #ifdef DEBUG
-          write(*,*) 'c2c_1m_y'
-          dim3d = shape(wk2_r2c)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i,j,k)),&
-                                aimag(wk2_r2c(i,j,k))
+            write (*, *) 'c2c_1m_y'
+            dim3d = shape(wk2_r2c)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
+                        aimag(wk2_r2c(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
-       else  ! out_c==wk2_r2c if 1D decomposition
-          !call nvtxStartRange("Z transpose_z_to_y")
-          call transpose_z_to_y(wk13,out_c,sp)
-          !call nvtxEndRange
-          !call nvtxStartRange("Z c2c_1m_y")
-          call c2c_1m_y(out_c,-1,plan(0,2))
-          !call nvtxEndRange
+         else  ! out_c==wk2_r2c if 1D decomposition
+            !call nvtxStartRange("Z transpose_z_to_y")
+            call transpose_z_to_y(wk13, out_c, sp)
+            !call nvtxEndRange
+            !call nvtxStartRange("Z c2c_1m_y")
+            call c2c_1m_y(out_c, -1, plan(0, 2))
+            !call nvtxEndRange
 #ifdef DEBUG
-          write(*,*) 'c2c_1m_y2'
-          dim3d = shape(out_c)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i,j,k)),&
-                                aimag(out_c(i,j,k))
+            write (*, *) 'c2c_1m_y2'
+            dim3d = shape(out_c)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
+                        aimag(out_c(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
-       end if
-
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          !call nvtxStartRange("Z1 transpose_y_to_x")
-          call transpose_y_to_x(wk2_r2c,out_c,sp)
-          !call nvtxEndRange
-       end if
-       !call nvtxStartRange("c2c_1m_x")
-       call c2c_1m_x(out_c,-1,plan(0,1))
-       !call nvtxEndRange
+         end if
+
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            !call nvtxStartRange("Z1 transpose_y_to_x")
+            call transpose_y_to_x(wk2_r2c, out_c, sp)
+            !call nvtxEndRange
+         end if
+         !call nvtxStartRange("c2c_1m_x")
+         call c2c_1m_x(out_c, -1, plan(0, 1))
+         !call nvtxEndRange
 #ifdef DEBUG
-       write(*,*) 'c2c_1m_x'
-       dim3d = shape(out_c)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i,j,k)),&
-                             aimag(out_c(i,j,k))
+         write (*, *) 'c2c_1m_x'
+         dim3d = shape(out_c)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
+                     aimag(out_c(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
-    end if
+      end if
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-  end subroutine fft_3d_r2c
-
+   end subroutine fft_3d_r2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D inverse FFT - complex to real
+   ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2r(in_c, out_r)
+   subroutine fft_3d_c2r(in_c, out_r)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
-    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
-    integer :: i,j,k
-    integer, dimension(3) :: dim3d
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
+      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
+      integer :: i, j, k
+      integer, dimension(3) :: dim3d
 #ifndef OVERWRITE
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in_c,1,plan(2,3))       
+         call c2c_1m_z(in_c, 1, plan(2, 3))
 #else
-       allocate (wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       wk1 = in_c
-       call c2c_1m_z(wk1,1,plan(2,3))
+         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         wk1 = in_c
+         call c2c_1m_z(wk1, 1, plan(2, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-       call transpose_z_to_y(in_c,wk2_r2c,sp)
+         call transpose_z_to_y(in_c, wk2_r2c, sp)
 #else
-       call transpose_z_to_y(wk1,wk2_r2c,sp)
+         call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-       call c2c_1m_y(wk2_r2c,1,plan(2,2))
+         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,wk13,sp)
-          call c2r_1m_x(wk13,out_r)
-       else
-          call c2r_1m_x(wk2_r2c,out_r)
-       end if
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, wk13, sp)
+            call c2r_1m_x(wk13, out_r)
+         else
+            call c2r_1m_x(wk2_r2c, out_r)
+         end if
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 #ifdef DEBUG
-       write(*,*) 'Back Init c2c_1m_x line 788'
-       dim3d = shape(in_c)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i,j,k)),&
-                             aimag(in_c(i,j,k))
+         write (*, *) 'Back Init c2c_1m_x line 788'
+         dim3d = shape(in_c)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
+                     aimag(in_c(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in_c,1,plan(2,1))
+         call c2c_1m_x(in_c, 1, plan(2, 1))
 #ifdef DEBUG
-       write(*,*) 'Back c2c_1m_x overwrite line 804'
-       dim3d = shape(in_c)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i,j,k)),&
-                             aimag(in_c(i,j,k))
+         write (*, *) 'Back c2c_1m_x overwrite line 804'
+         dim3d = shape(in_c)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
+                     aimag(in_c(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
 #else
-       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       wk1 = in_c
-       call c2c_1m_x(wk1,1,plan(2,1))
+         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         wk1 = in_c
+         call c2c_1m_x(wk1, 1, plan(2, 1))
 #ifdef DEBUG
-       write(*,*) 'Back2 c2c_1m_x line 821'
-       dim3d = shape(wk1)
-       do k = 1, dim3d(3),dim3d(1)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i,j,k)),&
-                             aimag(wk1(i,j,k))
+         write (*, *) 'Back2 c2c_1m_x line 821'
+         dim3d = shape(wk1)
+         do k = 1, dim3d(3), dim3d(1)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
+                     aimag(wk1(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in_c,wk2_r2c,sp)
+            call transpose_x_to_y(in_c, wk2_r2c, sp)
 #else
-          call transpose_x_to_y(wk1,wk2_r2c,sp)
+            call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-          call c2c_1m_y(wk2_r2c,1,plan(2,2))
+            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
 #ifdef DEBUG
-          write(*,*) 'Back c2c_1m_y line 844'
-          dim3d = shape(wk2_r2c)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i,j,k)),&
-                                aimag(wk2_r2c(i,j,k))
+            write (*, *) 'Back c2c_1m_y line 844'
+            dim3d = shape(wk2_r2c)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
+                        aimag(wk2_r2c(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
-       else  ! in_c==wk2_r2c if 1D decomposition
+         else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-          call c2c_1m_y(in_c,1,plan(2,2))
+            call c2c_1m_y(in_c, 1, plan(2, 2))
 #ifdef DEBUG
-          write(*,*) 'Back2 c2c_1m_y line 860'
-          dim3d = shape(in_c)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i,j,k)),&
-                                aimag(in_c(i,j,k))
+            write (*, *) 'Back2 c2c_1m_y line 860'
+            dim3d = shape(in_c)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
+                        aimag(in_c(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
 #else
-          call c2c_1m_y(wk1,1,plan(2,2))
+            call c2c_1m_y(wk1, 1, plan(2, 2))
 #ifdef DEBUG
-          write(*,*) 'Back3 c2c_1m_y line 875'
-          dim3d = shape(wk1)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i,j,k)),&
-                                aimag(wk1(i,j,k))
+            write (*, *) 'Back3 c2c_1m_y line 875'
+            dim3d = shape(wk1)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
+                        aimag(wk1(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,wk13,sp)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, wk13, sp)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in_c,wk13,sp)
+            call transpose_y_to_z(in_c, wk13, sp)
 #else
-          call transpose_y_to_z(wk1,wk13,sp)
+            call transpose_y_to_z(wk1, wk13, sp)
 #endif
-       end if
+         end if
 #ifdef DEBUG
-       write(*,*) 'Back2 after tr_y2z'
-       dim3d = shape(wk13)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-              print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk13(i,j,k)),&
-                               aimag(wk13(i,j,k))
+         write (*, *) 'Back2 after tr_y2z'
+         dim3d = shape(wk13)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk13(i, j, k)), &
+                     aimag(wk13(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
-       call c2r_1m_z(wk13,out_r)
+         call c2r_1m_z(wk13, out_r)
 #ifdef DEBUG
-       write(*,*) 'Back2 c2r_1m_z out_r line 902'
-       dim3d = shape(out_r)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_r(i,j,k))
+         write (*, *) 'Back2 c2r_1m_z out_r line 902'
+         dim3d = shape(out_r)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_r(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    deallocate (wk1)
+      deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-  end subroutine fft_3d_c2r
-
+   end subroutine fft_3d_c2r
 
 end module decomp_2d_fft
 
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index 91cc0670..fb7646ba 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,733 +13,720 @@
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use iso_c_binding
+   use decomp_2d  ! 2D decomposition module
+   use iso_c_binding
 
-  implicit none
+   implicit none
 
-  include "fftw3.f"
+   include "fftw3.f"
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
-  ! engine-specific global variables
-  integer, save :: plan_type = FFTW_MEASURE
+   ! engine-specific global variables
+   integer, save :: plan_type = FFTW_MEASURE
 
-  ! FFTW plans
-  ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
-  ! For c2c transforms: 
-  !     use plan(-1,j) for  forward transform; 
-  !     use plan( 1,j) for backward transform;
-  ! For r2c/c2r transforms:
-  !     use plan(0,j) for r2c transforms;
-  !     use plan(2,j) for c2r transforms;
-  type(C_PTR), save :: plan(-1:2,3)
+   ! FFTW plans
+   ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
+   ! For c2c transforms:
+   !     use plan(-1,j) for  forward transform;
+   !     use plan( 1,j) for backward transform;
+   ! For r2c/c2r transforms:
+   !     use plan(0,j) for r2c transforms;
+   !     use plan(2,j) for c2r transforms;
+   type(C_PTR), save :: plan(-1:2, 3)
 
-  ! This is defined in fftw3.f03 but not in fftw3.f
-  interface
-    subroutine fftw_cleanup() bind(C, name='fftw_cleanup')
-      import
-    end subroutine fftw_cleanup
-  end interface
+   ! This is defined in fftw3.f03 but not in fftw3.f
+   interface
+      subroutine fftw_cleanup() bind(C, name='fftw_cleanup')
+         import
+      end subroutine fftw_cleanup
+   end interface
 
-  ! common code used for all engines, including global variables, 
-  ! generic interface definitions and several subroutines
+   ! common code used for all engines, including global variables,
+   ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
-  subroutine c2c_1m_x_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
+   subroutine c2c_1m_x_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
-    complex(mytype), allocatable, dimension(:,:,:) :: a1
+      complex(mytype), allocatable, dimension(:, :, :) :: a1
 
-    allocate(a1(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)))
+      allocate (a1(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)))
 
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-         decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
-         isign, plan_type)
+      call dfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
+                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                               decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
+                               isign, plan_type)
 #else
-    call sfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-         decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
-         isign, plan_type)
+      call sfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
+                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                               decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
+                               isign, plan_type)
 #endif
 
-    deallocate(a1)
+      deallocate (a1)
 
-    return
-  end subroutine c2c_1m_x_plan
+      return
+   end subroutine c2c_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
-  subroutine c2c_1m_y_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
+   subroutine c2c_1m_y_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
-    complex(mytype), allocatable, dimension(:,:) :: a1
+      complex(mytype), allocatable, dimension(:, :) :: a1
 
-    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-    ! done one Z-plane at a time. So plan for 2D data sets here.
+      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+      ! done one Z-plane at a time. So plan for 2D data sets here.
 
-    allocate(a1(decomp%ysz(1),decomp%ysz(2)))
+      allocate (a1(decomp%ysz(1), decomp%ysz(2)))
 
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
-         a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
-         decomp%ysz(1), 1, isign, plan_type)
+      call dfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
+                               a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
+                               decomp%ysz(1), 1, isign, plan_type)
 #else
-    call sfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
-         a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
-         decomp%ysz(1), 1, isign, plan_type)
+      call sfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
+                               a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
+                               decomp%ysz(1), 1, isign, plan_type)
 #endif
 
-    deallocate(a1)
-
-    return
-  end subroutine c2c_1m_y_plan
+      deallocate (a1)
 
+      return
+   end subroutine c2c_1m_y_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
-  subroutine c2c_1m_z_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
+   subroutine c2c_1m_z_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
-    complex(mytype), allocatable, dimension(:,:,:) :: a1
+      complex(mytype), allocatable, dimension(:, :, :) :: a1
 
-    allocate(a1(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)))
+      allocate (a1(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)))
 
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+      call dfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #else
-    call sfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+      call sfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #endif
 
-    deallocate(a1)
+      deallocate (a1)
 
-    return
-  end subroutine c2c_1m_z_plan
+      return
+   end subroutine c2c_1m_z_plan
 
+   ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
+   subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
 
-  ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
-  subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
+      implicit none
 
-    implicit none
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      real(mytype), allocatable, dimension(:, :, :) :: a1
+      complex(mytype), allocatable, dimension(:, :, :) :: a2
 
-    real(mytype), allocatable, dimension(:,:,:) :: a1
-    complex(mytype), allocatable, dimension(:,:,:) :: a2
-
-    allocate(a1(decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)))
-    allocate(a2(decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)))
+      allocate (a1(decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)))
+      allocate (a2(decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)))
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-         plan_type)
+      call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
+                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                   decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                   plan_type)
 #else
-    call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-         plan_type)
+      call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
+                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                   decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                   plan_type)
 #endif
-    deallocate(a1,a2)    
-
-    return
-  end subroutine r2c_1m_x_plan
+      deallocate (a1, a2)
 
+      return
+   end subroutine r2c_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
-  subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
+   ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
+   subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-    complex(mytype), allocatable, dimension(:,:,:) :: a1
-    real(mytype), allocatable, dimension(:,:,:) :: a2
+      complex(mytype), allocatable, dimension(:, :, :) :: a1
+      real(mytype), allocatable, dimension(:, :, :) :: a2
 
-    allocate(a1(decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)))
-    allocate(a2(decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)))
+      allocate (a1(decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)))
+      allocate (a2(decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)))
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-         plan_type)
+      call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
+                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                   decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                   plan_type)
 #else
-    call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-         plan_type)
+      call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
+                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                   decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                   plan_type)
 #endif
-    deallocate(a1,a2)
+      deallocate (a1, a2)
 
-    return
-  end subroutine c2r_1m_x_plan
+      return
+   end subroutine c2r_1m_x_plan
 
+   ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
+   subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
 
-  ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
-  subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
+      implicit none
 
-    implicit none
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      real(mytype), allocatable, dimension(:, :, :) :: a1
+      complex(mytype), allocatable, dimension(:, :, :) :: a2
 
-    real(mytype), allocatable, dimension(:,:,:) :: a1
-    complex(mytype), allocatable, dimension(:,:,:) :: a2
-
-    allocate(a1(decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)))
-    allocate(a2(decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)))
+      allocate (a1(decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)))
+      allocate (a2(decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)))
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+      call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #else
-    call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+      call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #endif
-    deallocate(a1,a2)
-
-    return
-  end subroutine r2c_1m_z_plan
+      deallocate (a1, a2)
 
+      return
+   end subroutine r2c_1m_z_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
-  subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
+   ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
+   subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-    complex(mytype), allocatable, dimension(:,:,:) :: a1
-    real(mytype), allocatable, dimension(:,:,:) :: a2
+      complex(mytype), allocatable, dimension(:, :, :) :: a1
+      real(mytype), allocatable, dimension(:, :, :) :: a2
 
-    allocate(a1(decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)))
-    allocate(a2(decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)))
+      allocate (a1(decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)))
+      allocate (a2(decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)))
 
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+      call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #else
-    call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+      call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #endif
-    deallocate(a1,a2)     
-
-    return
-  end subroutine c2r_1m_z_plan
+      deallocate (a1, a2)
 
+      return
+   end subroutine c2r_1m_z_plan
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
-
-    implicit none
-
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the FFTW (version 3.x) engine *****'
-       write(*,*) ' '
-    end if
-
-    if (format == PHYSICAL_IN_X) then
-
-       ! For C2C transforms
-       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
-       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
-       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
-
-       ! For R2C/C2R tranforms
-       call r2c_1m_x_plan(plan(0,1), ph, sp)
-       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(0,3), sp, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(2,3), sp, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
-       call c2r_1m_x_plan(plan(2,1), sp, ph)
-
-    else if (format == PHYSICAL_IN_Z) then
-
-       ! For C2C transforms
-       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
-       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD ) 
-       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
-       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
-
-       ! For R2C/C2R tranforms
-       call r2c_1m_z_plan(plan(0,3), ph, sp)
-       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan(0,1), sp, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan(2,1), sp, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
-       call c2r_1m_z_plan(plan(2,3), sp, ph)
-
-    end if
-
-    return
-  end subroutine init_fft_engine
+   subroutine init_fft_engine
+
+      implicit none
+
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the FFTW (version 3.x) engine *****'
+         write (*, *) ' '
+      end if
+
+      if (format == PHYSICAL_IN_X) then
+
+         ! For C2C transforms
+         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
+         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
+         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_x_plan(plan(0, 1), ph, sp)
+         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(0, 3), sp, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(2, 3), sp, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
+         call c2r_1m_x_plan(plan(2, 1), sp, ph)
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! For C2C transforms
+         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
+         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
+         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_z_plan(plan(0, 3), ph, sp)
+         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(0, 1), sp, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(2, 1), sp, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
+         call c2r_1m_z_plan(plan(2, 3), sp, ph)
+
+      end if
+
+      return
+   end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
+   subroutine finalize_fft_engine
 
-    implicit none
+      implicit none
 
-    integer :: i,j
+      integer :: i, j
 
-    do j=1,3
-       do i=-1,2
+      do j = 1, 3
+         do i = -1, 2
 #ifdef DOUBLE_PREC
-          call dfftw_destroy_plan(plan(i,j))
+            call dfftw_destroy_plan(plan(i, j))
 #else
-          call sfftw_destroy_plan(plan(i,j))
+            call sfftw_destroy_plan(plan(i, j))
 #endif
-       end do
-    end do
+         end do
+      end do
 
-    call fftw_cleanup()
+      call fftw_cleanup()
 
-    return
-  end subroutine finalize_fft_engine
+      return
+   end subroutine finalize_fft_engine
 
+   ! Following routines calculate multiple one-dimensional FFTs to form
+   ! the basis of three-dimensional FFTs.
 
-  ! Following routines calculate multiple one-dimensional FFTs to form 
-  ! the basis of three-dimensional FFTs.
+   ! c2c transform, multiple 1D FFTs in x direction
+   subroutine c2c_1m_x(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in x direction
-  subroutine c2c_1m_x(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR), intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR), intent(IN) :: plan1
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft(plan1, inout, inout)
+      call dfftw_execute_dft(plan1, inout, inout)
 #else
-    call sfftw_execute_dft(plan1, inout, inout)
+      call sfftw_execute_dft(plan1, inout, inout)
 #endif
 
-    return
-  end subroutine c2c_1m_x
+      return
+   end subroutine c2c_1m_x
 
+   ! c2c transform, multiple 1D FFTs in y direction
+   subroutine c2c_1m_y(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in y direction
-  subroutine c2c_1m_y(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR), intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR), intent(IN) :: plan1
+      integer :: k, s3
 
-    integer :: k, s3
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
-    ! transform on one Z-plane at a time
-    s3 = size(inout,3)
-    do k=1,s3
+      ! transform on one Z-plane at a time
+      s3 = size(inout, 3)
+      do k = 1, s3
 #ifdef DOUBLE_PREC
-       call dfftw_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
+         call dfftw_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
 #else
-       call sfftw_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
+         call sfftw_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
 #endif
-    end do
+      end do
+
+      return
+   end subroutine c2c_1m_y
 
-    return
-  end subroutine c2c_1m_y
+   ! c2c transform, multiple 1D FFTs in z direction
+   subroutine c2c_1m_z(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in z direction
-  subroutine c2c_1m_z(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR), intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR), intent(IN) :: plan1
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft(plan1, inout, inout)
+      call dfftw_execute_dft(plan1, inout, inout)
 #else
-    call sfftw_execute_dft(plan1, inout, inout)
+      call sfftw_execute_dft(plan1, inout, inout)
 #endif
 
-    return
-  end subroutine c2c_1m_z
+      return
+   end subroutine c2c_1m_z
 
-  ! r2c transform, multiple 1D FFTs in x direction
-  subroutine r2c_1m_x(input, output)
+   ! r2c transform, multiple 1D FFTs in x direction
+   subroutine r2c_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft_r2c(plan(0,1), input, output)
+      call dfftw_execute_dft_r2c(plan(0, 1), input, output)
 #else
-    call sfftw_execute_dft_r2c(plan(0,1), input, output)
-#endif    
+      call sfftw_execute_dft_r2c(plan(0, 1), input, output)
+#endif
 
-    return
+      return
 
-  end subroutine r2c_1m_x
+   end subroutine r2c_1m_x
 
-  ! r2c transform, multiple 1D FFTs in z direction
-  subroutine r2c_1m_z(input, output)
+   ! r2c transform, multiple 1D FFTs in z direction
+   subroutine r2c_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft_r2c(plan(0,3), input, output)
+      call dfftw_execute_dft_r2c(plan(0, 3), input, output)
 #else
-    call sfftw_execute_dft_r2c(plan(0,3), input, output)
+      call sfftw_execute_dft_r2c(plan(0, 3), input, output)
 #endif
 
-    return
+      return
 
-  end subroutine r2c_1m_z
+   end subroutine r2c_1m_z
 
-  ! c2r transform, multiple 1D FFTs in x direction
-  subroutine c2r_1m_x(input, output)
+   ! c2r transform, multiple 1D FFTs in x direction
+   subroutine c2r_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft_c2r(plan(2,1), input, output)
+      call dfftw_execute_dft_c2r(plan(2, 1), input, output)
 #else
-    call sfftw_execute_dft_c2r(plan(2,1), input, output)
+      call sfftw_execute_dft_c2r(plan(2, 1), input, output)
 #endif
 
-    return
+      return
 
-  end subroutine c2r_1m_x
+   end subroutine c2r_1m_x
 
-  ! c2r transform, multiple 1D FFTs in z direction
-  subroutine c2r_1m_z(input, output)
+   ! c2r transform, multiple 1D FFTs in z direction
+   subroutine c2r_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(IN) :: input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft_c2r(plan(2,3), input, output)
+      call dfftw_execute_dft_c2r(plan(2, 3), input, output)
 #else
-    call sfftw_execute_dft_c2r(plan(2,3), input, output)
-#endif    
-
-    return
-
-  end subroutine c2r_1m_z
+      call sfftw_execute_dft_c2r(plan(2, 3), input, output)
+#endif
 
+      return
 
+   end subroutine c2r_1m_z
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D FFT - complex to complex
+   ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2c(in, out, isign)
+   subroutine fft_3d_c2c(in, out, isign)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out
-    integer, intent(IN) :: isign
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out
+      integer, intent(IN) :: isign
 
 #ifndef OVERWRITE
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
-    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
+      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
 
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in,isign,plan(isign,1))
+         call c2c_1m_x(in, isign, plan(isign, 1))
 #else
-       allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-       wk1 = in
-       call c2c_1m_x(wk1,isign,plan(isign,1))
+         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+         wk1 = in
+         call c2c_1m_x(wk1, isign, plan(isign, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-       if (dims(1)>1) then
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in,wk2_c2c,ph)
+            call transpose_x_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_x_to_y(wk1,wk2_c2c,ph)
+            call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else
 #ifdef OVERWRITE
-          call c2c_1m_y(in,isign,plan(isign,2))
+            call c2c_1m_y(in, isign, plan(isign, 2))
 #else
-          call c2c_1m_y(wk1,isign,plan(isign,2))
+            call c2c_1m_y(wk1, isign, plan(isign, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_c2c,out,ph)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_c2c, out, ph)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in,out,ph)
+            call transpose_y_to_z(in, out, ph)
 #else
-          call transpose_y_to_z(wk1,out,ph)
+            call transpose_y_to_z(wk1, out, ph)
 #endif
-       end if
-       call c2c_1m_z(out,isign,plan(isign,3))
+         end if
+         call c2c_1m_z(out, isign, plan(isign, 3))
 
-    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-         .OR. & 
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
+      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+               .OR. &
+               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in,isign,plan(isign,3))
+         call c2c_1m_z(in, isign, plan(isign, 3))
 #else
-       allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-       wk1 = in
-       call c2c_1m_z(wk1,isign,plan(isign,3))
+         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+         wk1 = in
+         call c2c_1m_z(wk1, isign, plan(isign, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,wk2_c2c,ph)
+            call transpose_z_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_z_to_y(wk1,wk2_c2c,ph)
+            call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else  ! out==wk2_c2c if 1D decomposition
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,out,ph)
+            call transpose_z_to_y(in, out, ph)
 #else
-          call transpose_z_to_y(wk1,out,ph)
+            call transpose_z_to_y(wk1, out, ph)
 #endif
-          call c2c_1m_y(out,isign,plan(isign,2))
-       end if
+            call c2c_1m_y(out, isign, plan(isign, 2))
+         end if
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_c2c,out,ph)
-       end if
-       call c2c_1m_x(out,isign,plan(isign,1))
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_c2c, out, ph)
+         end if
+         call c2c_1m_x(out, isign, plan(isign, 1))
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    deallocate (wk1)
+      deallocate (wk1)
 #endif
 
-    return
-  end subroutine fft_3d_c2c
-
+      return
+   end subroutine fft_3d_c2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D forward FFT - real to complex
+   ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_r2c(in_r, out_c)
-
-    implicit none
+   subroutine fft_3d_r2c(in_r, out_c)
 
-    real(mytype), dimension(:,:,:), intent(IN) :: in_r
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
+      implicit none
 
-    if (format==PHYSICAL_IN_X) then
+      real(mytype), dimension(:, :, :), intent(IN) :: in_r
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
-       ! ===== 1D FFTs in X =====
-       call r2c_1m_x(in_r,wk13)
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_x_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else
-          call c2c_1m_y(wk13,-1,plan(0,2))
-       end if
+         ! ===== 1D FFTs in X =====
+         call r2c_1m_x(in_r, wk13)
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,out_c,sp)
-       else
-          call transpose_y_to_z(wk13,out_c,sp)
-       end if
-       call c2c_1m_z(out_c,-1,plan(0,3))
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_x_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else
+            call c2c_1m_y(wk13, -1, plan(0, 2))
+         end if
 
-    else if (format==PHYSICAL_IN_Z) then
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, out_c, sp)
+         else
+            call transpose_y_to_z(wk13, out_c, sp)
+         end if
+         call c2c_1m_z(out_c, -1, plan(0, 3))
 
-       ! ===== 1D FFTs in Z =====
-       call r2c_1m_z(in_r,wk13)
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_z_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else  ! out_c==wk2_r2c if 1D decomposition
-          call transpose_z_to_y(wk13,out_c,sp)
-          call c2c_1m_y(out_c,-1,plan(0,2))
-       end if
+         ! ===== 1D FFTs in Z =====
+         call r2c_1m_z(in_r, wk13)
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,out_c,sp)
-       end if
-       call c2c_1m_x(out_c,-1,plan(0,1))
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_z_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else  ! out_c==wk2_r2c if 1D decomposition
+            call transpose_z_to_y(wk13, out_c, sp)
+            call c2c_1m_y(out_c, -1, plan(0, 2))
+         end if
 
-    end if
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, out_c, sp)
+         end if
+         call c2c_1m_x(out_c, -1, plan(0, 1))
 
-    return
-  end subroutine fft_3d_r2c
+      end if
 
+      return
+   end subroutine fft_3d_r2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D inverse FFT - complex to real
+   ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2r(in_c, out_r)
+   subroutine fft_3d_c2r(in_c, out_r)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
-    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
+      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
 #ifndef OVERWRITE
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in_c,1,plan(2,3))       
+         call c2c_1m_z(in_c, 1, plan(2, 3))
 #else
-       allocate (wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       wk1 = in_c
-       call c2c_1m_z(wk1,1,plan(2,3))
+         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         wk1 = in_c
+         call c2c_1m_z(wk1, 1, plan(2, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-       call transpose_z_to_y(in_c,wk2_r2c,sp)
+         call transpose_z_to_y(in_c, wk2_r2c, sp)
 #else
-       call transpose_z_to_y(wk1,wk2_r2c,sp)
+         call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-       call c2c_1m_y(wk2_r2c,1,plan(2,2))
+         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,wk13,sp)
-          call c2r_1m_x(wk13,out_r)
-       else
-          call c2r_1m_x(wk2_r2c,out_r)
-       end if
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, wk13, sp)
+            call c2r_1m_x(wk13, out_r)
+         else
+            call c2r_1m_x(wk2_r2c, out_r)
+         end if
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in_c,1,plan(2,1))
+         call c2c_1m_x(in_c, 1, plan(2, 1))
 #else
-       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       wk1 = in_c
-       call c2c_1m_x(wk1,1,plan(2,1))
+         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         wk1 = in_c
+         call c2c_1m_x(wk1, 1, plan(2, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in_c,wk2_r2c,sp)
+            call transpose_x_to_y(in_c, wk2_r2c, sp)
 #else
-          call transpose_x_to_y(wk1,wk2_r2c,sp)
+            call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-          call c2c_1m_y(wk2_r2c,1,plan(2,2))
-       else  ! in_c==wk2_r2c if 1D decomposition
+            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+         else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-          call c2c_1m_y(in_c,1,plan(2,2))
+            call c2c_1m_y(in_c, 1, plan(2, 2))
 #else
-          call c2c_1m_y(wk1,1,plan(2,2))
+            call c2c_1m_y(wk1, 1, plan(2, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,wk13,sp)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, wk13, sp)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in_c,wk13,sp)
+            call transpose_y_to_z(in_c, wk13, sp)
 #else
-          call transpose_y_to_z(wk1,wk13,sp)
+            call transpose_y_to_z(wk1, wk13, sp)
 #endif
-       end if
-       call c2r_1m_z(wk13,out_r)
+         end if
+         call c2r_1m_z(wk13, out_r)
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    deallocate (wk1)
+      deallocate (wk1)
 #endif
 
-    return
-  end subroutine fft_3d_c2r
-
+      return
+   end subroutine fft_3d_c2r
 
 end module decomp_2d_fft
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 5765ab66..e8c97081 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,1083 +9,1070 @@
 !
 !=======================================================================
 
-! This is the FFTW implementation of the FFT library using 
+! This is the FFTW implementation of the FFT library using
 ! the Fortran 2003 interface introduced in FFTW 3.3-beta1
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use, intrinsic :: iso_c_binding
-  
-  implicit none
-
-  include "fftw3.f03"
-  
-  private        ! Make everything private unless declared public
-
-  ! engine-specific global variables
-  integer, save :: plan_type = FFTW_MEASURE
-
-  ! FFTW plans
-  ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
-  ! For c2c transforms: 
-  !     use plan(-1,j) for  forward transform; 
-  !     use plan( 1,j) for backward transform;
-  ! For r2c/c2r transforms:
-  !     use plan(0,j) for r2c transforms;
-  !     use plan(2,j) for c2r transforms;
-  type(C_PTR), save :: plan(-1:2,3)
-
-  integer, parameter, public :: DECOMP_2D_FFT_FORWARD = -1
-  integer, parameter, public :: DECOMP_2D_FFT_BACKWARD = 1
-  
-  ! Physical space data can be stored in either X-pencil or Z-pencil
-  integer, parameter, public :: PHYSICAL_IN_X = 1
-  integer, parameter, public :: PHYSICAL_IN_Z = 3 
-
-  integer, save :: format                 ! input X-pencil or Z-pencil
-  
-  ! The libary can only be initialised once
-  logical, save :: initialised = .false. 
-
-  ! Global size of the FFT
-  integer, save :: nx_fft, ny_fft, nz_fft
-
-  ! 2D processor grid
-  ! FIXME this is already available in the module decomp_2d
-  integer, save, dimension(2) :: dims
-
-  ! Decomposition objects
-  TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
-  TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
-
-  ! Workspace to store the intermediate Y-pencil data
-  ! *** TODO: investigate how to use only one workspace array
-  complex(mytype), pointer :: wk2_c2c(:,:,:), wk2_r2c(:,:,:), wk13(:,:,:)
-  type(C_PTR) :: wk2_c2c_p, wk2_r2c_p, wk13_p
-
-  public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-       decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
-       decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
-  
-  ! Declare generic interfaces to handle different inputs
-  
-  interface decomp_2d_fft_init
-     module procedure fft_init_noarg
-     module procedure fft_init_arg
-     module procedure fft_init_general
-  end interface
-  
-  interface decomp_2d_fft_3d
-     module procedure fft_3d_c2c
-     module procedure fft_3d_r2c
-     module procedure fft_3d_c2r
-  end interface
-
-  
+   use decomp_2d  ! 2D decomposition module
+   use, intrinsic :: iso_c_binding
+
+   implicit none
+
+   include "fftw3.f03"
+
+   private        ! Make everything private unless declared public
+
+   ! engine-specific global variables
+   integer, save :: plan_type = FFTW_MEASURE
+
+   ! FFTW plans
+   ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
+   ! For c2c transforms:
+   !     use plan(-1,j) for  forward transform;
+   !     use plan( 1,j) for backward transform;
+   ! For r2c/c2r transforms:
+   !     use plan(0,j) for r2c transforms;
+   !     use plan(2,j) for c2r transforms;
+   type(C_PTR), save :: plan(-1:2, 3)
+
+   integer, parameter, public :: DECOMP_2D_FFT_FORWARD = -1
+   integer, parameter, public :: DECOMP_2D_FFT_BACKWARD = 1
+
+   ! Physical space data can be stored in either X-pencil or Z-pencil
+   integer, parameter, public :: PHYSICAL_IN_X = 1
+   integer, parameter, public :: PHYSICAL_IN_Z = 3
+
+   integer, save :: format                 ! input X-pencil or Z-pencil
+
+   ! The libary can only be initialised once
+   logical, save :: initialised = .false.
+
+   ! Global size of the FFT
+   integer, save :: nx_fft, ny_fft, nz_fft
+
+   ! 2D processor grid
+   ! FIXME this is already available in the module decomp_2d
+   integer, save, dimension(2) :: dims
+
+   ! Decomposition objects
+   TYPE(DECOMP_INFO), pointer, save :: ph => null()  ! physical space
+   TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
+
+   ! Workspace to store the intermediate Y-pencil data
+   ! *** TODO: investigate how to use only one workspace array
+   complex(mytype), pointer :: wk2_c2c(:, :, :), wk2_r2c(:, :, :), wk13(:, :, :)
+   type(C_PTR) :: wk2_c2c_p, wk2_r2c_p, wk13_p
+
+   public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
+             decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+             decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
+
+   ! Declare generic interfaces to handle different inputs
+
+   interface decomp_2d_fft_init
+      module procedure fft_init_noarg
+      module procedure fft_init_arg
+      module procedure fft_init_general
+   end interface
+
+   interface decomp_2d_fft_3d
+      module procedure fft_3d_c2c
+      module procedure fft_3d_r2c
+      module procedure fft_3d_c2r
+   end interface
+
 contains
-  
-  
+
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Initialise the FFT module
+   ! Initialise the FFT module
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_init_noarg
-    
-    implicit none
-    
-    call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
-    
-    return
-  end subroutine fft_init_noarg
+   subroutine fft_init_noarg
+
+      implicit none
+
+      call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
 
-  subroutine fft_init_arg(pencil)     ! allow to handle Z-pencil input
+      return
+   end subroutine fft_init_noarg
 
-    implicit none
+   subroutine fft_init_arg(pencil)     ! allow to handle Z-pencil input
 
-    integer, intent(IN) :: pencil
+      implicit none
 
-    call fft_init_general(pencil, nx_global, ny_global, nz_global)
+      integer, intent(IN) :: pencil
 
-    return
-  end subroutine fft_init_arg
+      call fft_init_general(pencil, nx_global, ny_global, nz_global)
 
-  ! Initialise the FFT library to perform arbitrary size transforms
-  subroutine fft_init_general(pencil, nx, ny, nz)
+      return
+   end subroutine fft_init_arg
 
-    implicit none
+   ! Initialise the FFT library to perform arbitrary size transforms
+   subroutine fft_init_general(pencil, nx, ny, nz)
 
-    integer, intent(IN) :: pencil
-    integer, intent(IN) :: nx, ny, nz
+      implicit none
 
-    integer :: errorcode
-    integer(C_SIZE_T) :: sz
+      integer, intent(IN) :: pencil
+      integer, intent(IN) :: nx, ny, nz
+
+      integer :: errorcode
+      integer(C_SIZE_T) :: sz
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
 #endif
 
-    if (initialised) then
-       errorcode = 4
-       call decomp_2d_abort(errorcode, &
-            'FFT library should only be initialised once')
-    end if
-    
-    format = pencil
-    nx_fft = nx
-    ny_fft = ny
-    nz_fft = nz
-
-    ! determine the processor grid in use
-    dims = get_decomp_dims()
-
-    ! for c2r/r2c interface:
-    ! if in physical space, a real array is of size: nx*ny*nz
-    ! in spectral space, the complex array is of size:
-    !         (nx/2+1)*ny*nz, if PHYSICAL_IN_X
-    !      or nx*ny*(nz/2+1), if PHYSICAL_IN_Z
-
-    if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
-       ph => decomp_main
-    else
-       if (.not.associated(ph)) allocate(ph)
-       call decomp_info_init(nx, ny, nz, ph)
-    endif
-    if (format==PHYSICAL_IN_X) then
-       call decomp_info_init(nx/2+1, ny, nz, sp)
-    else if (format==PHYSICAL_IN_Z) then
-       call decomp_info_init(nx, ny, nz/2+1, sp)
-    end if
-
-    sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)
-    wk2_c2c_p = fftw_alloc_complex(sz)
-    call c_f_pointer(wk2_c2c_p,wk2_c2c,[ph%ysz(1),ph%ysz(2),ph%ysz(3)])
-
-    sz = sp%ysz(1)*sp%ysz(2)*sp%ysz(3)
-    wk2_r2c_p = fftw_alloc_complex(sz)
-    call c_f_pointer(wk2_r2c_p,wk2_r2c,[sp%ysz(1),sp%ysz(2),sp%ysz(3)])
-
-
-    if (format==PHYSICAL_IN_X) then
-       sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
-       wk13_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk13_p,wk13,[sp%xsz(1),sp%xsz(2),sp%xsz(3)])
-    else if (format==PHYSICAL_IN_Z) then
-       sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
-       wk13_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk13_p,wk13,[sp%zsz(1),sp%zsz(2),sp%zsz(3)])
-    end if
-
-    call init_fft_engine
-    
-    initialised = .true.
+      if (initialised) then
+         errorcode = 4
+         call decomp_2d_abort(errorcode, &
+                              'FFT library should only be initialised once')
+      end if
+
+      format = pencil
+      nx_fft = nx
+      ny_fft = ny
+      nz_fft = nz
+
+      ! determine the processor grid in use
+      dims = get_decomp_dims()
+
+      ! for c2r/r2c interface:
+      ! if in physical space, a real array is of size: nx*ny*nz
+      ! in spectral space, the complex array is of size:
+      !         (nx/2+1)*ny*nz, if PHYSICAL_IN_X
+      !      or nx*ny*(nz/2+1), if PHYSICAL_IN_Z
+
+      if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
+         ph => decomp_main
+      else
+         if (.not. associated(ph)) allocate (ph)
+         call decomp_info_init(nx, ny, nz, ph)
+      end if
+      if (format == PHYSICAL_IN_X) then
+         call decomp_info_init(nx/2 + 1, ny, nz, sp)
+      else if (format == PHYSICAL_IN_Z) then
+         call decomp_info_init(nx, ny, nz/2 + 1, sp)
+      end if
+
+      sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)
+      wk2_c2c_p = fftw_alloc_complex(sz)
+      call c_f_pointer(wk2_c2c_p, wk2_c2c, [ph%ysz(1), ph%ysz(2), ph%ysz(3)])
+
+      sz = sp%ysz(1)*sp%ysz(2)*sp%ysz(3)
+      wk2_r2c_p = fftw_alloc_complex(sz)
+      call c_f_pointer(wk2_r2c_p, wk2_r2c, [sp%ysz(1), sp%ysz(2), sp%ysz(3)])
+
+      if (format == PHYSICAL_IN_X) then
+         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+         wk13_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk13_p, wk13, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
+      else if (format == PHYSICAL_IN_Z) then
+         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+         wk13_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk13_p, wk13, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
+      end if
+
+      call init_fft_engine
+
+      initialised = .true.
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
 #endif
 
-    return
-  end subroutine fft_init_general
+      return
+   end subroutine fft_init_general
 
-  
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Final clean up
+   ! Final clean up
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_2d_fft_finalize
-    
-    implicit none
+   subroutine decomp_2d_fft_finalize
+
+      implicit none
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-    if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
-      call decomp_info_finalize(ph)
-      deallocate(ph)
-    endif
-    nullify(ph)
-    call decomp_info_finalize(sp)
+      if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
+         call decomp_info_finalize(ph)
+         deallocate (ph)
+      end if
+      nullify (ph)
+      call decomp_info_finalize(sp)
 
-    call fftw_free(wk2_c2c_p)
-    call fftw_free(wk2_r2c_p)
-    call fftw_free(wk13_p)
+      call fftw_free(wk2_c2c_p)
+      call fftw_free(wk2_r2c_p)
+      call fftw_free(wk13_p)
 
-    call finalize_fft_engine
+      call finalize_fft_engine
 
-    initialised = .false.
+      initialised = .false.
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
 #endif
 
-    return
-  end subroutine decomp_2d_fft_finalize
-
+      return
+   end subroutine decomp_2d_fft_finalize
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Return the size, starting/ending index of the distributed array 
-  !  whose global size is (nx/2+1)*ny*nz, for defining data structures
-  !  in r2c and c2r interfaces
+   ! Return the size, starting/ending index of the distributed array
+   !  whose global size is (nx/2+1)*ny*nz, for defining data structures
+   !  in r2c and c2r interfaces
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_2d_fft_get_size(istart, iend, isize)
-    
-    implicit none
-    integer, dimension(3), intent(OUT) :: istart, iend, isize
-    
-    if (format==PHYSICAL_IN_X) then
-       istart = sp%zst
-       iend   = sp%zen
-       isize  = sp%zsz
-    else if (format==PHYSICAL_IN_Z) then
-       istart = sp%xst
-       iend   = sp%xen
-       isize  = sp%xsz
-    end if
-    
-    return
-  end subroutine decomp_2d_fft_get_size
+   subroutine decomp_2d_fft_get_size(istart, iend, isize)
+
+      implicit none
+      integer, dimension(3), intent(OUT) :: istart, iend, isize
+
+      if (format == PHYSICAL_IN_X) then
+         istart = sp%zst
+         iend = sp%zen
+         isize = sp%zsz
+      else if (format == PHYSICAL_IN_Z) then
+         istart = sp%xst
+         iend = sp%xen
+         isize = sp%xsz
+      end if
+
+      return
+   end subroutine decomp_2d_fft_get_size
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Return a pointer to the decomp_info object ph
-  !
-  ! The caller should not apply decomp_info_finalize on the pointer
+   ! Return a pointer to the decomp_info object ph
+   !
+   ! The caller should not apply decomp_info_finalize on the pointer
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  function decomp_2d_fft_get_ph()
+   function decomp_2d_fft_get_ph()
 
-    implicit none
+      implicit none
 
-    type(decomp_info), pointer :: decomp_2d_fft_get_ph
+      type(decomp_info), pointer :: decomp_2d_fft_get_ph
 
-    if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  -1, &
-                                                  'FFT library must be initialised first')
-    decomp_2d_fft_get_ph => ph
+      if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                     __LINE__, &
+                                                     -1, &
+                                                     'FFT library must be initialised first')
+      decomp_2d_fft_get_ph => ph
 
-  end function decomp_2d_fft_get_ph
+   end function decomp_2d_fft_get_ph
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Return a pointer to the decomp_info object sp
-  !
-  ! The caller should not apply decomp_info_finalize on the pointer
+   ! Return a pointer to the decomp_info object sp
+   !
+   ! The caller should not apply decomp_info_finalize on the pointer
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  function decomp_2d_fft_get_sp()
+   function decomp_2d_fft_get_sp()
 
-    implicit none
+      implicit none
 
-    type(decomp_info), pointer :: decomp_2d_fft_get_sp
+      type(decomp_info), pointer :: decomp_2d_fft_get_sp
 
-    if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  -1, &
-                                                  'FFT library must be initialised first')
-    decomp_2d_fft_get_sp => sp
+      if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                     __LINE__, &
+                                                     -1, &
+                                                     'FFT library must be initialised first')
+      decomp_2d_fft_get_sp => sp
 
-  end function decomp_2d_fft_get_sp
+   end function decomp_2d_fft_get_sp
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
-  subroutine c2c_1m_x_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
+   subroutine c2c_1m_x_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
 #ifdef DOUBLE_PREC
-    complex(C_DOUBLE_COMPLEX), pointer :: a1(:,:,:)
-    complex(C_DOUBLE_COMPLEX), pointer :: a1o(:,:,:)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1(:, :, :)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1o(:, :, :)
 #else
-    complex(C_FLOAT_COMPLEX), pointer :: a1(:,:,:)
-    complex(C_FLOAT_COMPLEX), pointer :: a1o(:,:,:)
+      complex(C_FLOAT_COMPLEX), pointer :: a1(:, :, :)
+      complex(C_FLOAT_COMPLEX), pointer :: a1o(:, :, :)
 #endif
-    type(C_PTR) :: a1_p
-    integer(C_SIZE_T) :: sz
+      type(C_PTR) :: a1_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1,[decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)])
-    call c_f_pointer(a1_p,a1o,[decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)])
+      sz = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
+      call c_f_pointer(a1_p, a1o, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 = fftw_plan_many_dft(1, decomp%xsz(1), &
-         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-         decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
-         isign, plan_type)
+      plan1 = fftw_plan_many_dft(1, decomp%xsz(1), &
+                                 decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                                 decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
+                                 isign, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft(1, decomp%xsz(1), &
-         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-         decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
-         isign, plan_type)
+      plan1 = fftwf_plan_many_dft(1, decomp%xsz(1), &
+                                  decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                                  decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
+                                  isign, plan_type)
 #endif
 
-    call fftw_free(a1_p)
+      call fftw_free(a1_p)
 
-    return
-  end subroutine c2c_1m_x_plan
+      return
+   end subroutine c2c_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
-  subroutine c2c_1m_y_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
+   subroutine c2c_1m_y_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
 #ifdef DOUBLE_PREC
-    complex(C_DOUBLE_COMPLEX), pointer :: a1(:,:)
-    complex(C_DOUBLE_COMPLEX), pointer :: a1o(:,:)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1(:, :)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1o(:, :)
 #else
-    complex(C_FLOAT_COMPLEX), pointer :: a1(:,:)
-    complex(C_FLOAT_COMPLEX), pointer :: a1o(:,:)
+      complex(C_FLOAT_COMPLEX), pointer :: a1(:, :)
+      complex(C_FLOAT_COMPLEX), pointer :: a1o(:, :)
 #endif
-    type(C_PTR) :: a1_p
-    integer(C_SIZE_T) :: sz
+      type(C_PTR) :: a1_p
+      integer(C_SIZE_T) :: sz
 
-    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-    ! done one Z-plane at a time. So plan for 2D data sets here.
-    sz = decomp%ysz(1)*decomp%ysz(2)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1,[decomp%ysz(1),decomp%ysz(2)])
-    call c_f_pointer(a1_p,a1o,[decomp%ysz(1),decomp%ysz(2)])
+      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+      ! done one Z-plane at a time. So plan for 2D data sets here.
+      sz = decomp%ysz(1)*decomp%ysz(2)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, [decomp%ysz(1), decomp%ysz(2)])
+      call c_f_pointer(a1_p, a1o, [decomp%ysz(1), decomp%ysz(2)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
-         a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
-         decomp%ysz(1), 1, isign, plan_type)
+      plan1 = fftw_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
+                                 a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
+                                 decomp%ysz(1), 1, isign, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
-         a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
-         decomp%ysz(1), 1, isign, plan_type)
+      plan1 = fftwf_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
+                                  a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
+                                  decomp%ysz(1), 1, isign, plan_type)
 #endif
 
-    call fftw_free(a1_p)
+      call fftw_free(a1_p)
 
-    return
-  end subroutine c2c_1m_y_plan
+      return
+   end subroutine c2c_1m_y_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
-  subroutine c2c_1m_z_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
+   subroutine c2c_1m_z_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
 #ifdef DOUBLE_PREC
-    complex(C_DOUBLE_COMPLEX), pointer :: a1(:,:,:)
-    complex(C_DOUBLE_COMPLEX), pointer :: a1o(:,:,:)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1(:, :, :)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1o(:, :, :)
 #else
-    complex(C_FLOAT_COMPLEX), pointer :: a1(:,:,:)
-    complex(C_FLOAT_COMPLEX), pointer :: a1o(:,:,:)
+      complex(C_FLOAT_COMPLEX), pointer :: a1(:, :, :)
+      complex(C_FLOAT_COMPLEX), pointer :: a1o(:, :, :)
 #endif
-    type(C_PTR) :: a1_p
-    integer(C_SIZE_T) :: sz
+      type(C_PTR) :: a1_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1,[decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)])
-    call c_f_pointer(a1_p,a1o,[decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)])
+      sz = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
+      call c_f_pointer(a1_p, a1o, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft(1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+      plan1 = fftw_plan_many_dft(1, decomp%zsz(3), &
+                                 decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+                                 decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+                                 decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft(1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+      plan1 = fftwf_plan_many_dft(1, decomp%zsz(3), &
+                                  decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+                                  decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+                                  decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #endif
 
-    call fftw_free(a1_p)
+      call fftw_free(a1_p)
 
-    return
-  end subroutine c2c_1m_z_plan
+      return
+   end subroutine c2c_1m_z_plan
 
-  ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
-  subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
+   ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
+   subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-    real(mytype), pointer :: a1(:,:,:)
-    complex(mytype), pointer :: a2(:,:,:)
-    type(C_PTR) :: a1_p, a2_p
-    integer(C_SIZE_T) :: sz
+      real(mytype), pointer :: a1(:, :, :)
+      complex(mytype), pointer :: a2(:, :, :)
+      type(C_PTR) :: a1_p, a2_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
-    a1_p = fftw_alloc_real(sz)
-    call c_f_pointer(a1_p,a1, &
-         [decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)])
-    sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
-    a2_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a2_p,a2, &
-         [decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)])
+      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+      a1_p = fftw_alloc_real(sz)
+      call c_f_pointer(a1_p, a1, &
+                       [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
+      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+      a2_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a2_p, a2, &
+                       [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-         plan_type)
+      plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
+                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                     decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                     plan_type)
 #else
-    plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-         plan_type)
+      plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
+                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                      decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                      plan_type)
 #endif
 
-    call fftw_free(a1_p)
-    call fftw_free(a2_p)
+      call fftw_free(a1_p)
+      call fftw_free(a2_p)
 
-    return
-  end subroutine r2c_1m_x_plan
+      return
+   end subroutine r2c_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
-  subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
+   ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
+   subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-    complex(mytype), pointer :: a1(:,:,:)
-    real(mytype), pointer :: a2(:,:,:)
-    type(C_PTR) :: a1_p, a2_p
-    integer(C_SIZE_T) :: sz
+      complex(mytype), pointer :: a1(:, :, :)
+      real(mytype), pointer :: a2(:, :, :)
+      type(C_PTR) :: a1_p, a2_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1, &
-         [decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)])
-    sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
-    a2_p = fftw_alloc_real(sz)
-    call c_f_pointer(a2_p,a2, &
-         [decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)])
+      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, &
+                       [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
+      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+      a2_p = fftw_alloc_real(sz)
+      call c_f_pointer(a2_p, a2, &
+                       [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-         plan_type)
+      plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
+                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                     decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                     plan_type)
 #else
-    plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-         plan_type)
+      plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
+                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                      decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                      plan_type)
 #endif
 
-    call fftw_free(a1_p)
-    call fftw_free(a2_p)
+      call fftw_free(a1_p)
+      call fftw_free(a2_p)
 
-    return
-  end subroutine c2r_1m_x_plan
+      return
+   end subroutine c2r_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
-  subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
+   ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
+   subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-    real(mytype), pointer :: a1(:,:,:)
-    complex(mytype), pointer :: a2(:,:,:)
-    type(C_PTR) :: a1_p, a2_p
-    integer(C_SIZE_T) :: sz
+      real(mytype), pointer :: a1(:, :, :)
+      complex(mytype), pointer :: a2(:, :, :)
+      type(C_PTR) :: a1_p, a2_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
-    a1_p = fftw_alloc_real(sz)
-    call c_f_pointer(a1_p,a1, &
-         [decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)])
-    sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
-    a2_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a2_p,a2, &
-         [decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)])
+      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+      a1_p = fftw_alloc_real(sz)
+      call c_f_pointer(a1_p, a1, &
+                       [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
+      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+      a2_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a2_p, a2, &
+                       [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+      plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+      plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #endif
 
-    call fftw_free(a1_p)
-    call fftw_free(a2_p)
+      call fftw_free(a1_p)
+      call fftw_free(a2_p)
 
-    return
-  end subroutine r2c_1m_z_plan
+      return
+   end subroutine r2c_1m_z_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
-  subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
+   ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
+   subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-    complex(mytype), pointer :: a1(:,:,:)
-    real(mytype), pointer :: a2(:,:,:)
-    type(C_PTR) :: a1_p, a2_p
-    integer(C_SIZE_T) :: sz
+      complex(mytype), pointer :: a1(:, :, :)
+      real(mytype), pointer :: a2(:, :, :)
+      type(C_PTR) :: a1_p, a2_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1, &
-         [decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)])
-    sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
-    a2_p = fftw_alloc_real(sz)
-    call c_f_pointer(a2_p,a2, &
-         [decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)])
+      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, &
+                       [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
+      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+      a2_p = fftw_alloc_real(sz)
+      call c_f_pointer(a2_p, a2, &
+                       [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+      plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+      plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #endif
 
-    call fftw_free(a1_p)
-    call fftw_free(a2_p)
-
-    return
-  end subroutine c2r_1m_z_plan
+      call fftw_free(a1_p)
+      call fftw_free(a2_p)
 
+      return
+   end subroutine c2r_1m_z_plan
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
-
-    implicit none
-
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the FFTW (F2003 interface) engine *****'
-       write(*,*) ' '
-    end if
-
-    if (format == PHYSICAL_IN_X) then
-
-       ! For C2C transforms
-       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
-       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
-       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
-       
-       ! For R2C/C2R tranforms
-       call r2c_1m_x_plan(plan(0,1), ph, sp)
-       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(0,3), sp, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(2,3), sp, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
-       call c2r_1m_x_plan(plan(2,1), sp, ph)
-
-    else if (format == PHYSICAL_IN_Z) then
-
-       ! For C2C transforms
-       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
-       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD ) 
-       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
-       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
-       
-       ! For R2C/C2R tranforms
-       call r2c_1m_z_plan(plan(0,3), ph, sp)
-       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan(0,1), sp, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan(2,1), sp, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
-       call c2r_1m_z_plan(plan(2,3), sp, ph)
-       
-    end if
-
-    return
-  end subroutine init_fft_engine
+   subroutine init_fft_engine
+
+      implicit none
+
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the FFTW (F2003 interface) engine *****'
+         write (*, *) ' '
+      end if
+
+      if (format == PHYSICAL_IN_X) then
+
+         ! For C2C transforms
+         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
+         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
+         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_x_plan(plan(0, 1), ph, sp)
+         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(0, 3), sp, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(2, 3), sp, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
+         call c2r_1m_x_plan(plan(2, 1), sp, ph)
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! For C2C transforms
+         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
+         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
+         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_z_plan(plan(0, 3), ph, sp)
+         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(0, 1), sp, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(2, 1), sp, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
+         call c2r_1m_z_plan(plan(2, 3), sp, ph)
+
+      end if
+
+      return
+   end subroutine init_fft_engine
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
+   subroutine finalize_fft_engine
+
+      implicit none
 
-    implicit none
+      integer :: i, j
 
-    integer :: i,j
-    
-    do j=1,3
-       do i=-1,2
+      do j = 1, 3
+         do i = -1, 2
 #ifdef DOUBLE_PREC
-          call fftw_destroy_plan(plan(i,j))
+            call fftw_destroy_plan(plan(i, j))
 #else
-          call fftwf_destroy_plan(plan(i,j))
+            call fftwf_destroy_plan(plan(i, j))
 #endif
-       end do
-    end do
+         end do
+      end do
 
-    call fftw_cleanup()
+      call fftw_cleanup()
 
-    return
-  end subroutine finalize_fft_engine
+      return
+   end subroutine finalize_fft_engine
 
+   ! Following routines calculate multiple one-dimensional FFTs to form
+   ! the basis of three-dimensional FFTs.
 
-  ! Following routines calculate multiple one-dimensional FFTs to form 
-  ! the basis of three-dimensional FFTs.
+   ! c2c transform, multiple 1D FFTs in x direction
+   subroutine c2c_1m_x(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in x direction
-  subroutine c2c_1m_x(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR) :: plan1
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft(plan1, inout, inout)
+      call fftw_execute_dft(plan1, inout, inout)
 #else
-    call fftwf_execute_dft(plan1, inout, inout)
+      call fftwf_execute_dft(plan1, inout, inout)
 #endif
 
-    return
-  end subroutine c2c_1m_x
+      return
+   end subroutine c2c_1m_x
 
+   ! c2c transform, multiple 1D FFTs in y direction
+   subroutine c2c_1m_y(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in y direction
-  subroutine c2c_1m_y(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR) :: plan1
+      integer :: k, s3
 
-    integer :: k, s3
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
-    s3 = size(inout,3)
+      s3 = size(inout, 3)
 
-    do k=1,s3  ! transform on one Z-plane at a time
+      do k = 1, s3  ! transform on one Z-plane at a time
 #ifdef DOUBLE_PREC
-       call fftw_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
+         call fftw_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
 #else
-       call fftwf_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
+         call fftwf_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
 #endif
-    end do
+      end do
+
+      return
+   end subroutine c2c_1m_y
 
-    return
-  end subroutine c2c_1m_y
+   ! c2c transform, multiple 1D FFTs in z direction
+   subroutine c2c_1m_z(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in z direction
-  subroutine c2c_1m_z(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR) :: plan1
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
 #ifdef DOUBLE_PREC
-       call fftw_execute_dft(plan1, inout, inout)
+      call fftw_execute_dft(plan1, inout, inout)
 #else
-       call fftwf_execute_dft(plan1, inout, inout)
+      call fftwf_execute_dft(plan1, inout, inout)
 #endif
 
-    return
-  end subroutine c2c_1m_z
+      return
+   end subroutine c2c_1m_z
 
-  ! r2c transform, multiple 1D FFTs in x direction
-  subroutine r2c_1m_x(input, output)
+   ! r2c transform, multiple 1D FFTs in x direction
+   subroutine r2c_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(INOUT)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(INOUT)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft_r2c(plan(0,1), input, output)
+      call fftw_execute_dft_r2c(plan(0, 1), input, output)
 #else
-    call fftwf_execute_dft_r2c(plan(0,1), input, output)
-#endif    
+      call fftwf_execute_dft_r2c(plan(0, 1), input, output)
+#endif
 
-    return
+      return
 
-  end subroutine r2c_1m_x
+   end subroutine r2c_1m_x
 
-  ! r2c transform, multiple 1D FFTs in z direction
-  subroutine r2c_1m_z(input, output)
+   ! r2c transform, multiple 1D FFTs in z direction
+   subroutine r2c_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(INOUT)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(INOUT)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft_r2c(plan(0,3), input, output)
+      call fftw_execute_dft_r2c(plan(0, 3), input, output)
 #else
-    call fftwf_execute_dft_r2c(plan(0,3), input, output)
+      call fftwf_execute_dft_r2c(plan(0, 3), input, output)
 #endif
 
-    return
+      return
 
-  end subroutine r2c_1m_z
+   end subroutine r2c_1m_z
 
-  ! c2r transform, multiple 1D FFTs in x direction
-  subroutine c2r_1m_x(input, output)
+   ! c2r transform, multiple 1D FFTs in x direction
+   subroutine c2r_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(INOUT)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft_c2r(plan(2,1), input, output)
+      call fftw_execute_dft_c2r(plan(2, 1), input, output)
 #else
-    call fftwf_execute_dft_c2r(plan(2,1), input, output)
+      call fftwf_execute_dft_c2r(plan(2, 1), input, output)
 #endif
 
-    return
+      return
 
-  end subroutine c2r_1m_x
+   end subroutine c2r_1m_x
 
-  ! c2r transform, multiple 1D FFTs in z direction
-  subroutine c2r_1m_z(input, output)
+   ! c2r transform, multiple 1D FFTs in z direction
+   subroutine c2r_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft_c2r(plan(2,3), input, output)
+      call fftw_execute_dft_c2r(plan(2, 3), input, output)
 #else
-    call fftwf_execute_dft_c2r(plan(2,3), input, output)
-#endif    
-
-    return
-
-  end subroutine c2r_1m_z
+      call fftwf_execute_dft_c2r(plan(2, 3), input, output)
+#endif
 
+      return
 
+   end subroutine c2r_1m_z
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D FFT - complex to complex
+   ! 3D FFT - complex to complex
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2c(in, out, isign)
-    
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out
-    integer, intent(IN) :: isign
+   subroutine fft_3d_c2c(in, out, isign)
+
+      implicit none
+
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out
+      integer, intent(IN) :: isign
 
 #ifndef OVERWRITE
-    complex(mytype), pointer :: wk1(:,:,:)
-    integer(C_SIZE_T) :: sz
-    type(C_PTR) :: wk1_p
+      complex(mytype), pointer :: wk1(:, :, :)
+      integer(C_SIZE_T) :: sz
+      type(C_PTR) :: wk1_p
 
-    wk1_p = c_null_ptr ! Initialise to NULL pointer
+      wk1_p = c_null_ptr ! Initialise to NULL pointer
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
-       
-       ! ===== 1D FFTs in X =====
+      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
+
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in,isign,plan(isign,1))
+         call c2c_1m_x(in, isign, plan(isign, 1))
 #else
-       sz = ph%xsz(1)*ph%xsz(2)*ph%xsz(3)
-       wk1_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk1_p, wk1, [ph%xsz(1),ph%xsz(2),ph%xsz(3)])
-       wk1 = in
-       call c2c_1m_x(wk1,isign,plan(isign,1))
+         sz = ph%xsz(1)*ph%xsz(2)*ph%xsz(3)
+         wk1_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk1_p, wk1, [ph%xsz(1), ph%xsz(2), ph%xsz(3)])
+         wk1 = in
+         call c2c_1m_x(wk1, isign, plan(isign, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-       if (dims(1)>1) then
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in,wk2_c2c,ph)
+            call transpose_x_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_x_to_y(wk1,wk2_c2c,ph)
+            call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else
 #ifdef OVERWRITE
-          call c2c_1m_y(in,isign,plan(isign,2))
+            call c2c_1m_y(in, isign, plan(isign, 2))
 #else
-          call c2c_1m_y(wk1,isign,plan(isign,2))
+            call c2c_1m_y(wk1, isign, plan(isign, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_c2c,out,ph)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_c2c, out, ph)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in,out,ph)
+            call transpose_y_to_z(in, out, ph)
 #else
-          call transpose_y_to_z(wk1,out,ph)
+            call transpose_y_to_z(wk1, out, ph)
 #endif
-       end if
-       call c2c_1m_z(out,isign,plan(isign,3))
+         end if
+         call c2c_1m_z(out, isign, plan(isign, 3))
 
-    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-         .OR. & 
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
+      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+               .OR. &
+               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in,isign,plan(isign,3))
+         call c2c_1m_z(in, isign, plan(isign, 3))
 #else
-       sz = ph%zsz(1)*ph%zsz(2)*ph%zsz(3)
-       wk1_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk1_p, wk1, [ph%zsz(1),ph%zsz(2),ph%zsz(3)])
-       wk1 = in
-       call c2c_1m_z(wk1,isign,plan(isign,3))
+         sz = ph%zsz(1)*ph%zsz(2)*ph%zsz(3)
+         wk1_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk1_p, wk1, [ph%zsz(1), ph%zsz(2), ph%zsz(3)])
+         wk1 = in
+         call c2c_1m_z(wk1, isign, plan(isign, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,wk2_c2c,ph)
+            call transpose_z_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_z_to_y(wk1,wk2_c2c,ph)
+            call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else  ! out==wk2_c2c if 1D decomposition
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,out,ph)
+            call transpose_z_to_y(in, out, ph)
 #else
-          call transpose_z_to_y(wk1,out,ph)
+            call transpose_z_to_y(wk1, out, ph)
 #endif
-          call c2c_1m_y(out,isign,plan(isign,2))
-       end if
+            call c2c_1m_y(out, isign, plan(isign, 2))
+         end if
+
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_c2c, out, ph)
+         end if
+         call c2c_1m_x(out, isign, plan(isign, 1))
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_c2c,out,ph)
-       end if
-       call c2c_1m_x(out,isign,plan(isign,1))
-       
-    end if
+      end if
 
 #ifndef OVERWRITE
-    call fftw_free(wk1_p)
+      call fftw_free(wk1_p)
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-    return
-  end subroutine fft_3d_c2c
+      return
+   end subroutine fft_3d_c2c
 
-  
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D forward FFT - real to complex
+   ! 3D forward FFT - real to complex
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_r2c(in_r, out_c)
-    
-    implicit none
-    
-    real(mytype), dimension(:,:,:), intent(INOUT) :: in_r
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
+   subroutine fft_3d_r2c(in_r, out_c)
+
+      implicit none
+
+      real(mytype), dimension(:, :, :), intent(INOUT) :: in_r
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-    if (format==PHYSICAL_IN_X) then
-
-       ! ===== 1D FFTs in X =====
-       call r2c_1m_x(in_r,wk13)
-
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_x_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else
-          call c2c_1m_y(wk13,-1,plan(0,2))
-       end if
-
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,out_c,sp)
-       else
-          call transpose_y_to_z(wk13,out_c,sp)
-       end if
-       call c2c_1m_z(out_c,-1,plan(0,3))
-                
-    else if (format==PHYSICAL_IN_Z) then
-
-       ! ===== 1D FFTs in Z =====
-       call r2c_1m_z(in_r,wk13)
-
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_z_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else  ! out_c==wk2_r2c if 1D decomposition
-          call transpose_z_to_y(wk13,out_c,sp)
-          call c2c_1m_y(out_c,-1,plan(0,2))
-       end if
-
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,out_c,sp)
-       end if
-       call c2c_1m_x(out_c,-1,plan(0,1))
-
-    end if
+      if (format == PHYSICAL_IN_X) then
+
+         ! ===== 1D FFTs in X =====
+         call r2c_1m_x(in_r, wk13)
+
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_x_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else
+            call c2c_1m_y(wk13, -1, plan(0, 2))
+         end if
+
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, out_c, sp)
+         else
+            call transpose_y_to_z(wk13, out_c, sp)
+         end if
+         call c2c_1m_z(out_c, -1, plan(0, 3))
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! ===== 1D FFTs in Z =====
+         call r2c_1m_z(in_r, wk13)
+
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_z_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else  ! out_c==wk2_r2c if 1D decomposition
+            call transpose_z_to_y(wk13, out_c, sp)
+            call c2c_1m_y(out_c, -1, plan(0, 2))
+         end if
+
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, out_c, sp)
+         end if
+         call c2c_1m_x(out_c, -1, plan(0, 1))
+
+      end if
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-    return
-  end subroutine fft_3d_r2c
-  
-  
+      return
+   end subroutine fft_3d_r2c
+
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D inverse FFT - complex to real
+   ! 3D inverse FFT - complex to real
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2r(in_c, out_r)
-    
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
-    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
+   subroutine fft_3d_c2r(in_c, out_r)
+
+      implicit none
+
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
+      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
 #ifndef OVERWRITE
-    complex(mytype), pointer :: wk1(:,:,:)
-    integer(C_SIZE_T) :: sz
-    type(C_PTR) :: wk1_p
+      complex(mytype), pointer :: wk1(:, :, :)
+      integer(C_SIZE_T) :: sz
+      type(C_PTR) :: wk1_p
 
-    wk1_p = c_null_ptr ! Initialise to NULL pointer
+      wk1_p = c_null_ptr ! Initialise to NULL pointer
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in_c,1,plan(2,3))       
+         call c2c_1m_z(in_c, 1, plan(2, 3))
 #else
-       sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
-       wk1_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk1_p, wk1, [sp%zsz(1),sp%zsz(2),sp%zsz(3)])
-       wk1 = in_c
-       call c2c_1m_z(wk1,1,plan(2,3))
+         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+         wk1_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk1_p, wk1, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
+         wk1 = in_c
+         call c2c_1m_z(wk1, 1, plan(2, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-       call transpose_z_to_y(in_c,wk2_r2c,sp)
+         call transpose_z_to_y(in_c, wk2_r2c, sp)
 #else
-       call transpose_z_to_y(wk1,wk2_r2c,sp)
+         call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-       call c2c_1m_y(wk2_r2c,1,plan(2,2))
+         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,wk13,sp)
-          call c2r_1m_x(wk13,out_r)
-       else
-          call c2r_1m_x(wk2_r2c,out_r)
-       end if
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, wk13, sp)
+            call c2r_1m_x(wk13, out_r)
+         else
+            call c2r_1m_x(wk2_r2c, out_r)
+         end if
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in_c,1,plan(2,1))
+         call c2c_1m_x(in_c, 1, plan(2, 1))
 #else
-       sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
-       wk1_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk1_p, wk1, [sp%xsz(1),sp%xsz(2),sp%xsz(3)])
-       wk1 = in_c
-       call c2c_1m_x(wk1,1,plan(2,1))
+         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+         wk1_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk1_p, wk1, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
+         wk1 = in_c
+         call c2c_1m_x(wk1, 1, plan(2, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in_c,wk2_r2c,sp)
+            call transpose_x_to_y(in_c, wk2_r2c, sp)
 #else
-          call transpose_x_to_y(wk1,wk2_r2c,sp)
+            call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-          call c2c_1m_y(wk2_r2c,1,plan(2,2))
-       else  ! in_c==wk2_r2c if 1D decomposition
+            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+         else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-          call c2c_1m_y(in_c,1,plan(2,2))
+            call c2c_1m_y(in_c, 1, plan(2, 2))
 #else
-          call c2c_1m_y(wk1,1,plan(2,2))
+            call c2c_1m_y(wk1, 1, plan(2, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,wk13,sp)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, wk13, sp)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in_c,wk13,sp)
+            call transpose_y_to_z(in_c, wk13, sp)
 #else
-          call transpose_y_to_z(wk1,wk13,sp)
+            call transpose_y_to_z(wk1, wk13, sp)
 #endif
-       end if
-       call c2r_1m_z(wk13,out_r)
+         end if
+         call c2r_1m_z(wk13, out_r)
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    call fftw_free(wk1_p)
+      call fftw_free(wk1_p)
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-    return
-  end subroutine fft_3d_c2r
+      return
+   end subroutine fft_3d_c2r
 
-  
 end module decomp_2d_fft
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index d66425bd..8bab0ae3 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,320 +13,316 @@
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use glassman
+   use decomp_2d  ! 2D decomposition module
+   use glassman
 
-  implicit none
+   implicit none
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
-  ! engine-specific global variables
-  complex(mytype), allocatable, dimension(:) :: buf, scratch
+   ! engine-specific global variables
+   complex(mytype), allocatable, dimension(:) :: buf, scratch
 
-  ! common code used for all engines, including global variables, 
-  ! generic interface definitions and several subroutines
+   ! common code used for all engines, including global variables,
+   ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
-
-    implicit none
+   subroutine init_fft_engine
 
-    integer :: cbuf_size
+      implicit none
 
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the generic FFT engine *****'
-       write(*,*) ' '
-    end if
+      integer :: cbuf_size
 
-    cbuf_size = max(ph%xsz(1), ph%ysz(2))
-    cbuf_size = max(cbuf_size, ph%zsz(3))
-    allocate(buf(cbuf_size))
-    allocate(scratch(cbuf_size))
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the generic FFT engine *****'
+         write (*, *) ' '
+      end if
 
-    return
-  end subroutine init_fft_engine
+      cbuf_size = max(ph%xsz(1), ph%ysz(2))
+      cbuf_size = max(cbuf_size, ph%zsz(3))
+      allocate (buf(cbuf_size))
+      allocate (scratch(cbuf_size))
 
+      return
+   end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
-
-    implicit none
+   subroutine finalize_fft_engine
 
-    if (allocated(buf)) deallocate(buf)
-    if (allocated(scratch)) deallocate(scratch)
+      implicit none
 
-    return
-  end subroutine finalize_fft_engine
+      if (allocated(buf)) deallocate (buf)
+      if (allocated(scratch)) deallocate (scratch)
 
+      return
+   end subroutine finalize_fft_engine
 
-  ! Following routines calculate multiple one-dimensional FFTs to form 
-  ! the basis of three-dimensional FFTs.
+   ! Following routines calculate multiple one-dimensional FFTs to form
+   ! the basis of three-dimensional FFTs.
 
-  ! c2c transform, multiple 1D FFTs in x direction
-  subroutine c2c_1m_x(inout, isign, decomp)
+   ! c2c transform, multiple 1D FFTs in x direction
+   subroutine c2c_1m_x(inout, isign, decomp)
 
-    !$acc routine(spcfft) seq
+      !$acc routine(spcfft) seq
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer :: i,j,k
+      integer :: i, j, k
 
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do k=1,decomp%xsz(3)
-       do j=1,decomp%xsz(2)
-          do i=1,decomp%xsz(1)
-             buf(i) = inout(i,j,k)
-          end do
-          call spcfft(buf,decomp%xsz(1),isign,scratch)
-          do i=1,decomp%xsz(1)
-             inout(i,j,k) = buf(i)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do k = 1, decomp%xsz(3)
+         do j = 1, decomp%xsz(2)
+            do i = 1, decomp%xsz(1)
+               buf(i) = inout(i, j, k)
+            end do
+            call spcfft(buf, decomp%xsz(1), isign, scratch)
+            do i = 1, decomp%xsz(1)
+               inout(i, j, k) = buf(i)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
 
-    return
+      return
 
-  end subroutine c2c_1m_x
+   end subroutine c2c_1m_x
 
-  ! c2c transform, multiple 1D FFTs in y direction
-  subroutine c2c_1m_y(inout, isign, decomp)
+   ! c2c transform, multiple 1D FFTs in y direction
+   subroutine c2c_1m_y(inout, isign, decomp)
 
-    !$acc routine(spcfft) seq
+      !$acc routine(spcfft) seq
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer :: i,j,k
+      integer :: i, j, k
 
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do k=1,decomp%ysz(3)
-       do i=1,decomp%ysz(1)
-          do j=1,decomp%ysz(2)
-             buf(j) = inout(i,j,k)
-          end do
-          call spcfft(buf,decomp%ysz(2),isign,scratch)
-          do j=1,decomp%ysz(2)
-             inout(i,j,k) = buf(j)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do k = 1, decomp%ysz(3)
+         do i = 1, decomp%ysz(1)
+            do j = 1, decomp%ysz(2)
+               buf(j) = inout(i, j, k)
+            end do
+            call spcfft(buf, decomp%ysz(2), isign, scratch)
+            do j = 1, decomp%ysz(2)
+               inout(i, j, k) = buf(j)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
 
-    return
+      return
 
-  end subroutine c2c_1m_y
+   end subroutine c2c_1m_y
 
-  ! c2c transform, multiple 1D FFTs in z direction
-  subroutine c2c_1m_z(inout, isign, decomp)
+   ! c2c transform, multiple 1D FFTs in z direction
+   subroutine c2c_1m_z(inout, isign, decomp)
 
-    !$acc routine(spcfft) seq
+      !$acc routine(spcfft) seq
+
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer :: i, j, k
 
-    integer :: i,j,k
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do j = 1, decomp%zsz(2)
+         do i = 1, decomp%zsz(1)
+            do k = 1, decomp%zsz(3)
+               buf(k) = inout(i, j, k)
+            end do
+            call spcfft(buf, decomp%zsz(3), isign, scratch)
+            do k = 1, decomp%zsz(3)
+               inout(i, j, k) = buf(k)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
 
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do j=1,decomp%zsz(2)
-       do i=1,decomp%zsz(1)
-          do k=1,decomp%zsz(3)
-             buf(k) = inout(i,j,k)
-          end do
-          call spcfft(buf,decomp%zsz(3),isign,scratch)
-          do k=1,decomp%zsz(3)
-             inout(i,j,k) = buf(k)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
+      return
 
-    return
+   end subroutine c2c_1m_z
 
-  end subroutine c2c_1m_z
-
-  ! r2c transform, multiple 1D FFTs in x direction
-  subroutine r2c_1m_x(input, output)
-
-    !$acc routine(spcfft) seq
-
-    implicit none
-
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
-
-    integer :: i,j,k, s1,s2,s3, d1
-
-    s1 = size(input,1)
-    s2 = size(input,2)
-    s3 = size(input,3)
-    d1 = size(output,1)
-
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do k=1,s3
-       do j=1,s2
-          ! Glassman's FFT is c2c only, 
-          ! needing some pre- and post-processing for r2c
-          ! pack real input in complex storage
-          do i=1,s1
-             buf(i) = cmplx(input(i,j,k),0._mytype, kind=mytype)
-          end do
-          call spcfft(buf,s1,-1,scratch)
-          ! note d1 ~ s1/2+1
-          ! simply drop the redundant part of the complex output
-          do i=1,d1
-             output(i,j,k) = buf(i)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
-
-    return
-
-  end subroutine r2c_1m_x
-
-  ! r2c transform, multiple 1D FFTs in z direction
-  subroutine r2c_1m_z(input, output)
-
-    !$acc routine(spcfft) seq
-
-    implicit none
-
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
-
-    integer :: i,j,k, s1,s2,s3, d3
-
-    s1 = size(input,1)
-    s2 = size(input,2)
-    s3 = size(input,3)
-    d3 = size(output,3)
-
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do j=1,s2
-       do i=1,s1
-          ! Glassman's FFT is c2c only, 
-          ! needing some pre- and post-processing for r2c
-          ! pack real input in complex storage
-          do k=1,s3
-             buf(k) = cmplx(input(i,j,k),0._mytype, kind=mytype)
-          end do
-          call spcfft(buf,s3,-1,scratch)
-          ! note d3 ~ s3/2+1
-          ! simply drop the redundant part of the complex output
-          do k=1,d3
-             output(i,j,k) = buf(k)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
-
-    return
-
-  end subroutine r2c_1m_z
-
-  ! c2r transform, multiple 1D FFTs in x direction
-  subroutine c2r_1m_x(input, output)
-
-    !$acc routine(spcfft) seq
-
-    implicit none
-
-    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
-
-    integer :: i,j,k, d1,d2,d3
-
-    d1 = size(output,1)
-    d2 = size(output,2)
-    d3 = size(output,3)
-
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do k=1,d3
-       do j=1,d2
-          ! Glassman's FFT is c2c only, 
-          ! needing some pre- and post-processing for c2r
-          do i=1,d1/2+1
-             buf(i) = input(i,j,k)
-          end do
-          ! expanding to a full-size complex array
-          ! For odd N, the storage is:
-          !  1, 2, ...... N/2+1   integer division rounded down
-          !     N, ...... N/2+2   => a(i) is conjugate of a(N+2-i)
-          ! For even N, the storage is:
-          !  1, 2, ...... N/2  , N/2+1
-          !     N, ...... N/2+2  again a(i) conjugate of a(N+2-i)
-          do i=d1/2+2,d1
-             buf(i) =  conjg(buf(d1+2-i))
-          end do
-          call spcfft(buf,d1,1,scratch)
-          do i=1,d1
-             ! simply drop imaginary part
-             output(i,j,k) = real(buf(i), kind=mytype)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
-
-    return
-
-  end subroutine c2r_1m_x
-
-  ! c2r transform, multiple 1D FFTs in z direction
-  subroutine c2r_1m_z(input, output)
-
-    !$acc routine(spcfft) seq
-
-    implicit none
-
-    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
-
-    integer :: i,j,k, d1,d2,d3
-
-    d1 = size(output,1)
-    d2 = size(output,2)
-    d3 = size(output,3)
-
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do j=1,d2
-       do i=1,d1
-          do k=1,d3/2+1
-             buf(k) = input(i,j,k)
-          end do
-          do k=d3/2+2,d3
-             buf(k) =  conjg(buf(d3+2-k))
-          end do
-          call spcfft(buf,d3,1,scratch)
-          do k=1,d3
-             output(i,j,k) = real(buf(k), kind=mytype)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
-
-    return
-
-  end subroutine c2r_1m_z
+   ! r2c transform, multiple 1D FFTs in x direction
+   subroutine r2c_1m_x(input, output)
 
+      !$acc routine(spcfft) seq
+
+      implicit none
+
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+
+      integer :: i, j, k, s1, s2, s3, d1
+
+      s1 = size(input, 1)
+      s2 = size(input, 2)
+      s3 = size(input, 3)
+      d1 = size(output, 1)
+
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do k = 1, s3
+         do j = 1, s2
+            ! Glassman's FFT is c2c only,
+            ! needing some pre- and post-processing for r2c
+            ! pack real input in complex storage
+            do i = 1, s1
+               buf(i) = cmplx(input(i, j, k), 0._mytype, kind=mytype)
+            end do
+            call spcfft(buf, s1, -1, scratch)
+            ! note d1 ~ s1/2+1
+            ! simply drop the redundant part of the complex output
+            do i = 1, d1
+               output(i, j, k) = buf(i)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
+
+      return
+
+   end subroutine r2c_1m_x
+
+   ! r2c transform, multiple 1D FFTs in z direction
+   subroutine r2c_1m_z(input, output)
+
+      !$acc routine(spcfft) seq
+
+      implicit none
+
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+
+      integer :: i, j, k, s1, s2, s3, d3
+
+      s1 = size(input, 1)
+      s2 = size(input, 2)
+      s3 = size(input, 3)
+      d3 = size(output, 3)
+
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do j = 1, s2
+         do i = 1, s1
+            ! Glassman's FFT is c2c only,
+            ! needing some pre- and post-processing for r2c
+            ! pack real input in complex storage
+            do k = 1, s3
+               buf(k) = cmplx(input(i, j, k), 0._mytype, kind=mytype)
+            end do
+            call spcfft(buf, s3, -1, scratch)
+            ! note d3 ~ s3/2+1
+            ! simply drop the redundant part of the complex output
+            do k = 1, d3
+               output(i, j, k) = buf(k)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
+
+      return
+
+   end subroutine r2c_1m_z
+
+   ! c2r transform, multiple 1D FFTs in x direction
+   subroutine c2r_1m_x(input, output)
+
+      !$acc routine(spcfft) seq
+
+      implicit none
+
+      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
+
+      integer :: i, j, k, d1, d2, d3
+
+      d1 = size(output, 1)
+      d2 = size(output, 2)
+      d3 = size(output, 3)
+
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do k = 1, d3
+         do j = 1, d2
+            ! Glassman's FFT is c2c only,
+            ! needing some pre- and post-processing for c2r
+            do i = 1, d1/2 + 1
+               buf(i) = input(i, j, k)
+            end do
+            ! expanding to a full-size complex array
+            ! For odd N, the storage is:
+            !  1, 2, ...... N/2+1   integer division rounded down
+            !     N, ...... N/2+2   => a(i) is conjugate of a(N+2-i)
+            ! For even N, the storage is:
+            !  1, 2, ...... N/2  , N/2+1
+            !     N, ...... N/2+2  again a(i) conjugate of a(N+2-i)
+            do i = d1/2 + 2, d1
+               buf(i) = conjg(buf(d1 + 2 - i))
+            end do
+            call spcfft(buf, d1, 1, scratch)
+            do i = 1, d1
+               ! simply drop imaginary part
+               output(i, j, k) = real(buf(i), kind=mytype)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
+
+      return
+
+   end subroutine c2r_1m_x
+
+   ! c2r transform, multiple 1D FFTs in z direction
+   subroutine c2r_1m_z(input, output)
+
+      !$acc routine(spcfft) seq
+
+      implicit none
+
+      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
+
+      integer :: i, j, k, d1, d2, d3
+
+      d1 = size(output, 1)
+      d2 = size(output, 2)
+      d3 = size(output, 3)
+
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do j = 1, d2
+         do i = 1, d1
+            do k = 1, d3/2 + 1
+               buf(k) = input(i, j, k)
+            end do
+            do k = d3/2 + 2, d3
+               buf(k) = conjg(buf(d3 + 2 - k))
+            end do
+            call spcfft(buf, d3, 1, scratch)
+            do k = 1, d3
+               output(i, j, k) = real(buf(k), kind=mytype)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
+
+      return
+
+   end subroutine c2r_1m_z
 
 #include "fft_common_3d.f90"
 
-
 end module decomp_2d_fft
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index b50120aa..cebed05c 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,666 +13,660 @@
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use MKL_DFTI   ! MKL FFT module
+   use decomp_2d  ! 2D decomposition module
+   use MKL_DFTI   ! MKL FFT module
 
-  implicit none
+   implicit none
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
-  ! engine-specific global variables
+   ! engine-specific global variables
 
-  ! Descriptors for MKL FFT, one for each set of 1D FFTs
-  !  for c2c transforms
-  type(DFTI_DESCRIPTOR), pointer :: c2c_x, c2c_y, c2c_z     
-  !  for r2c/c2r transforms, PHYSICAL_IN_X
-  type(DFTI_DESCRIPTOR), pointer :: r2c_x, c2c_y2, c2c_z2, c2r_x
-  !  for r2c/c2r transforms, PHYSICAL_IN_Z
-  type(DFTI_DESCRIPTOR), pointer :: r2c_z, c2c_x2, c2r_z
+   ! Descriptors for MKL FFT, one for each set of 1D FFTs
+   !  for c2c transforms
+   type(DFTI_DESCRIPTOR), pointer :: c2c_x, c2c_y, c2c_z
+   !  for r2c/c2r transforms, PHYSICAL_IN_X
+   type(DFTI_DESCRIPTOR), pointer :: r2c_x, c2c_y2, c2c_z2, c2r_x
+   !  for r2c/c2r transforms, PHYSICAL_IN_Z
+   type(DFTI_DESCRIPTOR), pointer :: r2c_z, c2c_x2, c2r_z
 
-  ! common code used for all engines, including global variables, 
-  ! generic interface definitions and several subroutines
+   ! common code used for all engines, including global variables,
+   ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
+   subroutine init_fft_engine
 
-    implicit none
+      implicit none
 
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the MKL engine *****'
-       write(*,*) ' '
-    end if
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the MKL engine *****'
+         write (*, *) ' '
+      end if
 #ifdef OVERWRITE
-    call decomp_2d_warning(0, "MKL FFT engine does not support the OVERWRITE flag.")
+      call decomp_2d_warning(0, "MKL FFT engine does not support the OVERWRITE flag.")
 #endif
 
-    ! For C2C transforms
-    call c2c_1m_x_plan(c2c_x, ph)
-    call c2c_1m_y_plan(c2c_y, ph)
-    call c2c_1m_z_plan(c2c_z, ph)
-
-    ! For R2C/C2R tranfroms with physical space in X-pencil
-    if (format == PHYSICAL_IN_X) then
-       call r2c_1m_x_plan(r2c_x, ph, sp, -1)
-       call c2c_1m_y_plan(c2c_y2, sp)
-       call c2c_1m_z_plan(c2c_z2, sp)
-       call r2c_1m_x_plan(c2r_x, ph, sp,  1)
+      ! For C2C transforms
+      call c2c_1m_x_plan(c2c_x, ph)
+      call c2c_1m_y_plan(c2c_y, ph)
+      call c2c_1m_z_plan(c2c_z, ph)
 
-       ! For R2C/C2R tranfroms with physical space in Z-pencil
-    else if (format == PHYSICAL_IN_Z) then
-       call r2c_1m_z_plan(r2c_z, ph, sp, -1)
-       call c2c_1m_y_plan(c2c_y2, sp)
-       call c2c_1m_x_plan(c2c_x2, sp)
-       call r2c_1m_z_plan(c2r_z, ph, sp,  1)
-    end if
+      ! For R2C/C2R tranfroms with physical space in X-pencil
+      if (format == PHYSICAL_IN_X) then
+         call r2c_1m_x_plan(r2c_x, ph, sp, -1)
+         call c2c_1m_y_plan(c2c_y2, sp)
+         call c2c_1m_z_plan(c2c_z2, sp)
+         call r2c_1m_x_plan(c2r_x, ph, sp, 1)
 
-    return
-  end subroutine init_fft_engine
+         ! For R2C/C2R tranfroms with physical space in Z-pencil
+      else if (format == PHYSICAL_IN_Z) then
+         call r2c_1m_z_plan(r2c_z, ph, sp, -1)
+         call c2c_1m_y_plan(c2c_y2, sp)
+         call c2c_1m_x_plan(c2c_x2, sp)
+         call r2c_1m_z_plan(c2r_z, ph, sp, 1)
+      end if
 
+      return
+   end subroutine init_fft_engine
 
-  ! Return an MKL plan for multiple 1D c2c FFTs in X direction
-  subroutine c2c_1m_x_plan(desc, decomp)
+   ! Return an MKL plan for multiple 1D c2c FFTs in X direction
+   subroutine c2c_1m_x_plan(desc, decomp)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer :: status
+      integer :: status
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_COMPLEX, 1, decomp%xsz(1))
-#else       
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_COMPLEX, 1, decomp%xsz(1))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_COMPLEX, 1, decomp%xsz(1))
+#else
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_COMPLEX, 1, decomp%xsz(1))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-         decomp%xsz(2)*decomp%xsz(3))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, decomp%xsz(1))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, decomp%xsz(1))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+                            decomp%xsz(2)*decomp%xsz(3))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, decomp%xsz(1))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, decomp%xsz(1))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
 
-    return
-  end subroutine c2c_1m_x_plan
+      return
+   end subroutine c2c_1m_x_plan
 
-  ! Return an MKL plan for multiple 1D c2c FFTs in Y direction
-  subroutine c2c_1m_y_plan(desc, decomp)
+   ! Return an MKL plan for multiple 1D c2c FFTs in Y direction
+   subroutine c2c_1m_y_plan(desc, decomp)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer :: status, strides(2)
+      integer :: status, strides(2)
 
-    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-    ! done one Z-plane at a time. So plan for 2D data sets here.
+      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+      ! done one Z-plane at a time. So plan for 2D data sets here.
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_COMPLEX, 1, decomp%ysz(2))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_COMPLEX, 1, decomp%ysz(2))
 #else
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_COMPLEX, 1, decomp%ysz(2))
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_COMPLEX, 1, decomp%ysz(2))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, decomp%ysz(1))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    strides(1) = 0
-    strides(2) = decomp%ysz(1)
-    status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-    return
-  end subroutine c2c_1m_y_plan
-
-  ! Return an MKL plan for multiple 1D c2c FFTs in Z direction
-  subroutine c2c_1m_z_plan(desc, decomp)
-
-    implicit none
-
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-
-    integer :: status, strides(2)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, decomp%ysz(1))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      strides(1) = 0
+      strides(2) = decomp%ysz(1)
+      status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+      return
+   end subroutine c2c_1m_y_plan
+
+   ! Return an MKL plan for multiple 1D c2c FFTs in Z direction
+   subroutine c2c_1m_z_plan(desc, decomp)
+
+      implicit none
+
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+
+      integer :: status, strides(2)
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_COMPLEX, 1, decomp%zsz(3))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_COMPLEX, 1, decomp%zsz(3))
 #else
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_COMPLEX, 1, decomp%zsz(3))
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_COMPLEX, 1, decomp%zsz(3))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-         decomp%zsz(1)*decomp%zsz(2))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    strides(1) = 0
-    strides(2) = decomp%zsz(1)*decomp%zsz(2)
-    status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-    return
-  end subroutine c2c_1m_z_plan
-
-  ! Return an MKL plan for multiple 1D r2c FFTs in X direction
-  subroutine r2c_1m_x_plan(desc, decomp_ph, decomp_sp, direction)
-
-    implicit none
-
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
-    integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
-
-    integer :: status
-
-    ! c2r and r2c plans are almost the same, just swap input/output
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+                            decomp%zsz(1)*decomp%zsz(2))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      strides(1) = 0
+      strides(2) = decomp%zsz(1)*decomp%zsz(2)
+      status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+      return
+   end subroutine c2c_1m_z_plan
+
+   ! Return an MKL plan for multiple 1D r2c FFTs in X direction
+   subroutine r2c_1m_x_plan(desc, decomp_ph, decomp_sp, direction)
+
+      implicit none
+
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
+      integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
+
+      integer :: status
+
+      ! c2r and r2c plans are almost the same, just swap input/output
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_REAL, 1, decomp_ph%xsz(1))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_REAL, 1, decomp_ph%xsz(1))
 #else
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_REAL, 1, decomp_ph%xsz(1))
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_REAL, 1, decomp_ph%xsz(1))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
-         DFTI_COMPLEX_COMPLEX)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    if (direction == -1) then  ! r2c
-       status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
-            decomp_ph%xsz(1))
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
-            decomp_sp%xsz(1))
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    else if (direction == 1) then  ! c2r
-       status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
-            decomp_sp%xsz(1))
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
-            decomp_ph%xsz(1))
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    end if
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-    return
-  end subroutine r2c_1m_x_plan
-
-  ! Return an MKL plan for multiple 1D r2c FFTs in Z direction
-  subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
-
-    implicit none
-
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
-    integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
-
-    integer :: status, strides(2)
-
-    ! c2r and r2c plans are almost the same, just swap input/output
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+                            decomp_ph%xsz(2)*decomp_ph%xsz(3))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
+                            DFTI_COMPLEX_COMPLEX)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      if (direction == -1) then  ! r2c
+         status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
+                               decomp_ph%xsz(1))
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+         status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
+                               decomp_sp%xsz(1))
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      else if (direction == 1) then  ! c2r
+         status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
+                               decomp_sp%xsz(1))
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+         status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
+                               decomp_ph%xsz(1))
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      end if
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+      return
+   end subroutine r2c_1m_x_plan
+
+   ! Return an MKL plan for multiple 1D r2c FFTs in Z direction
+   subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
+
+      implicit none
+
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
+      integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
+
+      integer :: status, strides(2)
+
+      ! c2r and r2c plans are almost the same, just swap input/output
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_REAL, 1, decomp_ph%zsz(3))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_REAL, 1, decomp_ph%zsz(3))
 #else
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_REAL, 1, decomp_ph%zsz(3))
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_REAL, 1, decomp_ph%zsz(3))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
-         DFTI_COMPLEX_COMPLEX)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    strides(1) = 0
-    strides(2) = decomp_ph%zsz(1)*decomp_ph%zsz(2)
-    if (direction == -1) then
-       status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-    else if (direction == 1) then
-       status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-    end if
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    strides(2) = decomp_sp%zsz(1)*decomp_sp%zsz(2)
-    if (direction == -1) then
-       status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-    else if (direction == 1) then
-       status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-    end if
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-    return
-  end subroutine r2c_1m_z_plan
-
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+                            decomp_ph%zsz(1)*decomp_ph%zsz(2))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
+                            DFTI_COMPLEX_COMPLEX)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      strides(1) = 0
+      strides(2) = decomp_ph%zsz(1)*decomp_ph%zsz(2)
+      if (direction == -1) then
+         status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+      else if (direction == 1) then
+         status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+      end if
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      strides(2) = decomp_sp%zsz(1)*decomp_sp%zsz(2)
+      if (direction == -1) then
+         status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+      else if (direction == 1) then
+         status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+      end if
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+      return
+   end subroutine r2c_1m_z_plan
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
-
-    implicit none
-
-    integer :: status
-
-    status = DftiFreeDescriptor(c2c_x)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    status = DftiFreeDescriptor(c2c_y)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    status = DftiFreeDescriptor(c2c_z)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    if (format==PHYSICAL_IN_X) then
-       status = DftiFreeDescriptor(r2c_x)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-       status = DftiFreeDescriptor(c2c_z2)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-       status = DftiFreeDescriptor(c2r_x)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    else if (format==PHYSICAL_IN_Z) then
-       status = DftiFreeDescriptor(r2c_z)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-       status = DftiFreeDescriptor(c2c_x2)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-       status = DftiFreeDescriptor(c2r_z)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    end if
-    status = DftiFreeDescriptor(c2c_y2)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-
-    return
-  end subroutine finalize_fft_engine
-
+   subroutine finalize_fft_engine
+
+      implicit none
+
+      integer :: status
+
+      status = DftiFreeDescriptor(c2c_x)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      status = DftiFreeDescriptor(c2c_y)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      status = DftiFreeDescriptor(c2c_z)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      if (format == PHYSICAL_IN_X) then
+         status = DftiFreeDescriptor(r2c_x)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+         status = DftiFreeDescriptor(c2c_z2)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+         status = DftiFreeDescriptor(c2r_x)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      else if (format == PHYSICAL_IN_Z) then
+         status = DftiFreeDescriptor(r2c_z)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+         status = DftiFreeDescriptor(c2c_x2)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+         status = DftiFreeDescriptor(c2r_z)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      end if
+      status = DftiFreeDescriptor(c2c_y2)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+
+      return
+   end subroutine finalize_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D FFT - complex to complex
+   ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2c(in, out, isign)
+   subroutine fft_3d_c2c(in, out, isign)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: in
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out
-    integer, intent(IN) :: isign   
+      complex(mytype), dimension(:, :, :), intent(IN) :: in
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out
+      integer, intent(IN) :: isign
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1,wk2,wk2b,wk3
-    integer :: k, status
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
+      integer :: k, status
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
-
-       ! ===== 1D FFTs in X =====
-       allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-       !       if (isign==DECOMP_2D_FFT_FORWARD) then
-       !          status = DftiComputeForward(c2c_x, in(:,1,1), wk1(:,1,1))
-       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-       !          status = DftiComputeBackward(c2c_x, in(:,1,1), wk1(:,1,1))
-       !       end if
-       status = wrapper_c2c(c2c_x, in, wk1, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-       ! ===== Swap X --> Y =====
-       allocate (wk2(ph%ysz(1),ph%ysz(2),ph%ysz(3)))
-       call transpose_x_to_y(wk1,wk2,ph)
-
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(ph%ysz(1),ph%ysz(2),ph%ysz(3))) 
-       do k=1,ph%xsz(3) ! one Z-plane at a time
-          !          if (isign==DECOMP_2D_FFT_FORWARD) then
-          !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-          !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
-          !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-          !          end if
-          status = wrapper_c2c(c2c_y, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
-
-       ! ===== Swap Y --> Z =====
-       allocate (wk3(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-       call transpose_y_to_z(wk2b,wk3,ph)
-
-       ! ===== 1D FFTs in Z =====
-       !       if (isign==DECOMP_2D_FFT_FORWARD) then
-       !          status = DftiComputeForward(c2c_z, wk3(:,1,1), out(:,1,1))
-       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-       !          status = DftiComputeBackward(c2c_z, wk3(:,1,1), out(:,1,1))
-       !       end if
-       status = wrapper_c2c(c2c_z, wk3, out, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-         .OR. & 
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
-
-       ! ===== 1D FFTs in Z =====
-       allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-       !       if (isign==DECOMP_2D_FFT_FORWARD) then
-       !          status = DftiComputeForward(c2c_z, in(:,1,1), wk1(:,1,1))
-       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-       !          status = DftiComputeBackward(c2c_z, in(:,1,1), wk1(:,1,1))
-       !       end if
-       status = wrapper_c2c(c2c_z, in, wk1, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-       ! ===== Swap Z --> Y =====
-       allocate (wk2(ph%ysz(1),ph%ysz(2),ph%ysz(3)))
-       call transpose_z_to_y(wk1,wk2,ph)
-
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(ph%ysz(1),ph%ysz(2),ph%ysz(3)))
-       do k=1,ph%xsz(3) ! one Z-plane at a time
-          !          if (isign==DECOMP_2D_FFT_FORWARD) then
-          !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-          !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
-          !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-          !          end if
-          status = wrapper_c2c(c2c_y, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
-
-       ! ===== Swap Y --> X =====
-       allocate (wk3(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-       call transpose_y_to_x(wk2b,wk3,ph)
-
-       ! ===== 1D FFTs in X =====
-       !       if (isign==DECOMP_2D_FFT_FORWARD) then
-       !          status = DftiComputeForward(c2c_x, wk3(:,1,1), out(:,1,1))
-       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-       !          status = DftiComputeBackward(c2c_x, wk3(:,1,1), out(:,1,1))
-       !       end if
-       status = wrapper_c2c(c2c_x, wk3, out, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-    end if
+      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
+
+         ! ===== 1D FFTs in X =====
+         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+         !       if (isign==DECOMP_2D_FFT_FORWARD) then
+         !          status = DftiComputeForward(c2c_x, in(:,1,1), wk1(:,1,1))
+         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+         !          status = DftiComputeBackward(c2c_x, in(:,1,1), wk1(:,1,1))
+         !       end if
+         status = wrapper_c2c(c2c_x, in, wk1, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+         ! ===== Swap X --> Y =====
+         allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+         call transpose_x_to_y(wk1, wk2, ph)
+
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+         do k = 1, ph%xsz(3) ! one Z-plane at a time
+            !          if (isign==DECOMP_2D_FFT_FORWARD) then
+            !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+            !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
+            !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+            !          end if
+            status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+
+         ! ===== Swap Y --> Z =====
+         allocate (wk3(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+         call transpose_y_to_z(wk2b, wk3, ph)
+
+         ! ===== 1D FFTs in Z =====
+         !       if (isign==DECOMP_2D_FFT_FORWARD) then
+         !          status = DftiComputeForward(c2c_z, wk3(:,1,1), out(:,1,1))
+         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+         !          status = DftiComputeBackward(c2c_z, wk3(:,1,1), out(:,1,1))
+         !       end if
+         status = wrapper_c2c(c2c_z, wk3, out, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+               .OR. &
+               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
+
+         ! ===== 1D FFTs in Z =====
+         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+         !       if (isign==DECOMP_2D_FFT_FORWARD) then
+         !          status = DftiComputeForward(c2c_z, in(:,1,1), wk1(:,1,1))
+         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+         !          status = DftiComputeBackward(c2c_z, in(:,1,1), wk1(:,1,1))
+         !       end if
+         status = wrapper_c2c(c2c_z, in, wk1, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+         ! ===== Swap Z --> Y =====
+         allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+         call transpose_z_to_y(wk1, wk2, ph)
+
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+         do k = 1, ph%xsz(3) ! one Z-plane at a time
+            !          if (isign==DECOMP_2D_FFT_FORWARD) then
+            !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+            !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
+            !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+            !          end if
+            status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+
+         ! ===== Swap Y --> X =====
+         allocate (wk3(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+         call transpose_y_to_x(wk2b, wk3, ph)
+
+         ! ===== 1D FFTs in X =====
+         !       if (isign==DECOMP_2D_FFT_FORWARD) then
+         !          status = DftiComputeForward(c2c_x, wk3(:,1,1), out(:,1,1))
+         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+         !          status = DftiComputeBackward(c2c_x, wk3(:,1,1), out(:,1,1))
+         !       end if
+         status = wrapper_c2c(c2c_x, wk3, out, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+      end if
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-    return
-  end subroutine fft_3d_c2c
-
+      return
+   end subroutine fft_3d_c2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D forward FFT - real to complex
+   ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_r2c(in_r, out_c)
+   subroutine fft_3d_r2c(in_r, out_c)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: in_r
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
+      real(mytype), dimension(:, :, :), intent(IN) :: in_r
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1,wk2,wk2b,wk3
-    integer :: k, status, isign
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
+      integer :: k, status, isign
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-    isign = DECOMP_2D_FFT_FORWARD
+      isign = DECOMP_2D_FFT_FORWARD
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in X =====
-       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       !       status = DftiComputeForward(r2c_x, in_r(:,1,1), wk1(:,1,1))       
-       status = wrapper_r2c(r2c_x, in_r, wk1)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
+         ! ===== 1D FFTs in X =====
+         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         !       status = DftiComputeForward(r2c_x, in_r(:,1,1), wk1(:,1,1))
+         status = wrapper_r2c(r2c_x, in_r, wk1)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
-       ! ===== Swap X --> Y =====
-       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       call transpose_x_to_y(wk1,wk2,sp)
+         ! ===== Swap X --> Y =====
+         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         call transpose_x_to_y(wk1, wk2, sp)
 
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       do k=1,sp%ysz(3)
-          !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         do k = 1, sp%ysz(3)
+            !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
 
-       ! ===== Swap Y --> Z =====
-       allocate (wk3(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       call transpose_y_to_z(wk2b,wk3,sp)
+         ! ===== Swap Y --> Z =====
+         allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         call transpose_y_to_z(wk2b, wk3, sp)
 
-       ! ===== 1D FFTs in Z =====
-       !       status = DftiComputeForward(c2c_z2, wk3(:,1,1), out_c(:,1,1))
-       status = wrapper_c2c(c2c_z2, wk3, out_c, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         ! ===== 1D FFTs in Z =====
+         !       status = DftiComputeForward(c2c_z2, wk3(:,1,1), out_c(:,1,1))
+         status = wrapper_c2c(c2c_z2, wk3, out_c, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in Z =====
-       allocate(wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       !       status = DftiComputeForward(r2c_z, in_r(:,1,1), wk1(:,1,1))
-       status = wrapper_r2c(r2c_z, in_r, wk1)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
+         ! ===== 1D FFTs in Z =====
+         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         !       status = DftiComputeForward(r2c_z, in_r(:,1,1), wk1(:,1,1))
+         status = wrapper_r2c(r2c_z, in_r, wk1)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
-       ! ===== Swap Z --> Y =====
-       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       call transpose_z_to_y(wk1,wk2,sp)
+         ! ===== Swap Z --> Y =====
+         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         call transpose_z_to_y(wk1, wk2, sp)
 
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       do k=1,sp%ysz(3)
-          !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         do k = 1, sp%ysz(3)
+            !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
 
-       ! ===== Swap Y --> X =====
-       allocate (wk3(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       call transpose_y_to_x(wk2b,wk3,sp)
+         ! ===== Swap Y --> X =====
+         allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         call transpose_y_to_x(wk2b, wk3, sp)
 
-       ! ===== 1D FFTs in X =====
-       !       status = DftiComputeForward(c2c_x2, wk3(:,1,1), out_c(:,1,1))
-       status = wrapper_c2c(c2c_x2, wk3, out_c, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         ! ===== 1D FFTs in X =====
+         !       status = DftiComputeForward(c2c_x2, wk3(:,1,1), out_c(:,1,1))
+         status = wrapper_c2c(c2c_x2, wk3, out_c, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-    end if
+      end if
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-    return
-  end subroutine fft_3d_r2c
-
+      return
+   end subroutine fft_3d_r2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D inverse FFT - complex to real
+   ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2r(in_c, out_r)
+   subroutine fft_3d_c2r(in_c, out_r)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: in_c
-    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
+      complex(mytype), dimension(:, :, :), intent(IN) :: in_c
+      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1,wk2,wk2b,wk3
-    integer :: k, status, isign
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
+      integer :: k, status, isign
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-    isign = DECOMP_2D_FFT_BACKWARD
+      isign = DECOMP_2D_FFT_BACKWARD
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in Z =====
-       allocate (wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       !       status = DftiComputeBackward(c2c_z2, in_c(:,1,1), wk1(:,1,1))
-       status = wrapper_c2c(c2c_z2, in_c, wk1, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         ! ===== 1D FFTs in Z =====
+         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         !       status = DftiComputeBackward(c2c_z2, in_c(:,1,1), wk1(:,1,1))
+         status = wrapper_c2c(c2c_z2, in_c, wk1, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-       ! ===== Swap Z --> Y =====
-       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       call transpose_z_to_y(wk1,wk2,sp)
+         ! ===== Swap Z --> Y =====
+         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         call transpose_z_to_y(wk1, wk2, sp)
 
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       do k=1,sp%ysz(3)
-          !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         do k = 1, sp%ysz(3)
+            !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
 
-       ! ===== Swap Y --> X =====
-       allocate (wk3(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       call transpose_y_to_x(wk2b,wk3,sp)
+         ! ===== Swap Y --> X =====
+         allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         call transpose_y_to_x(wk2b, wk3, sp)
 
-       ! ===== 1D FFTs in X =====
-       !       status = DftiComputeBackward(c2r_x, wk3(:,1,1), out_r(:,1,1))
-       status =  wrapper_c2r(c2r_x, wk3, out_r)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
+         ! ===== 1D FFTs in X =====
+         !       status = DftiComputeBackward(c2r_x, wk3(:,1,1), out_r(:,1,1))
+         status = wrapper_c2r(c2r_x, wk3, out_r)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in X =====
-       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       !       status = DftiComputeBackward(c2c_x2, in_c(:,1,1), wk1(:,1,1))
-       status = wrapper_c2c(c2c_x2, in_c, wk1, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         ! ===== 1D FFTs in X =====
+         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         !       status = DftiComputeBackward(c2c_x2, in_c(:,1,1), wk1(:,1,1))
+         status = wrapper_c2c(c2c_x2, in_c, wk1, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-       ! ===== Swap X --> Y =====
-       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       call transpose_x_to_y(wk1,wk2,sp)
+         ! ===== Swap X --> Y =====
+         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         call transpose_x_to_y(wk1, wk2, sp)
 
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       do k=1,sp%ysz(3)
-          !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         do k = 1, sp%ysz(3)
+            !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
 
-       ! ===== Swap Y --> Z =====
-       allocate (wk3(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       call transpose_y_to_z(wk2b,wk3,sp)
+         ! ===== Swap Y --> Z =====
+         allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         call transpose_y_to_z(wk2b, wk3, sp)
 
-       ! ===== 1D FFTs in Z =====
-       !       status = DftiComputeBackward(c2r_z, wk3(:,1,1), out_r(:,1,1))
-       status =  wrapper_c2r(c2r_z, wk3, out_r)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
+         ! ===== 1D FFTs in Z =====
+         !       status = DftiComputeBackward(c2r_z, wk3(:,1,1), out_r(:,1,1))
+         status = wrapper_c2r(c2r_z, wk3, out_r)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
 
-    end if
+      end if
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-    return
-  end subroutine fft_3d_c2r
-
+      return
+   end subroutine fft_3d_c2r
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Wrapper functions so that one can pass 3D arrays to DftiCompute
-  !  -- MKL accepts only 1D arrays as input/output for its multi-
-  !     dimensional FFTs.
-  !  -- Using EQUIVALENCE as suggested by MKL documents is impossible 
-  !     for allocated arrays, not to mention bad coding style
-  !  -- All code commented out above may well work but not safe. There 
-  !     is no guarantee that compiler wouldn't make copies of 1D arrays
-  !     (which would contain only one slice of the original 3D data) 
-  !     rather than referring to the same memory address, i.e. 3D array
-  !     A and 1D array A(:,1,1) may refer to different memory location.
-  !  -- Using the following wrappers is safe and standard conforming.
+   ! Wrapper functions so that one can pass 3D arrays to DftiCompute
+   !  -- MKL accepts only 1D arrays as input/output for its multi-
+   !     dimensional FFTs.
+   !  -- Using EQUIVALENCE as suggested by MKL documents is impossible
+   !     for allocated arrays, not to mention bad coding style
+   !  -- All code commented out above may well work but not safe. There
+   !     is no guarantee that compiler wouldn't make copies of 1D arrays
+   !     (which would contain only one slice of the original 3D data)
+   !     rather than referring to the same memory address, i.e. 3D array
+   !     A and 1D array A(:,1,1) may refer to different memory location.
+   !  -- Using the following wrappers is safe and standard conforming.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  integer function wrapper_c2c(desc, in, out, isign)
+   integer function wrapper_c2c(desc, in, out, isign)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    complex(mytype), dimension(*) :: in, out
-    integer :: isign, status
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      complex(mytype), dimension(*) :: in, out
+      integer :: isign, status
 
-    if (isign == DECOMP_2D_FFT_FORWARD) then
-       status = DftiComputeForward(desc, in, out)
-    else if (isign == DECOMP_2D_FFT_BACKWARD) then
-       status = DftiComputeBackward(desc, in, out)
-    end if
+      if (isign == DECOMP_2D_FFT_FORWARD) then
+         status = DftiComputeForward(desc, in, out)
+      else if (isign == DECOMP_2D_FFT_BACKWARD) then
+         status = DftiComputeBackward(desc, in, out)
+      end if
 
-    wrapper_c2c = status
+      wrapper_c2c = status
 
-    return
-  end function wrapper_c2c
+      return
+   end function wrapper_c2c
 
-  integer function wrapper_r2c(desc, in, out)
+   integer function wrapper_r2c(desc, in, out)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    real(mytype), dimension(*) :: in 
-    complex(mytype), dimension(*) :: out
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      real(mytype), dimension(*) :: in
+      complex(mytype), dimension(*) :: out
 
-    wrapper_r2c = DftiComputeForward(desc, in, out)
+      wrapper_r2c = DftiComputeForward(desc, in, out)
 
-    return
-  end function wrapper_r2c
+      return
+   end function wrapper_r2c
 
-  integer function wrapper_c2r(desc, in, out)
+   integer function wrapper_c2r(desc, in, out)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    complex(mytype), dimension(*) :: in 
-    real(mytype), dimension(*) :: out
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      complex(mytype), dimension(*) :: in
+      real(mytype), dimension(*) :: out
 
-    wrapper_c2r = DftiComputeBackward(desc, in, out)
+      wrapper_c2r = DftiComputeBackward(desc, in, out)
 
-    return
-  end function wrapper_c2r
+      return
+   end function wrapper_c2r
 
 end module decomp_2d_fft
diff --git a/src/glassman.f90 b/src/glassman.f90
index 89fcf41e..170fb905 100644
--- a/src/glassman.f90
+++ b/src/glassman.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,184 +9,181 @@
 !
 !=======================================================================
 
-! This module contains a few 'generic' FFT routines, making the 
+! This module contains a few 'generic' FFT routines, making the
 ! 2DECOMP&FFT library not dependent on any external libraries
 
 module glassman
 
-  use decomp_2d, only : mytype
+   use decomp_2d, only: mytype
 
-  implicit none
+   implicit none
 
 contains
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Following is a FFT implementation based on algorithm proposed by
-  ! Glassman, a general FFT algorithm supporting arbitrary input length.
-  !
-  ! W. E. Ferguson, Jr., "A simple derivation of Glassman general-n fast
-  ! Fourier transform," Comput. and Math. with Appls., vol. 8, no. 6, pp.
-  ! 401-411, 1982.
-  !
-  ! Original implemtation online at http://www.jjj.de/fft/fftpage.html
-  !
-  ! Updated  
-  !  -  to handle double-precision as well
-  !  -  unnecessary scaling code removed
+   ! Following is a FFT implementation based on algorithm proposed by
+   ! Glassman, a general FFT algorithm supporting arbitrary input length.
+   !
+   ! W. E. Ferguson, Jr., "A simple derivation of Glassman general-n fast
+   ! Fourier transform," Comput. and Math. with Appls., vol. 8, no. 6, pp.
+   ! 401-411, 1982.
+   !
+   ! Original implemtation online at http://www.jjj.de/fft/fftpage.html
+   !
+   ! Updated
+   !  -  to handle double-precision as well
+   !  -  unnecessary scaling code removed
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  PURE SUBROUTINE SPCFFT(U,N,ISIGN,WORK)
-
-    !$acc routine seq
-
-    IMPLICIT NONE
-
-    LOGICAL :: INU
-    INTEGER :: A,B,C,I
-    INTEGER, INTENT(IN) :: N, ISIGN
-    COMPLEX(mytype), INTENT(INOUT) :: U(*),WORK(*)
-
-    A = 1
-    B = N
-    C = 1
-    INU = .TRUE.
-
-    DO WHILE ( B .GT. 1 )
-       A = C * A
-       C = 2
-       DO WHILE ( MOD(B,C) .NE. 0 )
-          C = C + 1
-       END DO
-       B = B / C
-       IF ( INU ) THEN
-          CALL SPCPFT (A,B,C,U,WORK,ISIGN)
-       ELSE
-          CALL SPCPFT (A,B,C,WORK,U,ISIGN)
-       END IF
-       INU = ( .NOT. INU )
-    END DO
-
-    IF ( .NOT. INU ) THEN
-       DO I = 1, N
-          U(I) = WORK(I)
-       END DO
-    END IF
-
-    RETURN
-  END SUBROUTINE SPCFFT
-
-
-  PURE SUBROUTINE SPCPFT( A, B, C, UIN, UOUT, ISIGN )
-
-    !$acc routine seq
-
-    IMPLICIT NONE
-
-    INTEGER, INTENT(IN), VALUE :: A, B, C, ISIGN
-    INTEGER :: IA,IB,IC,JCR,JC
-
-    DOUBLE PRECISION :: ANGLE
-
-    COMPLEX(mytype), INTENT(IN) :: UIN(B,C,A)
-    COMPLEX(mytype), INTENT(OUT) :: UOUT(B,A,C)
-    COMPLEX(mytype) :: DELTA,OMEGA,SUM
-
-    ANGLE = 6.28318530717958_mytype / REAL( A * C, kind=mytype )
-    OMEGA = CMPLX( 1.0, 0.0, kind=mytype )
-
-    IF( ISIGN .EQ. 1 ) THEN
-       DELTA = CMPLX( COS(ANGLE), SIN(ANGLE), kind=mytype )
-    ELSE
-       DELTA = CMPLX( COS(ANGLE), -SIN(ANGLE), kind=mytype )
-    END IF
-
-    DO IC = 1, C
-       DO IA = 1, A
-          DO IB = 1, B
-             SUM = UIN( IB, C, IA )
-             DO JCR = 2, C
-                JC = C + 1 - JCR
-                SUM = UIN( IB, JC, IA ) + OMEGA * SUM
-             END DO
-             UOUT( IB, IA, IC ) = SUM
-          END DO
-          OMEGA = DELTA * OMEGA
-       END DO
-    END DO
-
-    RETURN
-  END SUBROUTINE SPCPFT
-
+   PURE SUBROUTINE SPCFFT(U, N, ISIGN, WORK)
+
+      !$acc routine seq
+
+      IMPLICIT NONE
+
+      LOGICAL :: INU
+      INTEGER :: A, B, C, I
+      INTEGER, INTENT(IN) :: N, ISIGN
+      COMPLEX(mytype), INTENT(INOUT) :: U(*), WORK(*)
+
+      A = 1
+      B = N
+      C = 1
+      INU = .TRUE.
+
+      DO WHILE (B .GT. 1)
+         A = C*A
+         C = 2
+         DO WHILE (MOD(B, C) .NE. 0)
+            C = C + 1
+         END DO
+         B = B/C
+         IF (INU) THEN
+            CALL SPCPFT(A, B, C, U, WORK, ISIGN)
+         ELSE
+            CALL SPCPFT(A, B, C, WORK, U, ISIGN)
+         END IF
+         INU = (.NOT. INU)
+      END DO
+
+      IF (.NOT. INU) THEN
+         DO I = 1, N
+            U(I) = WORK(I)
+         END DO
+      END IF
+
+      RETURN
+   END SUBROUTINE SPCFFT
+
+   PURE SUBROUTINE SPCPFT(A, B, C, UIN, UOUT, ISIGN)
+
+      !$acc routine seq
+
+      IMPLICIT NONE
+
+      INTEGER, INTENT(IN), VALUE :: A, B, C, ISIGN
+      INTEGER :: IA, IB, IC, JCR, JC
+
+      DOUBLE PRECISION :: ANGLE
+
+      COMPLEX(mytype), INTENT(IN) :: UIN(B, C, A)
+      COMPLEX(mytype), INTENT(OUT) :: UOUT(B, A, C)
+      COMPLEX(mytype) :: DELTA, OMEGA, SUM
+
+      ANGLE = 6.28318530717958_mytype/REAL(A*C, kind=mytype)
+      OMEGA = CMPLX(1.0, 0.0, kind=mytype)
+
+      IF (ISIGN .EQ. 1) THEN
+         DELTA = CMPLX(COS(ANGLE), SIN(ANGLE), kind=mytype)
+      ELSE
+         DELTA = CMPLX(COS(ANGLE), -SIN(ANGLE), kind=mytype)
+      END IF
+
+      DO IC = 1, C
+         DO IA = 1, A
+            DO IB = 1, B
+               SUM = UIN(IB, C, IA)
+               DO JCR = 2, C
+                  JC = C + 1 - JCR
+                  SUM = UIN(IB, JC, IA) + OMEGA*SUM
+               END DO
+               UOUT(IB, IA, IC) = SUM
+            END DO
+            OMEGA = DELTA*OMEGA
+         END DO
+      END DO
+
+      RETURN
+   END SUBROUTINE SPCPFT
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! A 3D real-to-complex routine implemented using the 1D FFT above
-  !   Input:   nx*ny*nz real numbers
-  !   Output:  (nx/2+1)*ny*nz complex numbers
-  ! Just like big FFT libraries (such as FFTW) do
+   ! A 3D real-to-complex routine implemented using the 1D FFT above
+   !   Input:   nx*ny*nz real numbers
+   !   Output:  (nx/2+1)*ny*nz complex numbers
+   ! Just like big FFT libraries (such as FFTW) do
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine glassman_3d_r2c(in_r,nx,ny,nz,out_c)
-
-    implicit none
-
-    integer, intent(IN) :: nx,ny,nz
-    real(mytype), dimension(nx,ny,nz) :: in_r
-    complex(mytype), dimension(nx/2+1,ny,nz) :: out_c
-
-    complex(mytype), allocatable, dimension(:) :: buf, scratch
-    integer :: maxsize, i,j,k
-
-    maxsize = max(nx, max(ny,nz))
-    allocate(buf(maxsize))
-    allocate(scratch(maxsize))
-
-    ! ===== 1D FFTs in X =====
-    do k=1,nz
-       do j=1,ny
-          ! Glassman's 1D FFT is c2c only, 
-          ! needing some pre- and post-processing for r2c
-          ! pack real input in complex storage
-          do i=1,nx
-             buf(i) = cmplx(in_r(i,j,k),0._mytype, kind=mytype)
-          end do
-          call spcfft(buf,nx,-1,scratch)
-          ! simply drop the redundant part of the complex output
-          do i=1,nx/2+1
-             out_c(i,j,k) = buf(i)
-          end do
-       end do
-    end do
-
-    ! ===== 1D FFTs in Y =====
-    do k=1,nz
-       do i=1,nx/2+1
-          do j=1,ny
-             buf(j) = out_c(i,j,k)
-          end do
-          call spcfft(buf,ny,-1,scratch)
-          do j=1,ny
-             out_c(i,j,k) = buf(j)
-          end do
-       end do
-    end do
-
-    ! ===== 1D FFTs in Z =====
-    do j=1,ny
-       do i=1,nx/2+1
-          do k=1,nz
-             buf(k) = out_c(i,j,k)
-          end do
-          call spcfft(buf,nz,-1,scratch)
-          do k=1,nz
-             out_c(i,j,k) = buf(k)
-          end do
-       end do
-    end do
-
-    deallocate(buf,scratch)
-
-    return
-  end subroutine glassman_3d_r2c
-
+   subroutine glassman_3d_r2c(in_r, nx, ny, nz, out_c)
+
+      implicit none
+
+      integer, intent(IN) :: nx, ny, nz
+      real(mytype), dimension(nx, ny, nz) :: in_r
+      complex(mytype), dimension(nx/2 + 1, ny, nz) :: out_c
+
+      complex(mytype), allocatable, dimension(:) :: buf, scratch
+      integer :: maxsize, i, j, k
+
+      maxsize = max(nx, max(ny, nz))
+      allocate (buf(maxsize))
+      allocate (scratch(maxsize))
+
+      ! ===== 1D FFTs in X =====
+      do k = 1, nz
+         do j = 1, ny
+            ! Glassman's 1D FFT is c2c only,
+            ! needing some pre- and post-processing for r2c
+            ! pack real input in complex storage
+            do i = 1, nx
+               buf(i) = cmplx(in_r(i, j, k), 0._mytype, kind=mytype)
+            end do
+            call spcfft(buf, nx, -1, scratch)
+            ! simply drop the redundant part of the complex output
+            do i = 1, nx/2 + 1
+               out_c(i, j, k) = buf(i)
+            end do
+         end do
+      end do
+
+      ! ===== 1D FFTs in Y =====
+      do k = 1, nz
+         do i = 1, nx/2 + 1
+            do j = 1, ny
+               buf(j) = out_c(i, j, k)
+            end do
+            call spcfft(buf, ny, -1, scratch)
+            do j = 1, ny
+               out_c(i, j, k) = buf(j)
+            end do
+         end do
+      end do
+
+      ! ===== 1D FFTs in Z =====
+      do j = 1, ny
+         do i = 1, nx/2 + 1
+            do k = 1, nz
+               buf(k) = out_c(i, j, k)
+            end do
+            call spcfft(buf, nz, -1, scratch)
+            do k = 1, nz
+               out_c(i, j, k) = buf(k)
+            end do
+         end do
+      end do
+
+      deallocate (buf, scratch)
+
+      return
+   end subroutine glassman_3d_r2c
 
 end module glassman
 
diff --git a/src/halo.f90 b/src/halo.f90
index ef324bf5..d0082f5c 100644
--- a/src/halo.f90
+++ b/src/halo.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -14,98 +14,98 @@
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   subroutine update_halo_real_short(in, out, level, opt_global, opt_pencil)
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: level      ! levels of halo cells required
-    real(mytype), dimension(:,:,:), intent(IN) :: in
-    real(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
-    logical, optional :: opt_global
-    integer, intent(in), optional :: opt_pencil
+     integer, intent(IN) :: level      ! levels of halo cells required
+     real(mytype), dimension(:, :, :), intent(IN) :: in
+     real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+     logical, optional :: opt_global
+     integer, intent(in), optional :: opt_pencil
 
-    call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
+     call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
 
   end subroutine update_halo_real_short
 
   subroutine update_halo_real(in, out, level, decomp, opt_global, opt_pencil)
 
-    implicit none
+     implicit none
+
+     integer, intent(IN) :: level      ! levels of halo cells required
+     real(mytype), dimension(:, :, :), intent(IN) :: in
+     real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+     TYPE(DECOMP_INFO), intent(in) :: decomp
+     logical, optional :: opt_global
+     integer, intent(in), optional :: opt_pencil
 
-    integer, intent(IN) :: level      ! levels of halo cells required
-    real(mytype), dimension(:,:,:), intent(IN) :: in    
-    real(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
-    TYPE(DECOMP_INFO), intent(in) :: decomp
-    logical, optional :: opt_global
-    integer, intent(in), optional :: opt_pencil
+     logical :: global
 
-    logical :: global
+     ! starting/ending index of array with halo cells
+     integer :: xs, ys, zs, xe, ye, ze
 
-    ! starting/ending index of array with halo cells
-    integer :: xs, ys, zs, xe, ye, ze
+     integer :: i, j, k, s1, s2, s3, ierror
+     integer :: data_type
 
-    integer :: i, j, k, s1, s2, s3, ierror
-    integer :: data_type
+     integer :: icount, ilength, ijump
+     integer :: halo12, halo21, halo31, halo32
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
 
-    integer :: icount, ilength, ijump 
-    integer :: halo12, halo21, halo31, halo32                 
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
+     integer :: ipencil
+     logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
 
-    integer :: ipencil
-    logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
-    
-    data_type = real_type
+     data_type = real_type
 
 #include "halo_common.f90"
 
-    return
+     return
   end subroutine update_halo_real
 
   subroutine update_halo_complex_short(in, out, level, opt_global, opt_pencil)
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: level      ! levels of halo cells required
-    complex(mytype), dimension(:,:,:), intent(IN) :: in
-    complex(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
-    logical, optional :: opt_global
-    integer, intent(in), optional :: opt_pencil
+     integer, intent(IN) :: level      ! levels of halo cells required
+     complex(mytype), dimension(:, :, :), intent(IN) :: in
+     complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+     logical, optional :: opt_global
+     integer, intent(in), optional :: opt_pencil
 
-    call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
+     call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
 
   end subroutine update_halo_complex_short
 
   subroutine update_halo_complex(in, out, level, decomp, opt_global, opt_pencil)
 
-    implicit none
+     implicit none
+
+     integer, intent(IN) :: level      ! levels of halo cells required
+     complex(mytype), dimension(:, :, :), intent(IN) :: in
+     complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+     TYPE(DECOMP_INFO), intent(in) :: decomp
+     logical, optional :: opt_global
+     integer, intent(in), optional :: opt_pencil
 
-    integer, intent(IN) :: level      ! levels of halo cells required
-    complex(mytype), dimension(:,:,:), intent(IN) :: in    
-    complex(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
-    TYPE(DECOMP_INFO), intent(in) :: decomp
-    logical, optional :: opt_global
-    integer, intent(in), optional :: opt_pencil
+     logical :: global
 
-    logical :: global
+     ! starting/ending index of array with halo cells
+     integer :: xs, ys, zs, xe, ye, ze
 
-    ! starting/ending index of array with halo cells
-    integer :: xs, ys, zs, xe, ye, ze
+     integer :: i, j, k, s1, s2, s3, ierror
+     integer :: data_type
 
-    integer :: i, j, k, s1, s2, s3, ierror
-    integer :: data_type
+     integer :: icount, ilength, ijump
+     integer :: halo12, halo21, halo31, halo32
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
 
-    integer :: icount, ilength, ijump 
-    integer :: halo12, halo21, halo31, halo32                 
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
+     integer :: ipencil
+     logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
 
-    integer :: ipencil
-    logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
-    
-    data_type = complex_type
+     data_type = complex_type
 
 #include "halo_common.f90"
 
-    return
+     return
   end subroutine update_halo_complex
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index 11859eae..13fa5d43 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,7 +9,7 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'update_halo_...' in halo.f90
 
     if (present(opt_global)) then
@@ -18,9 +18,9 @@
        global = .false.
     end if
 
-    s1 = size(in,1)
-    s2 = size(in,2)
-    s3 = size(in,3)
+    s1 = size(in, 1)
+    s2 = size(in, 2)
+    s3 = size(in, 3)
 
     if (present(opt_pencil)) then
        ipencil = opt_pencil
@@ -31,21 +31,21 @@
           if (first_call_x) then
              first_call_x = .false.
              call decomp_2d_warning(__FILE__, __LINE__, &
-                  0, "Deprecated interface - calling halo in X without explicit pencil")
+                                    0, "Deprecated interface - calling halo in X without explicit pencil")
           end if
        else if (s2 == decomp%ysz(2)) then
           ipencil = 2
           if (first_call_y) then
              first_call_y = .false.
              call decomp_2d_warning(__FILE__, __LINE__, &
-                  0, "Deprecated interface - calling halo in Y without explicit pencil")
+                                    0, "Deprecated interface - calling halo in Y without explicit pencil")
           end if
        else if (s3 == decomp%zsz(3)) then
           ipencil = 3
           if (first_call_z) then
              first_call_z = .false.
              call decomp_2d_warning(__FILE__, __LINE__, &
-                  0, "Deprecated interface - calling halo in Z without explicit pencil")
+                                    0, "Deprecated interface - calling halo in Z without explicit pencil")
           end if
        else
           ipencil = 0
@@ -56,23 +56,23 @@
     ! Calculate the starting index and ending index of output
     if (ipencil == 1) then  ! X-pencil input
        if (global) then
-          xs = decomp%xst(1) 
+          xs = decomp%xst(1)
           xe = decomp%xen(1)
           ys = decomp%xst(2) - level
           ye = decomp%xen(2) + level
           zs = decomp%xst(3) - level
           ze = decomp%xen(3) + level
        else
-          xs = 1 
+          xs = 1
           xe = s1
           ys = 1 - level
-          ye = s2 + level 
+          ye = s2 + level
           zs = 1 - level
           ze = s3 + level
        end if
     else if (ipencil == 2) then  ! Y-pencil input
        if (global) then
-          xs = decomp%yst(1) - level 
+          xs = decomp%yst(1) - level
           xe = decomp%yen(1) + level
           ys = decomp%yst(2)
           ye = decomp%yen(2)
@@ -88,7 +88,7 @@
        end if
     else if (ipencil == 3) then  ! Z-pencil input
        if (global) then
-          xs = decomp%zst(1) - level 
+          xs = decomp%zst(1) - level
           xe = decomp%zen(1) + level
           ys = decomp%zst(2) - level
           ye = decomp%zen(2) + level
@@ -109,49 +109,49 @@
        xs = 1; xe = 1
        ys = 1; ye = 1
        zs = 1; ze = 1
-       
+
        call decomp_2d_abort(__FILE__, __LINE__, 10, &
-            'Invalid data passed to update_halo')
+                            'Invalid data passed to update_halo')
     end if
-    allocate(out(xs:xe, ys:ye, zs:ze))
+    allocate (out(xs:xe, ys:ye, zs:ze))
 !    out = -1.0_mytype ! fill the halo for debugging
 
     ! copy input data to output
     if (global) then
        ! using global coordinate
-       ! note the input array passed in always has index starting from 1 
+       ! note the input array passed in always has index starting from 1
        ! need to work out the corresponding global index
        if (ipencil == 1) then
-          do k=decomp%xst(3),decomp%xen(3)
-             do j=decomp%xst(2),decomp%xen(2)
-                do i=1,s1  ! x all local
-                   out(i,j,k) = in(i,j-decomp%xst(2)+1,k-decomp%xst(3)+1)
+          do k = decomp%xst(3), decomp%xen(3)
+             do j = decomp%xst(2), decomp%xen(2)
+                do i = 1, s1  ! x all local
+                   out(i, j, k) = in(i, j - decomp%xst(2) + 1, k - decomp%xst(3) + 1)
                 end do
              end do
           end do
        else if (ipencil == 2) then
-          do k=decomp%yst(3),decomp%yen(3)
-             do j=1,s2  ! y all local
-                do i=decomp%yst(1),decomp%yen(1)
-                   out(i,j,k) = in(i-decomp%yst(1)+1,j,k-decomp%yst(3)+1)
+          do k = decomp%yst(3), decomp%yen(3)
+             do j = 1, s2  ! y all local
+                do i = decomp%yst(1), decomp%yen(1)
+                   out(i, j, k) = in(i - decomp%yst(1) + 1, j, k - decomp%yst(3) + 1)
                 end do
              end do
           end do
        else if (ipencil == 3) then
-          do k=1,s3  ! z all local
-             do j=decomp%zst(2),decomp%zen(2)
-                do i=decomp%zst(1),decomp%zen(1)
-                   out(i,j,k) = in(i-decomp%zst(1)+1,j-decomp%zst(2)+1,k)
+          do k = 1, s3  ! z all local
+             do j = decomp%zst(2), decomp%zen(2)
+                do i = decomp%zst(1), decomp%zen(1)
+                   out(i, j, k) = in(i - decomp%zst(1) + 1, j - decomp%zst(2) + 1, k)
                 end do
              end do
           end do
        end if
     else
        ! not using global coordinate
-       do k=1,s3
-          do j=1,s2
-             do i=1,s1
-                out(i,j,k) = in(i,j,k)
+       do k = 1, s3
+          do j = 1, s2
+             do i = 1, s1
+                out(i, j, k) = in(i, j, k)
              end do
           end do
        end do
@@ -159,19 +159,19 @@
 
     ! If needed, define MPI derived data type to pack halo data,
     ! then call MPI send/receive to exchange halo data
-    
+
     !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     ! X-pencil
     if (ipencil == 1) then
 
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'X-pencil input'
-          write(*,*) '=============='
-          write(*,*) 'Data on a y-z plane is shown'
-          write(*,*) 'Before halo exchange'
-          do j=ye,ys,-1
-             write(*,'(10F4.0)') (out(1,j,k),k=zs,ze)
+       if (nrank == 4) then
+          write (*, *) 'X-pencil input'
+          write (*, *) '=============='
+          write (*, *) 'Data on a y-z plane is shown'
+          write (*, *) 'Before halo exchange'
+          do j = ye, ys, -1
+             write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
           end do
        end if
 #endif
@@ -179,50 +179,50 @@
        ! *** east/west ***
        ! all data in local memory already, no halo exchange
 
-       ! *** north/south *** 
+       ! *** north/south ***
        tag_s = coord(1)
-       if (coord(1)==dims(1)-1 .AND. periodic_y) then
+       if (coord(1) == dims(1) - 1 .AND. periodic_y) then
           tag_n = 0
        else
           tag_n = coord(1) + 1
        end if
        icount = s3 + 2*level
-       ilength = level * s1
-       ijump = s1*(s2+2*level)
+       ilength = level*s1
+       ijump = s1*(s2 + 2*level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-            data_type, halo12, ierror)
+                            data_type, halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from south
-       call MPI_IRECV(out(xs,ys,zs), 1, halo12, &
-            neighbour(1,4), tag_s, DECOMP_2D_COMM_CART_X, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), 1, halo12, &
+                      neighbour(1, 4), tag_s, DECOMP_2D_COMM_CART_X, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from north
-       call MPI_IRECV(out(xs,ye-level+1,zs), 1, halo12, &
-            neighbour(1,3), tag_n, DECOMP_2D_COMM_CART_X, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xs, ye - level + 1, zs), 1, halo12, &
+                      neighbour(1, 3), tag_n, DECOMP_2D_COMM_CART_X, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to south
-       call MPI_ISSEND(out(xs,ys+level,zs), 1, halo12, &
-            neighbour(1,4), tag_s, DECOMP_2D_COMM_CART_X, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs, ys + level, zs), 1, halo12, &
+                       neighbour(1, 4), tag_s, DECOMP_2D_COMM_CART_X, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to north
-       call MPI_ISSEND(out(xs,ye-level-level+1,zs), 1, halo12, &
-            neighbour(1,3), tag_n, DECOMP_2D_COMM_CART_X, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xs, ye - level - level + 1, zs), 1, halo12, &
+                       neighbour(1, 3), tag_n, DECOMP_2D_COMM_CART_X, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
        call MPI_TYPE_FREE(halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'After exchange in Y'
-          do j=ye,ys,-1
-             write(*,'(10F4.0)') (out(1,j,k),k=zs,ze)
+       if (nrank == 4) then
+          write (*, *) 'After exchange in Y'
+          do j = ye, ys, -1
+             write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
           end do
        end if
 #endif
@@ -232,103 +232,103 @@
        ! all contiguous in memory, which can be sent/received using
        ! MPI directly
        tag_b = coord(2)
-       if (coord(2)==dims(2)-1 .AND. periodic_z) then
+       if (coord(2) == dims(2) - 1 .AND. periodic_z) then
           tag_t = 0
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s1 * (s2+2*level)) * level
+       icount = (s1*(s2 + 2*level))*level
        ! receive from bottom
-       call MPI_IRECV(out(xs,ys,zs), icount, data_type, &
-            neighbour(1,6), tag_b, DECOMP_2D_COMM_CART_X, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
+                      neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from top
-       call MPI_IRECV(out(xs,ys,ze-level+1), icount, data_type, &
-            neighbour(1,5), tag_t, DECOMP_2D_COMM_CART_X, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xs, ys, ze - level + 1), icount, data_type, &
+                      neighbour(1, 5), tag_t, DECOMP_2D_COMM_CART_X, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to bottom
-       call MPI_ISSEND(out(xs,ys,zs+level), icount, data_type, &
-            neighbour(1,6), tag_b, DECOMP_2D_COMM_CART_X, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs, ys, zs + level), icount, data_type, &
+                       neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to top
-       call MPI_ISSEND(out(xs,ys,ze-level-level+1), icount, data_type, &
-            neighbour(1,5), tag_t, DECOMP_2D_COMM_CART_X, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xs, ys, ze - level - level + 1), icount, data_type, &
+                       neighbour(1, 5), tag_t, DECOMP_2D_COMM_CART_X, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
-#ifdef HALO_DEBUG       
-       if (nrank==4) then
-          write(*,*) 'After exchange in Z'
-          do j=ye,ys,-1
-             write(*,'(10F4.0)') (out(1,j,k),k=zs,ze)
+#ifdef HALO_DEBUG
+       if (nrank == 4) then
+          write (*, *) 'After exchange in Z'
+          do j = ye, ys, -1
+             write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
           end do
        end if
-#endif       
+#endif
 
-    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   
-    ! Y-pencil   
+    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+       ! Y-pencil
     else if (ipencil == 2) then
 
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'Y-pencil input'
-          write(*,*) '=============='
-          write(*,*) 'Data on a x-z plane is shown'
-          write(*,*) 'Before halo exchange'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,1,k),k=zs,ze)
+       if (nrank == 4) then
+          write (*, *) 'Y-pencil input'
+          write (*, *) '=============='
+          write (*, *) 'Data on a x-z plane is shown'
+          write (*, *) 'Before halo exchange'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
           end do
        end if
 #endif
 
        ! *** east/west ***
        tag_w = coord(1)
-       if (coord(1)==dims(1)-1 .AND. periodic_x) then
+       if (coord(1) == dims(1) - 1 .AND. periodic_x) then
           tag_e = 0
        else
           tag_e = coord(1) + 1
        end if
-       icount = s2*(s3+2*level)
+       icount = s2*(s3 + 2*level)
        ilength = level
-       ijump = s1+2*level
+       ijump = s1 + 2*level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-            data_type, halo21, ierror)
+                            data_type, halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from west
-       call MPI_IRECV(out(xs,ys,zs), 1, halo21, &
-            neighbour(2,2), tag_w, DECOMP_2D_COMM_CART_Y, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), 1, halo21, &
+                      neighbour(2, 2), tag_w, DECOMP_2D_COMM_CART_Y, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from east
-       call MPI_IRECV(out(xe-level+1,ys,zs), 1, halo21, &
-            neighbour(2,1), tag_e, DECOMP_2D_COMM_CART_Y, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xe - level + 1, ys, zs), 1, halo21, &
+                      neighbour(2, 1), tag_e, DECOMP_2D_COMM_CART_Y, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to west
-       call MPI_ISSEND(out(xs+level,ys,zs), 1, halo21, &
-            neighbour(2,2), tag_w, DECOMP_2D_COMM_CART_Y, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs + level, ys, zs), 1, halo21, &
+                       neighbour(2, 2), tag_w, DECOMP_2D_COMM_CART_Y, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to east
-       call MPI_ISSEND(out(xe-level-level+1,ys,zs), 1, halo21, &
-            neighbour(2,1), tag_e, DECOMP_2D_COMM_CART_Y, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xe - level - level + 1, ys, zs), 1, halo21, &
+                       neighbour(2, 1), tag_e, DECOMP_2D_COMM_CART_Y, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
        call MPI_TYPE_FREE(halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-#ifdef HALO_DEBUG       
-       if (nrank==4) then
-          write(*,*) 'After exchange in X'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,1,k),k=zs,ze)
+#ifdef HALO_DEBUG
+       if (nrank == 4) then
+          write (*, *) 'After exchange in X'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
           end do
        end if
 #endif
@@ -341,150 +341,150 @@
        ! all contiguous in memory, which can be sent/received using
        ! MPI directly
        tag_b = coord(2)
-       if (coord(2)==dims(2)-1 .AND. periodic_z) then
+       if (coord(2) == dims(2) - 1 .AND. periodic_z) then
           tag_t = 0
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s2 * (s1+2*level)) * level
+       icount = (s2*(s1 + 2*level))*level
        ! receive from bottom
-       call MPI_IRECV(out(xs,ys,zs), icount, data_type, &
-            neighbour(2,6), tag_b, DECOMP_2D_COMM_CART_Y, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
+                      neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from top
-       call MPI_IRECV(out(xs,ys,ze-level+1), icount, data_type, &
-            neighbour(2,5), tag_t, DECOMP_2D_COMM_CART_Y, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xs, ys, ze - level + 1), icount, data_type, &
+                      neighbour(2, 5), tag_t, DECOMP_2D_COMM_CART_Y, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to bottom
-       call MPI_ISSEND(out(xs,ys,zs+level), icount, data_type, &
-            neighbour(2,6), tag_b, DECOMP_2D_COMM_CART_Y, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs, ys, zs + level), icount, data_type, &
+                       neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to top
-       call MPI_ISSEND(out(xs,ys,ze-level-level+1), icount, data_type, &
-            neighbour(2,5), tag_t, DECOMP_2D_COMM_CART_Y, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xs, ys, ze - level - level + 1), icount, data_type, &
+                       neighbour(2, 5), tag_t, DECOMP_2D_COMM_CART_Y, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'After exchange in Z'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,1,k),k=zs,ze)
+       if (nrank == 4) then
+          write (*, *) 'After exchange in Z'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
           end do
        end if
 #endif
 
-    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   
-    ! Z-pencil
-    else if (ipencil == 3) then   
+    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+       ! Z-pencil
+    else if (ipencil == 3) then
 
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'Z-pencil input'
-          write(*,*) '=============='
-          write(*,*) 'Data on a x-y plane is shown'
-          write(*,*) 'Before halo exchange'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,j,1),j=ys,ye)
+       if (nrank == 4) then
+          write (*, *) 'Z-pencil input'
+          write (*, *) '=============='
+          write (*, *) 'Data on a x-y plane is shown'
+          write (*, *) 'Before halo exchange'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
           end do
        end if
 #endif
        ! *** east/west ***
        tag_w = coord(1)
-       if (coord(1)==dims(1)-1 .AND. periodic_x) then
+       if (coord(1) == dims(1) - 1 .AND. periodic_x) then
           tag_e = 0
        else
           tag_e = coord(1) + 1
        end if
-       icount = (s2+2*level)*s3
+       icount = (s2 + 2*level)*s3
        ilength = level
-       ijump = s1+2*level
+       ijump = s1 + 2*level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-            data_type, halo31, ierror)
+                            data_type, halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from west
-       call MPI_IRECV(out(xs,ys,zs), 1, halo31, &
-            neighbour(3,2), tag_w, DECOMP_2D_COMM_CART_Z, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), 1, halo31, &
+                      neighbour(3, 2), tag_w, DECOMP_2D_COMM_CART_Z, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from east
-       call MPI_IRECV(out(xe-level+1,ys,zs), 1, halo31, &
-            neighbour(3,1), tag_e, DECOMP_2D_COMM_CART_Z, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xe - level + 1, ys, zs), 1, halo31, &
+                      neighbour(3, 1), tag_e, DECOMP_2D_COMM_CART_Z, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to west
-       call MPI_ISSEND(out(xs+level,ys,zs), 1, halo31, &
-            neighbour(3,2), tag_w, DECOMP_2D_COMM_CART_Z, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs + level, ys, zs), 1, halo31, &
+                       neighbour(3, 2), tag_w, DECOMP_2D_COMM_CART_Z, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to east
-       call MPI_ISSEND(out(xe-level-level+1,ys,zs), 1, halo31, &
-            neighbour(3,1), tag_e, DECOMP_2D_COMM_CART_Z, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xe - level - level + 1, ys, zs), 1, halo31, &
+                       neighbour(3, 1), tag_e, DECOMP_2D_COMM_CART_Z, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
-       call MPI_TYPE_FREE(halo31, ierror)       
+       call MPI_TYPE_FREE(halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'After exchange in X'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,j,1),j=ys,ye)
+       if (nrank == 4) then
+          write (*, *) 'After exchange in X'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
           end do
        end if
 #endif
 
-       ! *** north/south *** 
+       ! *** north/south ***
        tag_s = coord(2)
-       if (coord(2)==dims(2)-1 .AND. periodic_y) then
+       if (coord(2) == dims(2) - 1 .AND. periodic_y) then
           tag_n = 0
        else
           tag_n = coord(2) + 1
        end if
        icount = s3
-       ilength = level * (s1+2*level)
-       ijump = (s1+2*level) * (s2+2*level)
+       ilength = level*(s1 + 2*level)
+       ijump = (s1 + 2*level)*(s2 + 2*level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-            data_type, halo32, ierror)
+                            data_type, halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from south
-       call MPI_IRECV(out(xs,ys,zs), 1, halo32, &
-            neighbour(3,4), tag_s, DECOMP_2D_COMM_CART_Z, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), 1, halo32, &
+                      neighbour(3, 4), tag_s, DECOMP_2D_COMM_CART_Z, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from north
-       call MPI_IRECV(out(xs,ye-level+1,zs), 1, halo32, &
-            neighbour(3,3), tag_n, DECOMP_2D_COMM_CART_Z, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xs, ye - level + 1, zs), 1, halo32, &
+                      neighbour(3, 3), tag_n, DECOMP_2D_COMM_CART_Z, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to south
-       call MPI_ISSEND(out(xs,ys+level,zs), 1, halo32, &
-            neighbour(3,4), tag_s, DECOMP_2D_COMM_CART_Z, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs, ys + level, zs), 1, halo32, &
+                       neighbour(3, 4), tag_s, DECOMP_2D_COMM_CART_Z, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to north
-       call MPI_ISSEND(out(xs,ye-level-level+1,zs), 1, halo32, &
-            neighbour(3,3), tag_n, DECOMP_2D_COMM_CART_Z, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xs, ye - level - level + 1, zs), 1, halo32, &
+                       neighbour(3, 3), tag_n, DECOMP_2D_COMM_CART_Z, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
        call MPI_TYPE_FREE(halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'After exchange in Y'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,j,1),j=ys,ye)
+       if (nrank == 4) then
+          write (*, *) 'After exchange in Y'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
           end do
        end if
 #endif
diff --git a/src/io.f90 b/src/io.f90
index 2869145d..f547a840 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -15,2047 +15,2028 @@
 
 module decomp_2d_io
 
-  use decomp_2d
-  use MPI
+   use decomp_2d
+   use MPI
 #ifdef T3PIO
-  use t3pio
+   use t3pio
 #endif
 
 #ifdef ADIOS2
-  use adios2
+   use adios2
 #endif
 
-  implicit none
+   implicit none
 
-  integer, parameter, public :: decomp_2d_write_mode = 1, decomp_2d_read_mode = 2, &
-       decomp_2d_append_mode = 3
-  integer, parameter :: MAX_IOH = 10 ! How many live IO things should we handle?
-  character(len=*), parameter :: io_sep = "::"
-  integer, save :: nreg_io = 0
+   integer, parameter, public :: decomp_2d_write_mode = 1, decomp_2d_read_mode = 2, &
+                                 decomp_2d_append_mode = 3
+   integer, parameter :: MAX_IOH = 10 ! How many live IO things should we handle?
+   character(len=*), parameter :: io_sep = "::"
+   integer, save :: nreg_io = 0
 #ifndef ADIOS2
-  integer, dimension(MAX_IOH), save :: fh_registry
-  logical, dimension(MAX_IOH), target, save :: fh_live
-  character(len=1024), dimension(MAX_IOH), target, save :: fh_names
-  integer(kind=MPI_OFFSET_KIND), dimension(MAX_IOH), save :: fh_disp
+   integer, dimension(MAX_IOH), save :: fh_registry
+   logical, dimension(MAX_IOH), target, save :: fh_live
+   character(len=1024), dimension(MAX_IOH), target, save :: fh_names
+   integer(kind=MPI_OFFSET_KIND), dimension(MAX_IOH), save :: fh_disp
 #else
-  type(adios2_adios) :: adios
-  character(len=1024), dimension(MAX_IOH), target, save :: engine_names
-  logical, dimension(MAX_IOH), target, save :: engine_live
-  type(adios2_engine), dimension(MAX_IOH), save :: engine_registry
-#endif
-  
-  private        ! Make everything private unless declared public
-
-  public :: decomp_2d_write_one, decomp_2d_read_one, &
-       decomp_2d_write_var, decomp_2d_read_var, &
-       decomp_2d_write_scalar, decomp_2d_read_scalar, &
-       decomp_2d_write_plane, decomp_2d_write_every, &
-       decomp_2d_write_subdomain, &
-       decomp_2d_write_outflow, decomp_2d_read_inflow, &
-       decomp_2d_io_init, decomp_2d_io_finalise, & ! XXX: initialise/finalise 2decomp&fft IO module
-       decomp_2d_init_io, & ! XXX: initialise an io process - awful naming
-       decomp_2d_register_variable, &
-       decomp_2d_open_io, decomp_2d_close_io, &
-       decomp_2d_start_io, decomp_2d_end_io, &
-       gen_iodir_name
-
-  ! Generic interface to handle multiple data types
-
-  interface decomp_2d_write_one
-     module procedure write_one_real
-     module procedure write_one_complex
-     module procedure mpiio_write_real_coarse
-     module procedure mpiio_write_real_probe
-  end interface decomp_2d_write_one
-
-  interface decomp_2d_read_one
-     module procedure read_one_real
-     module procedure read_one_complex
-  end interface decomp_2d_read_one
-
-  interface decomp_2d_write_var
-     module procedure write_var_real
-     module procedure write_var_complex
-  end interface decomp_2d_write_var
-
-  interface decomp_2d_read_var
-     module procedure read_var_real
-     module procedure read_var_complex
-  end interface decomp_2d_read_var
-
-  interface decomp_2d_write_scalar
-     module procedure write_scalar_real
-     module procedure write_scalar_complex
-     module procedure write_scalar_integer
-     module procedure write_scalar_logical
-  end interface decomp_2d_write_scalar
-
-  interface decomp_2d_read_scalar
-     module procedure read_scalar_real
-     module procedure read_scalar_complex
-     module procedure read_scalar_integer
-     module procedure read_scalar_logical
-  end interface decomp_2d_read_scalar
-
-  interface decomp_2d_write_plane
-     module procedure write_plane_3d_real
-     module procedure write_plane_3d_complex
-     !     module procedure write_plane_2d
-  end interface decomp_2d_write_plane
-
-  interface decomp_2d_write_every
-     module procedure write_every_real
-     module procedure write_every_complex
-  end interface decomp_2d_write_every
-
-  interface decomp_2d_write_subdomain
-     module procedure write_subdomain
-  end interface decomp_2d_write_subdomain
-
-  interface decomp_2d_write_outflow
-     module procedure write_outflow
-  end interface decomp_2d_write_outflow
-
-  interface decomp_2d_read_inflow
-     module procedure read_inflow
-  end interface decomp_2d_read_inflow
+   type(adios2_adios) :: adios
+   character(len=1024), dimension(MAX_IOH), target, save :: engine_names
+   logical, dimension(MAX_IOH), target, save :: engine_live
+   type(adios2_engine), dimension(MAX_IOH), save :: engine_registry
+#endif
+
+   private        ! Make everything private unless declared public
+
+   public :: decomp_2d_write_one, decomp_2d_read_one, &
+             decomp_2d_write_var, decomp_2d_read_var, &
+             decomp_2d_write_scalar, decomp_2d_read_scalar, &
+             decomp_2d_write_plane, decomp_2d_write_every, &
+             decomp_2d_write_subdomain, &
+             decomp_2d_write_outflow, decomp_2d_read_inflow, &
+             decomp_2d_io_init, decomp_2d_io_finalise, & ! XXX: initialise/finalise 2decomp&fft IO module
+             decomp_2d_init_io, & ! XXX: initialise an io process - awful naming
+             decomp_2d_register_variable, &
+             decomp_2d_open_io, decomp_2d_close_io, &
+             decomp_2d_start_io, decomp_2d_end_io, &
+             gen_iodir_name
+
+   ! Generic interface to handle multiple data types
+
+   interface decomp_2d_write_one
+      module procedure write_one_real
+      module procedure write_one_complex
+      module procedure mpiio_write_real_coarse
+      module procedure mpiio_write_real_probe
+   end interface decomp_2d_write_one
+
+   interface decomp_2d_read_one
+      module procedure read_one_real
+      module procedure read_one_complex
+   end interface decomp_2d_read_one
+
+   interface decomp_2d_write_var
+      module procedure write_var_real
+      module procedure write_var_complex
+   end interface decomp_2d_write_var
+
+   interface decomp_2d_read_var
+      module procedure read_var_real
+      module procedure read_var_complex
+   end interface decomp_2d_read_var
+
+   interface decomp_2d_write_scalar
+      module procedure write_scalar_real
+      module procedure write_scalar_complex
+      module procedure write_scalar_integer
+      module procedure write_scalar_logical
+   end interface decomp_2d_write_scalar
+
+   interface decomp_2d_read_scalar
+      module procedure read_scalar_real
+      module procedure read_scalar_complex
+      module procedure read_scalar_integer
+      module procedure read_scalar_logical
+   end interface decomp_2d_read_scalar
+
+   interface decomp_2d_write_plane
+      module procedure write_plane_3d_real
+      module procedure write_plane_3d_complex
+      !     module procedure write_plane_2d
+   end interface decomp_2d_write_plane
+
+   interface decomp_2d_write_every
+      module procedure write_every_real
+      module procedure write_every_complex
+   end interface decomp_2d_write_every
+
+   interface decomp_2d_write_subdomain
+      module procedure write_subdomain
+   end interface decomp_2d_write_subdomain
+
+   interface decomp_2d_write_outflow
+      module procedure write_outflow
+   end interface decomp_2d_write_outflow
+
+   interface decomp_2d_read_inflow
+      module procedure read_inflow
+   end interface decomp_2d_read_inflow
 
 contains
 
-  subroutine decomp_2d_io_init()
+   subroutine decomp_2d_io_init()
 
 #ifdef ADIOS2
-  integer :: ierror
-  logical :: adios2_debug_mode
-  character(len=80) :: config_file="adios2_config.xml"
+      integer :: ierror
+      logical :: adios2_debug_mode
+      character(len=80) :: config_file = "adios2_config.xml"
 #endif
 
 #ifdef ADIOS2
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_start("io_init")
+      if (decomp_profiler_io) call decomp_profiler_start("io_init")
 #endif
 
   !! TODO: make this a runtime-option
-  adios2_debug_mode = .true.
+      adios2_debug_mode = .true.
 
-  call adios2_init(adios, trim(config_file), decomp_2d_comm, adios2_debug_mode, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-     "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
+      call adios2_init(adios, trim(config_file), decomp_2d_comm, adios2_debug_mode, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                                              "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
 
-  engine_live(:) = .false.
+      engine_live(:) = .false.
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_end("io_init")
+      if (decomp_profiler_io) call decomp_profiler_end("io_init")
 #endif
 #endif
-  
-  end subroutine decomp_2d_io_init
-  subroutine decomp_2d_io_finalise()
+
+   end subroutine decomp_2d_io_init
+   subroutine decomp_2d_io_finalise()
 
 #ifdef ADIOS2
-    use adios2
+      use adios2
 #endif
 
-    implicit none
+      implicit none
 
 #ifdef ADIOS2
-    integer :: ierror
+      integer :: ierror
 #endif
-    
+
 #ifdef ADIOS2
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_fin")
+      if (decomp_profiler_io) call decomp_profiler_start("io_fin")
 #endif
-    call adios2_finalize(adios, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-       "adios2_finalize")
+      call adios2_finalize(adios, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                                              "adios2_finalize")
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_fin")
+      if (decomp_profiler_io) call decomp_profiler_end("io_fin")
 #endif
 #endif
 
-  end subroutine decomp_2d_io_finalise
-  
+   end subroutine decomp_2d_io_finalise
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Using MPI-IO library to write a single 3D array to a file
+   ! Using MPI-IO library to write a single 3D array to a file
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_one_real(ipencil,var,filename,opt_decomp)
+   subroutine write_one_real(ipencil, var, filename, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    character(len=*), intent(IN) :: filename
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: ipencil
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      character(len=*), intent(IN) :: filename
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh, data_type, info, gs
+      TYPE(DECOMP_INFO) :: decomp
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, fh, data_type, info, gs
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_start("io_write_one_real")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_one_real")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_one.inc"
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_end("io_write_one_real")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_one_real")
 #endif
 
-    return
-  end subroutine write_one_real
+      return
+   end subroutine write_one_real
 
-  subroutine write_one_complex(ipencil,var,filename,opt_decomp)
+   subroutine write_one_complex(ipencil, var, filename, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil
-    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    character(len=*), intent(IN) :: filename
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: ipencil
+      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      character(len=*), intent(IN) :: filename
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh, data_type, info, gs
+      TYPE(DECOMP_INFO) :: decomp
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, fh, data_type, info, gs
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_start("io_write_one_cplx")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_one_cplx")
 #endif
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_write_one.inc"
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_end("io_write_one_cplx")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_one_cplx")
 #endif
 
-    return
-  end subroutine write_one_complex
-
+      return
+   end subroutine write_one_complex
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Using MPI-IO library to read from a file a single 3D array
+   ! Using MPI-IO library to read from a file a single 3D array
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine read_one_real(ipencil,var,dirname,varname,io_name,opt_decomp,reduce_prec)
+   subroutine read_one_real(ipencil, var, dirname, varname, io_name, opt_decomp, reduce_prec)
+
+      implicit none
 
-    implicit none
+      integer, intent(IN) :: ipencil
+      real(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      character(len=*), intent(IN) :: varname, dirname, io_name
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      logical, intent(in), optional :: reduce_prec
 
-    integer, intent(IN) :: ipencil
-    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    character(len=*), intent(IN) :: varname, dirname, io_name
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
-    logical, intent(in), optional :: reduce_prec
+      logical :: read_reduce_prec
 
-    logical :: read_reduce_prec
-    
-    integer :: idx
+      integer :: idx
 #ifndef ADIOS2
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    real(mytype_single), allocatable, dimension(:,:,:) :: varsingle
-    integer :: data_type
-    logical :: dir_exists
-    integer :: disp_bytes
-    integer :: ierror, newtype
-    character(len=:), allocatable :: full_io_name
-    logical :: opened_new
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      real(mytype_single), allocatable, dimension(:, :, :) :: varsingle
+      integer :: data_type
+      logical :: dir_exists
+      integer :: disp_bytes
+      integer :: ierror, newtype
+      character(len=:), allocatable :: full_io_name
+      logical :: opened_new
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_read_one_real")
+      if (decomp_profiler_io) call decomp_profiler_start("io_read_one_real")
 #endif
-    
-    read_reduce_prec = .true.
-    
-    idx = get_io_idx(io_name, dirname)
+
+      read_reduce_prec = .true.
+
+      idx = get_io_idx(io_name, dirname)
 #ifndef ADIOS2
-    opened_new = .false.
-    if (idx .lt. 1) then
-       ! Check file exists
-       full_io_name = trim(dirname)//"/"//trim(varname)
-       if (nrank==0) then
-          inquire(file=full_io_name, exist=dir_exists)
-          if (.not.dir_exists) then
-             print *, "ERROR: cannot read from", full_io_name, " directory doesn't exist!"
-             stop
-          end if
-       end if
-       
-       call decomp_2d_open_io(io_name, full_io_name, decomp_2d_read_mode)
-       idx = get_io_idx(io_name, full_io_name)
-       opened_new = .true.
-    else
-       full_io_name = "" ! Ensure string is not unset
-    end if
-    
-    if (present(reduce_prec)) then
-       if (.not. reduce_prec) then
-          read_reduce_prec = .false.
-       end if
-    end if
-    if (read_reduce_prec) then
-       data_type = real_type_single
-    else
-       data_type = real_type
-    end if
-    call MPI_TYPE_SIZE(data_type,disp_bytes,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    
-    if (present(opt_decomp)) then
-       decomp = opt_decomp
-    else
-       call get_decomp_info(decomp)
-    end if
-
-    ! determine subarray parameters
-    sizes(1) = decomp%xsz(1)
-    sizes(2) = decomp%ysz(2)
-    sizes(3) = decomp%zsz(3)
-
-    if (ipencil == 1) then
-       subsizes(1) = decomp%xsz(1)
-       subsizes(2) = decomp%xsz(2)
-       subsizes(3) = decomp%xsz(3)
-       starts(1) = decomp%xst(1)-1  ! 0-based index
-       starts(2) = decomp%xst(2)-1
-       starts(3) = decomp%xst(3)-1
-    else if (ipencil == 2) then
-       subsizes(1) = decomp%ysz(1)
-       subsizes(2) = decomp%ysz(2)
-       subsizes(3) = decomp%ysz(3)
-       starts(1) = decomp%yst(1)-1
-       starts(2) = decomp%yst(2)-1
-       starts(3) = decomp%yst(3)-1
-    else if (ipencil == 3) then
-       subsizes(1) = decomp%zsz(1)
-       subsizes(2) = decomp%zsz(2)
-       subsizes(3) = decomp%zsz(3)
-       starts(1) = decomp%zst(1)-1
-       starts(2) = decomp%zst(2)-1
-       starts(3) = decomp%zst(3)-1
-    else
-       call decomp_2d_abort(-1, "IO/read_one_real : Wrong value for ipencil")
-    endif
-
-    associate(fh => fh_registry(idx), &
-         disp => fh_disp(idx))
-      call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-           MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-      call MPI_TYPE_COMMIT(newtype,ierror)
-      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-      call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-           newtype,'native',MPI_INFO_NULL,ierror)
-      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      opened_new = .false.
+      if (idx .lt. 1) then
+         ! Check file exists
+         full_io_name = trim(dirname)//"/"//trim(varname)
+         if (nrank == 0) then
+            inquire (file=full_io_name, exist=dir_exists)
+            if (.not. dir_exists) then
+               print *, "ERROR: cannot read from", full_io_name, " directory doesn't exist!"
+               stop
+            end if
+         end if
+
+         call decomp_2d_open_io(io_name, full_io_name, decomp_2d_read_mode)
+         idx = get_io_idx(io_name, full_io_name)
+         opened_new = .true.
+      else
+         full_io_name = "" ! Ensure string is not unset
+      end if
+
+      if (present(reduce_prec)) then
+         if (.not. reduce_prec) then
+            read_reduce_prec = .false.
+         end if
+      end if
       if (read_reduce_prec) then
-         allocate (varsingle(xstV(1):xenV(1),xstV(2):xenV(2),xstV(3):xenV(3)))
-         call MPI_FILE_READ_ALL(fh, varsingle, &
-              subsizes(1)*subsizes(2)*subsizes(3), &
-              data_type, MPI_STATUS_IGNORE, ierror)
-         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-         var = real(varsingle,mytype)
-         deallocate(varsingle)
+         data_type = real_type_single
+      else
+         data_type = real_type
+      end if
+      call MPI_TYPE_SIZE(data_type, disp_bytes, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+
+      if (present(opt_decomp)) then
+         decomp = opt_decomp
       else
-         call MPI_FILE_READ_ALL(fh, var, &
-              subsizes(1)*subsizes(2)*subsizes(3), &
-              data_type, MPI_STATUS_IGNORE, ierror)
-         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-      endif
-      call MPI_TYPE_FREE(newtype,ierror)
-      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-
-      disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                  * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                  * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                  * int(disp_bytes, kind=MPI_OFFSET_KIND)
-    end associate
-
-    if (opened_new) then
-       call decomp_2d_close_io(io_name, full_io_name)
-       deallocate(full_io_name)
-    end if
+         call get_decomp_info(decomp)
+      end if
+
+      ! determine subarray parameters
+      sizes(1) = decomp%xsz(1)
+      sizes(2) = decomp%ysz(2)
+      sizes(3) = decomp%zsz(3)
+
+      if (ipencil == 1) then
+         subsizes(1) = decomp%xsz(1)
+         subsizes(2) = decomp%xsz(2)
+         subsizes(3) = decomp%xsz(3)
+         starts(1) = decomp%xst(1) - 1  ! 0-based index
+         starts(2) = decomp%xst(2) - 1
+         starts(3) = decomp%xst(3) - 1
+      else if (ipencil == 2) then
+         subsizes(1) = decomp%ysz(1)
+         subsizes(2) = decomp%ysz(2)
+         subsizes(3) = decomp%ysz(3)
+         starts(1) = decomp%yst(1) - 1
+         starts(2) = decomp%yst(2) - 1
+         starts(3) = decomp%yst(3) - 1
+      else if (ipencil == 3) then
+         subsizes(1) = decomp%zsz(1)
+         subsizes(2) = decomp%zsz(2)
+         subsizes(3) = decomp%zsz(3)
+         starts(1) = decomp%zst(1) - 1
+         starts(2) = decomp%zst(2) - 1
+         starts(3) = decomp%zst(3) - 1
+      else
+         call decomp_2d_abort(-1, "IO/read_one_real : Wrong value for ipencil")
+      end if
+
+      associate (fh => fh_registry(idx), &
+                 disp => fh_disp(idx))
+         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+         call MPI_TYPE_COMMIT(newtype, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+         call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                                newtype, 'native', MPI_INFO_NULL, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+         if (read_reduce_prec) then
+            allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
+            call MPI_FILE_READ_ALL(fh, varsingle, &
+                                   subsizes(1)*subsizes(2)*subsizes(3), &
+                                   data_type, MPI_STATUS_IGNORE, ierror)
+            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+            var = real(varsingle, mytype)
+            deallocate (varsingle)
+         else
+            call MPI_FILE_READ_ALL(fh, var, &
+                                   subsizes(1)*subsizes(2)*subsizes(3), &
+                                   data_type, MPI_STATUS_IGNORE, ierror)
+            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+         end if
+         call MPI_TYPE_FREE(newtype, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+
+         disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                *int(disp_bytes, kind=MPI_OFFSET_KIND)
+      end associate
+
+      if (opened_new) then
+         call decomp_2d_close_io(io_name, full_io_name)
+         deallocate (full_io_name)
+      end if
 #else
-    call adios2_read_one_real(var, dirname, varname, io_name)
+      call adios2_read_one_real(var, dirname, varname, io_name)
 
-    associate(pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
-    end associate
+      associate (pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
+      end associate
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_read_one_real")
+      if (decomp_profiler_io) call decomp_profiler_end("io_read_one_real")
 #endif
 
-  end subroutine read_one_real
-
+   end subroutine read_one_real
 
-  subroutine read_one_complex(ipencil,var,filename,opt_decomp)
+   subroutine read_one_complex(ipencil, var, filename, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil
-    complex(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    character(len=*), intent(IN) :: filename
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: ipencil
+      complex(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      character(len=*), intent(IN) :: filename
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer(kind=MPI_OFFSET_KIND) :: disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer(kind=MPI_OFFSET_KIND) :: disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, fh, data_type
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_read_one_cplx")
+      if (decomp_profiler_io) call decomp_profiler_start("io_read_one_cplx")
 #endif
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_read_one.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_read_one_cplx")
+      if (decomp_profiler_io) call decomp_profiler_end("io_read_one_cplx")
 #endif
 
-  end subroutine read_one_complex
+   end subroutine read_one_complex
 
 #ifdef ADIOS2
-  subroutine adios2_read_one_real(var,engine_name,varname,io_name)
+   subroutine adios2_read_one_real(var, engine_name, varname, io_name)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: engine_name
-    character(len=*), intent(in) :: io_name
-    character*(*), intent(in) :: varname
-    real(mytype), contiguous, dimension(:,:,:), intent(out) :: var
+      character(len=*), intent(in) :: engine_name
+      character(len=*), intent(in) :: io_name
+      character*(*), intent(in) :: varname
+      real(mytype), contiguous, dimension(:, :, :), intent(out) :: var
 
-    integer :: ierror
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
-    integer :: idx
+      integer :: ierror
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
+      integer :: idx
 
-    integer(kind=8) :: nsteps
-    integer(kind=8) :: curstep
+      integer(kind=8) :: nsteps
+      integer(kind=8) :: curstep
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("adios2_read_one_real")
-#endif
-
-    call adios2_at_io(io_handle, adios, io_name, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-    call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-    if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-       "ERROR: trying to read variable without registering first! "//trim(varname))
-
-    call adios2_variable_steps(nsteps, var_handle, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_variable_steps")
-    print *, "AVAILABLE steps for ", nsteps
-    
-    print *, "IO_NAME: ", io_name
-    print *, "ENGINE_NAME: ", engine_name
-    print *, "VAR_NAME: ", varname
-    idx = get_io_idx(io_name, engine_name)
-    print *, idx
-    call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_deferred, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
-
-    print *, "MAX: ", maxval(var)
-    
-    call adios2_current_step(curstep, engine_registry(idx), ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_current_step")
-    print *, "Current step: ", curstep
+      if (decomp_profiler_io) call decomp_profiler_start("adios2_read_one_real")
+#endif
+
+      call adios2_at_io(io_handle, adios, io_name, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+      call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+      if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                                                       "ERROR: trying to read variable without registering first! "//trim(varname))
+
+      call adios2_variable_steps(nsteps, var_handle, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_variable_steps")
+      print *, "AVAILABLE steps for ", nsteps
+
+      print *, "IO_NAME: ", io_name
+      print *, "ENGINE_NAME: ", engine_name
+      print *, "VAR_NAME: ", varname
+      idx = get_io_idx(io_name, engine_name)
+      print *, idx
+      call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_deferred, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
+
+      print *, "MAX: ", maxval(var)
+
+      call adios2_current_step(curstep, engine_registry(idx), ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_current_step")
+      print *, "Current step: ", curstep
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("adios2_read_one_real")
+      if (decomp_profiler_io) call decomp_profiler_end("adios2_read_one_real")
 #endif
-    
-  end subroutine adios2_read_one_real
+
+   end subroutine adios2_read_one_real
 #endif
-  
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write a 3D array as part of a big MPI-IO file, starting from 
-  !  displacement 'disp'; 'disp' will be updated after the writing
-  !  operation to prepare the writing of next chunk of data.
+   ! Write a 3D array as part of a big MPI-IO file, starting from
+   !  displacement 'disp'; 'disp' will be updated after the writing
+   !  operation to prepare the writing of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_var_real(fh,disp,ipencil,var,opt_decomp)
+   subroutine write_var_real(fh, disp, ipencil, var, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: ipencil
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: ipencil
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, data_type
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_var.inc"
 
-    return
-  end subroutine write_var_real
+      return
+   end subroutine write_var_real
 
-  subroutine write_var_complex(fh,disp,ipencil,var,opt_decomp)
+   subroutine write_var_complex(fh, disp, ipencil, var, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: ipencil
-    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: ipencil
+      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, data_type
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_write_var.inc"
 
-    return
-  end subroutine write_var_complex
-
+      return
+   end subroutine write_var_complex
 
-  subroutine write_outflow(dirname,varname,ntimesteps,var,io_name,opt_decomp)
+   subroutine write_outflow(dirname, varname, ntimesteps, var, io_name, opt_decomp)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: dirname, varname, io_name
-    integer, intent(IN) :: ntimesteps 
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      character(len=*), intent(in) :: dirname, varname, io_name
+      integer, intent(IN) :: ntimesteps
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, data_type
-    integer :: idx
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, data_type
+      integer :: idx
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
 #else
-    integer :: newtype
+      integer :: newtype
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_outflow")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_outflow")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_outflow.f90"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_outflow")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_outflow")
 #endif
 
-  end subroutine write_outflow
-
+   end subroutine write_outflow
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Read a 3D array as part of a big MPI-IO file, starting from 
-  !  displacement 'disp'; 'disp' will be updated after the reading
-  !  operation to prepare the reading of next chunk of data.
+   ! Read a 3D array as part of a big MPI-IO file, starting from
+   !  displacement 'disp'; 'disp' will be updated after the reading
+   !  operation to prepare the reading of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine read_var_real(fh,disp,ipencil,var,opt_decomp)
+   subroutine read_var_real(fh, disp, ipencil, var, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: ipencil
-    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: ipencil
+      real(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, data_type
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_read_var.inc"
 
-    return
-  end subroutine read_var_real
-
+      return
+   end subroutine read_var_real
 
-  subroutine read_var_complex(fh,disp,ipencil,var,opt_decomp)
+   subroutine read_var_complex(fh, disp, ipencil, var, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: ipencil
-    complex(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: ipencil
+      complex(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, data_type
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_read_var.inc"
 
-    return
-  end subroutine read_var_complex
+      return
+   end subroutine read_var_complex
 
+   subroutine read_inflow(dirname, varname, ntimesteps, var, io_name, opt_decomp)
 
-  subroutine read_inflow(dirname,varname,ntimesteps,var,io_name,opt_decomp)
+      implicit none
 
-    implicit none
+      character(len=*), intent(in) :: dirname, varname, io_name
+      integer, intent(IN) :: ntimesteps
+      real(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    character(len=*), intent(in) :: dirname, varname, io_name
-    integer, intent(IN) :: ntimesteps
-    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
-
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, data_type
-    integer :: idx
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, data_type
+      integer :: idx
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
 #else
-    integer :: newtype
+      integer :: newtype
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_read_inflow")
+      if (decomp_profiler_io) call decomp_profiler_start("io_read_inflow")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_read_inflow.f90"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_read_inflow")
+      if (decomp_profiler_io) call decomp_profiler_end("io_read_inflow")
 #endif
 
-  end subroutine read_inflow
+   end subroutine read_inflow
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write scalar variables as part of a big MPI-IO file, starting from 
-  !  displacement 'disp'; 'disp' will be updated after the reading
-  !  operation to prepare the reading of next chunk of data.
+   ! Write scalar variables as part of a big MPI-IO file, starting from
+   !  displacement 'disp'; 'disp' will be updated after the reading
+   !  operation to prepare the reading of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_scalar_real(fh,disp,n,var)
+   subroutine write_scalar_real(fh, disp, n, var)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh             ! file handle
-    integer(KIND=MPI_OFFSET_KIND), &
+      integer, intent(IN) :: fh             ! file handle
+      integer(KIND=MPI_OFFSET_KIND), &
          intent(INOUT) :: disp            ! displacement
-    integer, intent(IN) :: n              ! number of scalars
-    real(mytype), dimension(n), &
+      integer, intent(IN) :: n              ! number of scalars
+      real(mytype), dimension(n), &
          intent(IN) :: var                ! array of scalars
 
-    integer :: m, ierror
-
-    call MPI_FILE_SET_VIEW(fh,disp,real_type, &
-         real_type,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    if (nrank==0) then
-       m = n ! only one rank needs to write
-    else
-       m = 0
-    end if
-    call MPI_FILE_WRITE_ALL(fh, var, m, real_type, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(mytype_bytes, kind=MPI_OFFSET_KIND)
-
-    return
-  end subroutine write_scalar_real
-
-
-  subroutine write_scalar_complex(fh,disp,n,var)
-
-    implicit none
-
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    complex(mytype), dimension(n), intent(IN) :: var
-
-    integer :: m, ierror
-
-    call MPI_FILE_SET_VIEW(fh,disp,complex_type, &
-         complex_type,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    if (nrank==0) then
-       m = n
-    else
-       m = 0
-    end if
-    call MPI_FILE_WRITE_ALL(fh, var, m, complex_type, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-                * 2_MPI_OFFSET_KIND
-
-    return
-  end subroutine write_scalar_complex
-
-
-  subroutine write_scalar_integer(fh,disp,n,var)
-
-    implicit none
-
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    integer, dimension(n), intent(IN) :: var
-
-    integer :: m, ierror
-
-    call MPI_FILE_SET_VIEW(fh,disp,MPI_INTEGER, &
-         MPI_INTEGER,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    if (nrank==0) then
-       m = n
-    else
-       m = 0
-    end if
-    call MPI_FILE_WRITE_ALL(fh, var, m, MPI_INTEGER, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    call MPI_TYPE_SIZE(MPI_INTEGER,m,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(m, kind=MPI_OFFSET_KIND)
-
-    return
-  end subroutine write_scalar_integer
-
-
-  subroutine write_scalar_logical(fh,disp,n,var)
-
-    implicit none
-
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    logical, dimension(n), intent(IN) :: var
-
-    integer :: m, ierror
-
-    call MPI_FILE_SET_VIEW(fh,disp,MPI_LOGICAL, &
-         MPI_LOGICAL,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    if (nrank==0) then
-       m = n
-    else
-       m = 0
-    end if
-    call MPI_FILE_WRITE_ALL(fh, var, m, MPI_LOGICAL, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    call MPI_TYPE_SIZE(MPI_LOGICAL,m,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(m, kind=MPI_OFFSET_KIND)
-
-    return
-  end subroutine write_scalar_logical
+      integer :: m, ierror
+
+      call MPI_FILE_SET_VIEW(fh, disp, real_type, &
+                             real_type, 'native', MPI_INFO_NULL, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      if (nrank == 0) then
+         m = n ! only one rank needs to write
+      else
+         m = 0
+      end if
+      call MPI_FILE_WRITE_ALL(fh, var, m, real_type, &
+                              MPI_STATUS_IGNORE, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+
+      return
+   end subroutine write_scalar_real
+
+   subroutine write_scalar_complex(fh, disp, n, var)
+
+      implicit none
+
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      complex(mytype), dimension(n), intent(IN) :: var
+
+      integer :: m, ierror
+
+      call MPI_FILE_SET_VIEW(fh, disp, complex_type, &
+                             complex_type, 'native', MPI_INFO_NULL, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      if (nrank == 0) then
+         m = n
+      else
+         m = 0
+      end if
+      call MPI_FILE_WRITE_ALL(fh, var, m, complex_type, &
+                              MPI_STATUS_IGNORE, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+             *2_MPI_OFFSET_KIND
+
+      return
+   end subroutine write_scalar_complex
+
+   subroutine write_scalar_integer(fh, disp, n, var)
+
+      implicit none
+
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      integer, dimension(n), intent(IN) :: var
+
+      integer :: m, ierror
+
+      call MPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, &
+                             MPI_INTEGER, 'native', MPI_INFO_NULL, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      if (nrank == 0) then
+         m = n
+      else
+         m = 0
+      end if
+      call MPI_FILE_WRITE_ALL(fh, var, m, MPI_INTEGER, &
+                              MPI_STATUS_IGNORE, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(m, kind=MPI_OFFSET_KIND)
+
+      return
+   end subroutine write_scalar_integer
+
+   subroutine write_scalar_logical(fh, disp, n, var)
 
+      implicit none
+
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      logical, dimension(n), intent(IN) :: var
+
+      integer :: m, ierror
+
+      call MPI_FILE_SET_VIEW(fh, disp, MPI_LOGICAL, &
+                             MPI_LOGICAL, 'native', MPI_INFO_NULL, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      if (nrank == 0) then
+         m = n
+      else
+         m = 0
+      end if
+      call MPI_FILE_WRITE_ALL(fh, var, m, MPI_LOGICAL, &
+                              MPI_STATUS_IGNORE, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(m, kind=MPI_OFFSET_KIND)
+
+      return
+   end subroutine write_scalar_logical
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Read scalar variables as part of a big MPI-IO file, starting from 
-  !  displacement 'disp'; 'disp' will be updated after the reading
-  !  operation to prepare the reading of next chunk of data.
+   ! Read scalar variables as part of a big MPI-IO file, starting from
+   !  displacement 'disp'; 'disp' will be updated after the reading
+   !  operation to prepare the reading of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine read_scalar_real(fh,disp,n,var)
+   subroutine read_scalar_real(fh, disp, n, var)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh             ! file handle
-    integer(KIND=MPI_OFFSET_KIND), &
+      integer, intent(IN) :: fh             ! file handle
+      integer(KIND=MPI_OFFSET_KIND), &
          intent(INOUT) :: disp            ! displacement
-    integer, intent(IN) :: n              ! number of scalars
-    real(mytype), dimension(n), &
+      integer, intent(IN) :: n              ! number of scalars
+      real(mytype), dimension(n), &
          intent(INOUT) :: var             ! array of scalars
 
-    integer :: ierror
+      integer :: ierror
 
-    call MPI_FILE_SET_VIEW(fh,disp,real_type, &
-         real_type,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_READ_ALL(fh, var, n, real_type, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+      call MPI_FILE_SET_VIEW(fh, disp, real_type, &
+                             real_type, 'native', MPI_INFO_NULL, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_READ_ALL(fh, var, n, real_type, &
+                             MPI_STATUS_IGNORE, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 
-    return
-  end subroutine read_scalar_real
+      return
+   end subroutine read_scalar_real
 
+   subroutine read_scalar_complex(fh, disp, n, var)
 
-  subroutine read_scalar_complex(fh,disp,n,var)
+      implicit none
 
-    implicit none
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      complex(mytype), dimension(n), intent(INOUT) :: var
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    complex(mytype), dimension(n), intent(INOUT) :: var
+      integer :: ierror
 
-    integer :: ierror
+      call MPI_FILE_SET_VIEW(fh, disp, complex_type, &
+                             complex_type, 'native', MPI_INFO_NULL, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_READ_ALL(fh, var, n, complex_type, &
+                             MPI_STATUS_IGNORE, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+             *2_MPI_OFFSET_KIND
 
-    call MPI_FILE_SET_VIEW(fh,disp,complex_type, &
-         complex_type,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_READ_ALL(fh, var, n, complex_type, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-                * 2_MPI_OFFSET_KIND
+      return
+   end subroutine read_scalar_complex
 
-    return
-  end subroutine read_scalar_complex
+   subroutine read_scalar_integer(fh, disp, n, var)
 
+      implicit none
 
-  subroutine read_scalar_integer(fh,disp,n,var)
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      integer, dimension(n), intent(INOUT) :: var
 
-    implicit none
+      integer :: m, ierror
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    integer, dimension(n), intent(INOUT) :: var
+      call MPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, &
+                             MPI_INTEGER, 'native', MPI_INFO_NULL, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_READ_ALL(fh, var, n, MPI_INTEGER, &
+                             MPI_STATUS_IGNORE, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+      call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(m, kind=MPI_OFFSET_KIND)
 
-    integer :: m, ierror
+      return
+   end subroutine read_scalar_integer
 
-    call MPI_FILE_SET_VIEW(fh,disp,MPI_INTEGER, &
-         MPI_INTEGER,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_READ_ALL(fh, var, n, MPI_INTEGER, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    call MPI_TYPE_SIZE(MPI_INTEGER,m,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(m, kind=MPI_OFFSET_KIND)
+   subroutine read_scalar_logical(fh, disp, n, var)
 
-    return
-  end subroutine read_scalar_integer
+      implicit none
 
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      logical, dimension(n), intent(INOUT) :: var
 
-  subroutine read_scalar_logical(fh,disp,n,var)
+      integer :: m, ierror
 
-    implicit none
+      call MPI_FILE_SET_VIEW(fh, disp, MPI_LOGICAL, &
+                             MPI_LOGICAL, 'native', MPI_INFO_NULL, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_READ_ALL(fh, var, n, MPI_LOGICAL, &
+                             MPI_STATUS_IGNORE, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+      call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(m, kind=MPI_OFFSET_KIND)
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    logical, dimension(n), intent(INOUT) :: var
+      return
+   end subroutine read_scalar_logical
 
-    integer :: m, ierror
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   ! Write a 2D slice of the 3D data to a file
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   subroutine plane_extents(sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes)
 
-    call MPI_FILE_SET_VIEW(fh,disp,MPI_LOGICAL, &
-         MPI_LOGICAL,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_READ_ALL(fh, var, n, MPI_LOGICAL, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    call MPI_TYPE_SIZE(MPI_LOGICAL,m,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(m, kind=MPI_OFFSET_KIND)
+      integer, intent(in) :: iplane
+      type(decomp_info), intent(in), optional :: opt_decomp
+      integer, intent(in), optional :: opt_nplanes
 
-    return
-  end subroutine read_scalar_logical
+      integer, dimension(3), intent(out) :: sizes, subsizes, starts
 
+      integer :: nplanes
+      type(decomp_info) :: decomp
 
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write a 2D slice of the 3D data to a file
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine plane_extents (sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes)
-
-    integer, intent(in) :: iplane
-    type(decomp_info), intent(in), optional :: opt_decomp
-    integer, intent(in), optional :: opt_nplanes
-    
-    integer, dimension(3), intent(out) :: sizes, subsizes, starts
-
-    integer :: nplanes
-    type(decomp_info) :: decomp
-
-    if (present(opt_decomp)) then
-       decomp = opt_decomp
-    else
-       call get_decomp_info(decomp)
-    end if
-
-    if (present(opt_nplanes)) then
-       nplanes = opt_nplanes
-    else
-       nplanes = 1
-    end if
-    
-    if (iplane == 1) then
-       sizes(1) = nplanes
-       sizes(2) = decomp%ysz(2)
-       sizes(3) = decomp%zsz(3)
-       subsizes(1) = nplanes
-       subsizes(2) = decomp%xsz(2)
-       subsizes(3) = decomp%xsz(3)
-       starts(1) = 0
-       starts(2) = decomp%xst(2)-1
-       starts(3) = decomp%xst(3)-1
-    else if (iplane == 2) then
-       sizes(1) = decomp%xsz(1)
-       sizes(2) = nplanes
-       sizes(3) = decomp%zsz(3)
-       subsizes(1) = decomp%ysz(1)
-       subsizes(2) = nplanes
-       subsizes(3) = decomp%ysz(3)
-       starts(1) = decomp%yst(1)-1
-       starts(2) = 0
-       starts(3) = decomp%yst(3)-1
-    else if (iplane == 3) then
-       sizes(1) = decomp%xsz(1)
-       sizes(2) = decomp%ysz(2)
-       sizes(3) = nplanes
-       subsizes(1) = decomp%zsz(1)
-       subsizes(2) = decomp%zsz(2)
-       subsizes(3) = nplanes
-       starts(1) = decomp%zst(1)-1
-       starts(2) = decomp%zst(2)-1
-       starts(3) = 0
-    else
-       print *, "Can't work with plane ", iplane
-       stop
-    endif
-    
-  end subroutine plane_extents
-
-  subroutine write_plane_3d_real(ipencil,var,iplane,n,dirname,varname,io_name, &
-       opt_decomp)
-
-    implicit none
-
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
-    integer, intent(IN) :: n ! which plane to write (global coordinate)
-    character(len=*), intent(IN) :: dirname,varname,io_name
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
-
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    real(mytype), allocatable, dimension(:,:,:) :: wk2d
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: i,j,k, ierror, data_type
-
-    logical :: opened_new
-    integer :: idx
+      if (present(opt_decomp)) then
+         decomp = opt_decomp
+      else
+         call get_decomp_info(decomp)
+      end if
+
+      if (present(opt_nplanes)) then
+         nplanes = opt_nplanes
+      else
+         nplanes = 1
+      end if
+
+      if (iplane == 1) then
+         sizes(1) = nplanes
+         sizes(2) = decomp%ysz(2)
+         sizes(3) = decomp%zsz(3)
+         subsizes(1) = nplanes
+         subsizes(2) = decomp%xsz(2)
+         subsizes(3) = decomp%xsz(3)
+         starts(1) = 0
+         starts(2) = decomp%xst(2) - 1
+         starts(3) = decomp%xst(3) - 1
+      else if (iplane == 2) then
+         sizes(1) = decomp%xsz(1)
+         sizes(2) = nplanes
+         sizes(3) = decomp%zsz(3)
+         subsizes(1) = decomp%ysz(1)
+         subsizes(2) = nplanes
+         subsizes(3) = decomp%ysz(3)
+         starts(1) = decomp%yst(1) - 1
+         starts(2) = 0
+         starts(3) = decomp%yst(3) - 1
+      else if (iplane == 3) then
+         sizes(1) = decomp%xsz(1)
+         sizes(2) = decomp%ysz(2)
+         sizes(3) = nplanes
+         subsizes(1) = decomp%zsz(1)
+         subsizes(2) = decomp%zsz(2)
+         subsizes(3) = nplanes
+         starts(1) = decomp%zst(1) - 1
+         starts(2) = decomp%zst(2) - 1
+         starts(3) = 0
+      else
+         print *, "Can't work with plane ", iplane
+         stop
+      end if
+
+   end subroutine plane_extents
+
+   subroutine write_plane_3d_real(ipencil, var, iplane, n, dirname, varname, io_name, &
+                                  opt_decomp)
+
+      implicit none
+
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
+      integer, intent(IN) :: n ! which plane to write (global coordinate)
+      character(len=*), intent(IN) :: dirname, varname, io_name
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      real(mytype), allocatable, dimension(:, :, :) :: wk2d
+      TYPE(DECOMP_INFO) :: decomp
+      integer :: i, j, k, ierror, data_type
+
+      logical :: opened_new
+      integer :: idx
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
 #else
-    integer, dimension(3) :: sizes, subsizes, starts
-    logical :: dir_exists
-    character(len=:), allocatable :: full_io_name
-    integer :: newtype
+      integer, dimension(3) :: sizes, subsizes, starts
+      logical :: dir_exists
+      character(len=:), allocatable :: full_io_name
+      integer :: newtype
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_real")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_real")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_plane.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_real")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_real")
 #endif
 
-  end subroutine write_plane_3d_real
-
+   end subroutine write_plane_3d_real
 
-  subroutine write_plane_3d_complex(ipencil,var,iplane,n, &
-       dirname,varname,io_name,opt_decomp)
+   subroutine write_plane_3d_complex(ipencil, var, iplane, n, &
+                                     dirname, varname, io_name, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
-    integer, intent(IN) :: n ! which plane to write (global coordinate)
-    character(len=*), intent(IN) :: dirname,varname,io_name
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
+      integer, intent(IN) :: n ! which plane to write (global coordinate)
+      character(len=*), intent(IN) :: dirname, varname, io_name
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    complex(mytype), allocatable, dimension(:,:,:) :: wk2d
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: i,j,k, ierror, data_type
-    logical :: opened_new
-    integer :: idx
+      complex(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      complex(mytype), allocatable, dimension(:, :, :) :: wk2d
+      TYPE(DECOMP_INFO) :: decomp
+      integer :: i, j, k, ierror, data_type
+      logical :: opened_new
+      integer :: idx
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
 #else
-    integer, dimension(3) :: sizes, subsizes, starts
-    logical :: dir_exists
-    character(len=:), allocatable :: full_io_name
-    integer :: newtype
+      integer, dimension(3) :: sizes, subsizes, starts
+      logical :: dir_exists
+      character(len=:), allocatable :: full_io_name
+      integer :: newtype
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_cplx")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_cplx")
 #endif
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_write_plane.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_cplx")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_cplx")
 #endif
 
-  end subroutine write_plane_3d_complex
-
+   end subroutine write_plane_3d_complex
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write a 2D array to a file
+   ! Write a 2D array to a file
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !************** TO DO ***************
-  !* Consider handling distributed 2D data set
-  !  subroutine write_plane_2d(ipencil,var,filename)
-  !    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-  !    real(mytype), dimension(:,:), intent(IN) :: var ! 2D array
-  !    character(len=*), intent(IN) :: filename
-  !
-  !    if (ipencil==1) then
-  !       ! var should be defined as var(xsize(2)
-  !
-  !    else if (ipencil==2) then
-  !
-  !    else if (ipencil==3) then
-  !
-  !    end if
-  !
-  !    return
-  !  end subroutine write_plane_2d
-
+   !************** TO DO ***************
+   !* Consider handling distributed 2D data set
+   !  subroutine write_plane_2d(ipencil,var,filename)
+   !    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+   !    real(mytype), dimension(:,:), intent(IN) :: var ! 2D array
+   !    character(len=*), intent(IN) :: filename
+   !
+   !    if (ipencil==1) then
+   !       ! var should be defined as var(xsize(2)
+   !
+   !    else if (ipencil==2) then
+   !
+   !    else if (ipencil==3) then
+   !
+   !    end if
+   !
+   !    return
+   !  end subroutine write_plane_2d
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write 3D array data for every specified mesh point
+   ! Write 3D array data for every specified mesh point
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_every_real(ipencil,var,iskip,jskip,kskip, &
-       filename, from1)
+   subroutine write_every_real(ipencil, var, iskip, jskip, kskip, &
+                               filename, from1)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: iskip,jskip,kskip 
-    character(len=*), intent(IN) :: filename
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: iskip, jskip, kskip
+      character(len=*), intent(IN) :: filename
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
 
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: i,j,k, ierror, newtype, fh, key,color,newcomm, data_type
-    integer, dimension(3) :: xsz,ysz,zsz,xst,yst,zst,xen,yen,zen,skip
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: i, j, k, ierror, newtype, fh, key, color, newcomm, data_type
+      integer, dimension(3) :: xsz, ysz, zsz, xst, yst, zst, xen, yen, zen, skip
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_every_real")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_every_real")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_every.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_every_real")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_every_real")
 #endif
 
-  end subroutine write_every_real
-
+   end subroutine write_every_real
 
-  subroutine write_every_complex(ipencil,var,iskip,jskip,kskip, &
-       filename, from1)
+   subroutine write_every_complex(ipencil, var, iskip, jskip, kskip, &
+                                  filename, from1)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: iskip,jskip,kskip 
-    character(len=*), intent(IN) :: filename
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: iskip, jskip, kskip
+      character(len=*), intent(IN) :: filename
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: i,j,k, ierror, newtype, fh, key,color,newcomm, data_type
-    integer, dimension(3) :: xsz,ysz,zsz,xst,yst,zst,xen,yen,zen,skip
+      complex(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: i, j, k, ierror, newtype, fh, key, color, newcomm, data_type
+      integer, dimension(3) :: xsz, ysz, zsz, xst, yst, zst, xen, yen, zen, skip
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_every_cplx")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_every_cplx")
 #endif
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_write_every.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_every_cplx")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_every_cplx")
 #endif
 
-  end subroutine write_every_complex
+   end subroutine write_every_complex
 
-  subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
+   subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
 
-    implicit none
-    
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-    type(decomp_info), intent(in), optional :: opt_decomp
+      implicit none
 
-    integer, dimension(3) :: sizes, subsizes, starts
-    type(decomp_info) :: decomp
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
+      type(decomp_info), intent(in), optional :: opt_decomp
 
-    if ((icoarse.lt.0).or.(icoarse.gt.2)) then
-       call decomp_2d_abort(__FILE__, __LINE__, icoarse, "Error invalid value of icoarse")
-    endif
-    if ((ipencil.lt.1).or.(ipencil.gt.3)) then
-       call decomp_2d_abort(__FILE__, __LINE__, ipencil, "Error invalid value of ipencil ")
-    endif
+      integer, dimension(3) :: sizes, subsizes, starts
+      type(decomp_info) :: decomp
 
-    if (icoarse==0) then
+      if ((icoarse .lt. 0) .or. (icoarse .gt. 2)) then
+         call decomp_2d_abort(__FILE__, __LINE__, icoarse, "Error invalid value of icoarse")
+      end if
+      if ((ipencil .lt. 1) .or. (ipencil .gt. 3)) then
+         call decomp_2d_abort(__FILE__, __LINE__, ipencil, "Error invalid value of ipencil ")
+      end if
+
+      if (icoarse == 0) then
        !! Use full fields
-       
-       if (present(opt_decomp)) then
-          decomp = opt_decomp
-       else
-          call get_decomp_info(decomp)
-       endif
-
-       sizes(1) = decomp%xsz(1)
-       sizes(2) = decomp%ysz(2)
-       sizes(3) = decomp%zsz(3)
-
-       if (ipencil == 1) then
-          subsizes(1:3) = decomp%xsz(1:3)
-          starts(1:3) = decomp%xst(1:3) - 1
-       elseif (ipencil == 2) then
-          subsizes(1:3) = decomp%ysz(1:3)
-          starts(1:3) = decomp%yst(1:3) - 1
-       elseif (ipencil == 3) then
-          subsizes(1:3) = decomp%zsz(1:3)
-          starts(1:3) = decomp%zst(1:3) - 1
-       else
-          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
-       endif
-    elseif (icoarse==1) then
-       sizes(1) = xszS(1)
-       sizes(2) = yszS(2)
-       sizes(3) = zszS(3)
-
-       if (ipencil == 1) then
-          subsizes(1) = xszS(1)
-          subsizes(2) = xszS(2)
-          subsizes(3) = xszS(3)
-          starts(1) = xstS(1)-1  ! 0-based index
-          starts(2) = xstS(2)-1
-          starts(3) = xstS(3)-1
-       else if (ipencil == 2) then
-          subsizes(1) = yszS(1)
-          subsizes(2) = yszS(2)
-          subsizes(3) = yszS(3)
-          starts(1) = ystS(1)-1
-          starts(2) = ystS(2)-1
-          starts(3) = ystS(3)-1
-       else if (ipencil == 3) then
-          subsizes(1) = zszS(1)
-          subsizes(2) = zszS(2)
-          subsizes(3) = zszS(3)
-          starts(1) = zstS(1)-1
-          starts(2) = zstS(2)-1
-          starts(3) = zstS(3)-1
-       else
-          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
-       endif
-    elseif (icoarse==2) then
-       sizes(1) = xszV(1)
-       sizes(2) = yszV(2)
-       sizes(3) = zszV(3)
-
-       if (ipencil == 1) then
-          subsizes(1) = xszV(1)
-          subsizes(2) = xszV(2)
-          subsizes(3) = xszV(3)
-          starts(1) = xstV(1)-1  ! 0-based index
-          starts(2) = xstV(2)-1
-          starts(3) = xstV(3)-1
-       else if (ipencil == 2) then
-          subsizes(1) = yszV(1)
-          subsizes(2) = yszV(2)
-          subsizes(3) = yszV(3)
-          starts(1) = ystV(1)-1
-          starts(2) = ystV(2)-1
-          starts(3) = ystV(3)-1
-       else if (ipencil == 3) then
-          subsizes(1) = zszV(1)
-          subsizes(2) = zszV(2)
-          subsizes(3) = zszV(3)
-          starts(1) = zstV(1)-1
-          starts(2) = zstV(2)-1
-          starts(3) = zstV(3)-1
-       else
-          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
-       endif
-    endif
-    
-  end subroutine coarse_extents
-
-  subroutine mpiio_write_real_coarse(ipencil,var,dirname,varname,icoarse,io_name,opt_decomp,reduce_prec,opt_deferred_writes)
-
-    ! USE param
-    ! USE variables
-
-    implicit none
-
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    character(len=*), intent(in) :: dirname, varname, io_name
-    type(decomp_info), intent(in), optional :: opt_decomp
-    logical, intent(in), optional :: reduce_prec
-    logical, intent(in), optional :: opt_deferred_writes
-
-    logical :: write_reduce_prec
-    logical :: deferred_writes
-    
-    integer :: ierror
-    integer :: idx
-    logical :: opened_new
+
+         if (present(opt_decomp)) then
+            decomp = opt_decomp
+         else
+            call get_decomp_info(decomp)
+         end if
+
+         sizes(1) = decomp%xsz(1)
+         sizes(2) = decomp%ysz(2)
+         sizes(3) = decomp%zsz(3)
+
+         if (ipencil == 1) then
+            subsizes(1:3) = decomp%xsz(1:3)
+            starts(1:3) = decomp%xst(1:3) - 1
+         elseif (ipencil == 2) then
+            subsizes(1:3) = decomp%ysz(1:3)
+            starts(1:3) = decomp%yst(1:3) - 1
+         elseif (ipencil == 3) then
+            subsizes(1:3) = decomp%zsz(1:3)
+            starts(1:3) = decomp%zst(1:3) - 1
+         else
+            call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
+         end if
+      elseif (icoarse == 1) then
+         sizes(1) = xszS(1)
+         sizes(2) = yszS(2)
+         sizes(3) = zszS(3)
+
+         if (ipencil == 1) then
+            subsizes(1) = xszS(1)
+            subsizes(2) = xszS(2)
+            subsizes(3) = xszS(3)
+            starts(1) = xstS(1) - 1  ! 0-based index
+            starts(2) = xstS(2) - 1
+            starts(3) = xstS(3) - 1
+         else if (ipencil == 2) then
+            subsizes(1) = yszS(1)
+            subsizes(2) = yszS(2)
+            subsizes(3) = yszS(3)
+            starts(1) = ystS(1) - 1
+            starts(2) = ystS(2) - 1
+            starts(3) = ystS(3) - 1
+         else if (ipencil == 3) then
+            subsizes(1) = zszS(1)
+            subsizes(2) = zszS(2)
+            subsizes(3) = zszS(3)
+            starts(1) = zstS(1) - 1
+            starts(2) = zstS(2) - 1
+            starts(3) = zstS(3) - 1
+         else
+            call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
+         end if
+      elseif (icoarse == 2) then
+         sizes(1) = xszV(1)
+         sizes(2) = yszV(2)
+         sizes(3) = zszV(3)
+
+         if (ipencil == 1) then
+            subsizes(1) = xszV(1)
+            subsizes(2) = xszV(2)
+            subsizes(3) = xszV(3)
+            starts(1) = xstV(1) - 1  ! 0-based index
+            starts(2) = xstV(2) - 1
+            starts(3) = xstV(3) - 1
+         else if (ipencil == 2) then
+            subsizes(1) = yszV(1)
+            subsizes(2) = yszV(2)
+            subsizes(3) = yszV(3)
+            starts(1) = ystV(1) - 1
+            starts(2) = ystV(2) - 1
+            starts(3) = ystV(3) - 1
+         else if (ipencil == 3) then
+            subsizes(1) = zszV(1)
+            subsizes(2) = zszV(2)
+            subsizes(3) = zszV(3)
+            starts(1) = zstV(1) - 1
+            starts(2) = zstV(2) - 1
+            starts(3) = zstV(3) - 1
+         else
+            call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
+         end if
+      end if
+
+   end subroutine coarse_extents
+
+  subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_name, opt_decomp, reduce_prec, opt_deferred_writes)
+
+      ! USE param
+      ! USE variables
+
+      implicit none
+
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      character(len=*), intent(in) :: dirname, varname, io_name
+      type(decomp_info), intent(in), optional :: opt_decomp
+      logical, intent(in), optional :: reduce_prec
+      logical, intent(in), optional :: opt_deferred_writes
+
+      logical :: write_reduce_prec
+      logical :: deferred_writes
+
+      integer :: ierror
+      integer :: idx
+      logical :: opened_new
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
-    integer :: write_mode
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
+      integer :: write_mode
 #else
-    real(mytype_single), allocatable, dimension(:,:,:) :: varsingle
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: newtype
-    logical :: dir_exists
-    integer :: disp_bytes
-    character(len=:), allocatable :: full_io_name
+      real(mytype_single), allocatable, dimension(:, :, :) :: varsingle
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: newtype
+      logical :: dir_exists
+      integer :: disp_bytes
+      character(len=:), allocatable :: full_io_name
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_coarse")
+      if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_coarse")
 #endif
 
     !! Set defaults
-    write_reduce_prec = .true.
-    if (present(opt_deferred_writes)) then
-       deferred_writes = opt_deferred_writes
-    else
-       deferred_writes = .true.
-    end if
-    
-    opened_new = .false.
-    idx = get_io_idx(io_name, dirname)
+      write_reduce_prec = .true.
+      if (present(opt_deferred_writes)) then
+         deferred_writes = opt_deferred_writes
+      else
+         deferred_writes = .true.
+      end if
+
+      opened_new = .false.
+      idx = get_io_idx(io_name, dirname)
 #ifndef ADIOS2
-    if (present(reduce_prec)) then
-       if (.not. reduce_prec) then
-          write_reduce_prec = .false.
-       end if
-    end if
-    if (write_reduce_prec) then
-       call MPI_TYPE_SIZE(real_type_single,disp_bytes,ierror)
-    else
-       call MPI_TYPE_SIZE(real_type,disp_bytes,ierror)
-    end if
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    
+      if (present(reduce_prec)) then
+         if (.not. reduce_prec) then
+            write_reduce_prec = .false.
+         end if
+      end if
+      if (write_reduce_prec) then
+         call MPI_TYPE_SIZE(real_type_single, disp_bytes, ierror)
+      else
+         call MPI_TYPE_SIZE(real_type, disp_bytes, ierror)
+      end if
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+
     !! Use original MPIIO writers
-    if (present(opt_decomp)) then
-       call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
-    else
-       call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
-    end if
-    if (write_reduce_prec) then
-       allocate (varsingle(xstV(1):xenV(1),xstV(2):xenV(2),xstV(3):xenV(3)))
-       varsingle=real(var, mytype_single)
-    end if
-
-    if (write_reduce_prec) then
-       call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-            MPI_ORDER_FORTRAN, real_type_single, newtype, ierror)
-    else
-       call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-            MPI_ORDER_FORTRAN, real_type, newtype, ierror)
-    end if
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-    call MPI_TYPE_COMMIT(newtype,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-
-    if (idx .lt. 1) then
-       ! Create folder if needed
-       if (nrank==0) then
-          inquire(file=dirname, exist=dir_exists)
-          if (.not.dir_exists) then
-             call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
-          end if
-       end if
-       full_io_name = trim(dirname)//"/"//trim(varname)
-       call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
-       idx = get_io_idx(io_name, full_io_name)
-       opened_new = .true.
-    else
-       full_io_name = "" ! Ensure string is set
-    end if
-
-    if (write_reduce_prec) then
-       call MPI_FILE_SET_VIEW(fh_registry(idx),fh_disp(idx),real_type_single, &
-            newtype,'native',MPI_INFO_NULL,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-       call MPI_FILE_WRITE_ALL(fh_registry(idx), varsingle, &
-            subsizes(1)*subsizes(2)*subsizes(3), &
-            real_type_single, MPI_STATUS_IGNORE, ierror)
-    else
-       call MPI_FILE_SET_VIEW(fh_registry(idx),fh_disp(idx),real_type, &
-            newtype,'native',MPI_INFO_NULL,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-       call MPI_FILE_WRITE_ALL(fh_registry(idx), var, &
-            subsizes(1)*subsizes(2)*subsizes(3), &
-            real_type, MPI_STATUS_IGNORE, ierror)
-    end if
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    
-    fh_disp(idx) = fh_disp(idx) + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                * int(disp_bytes, kind=MPI_OFFSET_KIND)
-    
-    if (opened_new) then
-       call decomp_2d_close_io(io_name, full_io_name)
-       deallocate(full_io_name)
-    end if
-
-    call MPI_TYPE_FREE(newtype,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-    if (write_reduce_prec) then
-       deallocate(varsingle)
-    end if
+      if (present(opt_decomp)) then
+         call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
+      else
+         call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
+      end if
+      if (write_reduce_prec) then
+         allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
+         varsingle = real(var, mytype_single)
+      end if
+
+      if (write_reduce_prec) then
+         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                       MPI_ORDER_FORTRAN, real_type_single, newtype, ierror)
+      else
+         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                       MPI_ORDER_FORTRAN, real_type, newtype, ierror)
+      end if
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+      call MPI_TYPE_COMMIT(newtype, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+
+      if (idx .lt. 1) then
+         ! Create folder if needed
+         if (nrank == 0) then
+            inquire (file=dirname, exist=dir_exists)
+            if (.not. dir_exists) then
+               call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
+            end if
+         end if
+         full_io_name = trim(dirname)//"/"//trim(varname)
+         call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
+         idx = get_io_idx(io_name, full_io_name)
+         opened_new = .true.
+      else
+         full_io_name = "" ! Ensure string is set
+      end if
+
+      if (write_reduce_prec) then
+         call MPI_FILE_SET_VIEW(fh_registry(idx), fh_disp(idx), real_type_single, &
+                                newtype, 'native', MPI_INFO_NULL, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+         call MPI_FILE_WRITE_ALL(fh_registry(idx), varsingle, &
+                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 real_type_single, MPI_STATUS_IGNORE, ierror)
+      else
+         call MPI_FILE_SET_VIEW(fh_registry(idx), fh_disp(idx), real_type, &
+                                newtype, 'native', MPI_INFO_NULL, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+         call MPI_FILE_WRITE_ALL(fh_registry(idx), var, &
+                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 real_type, MPI_STATUS_IGNORE, ierror)
+      end if
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+
+      fh_disp(idx) = fh_disp(idx) + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                     *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                     *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                     *int(disp_bytes, kind=MPI_OFFSET_KIND)
+
+      if (opened_new) then
+         call decomp_2d_close_io(io_name, full_io_name)
+         deallocate (full_io_name)
+      end if
+
+      call MPI_TYPE_FREE(newtype, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+      if (write_reduce_prec) then
+         deallocate (varsingle)
+      end if
 #else
-    if (.not. engine_live(idx)) then
-       call decomp_2d_abort(__FILE__, __LINE__, -1, "ERROR: Engine is not live!")
-    end if
-    
-    call adios2_at_io(io_handle, adios, io_name, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-    call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-    if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-       "ERROR: trying to write variable before registering! "//trim(varname))
-
-    if (idx .lt. 1) call decomp_2d_abort(__FILE__, __LINE__, idx, &
-       "You haven't opened "//trim(io_name)//":"//trim(dirname))
-
-    if (deferred_writes) then
-       write_mode = adios2_mode_deferred
-    else
-       write_mode = adios2_mode_sync
-    end if
-
-    if (engine_registry(idx)%valid) then
-       call adios2_put(engine_registry(idx), var_handle, var, write_mode, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
-    else
-       call decomp_2d_abort(__FILE__, __LINE__, -1, &
-          "ERROR: decomp2d thinks engine is live, but adios2 engine object is not valid")
-    end if
-
-    associate(crs => icoarse, pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
-    end associate
+      if (.not. engine_live(idx)) then
+         call decomp_2d_abort(__FILE__, __LINE__, -1, "ERROR: Engine is not live!")
+      end if
+
+      call adios2_at_io(io_handle, adios, io_name, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+      call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+      if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                                                       "ERROR: trying to write variable before registering! "//trim(varname))
+
+      if (idx .lt. 1) call decomp_2d_abort(__FILE__, __LINE__, idx, &
+                                           "You haven't opened "//trim(io_name)//":"//trim(dirname))
+
+      if (deferred_writes) then
+         write_mode = adios2_mode_deferred
+      else
+         write_mode = adios2_mode_sync
+      end if
+
+      if (engine_registry(idx)%valid) then
+         call adios2_put(engine_registry(idx), var_handle, var, write_mode, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
+      else
+         call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                              "ERROR: decomp2d thinks engine is live, but adios2 engine object is not valid")
+      end if
+
+      associate (crs => icoarse, pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
+      end associate
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_coarse")
+      if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_coarse")
 #endif
 
-  end subroutine mpiio_write_real_coarse
+   end subroutine mpiio_write_real_coarse
 
-  subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplane, type, opt_decomp, opt_nplanes)
+   subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplane, type, opt_decomp, opt_nplanes)
 
-    use, intrinsic :: iso_fortran_env, only : real32, real64
+      use, intrinsic :: iso_fortran_env, only: real32, real64
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-    character(len=*), intent(in) :: io_name
-    integer, intent(in) :: type
-    integer, intent(in) :: iplane
-    type(decomp_info), intent(in), optional :: opt_decomp
-    integer, intent(in), optional :: opt_nplanes
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
+      character(len=*), intent(in) :: io_name
+      integer, intent(in) :: type
+      integer, intent(in) :: iplane
+      type(decomp_info), intent(in), optional :: opt_decomp
+      integer, intent(in), optional :: opt_nplanes
 
-    integer :: nplanes
-    character*(*), intent(in) :: varname
+      integer :: nplanes
+      character*(*), intent(in) :: varname
 #ifdef ADIOS2
-    integer, dimension(3) :: sizes, subsizes, starts
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
-    integer, parameter :: ndims = 3
-    logical, parameter :: adios2_constant_dims = .true.
-    integer :: data_type
-    integer :: ierror
-
-    if (iplane .eq. 0) then
-       if (present(opt_decomp)) then
-          call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
-       else
-          call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
-       endif
-    else
-       if (present(opt_nplanes)) then
-          nplanes = opt_nplanes
-       else
-          nplanes = 1
-       end if
-       if (present(opt_decomp)) then
-          call plane_extents(sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes=nplanes)
-       else
-          call plane_extents(sizes, subsizes, starts, iplane, opt_nplanes=nplanes)
-       endif
-    end if
-    
-    ! Check if variable already exists, if not create it
-    call adios2_at_io(io_handle, adios, io_name, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-    if (io_handle%valid) then
-       call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-       if (.not.var_handle % valid) then
+      integer, dimension(3) :: sizes, subsizes, starts
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
+      integer, parameter :: ndims = 3
+      logical, parameter :: adios2_constant_dims = .true.
+      integer :: data_type
+      integer :: ierror
+
+      if (iplane .eq. 0) then
+         if (present(opt_decomp)) then
+            call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
+         else
+            call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
+         end if
+      else
+         if (present(opt_nplanes)) then
+            nplanes = opt_nplanes
+         else
+            nplanes = 1
+         end if
+         if (present(opt_decomp)) then
+            call plane_extents(sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes=nplanes)
+         else
+            call plane_extents(sizes, subsizes, starts, iplane, opt_nplanes=nplanes)
+         end if
+      end if
+
+      ! Check if variable already exists, if not create it
+      call adios2_at_io(io_handle, adios, io_name, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+      if (io_handle%valid) then
+         call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+         if (.not. var_handle%valid) then
           !! New variable
-          if (nrank .eq. 0) then
-             print *, "Registering variable for IO: ", varname
-          endif
+            if (nrank .eq. 0) then
+               print *, "Registering variable for IO: ", varname
+            end if
 
-          ! Need to set the ADIOS2 data type
-          if (type.eq.kind(0._real64)) then
+            ! Need to set the ADIOS2 data type
+            if (type .eq. kind(0._real64)) then
              !! Double
-             data_type = adios2_type_dp
-          else if (type.eq.kind(0._real32)) then
+               data_type = adios2_type_dp
+            else if (type .eq. kind(0._real32)) then
              !! Single
-             data_type = adios2_type_real
-          else
-             call decomp_2d_abort(__FILE__, __LINE__, -1, "Trying to write unknown data type!")
-          endif
-
-          call adios2_define_variable(var_handle, io_handle, varname, data_type, &
-               ndims, int(sizes, kind=8), int(starts, kind=8), int(subsizes, kind=8), &
-               adios2_constant_dims, ierror)
-          if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-             "adios2_define_variable, ERROR registering variable "//trim(varname))
-       endif
-    else
-       call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to register variable with invalid IO!")
-    end if
+               data_type = adios2_type_real
+            else
+               call decomp_2d_abort(__FILE__, __LINE__, -1, "Trying to write unknown data type!")
+            end if
+
+            call adios2_define_variable(var_handle, io_handle, varname, data_type, &
+                                        ndims, int(sizes, kind=8), int(starts, kind=8), int(subsizes, kind=8), &
+                                        adios2_constant_dims, ierror)
+            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                                                    "adios2_define_variable, ERROR registering variable "//trim(varname))
+         end if
+      else
+         call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to register variable with invalid IO!")
+      end if
 #else
-    nplanes = 1 ! Silence unused variable
-    associate(crs => icoarse, nm =>io_name, pncl => ipencil, pln => iplane, &
-         opdcmp => opt_decomp, opnpl => opt_nplanes, tp => type, &
-         vnm => varname) ! Silence unused dummy argument
-    end associate
+      nplanes = 1 ! Silence unused variable
+      associate (crs => icoarse, nm => io_name, pncl => ipencil, pln => iplane, &
+                 opdcmp => opt_decomp, opnpl => opt_nplanes, tp => type, &
+                 vnm => varname) ! Silence unused dummy argument
+      end associate
 #endif
-    
-  end subroutine decomp_2d_register_variable
-  
-  subroutine mpiio_write_real_probe(ipencil,var,filename,nlength)
 
-    ! USE param
-    ! USE variables
+   end subroutine decomp_2d_register_variable
+
+   subroutine mpiio_write_real_probe(ipencil, var, filename, nlength)
+
+      ! USE param
+      ! USE variables
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    integer, intent(in) :: nlength
-    real(mytype), contiguous, dimension(:,:,:,:), intent(IN) :: var
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      integer, intent(in) :: nlength
+      real(mytype), contiguous, dimension(:, :, :, :), intent(IN) :: var
 
-    character(len=*) :: filename
+      character(len=*) :: filename
 
-    integer (kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(4) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(4) :: sizes, subsizes, starts
+      integer :: ierror, newtype, fh
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_probe")
-#endif
-
-    sizes(1) = xszP(1)
-    sizes(2) = yszP(2)
-    sizes(3) = zszP(3)
-    sizes(4) = nlength
-    if (ipencil == 1) then
-       subsizes(1) = xszP(1)
-       subsizes(2) = xszP(2)
-       subsizes(3) = xszP(3)
-       subsizes(4) = nlength
-       starts(1) = xstP(1)-1  ! 0-based index
-       starts(2) = xstP(2)-1
-       starts(3) = xstP(3)-1
-       starts(4) = 0
-    else if (ipencil == 2) then
-       subsizes(1) = yszP(1)
-       subsizes(2) = yszP(2)
-       subsizes(3) = yszP(3)
-       starts(1) = ystP(1)-1
-       starts(2) = ystP(2)-1
-       starts(3) = ystP(3)-1
-    else if (ipencil == 3) then
-       subsizes(1) = zszP(1)
-       subsizes(2) = zszP(2)
-       subsizes(3) = zszP(3)
-       starts(1) = zstP(1)-1
-       starts(2) = zstP(2)-1
-       starts(3) = zstP(3)-1
-    endif
-    !   print *,nrank,starts(1),starts(2),starts(3),starts(4)
-    call MPI_TYPE_CREATE_SUBARRAY(4, sizes, subsizes, starts,  &
-         MPI_ORDER_FORTRAN, real_type, newtype, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-    call MPI_TYPE_COMMIT(newtype,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-    call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-         MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-         fh, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
-    filesize = 0_MPI_OFFSET_KIND
-    call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
-    disp = 0_MPI_OFFSET_KIND
-    call MPI_FILE_SET_VIEW(fh,disp,real_type, &
-         newtype,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_WRITE_ALL(fh, var, &
-         subsizes(1)*subsizes(2)*subsizes(3)*subsizes(4), &
-         real_type, MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    call MPI_FILE_CLOSE(fh,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-    call MPI_TYPE_FREE(newtype,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+      if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_probe")
+#endif
+
+      sizes(1) = xszP(1)
+      sizes(2) = yszP(2)
+      sizes(3) = zszP(3)
+      sizes(4) = nlength
+      if (ipencil == 1) then
+         subsizes(1) = xszP(1)
+         subsizes(2) = xszP(2)
+         subsizes(3) = xszP(3)
+         subsizes(4) = nlength
+         starts(1) = xstP(1) - 1  ! 0-based index
+         starts(2) = xstP(2) - 1
+         starts(3) = xstP(3) - 1
+         starts(4) = 0
+      else if (ipencil == 2) then
+         subsizes(1) = yszP(1)
+         subsizes(2) = yszP(2)
+         subsizes(3) = yszP(3)
+         starts(1) = ystP(1) - 1
+         starts(2) = ystP(2) - 1
+         starts(3) = ystP(3) - 1
+      else if (ipencil == 3) then
+         subsizes(1) = zszP(1)
+         subsizes(2) = zszP(2)
+         subsizes(3) = zszP(3)
+         starts(1) = zstP(1) - 1
+         starts(2) = zstP(2) - 1
+         starts(3) = zstP(3) - 1
+      end if
+      !   print *,nrank,starts(1),starts(2),starts(3),starts(4)
+      call MPI_TYPE_CREATE_SUBARRAY(4, sizes, subsizes, starts, &
+                                    MPI_ORDER_FORTRAN, real_type, newtype, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+      call MPI_TYPE_COMMIT(newtype, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+      call MPI_FILE_OPEN(decomp_2d_comm, filename, &
+                         MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                         fh, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+      filesize = 0_MPI_OFFSET_KIND
+      call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+      disp = 0_MPI_OFFSET_KIND
+      call MPI_FILE_SET_VIEW(fh, disp, real_type, &
+                             newtype, 'native', MPI_INFO_NULL, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_WRITE_ALL(fh, var, &
+                              subsizes(1)*subsizes(2)*subsizes(3)*subsizes(4), &
+                              real_type, MPI_STATUS_IGNORE, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      call MPI_FILE_CLOSE(fh, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+      call MPI_TYPE_FREE(newtype, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_probe")
+      if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_probe")
 #endif
 
-  end subroutine mpiio_write_real_probe
+   end subroutine mpiio_write_real_probe
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write a 3D data set covering a smaller sub-domain only
+   ! Write a 3D data set covering a smaller sub-domain only
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_subdomain(ipencil,var,is,ie,js,je,ks,ke,filename)
+   subroutine write_subdomain(ipencil, var, is, ie, js, je, ks, ke, filename)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: is, ie, js, je, ks, ke
-    character(len=*), intent(IN) :: filename
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: is, ie, js, je, ks, ke
+      character(len=*), intent(IN) :: filename
 
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: color, key, errorcode, newcomm, ierror
-    integer :: newtype, fh, data_type, i, j, k
-    integer :: i1, i2, j1, j2, k1, k2
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: color, key, errorcode, newcomm, ierror
+      integer :: newtype, fh, data_type, i, j, k
+      integer :: i1, i2, j1, j2, k1, k2
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_subdomain")
-#endif
-
-    data_type = real_type
-
-    ! validate the input paramters
-    if (is<1 .OR. ie>nx_global .OR. js<1 .OR. je>ny_global .OR. &
-         ks<1 .OR. ke>nz_global) then
-       errorcode = 10
-       call decomp_2d_abort(errorcode, &
-            'Invalid subdomain specified in I/O')
-    end if
-
-    ! create a communicator for all those MPI ranks containing the subdomain
-    color = 1
-    key = 1
-    if (ipencil==1) then
-       if (xstart(1)>ie .OR. xend(1)<is .OR. xstart(2)>je .OR. xend(2)<js &
-            .OR. xstart(3)>ke .OR. xend(3)<ks) then
-          color = 2
-       end if
-    else if (ipencil==2) then
-       if (ystart(1)>ie .OR. yend(1)<is .OR. ystart(2)>je .OR. yend(2)<js &
-            .OR. ystart(3)>ke .OR. yend(3)<ks) then
-          color = 2
-       end if
-    else if (ipencil==3) then
-       if (zstart(1)>ie .OR. zend(1)<is .OR. zstart(2)>je .OR. zend(2)<js &
-            .OR. zstart(3)>ke .OR. zend(3)<ks) then
-          color = 2
-       end if
-    end if
-    call MPI_COMM_SPLIT(decomp_2d_comm,color,key,newcomm,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-
-    if (color==1) then ! only ranks in this group do IO collectively
-
-       ! generate MPI-IO subarray information
-
-       ! global size of the sub-domain to write
-       sizes(1) = ie - is + 1
-       sizes(2) = je - js + 1
-       sizes(3) = ke - ks + 1
-
-       ! 'subsizes' & 'starts' as required by MPI_TYPE_CREATE_SUBARRAY
-       ! note the special code whe subdomain only occupy part of the pencil
-       if (ipencil==1) then
-
-          subsizes(1) = xsize(1)
-          starts(1) = xstart(1) - is
-          if (xend(1)>ie .AND. xstart(1)<is) then
-             subsizes(1) = ie - is + 1
-             starts(1) = 0
-          else if (xstart(1)<is) then
-             subsizes(1) = xend(1) - is + 1
-             starts(1) = 0
-          else if (xend(1)>ie) then
-             subsizes(1) = ie - xstart(1) + 1
-          end if
-          subsizes(2) = xsize(2)
-          starts(2) = xstart(2) - js
-          if (xend(2)>je .AND. xstart(2)<js) then
-             subsizes(2) = je - js + 1
-             starts(2) = 0
-          else if (xstart(2)<js) then
-             subsizes(2) = xend(2) - js + 1
-             starts(2) = 0
-          else if (xend(2)>je) then
-             subsizes(2) = je - xstart(2) + 1
-          end if
-          subsizes(3) = xsize(3)
-          starts(3) = xstart(3) - ks
-          if (xend(3)>ke .AND. xstart(3)<ks) then
-             subsizes(3) = ke - ks + 1
-             starts(3) = 0
-          else if (xstart(3)<ks) then
-             subsizes(3) = xend(3) - ks + 1
-             starts(3) = 0
-          else if (xend(3)>ke) then
-             subsizes(3) = ke - xstart(3) + 1
-          end if
-
-       else if (ipencil==2) then
-
-          ! TODO 
-
-       else if (ipencil==3) then
-
-          ! TODO
-
-       end if
-
-
-       ! copy data from orginal to a temp array
-       ! pay attention to blocks only partially cover the sub-domain
-       if (ipencil==1) then
-
-          if (xend(1)>ie .AND. xstart(1)<is) then
-             i1 = is
-             i2 = ie
-          else if (xend(1)>ie) then
-             i1 = xstart(1)
-             i2 = ie
-          else if (xstart(1)<is) then
-             i1 = is
-             i2 = xend(1)
-          else
-             i1 = xstart(1)
-             i2 = xend(1)
-          end if
-
-          if (xend(2)>je .AND. xstart(2)<js) then
-             j1 = js
-             j2 = je
-          else if (xend(2)>je) then
-             j1 = xstart(2)
-             j2 = je
-          else if (xstart(2)<js) then
-             j1 = js
-             j2 = xend(2)
-          else
-             j1 = xstart(2)
-             j2 = xend(2)
-          end if
-
-          if (xend(3)>ke .AND. xstart(3)<ks) then
-             k1 = ks
-             k2 = ke
-          else if (xend(3)>ke) then
-             k1 = xstart(3)
-             k2 = ke
-          else if (xstart(3)<ks) then
-             k1 = ks
-             k2 = xend(3)
-          else
-             k1 = xstart(3)
-             k2 = xend(3)
-          end if
-
-          allocate(wk(i1:i2, j1:j2, k1:k2))
-          allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-          wk2 = var
-          do k=k1,k2
-             do j=j1,j2
-                do i=i1,i2
-                   wk(i,j,k) = wk2(i,j,k)
-                end do
-             end do
-          end do
-
-       else if (ipencil==2) then
-
-          ! TODO
-
-       else if (ipencil==3) then
-
-          ! TODO
-
-       end if
-
-       deallocate(wk2)
-
-       ! MPI-IO
-       call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-            MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-       call MPI_TYPE_COMMIT(newtype,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-       call MPI_FILE_OPEN(newcomm, filename, &
-            MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-            fh, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
-       filesize = 0_MPI_OFFSET_KIND
-       call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
-       disp = 0_MPI_OFFSET_KIND
-       call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-            newtype,'native',MPI_INFO_NULL,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-       call MPI_FILE_WRITE_ALL(fh, wk, &
-            subsizes(1)*subsizes(2)*subsizes(3), &
-            data_type, MPI_STATUS_IGNORE, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-       call MPI_FILE_CLOSE(fh,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-       call MPI_TYPE_FREE(newtype,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-
-       deallocate(wk)
-
-    end if
-
-    call decomp_mpi_comm_free(newcomm)
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_subdomain")
+#endif
+
+      data_type = real_type
+
+      ! validate the input paramters
+      if (is < 1 .OR. ie > nx_global .OR. js < 1 .OR. je > ny_global .OR. &
+          ks < 1 .OR. ke > nz_global) then
+         errorcode = 10
+         call decomp_2d_abort(errorcode, &
+                              'Invalid subdomain specified in I/O')
+      end if
+
+      ! create a communicator for all those MPI ranks containing the subdomain
+      color = 1
+      key = 1
+      if (ipencil == 1) then
+         if (xstart(1) > ie .OR. xend(1) < is .OR. xstart(2) > je .OR. xend(2) < js &
+             .OR. xstart(3) > ke .OR. xend(3) < ks) then
+            color = 2
+         end if
+      else if (ipencil == 2) then
+         if (ystart(1) > ie .OR. yend(1) < is .OR. ystart(2) > je .OR. yend(2) < js &
+             .OR. ystart(3) > ke .OR. yend(3) < ks) then
+            color = 2
+         end if
+      else if (ipencil == 3) then
+         if (zstart(1) > ie .OR. zend(1) < is .OR. zstart(2) > je .OR. zend(2) < js &
+             .OR. zstart(3) > ke .OR. zend(3) < ks) then
+            color = 2
+         end if
+      end if
+      call MPI_COMM_SPLIT(decomp_2d_comm, color, key, newcomm, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+
+      if (color == 1) then ! only ranks in this group do IO collectively
+
+         ! generate MPI-IO subarray information
+
+         ! global size of the sub-domain to write
+         sizes(1) = ie - is + 1
+         sizes(2) = je - js + 1
+         sizes(3) = ke - ks + 1
+
+         ! 'subsizes' & 'starts' as required by MPI_TYPE_CREATE_SUBARRAY
+         ! note the special code whe subdomain only occupy part of the pencil
+         if (ipencil == 1) then
+
+            subsizes(1) = xsize(1)
+            starts(1) = xstart(1) - is
+            if (xend(1) > ie .AND. xstart(1) < is) then
+               subsizes(1) = ie - is + 1
+               starts(1) = 0
+            else if (xstart(1) < is) then
+               subsizes(1) = xend(1) - is + 1
+               starts(1) = 0
+            else if (xend(1) > ie) then
+               subsizes(1) = ie - xstart(1) + 1
+            end if
+            subsizes(2) = xsize(2)
+            starts(2) = xstart(2) - js
+            if (xend(2) > je .AND. xstart(2) < js) then
+               subsizes(2) = je - js + 1
+               starts(2) = 0
+            else if (xstart(2) < js) then
+               subsizes(2) = xend(2) - js + 1
+               starts(2) = 0
+            else if (xend(2) > je) then
+               subsizes(2) = je - xstart(2) + 1
+            end if
+            subsizes(3) = xsize(3)
+            starts(3) = xstart(3) - ks
+            if (xend(3) > ke .AND. xstart(3) < ks) then
+               subsizes(3) = ke - ks + 1
+               starts(3) = 0
+            else if (xstart(3) < ks) then
+               subsizes(3) = xend(3) - ks + 1
+               starts(3) = 0
+            else if (xend(3) > ke) then
+               subsizes(3) = ke - xstart(3) + 1
+            end if
+
+         else if (ipencil == 2) then
+
+            ! TODO
+
+         else if (ipencil == 3) then
+
+            ! TODO
+
+         end if
+
+         ! copy data from orginal to a temp array
+         ! pay attention to blocks only partially cover the sub-domain
+         if (ipencil == 1) then
+
+            if (xend(1) > ie .AND. xstart(1) < is) then
+               i1 = is
+               i2 = ie
+            else if (xend(1) > ie) then
+               i1 = xstart(1)
+               i2 = ie
+            else if (xstart(1) < is) then
+               i1 = is
+               i2 = xend(1)
+            else
+               i1 = xstart(1)
+               i2 = xend(1)
+            end if
+
+            if (xend(2) > je .AND. xstart(2) < js) then
+               j1 = js
+               j2 = je
+            else if (xend(2) > je) then
+               j1 = xstart(2)
+               j2 = je
+            else if (xstart(2) < js) then
+               j1 = js
+               j2 = xend(2)
+            else
+               j1 = xstart(2)
+               j2 = xend(2)
+            end if
+
+            if (xend(3) > ke .AND. xstart(3) < ks) then
+               k1 = ks
+               k2 = ke
+            else if (xend(3) > ke) then
+               k1 = xstart(3)
+               k2 = ke
+            else if (xstart(3) < ks) then
+               k1 = ks
+               k2 = xend(3)
+            else
+               k1 = xstart(3)
+               k2 = xend(3)
+            end if
+
+            allocate (wk(i1:i2, j1:j2, k1:k2))
+            allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+            wk2 = var
+            do k = k1, k2
+               do j = j1, j2
+                  do i = i1, i2
+                     wk(i, j, k) = wk2(i, j, k)
+                  end do
+               end do
+            end do
+
+         else if (ipencil == 2) then
+
+            ! TODO
+
+         else if (ipencil == 3) then
+
+            ! TODO
+
+         end if
+
+         deallocate (wk2)
+
+         ! MPI-IO
+         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+         call MPI_TYPE_COMMIT(newtype, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+         call MPI_FILE_OPEN(newcomm, filename, &
+                            MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                            fh, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+         filesize = 0_MPI_OFFSET_KIND
+         call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+         disp = 0_MPI_OFFSET_KIND
+         call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                                newtype, 'native', MPI_INFO_NULL, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+         call MPI_FILE_WRITE_ALL(fh, wk, &
+                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 data_type, MPI_STATUS_IGNORE, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+         call MPI_FILE_CLOSE(fh, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+         call MPI_TYPE_FREE(newtype, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+
+         deallocate (wk)
+
+      end if
+
+      call decomp_mpi_comm_free(newcomm)
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_subdomain")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_subdomain")
 #endif
 
-  end subroutine write_subdomain
+   end subroutine write_subdomain
 
-  subroutine decomp_2d_init_io(io_name)
+   subroutine decomp_2d_init_io(io_name)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: io_name
+      character(len=*), intent(in) :: io_name
 #ifdef ADIOS2
-    integer :: ierror
-    type(adios2_io) :: io
+      integer :: ierror
+      type(adios2_io) :: io
 #endif
-    
-    if (nrank .eq. 0) then
-       print *, "Initialising IO for ", io_name
-    end if
+
+      if (nrank .eq. 0) then
+         print *, "Initialising IO for ", io_name
+      end if
 
 #ifdef ADIOS2
-    if (adios%valid) then
-       call adios2_declare_io(io, adios, io_name, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_declare_io "//trim(io_name))
-    else
-       call decomp_2d_abort(__FILE__, __LINE__, -1, "couldn't declare IO - adios object not valid")
-    end if
-#endif
-    
-  end subroutine decomp_2d_init_io
-  
-  subroutine decomp_2d_open_io(io_name, io_dir, mode)
-
-    implicit none
-
-    character(len=*), intent(in) :: io_name, io_dir
-    integer, intent(in) :: mode
-
-    logical, dimension(:), pointer :: live_ptrh
-    character(len=1024), dimension(:), pointer :: names_ptr
-    character(len=(len(io_name)+len(io_sep)+len(io_dir))) :: full_name
-    
-    integer :: idx, ierror
-    integer :: access_mode
+      if (adios%valid) then
+         call adios2_declare_io(io, adios, io_name, ierror)
+         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_declare_io "//trim(io_name))
+      else
+         call decomp_2d_abort(__FILE__, __LINE__, -1, "couldn't declare IO - adios object not valid")
+      end if
+#endif
+
+   end subroutine decomp_2d_init_io
+
+   subroutine decomp_2d_open_io(io_name, io_dir, mode)
+
+      implicit none
+
+      character(len=*), intent(in) :: io_name, io_dir
+      integer, intent(in) :: mode
+
+      logical, dimension(:), pointer :: live_ptrh
+      character(len=1024), dimension(:), pointer :: names_ptr
+      character(len=(len(io_name) + len(io_sep) + len(io_dir))) :: full_name
+
+      integer :: idx, ierror
+      integer :: access_mode
 #ifndef ADIOS2
-    integer(MPI_OFFSET_KIND) :: filesize
+      integer(MPI_OFFSET_KIND) :: filesize
 #else
-    type(adios2_io) :: io
+      type(adios2_io) :: io
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_open_close")
+      if (decomp_profiler_io) call decomp_profiler_start("io_open_close")
 #endif
 
 #ifndef ADIOS2
-    live_ptrh => fh_live
-    names_ptr => fh_names
+      live_ptrh => fh_live
+      names_ptr => fh_names
 #else
-    live_ptrh => engine_live
-    names_ptr => engine_names
+      live_ptrh => engine_live
+      names_ptr => engine_names
 #endif
-    
-    idx = get_io_idx(io_name, io_dir)
-    if (idx .lt. 1) then
+
+      idx = get_io_idx(io_name, io_dir)
+      if (idx .lt. 1) then
        !! New io destination
-       if (nreg_io .lt. MAX_IOH) then
-          nreg_io = nreg_io + 1
-          do idx = 1, MAX_IOH
-             if (.not. live_ptrh(idx)) then
-                live_ptrh(idx) = .true.
-                exit
-             end if
-          end do
-          
-          full_name = io_name//io_sep//io_dir
-          names_ptr(idx) = full_name
-
-          if (mode .eq. decomp_2d_write_mode) then
+         if (nreg_io .lt. MAX_IOH) then
+            nreg_io = nreg_io + 1
+            do idx = 1, MAX_IOH
+               if (.not. live_ptrh(idx)) then
+                  live_ptrh(idx) = .true.
+                  exit
+               end if
+            end do
+
+            full_name = io_name//io_sep//io_dir
+            names_ptr(idx) = full_name
+
+            if (mode .eq. decomp_2d_write_mode) then
              !! Setup writers
 #ifndef ADIOS2
-             filesize = 0_MPI_OFFSET_KIND
-             fh_disp(idx) = 0_MPI_OFFSET_KIND
-             access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
+               filesize = 0_MPI_OFFSET_KIND
+               fh_disp(idx) = 0_MPI_OFFSET_KIND
+               access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
 #else
-             access_mode = adios2_mode_write
+               access_mode = adios2_mode_write
 #endif
-          else if (mode .eq. decomp_2d_read_mode) then
+            else if (mode .eq. decomp_2d_read_mode) then
              !! Setup readers
 #ifndef ADIOS2
-             fh_disp(idx) = 0_MPI_OFFSET_KIND
-             access_mode = MPI_MODE_RDONLY
+               fh_disp(idx) = 0_MPI_OFFSET_KIND
+               access_mode = MPI_MODE_RDONLY
 #else
-             access_mode = adios2_mode_read
+               access_mode = adios2_mode_read
 #endif
-          else if (mode .eq. decomp_2d_append_mode) then
+            else if (mode .eq. decomp_2d_append_mode) then
 #ifndef ADIOS2
-             filesize = 0_MPI_OFFSET_KIND
-             fh_disp(idx) = 0_MPI_OFFSET_KIND
-             access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
+               filesize = 0_MPI_OFFSET_KIND
+               fh_disp(idx) = 0_MPI_OFFSET_KIND
+               access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
 #else
-             access_mode = adios2_mode_append
+               access_mode = adios2_mode_append
 #endif
-          else
-             print *, "ERROR: Unknown mode!"
-             stop
-          endif
+            else
+               print *, "ERROR: Unknown mode!"
+               stop
+            end if
 
           !! Open IO
 #ifndef ADIOS2
-          call MPI_FILE_OPEN(decomp_2d_comm, io_dir, &
-               access_mode, MPI_INFO_NULL, &
-               fh_registry(idx), ierror)
-          if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
-          if (mode .eq. decomp_2d_write_mode) then
+            call MPI_FILE_OPEN(decomp_2d_comm, io_dir, &
+                               access_mode, MPI_INFO_NULL, &
+                               fh_registry(idx), ierror)
+            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+            if (mode .eq. decomp_2d_write_mode) then
              !! Guarantee overwriting
-             call MPI_FILE_SET_SIZE(fh_registry(idx), filesize, ierror)
-             if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
-          end if
+               call MPI_FILE_SET_SIZE(fh_registry(idx), filesize, ierror)
+               if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+            end if
 #else
-          call adios2_at_io(io, adios, io_name, ierror)
-          if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-          if (io%valid) then
-             call adios2_open(engine_registry(idx), io, trim(gen_iodir_name(io_dir, io_name)), access_mode, ierror)
-             if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "ERROR opening engine!")
-          else
-             call decomp_2d_abort(__FILE__, __LINE__, -1, "Couldn't find IO handle")
-          end if
-#endif
-       end if
-    end if
+            call adios2_at_io(io, adios, io_name, ierror)
+            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+            if (io%valid) then
+               call adios2_open(engine_registry(idx), io, trim(gen_iodir_name(io_dir, io_name)), access_mode, ierror)
+               if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "ERROR opening engine!")
+            else
+               call decomp_2d_abort(__FILE__, __LINE__, -1, "Couldn't find IO handle")
+            end if
+#endif
+         end if
+      end if
+
+   end subroutine decomp_2d_open_io
 
-  end subroutine decomp_2d_open_io
+   subroutine decomp_2d_close_io(io_name, io_dir)
 
-  subroutine decomp_2d_close_io(io_name, io_dir)
+      implicit none
 
-    implicit none
+      character(len=*), intent(in) :: io_name, io_dir
 
-    character(len=*), intent(in) :: io_name, io_dir
-    
-    character(len=1024), dimension(:), pointer :: names_ptr
-    logical, dimension(:), pointer :: live_ptrh
-    integer :: idx, ierror
+      character(len=1024), dimension(:), pointer :: names_ptr
+      logical, dimension(:), pointer :: live_ptrh
+      integer :: idx, ierror
 
-    idx = get_io_idx(io_name, io_dir)
+      idx = get_io_idx(io_name, io_dir)
 #ifndef ADIOS2
-    names_ptr => fh_names
-    live_ptrh => fh_live
-    call MPI_FILE_CLOSE(fh_registry(idx), ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+      names_ptr => fh_names
+      live_ptrh => fh_live
+      call MPI_FILE_CLOSE(fh_registry(idx), ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
 #else
-    names_ptr => engine_names
-    live_ptrh => engine_live
-    call adios2_close(engine_registry(idx), ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_close")
+      names_ptr => engine_names
+      live_ptrh => engine_live
+      call adios2_close(engine_registry(idx), ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_close")
 #endif
-    names_ptr(idx) = ""
-    live_ptrh(idx) = .false.
-    nreg_io = nreg_io - 1
+      names_ptr(idx) = ""
+      live_ptrh(idx) = .false.
+      nreg_io = nreg_io - 1
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_open_close")
+      if (decomp_profiler_io) call decomp_profiler_end("io_open_close")
 #endif
 
-  end subroutine decomp_2d_close_io
+   end subroutine decomp_2d_close_io
 
-  subroutine decomp_2d_start_io(io_name, io_dir)
+   subroutine decomp_2d_start_io(io_name, io_dir)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: io_name, io_dir
+      character(len=*), intent(in) :: io_name, io_dir
 #ifdef ADIOS2
-    integer :: idx, ierror
-
-    idx = get_io_idx(io_name, io_dir)
-    associate(engine => engine_registry(idx))
-      if (engine%valid) then
-         call adios2_begin_step(engine, ierror)
-         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_begin_step")
-      else
-         call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to begin step with invalid engine")
-      end if
-    end associate
+      integer :: idx, ierror
+
+      idx = get_io_idx(io_name, io_dir)
+      associate (engine => engine_registry(idx))
+         if (engine%valid) then
+            call adios2_begin_step(engine, ierror)
+            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_begin_step")
+         else
+            call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to begin step with invalid engine")
+         end if
+      end associate
 #else
-    associate(nm => io_name, dr => io_dir) ! Silence unused dummy argument
-    end associate
+      associate (nm => io_name, dr => io_dir) ! Silence unused dummy argument
+      end associate
 #endif
-    
-  end subroutine decomp_2d_start_io
 
-  subroutine decomp_2d_end_io(io_name, io_dir)
+   end subroutine decomp_2d_start_io
 
-    implicit none
+   subroutine decomp_2d_end_io(io_name, io_dir)
 
-    character(len=*), intent(in) :: io_name, io_dir
-#ifdef ADIOS2
-    integer :: idx, ierror
+      implicit none
 
-    idx = get_io_idx(io_name, io_dir)
-    associate(engine => engine_registry(idx))
-      if (engine%valid) then
-         call adios2_end_step(engine, ierror)
-         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_end_step")
-      else
-         call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to end step with invalid engine")
-      end if
-    end associate
+      character(len=*), intent(in) :: io_name, io_dir
+#ifdef ADIOS2
+      integer :: idx, ierror
+
+      idx = get_io_idx(io_name, io_dir)
+      associate (engine => engine_registry(idx))
+         if (engine%valid) then
+            call adios2_end_step(engine, ierror)
+            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_end_step")
+         else
+            call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to end step with invalid engine")
+         end if
+      end associate
 #else
-    associate(nm => io_name, dr => io_dir) ! Silence unused dummy argument
-    end associate
+      associate (nm => io_name, dr => io_dir) ! Silence unused dummy argument
+      end associate
 #endif
 
-  end subroutine decomp_2d_end_io
-  
-  integer function get_io_idx(io_name, engine_name)
+   end subroutine decomp_2d_end_io
+
+   integer function get_io_idx(io_name, engine_name)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: io_name
-    character(len=*), intent(in) :: engine_name
+      character(len=*), intent(in) :: io_name
+      character(len=*), intent(in) :: engine_name
 
-    character(len=(len(io_name)+len(io_sep)+len(engine_name))) :: full_name
-    integer :: idx
-    logical :: found
+      character(len=(len(io_name) + len(io_sep) + len(engine_name))) :: full_name
+      integer :: idx
+      logical :: found
 
-    character(len=1024), dimension(:), pointer :: names_ptr
+      character(len=1024), dimension(:), pointer :: names_ptr
 
 #ifndef ADIOS2
-    names_ptr => fh_names
+      names_ptr => fh_names
 #else
-    names_ptr => engine_names
+      names_ptr => engine_names
 #endif
 
-    full_name = io_name//io_sep//engine_name
-    
-    found = .false.
-    do idx = 1, MAX_IOH
-       if (names_ptr(idx) .eq. full_name) then
-          found = .true.
-          exit
-       end if
-    end do
+      full_name = io_name//io_sep//engine_name
+
+      found = .false.
+      do idx = 1, MAX_IOH
+         if (names_ptr(idx) .eq. full_name) then
+            found = .true.
+            exit
+         end if
+      end do
+
+      if (.not. found) then
+         idx = -1
+      end if
 
-    if (.not. found) then
-       idx = -1
-    end if
+      get_io_idx = idx
 
-    get_io_idx = idx
-    
-  end function get_io_idx
+   end function get_io_idx
 
-  function gen_iodir_name(io_dir, io_name)
+   function gen_iodir_name(io_dir, io_name)
 
-    character(len=*), intent(in) :: io_dir, io_name
-    character(len=(len(io_dir) + 5)) :: gen_iodir_name
+      character(len=*), intent(in) :: io_dir, io_name
+      character(len=(len(io_dir) + 5)) :: gen_iodir_name
 #ifdef ADIOS2
-    integer :: ierror
-    type(adios2_io) :: io
-    character(len=5) :: ext
+      integer :: ierror
+      type(adios2_io) :: io
+      character(len=5) :: ext
 #endif
 
 #ifndef ADIOS2
-    associate(nm => io_name) ! Silence unused dummy argument
-    end associate
-    write(gen_iodir_name, "(A)") io_dir
+      associate (nm => io_name) ! Silence unused dummy argument
+      end associate
+      write (gen_iodir_name, "(A)") io_dir
 #else
-    call adios2_at_io(io, adios, io_name, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-    if (io%engine_type .eq. "BP4") then
-       ext = ".bp4"
-    else if (io%engine_type .eq. "HDF5") then
-       ext = ".hdf5"
-    else if (io%engine_type .eq. "SST") then
-       ext = ""
-    else
-       print *, "ERROR: Unkown engine type! ", io%engine_type
-       print *, "-  IO: ", io_name
-       print *, "- DIR:", io_dir
-       stop
-    endif
-    write(gen_iodir_name, "(A,A)") io_dir, trim(ext)
-#endif
-    
-  end function gen_iodir_name
+      call adios2_at_io(io, adios, io_name, ierror)
+      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+      if (io%engine_type .eq. "BP4") then
+         ext = ".bp4"
+      else if (io%engine_type .eq. "HDF5") then
+         ext = ".hdf5"
+      else if (io%engine_type .eq. "SST") then
+         ext = ""
+      else
+         print *, "ERROR: Unkown engine type! ", io%engine_type
+         print *, "-  IO: ", io_name
+         print *, "- DIR:", io_dir
+         stop
+      end if
+      write (gen_iodir_name, "(A,A)") io_dir, trim(ext)
+#endif
+
+   end function gen_iodir_name
 
 end module decomp_2d_io
diff --git a/src/io_read_inflow.f90 b/src/io_read_inflow.f90
index 3e5a9fb1..4d2df417 100644
--- a/src/io_read_inflow.f90
+++ b/src/io_read_inflow.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,12 +9,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'read_var_...' in io.f90
 
-! Using MPI-IO to read a distributed 3D variable from a file. File 
-! operations (open/close) need to be done in calling application. This 
-! allows multiple variables to be read from a single file. Together 
+! Using MPI-IO to read a distributed 3D variable from a file. File
+! operations (open/close) need to be done in calling application. This
+! allows multiple variables to be read from a single file. Together
 ! with the corresponding write operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -25,62 +25,62 @@
 end if
 
 ! Create file type and set file view
-sizes(1) = ntimesteps 
+sizes(1) = ntimesteps
 sizes(2) = decomp%ysz(2)
 sizes(3) = decomp%zsz(3)
 subsizes(1) = ntimesteps
 subsizes(2) = decomp%xsz(2)
 subsizes(3) = decomp%xsz(3)
 starts(1) = 0  ! 0-based index
-starts(2) = decomp%xst(2)-1
-starts(3) = decomp%xst(3)-1
+starts(2) = decomp%xst(2) - 1
+starts(3) = decomp%xst(3) - 1
 
 idx = get_io_idx(io_name, dirname)
 
 #ifndef ADIOS2
 !! Use default MPIIO
-associate(fh=>fh_registry(idx), &
-  disp => fh_disp(idx))
-  call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-  call MPI_TYPE_COMMIT(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-  call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-       newtype,'native',MPI_INFO_NULL,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-  call MPI_FILE_READ_ALL(fh, var, &
-       subsizes(1)*subsizes(2)*subsizes(3), &
-       data_type, MPI_STATUS_IGNORE, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-  call MPI_TYPE_FREE(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+associate (fh => fh_registry(idx), &
+           disp => fh_disp(idx))
+   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+   call MPI_TYPE_COMMIT(newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                          newtype, 'native', MPI_INFO_NULL, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+   call MPI_FILE_READ_ALL(fh, var, &
+                          subsizes(1)*subsizes(2)*subsizes(3), &
+                          data_type, MPI_STATUS_IGNORE, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+   call MPI_TYPE_FREE(newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-  ! update displacement for the next read operation
-  disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-              * int(sizes(2), kind=MPI_OFFSET_KIND) &
-              * int(sizes(3), kind=MPI_OFFSET_KIND) &
-              * int(mytype_bytes, kind=MPI_OFFSET_KIND)
-  if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                             * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                             * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+   ! update displacement for the next read operation
+   disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+          *int(sizes(2), kind=MPI_OFFSET_KIND) &
+          *int(sizes(3), kind=MPI_OFFSET_KIND) &
+          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+   if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
-associate(vnm => varname) ! Silence unused argument
+associate (vnm => varname) ! Silence unused argument
 end associate
 
 #else
 !! Use ADIOS2
 call adios2_at_io(io_handle, adios, io_name, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-   "trying to write variable before registering! "//trim(varname))
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                                                 "trying to write variable before registering! "//trim(varname))
 
 !! Note - need to use sync mode as we are using a view into the array - unsure how this works with deferred writes
 ! call adios2_set_step_selection(var_handle, int(0, kind=8), int(1, kind=8), ierror)
 call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_sync, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
 #endif
diff --git a/src/io_read_one.inc b/src/io_read_one.inc
index 211cbc2c..8b8c2f5c 100644
--- a/src/io_read_one.inc
+++ b/src/io_read_one.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,7 +10,7 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'mpiio_read_one_...' in io.f90
 
 ! Using MPI-IO to write a distributed 3D array into a file
@@ -30,43 +30,43 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1)-1  ! 0-based index
-   starts(2) = decomp%xst(2)-1
-   starts(3) = decomp%xst(3)-1
+   starts(1) = decomp%xst(1) - 1  ! 0-based index
+   starts(2) = decomp%xst(2) - 1
+   starts(3) = decomp%xst(3) - 1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1)-1
-   starts(2) = decomp%yst(2)-1
-   starts(3) = decomp%yst(3)-1
+   starts(1) = decomp%yst(1) - 1
+   starts(2) = decomp%yst(2) - 1
+   starts(3) = decomp%yst(3) - 1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1)-1
-   starts(2) = decomp%zst(2)-1
-   starts(3) = decomp%zst(3)-1
-endif
+   starts(1) = decomp%zst(1) - 1
+   starts(2) = decomp%zst(2) - 1
+   starts(3) = decomp%zst(3) - 1
+end if
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
 call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-     MPI_MODE_RDONLY, MPI_INFO_NULL, &
-     fh, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+                   MPI_MODE_RDONLY, MPI_INFO_NULL, &
+                   fh, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
 disp = 0_MPI_OFFSET_KIND
-call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-     newtype,'native',MPI_INFO_NULL,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                       newtype, 'native', MPI_INFO_NULL, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_READ_ALL(fh, var, &
-     subsizes(1)*subsizes(2)*subsizes(3), &
-     data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-call MPI_FILE_CLOSE(fh,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-call MPI_TYPE_FREE(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                       subsizes(1)*subsizes(2)*subsizes(3), &
+                       data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+call MPI_FILE_CLOSE(fh, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+call MPI_TYPE_FREE(newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
diff --git a/src/io_read_var.inc b/src/io_read_var.inc
index 489247cb..816dd6eb 100644
--- a/src/io_read_var.inc
+++ b/src/io_read_var.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,12 +10,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'read_var_...' in io.f90
 
-! Using MPI-IO to read a distributed 3D variable from a file. File 
-! operations (open/close) need to be done in calling application. This 
-! allows multiple variables to be read from a single file. Together 
+! Using MPI-IO to read a distributed 3D variable from a file. File
+! operations (open/close) need to be done in calling application. This
+! allows multiple variables to be read from a single file. Together
 ! with the corresponding write operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -33,46 +33,46 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1)-1  ! 0-based index
-   starts(2) = decomp%xst(2)-1
-   starts(3) = decomp%xst(3)-1
+   starts(1) = decomp%xst(1) - 1  ! 0-based index
+   starts(2) = decomp%xst(2) - 1
+   starts(3) = decomp%xst(3) - 1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1)-1
-   starts(2) = decomp%yst(2)-1
-   starts(3) = decomp%yst(3)-1
+   starts(1) = decomp%yst(1) - 1
+   starts(2) = decomp%yst(2) - 1
+   starts(3) = decomp%yst(3) - 1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1)-1
-   starts(2) = decomp%zst(2)-1
-   starts(3) = decomp%zst(3)-1
-endif
+   starts(1) = decomp%zst(1) - 1
+   starts(2) = decomp%zst(2) - 1
+   starts(3) = decomp%zst(3) - 1
+end if
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-     newtype,'native',MPI_INFO_NULL,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                       newtype, 'native', MPI_INFO_NULL, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_READ_ALL(fh, var, &
-     subsizes(1)*subsizes(2)*subsizes(3), &
-     data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-call MPI_TYPE_FREE(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                       subsizes(1)*subsizes(2)*subsizes(3), &
+                       data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+call MPI_TYPE_FREE(newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 ! update displacement for the next read operation
 disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-            * int(sizes(2), kind=MPI_OFFSET_KIND) &
-            * int(sizes(3), kind=MPI_OFFSET_KIND) &
-            * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+       *int(sizes(2), kind=MPI_OFFSET_KIND) &
+       *int(sizes(3), kind=MPI_OFFSET_KIND) &
+       *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                           * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                           * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                           * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                      *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                      *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                      *int(mytype_bytes, kind=MPI_OFFSET_KIND)
diff --git a/src/io_write_every.inc b/src/io_write_every.inc
index cbf98b7a..a0fe74d3 100644
--- a/src/io_write_every.inc
+++ b/src/io_write_every.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,12 +10,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'write_every_...' in io.f90
 
 ! To write every few points of a 3D array to a file
 
-! work out the distribution parameters, which may be different from 
+! work out the distribution parameters, which may be different from
 ! the default distribution used by the decomposition library
 !  For exmample if nx=17 and p_row=4
 !    distribution is: 4 4 4 5
@@ -38,199 +38,199 @@
 !             4th block (13-17) contains then 12th & 15th point
 !    giving a 1 2 2 1 distribution
 
-skip(1)=iskip
-skip(2)=jskip
-skip(3)=kskip
+skip(1) = iskip
+skip(2) = jskip
+skip(3) = kskip
 
-do i=1,3
+do i = 1, 3
    if (from1) then
-      xst(i) = (xstart(i)+skip(i)-1)/skip(i)
-      if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xst(i)=xst(i)+1
-      xen(i) = (xend(i)+skip(i)-1)/skip(i)
+      xst(i) = (xstart(i) + skip(i) - 1)/skip(i)
+      if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xst(i) = xst(i) + 1
+      xen(i) = (xend(i) + skip(i) - 1)/skip(i)
    else
       xst(i) = xstart(i)/skip(i)
-      if (mod(xstart(i),skip(i))/=0) xst(i)=xst(i)+1
+      if (mod(xstart(i), skip(i)) /= 0) xst(i) = xst(i) + 1
       xen(i) = xend(i)/skip(i)
    end if
-   xsz(i) = xen(i)-xst(i)+1
+   xsz(i) = xen(i) - xst(i) + 1
 end do
 
-do i=1,3
+do i = 1, 3
    if (from1) then
-      yst(i) = (ystart(i)+skip(i)-1)/skip(i)
-      if (mod(ystart(i)+skip(i)-1,skip(i))/=0) yst(i)=yst(i)+1
-      yen(i) = (yend(i)+skip(i)-1)/skip(i)
+      yst(i) = (ystart(i) + skip(i) - 1)/skip(i)
+      if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) yst(i) = yst(i) + 1
+      yen(i) = (yend(i) + skip(i) - 1)/skip(i)
    else
       yst(i) = ystart(i)/skip(i)
-      if (mod(ystart(i),skip(i))/=0) yst(i)=yst(i)+1
+      if (mod(ystart(i), skip(i)) /= 0) yst(i) = yst(i) + 1
       yen(i) = yend(i)/skip(i)
    end if
-   ysz(i) = yen(i)-yst(i)+1
+   ysz(i) = yen(i) - yst(i) + 1
 end do
 
-do i=1,3
+do i = 1, 3
    if (from1) then
-      zst(i) = (zstart(i)+skip(i)-1)/skip(i)
-      if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zst(i)=zst(i)+1
-      zen(i) = (zend(i)+skip(i)-1)/skip(i)
+      zst(i) = (zstart(i) + skip(i) - 1)/skip(i)
+      if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zst(i) = zst(i) + 1
+      zen(i) = (zend(i) + skip(i) - 1)/skip(i)
    else
       zst(i) = zstart(i)/skip(i)
-      if (mod(zstart(i),skip(i))/=0) zst(i)=zst(i)+1
+      if (mod(zstart(i), skip(i)) /= 0) zst(i) = zst(i) + 1
       zen(i) = zend(i)/skip(i)
    end if
-   zsz(i) = zen(i)-zst(i)+1
+   zsz(i) = zen(i) - zst(i) + 1
 end do
 
-! if 'skip' value is large it is possible that some ranks do not 
-! contain any points to be written. Subarray constructor requires 
+! if 'skip' value is large it is possible that some ranks do not
+! contain any points to be written. Subarray constructor requires
 ! nonzero size so it is not possible to use decomp_2d_comm for IO.
 ! Create a sub communicator for this...
 color = 1
 key = 0  ! rank order doesn't matter
-if (ipencil==1) then
-   if (xsz(1)==0 .or. xsz(2)==0 .or. xsz(3)==0) then
+if (ipencil == 1) then
+   if (xsz(1) == 0 .or. xsz(2) == 0 .or. xsz(3) == 0) then
       color = 2
    end if
-else if (ipencil==2) then
-   if (ysz(1)==0 .or. ysz(2)==0 .or. ysz(3)==0) then
+else if (ipencil == 2) then
+   if (ysz(1) == 0 .or. ysz(2) == 0 .or. ysz(3) == 0) then
       color = 2
    end if
-else if (ipencil==3) then
-   if (zsz(1)==0 .or. zsz(2)==0 .or. zsz(3)==0) then
+else if (ipencil == 3) then
+   if (zsz(1) == 0 .or. zsz(2) == 0 .or. zsz(3) == 0) then
       color = 2
    end if
 end if
-call MPI_COMM_SPLIT(decomp_2d_comm,color,key,newcomm,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+call MPI_COMM_SPLIT(decomp_2d_comm, color, key, newcomm, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
 
-if (color==1) then ! only ranks in this group do IO collectively
+if (color == 1) then ! only ranks in this group do IO collectively
 
    ! generate subarray information
    sizes(1) = xsz(1)
    sizes(2) = ysz(2)
    sizes(3) = zsz(3)
-   if (ipencil==1) then
+   if (ipencil == 1) then
       subsizes(1) = xsz(1)
       subsizes(2) = xsz(2)
       subsizes(3) = xsz(3)
-      starts(1) = xst(1)-1
-      starts(2) = xst(2)-1
-      starts(3) = xst(3)-1
-   else if (ipencil==2) then
+      starts(1) = xst(1) - 1
+      starts(2) = xst(2) - 1
+      starts(3) = xst(3) - 1
+   else if (ipencil == 2) then
       subsizes(1) = ysz(1)
       subsizes(2) = ysz(2)
       subsizes(3) = ysz(3)
-      starts(1) = yst(1)-1
-      starts(2) = yst(2)-1
-      starts(3) = yst(3)-1
-   else if (ipencil==3) then
+      starts(1) = yst(1) - 1
+      starts(2) = yst(2) - 1
+      starts(3) = yst(3) - 1
+   else if (ipencil == 3) then
       subsizes(1) = zsz(1)
       subsizes(2) = zsz(2)
       subsizes(3) = zsz(3)
-      starts(1) = zst(1)-1
-      starts(2) = zst(2)-1
-      starts(3) = zst(3)-1
+      starts(1) = zst(1) - 1
+      starts(2) = zst(2) - 1
+      starts(3) = zst(3) - 1
    end if
 
    ! copy data from original array
-   ! needs a copy of original array in global coordinate 
-   if (ipencil==1) then
-      allocate(wk(xst(1):xen(1),xst(2):xen(2),xst(3):xen(3)))
-      allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-      wk2=var
+   ! needs a copy of original array in global coordinate
+   if (ipencil == 1) then
+      allocate (wk(xst(1):xen(1), xst(2):xen(2), xst(3):xen(3)))
+      allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      wk2 = var
       if (from1) then
-         do k=xst(3),xen(3)
-            do j=xst(2),xen(2)
-               do i=xst(1),xen(1)
-                  wk(i,j,k) = wk2((i-1)*iskip+1,(j-1)*jskip+1,(k-1)*kskip+1)
+         do k = xst(3), xen(3)
+            do j = xst(2), xen(2)
+               do i = xst(1), xen(1)
+                  wk(i, j, k) = wk2((i - 1)*iskip + 1, (j - 1)*jskip + 1, (k - 1)*kskip + 1)
                end do
             end do
          end do
       else
-         do k=xst(3),xen(3)
-            do j=xst(2),xen(2)
-               do i=xst(1),xen(1)
-                  wk(i,j,k) = wk2(i*iskip,j*jskip,k*kskip)
+         do k = xst(3), xen(3)
+            do j = xst(2), xen(2)
+               do i = xst(1), xen(1)
+                  wk(i, j, k) = wk2(i*iskip, j*jskip, k*kskip)
                end do
             end do
          end do
       end if
-   else if (ipencil==2) then
-      allocate(wk(yst(1):yen(1),yst(2):yen(2),yst(3):yen(3)))
-      allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
-      wk2=var
+   else if (ipencil == 2) then
+      allocate (wk(yst(1):yen(1), yst(2):yen(2), yst(3):yen(3)))
+      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      wk2 = var
       if (from1) then
-         do k=yst(3),yen(3)
-            do j=yst(2),yen(2)
-               do i=yst(1),yen(1)
-                  wk(i,j,k) = wk2((i-1)*iskip+1,(j-1)*jskip+1,(k-1)*kskip+1)
+         do k = yst(3), yen(3)
+            do j = yst(2), yen(2)
+               do i = yst(1), yen(1)
+                  wk(i, j, k) = wk2((i - 1)*iskip + 1, (j - 1)*jskip + 1, (k - 1)*kskip + 1)
                end do
             end do
          end do
       else
-         do k=yst(3),yen(3)
-            do j=yst(2),yen(2)
-               do i=yst(1),yen(1)
-                  wk(i,j,k) = wk2(i*iskip,j*jskip,k*kskip)
+         do k = yst(3), yen(3)
+            do j = yst(2), yen(2)
+               do i = yst(1), yen(1)
+                  wk(i, j, k) = wk2(i*iskip, j*jskip, k*kskip)
                end do
             end do
          end do
       end if
-   else if (ipencil==3) then
-      allocate(wk(zst(1):zen(1),zst(2):zen(2),zst(3):zen(3)))
-      allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-      wk2=var
+   else if (ipencil == 3) then
+      allocate (wk(zst(1):zen(1), zst(2):zen(2), zst(3):zen(3)))
+      allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      wk2 = var
       if (from1) then
-         do k=zst(3),zen(3)
-            do j=zst(2),zen(2)
-               do i=zst(1),zen(1)
-                  wk(i,j,k) = wk2((i-1)*iskip+1,(j-1)*jskip+1,(k-1)*kskip+1)
+         do k = zst(3), zen(3)
+            do j = zst(2), zen(2)
+               do i = zst(1), zen(1)
+                  wk(i, j, k) = wk2((i - 1)*iskip + 1, (j - 1)*jskip + 1, (k - 1)*kskip + 1)
                end do
             end do
          end do
       else
-         do k=zst(3),zen(3)
-            do j=zst(2),zen(2)
-               do i=zst(1),zen(1)
-                  wk(i,j,k) = wk2(i*iskip,j*jskip,k*kskip)
+         do k = zst(3), zen(3)
+            do j = zst(2), zen(2)
+               do i = zst(1), zen(1)
+                  wk(i, j, k) = wk2(i*iskip, j*jskip, k*kskip)
                end do
             end do
          end do
       end if
    end if
-   deallocate(wk2)
+   deallocate (wk2)
 
    ! MPI-IO
-   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-        MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-   call MPI_TYPE_COMMIT(newtype,ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+   call MPI_TYPE_COMMIT(newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
    call MPI_FILE_OPEN(newcomm, filename, &
-        MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-        fh, ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+                      MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                      fh, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
    filesize = 0_MPI_OFFSET_KIND
-   call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+   call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
    disp = 0_MPI_OFFSET_KIND
-   call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-        newtype,'native',MPI_INFO_NULL,ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                          newtype, 'native', MPI_INFO_NULL, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
    call MPI_FILE_WRITE_ALL(fh, wk, &
-        subsizes(1)*subsizes(2)*subsizes(3), &
-        data_type, MPI_STATUS_IGNORE, ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-   call MPI_FILE_CLOSE(fh,ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-   call MPI_TYPE_FREE(newtype,ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                           subsizes(1)*subsizes(2)*subsizes(3), &
+                           data_type, MPI_STATUS_IGNORE, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+   call MPI_FILE_CLOSE(fh, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+   call MPI_TYPE_FREE(newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-   deallocate(wk)
+   deallocate (wk)
 
 end if ! color==1
 
 call decomp_mpi_comm_free(newcomm)
 
 call MPI_BARRIER(decomp_2d_comm, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
diff --git a/src/io_write_one.inc b/src/io_write_one.inc
index 29facac5..e3972fb3 100644
--- a/src/io_write_one.inc
+++ b/src/io_write_one.inc
@@ -1,17 +1,17 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG) 
+! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
 ! Copyright (C) 2021               the University of Edinburgh (UoE)
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'mpiio_write_one_...' in io.f90
 
 ! Using MPI-IO to write a distributed 3D array into a file
@@ -31,67 +31,67 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1)-1  ! 0-based index
-   starts(2) = decomp%xst(2)-1
-   starts(3) = decomp%xst(3)-1
+   starts(1) = decomp%xst(1) - 1  ! 0-based index
+   starts(2) = decomp%xst(2) - 1
+   starts(3) = decomp%xst(3) - 1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1)-1
-   starts(2) = decomp%yst(2)-1
-   starts(3) = decomp%yst(3)-1
+   starts(1) = decomp%yst(1) - 1
+   starts(2) = decomp%yst(2) - 1
+   starts(3) = decomp%yst(3) - 1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1)-1
-   starts(2) = decomp%zst(2)-1
-   starts(3) = decomp%zst(3)-1
-endif
+   starts(1) = decomp%zst(1) - 1
+   starts(2) = decomp%zst(2) - 1
+   starts(3) = decomp%zst(3) - 1
+end if
 
 #ifdef T3PIO
 call MPI_INFO_CREATE(info, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_CREATE")
-gs = ceiling(real(sizes(1),mytype)*real(sizes(2),mytype)* &
-     real(sizes(3),mytype)/1024./1024.)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_CREATE")
+gs = ceiling(real(sizes(1), mytype)*real(sizes(2), mytype)* &
+             real(sizes(3), mytype)/1024./1024.)
 call t3pio_set_info(decomp_2d_comm, info, "./", ierror, &
-     GLOBAL_SIZE=gs, factor=1)
+                    GLOBAL_SIZE=gs, factor=1)
 #else
 gs = 1 ! Silence unused variable
 info = 1 ! Silence unused variable
 #endif
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
 #ifdef T3PIO
 call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-     MPI_MODE_CREATE+MPI_MODE_WRONLY, info, fh, ierror)
+                   MPI_MODE_CREATE + MPI_MODE_WRONLY, info, fh, ierror)
 #else
 call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-     MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-     fh, ierror)
+                   MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                   fh, ierror)
 #endif
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
 filesize = 0_MPI_OFFSET_KIND
-call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
 disp = 0_MPI_OFFSET_KIND
-call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-     newtype,'native',MPI_INFO_NULL,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                       newtype, 'native', MPI_INFO_NULL, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_WRITE_ALL(fh, var, &
-     subsizes(1)*subsizes(2)*subsizes(3), &
-     data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-call MPI_FILE_CLOSE(fh,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-call MPI_TYPE_FREE(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                        subsizes(1)*subsizes(2)*subsizes(3), &
+                        data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+call MPI_FILE_CLOSE(fh, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+call MPI_TYPE_FREE(newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef T3PIO
-call MPI_INFO_FREE(info,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_FREE")
+call MPI_INFO_FREE(info, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_FREE")
 #endif
diff --git a/src/io_write_outflow.f90 b/src/io_write_outflow.f90
index ccbd0314..4f6f3845 100644
--- a/src/io_write_outflow.f90
+++ b/src/io_write_outflow.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,12 +9,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'write_var_...' in io.f90
 
-! Using MPI-IO to write a distributed 3D variable to a file. File 
+! Using MPI-IO to write a distributed 3D variable to a file. File
 ! operations (open/close) need to be done in calling application. This
-! allows multiple variables to be written to a single file. Together 
+! allows multiple variables to be written to a single file. Together
 ! with the corresponding read operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -25,62 +25,62 @@
 end if
 
 ! Create file type and set file view
-sizes(1) = ntimesteps 
+sizes(1) = ntimesteps
 sizes(2) = decomp%ysz(2)
 sizes(3) = decomp%zsz(3)
-subsizes(1) = ntimesteps 
+subsizes(1) = ntimesteps
 subsizes(2) = decomp%xsz(2)
 subsizes(3) = decomp%xsz(3)
 starts(1) = 0  ! 0-based index
-starts(2) = decomp%xst(2)-1
-starts(3) = decomp%xst(3)-1
+starts(2) = decomp%xst(2) - 1
+starts(3) = decomp%xst(3) - 1
 
 idx = get_io_idx(io_name, dirname)
 
 #ifndef ADIOS2
 !! Use default MPIIO
-associate(fh=>fh_registry(idx), &
-     disp=>fh_disp(idx))
-  call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-  call MPI_TYPE_COMMIT(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-  call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-       newtype,'native',MPI_INFO_NULL,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-  call MPI_FILE_WRITE_ALL(fh, var, &
-       subsizes(1)*subsizes(2)*subsizes(3), &
-       data_type, MPI_STATUS_IGNORE, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-  call MPI_TYPE_FREE(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+associate (fh => fh_registry(idx), &
+           disp => fh_disp(idx))
+   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+   call MPI_TYPE_COMMIT(newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                          newtype, 'native', MPI_INFO_NULL, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+   call MPI_FILE_WRITE_ALL(fh, var, &
+                           subsizes(1)*subsizes(2)*subsizes(3), &
+                           data_type, MPI_STATUS_IGNORE, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+   call MPI_TYPE_FREE(newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-  ! update displacement for the next write operation
-  disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-              * int(sizes(2), kind=MPI_OFFSET_KIND) &
-              * int(sizes(3), kind=MPI_OFFSET_KIND) &
-              * int(mytype_bytes, kind=MPI_OFFSET_KIND)
-  if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                             * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                             * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+   ! update displacement for the next write operation
+   disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+          *int(sizes(2), kind=MPI_OFFSET_KIND) &
+          *int(sizes(3), kind=MPI_OFFSET_KIND) &
+          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+   if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
-associate(vnm => varname) ! Silence unused dummy argument
+associate (vnm => varname) ! Silence unused dummy argument
 end associate
 
 #else
 !! Use ADIOS2
 call adios2_at_io(io_handle, adios, io_name, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not.var_handle % valid) then
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+if (.not. var_handle%valid) then
    call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to write variable before registering!"//trim(varname))
-endif
+end if
 
 !! Note - need to use sync mode as we are using a view into the array - unsure how this works with deferred writes
 call adios2_put(engine_registry(idx), var_handle, var, adios2_mode_sync, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
 #endif
diff --git a/src/io_write_plane.inc b/src/io_write_plane.inc
index 088fe097..b48db97d 100644
--- a/src/io_write_plane.inc
+++ b/src/io_write_plane.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,7 +10,7 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'mpiio_write_plane_3d_...' in io.f90
 
 ! It is much easier to implement if all mpi ranks participate I/O.
@@ -24,81 +24,81 @@ end if
 
 opened_new = .false.
 
-if (iplane==1) then
-   allocate(wk(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)))
-   if (ipencil==1) then
+if (iplane == 1) then
+   allocate (wk(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)))
+   if (ipencil == 1) then
       wk = var
-   else if (ipencil==2) then
-      call transpose_y_to_x(var,wk,decomp)
-   else if (ipencil==3) then
-      allocate(wk2(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)))
-      call transpose_z_to_y(var,wk2,decomp)
-      call transpose_y_to_x(wk2,wk,decomp)
-      deallocate(wk2)
+   else if (ipencil == 2) then
+      call transpose_y_to_x(var, wk, decomp)
+   else if (ipencil == 3) then
+      allocate (wk2(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)))
+      call transpose_z_to_y(var, wk2, decomp)
+      call transpose_y_to_x(wk2, wk, decomp)
+      deallocate (wk2)
    end if
-   allocate(wk2d(1,decomp%xsz(2),decomp%xsz(3)))
-   if (n.ge.1) then
-      do k=1,decomp%xsz(3)
-         do j=1,decomp%xsz(2)
-            wk2d(1,j,k)=wk(n,j,k)
+   allocate (wk2d(1, decomp%xsz(2), decomp%xsz(3)))
+   if (n .ge. 1) then
+      do k = 1, decomp%xsz(3)
+         do j = 1, decomp%xsz(2)
+            wk2d(1, j, k) = wk(n, j, k)
          end do
       end do
    else
-      do k=1,decomp%xsz(3)
-         do j=1,decomp%xsz(2)
-            wk2d(1,j,k)=sum(wk(:,j,k))/real(decomp%xsz(1),kind=mytype)
+      do k = 1, decomp%xsz(3)
+         do j = 1, decomp%xsz(2)
+            wk2d(1, j, k) = sum(wk(:, j, k))/real(decomp%xsz(1), kind=mytype)
          end do
       end do
-   endif
-else if (iplane==2) then
-   allocate(wk(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)))
-   if (ipencil==1) then
-      call transpose_x_to_y(var,wk,decomp)
-   else if (ipencil==2) then
+   end if
+else if (iplane == 2) then
+   allocate (wk(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)))
+   if (ipencil == 1) then
+      call transpose_x_to_y(var, wk, decomp)
+   else if (ipencil == 2) then
       wk = var
-   else if (ipencil==3) then
-      call transpose_z_to_y(var,wk,decomp)
+   else if (ipencil == 3) then
+      call transpose_z_to_y(var, wk, decomp)
    end if
-   allocate(wk2d(decomp%ysz(1),1,decomp%ysz(3)))
-   if (n.ge.1) then
-      do k=1,decomp%ysz(3)
-         do i=1,decomp%ysz(1)
-            wk2d(i,1,k)=wk(i,n,k)
+   allocate (wk2d(decomp%ysz(1), 1, decomp%ysz(3)))
+   if (n .ge. 1) then
+      do k = 1, decomp%ysz(3)
+         do i = 1, decomp%ysz(1)
+            wk2d(i, 1, k) = wk(i, n, k)
          end do
       end do
    else
-      do k=1,decomp%ysz(3)
-         do i=1,decomp%ysz(1)
-            wk2d(i,1,k)=sum(wk(i,:,k))/real(decomp%ysz(2),kind=mytype)
+      do k = 1, decomp%ysz(3)
+         do i = 1, decomp%ysz(1)
+            wk2d(i, 1, k) = sum(wk(i, :, k))/real(decomp%ysz(2), kind=mytype)
          end do
       end do
-   endif
-else if (iplane==3) then
-   allocate(wk(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)))
-   if (ipencil==1) then
-      allocate(wk2(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)))
-      call transpose_x_to_y(var,wk2,decomp)
-      call transpose_y_to_z(wk2,wk,decomp)
-      deallocate(wk2)
-   else if (ipencil==2) then
-      call transpose_y_to_z(var,wk,decomp)
-   else if (ipencil==3) then
+   end if
+else if (iplane == 3) then
+   allocate (wk(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)))
+   if (ipencil == 1) then
+      allocate (wk2(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)))
+      call transpose_x_to_y(var, wk2, decomp)
+      call transpose_y_to_z(wk2, wk, decomp)
+      deallocate (wk2)
+   else if (ipencil == 2) then
+      call transpose_y_to_z(var, wk, decomp)
+   else if (ipencil == 3) then
       wk = var
    end if
-   allocate(wk2d(decomp%zsz(1),decomp%zsz(2),1))
-   if (n.ge.1) then
-      do j=1,decomp%zsz(2)
-         do i=1,decomp%zsz(1) 
-            wk2d(i,j,1)=wk(i,j,n)
+   allocate (wk2d(decomp%zsz(1), decomp%zsz(2), 1))
+   if (n .ge. 1) then
+      do j = 1, decomp%zsz(2)
+         do i = 1, decomp%zsz(1)
+            wk2d(i, j, 1) = wk(i, j, n)
          end do
       end do
    else
-      do j=1,decomp%zsz(2)
-         do i=1,decomp%zsz(1)
-            wk2d(i,j,1)=sum(wk(i,j,:))/real(decomp%zsz(3),kind=mytype)
+      do j = 1, decomp%zsz(2)
+         do i = 1, decomp%zsz(1)
+            wk2d(i, j, 1) = sum(wk(i, j, :))/real(decomp%zsz(3), kind=mytype)
          end do
       end do
-   endif
+   end if
 end if
 
 idx = get_io_idx(io_name, dirname)
@@ -108,13 +108,13 @@ idx = get_io_idx(io_name, dirname)
 
 if (idx .lt. 1) then
    ! Create folder if needed
-   if (nrank==0) then
-      inquire(file=dirname, exist=dir_exists)
-      if (.not.dir_exists) then
+   if (nrank == 0) then
+      inquire (file=dirname, exist=dir_exists)
+      if (.not. dir_exists) then
          call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
       end if
    end if
-   allocate(character(len(trim(dirname)) + 1 + len(trim(varname))) :: full_io_name)
+   allocate (character(len(trim(dirname)) + 1 + len(trim(varname))) :: full_io_name)
    full_io_name = dirname//"/"//varname
    call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
    idx = get_io_idx(io_name, full_io_name)
@@ -122,40 +122,40 @@ if (idx .lt. 1) then
 end if
 
 call plane_extents(sizes, subsizes, starts, iplane, decomp)
-associate(fh=>fh_registry(idx), &
-     disp=>fh_disp(idx))
-  call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-  call MPI_TYPE_COMMIT(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-  call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-       newtype,'native',MPI_INFO_NULL,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-  call MPI_FILE_WRITE_ALL(fh, wk2d, &
-       subsizes(1)*subsizes(2)*subsizes(3), &
-       data_type, MPI_STATUS_IGNORE, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-  call MPI_TYPE_FREE(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+associate (fh => fh_registry(idx), &
+           disp => fh_disp(idx))
+   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+   call MPI_TYPE_COMMIT(newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                          newtype, 'native', MPI_INFO_NULL, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+   call MPI_FILE_WRITE_ALL(fh, wk2d, &
+                           subsizes(1)*subsizes(2)*subsizes(3), &
+                           data_type, MPI_STATUS_IGNORE, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+   call MPI_TYPE_FREE(newtype, ierror)
+   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 end associate
-    
+
 if (opened_new) then
    call decomp_2d_close_io(io_name, full_io_name)
-   deallocate(full_io_name)
+   deallocate (full_io_name)
 end if
 #else
 !! Write using ADIOS2
 call adios2_at_io(io_handle, adios, io_name, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-   "trying to write variable before registering! "//trim(varname))
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                                                 "trying to write variable before registering! "//trim(varname))
 
 !! Note - need to use sync mode as the array for the output plane gets reused.
 call adios2_put(engine_registry(idx), var_handle, wk2d, adios2_mode_sync, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
 #endif
 
-deallocate(wk,wk2d)
+deallocate (wk, wk2d)
diff --git a/src/io_write_var.inc b/src/io_write_var.inc
index 23c8b617..338f42fe 100644
--- a/src/io_write_var.inc
+++ b/src/io_write_var.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,12 +10,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'write_var_...' in io.f90
 
-! Using MPI-IO to write a distributed 3D variable to a file. File 
+! Using MPI-IO to write a distributed 3D variable to a file. File
 ! operations (open/close) need to be done in calling application. This
-! allows multiple variables to be written to a single file. Together 
+! allows multiple variables to be written to a single file. Together
 ! with the corresponding read operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -33,46 +33,46 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1)-1  ! 0-based index
-   starts(2) = decomp%xst(2)-1
-   starts(3) = decomp%xst(3)-1
+   starts(1) = decomp%xst(1) - 1  ! 0-based index
+   starts(2) = decomp%xst(2) - 1
+   starts(3) = decomp%xst(3) - 1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1)-1
-   starts(2) = decomp%yst(2)-1
-   starts(3) = decomp%yst(3)-1
+   starts(1) = decomp%yst(1) - 1
+   starts(2) = decomp%yst(2) - 1
+   starts(3) = decomp%yst(3) - 1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1)-1
-   starts(2) = decomp%zst(2)-1
-   starts(3) = decomp%zst(3)-1
-endif
+   starts(1) = decomp%zst(1) - 1
+   starts(2) = decomp%zst(2) - 1
+   starts(3) = decomp%zst(3) - 1
+end if
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-     newtype,'native',MPI_INFO_NULL,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                       newtype, 'native', MPI_INFO_NULL, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_WRITE_ALL(fh, var, &
-     subsizes(1)*subsizes(2)*subsizes(3), &
-     data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-call MPI_TYPE_FREE(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                        subsizes(1)*subsizes(2)*subsizes(3), &
+                        data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+call MPI_TYPE_FREE(newtype, ierror)
+if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 ! update displacement for the next write operation
 disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-            * int(sizes(2), kind=MPI_OFFSET_KIND) &
-            * int(sizes(3), kind=MPI_OFFSET_KIND) &
-            * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+       *int(sizes(2), kind=MPI_OFFSET_KIND) &
+       *int(sizes(3), kind=MPI_OFFSET_KIND) &
+       *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                           * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                           * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                           * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                      *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                      *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                      *int(mytype_bytes, kind=MPI_OFFSET_KIND)
diff --git a/src/log.f90 b/src/log.f90
index 866cd97e..53e3165a 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,296 +10,296 @@
 !
 !=======================================================================
 
-submodule (decomp_2d) d2d_log
+submodule(decomp_2d) d2d_log
 
-  implicit none
+   implicit none
 
-  contains
+contains
 
-  !
-  ! Print some information about decomp_2d
-  !
-  module subroutine d2d_listing(given_io_unit)
+   !
+   ! Print some information about decomp_2d
+   !
+   module subroutine d2d_listing(given_io_unit)
 
-    use iso_fortran_env, only : output_unit, compiler_version, compiler_options
+      use iso_fortran_env, only: output_unit, compiler_version, compiler_options
 
-    implicit none
+      implicit none
 
-    ! Argument
-    integer, intent(in), optional :: given_io_unit
+      ! Argument
+      integer, intent(in), optional :: given_io_unit
 
-    ! Local variable
-    integer :: io_unit
-    integer :: version, subversion, ierror
+      ! Local variable
+      integer :: io_unit
+      integer :: version, subversion, ierror
 #ifdef DEBUG
-    character(len=64) :: fname
+      character(len=64) :: fname
 #endif
 
-    ! Output log if needed
-    if (decomp_log == D2D_LOG_QUIET) return
-    if (decomp_log == D2D_LOG_STDOUT .and. nrank /= 0) return
-    if (decomp_log == D2D_LOG_TOFILE .and. nrank /= 0) return
+      ! Output log if needed
+      if (decomp_log == D2D_LOG_QUIET) return
+      if (decomp_log == D2D_LOG_STDOUT .and. nrank /= 0) return
+      if (decomp_log == D2D_LOG_TOFILE .and. nrank /= 0) return
 
-    ! If no IO unit provided, use stdout
-    if (present(given_io_unit)) then
-       io_unit = given_io_unit
-    else
-       io_unit = output_unit
-    endif
+      ! If no IO unit provided, use stdout
+      if (present(given_io_unit)) then
+         io_unit = given_io_unit
+      else
+         io_unit = output_unit
+      end if
 
-    ! Header
-    write (io_unit, *) '==========================================================='
-    write (io_unit, *) '=================== Decomp2D - log ========================'
-    write (io_unit, *) '==========================================================='
+      ! Header
+      write (io_unit, *) '==========================================================='
+      write (io_unit, *) '=================== Decomp2D - log ========================'
+      write (io_unit, *) '==========================================================='
 
-    ! Git hash if available
+      ! Git hash if available
 #if defined(VERSION)
-    write (io_unit, *) 'Git version        : ', VERSION
+      write (io_unit, *) 'Git version        : ', VERSION
 #else
-    write (io_unit, *) 'Git version        : unknown'
+      write (io_unit, *) 'Git version        : unknown'
 #endif
 
-    ! Basic info
+      ! Basic info
 #ifdef DEBUG
-    if (decomp_debug >= D2D_DEBUG_LEVEL_INFO) &
-       write (io_unit, *) 'I am mpi rank ', nrank
+      if (decomp_debug >= D2D_DEBUG_LEVEL_INFO) &
+         write (io_unit, *) 'I am mpi rank ', nrank
 #endif
-    write (io_unit, *) 'Total ranks ', nproc
-    write (io_unit, *) 'Global data size : ', nx_global, ny_global, nz_global
-    write (io_unit, *) 'p_row, p_col : ', dims(1), dims(2)
-    write (io_unit, *) 'Periodicity : ', periodic_x, periodic_y, periodic_z
-    write (io_unit, *) 'Number of bytes / float number : ', mytype_bytes
-    write (io_unit, *) '==========================================================='
-
-    ! Show detected flags, compiler options, version of the MPI library
-    write (io_unit, *) 'Compile flags detected :'
+      write (io_unit, *) 'Total ranks ', nproc
+      write (io_unit, *) 'Global data size : ', nx_global, ny_global, nz_global
+      write (io_unit, *) 'p_row, p_col : ', dims(1), dims(2)
+      write (io_unit, *) 'Periodicity : ', periodic_x, periodic_y, periodic_z
+      write (io_unit, *) 'Number of bytes / float number : ', mytype_bytes
+      write (io_unit, *) '==========================================================='
+
+      ! Show detected flags, compiler options, version of the MPI library
+      write (io_unit, *) 'Compile flags detected :'
 #ifdef DOUBLE_PREC
 #ifdef SAVE_SINGLE
-    write (io_unit, *) 'Numerical precision: Double, saving in single'
+      write (io_unit, *) 'Numerical precision: Double, saving in single'
 #else
-    write (io_unit, *) 'Numerical precision: Double'
+      write (io_unit, *) 'Numerical precision: Double'
 #endif
 #else
-    write (io_unit, *) 'Numerical precision: Single'
+      write (io_unit, *) 'Numerical precision: Single'
 #endif
-    write (io_unit, *) 'Compiled with ', compiler_version()
-    write (io_unit, *) 'Compiler options : ', compiler_options()
-    call MPI_Get_version(version, subversion, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_Get_version")
-    write (io_unit, '(" Version of the MPI library : ",I0,".",I0)') version, subversion
+      write (io_unit, *) 'Compiled with ', compiler_version()
+      write (io_unit, *) 'Compiler options : ', compiler_options()
+      call MPI_Get_version(version, subversion, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_Get_version")
+      write (io_unit, '(" Version of the MPI library : ",I0,".",I0)') version, subversion
 #ifdef DEBUG
-    write (io_unit, *) 'Compile flag DEBUG detected'
-    write (io_unit, *) '   debug level : ', decomp_debug
+      write (io_unit, *) 'Compile flag DEBUG detected'
+      write (io_unit, *) '   debug level : ', decomp_debug
 #endif
 #ifdef PROFILER
-    write (io_unit, *) 'Compile flag PROFILER detected'
+      write (io_unit, *) 'Compile flag PROFILER detected'
 #endif
 #ifdef SHM
-    write (io_unit, *) 'Compile flag SHM detected'
+      write (io_unit, *) 'Compile flag SHM detected'
 #endif
 #ifdef EVEN
-    write (io_unit, *) 'Compile flag EVEN detected'
+      write (io_unit, *) 'Compile flag EVEN detected'
 #endif
 #ifdef OCC
-    write (io_unit, *) 'Compile flag OCC detected'
+      write (io_unit, *) 'Compile flag OCC detected'
 #endif
 #ifdef OVERWRITE
-    write (io_unit, *) 'Compile flag OVERWRITE detected'
+      write (io_unit, *) 'Compile flag OVERWRITE detected'
 #endif
 #ifdef HALO_DEBUG
-    write (io_unit, *) 'Compile flag HALO_DEBUG detected'
+      write (io_unit, *) 'Compile flag HALO_DEBUG detected'
 #endif
 #ifdef SHM_DEBUG
-    write (io_unit, *) 'Compile flag SHM_DEBUG detected'
+      write (io_unit, *) 'Compile flag SHM_DEBUG detected'
 #endif
 #ifdef _GPU
-    write (io_unit, *) 'Compile flag _GPU detected'
+      write (io_unit, *) 'Compile flag _GPU detected'
 #endif
 #ifdef _NCCL
-    write (io_unit, *) 'Compile flag _NCCL detected'
+      write (io_unit, *) 'Compile flag _NCCL detected'
 #endif
-    write (io_unit, *) '==========================================================='
-    write (io_unit, *) 'Profiler id : ', decomp_profiler
+      write (io_unit, *) '==========================================================='
+      write (io_unit, *) 'Profiler id : ', decomp_profiler
 #ifdef PROFILER
-    call decomp_profiler_log(io_unit)
-    write(io_unit, *) "   Profiling transpose : ", decomp_profiler_transpose
-    write(io_unit, *) "   Profiling IO : ", decomp_profiler_io
-    write(io_unit, *) "   Profiling FFT : ", decomp_profiler_fft
-    write(io_unit, *) "   Profiling decomp_2d : ", decomp_profiler_d2d
+      call decomp_profiler_log(io_unit)
+      write (io_unit, *) "   Profiling transpose : ", decomp_profiler_transpose
+      write (io_unit, *) "   Profiling IO : ", decomp_profiler_io
+      write (io_unit, *) "   Profiling FFT : ", decomp_profiler_fft
+      write (io_unit, *) "   Profiling decomp_2d : ", decomp_profiler_d2d
 #endif
-    write (io_unit, *) '==========================================================='
-    ! Info about each decomp_info object
-    call decomp_info_print(decomp_main, io_unit, "decomp_main")
-    call decomp_info_print(phG, io_unit, "phG")
-    call decomp_info_print(ph1, io_unit, "ph1")
-    call decomp_info_print(ph2, io_unit, "ph2")
-    call decomp_info_print(ph3, io_unit, "ph3")
-    call decomp_info_print(ph4, io_unit, "ph4")
+      write (io_unit, *) '==========================================================='
+      ! Info about each decomp_info object
+      call decomp_info_print(decomp_main, io_unit, "decomp_main")
+      call decomp_info_print(phG, io_unit, "phG")
+      call decomp_info_print(ph1, io_unit, "ph1")
+      call decomp_info_print(ph2, io_unit, "ph2")
+      call decomp_info_print(ph3, io_unit, "ph3")
+      call decomp_info_print(ph4, io_unit, "ph4")
 #ifdef SHM_DEBUG
-    write (io_unit, *) '==========================================================='
-    call print_smp(io_unit)
+      write (io_unit, *) '==========================================================='
+      call print_smp(io_unit)
 #endif
-    write (io_unit, *) '==========================================================='
-    write (io_unit, *) '==========================================================='
+      write (io_unit, *) '==========================================================='
+      write (io_unit, *) '==========================================================='
 #ifdef DEBUG
-    !
-    ! In DEBUG mode, rank 0 will also print environment variables
-    !
-    ! At high debug level, all ranks will print env. variables
-    !
-    ! The system call, if writing to a file, is not blocking if supported
-    !
-    if (nrank == 0 .or. decomp_debug >= D2D_DEBUG_LEVEL_INFO) then
-       write (io_unit, *) '============== Environment variables ======================'
-       write (io_unit, *) '==========================================================='
-       write (io_unit, *) '==========================================================='
-       if (io_unit == output_unit ) then
-          call execute_command_line("env", wait = .true.)
-       else
-          inquire(unit = io_unit, name = fname, iostat = ierror)
-          if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                                __LINE__, &
-                                                ierror, &
-                                                "No name for the log file")
-          call execute_command_line("env >> "//trim(fname), wait = .false.)
-       endif
-    endif
+      !
+      ! In DEBUG mode, rank 0 will also print environment variables
+      !
+      ! At high debug level, all ranks will print env. variables
+      !
+      ! The system call, if writing to a file, is not blocking if supported
+      !
+      if (nrank == 0 .or. decomp_debug >= D2D_DEBUG_LEVEL_INFO) then
+         write (io_unit, *) '============== Environment variables ======================'
+         write (io_unit, *) '==========================================================='
+         write (io_unit, *) '==========================================================='
+         if (io_unit == output_unit) then
+            call execute_command_line("env", wait=.true.)
+         else
+            inquire (unit=io_unit, name=fname, iostat=ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  ierror, &
+                                                  "No name for the log file")
+            call execute_command_line("env >> "//trim(fname), wait=.false.)
+         end if
+      end if
 #endif
 
-  end subroutine d2d_listing
+   end subroutine d2d_listing
 
-  !
-  ! Print some information about given decomp_info object
-  !
-  module subroutine decomp_info_print(d2d, io_unit, d2dname)
+   !
+   ! Print some information about given decomp_info object
+   !
+   module subroutine decomp_info_print(d2d, io_unit, d2dname)
 
-    implicit none
+      implicit none
 
-    ! Arguments
-    type(decomp_info), intent(in) :: d2d
-    integer, intent(in) :: io_unit
-    character(len=*), intent(in) :: d2dname
+      ! Arguments
+      type(decomp_info), intent(in) :: d2d
+      integer, intent(in) :: io_unit
+      character(len=*), intent(in) :: d2dname
 
-    ! Nothing to print if not initialized
-    if (.not.allocated(d2d%x1dist)) then
-      write (io_unit, *) 'Uninitialized decomp_info ', d2dname
-      return
-    endif
+      ! Nothing to print if not initialized
+      if (.not. allocated(d2d%x1dist)) then
+         write (io_unit, *) 'Uninitialized decomp_info ', d2dname
+         return
+      end if
 
-    !
-    ! If DEBUG mode, print everything
-    ! Otherwise, print only global size
-    !
-    write (io_unit, *) 'Decomp_info : ', d2dname
-    write (io_unit, *) '   Global size : ', d2d%xsz(1), d2d%ysz(2), d2d%zsz(3)
+      !
+      ! If DEBUG mode, print everything
+      ! Otherwise, print only global size
+      !
+      write (io_unit, *) 'Decomp_info : ', d2dname
+      write (io_unit, *) '   Global size : ', d2d%xsz(1), d2d%ysz(2), d2d%zsz(3)
 #ifdef DEBUG
-    write (io_unit, *) '   xsz, xst, xen : ', d2d%xsz, d2d%xst, d2d%xen
-    write (io_unit, *) '   ysz, yst, yen : ', d2d%ysz, d2d%yst, d2d%yen
-    write (io_unit, *) '   zsz, zst, zen : ', d2d%zsz, d2d%zst, d2d%zen
-    write (io_unit, *) '   x1dist : ', d2d%x1dist
-    write (io_unit, *) '   y1dist : ', d2d%y1dist
-    write (io_unit, *) '   y2dist : ', d2d%y2dist
-    write (io_unit, *) '   z2dist : ', d2d%z2dist
-    write (io_unit, *) '   x1cnts : ', d2d%x1cnts
-    write (io_unit, *) '   y1cnts : ', d2d%y1cnts
-    write (io_unit, *) '   y2cnts : ', d2d%y2cnts
-    write (io_unit, *) '   z2cnts : ', d2d%z2cnts
-    write (io_unit, *) '   x1disp : ', d2d%x1disp
-    write (io_unit, *) '   y1disp : ', d2d%y1disp
-    write (io_unit, *) '   y2disp : ', d2d%y2disp
-    write (io_unit, *) '   z2disp : ', d2d%z2disp
-    write (io_unit, *) '   x1count : ', d2d%x1count
-    write (io_unit, *) '   y1count : ', d2d%y1count
-    write (io_unit, *) '   y2count : ', d2d%y2count
-    write (io_unit, *) '   z2count : ', d2d%z2count
-    write (io_unit, *) '   even : ', d2d%even
+      write (io_unit, *) '   xsz, xst, xen : ', d2d%xsz, d2d%xst, d2d%xen
+      write (io_unit, *) '   ysz, yst, yen : ', d2d%ysz, d2d%yst, d2d%yen
+      write (io_unit, *) '   zsz, zst, zen : ', d2d%zsz, d2d%zst, d2d%zen
+      write (io_unit, *) '   x1dist : ', d2d%x1dist
+      write (io_unit, *) '   y1dist : ', d2d%y1dist
+      write (io_unit, *) '   y2dist : ', d2d%y2dist
+      write (io_unit, *) '   z2dist : ', d2d%z2dist
+      write (io_unit, *) '   x1cnts : ', d2d%x1cnts
+      write (io_unit, *) '   y1cnts : ', d2d%y1cnts
+      write (io_unit, *) '   y2cnts : ', d2d%y2cnts
+      write (io_unit, *) '   z2cnts : ', d2d%z2cnts
+      write (io_unit, *) '   x1disp : ', d2d%x1disp
+      write (io_unit, *) '   y1disp : ', d2d%y1disp
+      write (io_unit, *) '   y2disp : ', d2d%y2disp
+      write (io_unit, *) '   z2disp : ', d2d%z2disp
+      write (io_unit, *) '   x1count : ', d2d%x1count
+      write (io_unit, *) '   y1count : ', d2d%y1count
+      write (io_unit, *) '   y2count : ', d2d%y2count
+      write (io_unit, *) '   z2count : ', d2d%z2count
+      write (io_unit, *) '   even : ', d2d%even
 #ifdef SHM
-    write (io_unit, *) '   listing of the SHM part is not yet implemented'
+      write (io_unit, *) '   listing of the SHM part is not yet implemented'
 #endif
 #endif
 
-  end subroutine decomp_info_print
+   end subroutine decomp_info_print
 
 #ifdef SHM_DEBUG
 
-  ! For debugging, print the shared-memory structure
-  module subroutine print_smp(io_unit)
-
-    implicit none
-    
-    ! Argument
-    integer, intent(in) :: io_unit
-
-    ! print out shared-memory information
-    write(io_unit,*)'I am mpi rank ', nrank, 'Total ranks ', nproc
-    write(io_unit,*)' '
-    write(io_unit,*)'Global data size:'
-    write(io_unit,*)'nx*ny*nz', nx,ny,nz
-    write(io_unit,*)' '
-    write(io_unit,*)'2D processor grid:'
-    write(io_unit,*)'p_row*p_col:', dims(1), dims(2)
-    write(io_unit,*)' '
-    write(io_unit,*)'Portion of global data held locally:'
-    write(io_unit,*)'xsize:',xsize
-    write(io_unit,*)'ysize:',ysize
-    write(io_unit,*)'zsize:',zsize
-    write(io_unit,*)' '
-    write(io_unit,*)'How pensils are to be divided and sent in alltoallv:'
-    write(io_unit,*)'x1dist:',decomp_main%x1dist
-    write(io_unit,*)'y1dist:',decomp_main%y1dist
-    write(io_unit,*)'y2dist:',decomp_main%y2dist
-    write(io_unit,*)'z2dist:',decomp_main%z2dist
-    write(io_unit,*)' '
-    write(io_unit,*)'######Shared buffer set up after this point######'
-    write(io_unit,*)' '
-    write(io_unit,*) 'col communicator detais:'
-    call print_smp_info(decomp_main%COL_INFO, io_unit)
-    write(io_unit,*)' '
-    write(io_unit,*) 'row communicator detais:'
-    call print_smp_info(decomp_main%ROW_INFO; io_unit)
-    write(io_unit,*)' '
-    write(io_unit,*)'Buffer count and displacement of per-core buffers'
-    write(io_unit,*)'x1cnts:',decomp_main%x1cnts
-    write(io_unit,*)'y1cnts:',decomp_main%y1cnts
-    write(io_unit,*)'y2cnts:',decomp_main%y2cnts
-    write(io_unit,*)'z2cnts:',decomp_main%z2cnts
-    write(io_unit,*)'x1disp:',decomp_main%x1disp
-    write(io_unit,*)'y1disp:',decomp_main%y1disp
-    write(io_unit,*)'y2disp:',decomp_main%y2disp
-    write(io_unit,*)'z2disp:',decomp_main%z2disp
-    write(io_unit,*)' '
-    write(io_unit,*)'Buffer count and displacement of shared buffers'
-    write(io_unit,*)'x1cnts:',decomp_main%x1cnts_s
-    write(io_unit,*)'y1cnts:',decomp_main%y1cnts_s
-    write(io_unit,*)'y2cnts:',decomp_main%y2cnts_s
-    write(io_unit,*)'z2cnts:',decomp_main%z2cnts_s
-    write(io_unit,*)'x1disp:',decomp_main%x1disp_s
-    write(io_unit,*)'y1disp:',decomp_main%y1disp_s
-    write(io_unit,*)'y2disp:',decomp_main%y2disp_s
-    write(io_unit,*)'z2disp:',decomp_main%z2disp_s
-
-  end subroutine print_smp
-
-  ! For debugging, print the shared-memory structure
-  module subroutine print_smp_info(s, io_unit)
-
-    implicit none
-
-    ! Argument
-    TYPE(SMP_INFO), intent(in) :: s
-    integer, intent(in) :: io_unit
-
-    write(io_unit,*) 'size of current communicator:', s%NCPU
-    write(io_unit,*) 'rank in current communicator:', s%NODE_ME
-    write(io_unit,*) 'number of SMP-nodes in this communicator:', s%NSMP
-    write(io_unit,*) 'SMP-node id (1 ~ NSMP):', s%SMP_ME
-    write(io_unit,*) 'NCORE - number of cores on this SMP-node', s%NCORE
-    write(io_unit,*) 'core id (1 ~ NCORE):', s%CORE_ME
-    write(io_unit,*) 'maximum no. cores on any SMP-node:', s%MAXCORE
-    write(io_unit,*) 'size of SMP shared memory SND buffer:', s%N_SND
-    write(io_unit,*) 'size of SMP shared memory RCV buffer:', s%N_RCV
-
-  end subroutine print_smp_info
+   ! For debugging, print the shared-memory structure
+   module subroutine print_smp(io_unit)
+
+      implicit none
+
+      ! Argument
+      integer, intent(in) :: io_unit
+
+      ! print out shared-memory information
+      write (io_unit, *) 'I am mpi rank ', nrank, 'Total ranks ', nproc
+      write (io_unit, *) ' '
+      write (io_unit, *) 'Global data size:'
+      write (io_unit, *) 'nx*ny*nz', nx, ny, nz
+      write (io_unit, *) ' '
+      write (io_unit, *) '2D processor grid:'
+      write (io_unit, *) 'p_row*p_col:', dims(1), dims(2)
+      write (io_unit, *) ' '
+      write (io_unit, *) 'Portion of global data held locally:'
+      write (io_unit, *) 'xsize:', xsize
+      write (io_unit, *) 'ysize:', ysize
+      write (io_unit, *) 'zsize:', zsize
+      write (io_unit, *) ' '
+      write (io_unit, *) 'How pensils are to be divided and sent in alltoallv:'
+      write (io_unit, *) 'x1dist:', decomp_main%x1dist
+      write (io_unit, *) 'y1dist:', decomp_main%y1dist
+      write (io_unit, *) 'y2dist:', decomp_main%y2dist
+      write (io_unit, *) 'z2dist:', decomp_main%z2dist
+      write (io_unit, *) ' '
+      write (io_unit, *) '######Shared buffer set up after this point######'
+      write (io_unit, *) ' '
+      write (io_unit, *) 'col communicator detais:'
+      call print_smp_info(decomp_main%COL_INFO, io_unit)
+      write (io_unit, *) ' '
+      write (io_unit, *) 'row communicator detais:'
+      call print_smp_info(decomp_main%ROW_INFO; io_unit)
+      write (io_unit, *) ' '
+      write (io_unit, *) 'Buffer count and displacement of per-core buffers'
+      write (io_unit, *) 'x1cnts:', decomp_main%x1cnts
+      write (io_unit, *) 'y1cnts:', decomp_main%y1cnts
+      write (io_unit, *) 'y2cnts:', decomp_main%y2cnts
+      write (io_unit, *) 'z2cnts:', decomp_main%z2cnts
+      write (io_unit, *) 'x1disp:', decomp_main%x1disp
+      write (io_unit, *) 'y1disp:', decomp_main%y1disp
+      write (io_unit, *) 'y2disp:', decomp_main%y2disp
+      write (io_unit, *) 'z2disp:', decomp_main%z2disp
+      write (io_unit, *) ' '
+      write (io_unit, *) 'Buffer count and displacement of shared buffers'
+      write (io_unit, *) 'x1cnts:', decomp_main%x1cnts_s
+      write (io_unit, *) 'y1cnts:', decomp_main%y1cnts_s
+      write (io_unit, *) 'y2cnts:', decomp_main%y2cnts_s
+      write (io_unit, *) 'z2cnts:', decomp_main%z2cnts_s
+      write (io_unit, *) 'x1disp:', decomp_main%x1disp_s
+      write (io_unit, *) 'y1disp:', decomp_main%y1disp_s
+      write (io_unit, *) 'y2disp:', decomp_main%y2disp_s
+      write (io_unit, *) 'z2disp:', decomp_main%z2disp_s
+
+   end subroutine print_smp
+
+   ! For debugging, print the shared-memory structure
+   module subroutine print_smp_info(s, io_unit)
+
+      implicit none
+
+      ! Argument
+      TYPE(SMP_INFO), intent(in) :: s
+      integer, intent(in) :: io_unit
+
+      write (io_unit, *) 'size of current communicator:', s%NCPU
+      write (io_unit, *) 'rank in current communicator:', s%NODE_ME
+      write (io_unit, *) 'number of SMP-nodes in this communicator:', s%NSMP
+      write (io_unit, *) 'SMP-node id (1 ~ NSMP):', s%SMP_ME
+      write (io_unit, *) 'NCORE - number of cores on this SMP-node', s%NCORE
+      write (io_unit, *) 'core id (1 ~ NCORE):', s%CORE_ME
+      write (io_unit, *) 'maximum no. cores on any SMP-node:', s%MAXCORE
+      write (io_unit, *) 'size of SMP shared memory SND buffer:', s%N_SND
+      write (io_unit, *) 'size of SMP shared memory RCV buffer:', s%N_RCV
+
+   end subroutine print_smp_info
 #endif
 
 end submodule d2d_log
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index c5d83fa4..bd783d7e 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,480 +13,474 @@
 
   subroutine transpose_x_to_y_real_short(src, dst)
 
-    implicit none
+     implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_x_to_y(src, dst, decomp_main)
+     call transpose_x_to_y(src, dst, decomp_main)
 
   end subroutine transpose_x_to_y_real_short
 
   subroutine transpose_x_to_y_real(src, dst, decomp)
 
-    implicit none
-    
-    Real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
 
+     Real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU) && defined(_NCCL)
-    integer :: col_rank_id, cuda_stat
-    type(ncclResult) :: nccl_stat
+     integer :: col_rank_id, cuda_stat
+     type(ncclResult) :: nccl_stat
 #endif
 
 #ifdef SHM
-    real(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     real(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-    ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%COL_INFO%SND_P
-    call mem_split_xy_real(src, s1, s2, s3, work1, dims(1), &
-         decomp%x1dist, decomp)
+     work1_p = decomp%COL_INFO%SND_P
+     call mem_split_xy_real(src, s1, s2, s3, work1, dims(1), &
+                            decomp%x1dist, decomp)
 #else
-  
+
 #if defined(_GPU)
-    call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
+                            decomp%x1dist, decomp)
 #else
-    call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
+                            decomp%x1dist, decomp)
 #endif
 
 #endif
 
-    ! define receive buffer
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%COL_INFO%RCV_P
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%COL_INFO%RCV_P
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
-    ! transpose using MPI_ALLTOALL(V)
+
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-    if (decomp%COL_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
-            real_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
-            real_type, decomp%COL_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%COL_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
+                           real_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
+                           real_type, decomp%COL_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    call MPI_ALLTOALL(work1_r, decomp%x1count, &
-         real_type, work2_r, decomp%y1count, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_r, decomp%x1count, &
+                       real_type, work2_r, decomp%y1count, &
+                       real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    nccl_stat = ncclGroupStart()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-    do col_rank_id = 0, (col_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d( decomp%x1disp(col_rank_id)+1 ), decomp%x1cnts(col_rank_id), &
-          ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+     nccl_stat = ncclGroupStart()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+     do col_rank_id = 0, (col_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
+                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d( decomp%y1disp(col_rank_id)+1 ), decomp%y1cnts(col_rank_id), &
-          ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+        nccl_stat = ncclRecv(work2_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
+                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-    end do
-    nccl_stat = ncclGroupEnd()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     end do
+     nccl_stat = ncclGroupEnd()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-    call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
-         real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
+                        real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-    call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
-         real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
+                        real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_xy_real(work2, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_xy_real(work2, d1, d2, d3, dst, dims(1), &
+                            decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+                            decomp%y1dist, decomp)
 #else
-    call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
+                            decomp%y1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
 #endif
 
-    return
+     return
   end subroutine transpose_x_to_y_real
 
-
   subroutine transpose_x_to_y_complex_short(src, dst)
 
-    implicit none
+     implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_x_to_y(src, dst, decomp_main)
+     call transpose_x_to_y(src, dst, decomp_main)
 
   end subroutine transpose_x_to_y_complex_short
 
   subroutine transpose_x_to_y_complex(src, dst, decomp)
 
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #ifdef SHM
-    complex(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     complex(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
-    
-    ! rearrange source array as send buffer
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
+
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%COL_INFO%SND_P_c
-    call mem_split_xy_complex(src, s1, s2, s3, work1, dims(1), &
-         decomp%x1dist, decomp)
+     work1_p = decomp%COL_INFO%SND_P_c
+     call mem_split_xy_complex(src, s1, s2, s3, work1, dims(1), &
+                               decomp%x1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+                               decomp%x1dist, decomp)
 #else
-    call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
+                               decomp%x1dist, decomp)
 #endif
 
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%COL_INFO%RCV_P_c
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%COL_INFO%RCV_P_c
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
-    ! transpose using MPI_ALLTOALL(V)
+
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-    if (decomp%COL_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
-            complex_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
-            complex_type, decomp%COL_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%COL_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
+                           complex_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
+                           complex_type, decomp%COL_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    call MPI_ALLTOALL(work1_c, decomp%x1count, &
-         complex_type, work2_c, decomp%y1count, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_c, decomp%x1count, &
+                       complex_type, work2_c, decomp%y1count, &
+                       complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-    call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
-         complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-    call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
-         complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_xy_complex(work2, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_xy_complex(work2, d1, d2, d3, dst, dims(1), &
+                               decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+                               decomp%y1dist, decomp)
 #else
-    call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
+                               decomp%y1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
 #endif
 
-    return
+     return
   end subroutine transpose_x_to_y_complex
 
+  subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_xy_real(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    real(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
-    
-    integer :: i,j,k, m,i1,i2,pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%x1disp_o(m) + 1
+        pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%x1count + 1
+        pos = m*decomp%x1count + 1
 #else
-       pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=1,n2
-             do i=i1,i2
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_xy_real
 
+  subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_xy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    complex(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%x1disp_o(m) + 1
+        pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%x1count + 1
+        pos = m*decomp%x1count + 1
 #else
-       pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=1,n2
-             do i=i1,i2
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_xy_complex
 
+  subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_xy_real(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(*), intent(IN) :: in
-    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
-        
-    integer :: i,j,k, m,i1,i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y1disp_o(m) + 1
+        pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y1count + 1
+        pos = m*decomp%y1count + 1
 #else
-       pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_xy_real
 
+  subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_xy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(*), intent(IN) :: in
-    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y1disp_o(m) + 1
+        pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y1count + 1
+        pos = m*decomp%y1count + 1
 #else
-       pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_xy_complex
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index 1e7c62c2..a742ead6 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,479 +13,474 @@
 
   subroutine transpose_y_to_x_real_short(src, dst)
 
-    implicit none
+     implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_y_to_x(src, dst, decomp_main)
+     call transpose_y_to_x(src, dst, decomp_main)
 
   end subroutine transpose_y_to_x_real_short
 
   subroutine transpose_y_to_x_real(src, dst, decomp)
 
-    implicit none
-    
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU) && defined(_NCCL)
-    type(ncclResult) :: nccl_stat
-    integer :: col_rank_id, cuda_stat
+     type(ncclResult) :: nccl_stat
+     integer :: col_rank_id, cuda_stat
 #endif
 
 #ifdef SHM
-    real(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     real(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-    ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%COL_INFO%SND_P
-    call mem_split_yx_real(src, s1, s2, s3, work1, dims(1), &
-         decomp%y1dist, decomp)
+     work1_p = decomp%COL_INFO%SND_P
+     call mem_split_yx_real(src, s1, s2, s3, work1, dims(1), &
+                            decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
+                            decomp%y1dist, decomp)
 #else
-    call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
+                            decomp%y1dist, decomp)
 #endif
 
 #endif
 
-    ! define receive buffer
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%COL_INFO%RCV_P
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%COL_INFO%RCV_P
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
-    ! transpose using MPI_ALLTOALL(V)
+
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-    if (decomp%COL_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
-            real_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
-            real_type, decomp%COL_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%COL_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
+                           real_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
+                           real_type, decomp%COL_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    call MPI_ALLTOALL(work1_r, decomp%y1count, &
-         real_type, work2_r, decomp%x1count, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_r, decomp%y1count, &
+                       real_type, work2_r, decomp%x1count, &
+                       real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    nccl_stat = ncclGroupStart()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-    do col_rank_id = 0, (col_comm_size - 1)
-     nccl_stat = ncclSend(work1_r_d( decomp%y1disp(col_rank_id)+1 ), decomp%y1cnts(col_rank_id), &
-       ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-     nccl_stat = ncclRecv(work2_r_d( decomp%x1disp(col_rank_id)+1 ), decomp%x1cnts(col_rank_id), &
-       ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-    end do
-    nccl_stat = ncclGroupEnd()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     nccl_stat = ncclGroupStart()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+     do col_rank_id = 0, (col_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
+                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
+        nccl_stat = ncclRecv(work2_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
+                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
+     end do
+     nccl_stat = ncclGroupEnd()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-    call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
-         real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
+                        real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-    call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
-         real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
+                        real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_yx_real(work2, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_yx_real(work2, d1, d2, d3, dst, dims(1), &
+                            decomp%x1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+                            decomp%x1dist, decomp)
 #else
-    call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
+                            decomp%x1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
 #endif
 
-    return
+     return
   end subroutine transpose_y_to_x_real
 
-
   subroutine transpose_y_to_x_complex_short(src, dst)
 
-    implicit none
+     implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_y_to_x(src, dst, decomp_main)
+     call transpose_y_to_x(src, dst, decomp_main)
 
   end subroutine transpose_y_to_x_complex_short
 
   subroutine transpose_y_to_x_complex(src, dst, decomp)
 
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #ifdef SHM
-    complex(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     complex(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
-    
-    ! rearrange source array as send buffer
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
+
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%COL_INFO%SND_P_c
-    call mem_split_yx_complex(src, s1, s2, s3, work1, dims(1), &
-         decomp%y1dist, decomp)
+     work1_p = decomp%COL_INFO%SND_P_c
+     call mem_split_yx_complex(src, s1, s2, s3, work1, dims(1), &
+                               decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+                               decomp%y1dist, decomp)
 #else
-    call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
+                               decomp%y1dist, decomp)
 #endif
 
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%COL_INFO%RCV_P_c
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%COL_INFO%RCV_P_c
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
-    ! transpose using MPI_ALLTOALL(V)
+
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-    if (decomp%COL_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
-            complex_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
-            complex_type, decomp%COL_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%COL_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
+                           complex_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
+                           complex_type, decomp%COL_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    call MPI_ALLTOALL(work1_c, decomp%y1count, &
-         complex_type, work2_c, decomp%x1count, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_c, decomp%y1count, &
+                       complex_type, work2_c, decomp%x1count, &
+                       complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-    call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
-         complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-    call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
-         complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_yx_complex(work2, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_yx_complex(work2, d1, d2, d3, dst, dims(1), &
+                               decomp%x1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+                               decomp%x1dist, decomp)
 #else
-    call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
+                               decomp%x1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
 #endif
 
-    return
+     return
   end subroutine transpose_y_to_x_complex
 
+  subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_yx_real(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    real(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y1disp_o(m) + 1
+        pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y1count + 1
+        pos = m*decomp%y1count + 1
 #else
-       pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_yx_real
 
+  subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_yx_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    complex(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y1disp_o(m) + 1
+        pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y1count + 1
+        pos = m*decomp%y1count + 1
 #else
-       pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_yx_complex
 
+  subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_yx_real(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(*), intent(IN) :: in
-    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%x1disp_o(m) + 1
+        pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%x1count + 1
+        pos = m*decomp%x1count + 1
 #else
-       pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=1,n2
-             do i=i1,i2
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_yx_real
 
+  subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_yx_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(*), intent(IN) :: in
-    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%x1disp_o(m) + 1
+        pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%x1count + 1
+        pos = m*decomp%x1count + 1
 #else
-       pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=1,n2
-             do i=i1,i2
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_yx_complex
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 70f427cf..62bc9071 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,488 +13,484 @@
 
   subroutine transpose_y_to_z_real_short(src, dst)
 
-    implicit none
+     implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_y_to_z(src, dst, decomp_main)
+     call transpose_y_to_z(src, dst, decomp_main)
 
   end subroutine transpose_y_to_z_real_short
 
   subroutine transpose_y_to_z_real(src, dst, decomp)
 
-    implicit none
-    
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    type(ncclResult) :: nccl_stat
-    integer :: row_rank_id, cuda_stat
+     type(ncclResult) :: nccl_stat
+     integer :: row_rank_id, cuda_stat
 #endif
-    integer :: istat
+     integer :: istat
 #endif
 
 #ifdef SHM
-    real(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     real(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-    ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%ROW_INFO%SND_P
-    call mem_split_yz_real(src, s1, s2, s3, work1, dims(2), &
-         decomp%y2dist, decomp)
+     work1_p = decomp%ROW_INFO%SND_P
+     call mem_split_yz_real(src, s1, s2, s3, work1, dims(2), &
+                            decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
+                            decomp%y2dist, decomp)
 #else
-    call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
+                            decomp%y2dist, decomp)
 #endif
 
 #endif
 
-    ! define receive buffer
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%ROW_INFO%RCV_P
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%ROW_INFO%RCV_P
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
+
 #ifdef SHM
-    if (decomp%ROW_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
-            real_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
-            real_type, decomp%ROW_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%ROW_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
+                           real_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
+                           real_type, decomp%ROW_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    if (decomp%even) then
-       call MPI_ALLTOALL(work1_r, decomp%y2count, &
-            real_type, dst, decomp%z2count, &
-            real_type, DECOMP_2D_COMM_ROW, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    else
-       call MPI_ALLTOALL(work1_r, decomp%y2count, &
-            real_type, work2_r, decomp%z2count, &
-            real_type, DECOMP_2D_COMM_ROW, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
-    end if
+     if (decomp%even) then
+        call MPI_ALLTOALL(work1_r, decomp%y2count, &
+                          real_type, dst, decomp%z2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     else
+        call MPI_ALLTOALL(work1_r, decomp%y2count, &
+                          real_type, work2_r, decomp%z2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     end if
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    nccl_stat = ncclGroupStart()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-    do row_rank_id = 0, (row_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d( decomp%y2disp(row_rank_id)+1 ), decomp%y2cnts(row_rank_id), &
-          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+     nccl_stat = ncclGroupStart()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+     do row_rank_id = 0, (row_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
+                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d( decomp%z2disp(row_rank_id)+1 ), decomp%z2cnts(row_rank_id), &
-          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+        nccl_stat = ncclRecv(work2_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
+                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-    end do
-    nccl_stat = ncclGroupEnd()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     end do
+     nccl_stat = ncclGroupEnd()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-    call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
-         real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
-         real_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
+                        real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-    call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
-         real_type, dst, decomp%z2cnts, decomp%z2disp, &
-         real_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
+                        real_type, dst, decomp%z2cnts, decomp%z2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_yz_real(work2, d1, d2, d3, dst, dims(2), &
-         decomp%z2dist, decomp)
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_yz_real(work2, d1, d2, d3, dst, dims(2), &
+                            decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-    if (.not. decomp%even) then
-       call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
-         decomp%z2dist, decomp)
-    end if
+     if (.not. decomp%even) then
+        call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
+                               decomp%z2dist, decomp)
+     end if
 #else
-    ! note the receive buffer is already in natural (i,j,k) order
-    ! so no merge operation needed
+     ! note the receive buffer is already in natural (i,j,k) order
+     ! so no merge operation needed
 
 #if defined(_GPU)
-    istat = cudaMemcpy( dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
+     istat = cudaMemcpy(dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
 #endif
 
 #endif
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
 #endif
 
-    return
+     return
   end subroutine transpose_y_to_z_real
 
-
   subroutine transpose_y_to_z_complex_short(src, dst)
 
-    implicit none
+     implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_y_to_z(src, dst, decomp_main)
+     call transpose_y_to_z(src, dst, decomp_main)
 
   end subroutine transpose_y_to_z_complex_short
 
   subroutine transpose_y_to_z_complex(src, dst, decomp)
 
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-    integer :: istat
+     integer :: istat
 #endif
 
 #ifdef SHM
-    complex(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     complex(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
-    
-    ! rearrange source array as send buffer
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
+
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%ROW_INFO%SND_P_c
-    call mem_split_yz_complex(src, s1, s2, s3, work1, dims(2), &
-         decomp%y2dist, decomp)
+     work1_p = decomp%ROW_INFO%SND_P_c
+     call mem_split_yz_complex(src, s1, s2, s3, work1, dims(2), &
+                               decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
+                               decomp%y2dist, decomp)
 #else
-    call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
+                               decomp%y2dist, decomp)
 #endif
 
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%ROW_INFO%RCV_P_c
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%ROW_INFO%RCV_P_c
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
+
 #ifdef SHM
-    if (decomp%ROW_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
-            complex_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
-            complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%ROW_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
+                           complex_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
+                           complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    if (decomp%even) then
-       call MPI_ALLTOALL(work1_c, decomp%y2count, &
-            complex_type, dst, decomp%z2count, &
-            complex_type, DECOMP_2D_COMM_ROW, ierror)
-    else
-       call MPI_ALLTOALL(work1_c, decomp%y2count, &
-            complex_type, work2_c, decomp%z2count, &
-            complex_type, DECOMP_2D_COMM_ROW, ierror)
-    end if
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     if (decomp%even) then
+        call MPI_ALLTOALL(work1_c, decomp%y2count, &
+                          complex_type, dst, decomp%z2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     else
+        call MPI_ALLTOALL(work1_c, decomp%y2count, &
+                          complex_type, work2_c, decomp%z2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-    call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
-         complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
-         complex_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-    call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
-         complex_type, dst, decomp%z2cnts, decomp%z2disp, &
-         complex_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, dst, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_yz_complex(work2, d1, d2, d3, dst, dims(2), &
-         decomp%z2dist, decomp)
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_yz_complex(work2, d1, d2, d3, dst, dims(2), &
+                               decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-    if (.not. decomp%even) then
-       call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
-         decomp%z2dist, decomp)
-    end if
+     if (.not. decomp%even) then
+        call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
+                                  decomp%z2dist, decomp)
+     end if
 #else
-    ! note the receive buffer is already in natural (i,j,k) order
-    ! so no merge operation needed
+     ! note the receive buffer is already in natural (i,j,k) order
+     ! so no merge operation needed
 
 #if defined(_GPU)
-    istat = cudaMemcpy( dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
+     istat = cudaMemcpy(dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
 #endif
 
 #endif
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
 #endif
 
-    return
+     return
   end subroutine transpose_y_to_z_complex
 
-  subroutine mem_split_yz_real(in,n1,n2,n3,out,iproc,dist,decomp)
+  subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    real(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y2disp_o(m) + 1
+        pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y2count + 1
+        pos = m*decomp%y2count + 1
 #else
-       pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_yz_real
 
+  subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_yz_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    complex(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y2disp_o(m) + 1
+        pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y2count + 1
+        pos = m*decomp%y2count + 1
 #else
-       pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_yz_complex
 
+  subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
+
+     implicit none
+
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-  subroutine mem_merge_yz_real(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(*), intent(IN) :: in
-    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
-    integer :: i,j,k, m,i1,i2, pos
-
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%z2disp_o(m) + 1
+        pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%z2count + 1
+        pos = m*decomp%z2count + 1
 #else
-       pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-       do k=i1,i2
-          do j=1,n2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-    end do
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
 
-    return
+     return
   end subroutine mem_merge_yz_real
 
+  subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+
+     implicit none
+
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+
+     integer :: i, j, k, m, i1, i2, pos
 
-  subroutine mem_merge_yz_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-    
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(*), intent(IN) :: in
-    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
-    integer :: i,j,k, m,i1,i2, pos
-
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%z2disp_o(m) + 1
+        pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%z2count + 1
+        pos = m*decomp%z2count + 1
 #else
-       pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-       do k=i1,i2
-          do j=1,n2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-    end do
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
 
-    return
+     return
   end subroutine mem_merge_yz_complex
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index 33030a7d..a2f86844 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,490 +13,485 @@
 
   subroutine transpose_z_to_y_real_short(src, dst)
 
-    implicit none
+     implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_z_to_y(src, dst, decomp_main)
+     call transpose_z_to_y(src, dst, decomp_main)
 
   end subroutine transpose_z_to_y_real_short
 
   subroutine transpose_z_to_y_real(src, dst, decomp)
 
-    implicit none
-    
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    type(ncclResult) :: nccl_stat
-    integer :: row_rank_id, cuda_stat
+     type(ncclResult) :: nccl_stat
+     integer :: row_rank_id, cuda_stat
 #endif
-    integer :: istat
+     integer :: istat
 #endif
 
 #ifdef SHM
-    real(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     real(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-    ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%ROW_INFO%SND_P
-    call mem_split_zy_real(src, s1, s2, s3, work1, dims(2), &
-         decomp%z2dist, decomp)
+     work1_p = decomp%ROW_INFO%SND_P
+     call mem_split_zy_real(src, s1, s2, s3, work1, dims(2), &
+                            decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-    if (.not. decomp%even) then
-       call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
-            decomp%z2dist, decomp)
-    end if
+     if (.not. decomp%even) then
+        call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
+                               decomp%z2dist, decomp)
+     end if
 #else
-    ! note the src array is suitable to be a send buffer
-    ! so no split operation needed
+     ! note the src array is suitable to be a send buffer
+     ! so no split operation needed
 
 #if defined(_GPU)
-    istat = cudaMemcpy( work1_r_d, src, s1*s2*s3 )
+     istat = cudaMemcpy(work1_r_d, src, s1*s2*s3)
 #endif
 
 #endif
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%ROW_INFO%RCV_P
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%ROW_INFO%RCV_P
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
+
 #ifdef SHM
-    if (decomp%ROW_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
-            real_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
-            real_type, decomp%ROW_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%ROW_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
+                           real_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
+                           real_type, decomp%ROW_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    if (decomp%even) then
-       call MPI_ALLTOALL(src, decomp%z2count, &
-            real_type, work2_r, decomp%y2count, &
-            real_type, DECOMP_2D_COMM_ROW, ierror)
-    else
-       call MPI_ALLTOALL(work1_r, decomp%z2count, &
-            real_type, work2_r, decomp%y2count, &
-            real_type, DECOMP_2D_COMM_ROW, ierror)
-    end if
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     if (decomp%even) then
+        call MPI_ALLTOALL(src, decomp%z2count, &
+                          real_type, work2_r, decomp%y2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+     else
+        call MPI_ALLTOALL(work1_r, decomp%z2count, &
+                          real_type, work2_r, decomp%y2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    nccl_stat = ncclGroupStart()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-    do row_rank_id = 0, (row_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d( decomp%z2disp(row_rank_id)+1 ), decomp%z2cnts(row_rank_id), &
-          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+     nccl_stat = ncclGroupStart()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+     do row_rank_id = 0, (row_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
+                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d( decomp%y2disp(row_rank_id)+1 ), decomp%y2cnts(row_rank_id), &
-          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+        nccl_stat = ncclRecv(work2_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
+                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-    end do
-    nccl_stat = ncclGroupEnd()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     end do
+     nccl_stat = ncclGroupEnd()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-    call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
-         real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
-         real_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
+                        real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-    call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-         real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
-         real_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+                        real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_zy_real(work2, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_zy_real(work2, d1, d2, d3, dst, dims(2), &
+                            decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
+                            decomp%y2dist, decomp)
 #else
-    call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
+                            decomp%y2dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
 #endif
 
-    return
+     return
   end subroutine transpose_z_to_y_real
 
-
   subroutine transpose_z_to_y_complex_short(src, dst)
 
-    implicit none
+     implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_z_to_y(src, dst, decomp_main)
+     call transpose_z_to_y(src, dst, decomp_main)
 
   end subroutine transpose_z_to_y_complex_short
 
   subroutine transpose_z_to_y_complex(src, dst, decomp)
 
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-    integer :: istat
+     integer :: istat
 #endif
 
 #ifdef SHM
-    complex(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     complex(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
-    
-    ! rearrange source array as send buffer
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
+
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%ROW_INFO%SND_P_c
-    call mem_split_zy_complex(src, s1, s2, s3, work1, dims(2), &
-         decomp%z2dist, decomp)
+     work1_p = decomp%ROW_INFO%SND_P_c
+     call mem_split_zy_complex(src, s1, s2, s3, work1, dims(2), &
+                               decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-    if (.not. decomp%even) then
-       call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
-            decomp%z2dist, decomp)
-    end if
+     if (.not. decomp%even) then
+        call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
+                                  decomp%z2dist, decomp)
+     end if
 #else
-    ! note the src array is suitable to be a send buffer
-    ! so no split operation needed
+     ! note the src array is suitable to be a send buffer
+     ! so no split operation needed
 
 #if defined(_GPU)
-    istat = cudaMemcpy( work1_c_d, src, s1*s2*s3 )
+     istat = cudaMemcpy(work1_c_d, src, s1*s2*s3)
 #endif
 
 #endif
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%ROW_INFO%RCV_P_c
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%ROW_INFO%RCV_P_c
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
+
 #ifdef SHM
-    if (decomp%ROW_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
-            complex_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
-            complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%ROW_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
+                           complex_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
+                           complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    if (decomp%even) then
-       call MPI_ALLTOALL(src, decomp%z2count, &
-            complex_type, work2_c, decomp%y2count, &
-            complex_type, DECOMP_2D_COMM_ROW, ierror)
-    else
-       call MPI_ALLTOALL(work1_c, decomp%z2count, &
-            complex_type, work2_c, decomp%y2count, &
-            complex_type, DECOMP_2D_COMM_ROW, ierror)
-    end if
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     if (decomp%even) then
+        call MPI_ALLTOALL(src, decomp%z2count, &
+                          complex_type, work2_c, decomp%y2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     else
+        call MPI_ALLTOALL(work1_c, decomp%z2count, &
+                          complex_type, work2_c, decomp%y2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-    call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
-         complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
-         complex_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-    call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-         complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
-         complex_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_zy_complex(work2, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_zy_complex(work2, d1, d2, d3, dst, dims(2), &
+                               decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
+                               decomp%y2dist, decomp)
 #else
-    call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
+                               decomp%y2dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
 #endif
 
-    return
+     return
   end subroutine transpose_z_to_y_complex
 
-
   ! pack/unpack ALLTOALL(V) buffers
 
-  subroutine mem_split_zy_real(in,n1,n2,n3,out,iproc,dist,decomp)
+  subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    real(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
-    integer :: i,j,k, m,i1,i2,pos
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%z2disp_o(m) + 1
+        pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%z2count + 1
+        pos = m*decomp%z2count + 1
 #else
-       pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-       do k=i1,i2
-          do j=1,n2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-    end do
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
 
-    return
+     return
   end subroutine mem_split_zy_real
 
+  subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_zy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    complex(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%z2disp_o(m) + 1
+        pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%z2count + 1
+        pos = m*decomp%z2count + 1
 #else
-       pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-       do k=i1,i2
-          do j=1,n2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-    end do
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
 
-    return
+     return
   end subroutine mem_split_zy_complex
 
+  subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_zy_real(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(*), intent(IN) :: in
-    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y2disp_o(m) + 1
+        pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y2count + 1
+        pos = m*decomp%y2count + 1
 #else
-       pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_zy_real
 
+  subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+
+     implicit none
+
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-  subroutine mem_merge_zy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-    
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(*), intent(IN) :: in
-    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y2disp_o(m) + 1
+        pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y2count + 1
+        pos = m*decomp%y2count + 1
 #else
-       pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_zy_complex

From 42dcb0fafe6337e66e284fe0ced51adcf954cc83 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 8 Dec 2022 09:45:54 +0100
Subject: [PATCH 028/436] Revert "Apply fprettify --enable-replacements
 --c-relations"

This reverts commit 13814db122af04f72d1243204c9b7e7a6b44925f.
---
 examples/fft_physical_x/fft_grid_x.f90     |  234 +-
 examples/fft_physical_x/fft_physical_x.f90 |  377 +--
 examples/fft_physical_z/fft_physical_z.f90 |  378 +--
 examples/halo_test/halo_test.f90           |  772 ++---
 examples/init_test/init_test.f90           |  120 +-
 examples/io_test/io_bench.f90              |   50 +-
 examples/io_test/io_plane_test.f90         |  230 +-
 examples/io_test/io_read.f90               |  128 +-
 examples/io_test/io_test.f90               |  210 +-
 examples/io_test/io_var_test.f90           |  552 ++--
 examples/test2d/test2d.f90                 |  260 +-
 src/alloc.f90                              |  424 +--
 src/decomp_2d.f90                          | 3077 +++++++++----------
 src/decomp_2d_init_fin.f90                 |  591 ++--
 src/factor.f90                             |  150 +-
 src/fft_common.f90                         |  221 +-
 src/fft_common_3d.f90                      |  284 +-
 src/fft_cufft.f90                          | 1558 +++++-----
 src/fft_fftw3.f90                          |  937 +++---
 src/fft_fftw3_f03.f90                      | 1507 ++++-----
 src/fft_generic.f90                        |  536 ++--
 src/fft_mkl.f90                            | 1048 +++----
 src/glassman.f90                           |  329 +-
 src/halo.f90                               |  120 +-
 src/halo_common.f90                        |  366 +--
 src/io.f90                                 | 3219 ++++++++++----------
 src/io_read_inflow.f90                     |   80 +-
 src/io_read_one.inc                        |   64 +-
 src/io_read_var.inc                        |   72 +-
 src/io_write_every.inc                     |  218 +-
 src/io_write_one.inc                       |   84 +-
 src/io_write_outflow.f90                   |   80 +-
 src/io_write_plane.inc                     |  172 +-
 src/io_write_var.inc                       |   70 +-
 src/log.f90                                |  454 +--
 src/transpose_x_to_y.f90                   |  590 ++--
 src/transpose_y_to_x.f90                   |  589 ++--
 src/transpose_y_to_z.f90                   |  610 ++--
 src/transpose_z_to_y.f90                   |  605 ++--
 39 files changed, 10736 insertions(+), 10630 deletions(-)

diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 7955a81e..5b424138 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -1,130 +1,130 @@
 program fft_physical_x
 
-   use decomp_2d
-   use decomp_2d_fft
-   use MPI
-#if defined(_GPU)
-   use cudafor
-   use cufft
-   use openacc
+  use decomp_2d
+  use decomp_2d_fft
+  use MPI
+#if defined(_GPU) 
+  use cudafor
+  use cufft
+  use openacc 
 #endif
 
-   implicit none
-
-   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
-   integer :: nx, ny, nz
-   integer :: p_row = 0, p_col = 0
-   integer :: resize_domain
-   integer :: nranks_tot
-
-   integer, parameter :: ntest = 10  ! repeat test this times
-
-   type(decomp_info), pointer :: ph => null()
-   complex(mytype), allocatable, dimension(:, :, :) :: in, out
-
-   real(mytype) :: dr, di, error, err_all, n1, flops
-   integer :: ierror, i, j, k, m
-   real(mytype) :: t1, t2, t3, t4
-
-   call MPI_INIT(ierror)
-   ! To resize the domain we need to know global number of ranks
-   ! This operation is also done as part of decomp_2d_init
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
-   call decomp_2d_init(nx + 1, ny + 1, nz + 1, p_row, p_col)
+  implicit none
+  
+  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
+  integer :: nx, ny, nz
+  integer :: p_row=0, p_col=0
+  integer :: resize_domain
+  integer :: nranks_tot
+  
+  integer, parameter :: ntest = 10  ! repeat test this times
+ 
+  type(decomp_info), pointer :: ph=>null()
+  complex(mytype), allocatable, dimension(:,:,:) :: in, out
+  
+  real(mytype) :: dr,di, error, err_all, n1,flops
+  integer :: ierror, i,j,k,m
+  real(mytype) :: t1, t2, t3 ,t4
+  
+  call MPI_INIT(ierror)
+  ! To resize the domain we need to know global number of ranks
+  ! This operation is also done as part of decomp_2d_init
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+  resize_domain = int(nranks_tot/4)+1
+  nx = nx_base*resize_domain
+  ny = ny_base*resize_domain
+  nz = nz_base*resize_domain
+  call decomp_2d_init(nx+1,ny+1,nz+1,p_row,p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Test the c2c interface
+  ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-   call decomp_2d_fft_init(PHYSICAL_IN_X, nx, ny, nz) ! force the default x pencil
-   ph => decomp_2d_fft_get_ph()
-   !  input is X-pencil data
-   ! output is Z-pencil data
-   call alloc_x(in, ph, .true.)
-   call alloc_z(out, ph, .true.)
-   ! initilise input
-   do k = ph%xst(3), ph%xen(3)
-      do j = ph%xst(2), ph%xen(2)
-         do i = ph%xst(1), ph%xen(1)
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-            di = dr
-            in(i, j, k) = cmplx(dr, di, mytype)
-         end do
-      end do
-   end do
-
-   t2 = 0._mytype
-   t4 = 0._mytype
-   do m = 1, ntest
-
-      ! forward FFT
-      t1 = MPI_WTIME()
-      call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
-      t2 = t2 + MPI_WTIME() - t1
-
-      ! inverse FFT
-      t3 = MPI_WTIME()
-      call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
-      t4 = t4 + MPI_WTIME() - t3
-
-      ! normalisation - note 2DECOMP&FFT doesn't normalise
-      !$acc kernels
-      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
-      !$acc end kernels
-
-   end do
+  call decomp_2d_fft_init(PHYSICAL_IN_X, nx, ny, nz) ! force the default x pencil
+  ph => decomp_2d_fft_get_ph()
+  !  input is X-pencil data
+  ! output is Z-pencil data
+  call alloc_x(in, ph, .true.)
+  call alloc_z(out, ph, .true.)
+  ! initilise input
+  do k=ph%xst(3),ph%xen(3)
+     do j=ph%xst(2),ph%xen(2)
+        do i=ph%xst(1),ph%xen(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           di = dr
+           in(i,j,k) = cmplx(dr,di,mytype)
+        end do
+     end do
+  end do
+  
+  t2 = 0._mytype
+  t4 = 0._mytype
+  do m=1,ntest
+     
+     ! forward FFT
+     t1 = MPI_WTIME()
+     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+     t2 = t2 + MPI_WTIME() - t1
+
+     ! inverse FFT
+     t3 = MPI_WTIME()
+     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+     t4 = t4 + MPI_WTIME() - t3
+  
+     ! normalisation - note 2DECOMP&FFT doesn't normalise
+     !$acc kernels
+     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
+     !$acc end kernels
+
+  end do
 #if defined(_GPU)
-   ierror = cudaDeviceSynchronize()
+  ierror = cudaDeviceSynchronize()
 #endif
-
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
-
-   ! checking accuracy
-   error = 0._mytype
-   do k = ph%xst(3), ph%xen(3)
-      do j = ph%xst(2), ph%xen(2)
-         do i = ph%xst(1), ph%xen(1)
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-            di = dr
-            dr = dr - real(in(i, j, k), mytype)
-            di = di - aimag(in(i, j, k))
-            error = error + sqrt(dr*dr + di*di)
-         end do
-      end do
-   end do
-   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
-
-   if (nrank == 0) then
-      write (*, *) '===== c2c interface ====='
-      write (*, *) 'error / mesh point: ', err_all
-      write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
-      n1 = n1**(1._mytype/3._mytype)
-      ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
-      ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops*3._mytype*n1**2
-      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
-   end if
-
-   deallocate (in, out)
-   nullify (ph)
-   call decomp_2d_fft_finalize
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
+  
+  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t1 = t1 / real(nproc,mytype)
+  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t3 = t3 / real(nproc,mytype)
+  
+  ! checking accuracy
+  error = 0._mytype
+  do k=ph%xst(3),ph%xen(3) 
+     do j=ph%xst(2),ph%xen(2)
+        do i=ph%xst(1),ph%xen(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           di = dr
+           dr = dr - real(in(i,j,k),mytype)
+           di = di - aimag(in(i,j,k))
+           error = error + sqrt(dr*dr + di*di)
+        end do
+     end do
+  end do
+  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
+  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
+
+  if (nrank==0) then
+     write(*,*) '===== c2c interface ====='
+     write(*,*) 'error / mesh point: ', err_all 
+     write(*,*) 'time (sec): ', t1,t3
+     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
+     n1 = n1 ** (1._mytype/3._mytype)
+     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
+     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+     flops = flops * 3._mytype * n1**2  
+     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
+     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
+  end if
+  
+  deallocate(in,out)
+  nullify(ph)
+  call decomp_2d_fft_finalize
+  call decomp_2d_finalize
+  call MPI_FINALIZE(ierror)
 
 end program fft_physical_x
 
diff --git a/examples/fft_physical_x/fft_physical_x.f90 b/examples/fft_physical_x/fft_physical_x.f90
index 07daf7c9..9a999e95 100644
--- a/examples/fft_physical_x/fft_physical_x.f90
+++ b/examples/fft_physical_x/fft_physical_x.f90
@@ -1,206 +1,207 @@
 program fft_physical_x
 
-   use decomp_2d
-   use decomp_2d_fft
-   use MPI
-#if defined(_GPU)
-   use cudafor
-   use cufft
-   use openacc
+  use decomp_2d
+  use decomp_2d_fft
+  use MPI
+#if defined(_GPU) 
+  use cudafor
+  use cufft
+  use openacc 
 #endif
 
-   implicit none
-
-   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
-   integer :: nx, ny, nz
-   integer :: p_row = 0, p_col = 0
-   integer :: resize_domain
-   integer :: nranks_tot
-
-   integer, parameter :: ntest = 10  ! repeat test this times
-
-   complex(mytype), allocatable, dimension(:, :, :) :: in, out
-   real(mytype), allocatable, dimension(:, :, :) :: in_r
-
-   integer, dimension(3) :: fft_start, fft_end, fft_size
-
-   real(mytype) :: dr, di, error, err_all, n1, flops
-   integer :: ierror, i, j, k, m
-   real(mytype) :: t1, t2, t3, t4
-
-   call MPI_INIT(ierror)
-   ! To resize the domain we need to know global number of ranks
-   ! This operation is also done as part of decomp_2d_init
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
-   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+  implicit none
+  
+  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
+  integer :: nx, ny, nz
+  integer :: p_row=0, p_col=0
+  integer :: resize_domain
+  integer :: nranks_tot
+  
+  integer, parameter :: ntest = 10  ! repeat test this times
+  
+  complex(mytype), allocatable, dimension(:,:,:) :: in, out
+  real(mytype), allocatable, dimension(:,:,:) :: in_r
+  
+  integer, dimension(3) :: fft_start, fft_end, fft_size
+  
+  real(mytype) :: dr,di, error, err_all, n1,flops
+  integer :: ierror, i,j,k,m
+  real(mytype) :: t1, t2, t3 ,t4
+  
+  call MPI_INIT(ierror)
+  ! To resize the domain we need to know global number of ranks
+  ! This operation is also done as part of decomp_2d_init
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+  resize_domain = int(nranks_tot/4)+1
+  nx = nx_base*resize_domain
+  ny = ny_base*resize_domain
+  nz = nz_base*resize_domain
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Test the c2c interface
+  ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-   call decomp_2d_fft_init(PHYSICAL_IN_X) ! force the default x pencil
-
-   !  input is X-pencil data
-   ! output is Z-pencil data
-   allocate (in(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (out(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   ! initilise input
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-            di = dr
-            in(i, j, k) = cmplx(dr, di, mytype)
-         end do
-      end do
-   end do
-
-   t2 = 0._mytype
-   t4 = 0._mytype
-   do m = 1, ntest
-
-      ! forward FFT
-      t1 = MPI_WTIME()
-      call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
-      t2 = t2 + MPI_WTIME() - t1
-
-      ! inverse FFT
-      t3 = MPI_WTIME()
-      call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
-      t4 = t4 + MPI_WTIME() - t3
-
-      ! normalisation - note 2DECOMP&FFT doesn't normalise
-      !$acc kernels
-      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
-      !$acc end kernels
-
-   end do
+  call decomp_2d_fft_init(PHYSICAL_IN_X) ! force the default x pencil
+
+  !  input is X-pencil data
+  ! output is Z-pencil data
+  allocate (in(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
+  allocate (out(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
+  ! initilise input
+  do k=xstart(3),xend(3)
+     do j=xstart(2),xend(2)
+        do i=xstart(1),xend(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           di = dr
+           in(i,j,k) = cmplx(dr,di,mytype)
+        end do
+     end do
+  end do
+  
+  t2 = 0._mytype
+  t4 = 0._mytype
+  do m=1,ntest
+     
+     ! forward FFT
+     t1 = MPI_WTIME()
+     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+     t2 = t2 + MPI_WTIME() - t1
+
+     ! inverse FFT
+     t3 = MPI_WTIME()
+     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+     t4 = t4 + MPI_WTIME() - t3
+  
+     ! normalisation - note 2DECOMP&FFT doesn't normalise
+     !$acc kernels
+     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
+     !$acc end kernels
+
+  end do
 #if defined(_GPU)
-   ierror = cudaDeviceSynchronize()
+  ierror = cudaDeviceSynchronize()
 #endif
+  
+  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t1 = t1 / real(nproc,mytype)
+  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t3 = t3 / real(nproc,mytype)
+  
+  ! checking accuracy
+  error = 0._mytype
+  do k=xstart(3),xend(3)
+     do j=xstart(2),xend(2)
+        do i=xstart(1),xend(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           di = dr
+           dr = dr - real(in(i,j,k),mytype)
+           di = di - aimag(in(i,j,k))
+           error = error + sqrt(dr*dr + di*di)
+        end do
+     end do
+  end do
+  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
+  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
+
+  if (nrank==0) then
+     write(*,*) '===== c2c interface ====='
+     write(*,*) 'error / mesh point: ', err_all 
+     write(*,*) 'time (sec): ', t1,t3
+     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
+     n1 = n1 ** (1._mytype/3._mytype)
+     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
+     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+     flops = flops * 3._mytype * n1**2  
+     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
+     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
+  end if
+  
+  deallocate(in,out)
+  call decomp_2d_fft_finalize
 
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
-
-   ! checking accuracy
-   error = 0._mytype
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-            di = dr
-            dr = dr - real(in(i, j, k), mytype)
-            di = di - aimag(in(i, j, k))
-            error = error + sqrt(dr*dr + di*di)
-         end do
-      end do
-   end do
-   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
-
-   if (nrank == 0) then
-      write (*, *) '===== c2c interface ====='
-      write (*, *) 'error / mesh point: ', err_all
-      write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
-      n1 = n1**(1._mytype/3._mytype)
-      ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
-      ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops*3._mytype*n1**2
-      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
-   end if
-
-   deallocate (in, out)
-   call decomp_2d_fft_finalize
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Test the r2c/c2r interface
+  ! Test the r2c/c2r interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   call decomp_2d_fft_init(PHYSICAL_IN_X)
-
-   allocate (in_r(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   call decomp_2d_fft_get_size(fft_start, fft_end, fft_size)
-   allocate (out(fft_start(1):fft_end(1), &
-                 fft_start(2):fft_end(2), &
-                 fft_start(3):fft_end(3)))
-
-   ! initilise input
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-         end do
-      end do
-   end do
-
-   t2 = 0._mytype
-   t4 = 0._mytype
-   do m = 1, ntest
-
-      ! 3D r2c FFT
-      t1 = MPI_WTIME()
-      call decomp_2d_fft_3d(in_r, out)
-      t2 = t2 + MPI_WTIME() - t1
-
-      ! 3D inverse FFT
-      t3 = MPI_WTIME()
-      call decomp_2d_fft_3d(out, in_r)
-      t4 = t4 + MPI_WTIME() - t3
-
-      ! normalisation - note 2DECOMP&FFT doesn't normalise
-      !$acc kernels
-      in_r = in_r/(real(nx, mytype)*real(ny, mytype)*real(nz, mytype))
-      !$acc end kernels
-
-   end do
+  call decomp_2d_fft_init(PHYSICAL_IN_X)
+  
+  allocate (in_r(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
+  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
+  allocate (out(fft_start(1):fft_end(1), &
+                fft_start(2):fft_end(2), &
+                fft_start(3):fft_end(3)))
+  
+  ! initilise input
+  do k=xstart(3),xend(3)
+     do j=xstart(2),xend(2)
+        do i=xstart(1),xend(1)
+           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+        end do
+     end do
+  end do
+  
+  t2 = 0._mytype
+  t4 = 0._mytype
+  do m=1,ntest
+  
+     ! 3D r2c FFT
+     t1 = MPI_WTIME()
+     call decomp_2d_fft_3d(in_r, out)
+     t2 = t2 + MPI_WTIME() - t1
+  
+     ! 3D inverse FFT
+     t3 = MPI_WTIME()
+     call decomp_2d_fft_3d(out, in_r)
+     t4 = t4 + MPI_WTIME() - t3
+  
+     ! normalisation - note 2DECOMP&FFT doesn't normalise
+     !$acc kernels
+     in_r = in_r / (real(nx,mytype)*real(ny,mytype)*real(nz,mytype))
+     !$acc end kernels
+
+  end do
 #if defined(_GPU)
-   ierror = cudaDeviceSynchronize()
-#endif
-
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
-
-   ! checking accuracy
-   error = 0._mytype
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-            error = error + abs(in_r(i, j, k) - dr)
-         end do
-      end do
-   end do
-   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
-
-   if (nrank == 0) then
-      write (*, *) '===== r2c/c2r interface ====='
-      write (*, *) 'error / mesh point: ', err_all
-      write (*, *) 'time (sec): ', t1, t3
-   end if
-
-   deallocate (in_r, out)
-   call decomp_2d_fft_finalize
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
+  ierror = cudaDeviceSynchronize()
+#endif 
+
+  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t1 = t1 / real(nproc,mytype)
+  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t3 = t3 / real(nproc,mytype)
+  
+  ! checking accuracy
+  error = 0._mytype
+  do k=xstart(3),xend(3)
+     do j=xstart(2),xend(2)
+        do i=xstart(1),xend(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           error = error + abs(in_r(i,j,k)-dr)
+        end do
+     end do
+  end do
+  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
+  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
+  
+  if (nrank==0) then
+     write(*,*) '===== r2c/c2r interface ====='
+     write(*,*) 'error / mesh point: ', err_all
+     write(*,*) 'time (sec): ', t1,t3
+  end if
+  
+  deallocate(in_r,out)
+  call decomp_2d_fft_finalize
+  call decomp_2d_finalize
+  call MPI_FINALIZE(ierror)
 
 end program fft_physical_x
 
diff --git a/examples/fft_physical_z/fft_physical_z.f90 b/examples/fft_physical_z/fft_physical_z.f90
index fd8e45e5..5610b8e9 100644
--- a/examples/fft_physical_z/fft_physical_z.f90
+++ b/examples/fft_physical_z/fft_physical_z.f90
@@ -1,206 +1,208 @@
 program fft_physical_z
 
-   use decomp_2d
-   use decomp_2d_fft
-   use MPI
-#if defined(_GPU)
-   use cudafor
-   use cufft
-   use openacc
+  use decomp_2d
+  use decomp_2d_fft
+  use MPI
+#if defined(_GPU) 
+  use cudafor
+  use cufft
+  use openacc 
 #endif
 
-   implicit none
-
-   !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
-   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
-   integer :: nx, ny, nz
-   integer :: p_row = 0, p_col = 0
-   integer :: resize_domain
-   integer :: nranks_tot
-
-   integer, parameter :: ntest = 10  ! repeat test this times
-
-   complex(mytype), allocatable, dimension(:, :, :) :: in, out
-   real(mytype), allocatable, dimension(:, :, :) :: in_r
-
-   integer, dimension(3) :: fft_start, fft_end, fft_size
-
-   real(mytype) :: dr, di, error, err_all, n1, flops
-   integer :: ierror, i, j, k, m
-   real(mytype) :: t1, t2, t3, t4
-
-   call MPI_INIT(ierror)
-   ! To resize the domain we need to know global number of ranks
-   ! This operation is also done as part of decomp_2d_init
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
-   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+  implicit none
+  
+  !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
+  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
+  integer :: nx, ny, nz
+  integer :: p_row=0, p_col=0
+  integer :: resize_domain
+  integer :: nranks_tot
+  
+  integer, parameter :: ntest = 10  ! repeat test this times
+  
+  complex(mytype), allocatable, dimension(:,:,:) :: in, out
+  real(mytype), allocatable, dimension(:,:,:) :: in_r
+
+  integer, dimension(3) :: fft_start, fft_end, fft_size
+  
+  real(mytype) :: dr,di, error, err_all, n1,flops
+  integer :: ierror, i,j,k,m
+  real(mytype) :: t1, t2, t3 ,t4
+  
+  call MPI_INIT(ierror)
+  ! To resize the domain we need to know global number of ranks
+  ! This operation is also done as part of decomp_2d_init
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+  resize_domain = int(nranks_tot/4)+1
+  nx = nx_base*resize_domain
+  ny = ny_base*resize_domain
+  nz = nz_base*resize_domain
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Test the c2c interface
+  ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
-   !  input is Z-pencil data
-   ! output is X-pencil data
-   allocate (in(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   allocate (out(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   ! initilise input
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-            di = dr
-            in(i, j, k) = cmplx(dr, di, mytype)
-         end do
-      end do
-   end do
-
-   t2 = 0._mytype
-   t4 = 0._mytype
-   do m = 1, ntest
-
-      ! forward FFT
-      t1 = MPI_WTIME()
-      call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
-      t2 = t2 + MPI_WTIME() - t1
-
-      ! inverse FFT
-      t3 = MPI_WTIME()
-      call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
-      t4 = t4 + MPI_WTIME() - t3
-
-      ! normalisation - note 2DECOMP&FFT doesn't normalise
-      !$acc kernels
-      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
-      !$acc end kernels
-
-   end do
+  call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
+  !  input is Z-pencil data
+  ! output is X-pencil data
+  allocate (in(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
+  allocate (out(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
+  ! initilise input
+  do k=zstart(3),zend(3)
+     do j=zstart(2),zend(2)
+        do i=zstart(1),zend(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           di = dr
+           in(i,j,k) = cmplx(dr,di,mytype)
+        end do
+     end do
+  end do
+  
+  t2 = 0._mytype
+  t4 = 0._mytype
+  do m=1,ntest
+     
+     ! forward FFT
+     t1 = MPI_WTIME()
+     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+     t2 = t2 + MPI_WTIME() - t1
+
+     ! inverse FFT
+     t3 = MPI_WTIME()
+     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+     t4 = t4 + MPI_WTIME() - t3
+  
+     ! normalisation - note 2DECOMP&FFT doesn't normalise
+     !$acc kernels
+     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
+     !$acc end kernels
+
+  end do
 #if defined(_GPU)
-   ierror = cudaDeviceSynchronize()
+  ierror = cudaDeviceSynchronize()
 #endif
+  
+
+  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t1 = t1 / real(nproc,mytype)
+  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t3 = t3 / real(nproc,mytype)
+  
+  ! checking accuracy
+  error = 0._mytype
+  do k=zstart(3),zend(3)
+     do j=zstart(2),zend(2)
+        do i=zstart(1),zend(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           di = dr
+           dr = dr - real(in(i,j,k),mytype)
+           di = di - aimag(in(i,j,k))
+           error = error + sqrt(dr*dr + di*di)
+        end do
+     end do
+  end do
+  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
+  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
+
+  if (nrank==0) then
+     write(*,*) '===== c2c interface ====='
+     write(*,*) 'error / mesh point: ', err_all 
+     write(*,*) 'time (sec): ', t1,t3
+     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
+     n1 = n1 ** (1._mytype/3._mytype)
+     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
+     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+     flops = flops * 3._mytype * n1**2  
+     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
+     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
+  end if
+  
+  deallocate(in,out)
+  call decomp_2d_fft_finalize
 
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
-
-   ! checking accuracy
-   error = 0._mytype
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-            di = dr
-            dr = dr - real(in(i, j, k), mytype)
-            di = di - aimag(in(i, j, k))
-            error = error + sqrt(dr*dr + di*di)
-         end do
-      end do
-   end do
-   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
-
-   if (nrank == 0) then
-      write (*, *) '===== c2c interface ====='
-      write (*, *) 'error / mesh point: ', err_all
-      write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
-      n1 = n1**(1._mytype/3._mytype)
-      ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
-      ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops*3._mytype*n1**2
-      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
-   end if
-
-   deallocate (in, out)
-   call decomp_2d_fft_finalize
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Test the r2c/c2r interface
+  ! Test the r2c/c2r interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
-
-   allocate (in_r(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   call decomp_2d_fft_get_size(fft_start, fft_end, fft_size)
-   allocate (out(fft_start(1):fft_end(1), &
-                 fft_start(2):fft_end(2), &
-                 fft_start(3):fft_end(3)))
-
-   ! initilise input
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-         end do
-      end do
-   end do
-
-   t2 = 0._mytype
-   t4 = 0._mytype
-   do m = 1, ntest
-
-      ! 3D r2c FFT
-      t1 = MPI_WTIME()
-      call decomp_2d_fft_3d(in_r, out)
-      t2 = t2 + MPI_WTIME() - t1
-
-      ! 3D inverse FFT
-      t3 = MPI_WTIME()
-      call decomp_2d_fft_3d(out, in_r)
-      t4 = t4 + MPI_WTIME() - t3
-
-      !$acc kernels
-      in_r = in_r/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
-      !$acc end kernels
-
-   end do
+  call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
+  
+  allocate (in_r(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
+  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
+  allocate (out(fft_start(1):fft_end(1), &
+                fft_start(2):fft_end(2), &
+                fft_start(3):fft_end(3)))
+  
+  ! initilise input
+  do k=zstart(3),zend(3)
+     do j=zstart(2),zend(2)
+        do i=zstart(1),zend(1)
+           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+        end do
+     end do
+  end do
+  
+  t2 = 0._mytype
+  t4 = 0._mytype
+  do m=1,ntest
+     
+     ! 3D r2c FFT
+     t1 = MPI_WTIME()
+     call decomp_2d_fft_3d(in_r, out)
+     t2 = t2 + MPI_WTIME() - t1
+  
+     ! 3D inverse FFT
+     t3 = MPI_WTIME()
+     call decomp_2d_fft_3d(out, in_r)
+     t4 = t4 + MPI_WTIME() - t3
+  
+     !$acc kernels
+     in_r = in_r / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
+     !$acc end kernels     
+
+  end do
 #if defined(_GPU)
-   ierror = cudaDeviceSynchronize()
-#endif
-
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
-                      MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
-
-   ! checking accuracy
-   error = 0._mytype
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
-            error = error + abs(in_r(i, j, k) - dr)
-            !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
-         end do
-      end do
-   end do
+  ierror = cudaDeviceSynchronize()
+#endif 
+
+  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t1 = t1 / real(nproc,mytype)
+  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
+       MPI_COMM_WORLD,ierror)
+  t3 = t3 / real(nproc,mytype)
+  
+  ! checking accuracy
+  error = 0._mytype
+  do k=zstart(3),zend(3)
+     do j=zstart(2),zend(2)
+        do i=zstart(1),zend(1)
+           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
+                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
+           error = error + abs(in_r(i,j,k)-dr)
+           !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
+        end do
+     end do
+  end do
 !10 format('in_r final ', I2,1x,I2,1x,I2,1x,I2,1x,F12.6,1x,F12.6)
 
-   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
-
-   if (nrank == 0) then
-      write (*, *) '===== r2c/c2r interface ====='
-      write (*, *) 'error / mesh point: ', err_all
-      write (*, *) 'time (sec): ', t1, t3
-   end if
-
-   deallocate (in_r, out)
-   call decomp_2d_fft_finalize
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
+  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
+  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
+  
+  if (nrank==0) then
+     write(*,*) '===== r2c/c2r interface ====='
+     write(*,*) 'error / mesh point: ', err_all
+     write(*,*) 'time (sec): ', t1,t3
+  end if
+  
+  deallocate(in_r,out)
+  call decomp_2d_fft_finalize
+  call decomp_2d_finalize
+  call MPI_FINALIZE(ierror)
 
 end program fft_physical_z
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 2222d3c4..9f52a2bd 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -6,495 +6,495 @@
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 program halo_test
 
-   use mpi
+  use mpi
 
-   use decomp_2d
+  use decomp_2d
 
-   implicit none
+  implicit none
 
-   integer, parameter :: nx = 171, ny = 132, nz = 113
-   integer :: p_row = 0, p_col = 0
+  integer, parameter :: nx=171, ny=132, nz=113
+  integer :: p_row=0, p_col=0
 
-   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
-   real(mytype), allocatable, dimension(:, :, :) :: v1, v2, v3
-   real(mytype), allocatable, dimension(:, :, :) :: w1, w2, w3
-   real(mytype), allocatable, dimension(:, :, :) :: wk2, wk3
-   real(mytype), allocatable, dimension(:, :, :) :: uh, vh, wh
-   real(mytype), allocatable, dimension(:, :, :) :: div1, div2, div3, div4
+  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
+  real(mytype), allocatable, dimension(:,:,:) :: v1, v2, v3
+  real(mytype), allocatable, dimension(:,:,:) :: w1, w2, w3
+  real(mytype), allocatable, dimension(:,:,:) :: wk2, wk3
+  real(mytype), allocatable, dimension(:,:,:) :: uh, vh, wh
+  real(mytype), allocatable, dimension(:,:,:) :: div1, div2, div3, div4
 
-   integer :: i, j, k, ierror, n
+  integer :: i,j,k, ierror, n
 
-   integer, allocatable, dimension(:) :: seed
+  integer, allocatable, dimension(:) :: seed
 
-   real(mytype) :: err
-   integer :: xlast, ylast, zlast
+  real(mytype) :: err
+  integer :: xlast, ylast, zlast
 
-   integer :: nx_expected, ny_expected, nz_expected
+  integer :: nx_expected, ny_expected, nz_expected
+  
+  logical :: passing, all_pass
 
-   logical :: passing, all_pass
+  call MPI_INIT(ierror)
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
 
-   call MPI_INIT(ierror)
-   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+  xlast = xsize(1) - 1
+  if (xend(2) == ny) then
+     ylast = xsize(2) - 1
+  else
+     ylast = xsize(2)
+  end if
+  if (xend(3) == nz) then
+     zlast = xsize(3) - 1
+  else
+     zlast = xsize(3)
+  end if
 
-   xlast = xsize(1) - 1
-   if (xend(2) == ny) then
-      ylast = xsize(2) - 1
-   else
-      ylast = xsize(2)
-   end if
-   if (xend(3) == nz) then
-      zlast = xsize(3) - 1
-   else
-      zlast = xsize(3)
-   end if
+  call initialise()
+  call test_div_transpose()
+  call test_div_haloX()
+  call test_div_haloY()
+  call test_div_haloZ()
 
-   call initialise()
-   call test_div_transpose()
-   call test_div_haloX()
-   call test_div_haloY()
-   call test_div_haloZ()
+  if (nrank == 0) then
+     write(*,*) '-----------------------------------------------'
+     write(*,*) "All pass: ", all_pass
+     write(*,*) '==============================================='
+  end if
 
-   if (nrank == 0) then
-      write (*, *) '-----------------------------------------------'
-      write (*, *) "All pass: ", all_pass
-      write (*, *) '==============================================='
-   end if
+  deallocate(u1,v1,w1,u2,v2,w2,u3,v3,w3)
+  deallocate(div1,div2,div3,div4)
 
-   deallocate (u1, v1, w1, u2, v2, w2, u3, v3, w3)
-   deallocate (div1, div2, div3, div4)
+  call decomp_2d_finalize 
 
-   call decomp_2d_finalize
+  if (.not. all_pass) call decomp_2d_abort(1, "Error in halo_test")
 
-   if (.not. all_pass) call decomp_2d_abort(1, "Error in halo_test")
-
-   call MPI_FINALIZE(ierror)
+  call MPI_FINALIZE(ierror)
 
 contains
 
-   subroutine initialise()
+  subroutine initialise()
 
-      ! initialise u,v,w with random numbers in X-pencil
+    ! initialise u,v,w with random numbers in X-pencil
 #ifdef HALO_GLOBAL
-      allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      allocate (v1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      allocate (w1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+    allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+    allocate(v1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+    allocate(w1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
 #else
-      allocate (u1(xsize(1), xsize(2), xsize(3)))
-      allocate (v1(xsize(1), xsize(2), xsize(3)))
-      allocate (w1(xsize(1), xsize(2), xsize(3)))
+    allocate(u1(xsize(1), xsize(2), xsize(3)))
+    allocate(v1(xsize(1), xsize(2), xsize(3)))
+    allocate(w1(xsize(1), xsize(2), xsize(3)))
 #endif
 
-      call random_seed(size=n)
-      allocate (seed(n))
-      seed = nrank + 1
-      call random_seed(put=seed)
-      call random_number(u1)
-      call random_number(v1)
-      call random_number(w1)
+    call random_seed(size = n)
+    allocate(seed(n))
+    seed = nrank+1
+    call random_seed(put=seed)
+    call random_number(u1)
+    call random_number(v1)
+    call random_number(w1)
 
-      all_pass = .true.
+    all_pass = .true.
 
-   end subroutine initialise
+  end subroutine initialise
 
-   !=====================================================================
-   ! Calculate divergence using global transposition
-   !=====================================================================
-   subroutine test_div_transpose()
+  !=====================================================================
+  ! Calculate divergence using global transposition
+  !=====================================================================
+  subroutine test_div_transpose()
 
-      integer :: i1, in ! I loop start/end
-      integer :: j1, jn ! J loop start/end
-      integer :: k1, kn ! K loop start/end
+    integer :: i1, in ! I loop start/end
+    integer :: j1, jn ! J loop start/end
+    integer :: k1, kn ! K loop start/end
 
-      ! du/dx calculated on X-pencil
+    ! du/dx calculated on X-pencil
 #ifdef HALO_GLOBAL
-      allocate (div1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      k1 = xstart(3); kn = xend(3)
-      j1 = xstart(2); jn = xend(2)
+    allocate(div1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+    k1 = xstart(3); kn = xend(3)
+    j1 = xstart(2); jn = xend(2)
 #else
-      allocate (div1(xsize(1), xsize(2), xsize(3)))
-      k1 = 1; kn = xsize(3)
-      j1 = 1; jn = xsize(2)
+    allocate(div1(xsize(1), xsize(2), xsize(3)))
+    k1 = 1; kn = xsize(3)
+    j1 = 1; jn = xsize(2)
 #endif
-      i1 = 2; in = xsize(1) - 1
-
-      div1 = 0.0_mytype
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
-               div1(i, j, k) = u1(i + 1, j, k) - u1(i - 1, j, k)
-            end do
-         end do
-      end do
-
-      ! dv/dy calculated on Y-pencil
+    i1 = 2; in = xsize(1) - 1
+
+    div1 = 0.0_mytype
+    do k=k1,kn
+       do j=j1,jn
+          do i=i1,in
+             div1(i,j,k) = u1(i+1,j,k)-u1(i-1,j,k)
+          end do
+       end do
+    end do
+
+    ! dv/dy calculated on Y-pencil
 #ifdef HALO_GLOBAL
-      allocate (v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      k1 = ystart(3); kn = yend(3)
-      i1 = ystart(1); in = yend(1)
+    allocate(v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+    allocate(wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+    k1 = ystart(3); kn = yend(3)
+    i1 = ystart(1); in = yend(1)
 #else
-      allocate (v2(ysize(1), ysize(2), ysize(3)))
-      allocate (wk2(ysize(1), ysize(2), ysize(3)))
-      k1 = 1; kn = ysize(3)
-      i1 = 1; in = ysize(1)
+    allocate(v2(ysize(1), ysize(2), ysize(3)))
+    allocate(wk2(ysize(1), ysize(2), ysize(3)))
+    k1 = 1; kn = ysize(3)
+    i1 = 1; in = ysize(1)
 #endif
-      j1 = 2; jn = ysize(2) - 1
+    j1 = 2; jn = ysize(2) - 1
 
-      call transpose_x_to_y(v1, v2)
-      call transpose_x_to_y(div1, wk2)
+    call transpose_x_to_y(v1,v2)
+    call transpose_x_to_y(div1,wk2)
 
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
-               wk2(i, j, k) = wk2(i, j, k) + v2(i, j + 1, k) - v2(i, j - 1, k)
-            end do
-         end do
-      end do
+    do k=k1,kn
+       do j=j1,jn
+          do i=i1,in
+             wk2(i,j,k) = wk2(i,j,k) + v2(i,j+1,k)-v2(i,j-1,k)
+          end do
+       end do
+    end do
 
-      ! dw/dz calculated on Z-pencil
+    ! dw/dz calculated on Z-pencil
 #ifdef HALO_GLOBAL
-      allocate (w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-      allocate (wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-      j1 = zstart(2); jn = zend(2)
-      i1 = zstart(1); in = zend(1)
+    allocate(w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+    allocate(w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+    allocate(wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+    j1 = zstart(2); jn = zend(2)
+    i1 = zstart(1); in = zend(1)
 #else
-      allocate (w2(ysize(1), ysize(2), ysize(3)))
-      allocate (w3(zsize(1), zsize(2), zsize(3)))
-      allocate (wk3(zsize(1), zsize(2), zsize(3)))
-      j1 = 1; jn = zsize(2)
-      i1 = 1; in = zsize(1)
+    allocate(w2(ysize(1), ysize(2), ysize(3)))
+    allocate(w3(zsize(1), zsize(2), zsize(3)))
+    allocate(wk3(zsize(1), zsize(2), zsize(3)))
+    j1 = 1; jn = zsize(2)
+    i1 = 1; in = zsize(1)
 #endif
-      k1 = 2; kn = zsize(3) - 1
-
-      call transpose_x_to_y(w1, w2)
-      call transpose_y_to_z(w2, w3)
-      call transpose_y_to_z(wk2, wk3)
-
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
-               wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
-            end do
-         end do
-      end do
-
-      ! result in X-pencil
-      call transpose_z_to_y(wk3, wk2)
-      call transpose_y_to_x(wk2, div1)
-
-      if (nrank == 0) then
-         write (*, *) 'Calculated via global transposition'
+    k1 = 2; kn = zsize(3) - 1
+    
+    call transpose_x_to_y(w1,w2)
+    call transpose_y_to_z(w2,w3)
+    call transpose_y_to_z(wk2,wk3)
+
+    do k=k1,kn
+       do j=j1,jn
+          do i=i1,in
+             wk3(i,j,k) = wk3(i,j,k) + w3(i,j,k+1)-w3(i,j,k-1)
+          end do
+       end do
+    end do
+
+    ! result in X-pencil
+    call transpose_z_to_y(wk3,wk2)
+    call transpose_y_to_x(wk2,div1)
+
+    if (nrank==0) then
+       write(*,*) 'Calculated via global transposition'
 #ifdef DEBUG
-         write (*, *) (div1(i, i, i), i=2, 13)
+       write(*,*) (div1(i,i,i), i=2,13)
 #endif
-      end if
-
-      deallocate (v2, w2, w3, wk2, wk3)
-
-   end subroutine test_div_transpose
-
-   !=====================================================================
-   ! Calculate divergence using halo-cell exchange (data in X-pencil)
-   !=====================================================================
-   subroutine test_div_haloX()
-
-      integer :: i1, in ! I loop start/end
-      integer :: j1, jn ! J loop start/end
-      integer :: k1, kn ! K loop start/end
-
-      ! Expected sizes
-      nx_expected = nx
-      ny_expected = xsize(2) + 2
-      nz_expected = xsize(3) + 2
-
+    end if
+
+    deallocate(v2,w2,w3,wk2,wk3)
+
+  end subroutine test_div_transpose
+
+  !=====================================================================
+  ! Calculate divergence using halo-cell exchange (data in X-pencil)
+  !=====================================================================
+  subroutine test_div_haloX()
+
+    integer :: i1, in ! I loop start/end
+    integer :: j1, jn ! J loop start/end
+    integer :: k1, kn ! K loop start/end
+    
+    ! Expected sizes
+    nx_expected = nx
+    ny_expected = xsize(2) + 2
+    nz_expected = xsize(3) + 2
+    
 #ifdef HALO_GLOBAL
-      allocate (div2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      call update_halo(v1, vh, 1, opt_global=.true., opt_pencil=1)
-      call update_halo(w1, wh, 1, opt_global=.true., opt_pencil=1)
-
-      k1 = xstart(3); kn = xend(3)
-      j1 = xstart(2); jn = xend(2)
+    allocate(div2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+    call update_halo(v1,vh,1,opt_global=.true.,opt_pencil=1)
+    call update_halo(w1,wh,1,opt_global=.true.,opt_pencil=1)
+    
+    k1 = xstart(3); kn = xend(3)
+    j1 = xstart(2); jn = xend(2)
 #else
-      allocate (div2(xsize(1), xsize(2), xsize(3)))
-      call update_halo(v1, vh, 1, opt_pencil=1)
-      call update_halo(w1, wh, 1, opt_pencil=1)
+    allocate(div2(xsize(1), xsize(2), xsize(3)))
+    call update_halo(v1,vh,1,opt_pencil=1)
+    call update_halo(w1,wh,1,opt_pencil=1)
 
-      k1 = 1; kn = xsize(3)
-      j1 = 1; jn = xsize(2)
+    k1 = 1; kn = xsize(3)
+    j1 = 1; jn = xsize(2)
 #endif
-      i1 = 2; in = xsize(1) - 1
-
-      call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "X:v")
-      call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
-
-      div2 = 0.0_mytype
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
-               div2(i, j, k) = (u1(i + 1, j, k) - u1(i - 1, j, k)) &
-                               + (vh(i, j + 1, k) - vh(i, j - 1, k)) &
-                               + (wh(i, j, k + 1) - wh(i, j, k - 1))
-            end do
-         end do
-      end do
-
-      ! Compute error
-      call check_err(div2, "X")
-
-      deallocate (vh, wh)
-
-   end subroutine test_div_haloX
-
-   !=====================================================================
-   ! Calculate divergence using halo-cell exchange (data in Y-pencil)
-   !=====================================================================
-   subroutine test_div_haloY()
-
-      integer :: i1, in ! I loop start/end
-      integer :: j1, jn ! J loop start/end
-      integer :: k1, kn ! K loop start/end
-
-      ! Expected sizes
-      nx_expected = ysize(1) + 2
-      ny_expected = ny
-      nz_expected = ysize(3) + 2
-
+    i1 = 2; in = xsize(1) - 1
+
+    call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "X:v")
+    call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
+    
+    div2 = 0.0_mytype
+    do k=k1,kn
+       do j=j1,jn
+          do i=i1,in
+             div2(i,j,k) = (u1(i+1,j,k)-u1(i-1,j,k)) &
+                  + (vh(i,j+1,k)-vh(i,j-1,k)) &
+                  + (wh(i,j,k+1)-wh(i,j,k-1))
+          end do
+       end do
+    end do
+
+    ! Compute error
+    call check_err(div2, "X")
+           
+    deallocate(vh,wh)
+
+  end subroutine test_div_haloX
+
+  !=====================================================================
+  ! Calculate divergence using halo-cell exchange (data in Y-pencil)
+  !=====================================================================
+  subroutine test_div_haloY()
+
+    integer :: i1, in ! I loop start/end
+    integer :: j1, jn ! J loop start/end
+    integer :: k1, kn ! K loop start/end
+
+    ! Expected sizes
+    nx_expected = ysize(1) + 2
+    ny_expected = ny
+    nz_expected = ysize(3) + 2
+  
 #ifdef HALO_GLOBAL
-      allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (div3(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+    allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+    allocate(v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+    allocate(w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+    allocate(div3(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+    allocate(wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
 #else
-      allocate (u2(ysize(1), ysize(2), ysize(3)))
-      allocate (v2(ysize(1), ysize(2), ysize(3)))
-      allocate (w2(ysize(1), ysize(2), ysize(3)))
-      allocate (div3(xsize(1), xsize(2), xsize(3)))
-      allocate (wk2(ysize(1), ysize(2), ysize(3)))
+    allocate(u2(ysize(1), ysize(2), ysize(3)))
+    allocate(v2(ysize(1), ysize(2), ysize(3)))
+    allocate(w2(ysize(1), ysize(2), ysize(3)))
+    allocate(div3(xsize(1), xsize(2), xsize(3)))
+    allocate(wk2(ysize(1), ysize(2), ysize(3)))
 #endif
-      call transpose_x_to_y(u1, u2)
-      call transpose_x_to_y(v1, v2)
-      call transpose_x_to_y(w1, w2)
+    call transpose_x_to_y(u1,u2)
+    call transpose_x_to_y(v1,v2)
+    call transpose_x_to_y(w1,w2)
 
-      ! du/dx
+    ! du/dx
 #ifdef HALO_GLOBAL
-      call update_halo(u2, uh, 1, opt_global=.true., opt_pencil=2)
-      call update_halo(w2, wh, 1, opt_global=.true., opt_pencil=2)
-      k1 = ystart(3); kn = yend(3)
-      i1 = ystart(1); in = yend(1)
+    call update_halo(u2,uh,1,opt_global=.true.,opt_pencil=2)
+    call update_halo(w2,wh,1,opt_global=.true.,opt_pencil=2)
+    k1 = ystart(3); kn = yend(3)
+    i1 = ystart(1); in = yend(1)
 #else
-      call update_halo(u2, uh, 1, opt_pencil=2)
-      call update_halo(w2, wh, 1, opt_pencil=2)
-      k1 = 1; kn = ysize(3)
-      i1 = 1; in = ysize(1)
+    call update_halo(u2,uh,1,opt_pencil=2)
+    call update_halo(w2,wh,1,opt_pencil=2)
+    k1 = 1; kn = ysize(3)
+    i1 = 1; in = ysize(1)
 #endif
-      j1 = 2; jn = ysize(2) - 1
+    j1 = 2; jn = ysize(2) - 1
 
-      call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Y:u")
-      call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "Y:w")
+    call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Y:u")
+    call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "Y:w")
 
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
-               wk2(i, j, k) = (uh(i + 1, j, k) - uh(i - 1, j, k)) &
-                              + (v2(i, j + 1, k) - v2(i, j - 1, k)) &
-                              + (wh(i, j, k + 1) - wh(i, j, k - 1))
-            end do
-         end do
-      end do
+    do k=k1,kn
+       do j=j1,jn
+          do i=i1,in
+             wk2(i,j,k) = (uh(i+1,j,k)-uh(i-1,j,k)) &
+                  + (v2(i,j+1,k)-v2(i,j-1,k)) &
+                  + (wh(i,j,k+1)-wh(i,j,k-1))
+          end do
+       end do
+    end do
 
-      call transpose_y_to_x(wk2, div3)
+    call transpose_y_to_x(wk2,div3)
 
-      ! Compute error
-      call check_err(div3, "Y")
+    ! Compute error
+    call check_err(div3, "Y")
 
-      deallocate (uh, wh, wk2)
+    deallocate(uh,wh,wk2)
 
-   end subroutine test_div_haloY
+  end subroutine test_div_haloY
 
-   !=====================================================================
-   ! Calculate divergence using halo-cell exchange (data in Z-pencil)
-   !=====================================================================
-   subroutine test_div_haloZ()
+  !=====================================================================
+  ! Calculate divergence using halo-cell exchange (data in Z-pencil)
+  !=====================================================================
+  subroutine test_div_haloZ()
 
-      ! Expected sizes
-      nx_expected = zsize(1) + 2
-      ny_expected = zsize(2) + 2
-      nz_expected = nz
+  ! Expected sizes
+  nx_expected = zsize(1) + 2
+  ny_expected = zsize(2) + 2
+  nz_expected = nz
 
 #ifdef HALO_GLOBAL
-      allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-      allocate (v3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-      allocate (w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+  allocate(v3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+  allocate(w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
 #else
-      allocate (u3(zsize(1), zsize(2), zsize(3)))
-      allocate (v3(zsize(1), zsize(2), zsize(3)))
-      allocate (w3(zsize(1), zsize(2), zsize(3)))
+  allocate(u3(zsize(1), zsize(2), zsize(3)))
+  allocate(v3(zsize(1), zsize(2), zsize(3)))
+  allocate(w3(zsize(1), zsize(2), zsize(3)))
 #endif
-      call transpose_y_to_z(u2, u3)
-      call transpose_y_to_z(v2, v3)
-      call transpose_y_to_z(w2, w3)
+  call transpose_y_to_z(u2,u3)
+  call transpose_y_to_z(v2,v3)
+  call transpose_y_to_z(w2,w3)
 
 #ifdef HALO_GLOBAL
-      allocate (div4(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+  allocate(div4(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  allocate(wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  allocate(wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
 #else
-      allocate (div4(xsize(1), xsize(2), xsize(3)))
-      allocate (wk2(ysize(1), ysize(2), ysize(3)))
-      allocate (wk3(zsize(1), zsize(2), zsize(3)))
+  allocate(div4(xsize(1), xsize(2), xsize(3)))
+  allocate(wk2(ysize(1), ysize(2), ysize(3)))
+  allocate(wk3(zsize(1), zsize(2), zsize(3)))
 #endif
 
-      ! du/dx
+  ! du/dx
 #ifdef HALO_GLOBAL
-      call update_halo(u3, uh, 1, opt_global=.true., opt_pencil=3)
+  call update_halo(u3,uh,1,opt_global=.true.,opt_pencil=3)
 #else
-      call update_halo(u3, uh, 1, opt_pencil=3)
+  call update_halo(u3,uh,1,opt_pencil=3)
 #endif
 
-      call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
-
+  call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
+  
 #ifdef HALO_GLOBAL
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
+  do j=zstart(2),zend(2)
+     do i=zstart(1),zend(1)
 #else
-            do j = 1, zsize(2)
-               do i = 1, zsize(1)
+  do j=1,zsize(2)
+     do i=1,zsize(1)
 #endif
-                  do k = 2, zsize(3) - 1
-                     wk3(i, j, k) = uh(i + 1, j, k) - uh(i - 1, j, k)
-                  end do
-               end do
-            end do
+        do k=2,zsize(3)-1
+           wk3(i,j,k) = uh(i+1,j,k)-uh(i-1,j,k)
+        end do
+     end do
+  end do
 
-            ! dv/dy
+  ! dv/dy
 #ifdef HALO_GLOBAL
-            call update_halo(v3, vh, 1, opt_global=.true., opt_pencil=3)
+  call update_halo(v3,vh,1,opt_global=.true.,opt_pencil=3)
 #else
-            call update_halo(v3, vh, 1, opt_pencil=3)
+  call update_halo(v3,vh,1,opt_pencil=3)
 #endif
 
-            call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
-
+  call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
+  
 #ifdef HALO_GLOBAL
-            do j = zstart(2), zend(2)
-               do i = zstart(1), zend(1)
+  do j=zstart(2),zend(2)
+     do i=zstart(1),zend(1)
 #else
-                  do j = 1, zsize(2)
-                     do i = 1, zsize(1)
+  do j=1,zsize(2)
+     do i=1,zsize(1)
 #endif
-                        do k = 2, zsize(3) - 1
-                           wk3(i, j, k) = wk3(i, j, k) + vh(i, j + 1, k) - vh(i, j - 1, k)
-                        end do
-                     end do
-                  end do
+        do k=2,zsize(3)-1
+           wk3(i,j,k) = wk3(i,j,k) + vh(i,j+1,k)-vh(i,j-1,k)
+        end do
+     end do
+  end do
 
-                  ! dw/dz
+  ! dw/dz
 #ifdef HALO_GLOBAL
-                  do j = zstart(2), zend(2)
-                     do i = zstart(1), zend(1)
+  do j=zstart(2),zend(2)
+     do i=zstart(1),zend(1)
 #else
-                        do j = 1, zsize(2)
-                           do i = 1, zsize(1)
+  do j=1,zsize(2)
+     do i=1,zsize(1)
 #endif
-                              do k = 2, zsize(3) - 1
-                                 wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
-                              end do
-                           end do
-                        end do
-
-                        call transpose_z_to_y(wk3, wk2)
-                        call transpose_y_to_x(wk2, div4)
-
-                        ! Compute error
-                        call check_err(div4, "Z")
-
-                        deallocate (uh, vh, wk2, wk3)
-                        end subroutine test_div_haloZ
-
-                        subroutine check_err(divh, pencil)
-
-                           real(mytype), dimension(:, :, :), intent(in) :: divh
-                           character(len=*), intent(in) :: pencil
-
-                           real(mytype), dimension(:, :, :), allocatable :: tmp
-
-                           real(mytype) :: divmag
-
-                           ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
-                           !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
-                           !      causes a SEGFAULT. Explicitly computing the difference in a temporary
-                           !      array seems to be OK.
-                           allocate (tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
-                           tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - div1(2:xlast, 2:ylast, 2:zlast)
-                           err = mag(tmp(2:xlast, 2:ylast, 2:zlast))
-                           deallocate (tmp)
-                           divmag = mag(div1(2:xlast, 2:ylast, 2:zlast))
-                           if (err < epsilon(divmag)*divmag) then
-                              passing = .true.
-                           else
-                              passing = .false.
-                           end if
-                           all_pass = all_pass .and. passing
-
-                           if (nrank == 0) then
-                              write (*, *) '-----------------------------------------------'
-                              write (*, *) 'Calculated via halo exchange (data in '//pencil//'-pencil)'
+        do k=2,zsize(3)-1
+           wk3(i,j,k) = wk3(i,j,k) + w3(i,j,k+1)-w3(i,j,k-1)
+        end do
+     end do
+  end do
+
+  call transpose_z_to_y(wk3,wk2)
+  call transpose_y_to_x(wk2,div4)
+
+  ! Compute error
+  call check_err(div4, "Z")
+
+  deallocate(uh,vh,wk2,wk3)
+  end subroutine test_div_haloZ
+
+  subroutine check_err(divh, pencil)
+
+    real(mytype), dimension(:,:,:), intent(in) :: divh
+    character(len=*), intent(in) :: pencil
+
+    real(mytype), dimension(:,:,:), allocatable :: tmp
+
+    real(mytype) :: divmag
+
+    ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
+    !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
+    !      causes a SEGFAULT. Explicitly computing the difference in a temporary
+    !      array seems to be OK.
+    allocate(tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
+    tmp(2:xlast,2:ylast,2:zlast) = divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast)
+    err = mag(tmp(2:xlast,2:ylast,2:zlast))
+    deallocate(tmp)
+    divmag = mag(div1(2:xlast,2:ylast,2:zlast))
+    if (err < epsilon(divmag) * divmag) then
+       passing = .true.
+    else
+       passing = .false.
+    end if
+    all_pass = all_pass .and. passing
+
+    if (nrank==0) then
+       write(*,*) '-----------------------------------------------'
+       write(*,*) 'Calculated via halo exchange (data in '//pencil//'-pencil)'
 #ifdef DEBUG
-                              write (*, *) (divh(i, i, i), i=2, 13)
+       write(*,*) (divh(i,i,i), i=2,13)
 #endif
-                              write (*, *) 'Error: ', err, '; Relative: ', err/divmag
-                              write (*, *) 'Pass: ', passing
-                           end if
-
-                        end subroutine check_err
-
-                        real(mytype) function mag(a)
-
-                           real(mytype), dimension(:, :, :), intent(in) :: a
+       write(*,*) 'Error: ', err, '; Relative: ', err / divmag
+       write(*,*) 'Pass: ', passing
+    end if
 
-                           real(mytype) :: lmag, gmag
+  end subroutine check_err
 
-                           lmag = sum(a(:, :, :)**2)
-                           call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-                           if (ierror /= 0) then
-                              call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-                                                   "halo_test::mag::MPI_Allreduce")
-                           end if
+  real(mytype) function mag(a)
 
-                           mag = sqrt(gmag/(nx - 2)/(ny - 2)/(nz - 2))
+    real(mytype), dimension(:,:,:), intent(in) :: a
+    
+    real(mytype) :: lmag, gmag
 
-                        end function mag
+    lmag = sum(a(:,:,:)**2)
+    call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+    if (ierror /= 0) then
+       call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+            "halo_test::mag::MPI_Allreduce")
+    endif
 
-                        subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
+    mag = sqrt(gmag / (nx - 2) / (ny - 2) / (nz - 2))
 
-                           real(mytype), dimension(:, :, :), intent(in) :: arrh
-                           integer, intent(in) :: nx_expected, ny_expected, nz_expected
-                           character(len=*), intent(in) :: tag
+  end function mag
 
-                           integer :: nx, ny, nz
+  subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
 
-                           character(len=128) :: rank_lbl
+    real(mytype), dimension(:,:,:), intent(in) :: arrh
+    integer, intent(in) :: nx_expected, ny_expected, nz_expected
+    character(len=*), intent(in) :: tag
 
-                           nx = size(arrh, 1)
-                           ny = size(arrh, 2)
-                           nz = size(arrh, 3)
+    integer :: nx, ny, nz
 
-                           write (rank_lbl, "(A,I0,A)") "Rank", nrank, ":"
+    character(len=128) :: rank_lbl
 
-                           if ((nx /= nx_expected) .or. &
-                               (ny /= ny_expected) .or. &
-                               (nz /= nz_expected)) then
-                              write (*, *) trim(rank_lbl), " ", tag, ":ERROR: halo size"
-                              write (*, *) trim(rank_lbl), " ", "+ Expected: ", nx_expected, " ", ny_expected, " ", nz_expected, " "
-                              write (*, *) trim(rank_lbl), " ", "+ Got:      ", nx, " ", ny, " ", nz, " "
+    nx = size(arrh, 1)
+    ny = size(arrh, 2)
+    nz = size(arrh, 3)
 
-                              all_pass = .false.
-                           else
-                              write (*, *) trim(rank_lbl), " ", tag, ":PASS"
-                           end if
+    write(rank_lbl, "(A,I0,A)") "Rank", nrank, ":"
 
-                        end subroutine test_halo_size
+    if ((nx /= nx_expected) .or. &
+         (ny /= ny_expected) .or. &
+         (nz /= nz_expected)) then
+       write(*,*) trim(rank_lbl), " ", tag, ":ERROR: halo size"
+       write(*,*) trim(rank_lbl), " ", "+ Expected: ", nx_expected, " ", ny_expected, " ", nz_expected, " "
+       write(*,*) trim(rank_lbl), " ", "+ Got:      ", nx, " ", ny, " ", nz, " "
 
-                        end program halo_test
+       all_pass = .false.
+    else
+       write(*,*) trim(rank_lbl), " ", tag, ":PASS"
+    end if
+    
+  end subroutine test_halo_size
+  
+end program halo_test
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 25953391..5f9c5dd1 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -4,71 +4,71 @@
 
 program init_test
 
-   use MPI
-   use decomp_2d
+  use MPI
+  use decomp_2d
+  
+  implicit none
 
-   implicit none
+  integer, parameter :: nx = 5
+  integer, parameter :: ny = 6
+  integer, parameter :: nz = 7
+  integer, parameter :: nexpect = nx * ny * nz
 
-   integer, parameter :: nx = 5
-   integer, parameter :: ny = 6
-   integer, parameter :: nz = 7
-   integer, parameter :: nexpect = nx*ny*nz
+  integer :: p_row, p_col
 
-   integer :: p_row, p_col
+  integer :: ierr
 
-   integer :: ierr
-
-   call MPI_Init(ierr)
-
-   p_row = 0; p_col = 0
-   call run(p_row, p_col)
-
-   call MPI_Finalize(ierr)
+  call MPI_Init(ierr)
+  
+  p_row = 0; p_col = 0
+  call run(p_row, p_col)
 
+  call MPI_Finalize(ierr)
+  
 contains
 
-   subroutine run(p_row, p_col)
-
-      integer, intent(inout) :: p_row, p_col
-
-      call decomp_2d_init(nx, ny, nz, p_row, p_col)
-
-      call check_axis("X")
-      call check_axis("Y")
-      call check_axis("Z")
-
-      call decomp_2d_finalize()
-
-   end subroutine run
-
-   subroutine check_axis(axis)
-
-      character(len=*), intent(in) :: axis
-
-      integer :: suml
-      integer :: sumg
-      integer, dimension(3) :: sizes
-
-      if (axis == "X") then
-         sizes = xsize
-      else if (axis == "Y") then
-         sizes = ysize
-      else if (axis == "Z") then
-         sizes = zsize
-      else
-         sizes = 0
-         print *, "ERROR: unknown axis requested!"
-         stop 1
-      end if
-
-      suml = product(sizes)
-      call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierr)
-
-      if (sumg /= nexpect) then
-         print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
-         stop 1
-      end if
-
-   end subroutine check_axis
-
+  subroutine run(p_row, p_col)
+
+    integer, intent(inout) :: p_row, p_col
+
+    call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+    call check_axis("X")
+    call check_axis("Y")
+    call check_axis("Z")
+    
+    call decomp_2d_finalize()
+    
+  end subroutine run
+
+  subroutine check_axis(axis)
+
+    character(len=*), intent(in) :: axis
+
+    integer :: suml
+    integer :: sumg
+    integer, dimension(3) :: sizes
+
+    if (axis == "X") then
+       sizes = xsize
+    else if (axis == "Y") then
+       sizes = ysize
+    else if (axis == "Z") then
+       sizes = zsize
+    else
+       sizes = 0
+       print *, "ERROR: unknown axis requested!"
+       stop 1
+    end if
+
+    suml = product(sizes)
+    call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierr)
+    
+    if (sumg /= nexpect) then
+       print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
+       stop 1
+    end if
+    
+  end subroutine check_axis
+  
 end program init_test
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index 4e3200ca..a8e05390 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -1,36 +1,36 @@
 program io_bench
 
-   use decomp_2d
-   use decomp_2d_io
-   use MPI
+  use decomp_2d
+  use decomp_2d_io
+  use MPI
 
-   implicit none
+  implicit none
+  
+  integer, parameter :: nx=100, ny=100, nz=100
+  integer :: p_row=0, p_col=0
 
-   integer, parameter :: nx = 100, ny = 100, nz = 100
-   integer :: p_row = 0, p_col = 0
+  real(mytype), allocatable, dimension(:,:,:) :: u1
+  
+  double precision :: t1, t2
+  integer :: ierror
 
-   real(mytype), allocatable, dimension(:, :, :) :: u1
+  call MPI_INIT(ierror)
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
 
-   double precision :: t1, t2
-   integer :: ierror
+  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  call random_number(u1)
 
-   call MPI_INIT(ierror)
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+  t1 = MPI_WTIME()
+  call decomp_2d_write_one(1,u1,'io.dat')
+  t2 = MPI_WTIME()
 
-   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   call random_number(u1)
+  if (nrank==0) write(*,*) 'I/O time: ', t2-t1
 
-   t1 = MPI_WTIME()
-   call decomp_2d_write_one(1, u1, 'io.dat')
-   t2 = MPI_WTIME()
-
-   if (nrank == 0) write (*, *) 'I/O time: ', t2 - t1
-
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
-   deallocate (u1)
+  call decomp_2d_finalize 
+  call MPI_FINALIZE(ierror)
+  deallocate(u1)
 
 end program io_bench
-
+  
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 7bebfa03..42a6d221 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -1,128 +1,128 @@
 program io_plane_test
 
-   use mpi
-
-   use decomp_2d
-   use decomp_2d_io
-
-   implicit none
-
-   integer, parameter :: nx = 17, ny = 13, nz = 11
-   integer :: p_row = 0, p_col = 0
-
-   real(mytype), dimension(nx, ny, nz) :: data1
-   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
-
-   real(mytype), allocatable, dimension(:, :, :) :: work
-
-   integer :: i, j, k, m, ierror, iol
-
-   call MPI_INIT(ierror)
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-   call decomp_2d_init(nx, ny, nz, p_row, p_col)
-
-   ! ***** global data *****
-   m = 1
-   do k = 1, nz
-      do j = 1, ny
-         do i = 1, nx
-            data1(i, j, k) = real(m, mytype)
-            m = m + 1
-         end do
-      end do
-   end do
-
-   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-   ! original X-pensil based data
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            u1(i, j, k) = data1(i, j, k)
-         end do
-      end do
-   end do
-   call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
-
-   ! Y-pencil data
-   call transpose_x_to_y(u1, u2)
-   call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(2, u2, 2, ny/2, '.', 'y_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(2, u2, 3, nz/2, '.', 'y_pencil-z_plane.dat', 'test')
-
-   ! Z-pencil data
-   call transpose_y_to_z(u2, u3)
-   call decomp_2d_write_plane(3, u3, 1, nx/2, '.', 'z_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(3, u3, 2, ny/2, '.', 'z_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(3, u3, 3, nz/2, '.', 'z_pencil-z_plane.dat', 'test')
-
-   ! Attemp to read the files
-   if (nrank == 0) then
-      inquire (iolength=iol) data1(1, 1, 1)
-
-      ! X-plane
-      allocate (work(1, ny, nz))
-      open (10, FILE='x_pencil-x_plane.dat', FORM='unformatted', &
-            ACCESS='DIRECT', RECL=iol)
-      m = 1
-      do k = 1, nz
-         do j = 1, ny
-            read (10, rec=m) work(1, j, k)
-            m = m + 1
-         end do
+  use mpi
+
+  use decomp_2d
+  use decomp_2d_io
+
+  implicit none
+
+  integer, parameter :: nx=17, ny=13, nz=11
+  integer :: p_row=0, p_col=0
+
+  real(mytype), dimension(nx,ny,nz) :: data1
+  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
+
+  real(mytype), allocatable, dimension(:,:,:) :: work
+  
+  integer :: i,j,k, m, ierror, iol
+
+  call MPI_INIT(ierror)
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+  
+  ! ***** global data *****
+  m = 1
+  do k=1,nz
+     do j=1,ny
+        do i=1,nx
+           data1(i,j,k) = real(m,mytype)
+           m = m+1
+        end do
+     end do
+  end do
+
+  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+  ! original X-pensil based data 
+  do k=xstart(3),xend(3)
+    do j=xstart(2),xend(2)
+      do i=xstart(1),xend(1)
+        u1(i,j,k) = data1(i,j,k)
       end do
+    end do
+  end do
+  call decomp_2d_write_plane(1,u1,1,nx/2,'.','x_pencil-x_plane.dat','test')
+  call decomp_2d_write_plane(1,u1,2,ny/2,'.','x_pencil-y_plane.dat','test')
+  call decomp_2d_write_plane(1,u1,3,nz/2,'.','x_pencil-z_plane.dat','test')
+
+  ! Y-pencil data
+  call transpose_x_to_y(u1,u2)
+  call decomp_2d_write_plane(2,u2,1,nx/2,'.','y_pencil-x_plane.dat','test')
+  call decomp_2d_write_plane(2,u2,2,ny/2,'.','y_pencil-y_plane.dat','test')
+  call decomp_2d_write_plane(2,u2,3,nz/2,'.','y_pencil-z_plane.dat','test')
+
+  ! Z-pencil data
+  call transpose_y_to_z(u2,u3)
+  call decomp_2d_write_plane(3,u3,1,nx/2,'.','z_pencil-x_plane.dat','test')
+  call decomp_2d_write_plane(3,u3,2,ny/2,'.','z_pencil-y_plane.dat','test')
+  call decomp_2d_write_plane(3,u3,3,nz/2,'.','z_pencil-z_plane.dat','test')
+
+  ! Attemp to read the files
+  if (nrank==0) then
+     inquire(iolength=iol) data1(1,1,1)
+
+     ! X-plane
+     allocate(work(1,ny,nz))
+     open(10, FILE='x_pencil-x_plane.dat', FORM='unformatted', &
+          ACCESS='DIRECT', RECL=iol)
+     m=1
+     do k=1,nz
+        do j=1,ny
+           read(10,rec=m) work(1,j,k)
+           m=m+1
+        end do
+     end do
 !     write(*,*) ' '
 !     write(*,'(15I5)') int(work)
-      close (10)
-      deallocate (work)
-
-      write (*, *) 'passed self test x-plane'
-
-      ! Y-plane
-      allocate (work(nx, 1, nz))
-      open (10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
-            ACCESS='DIRECT', RECL=iol)
-      m = 1
-      do k = 1, nz
-         do i = 1, nx
-            read (10, rec=m) work(i, 1, k)
-            m = m + 1
-         end do
-      end do
+     close(10)
+     deallocate(work)
+
+     write(*,*) 'passed self test x-plane'
+
+     ! Y-plane
+     allocate(work(nx,1,nz))
+     open(10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
+          ACCESS='DIRECT', RECL=iol)
+     m=1
+     do k=1,nz
+        do i=1,nx
+           read(10,rec=m) work(i,1,k)
+           m=m+1
+        end do
+     end do
 !     write(*,*) ' '
 !     write(*,'(15I5)') int(work)
-      close (10)
-      deallocate (work)
-
-      write (*, *) 'passed self test y-plane'
-
-      ! Z-plane
-      allocate (work(nx, ny, 1))
-      open (10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
-            ACCESS='DIRECT', RECL=iol)
-      m = 1
-      do j = 1, ny
-         do i = 1, nx
-            read (10, rec=m) work(i, j, 1)
-            m = m + 1
-         end do
-      end do
+     close(10)
+     deallocate(work)
+
+     write(*,*) 'passed self test y-plane'
+
+     ! Z-plane
+     allocate(work(nx,ny,1))
+     open(10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
+          ACCESS='DIRECT', RECL=iol)
+     m=1
+     do j=1,ny
+        do i=1,nx
+           read(10,rec=m) work(i,j,1)
+           m=m+1
+        end do
+     end do
 !     write(*,*) ' '
 !     write(*,'(15I5)') int(work)
-      close (10)
-      deallocate (work)
-
-      write (*, *) 'passed self test z-plane'
+     close(10)
+     deallocate(work)
 
-   end if
+     write(*,*) 'passed self test z-plane'
 
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
-   deallocate (u1, u2, u3)
+  end if
 
+  call decomp_2d_finalize 
+  call MPI_FINALIZE(ierror)
+  deallocate(u1,u2,u3)
+  
 end program io_plane_test
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 694f1f21..38c9cb85 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -1,86 +1,86 @@
 program io_read
 
-   use mpi
+  use mpi
 
-   use decomp_2d
-   use decomp_2d_io
+  use decomp_2d
+  use decomp_2d_io
 
-   implicit none
+  implicit none
 
-   integer, parameter :: nx = 17, ny = 13, nz = 11
-   ! use different number of processes
-   integer :: p_row = 0, p_col = 0
+  integer, parameter :: nx=17, ny=13, nz=11
+  ! use different number of processes
+  integer :: p_row=0, p_col=0
 
 #ifdef COMPLEX_TEST
-   complex(mytype), dimension(nx, ny, nz) :: data1
+  complex(mytype), dimension(nx,ny,nz) :: data1
 
-   complex(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
+  complex(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
 #else
-   real(mytype), dimension(nx, ny, nz) :: data1
+  real(mytype), dimension(nx,ny,nz) :: data1
 
-   real(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
+  real(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
 #endif
 
-   real(mytype), parameter :: eps = 1.0E-7_mytype
-
-   integer :: i, j, k, m, ierror
-
-   call MPI_INIT(ierror)
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-   call decomp_2d_init(nx, ny, nz, p_row, p_col)
-
-   ! ***** global data *****
-   m = 1
-   do k = 1, nz
-      do j = 1, ny
-         do i = 1, nx
+  real(mytype), parameter :: eps = 1.0E-7_mytype
+ 
+  integer :: i,j,k, m, ierror
+  
+  call MPI_INIT(ierror)
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+
+  ! ***** global data *****
+  m = 1
+  do k=1,nz
+     do j=1,ny
+        do i=1,nx
 #ifdef COMPLEX_TEST
-            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
+           data1(i,j,k) = cmplx(real(m,mytype), real(nx*ny*nz-m,mytype)) 
 #else
-            data1(i, j, k) = real(m, mytype)
+           data1(i,j,k) = real(m,mytype)
 #endif
-            m = m + 1
-         end do
-      end do
-   end do
-
-   allocate (u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-   ! read back to different arrays
-   call decomp_2d_read_one(1, u1b, '.', 'u1.dat', 'test', reduce_prec=.false.)
-   call decomp_2d_read_one(2, u2b, '.', 'u2.dat', 'test', reduce_prec=.false.)
-   call decomp_2d_read_one(3, u3b, '.', 'u3.dat', 'test', reduce_prec=.false.)
-
-   ! Check against the global data array
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            if (abs((data1(i, j, k) - u1b(i, j, k))) > eps) stop 4
-         end do
+           m = m+1
+        end do
+     end do
+  end do
+
+  allocate(u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  allocate(u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  allocate(u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+  ! read back to different arrays
+  call decomp_2d_read_one(1,u1b,'.','u1.dat','test',reduce_prec=.false.)
+  call decomp_2d_read_one(2,u2b,'.','u2.dat','test',reduce_prec=.false.)
+  call decomp_2d_read_one(3,u3b,'.','u3.dat','test',reduce_prec=.false.)
+
+  ! Check against the global data array
+  do k=xstart(3),xend(3)
+    do j=xstart(2),xend(2)
+      do i=xstart(1),xend(1)
+        if (abs((data1(i,j,k)-u1b(i,j,k))) > eps) stop 4
       end do
-   end do
+    end do
+  end do
 
-   do k = ystart(3), yend(3)
-      do j = ystart(2), yend(2)
-         do i = ystart(1), yend(1)
-            if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
-         end do
+  do k=ystart(3),yend(3)
+    do j=ystart(2),yend(2)
+      do i=ystart(1),yend(1)
+        if (abs((data1(i,j,k)-u2b(i,j,k))) > eps) stop 5
       end do
-   end do
-
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
-         end do
+    end do
+  end do
+  
+  do k=zstart(3),zend(3)
+    do j=zstart(2),zend(2)
+      do i=zstart(1),zend(1)
+        if (abs((data1(i,j,k)-u3b(i,j,k))) > eps) stop 6
       end do
-   end do
+    end do
+  end do
 
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
-   deallocate (u1b, u2b, u3b)
+  call decomp_2d_finalize 
+  call MPI_FINALIZE(ierror)
+  deallocate(u1b,u2b,u3b)
 
 end program io_read
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index eecdcfeb..b38aa2a0 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -1,133 +1,133 @@
 program io_test
 
-   use mpi
+  use mpi
 
-   use decomp_2d
-   use decomp_2d_io
+  use decomp_2d
+  use decomp_2d_io
 
-   implicit none
+  implicit none
 
-   integer, parameter :: nx = 17, ny = 13, nz = 11
-   integer :: p_row = 0, p_col = 0
+  integer, parameter :: nx=17, ny=13, nz=11
+  integer :: p_row=0, p_col=0
 
 #ifdef COMPLEX_TEST
-   complex(mytype), dimension(nx, ny, nz) :: data1
+  complex(mytype), dimension(nx,ny,nz) :: data1
 
-   complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
-   complex(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
+  complex(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
+  complex(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
 #else
-   real(mytype), dimension(nx, ny, nz) :: data1
+  real(mytype), dimension(nx,ny,nz) :: data1
 
-   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
-   real(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
+  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
+  real(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
 #endif
 
-   real(mytype), parameter :: eps = 1.0E-7_mytype
-
-   integer :: i, j, k, m, ierror
-
-   call MPI_INIT(ierror)
-   call decomp_2d_init(nx, ny, nz, p_row, p_col)
-
-   ! ***** global data *****
-   m = 1
-   do k = 1, nz
-      do j = 1, ny
-         do i = 1, nx
+  real(mytype), parameter :: eps = 1.0E-7_mytype
+ 
+  integer :: i,j,k, m, ierror
+  
+  call MPI_INIT(ierror)
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+
+  ! ***** global data *****
+  m = 1
+  do k=1,nz
+     do j=1,ny
+        do i=1,nx
 #ifdef COMPLEX_TEST
-            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
+           data1(i,j,k) = cmplx(real(m,mytype), real(nx*ny*nz-m,mytype)) 
 #else
-            data1(i, j, k) = real(m, mytype)
+           data1(i,j,k) = real(m,mytype)
 #endif
-            m = m + 1
-         end do
+           m = m+1
+        end do
+     end do
+  end do
+
+  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+  allocate(u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  allocate(u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  allocate(u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+  ! original x-pencil based data 
+  do k=xstart(3),xend(3)
+    do j=xstart(2),xend(2)
+      do i=xstart(1),xend(1)
+        u1(i,j,k) = data1(i,j,k)
       end do
-   end do
-
-   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-   allocate (u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-   ! original x-pencil based data
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            u1(i, j, k) = data1(i, j, k)
-         end do
+    end do
+  end do
+
+  ! transpose
+  call transpose_x_to_y(u1,u2)
+  call transpose_y_to_z(u2,u3)
+
+  ! write to disk
+  call decomp_2d_write_one(1,u1,'.','u1.dat',0,'test')
+  call decomp_2d_write_one(2,u2,'.','u2.dat',0,'test')
+  call decomp_2d_write_one(3,u3,'.','u3.dat',0,'test')
+
+  ! read back to different arrays
+  call decomp_2d_read_one(1,u1b,'.','u1.dat','test',reduce_prec=.false.)
+  call decomp_2d_read_one(2,u2b,'.','u2.dat','test',reduce_prec=.false.)
+  call decomp_2d_read_one(3,u3b,'.','u3.dat','test',reduce_prec=.false.)
+
+  ! compare  
+  do k=xstart(3),xend(3)
+    do j=xstart(2),xend(2)
+      do i=xstart(1),xend(1)
+        if (abs((u1(i,j,k)-u1b(i,j,k))) > eps) stop 1
       end do
-   end do
-
-   ! transpose
-   call transpose_x_to_y(u1, u2)
-   call transpose_y_to_z(u2, u3)
-
-   ! write to disk
-   call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
-   call decomp_2d_write_one(2, u2, '.', 'u2.dat', 0, 'test')
-   call decomp_2d_write_one(3, u3, '.', 'u3.dat', 0, 'test')
-
-   ! read back to different arrays
-   call decomp_2d_read_one(1, u1b, '.', 'u1.dat', 'test', reduce_prec=.false.)
-   call decomp_2d_read_one(2, u2b, '.', 'u2.dat', 'test', reduce_prec=.false.)
-   call decomp_2d_read_one(3, u3b, '.', 'u3.dat', 'test', reduce_prec=.false.)
-
-   ! compare
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) stop 1
-         end do
-      end do
-   end do
+    end do
+  end do
 
-   do k = ystart(3), yend(3)
-      do j = ystart(2), yend(2)
-         do i = ystart(1), yend(1)
-            if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 2
-         end do
+  do k=ystart(3),yend(3)
+    do j=ystart(2),yend(2)
+      do i=ystart(1),yend(1)
+        if (abs((u2(i,j,k)-u2b(i,j,k))) > eps) stop 2
       end do
-   end do
+    end do
+  end do
 
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            if (abs((u3(i, j, k) - u3b(i, j, k))) > eps) stop 3
-         end do
+  do k=zstart(3),zend(3)
+    do j=zstart(2),zend(2)
+      do i=zstart(1),zend(1)
+        if (abs((u3(i,j,k)-u3b(i,j,k))) > eps) stop 3
       end do
-   end do
-
-   ! Also check against the global data array
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            if (abs(data1(i, j, k) - u1b(i, j, k)) > eps) stop 4
-         end do
+    end do
+  end do
+
+  ! Also check against the global data array
+  do k=xstart(3),xend(3)
+    do j=xstart(2),xend(2)
+      do i=xstart(1),xend(1)
+        if (abs(data1(i,j,k)-u1b(i,j,k)) > eps) stop 4
       end do
-   end do
+    end do
+  end do
 
-   do k = ystart(3), yend(3)
-      do j = ystart(2), yend(2)
-         do i = ystart(1), yend(1)
-            if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
-         end do
+  do k=ystart(3),yend(3)
+    do j=ystart(2),yend(2)
+      do i=ystart(1),yend(1)
+        if (abs((data1(i,j,k)-u2b(i,j,k))) > eps) stop 5
       end do
-   end do
-
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
-         end do
+    end do
+  end do
+  
+  do k=zstart(3),zend(3)
+    do j=zstart(2),zend(2)
+      do i=zstart(1),zend(1)
+        if (abs((data1(i,j,k)-u3b(i,j,k))) > eps) stop 6
       end do
-   end do
+    end do
+  end do
 
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
-   deallocate (u1, u2, u3)
-   deallocate (u1b, u2b, u3b)
+  call decomp_2d_finalize 
+  call MPI_FINALIZE(ierror)
+  deallocate(u1,u2,u3)
+  deallocate(u1b,u2b,u3b)
 
 end program io_test
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 7f91e01d..2d25dcf9 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -3,283 +3,283 @@
 
 program io_var_test
 
-   use decomp_2d
-   use decomp_2d_io
-   use MPI
-
-   implicit none
-
-   integer, parameter :: nx = 17, ny = 13, nz = 11
-   integer :: p_row, p_col
-
-   real(mytype), parameter :: eps = 1.0E-7
-
-   ! for global data
-   real(mytype), dimension(nx, ny, nz) :: data1
-   real(mytype), allocatable, dimension(:, :, :) :: data1_large
-   complex(mytype), dimension(nx, ny, nz) :: cdata1
-
-   ! for distributed data
-   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
-   real(mytype), allocatable, dimension(:, :, :) :: u1l, u2l, u3l
-   complex(mytype), allocatable, dimension(:, :, :) :: cu1, cu2, cu3
-
-   ! another copy
-   real(mytype), allocatable, dimension(:, :, :) :: u1_b, u2_b, u3_b
-   real(mytype), allocatable, dimension(:, :, :) :: u1l_b, u2l_b, u3l_b
-   complex(mytype), allocatable, dimension(:, :, :) :: cu1_b, cu2_b, cu3_b
-
-   real(mytype), allocatable, dimension(:) :: tmp
-   complex(mytype), allocatable, dimension(:) :: ctmp
-   integer, allocatable, dimension(:) :: itmp
-
-   TYPE(DECOMP_INFO) :: large
-
-   integer :: i, j, k, m, ierror, fh
-   character(len=15) :: filename, arg
-   integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-
-   allocate (data1_large(nx*2, ny*2, nz*2))
-
-   call MPI_INIT(ierror)
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-
-   ! Defaults
-   p_row = 0
-   p_col = 0
-
-   ! Read commandline input
-   i = command_argument_count()
-   if (i /= 2) then
-      call MPI_ABORT(MPI_COMM_WORLD, 1, ierror)
-   else
-      call get_command_argument(1, arg)
-      read (arg, '(I10)') i
-      p_row = i
-      call get_command_argument(2, arg)
-      read (arg, '(I10)') i
-      p_col = i
-   end if
-
-   call decomp_2d_init(nx, ny, nz, p_row, p_col)
-
-   ! also create a data set over a large domain
-   call decomp_info_init(nx*2, ny*2, nz*2, large)
-
-   ! initialise global data
-   m = 1
-   do k = 1, nz
-      do j = 1, ny
-         do i = 1, nx
-            data1(i, j, k) = real(m, mytype)
-            cdata1(i, j, k) = cmplx(real(m, mytype), real(m, mytype), kind=mytype)
-            m = m + 1
-         end do
-      end do
-   end do
-
-   m = 1
-   do k = 1, nz*2
-      do j = 1, ny*2
-         do i = 1, nx*2
-            data1_large(i, j, k) = real(m, mytype)
-            m = m + 1
-         end do
-      end do
-   end do
-
-   ! allocate memory
-   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   allocate (u1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-   allocate (cu1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (cu2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (cu3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   allocate (cu1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (cu2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (cu3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-   allocate (u1l(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
-                 large%xst(3):large%xen(3)))
-   allocate (u2l(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
-                 large%yst(3):large%yen(3)))
-   allocate (u3l(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
-                 large%zst(3):large%zen(3)))
-   allocate (u1l_b(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
-                   large%xst(3):large%xen(3)))
-   allocate (u2l_b(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
-                   large%yst(3):large%yen(3)))
-   allocate (u3l_b(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
-                   large%zst(3):large%zen(3)))
-
-   ! distribute the data
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            u1(i, j, k) = data1(i, j, k)
-            cu1(i, j, k) = cdata1(i, j, k)
-         end do
-      end do
-   end do
-   do k = large%xst(3), large%xen(3)
-      do j = large%xst(2), large%xen(2)
-         do i = large%xst(1), large%xen(1)
-            u1l(i, j, k) = data1_large(i, j, k)
-         end do
-      end do
-   end do
-
-   ! transpose
-   call transpose_x_to_y(u1, u2)
-   call transpose_y_to_z(u2, u3)
-   call transpose_x_to_y(u1l, u2l, large)
-   call transpose_y_to_z(u2l, u3l, large)
-   call transpose_x_to_y(cu1, cu2)
-   call transpose_y_to_z(cu2, cu3)
-
-   ! open file for IO
-   write (filename, '(A,I3.3)') 'io_var_data.', nproc
-   call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
-                      MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
-                      fh, ierror)
-   filesize = 0_MPI_OFFSET_KIND
-   call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
-   disp = 0_MPI_OFFSET_KIND
-
-   ! test writing scalar data
-   allocate (tmp(2))
-   tmp(1) = 1._mytype
-   tmp(2) = 2._mytype
-   allocate (ctmp(3))
-   ctmp(1) = cmplx(1.0, 1.0, mytype)
-   ctmp(2) = cmplx(2.0, 2.0, mytype)
-   ctmp(3) = cmplx(3.0, 3.0, mytype)
-   allocate (itmp(3))
-   call decomp_2d_write_scalar(fh, disp, 2, tmp)
-   call decomp_2d_write_scalar(fh, disp, 3, ctmp)
-   call decomp_2d_write_scalar(fh, disp, 3, (/nx, ny, nz/))
-
-   ! test the IO routines by writing all data to disk
-   call decomp_2d_write_var(fh, disp, 1, u1)
-   call decomp_2d_write_var(fh, disp, 2, u2)
-   call decomp_2d_write_var(fh, disp, 3, u3)
-   call decomp_2d_write_var(fh, disp, 1, u1l, large)
-   call decomp_2d_write_var(fh, disp, 2, u2l, large)
-   call decomp_2d_write_var(fh, disp, 3, u3l, large)
-   call decomp_2d_write_var(fh, disp, 1, cu1)
-   call decomp_2d_write_var(fh, disp, 2, cu2)
-   call decomp_2d_write_var(fh, disp, 3, cu3)
-
-   call MPI_FILE_CLOSE(fh, ierror)
-
-   if (nrank == 0) write (*, *) 'disp=', disp
-
-   ! read data back in from file
-   call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
-                      MPI_MODE_RDONLY, MPI_INFO_NULL, &
-                      fh, ierror)
-   ! skip the scalars (2 real, 3 cmplx, 3 int)
+  use decomp_2d
+  use decomp_2d_io
+  use MPI
+
+  implicit none
+
+  integer, parameter :: nx=17, ny=13, nz=11
+  integer :: p_row, p_col
+
+  real(mytype), parameter :: eps = 1.0E-7
+
+  ! for global data
+  real(mytype), dimension(nx,ny,nz) :: data1
+  real(mytype), allocatable, dimension(:,:,:) :: data1_large
+  complex(mytype), dimension(nx,ny,nz) :: cdata1
+
+  ! for distributed data
+  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
+  real(mytype), allocatable, dimension(:,:,:) :: u1l, u2l, u3l
+  complex(mytype), allocatable, dimension(:,:,:) :: cu1, cu2, cu3
+
+  ! another copy
+  real(mytype), allocatable, dimension(:,:,:) :: u1_b, u2_b, u3_b
+  real(mytype), allocatable, dimension(:,:,:) :: u1l_b, u2l_b, u3l_b
+  complex(mytype), allocatable, dimension(:,:,:) :: cu1_b, cu2_b, cu3_b
+
+  real(mytype), allocatable, dimension(:) :: tmp
+  complex(mytype), allocatable, dimension(:) :: ctmp
+  integer, allocatable, dimension(:) :: itmp
+
+  TYPE(DECOMP_INFO) :: large
+
+  integer :: i,j,k, m, ierror, fh
+  character(len=15) :: filename, arg
+  integer (kind=MPI_OFFSET_KIND) :: filesize, disp
+
+  allocate(data1_large(nx*2,ny*2,nz*2))
+
+  call MPI_INIT(ierror)
+  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+
+  ! Defaults
+  p_row = 0
+  p_col = 0
+
+  ! Read commandline input
+  i = command_argument_count()
+  if (i/=2) then
+     call MPI_ABORT(MPI_COMM_WORLD, 1, ierror)
+  else
+     call get_command_argument(1, arg)
+     read(arg, '(I10)') i
+     p_row = i
+     call get_command_argument(2, arg)
+     read(arg, '(I10)') i
+     p_col = i
+  end if
+  
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+
+  ! also create a data set over a large domain
+  call decomp_info_init(nx*2, ny*2, nz*2, large)
+
+  ! initialise global data
+  m = 1
+  do k=1,nz
+     do j=1,ny
+        do i=1,nx
+           data1(i,j,k) = real(m,mytype)
+           cdata1(i,j,k) = cmplx(real(m,mytype),real(m,mytype), kind=mytype)
+           m = m+1
+        end do
+     end do
+  end do
+
+  m = 1
+  do k=1,nz*2
+     do j=1,ny*2
+        do i=1,nx*2
+           data1_large(i,j,k) = real(m,mytype)
+           m = m+1
+        end do
+     end do
+  end do
+
+  ! allocate memory
+  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+  allocate(u1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  allocate(u2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  allocate(u3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+  allocate(cu1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  allocate(cu2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  allocate(cu3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+  allocate(cu1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  allocate(cu2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  allocate(cu3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+  allocate(u1l(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
+       large%xst(3):large%xen(3)))
+  allocate(u2l(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
+       large%yst(3):large%yen(3)))
+  allocate(u3l(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
+       large%zst(3):large%zen(3)))
+  allocate(u1l_b(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
+       large%xst(3):large%xen(3)))
+  allocate(u2l_b(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
+       large%yst(3):large%yen(3)))
+  allocate(u3l_b(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
+       large%zst(3):large%zen(3)))
+
+  ! distribute the data
+  do k=xstart(3),xend(3)
+     do j=xstart(2),xend(2)
+        do i=xstart(1),xend(1)
+           u1(i,j,k) = data1(i,j,k)
+           cu1(i,j,k) = cdata1(i,j,k)
+        end do
+     end do
+  end do
+  do k=large%xst(3),large%xen(3)
+     do j=large%xst(2),large%xen(2)
+        do i=large%xst(1),large%xen(1)
+           u1l(i,j,k) = data1_large(i,j,k)
+        end do
+     end do
+  end do
+
+  ! transpose
+  call transpose_x_to_y(u1,u2)
+  call transpose_y_to_z(u2,u3)
+  call transpose_x_to_y(u1l,u2l,large)
+  call transpose_y_to_z(u2l,u3l,large)
+  call transpose_x_to_y(cu1,cu2)
+  call transpose_y_to_z(cu2,cu3)
+
+  ! open file for IO
+  write(filename,'(A,I3.3)') 'io_var_data.', nproc
+  call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
+       MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
+       fh, ierror)
+  filesize = 0_MPI_OFFSET_KIND
+  call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
+  disp = 0_MPI_OFFSET_KIND
+
+  ! test writing scalar data
+  allocate(tmp(2))
+  tmp(1) = 1._mytype
+  tmp(2) = 2._mytype
+  allocate(ctmp(3))
+  ctmp(1) = cmplx(1.0,1.0,mytype)
+  ctmp(2) = cmplx(2.0,2.0,mytype)
+  ctmp(3) = cmplx(3.0,3.0,mytype)
+  allocate(itmp(3))
+  call decomp_2d_write_scalar(fh,disp,2,tmp)
+  call decomp_2d_write_scalar(fh,disp,3,ctmp)
+  call decomp_2d_write_scalar(fh,disp,3,(/nx,ny,nz/))
+  
+  ! test the IO routines by writing all data to disk
+  call decomp_2d_write_var(fh,disp,1,u1)
+  call decomp_2d_write_var(fh,disp,2,u2)
+  call decomp_2d_write_var(fh,disp,3,u3)
+  call decomp_2d_write_var(fh,disp,1,u1l,large)
+  call decomp_2d_write_var(fh,disp,2,u2l,large)
+  call decomp_2d_write_var(fh,disp,3,u3l,large)
+  call decomp_2d_write_var(fh,disp,1,cu1)
+  call decomp_2d_write_var(fh,disp,2,cu2)
+  call decomp_2d_write_var(fh,disp,3,cu3)
+
+  call MPI_FILE_CLOSE(fh,ierror)
+
+  if (nrank==0) write(*,*) 'disp=',disp
+
+  ! read data back in from file
+  call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
+       MPI_MODE_RDONLY, MPI_INFO_NULL, &
+       fh, ierror)
+  ! skip the scalars (2 real, 3 cmplx, 3 int)
 #ifdef DOUBLE_PREC
-   ! if double precision: 2*8+3*8*2+3*4
-   disp = 76_MPI_OFFSET_KIND
+  ! if double precision: 2*8+3*8*2+3*4
+  disp = 76_MPI_OFFSET_KIND
 #else
-   ! if single precision: 2*4+3*4*2+3*4
-   disp = 44_MPI_OFFSET_KIND
+  ! if single precision: 2*4+3*4*2+3*4
+  disp = 44_MPI_OFFSET_KIND
 #endif
-
-   call decomp_2d_read_var(fh, disp, 1, u1_b)
-   call decomp_2d_read_var(fh, disp, 2, u2_b)
-   call decomp_2d_read_var(fh, disp, 3, u3_b)
-   call decomp_2d_read_var(fh, disp, 1, u1l_b, large)
-   call decomp_2d_read_var(fh, disp, 2, u2l_b, large)
-   call decomp_2d_read_var(fh, disp, 3, u3l_b, large)
-   call decomp_2d_read_var(fh, disp, 1, cu1_b)
-   call decomp_2d_read_var(fh, disp, 2, cu2_b)
-   call decomp_2d_read_var(fh, disp, 3, cu3_b)
-
-   disp = 0_MPI_OFFSET_KIND
-   call decomp_2d_read_scalar(fh, disp, 2, tmp)
-   call decomp_2d_read_scalar(fh, disp, 3, ctmp)
-   call decomp_2d_read_scalar(fh, disp, 3, itmp)
-   if (nrank == 0) then
-      write (*, '(2F8.3)') tmp
-      write (*, 20) ctmp
-20    format(3(:, '(', F5.2, ',', F5.2, ')'))
-      write (*, '(A,3I5)') 'nx,ny,nz', itmp
-   end if
-
-   call MPI_FILE_CLOSE(fh, ierror)
-   deallocate (tmp, ctmp, itmp)
-
-   ! validate the data
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            if (abs(u1(i, j, k) - u1_b(i, j, k)) > eps) stop 1
-            if (abs(cu1(i, j, k) - cu1_b(i, j, k)) > eps) stop 2
-         end do
-      end do
-   end do
-
-   do k = ystart(3), yend(3)
-      do j = ystart(2), yend(2)
-         do i = ystart(1), yend(1)
-            if (abs(u2(i, j, k) - u2_b(i, j, k)) > eps) stop 3
-            if (abs(cu2(i, j, k) - cu2_b(i, j, k)) > eps) stop 4
-         end do
-      end do
-   end do
-
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            if (abs(u3(i, j, k) - u3_b(i, j, k)) > eps) stop 5
-            if (abs(cu3(i, j, k) - cu3_b(i, j, k)) > eps) stop 6
-         end do
-      end do
-   end do
-
-   do k = large%xst(3), large%xen(3)
-      do j = large%xst(2), large%xen(2)
-         do i = large%xst(1), large%xen(1)
-            if (abs(u1l(i, j, k) - u1l_b(i, j, k)) > eps) stop 7
-         end do
-      end do
-   end do
-
-   do k = large%yst(3), large%yen(3)
-      do j = large%yst(2), large%yen(2)
-         do i = large%yst(1), large%yen(1)
-            if (abs(u2l(i, j, k) - u2l_b(i, j, k)) > eps) stop 8
-         end do
-      end do
-   end do
-
-   do k = large%zst(3), large%zen(3)
-      do j = large%zst(2), large%zen(2)
-         do i = large%zst(1), large%zen(1)
-            if (abs(u3l(i, j, k) - u3l_b(i, j, k)) > eps) stop 9
-         end do
-      end do
-   end do
-
-   if (nrank == 0) write (*, *) 'passed self test'
-
-   ! clean up
-   call decomp_info_finalize(large)
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
-   deallocate (u1, u2, u3)
-   deallocate (u1l, u2l, u3l)
-   deallocate (cu1, cu2, cu3)
-   deallocate (u1_b, u2_b, u3_b)
-   deallocate (u1l_b, u2l_b, u3l_b)
-   deallocate (cu1_b, cu2_b, cu3_b)
-   deallocate (data1_large)
-
+  
+  call decomp_2d_read_var(fh,disp,1,u1_b)
+  call decomp_2d_read_var(fh,disp,2,u2_b)
+  call decomp_2d_read_var(fh,disp,3,u3_b)
+  call decomp_2d_read_var(fh,disp,1,u1l_b,large)
+  call decomp_2d_read_var(fh,disp,2,u2l_b,large)
+  call decomp_2d_read_var(fh,disp,3,u3l_b,large)
+  call decomp_2d_read_var(fh,disp,1,cu1_b)
+  call decomp_2d_read_var(fh,disp,2,cu2_b)
+  call decomp_2d_read_var(fh,disp,3,cu3_b)
+
+  disp = 0_MPI_OFFSET_KIND
+  call decomp_2d_read_scalar(fh,disp,2,tmp)
+  call decomp_2d_read_scalar(fh,disp,3,ctmp)
+  call decomp_2d_read_scalar(fh,disp,3,itmp)
+  if (nrank==0) then
+     write(*,'(2F8.3)') tmp
+     write(*,20) ctmp
+20   format(3(:,'(',F5.2,',',F5.2,')'))
+     write(*,'(A,3I5)') 'nx,ny,nz', itmp
+  end if
+
+  call MPI_FILE_CLOSE(fh,ierror)
+  deallocate(tmp, ctmp, itmp)
+
+  ! validate the data
+  do k=xstart(3),xend(3)
+     do j=xstart(2),xend(2)
+        do i=xstart(1),xend(1)
+           if (abs(u1(i,j,k)-u1_b(i,j,k)) > eps) stop 1
+           if (abs(cu1(i,j,k)-cu1_b(i,j,k)) > eps) stop 2
+        end do
+     end do
+  end do
+
+  do k=ystart(3),yend(3)
+     do j=ystart(2),yend(2)
+        do i=ystart(1),yend(1)
+           if (abs(u2(i,j,k)-u2_b(i,j,k)) > eps) stop 3
+           if (abs(cu2(i,j,k)-cu2_b(i,j,k)) > eps) stop 4
+        end do
+     end do
+  end do
+
+  do k=zstart(3),zend(3)
+     do j=zstart(2),zend(2)
+        do i=zstart(1),zend(1)
+           if (abs(u3(i,j,k)-u3_b(i,j,k)) > eps) stop 5
+           if (abs(cu3(i,j,k)-cu3_b(i,j,k)) > eps) stop 6
+        end do
+     end do
+  end do
+
+  do k=large%xst(3),large%xen(3)
+     do j=large%xst(2),large%xen(2)
+        do i=large%xst(1),large%xen(1)
+           if (abs(u1l(i,j,k)-u1l_b(i,j,k)) > eps) stop 7
+        end do
+     end do
+  end do
+
+  do k=large%yst(3),large%yen(3)
+     do j=large%yst(2),large%yen(2)
+        do i=large%yst(1),large%yen(1)
+           if (abs(u2l(i,j,k)-u2l_b(i,j,k)) > eps) stop 8
+        end do
+     end do
+  end do
+
+  do k=large%zst(3),large%zen(3)
+     do j=large%zst(2),large%zen(2)
+        do i=large%zst(1),large%zen(1)
+           if (abs(u3l(i,j,k)-u3l_b(i,j,k)) > eps) stop 9
+        end do
+     end do
+  end do
+
+  if (nrank==0) write(*,*) 'passed self test'
+
+  ! clean up
+  call decomp_info_finalize(large)
+  call decomp_2d_finalize
+  call MPI_FINALIZE(ierror)
+  deallocate(u1,u2,u3)
+  deallocate(u1l,u2l,u3l)
+  deallocate(cu1,cu2,cu3)
+  deallocate(u1_b,u2_b,u3_b)
+  deallocate(u1l_b,u2l_b,u3l_b)
+  deallocate(cu1_b,cu2_b,cu3_b)
+  deallocate(data1_large)
+  
 end program io_var_test
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 1723d013..9c19841a 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -1,166 +1,166 @@
 program test2d
-
-   use mpi
-
-   use decomp_2d
-   ! use decomp_2d_io
-
-   implicit none
-
-   integer, parameter :: nx = 17, ny = 13, nz = 11
-   integer :: p_row = 0, p_col = 0
-
-   real(mytype), dimension(nx, ny, nz) :: data1
-
-   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
-
-   integer :: i, j, k, m, ierror
-   logical :: error_flag
-
-   ! Init
-   error_flag = .false.
-   call MPI_INIT(ierror)
-   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_INIT")
-   call decomp_2d_init(nx, ny, nz, p_row, p_col)
-
-   ! ***** global data *****
-   m = 1
-   do k = 1, nz
-      do j = 1, ny
-         do i = 1, nx
-            data1(i, j, k) = float(m)
-            m = m + 1
-         end do
-      end do
-   end do
+ 
+  use mpi
+
+  use decomp_2d
+  ! use decomp_2d_io
+
+  implicit none
+
+  integer, parameter :: nx=17, ny=13, nz=11
+  integer :: p_row=0, p_col=0
+
+  real(mytype), dimension(nx,ny,nz) :: data1
+  
+  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
+  
+  integer :: i,j,k, m, ierror
+  logical :: error_flag
+
+  ! Init
+  error_flag = .false.
+  call MPI_INIT(ierror)
+  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_INIT")
+  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+
+  ! ***** global data *****
+  m = 1
+  do k=1,nz
+     do j=1,ny
+        do i=1,nx
+           data1(i,j,k) = float(m)
+           m = m+1
+        end do
+     end do
+  end do
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Testing the swap routines
+  ! Testing the swap routines
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
-   !allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   !allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   !allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   call alloc_x(u1, opt_global=.true.)
-   call alloc_y(u2, opt_global=.true.)
-   call alloc_z(u3, opt_global=.true.)
-
-   ! original x-pensil based data
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            u1(i, j, k) = data1(i, j, k)
-         end do
+  
+  !allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+  !allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+  !allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+  call alloc_x(u1, opt_global=.true.)
+  call alloc_y(u2, opt_global=.true.)
+  call alloc_z(u3, opt_global=.true.)
+ 
+  ! original x-pensil based data 
+  do k=xstart(3),xend(3)
+    do j=xstart(2),xend(2)
+      do i=xstart(1),xend(1)
+        u1(i,j,k) = data1(i,j,k)
       end do
-   end do
+    end do
+  end do
 
 10 format(15I5)
 
 #ifdef DEBUG
-   if (nrank == 0) then
-      write (*, *) 'Numbers held on Rank 0'
-      write (*, *) ' '
-      write (*, *) 'X-pencil'
-      write (*, 10) int(u1)
-   end if
+  if (nrank==0) then 
+     write(*,*) 'Numbers held on Rank 0'
+     write(*,*) ' '
+     write(*,*) 'X-pencil'
+     write(*,10) int(u1)
+  end if
 #endif
 
-   ! call decomp_2d_write_one(1,u1,'u1.dat')
+  ! call decomp_2d_write_one(1,u1,'u1.dat')
 
   !!!!!!!!!!!!!!!!!!!!!!!
-   ! x-pensil ==> y-pensil
-   call transpose_x_to_y(u1, u2)
+  ! x-pensil ==> y-pensil
+  call transpose_x_to_y(u1,u2)
 
 #ifdef DEBUG
-   if (nrank == 0) then
-      write (*, *) ' '
-      write (*, *) 'Y-pencil'
-      write (*, 10) int(u2)
-   end if
+  if (nrank==0) then
+     write(*,*) ' '
+     write(*,*) 'Y-pencil'
+     write(*,10) int(u2)
+  end if
 #endif
 
-   ! call decomp_2d_write_one(2,u2,'u2.dat')
-   ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
+  ! call decomp_2d_write_one(2,u2,'u2.dat')
+  ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte 
 
-   ! also check the transposition this way
-   do k = ystart(3), yend(3)
-      do j = ystart(2), yend(2)
-         do i = ystart(1), yend(1)
-            if (abs(u2(i, j, k) - data1(i, j, k)) .gt. 0) error_flag = .true.
-         end do
+  ! also check the transposition this way
+  do k=ystart(3),yend(3)
+    do j=ystart(2),yend(2)
+      do i=ystart(1),yend(1)
+        if (abs(u2(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
       end do
-   end do
-   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-   if (error_flag) call decomp_2d_abort(1, "error swaping x->y")
+    end do
+  end do
+  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+  if (error_flag) call decomp_2d_abort(1, "error swaping x->y")
 
   !!!!!!!!!!!!!!!!!!!!!!!
-   ! y-pensil ==> z-pensil
-   call transpose_y_to_z(u2, u3)
+  ! y-pensil ==> z-pensil
+  call transpose_y_to_z(u2,u3)
 
 #ifdef DEBUG
-   if (nrank == 0) then
-      write (*, *) ' '
-      write (*, *) 'Z-pencil'
-      write (*, 10) int(u3)
-   end if
+  if (nrank==0) then
+     write(*,*) ' '
+     write(*,*) 'Z-pencil'
+     write(*,10) int(u3)
+  end if
 #endif
 
-   ! call decomp_2d_write_one(3,u3,'u3.dat')
-   ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
+  ! call decomp_2d_write_one(3,u3,'u3.dat')
+  ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            if (abs(u3(i, j, k) - data1(i, j, k)) .gt. 0) error_flag = .true.
-         end do
+  do k=zstart(3),zend(3)
+    do j=zstart(2),zend(2)
+      do i=zstart(1),zend(1)
+        if (abs(u3(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
       end do
-   end do
-   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-   if (error_flag) call decomp_2d_abort(2, "error swaping y->z")
+    end do
+  end do
+  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+  if (error_flag) call decomp_2d_abort(2, "error swaping y->z")
 
   !!!!!!!!!!!!!!!!!!!!!!!
-   ! z-pensil ==> y-pensil
-   call transpose_z_to_y(u3, u2)
-   ! call decomp_2d_write_one(2,u2,'u2b.dat')
-
-   do k = ystart(3), yend(3)
-      do j = ystart(2), yend(2)
-         do i = ystart(1), yend(1)
-            if (abs(u2(i, j, k) - data1(i, j, k)) .gt. 0) error_flag = .true.
-         end do
+  ! z-pensil ==> y-pensil
+  call transpose_z_to_y(u3,u2)
+  ! call decomp_2d_write_one(2,u2,'u2b.dat')
+
+  do k=ystart(3),yend(3)
+    do j=ystart(2),yend(2)
+      do i=ystart(1),yend(1)
+        if (abs(u2(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
       end do
-   end do
-   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-   if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
+    end do
+  end do
+  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+  if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
 
   !!!!!!!!!!!!!!!!!!!!!!!
-   ! y-pensil ==> x-pensil
-   call transpose_y_to_x(u2, u1)
-   ! call decomp_2d_write_one(1,u1,'u1b.dat')
-
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            if (abs(u1(i, j, k) - data1(i, j, k)) .gt. 0) error_flag = .true.
-         end do
+  ! y-pensil ==> x-pensil
+  call transpose_y_to_x(u2,u1)
+  ! call decomp_2d_write_one(1,u1,'u1b.dat')
+
+  do k=xstart(3),xend(3)
+    do j=xstart(2),xend(2)
+      do i=xstart(1),xend(1)
+        if (abs(u1(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
       end do
-   end do
-   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-   if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
-
-   if (nrank == 0) then
-      write (*, *) " "
-      write (*, *) "test2d completed"
-      write (*, *) " "
-   end if
-
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
-   deallocate (u1, u2, u3)
+    end do
+  end do
+  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+  if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
+
+  if (nrank == 0) then
+    write(*,*) " "
+    write(*,*) "test2d completed"
+    write(*,*) " "
+  endif
+
+  call decomp_2d_finalize 
+  call MPI_FINALIZE(ierror)
+  deallocate(u1,u2,u3)
 
 end program test2d
 
diff --git a/src/alloc.f90 b/src/alloc.f90
index 51c893ef..ffcbfa14 100644
--- a/src/alloc.f90
+++ b/src/alloc.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -16,281 +16,281 @@
   ! X-pencil real arrays
   subroutine alloc_x_real_short(var, opt_global)
 
-     implicit none
+    implicit none
 
-     real(mytype), allocatable, dimension(:, :, :) :: var
-     logical, intent(IN), optional :: opt_global
+    real(mytype), allocatable, dimension(:,:,:) :: var
+    logical, intent(IN), optional :: opt_global
 
-     call alloc_x(var, decomp_main, opt_global)
+    call alloc_x(var, decomp_main, opt_global)
 
   end subroutine alloc_x_real_short
 
   subroutine alloc_x_real(var, decomp, opt_global)
 
-     implicit none
-
-     real(mytype), allocatable, dimension(:, :, :) :: var
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
-     logical, intent(IN), optional :: opt_global
-
-     logical :: global
-     integer :: alloc_stat, errorcode
-
-     if (present(opt_global)) then
-        global = opt_global
-     else
-        global = .false.
-     end if
-
-     if (global) then
-        allocate (var(decomp%xst(1):decomp%xen(1), &
-                      decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
-                  stat=alloc_stat)
-     else
-        allocate (var(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)), &
-                  stat=alloc_stat)
-     end if
-
-     if (alloc_stat /= 0) then
-        errorcode = 8
-        call decomp_2d_abort(errorcode, &
-                             'Memory allocation failed when creating new arrays')
-     end if
-
-     return
+    implicit none
+
+    real(mytype), allocatable, dimension(:,:,:) :: var
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    logical, intent(IN), optional :: opt_global
+
+    logical :: global
+    integer :: alloc_stat, errorcode
+
+    if (present(opt_global)) then
+       global = opt_global
+    else
+       global = .false.
+    end if
+
+    if (global) then
+       allocate(var(decomp%xst(1):decomp%xen(1), &
+            decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
+            stat=alloc_stat)
+    else
+       allocate(var(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)), &
+            stat=alloc_stat)
+    end if
+    
+    if (alloc_stat /= 0) then
+       errorcode = 8
+       call decomp_2d_abort(errorcode, &
+            'Memory allocation failed when creating new arrays')
+    end if
+
+    return
   end subroutine alloc_x_real
 
   ! X-pencil complex arrays
   subroutine alloc_x_complex_short(var, opt_global)
 
-     implicit none
+    implicit none
 
-     complex(mytype), allocatable, dimension(:, :, :) :: var
-     logical, intent(IN), optional :: opt_global
+    complex(mytype), allocatable, dimension(:,:,:) :: var
+    logical, intent(IN), optional :: opt_global
 
-     call alloc_x(var, decomp_main, opt_global)
+    call alloc_x(var, decomp_main, opt_global)
 
   end subroutine alloc_x_complex_short
 
   subroutine alloc_x_complex(var, decomp, opt_global)
 
-     implicit none
-
-     complex(mytype), allocatable, dimension(:, :, :) :: var
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
-     logical, intent(IN), optional :: opt_global
-
-     logical :: global
-     integer :: alloc_stat, errorcode
-
-     if (present(opt_global)) then
-        global = opt_global
-     else
-        global = .false.
-     end if
-
-     if (global) then
-        allocate (var(decomp%xst(1):decomp%xen(1), &
-                      decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
-                  stat=alloc_stat)
-     else
-        allocate (var(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)), &
-                  stat=alloc_stat)
-     end if
-
-     if (alloc_stat /= 0) then
-        errorcode = 8
-        call decomp_2d_abort(errorcode, &
-                             'Memory allocation failed when creating new arrays')
-     end if
-
-     return
+    implicit none
+
+    complex(mytype), allocatable, dimension(:,:,:) :: var
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    logical, intent(IN), optional :: opt_global
+
+    logical :: global
+    integer :: alloc_stat, errorcode
+
+    if (present(opt_global)) then
+       global = opt_global
+    else
+       global = .false.
+    end if
+
+    if (global) then
+       allocate(var(decomp%xst(1):decomp%xen(1), &
+            decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
+            stat=alloc_stat)
+    else
+       allocate(var(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)), &
+            stat=alloc_stat)
+    end if
+    
+    if (alloc_stat /= 0) then
+       errorcode = 8
+       call decomp_2d_abort(errorcode, &
+            'Memory allocation failed when creating new arrays')
+    end if
+
+    return
   end subroutine alloc_x_complex
 
   ! Y-pencil real arrays
   subroutine alloc_y_real_short(var, opt_global)
 
-     implicit none
+    implicit none
 
-     real(mytype), allocatable, dimension(:, :, :) :: var
-     logical, intent(IN), optional :: opt_global
+    real(mytype), allocatable, dimension(:,:,:) :: var
+    logical, intent(IN), optional :: opt_global
 
-     call alloc_y(var, decomp_main, opt_global)
+    call alloc_y(var, decomp_main, opt_global)
 
   end subroutine alloc_y_real_short
 
   subroutine alloc_y_real(var, decomp, opt_global)
 
-     implicit none
-
-     real(mytype), allocatable, dimension(:, :, :) :: var
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
-     logical, intent(IN), optional :: opt_global
-
-     logical :: global
-     integer :: alloc_stat, errorcode
-
-     if (present(opt_global)) then
-        global = opt_global
-     else
-        global = .false.
-     end if
-
-     if (global) then
-        allocate (var(decomp%yst(1):decomp%yen(1), &
-                      decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
-                  stat=alloc_stat)
-     else
-        allocate (var(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)), &
-                  stat=alloc_stat)
-     end if
-
-     if (alloc_stat /= 0) then
-        errorcode = 8
-        call decomp_2d_abort(errorcode, &
-                             'Memory allocation failed when creating new arrays')
-     end if
-
-     return
+    implicit none
+
+    real(mytype), allocatable, dimension(:,:,:) :: var
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    logical, intent(IN), optional :: opt_global
+
+    logical :: global
+    integer :: alloc_stat, errorcode
+
+    if (present(opt_global)) then
+       global = opt_global
+    else
+       global = .false.
+    end if
+
+    if (global) then
+       allocate(var(decomp%yst(1):decomp%yen(1), &
+            decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
+            stat=alloc_stat)
+    else
+       allocate(var(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)), &
+            stat=alloc_stat)
+    end if
+    
+    if (alloc_stat /= 0) then
+       errorcode = 8
+       call decomp_2d_abort(errorcode, &
+            'Memory allocation failed when creating new arrays')
+    end if
+
+    return
   end subroutine alloc_y_real
 
   ! Y-pencil complex arrays
   subroutine alloc_y_complex_short(var, opt_global)
 
-     implicit none
+    implicit none
 
-     complex(mytype), allocatable, dimension(:, :, :) :: var
-     logical, intent(IN), optional :: opt_global
+    complex(mytype), allocatable, dimension(:,:,:) :: var
+    logical, intent(IN), optional :: opt_global
 
-     call alloc_y(var, decomp_main, opt_global)
+    call alloc_y(var, decomp_main, opt_global)
 
   end subroutine alloc_y_complex_short
 
   subroutine alloc_y_complex(var, decomp, opt_global)
 
-     implicit none
-
-     complex(mytype), allocatable, dimension(:, :, :) :: var
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
-     logical, intent(IN), optional :: opt_global
-
-     logical :: global
-     integer :: alloc_stat, errorcode
-
-     if (present(opt_global)) then
-        global = opt_global
-     else
-        global = .false.
-     end if
-
-     if (global) then
-        allocate (var(decomp%yst(1):decomp%yen(1), &
-                      decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
-                  stat=alloc_stat)
-     else
-        allocate (var(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)), &
-                  stat=alloc_stat)
-     end if
-
-     if (alloc_stat /= 0) then
-        errorcode = 8
-        call decomp_2d_abort(errorcode, &
-                             'Memory allocation failed when creating new arrays')
-     end if
-
-     return
+    implicit none
+
+    complex(mytype), allocatable, dimension(:,:,:) :: var
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    logical, intent(IN), optional :: opt_global
+
+    logical :: global
+    integer :: alloc_stat, errorcode
+
+    if (present(opt_global)) then
+       global = opt_global
+    else
+       global = .false.
+    end if
+
+    if (global) then
+       allocate(var(decomp%yst(1):decomp%yen(1), &
+            decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
+            stat=alloc_stat)
+    else
+       allocate(var(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)), &
+            stat=alloc_stat)
+    end if
+    
+    if (alloc_stat /= 0) then
+       errorcode = 8
+       call decomp_2d_abort(errorcode, &
+            'Memory allocation failed when creating new arrays')
+    end if
+
+    return
   end subroutine alloc_y_complex
 
   ! Z-pencil real arrays
   subroutine alloc_z_real_short(var, opt_global)
 
-     implicit none
+    implicit none
 
-     real(mytype), allocatable, dimension(:, :, :) :: var
-     logical, intent(IN), optional :: opt_global
+    real(mytype), allocatable, dimension(:,:,:) :: var
+    logical, intent(IN), optional :: opt_global
 
-     call alloc_z(var, decomp_main, opt_global)
+    call alloc_z(var, decomp_main, opt_global)
 
   end subroutine alloc_z_real_short
 
   subroutine alloc_z_real(var, decomp, opt_global)
 
-     implicit none
-
-     real(mytype), allocatable, dimension(:, :, :) :: var
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
-     logical, intent(IN), optional :: opt_global
-
-     logical :: global
-     integer :: alloc_stat, errorcode
-
-     if (present(opt_global)) then
-        global = opt_global
-     else
-        global = .false.
-     end if
-
-     if (global) then
-        allocate (var(decomp%zst(1):decomp%zen(1), &
-                      decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
-                  stat=alloc_stat)
-     else
-        allocate (var(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)), &
-                  stat=alloc_stat)
-     end if
-
-     if (alloc_stat /= 0) then
-        errorcode = 8
-        call decomp_2d_abort(errorcode, &
-                             'Memory allocation failed when creating new arrays')
-     end if
-
-     return
+    implicit none
+
+    real(mytype), allocatable, dimension(:,:,:) :: var
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    logical, intent(IN), optional :: opt_global
+
+    logical :: global
+    integer :: alloc_stat, errorcode
+
+    if (present(opt_global)) then
+       global = opt_global
+    else
+       global = .false.
+    end if
+
+    if (global) then
+       allocate(var(decomp%zst(1):decomp%zen(1), &
+            decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
+            stat=alloc_stat)
+    else
+       allocate(var(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)), &
+            stat=alloc_stat)
+    end if
+    
+    if (alloc_stat /= 0) then
+       errorcode = 8
+       call decomp_2d_abort(errorcode, &
+            'Memory allocation failed when creating new arrays')
+    end if
+
+    return
   end subroutine alloc_z_real
 
   ! Z-pencil complex arrays
   subroutine alloc_z_complex_short(var, opt_global)
 
-     implicit none
+    implicit none
 
-     complex(mytype), allocatable, dimension(:, :, :) :: var
-     logical, intent(IN), optional :: opt_global
+    complex(mytype), allocatable, dimension(:,:,:) :: var
+    logical, intent(IN), optional :: opt_global
 
-     call alloc_z(var, decomp_main, opt_global)
+    call alloc_z(var, decomp_main, opt_global)
 
   end subroutine alloc_z_complex_short
 
   subroutine alloc_z_complex(var, decomp, opt_global)
 
-     implicit none
-
-     complex(mytype), allocatable, dimension(:, :, :) :: var
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
-     logical, intent(IN), optional :: opt_global
-
-     logical :: global
-     integer :: alloc_stat, errorcode
-
-     if (present(opt_global)) then
-        global = opt_global
-     else
-        global = .false.
-     end if
-
-     if (global) then
-        allocate (var(decomp%zst(1):decomp%zen(1), &
-                      decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
-                  stat=alloc_stat)
-     else
-        allocate (var(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)), &
-                  stat=alloc_stat)
-     end if
-
-     if (alloc_stat /= 0) then
-        errorcode = 8
-        call decomp_2d_abort(errorcode, &
-                             'Memory allocation failed when creating new arrays')
-     end if
-
-     return
+    implicit none
+
+    complex(mytype), allocatable, dimension(:,:,:) :: var
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    logical, intent(IN), optional :: opt_global
+
+    logical :: global
+    integer :: alloc_stat, errorcode
+
+    if (present(opt_global)) then
+       global = opt_global
+    else
+       global = .false.
+    end if
+
+    if (global) then
+       allocate(var(decomp%zst(1):decomp%zen(1), &
+            decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
+            stat=alloc_stat)
+    else
+       allocate(var(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)), &
+            stat=alloc_stat)
+    end if
+    
+    if (alloc_stat /= 0) then
+       errorcode = 8
+       call decomp_2d_abort(errorcode, &
+            'Memory allocation failed when creating new arrays')
+    end if
+
+    return
   end subroutine alloc_z_complex
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 155f20d8..11953900 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -14,362 +14,362 @@
 
 module decomp_2d
 
-   use MPI
-   use, intrinsic :: iso_fortran_env, only: real32, real64
-   use factor
+  use MPI
+  use, intrinsic :: iso_fortran_env, only : real32, real64
+  use factor
 #if defined(_GPU)
-   use cudafor
+  use cudafor
 #if defined(_NCCL)
-   use nccl
+  use nccl
 #endif
 #endif
 
-   implicit none
+  implicit none
 
-   private        ! Make everything private unless declared public
+  private        ! Make everything private unless declared public
 
 #ifdef DOUBLE_PREC
-   integer, parameter, public :: mytype = KIND(0._real64)
-   integer, parameter, public :: real_type = MPI_DOUBLE_PRECISION
-   integer, parameter, public :: real2_type = MPI_2DOUBLE_PRECISION
-   integer, parameter, public :: complex_type = MPI_DOUBLE_COMPLEX
+  integer, parameter, public :: mytype = KIND(0._real64)
+  integer, parameter, public :: real_type = MPI_DOUBLE_PRECISION
+  integer, parameter, public :: real2_type = MPI_2DOUBLE_PRECISION
+  integer, parameter, public :: complex_type = MPI_DOUBLE_COMPLEX
 #ifdef SAVE_SINGLE
-   integer, parameter, public :: mytype_single = KIND(0._real32)
-   integer, parameter, public :: real_type_single = MPI_REAL
+  integer, parameter, public :: mytype_single = KIND(0._real32)
+  integer, parameter, public :: real_type_single = MPI_REAL
 #else
-   integer, parameter, public :: mytype_single = KIND(0._real64)
-   integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
+  integer, parameter, public :: mytype_single = KIND(0._real64)
+  integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
 #endif
 #else
-   integer, parameter, public :: mytype = KIND(0._real32)
-   integer, parameter, public :: real_type = MPI_REAL
-   integer, parameter, public :: real2_type = MPI_2REAL
-   integer, parameter, public :: complex_type = MPI_COMPLEX
-   integer, parameter, public :: mytype_single = KIND(0._real32)
-   integer, parameter, public :: real_type_single = MPI_REAL
+  integer, parameter, public :: mytype = KIND(0._real32)
+  integer, parameter, public :: real_type = MPI_REAL
+  integer, parameter, public :: real2_type = MPI_2REAL
+  integer, parameter, public :: complex_type = MPI_COMPLEX
+  integer, parameter, public :: mytype_single = KIND(0._real32)
+  integer, parameter, public :: real_type_single = MPI_REAL
 #endif
 
-   integer, save, public :: mytype_bytes
-
-   ! some key global variables
-   integer, save, public :: nx_global, ny_global, nz_global  ! global size
-
-   integer, save, public :: nrank = -1 ! local MPI rank
-   integer, save, public :: nproc = -1 ! total number of processors
-   integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
-
-   ! parameters for 2D Cartesian topology
-   integer, save, dimension(2) :: dims, coord
-   integer, save, public :: DECOMP_2D_COMM_CART_X = MPI_COMM_NULL
-   integer, save, public :: DECOMP_2D_COMM_CART_Y = MPI_COMM_NULL
-   integer, save, public :: DECOMP_2D_COMM_CART_Z = MPI_COMM_NULL
-   integer, save :: DECOMP_2D_COMM_ROW = MPI_COMM_NULL
-   integer, save :: DECOMP_2D_COMM_COL = MPI_COMM_NULL
-
-   ! define neighboring blocks (to be used in halo-cell support)
-   !  first dimension 1=X-pencil, 2=Y-pencil, 3=Z-pencil
-   ! second dimension 1=east, 2=west, 3=north, 4=south, 5=top, 6=bottom
-   integer, save, dimension(3, 6) :: neighbour
-
-   ! flags for periodic condition in three dimensions
-   logical, save :: periodic_x, periodic_y, periodic_z
-
-   !
-   ! Output for the log can be changed by the external code before calling decomp_2d_init
-   !
-   !    0 => No log output
-   !    1 => Master rank log output to stdout
-   !    2 => Master rank log output to the file "decomp_2d_setup.log"
-   !    3 => All ranks log output to a dedicated file
-   !
-   ! The default value is 2 (3 for debug builds)
-   !
-   integer, parameter, public :: D2D_LOG_QUIET = 0
-   integer, parameter, public :: D2D_LOG_STDOUT = 1
-   integer, parameter, public :: D2D_LOG_TOFILE = 2
-   integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
+  integer, save, public :: mytype_bytes
+
+  ! some key global variables
+  integer, save, public :: nx_global, ny_global, nz_global  ! global size
+
+  integer, save, public :: nrank = -1 ! local MPI rank 
+  integer, save, public :: nproc = -1 ! total number of processors
+  integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
+
+  ! parameters for 2D Cartesian topology 
+  integer, save, dimension(2) :: dims, coord
+  integer, save, public :: DECOMP_2D_COMM_CART_X = MPI_COMM_NULL
+  integer, save, public :: DECOMP_2D_COMM_CART_Y = MPI_COMM_NULL
+  integer, save, public :: DECOMP_2D_COMM_CART_Z = MPI_COMM_NULL
+  integer, save :: DECOMP_2D_COMM_ROW = MPI_COMM_NULL
+  integer, save :: DECOMP_2D_COMM_COL = MPI_COMM_NULL
+
+  ! define neighboring blocks (to be used in halo-cell support)
+  !  first dimension 1=X-pencil, 2=Y-pencil, 3=Z-pencil
+  ! second dimension 1=east, 2=west, 3=north, 4=south, 5=top, 6=bottom 
+  integer, save, dimension(3,6) :: neighbour 
+
+  ! flags for periodic condition in three dimensions
+  logical, save :: periodic_x, periodic_y, periodic_z
+
+  !
+  ! Output for the log can be changed by the external code before calling decomp_2d_init
+  !
+  !    0 => No log output
+  !    1 => Master rank log output to stdout
+  !    2 => Master rank log output to the file "decomp_2d_setup.log"
+  !    3 => All ranks log output to a dedicated file
+  !
+  ! The default value is 2 (3 for debug builds)
+  !
+  integer, parameter, public :: D2D_LOG_QUIET = 0
+  integer, parameter, public :: D2D_LOG_STDOUT = 1
+  integer, parameter, public :: D2D_LOG_TOFILE = 2
+  integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
 #ifdef DEBUG
-   integer, public, save :: decomp_log = D2D_LOG_TOFILE_FULL
+  integer, public, save :: decomp_log = D2D_LOG_TOFILE_FULL
 #else
-   integer, public, save :: decomp_log = D2D_LOG_TOFILE
+  integer, public, save :: decomp_log = D2D_LOG_TOFILE
 #endif
 
-   !
-   ! Debug level can be changed by the external code before calling decomp_2d_init
-   !
-   ! The environment variable "DECOMP_2D_DEBUG" can be used to change the debug level
-   !
-   ! Debug checks are performed only when the preprocessor variable DEBUG is defined
-   !
-   enum, bind(c)
-      enumerator :: D2D_DEBUG_LEVEL_OFF = 0
-      enumerator :: D2D_DEBUG_LEVEL_CRITICAL = 1
-      enumerator :: D2D_DEBUG_LEVEL_ERROR = 2
-      enumerator :: D2D_DEBUG_LEVEL_WARN = 3
-      enumerator :: D2D_DEBUG_LEVEL_INFO = 4
-      enumerator :: D2D_DEBUG_LEVEL_DEBUG = 5
-      enumerator :: D2D_DEBUG_LEVEL_TRACE = 6
-   end enum
+  !
+  ! Debug level can be changed by the external code before calling decomp_2d_init
+  !
+  ! The environment variable "DECOMP_2D_DEBUG" can be used to change the debug level
+  !
+  ! Debug checks are performed only when the preprocessor variable DEBUG is defined
+  !
+  enum, bind(c)
+     enumerator :: D2D_DEBUG_LEVEL_OFF = 0
+     enumerator :: D2D_DEBUG_LEVEL_CRITICAL = 1
+     enumerator :: D2D_DEBUG_LEVEL_ERROR = 2
+     enumerator :: D2D_DEBUG_LEVEL_WARN = 3
+     enumerator :: D2D_DEBUG_LEVEL_INFO = 4
+     enumerator :: D2D_DEBUG_LEVEL_DEBUG = 5
+     enumerator :: D2D_DEBUG_LEVEL_TRACE = 6
+  end enum
 #ifdef DEBUG
-   integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_INFO
+  integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_INFO
 #else
-   integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_OFF
+  integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_OFF
 #endif
 
 #if defined(_GPU)
 #if defined(_NCCL)
-   integer, save :: row_rank, col_rank
+  integer, save :: row_rank, col_rank
 #endif
 #endif
 
 #ifdef SHM
-   ! derived type to store shared-memory info
-   TYPE, public :: SMP_INFO
-      integer MPI_COMM          ! SMP associated with this communicator
-      integer NODE_ME           ! rank in this communicator
-      integer NCPU              ! size of this communicator
-      integer SMP_COMM          ! communicator for SMP-node masters
-      integer CORE_COMM         ! communicator for cores on SMP-node
-      integer SMP_ME            ! SMP-node id starting from 1 ... NSMP
-      integer NSMP              ! number of SMP-nodes in this communicator
-      integer CORE_ME           ! core id starting from 1 ... NCORE
-      integer NCORE             ! number of cores on this SMP-node
-      integer MAXCORE           ! maximum no. cores on any SMP-node
-      integer N_SND             ! size of SMP shared memory buffer
-      integer N_RCV             ! size of SMP shared memory buffer
-      integer(8) SND_P          ! SNDBUF address (cray pointer), for real
-      integer(8) RCV_P          ! RCVBUF address (cray pointer), for real
-      integer(8) SND_P_c        ! for complex
-      integer(8) RCV_P_c        ! for complex
-   END TYPE SMP_INFO
+  ! derived type to store shared-memory info
+  TYPE, public :: SMP_INFO
+     integer MPI_COMM          ! SMP associated with this communicator
+     integer NODE_ME           ! rank in this communicator
+     integer NCPU              ! size of this communicator
+     integer SMP_COMM          ! communicator for SMP-node masters
+     integer CORE_COMM         ! communicator for cores on SMP-node
+     integer SMP_ME            ! SMP-node id starting from 1 ... NSMP
+     integer NSMP              ! number of SMP-nodes in this communicator
+     integer CORE_ME           ! core id starting from 1 ... NCORE
+     integer NCORE             ! number of cores on this SMP-node
+     integer MAXCORE           ! maximum no. cores on any SMP-node
+     integer N_SND             ! size of SMP shared memory buffer
+     integer N_RCV             ! size of SMP shared memory buffer
+     integer(8) SND_P          ! SNDBUF address (cray pointer), for real 
+     integer(8) RCV_P          ! RCVBUF address (cray pointer), for real
+     integer(8) SND_P_c        ! for complex
+     integer(8) RCV_P_c        ! for complex
+  END TYPE SMP_INFO
 #endif
 
-   ! derived type to store decomposition info for a given global data size
-   TYPE, public :: DECOMP_INFO
-      ! staring/ending index and size of data held by current processor
-      integer, dimension(3) :: xst, xen, xsz  ! x-pencil
-      integer, dimension(3) :: yst, yen, ysz  ! y-pencil
-      integer, dimension(3) :: zst, zen, zsz  ! z-pencil
+  ! derived type to store decomposition info for a given global data size
+  TYPE, public :: DECOMP_INFO
+     ! staring/ending index and size of data held by current processor
+     integer, dimension(3) :: xst, xen, xsz  ! x-pencil
+     integer, dimension(3) :: yst, yen, ysz  ! y-pencil
+     integer, dimension(3) :: zst, zen, zsz  ! z-pencil
 
-      ! in addition to local information, processors also need to know
-      ! some global information for global communications to work
+     ! in addition to local information, processors also need to know 
+     ! some global information for global communications to work 
 
-      ! how each dimension is distributed along pencils
-      integer, allocatable, dimension(:) :: &
-         x1dist, y1dist, y2dist, z2dist
+     ! how each dimension is distributed along pencils
+     integer, allocatable, dimension(:) :: &
+          x1dist, y1dist, y2dist, z2dist
 
-      ! send/receive buffer counts and displacements for MPI_ALLTOALLV
-      integer, allocatable, dimension(:) :: &
-         x1cnts, y1cnts, y2cnts, z2cnts
-      integer, allocatable, dimension(:) :: &
-         x1disp, y1disp, y2disp, z2disp
+     ! send/receive buffer counts and displacements for MPI_ALLTOALLV
+     integer, allocatable, dimension(:) :: &
+          x1cnts, y1cnts, y2cnts, z2cnts
+     integer, allocatable, dimension(:) :: &
+          x1disp, y1disp, y2disp, z2disp
 
-      ! buffer counts for MPI_ALLTOALL: either for evenly distributed data
-      ! or for padded-alltoall
-      integer :: x1count, y1count, y2count, z2count
+     ! buffer counts for MPI_ALLTOALL: either for evenly distributed data
+     ! or for padded-alltoall
+     integer :: x1count, y1count, y2count, z2count
 
-      ! evenly distributed data
-      logical :: even
+     ! evenly distributed data
+     logical :: even
 
 #ifdef SHM
-      ! For shared-memory implementation
-
-      ! one instance of this derived type for each communicator
-      ! shared moemory info, such as which MPI rank belongs to which node
-      TYPE(SMP_INFO) :: ROW_INFO, COL_INFO
-
-      ! shared send/recv buffers for ALLTOALLV
-      integer, allocatable, dimension(:) :: x1cnts_s, y1cnts_s, &
-                                            y2cnts_s, z2cnts_s
-      integer, allocatable, dimension(:) :: x1disp_s, y1disp_s, &
-                                            y2disp_s, z2disp_s
-      ! A copy of original buffer displacement (will be overwriten)
-      integer, allocatable, dimension(:) :: x1disp_o, y1disp_o, &
-                                            y2disp_o, z2disp_o
+     ! For shared-memory implementation
+
+     ! one instance of this derived type for each communicator
+     ! shared moemory info, such as which MPI rank belongs to which node
+     TYPE(SMP_INFO) :: ROW_INFO, COL_INFO
+
+     ! shared send/recv buffers for ALLTOALLV
+     integer, allocatable, dimension(:) :: x1cnts_s, y1cnts_s, &
+          y2cnts_s, z2cnts_s
+     integer, allocatable, dimension(:) :: x1disp_s, y1disp_s, &
+          y2disp_s, z2disp_s
+     ! A copy of original buffer displacement (will be overwriten)
+     integer, allocatable, dimension(:) :: x1disp_o, y1disp_o, &
+          y2disp_o, z2disp_o
 #endif
-   END TYPE DECOMP_INFO
+  END TYPE DECOMP_INFO
 
-   ! main (default) decomposition information for global size nx*ny*nz
-   TYPE(DECOMP_INFO), target, save, public :: decomp_main
-   ! FIXME The extra decomp_info objects should be defined in the external code, not here
-   !       Currently keeping them to avoid breaking external codes
-   TYPE(DECOMP_INFO), save, public :: phG, ph1, ph2, ph3, ph4
+  ! main (default) decomposition information for global size nx*ny*nz
+  TYPE(DECOMP_INFO), target, save, public :: decomp_main
+  ! FIXME The extra decomp_info objects should be defined in the external code, not here
+  !       Currently keeping them to avoid breaking external codes
+  TYPE(DECOMP_INFO), save, public :: phG,ph1,ph2,ph3,ph4
 
-   ! staring/ending index and size of data held by current processor
-   ! duplicate 'decomp_main', needed by apps to define data structure
-   integer, save, dimension(3), public :: xstart, xend, xsize  ! x-pencil
-   integer, save, dimension(3), public :: ystart, yend, ysize  ! y-pencil
-   integer, save, dimension(3), public :: zstart, zend, zsize  ! z-pencil
+  ! staring/ending index and size of data held by current processor
+  ! duplicate 'decomp_main', needed by apps to define data structure 
+  integer, save, dimension(3), public :: xstart, xend, xsize  ! x-pencil
+  integer, save, dimension(3), public :: ystart, yend, ysize  ! y-pencil
+  integer, save, dimension(3), public :: zstart, zend, zsize  ! z-pencil
 
-   ! These are the buffers used by MPI_ALLTOALL(V) calls
-   integer, save :: decomp_buf_size = 0
-   real(mytype), allocatable, dimension(:) :: work1_r, work2_r
-   complex(mytype), allocatable, dimension(:) :: work1_c, work2_c
+  ! These are the buffers used by MPI_ALLTOALL(V) calls
+  integer, save :: decomp_buf_size = 0
+  real(mytype),    allocatable, dimension(:) :: work1_r, work2_r
+  complex(mytype), allocatable, dimension(:) :: work1_c, work2_c
 
 #if defined(_GPU)
-   real(mytype), allocatable, dimension(:), device :: work1_r_d, work2_r_d
-   complex(mytype), allocatable, dimension(:), device :: work1_c_d, work2_c_d
+  real(mytype), allocatable, dimension(:), device :: work1_r_d, work2_r_d
+  complex(mytype), allocatable, dimension(:), device :: work1_c_d, work2_c_d
 
 #if defined(_NCCL)
-   integer col_comm_size, row_comm_size
-   integer, allocatable, dimension(:) :: local_to_global_col, local_to_global_row
-   type(ncclUniqueId) :: nccl_uid_2decomp
-   type(ncclComm) :: nccl_comm_2decomp
-   integer(kind=cuda_stream_kind) :: cuda_stream_2decomp
+  integer col_comm_size, row_comm_size
+  integer, allocatable, dimension(:) :: local_to_global_col, local_to_global_row
+  type(ncclUniqueId) :: nccl_uid_2decomp
+  type(ncclComm) :: nccl_comm_2decomp
+  integer(kind=cuda_stream_kind) :: cuda_stream_2decomp
 #endif
 #endif
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! To define smaller arrays using every several mesh points
-   integer, save, dimension(3), public :: xszS, yszS, zszS, xstS, ystS, zstS, xenS, yenS, zenS
-   integer, save, dimension(3), public :: xszV, yszV, zszV, xstV, ystV, zstV, xenV, yenV, zenV
-   integer, save, dimension(3), public :: xszP, yszP, zszP, xstP, ystP, zstP, xenP, yenP, zenP
-   logical, save :: coarse_mesh_starts_from_1
-   integer, save :: iskipS, jskipS, kskipS
-   integer, save :: iskipV, jskipV, kskipV
-   integer, save :: iskipP, jskipP, kskipP
-
-   !
-   ! Profiler section
-   !
-   ! Integer to select the profiling tool
-   !    0 => no profiling, default
-   !    1 => Caliper (https://github.com/LLNL/Caliper)
-   !
-   enum, bind(c)
-      enumerator :: decomp_profiler_none = 0
-      enumerator :: decomp_profiler_caliper = 1
-   end enum
-   integer(kind(decomp_profiler_none)), save, public :: decomp_profiler = decomp_profiler_none
-   ! Default : profile everything
-   logical, save, public :: decomp_profiler_transpose = .true.
-   logical, save, public :: decomp_profiler_io = .true.
-   logical, save, public :: decomp_profiler_fft = .true.
-   logical, save, public :: decomp_profiler_d2d = .true.
-
-   ! public user routines
-   public :: decomp_2d_init, decomp_2d_finalize, &
-             transpose_x_to_y, transpose_y_to_z, &
-             transpose_z_to_y, transpose_y_to_x, &
-             decomp_info_init, decomp_info_finalize, partition, &
-             decomp_info_print, decomp_profiler_prep, &
-             decomp_profiler_start, decomp_profiler_end, &
-             init_coarser_mesh_statS, fine_to_coarseS, &
-             init_coarser_mesh_statV, fine_to_coarseV, &
-             init_coarser_mesh_statP, fine_to_coarseP, &
-             alloc_x, alloc_y, alloc_z, &
-             update_halo, decomp_2d_abort, &
-             decomp_2d_warning, get_decomp_info, &
-             decomp_mpi_comm_free, get_decomp_dims
+  ! To define smaller arrays using every several mesh points
+  integer, save, dimension(3), public :: xszS,yszS,zszS,xstS,ystS,zstS,xenS,yenS,zenS
+  integer, save, dimension(3), public :: xszV,yszV,zszV,xstV,ystV,zstV,xenV,yenV,zenV
+  integer, save, dimension(3), public :: xszP,yszP,zszP,xstP,ystP,zstP,xenP,yenP,zenP
+  logical, save :: coarse_mesh_starts_from_1
+  integer, save :: iskipS, jskipS, kskipS
+  integer, save :: iskipV, jskipV, kskipV
+  integer, save :: iskipP, jskipP, kskipP
+
+  !
+  ! Profiler section
+  !
+  ! Integer to select the profiling tool
+  !    0 => no profiling, default
+  !    1 => Caliper (https://github.com/LLNL/Caliper)
+  !
+  enum, bind(c)
+    enumerator :: decomp_profiler_none = 0
+    enumerator :: decomp_profiler_caliper = 1
+  end enum
+  integer(kind(decomp_profiler_none)), save, public :: decomp_profiler = decomp_profiler_none
+  ! Default : profile everything
+  logical, save, public :: decomp_profiler_transpose = .true.
+  logical, save, public :: decomp_profiler_io = .true.
+  logical, save, public :: decomp_profiler_fft = .true.
+  logical, save, public :: decomp_profiler_d2d = .true.
+
+  ! public user routines
+  public :: decomp_2d_init, decomp_2d_finalize, &
+       transpose_x_to_y, transpose_y_to_z, &
+       transpose_z_to_y, transpose_y_to_x, &
+       decomp_info_init, decomp_info_finalize, partition, &
+       decomp_info_print, decomp_profiler_prep, &
+       decomp_profiler_start, decomp_profiler_end, &
+       init_coarser_mesh_statS,fine_to_coarseS,&
+       init_coarser_mesh_statV,fine_to_coarseV,&
+       init_coarser_mesh_statP,fine_to_coarseP,&
+       alloc_x, alloc_y, alloc_z, &
+       update_halo, decomp_2d_abort, &
+       decomp_2d_warning, get_decomp_info, &
+       decomp_mpi_comm_free, get_decomp_dims
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! These are routines to perform global data transpositions
-   !
-   !   Four combinations are available, enough to cover all situations
-   !    - transpose_x_to_y (X-pencil --> Y-pencil)
-   !    - transpose_y_to_z (Y-pencil --> Z-pencil)
-   !    - transpose_z_to_y (Z-pencil --> Y-pencil)
-   !    - transpose_y_to_x (Y-pencil --> X-pencil)
-   !
-   !   Generic interface provided here to support multiple data types
-   !    - real and complex types supported through generic interface
-   !    - single/double precision supported through pre-processing
-   !       * see 'mytype' variable at the beginning
-   !    - an optional argument can be supplied to transpose data whose
-   !      global size is not the default nx*ny*nz
-   !       * as the case in fft r2c/c2r interface
+  ! These are routines to perform global data transpositions
+  ! 
+  !   Four combinations are available, enough to cover all situations
+  !    - transpose_x_to_y (X-pencil --> Y-pencil)
+  !    - transpose_y_to_z (Y-pencil --> Z-pencil)
+  !    - transpose_z_to_y (Z-pencil --> Y-pencil)
+  !    - transpose_y_to_x (Y-pencil --> X-pencil)
+  !
+  !   Generic interface provided here to support multiple data types
+  !    - real and complex types supported through generic interface
+  !    - single/double precision supported through pre-processing
+  !       * see 'mytype' variable at the beginning
+  !    - an optional argument can be supplied to transpose data whose 
+  !      global size is not the default nx*ny*nz 
+  !       * as the case in fft r2c/c2r interface 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-   interface decomp_2d_init
-      module procedure decomp_2d_init_ref
-   end interface decomp_2d_init
-
-   interface decomp_2d_finalize
-      module procedure decomp_2d_finalize_ref
-   end interface decomp_2d_finalize
-
-   interface transpose_x_to_y
-      module procedure transpose_x_to_y_real
-      module procedure transpose_x_to_y_real_short
-      module procedure transpose_x_to_y_complex
-      module procedure transpose_x_to_y_complex_short
-   end interface transpose_x_to_y
-
-   interface transpose_y_to_z
-      module procedure transpose_y_to_z_real
-      module procedure transpose_y_to_z_real_short
-      module procedure transpose_y_to_z_complex
-      module procedure transpose_y_to_z_complex_short
-   end interface transpose_y_to_z
-
-   interface transpose_z_to_y
-      module procedure transpose_z_to_y_real
-      module procedure transpose_z_to_y_real_short
-      module procedure transpose_z_to_y_complex
-      module procedure transpose_z_to_y_complex_short
-   end interface transpose_z_to_y
-
-   interface transpose_y_to_x
-      module procedure transpose_y_to_x_real
-      module procedure transpose_y_to_x_real_short
-      module procedure transpose_y_to_x_complex
-      module procedure transpose_y_to_x_complex_short
-   end interface transpose_y_to_x
-
-   interface update_halo
-      module procedure update_halo_real
-      module procedure update_halo_real_short
-      module procedure update_halo_complex
-      module procedure update_halo_complex_short
-   end interface update_halo
-
-   interface alloc_x
-      module procedure alloc_x_real
-      module procedure alloc_x_real_short
-      module procedure alloc_x_complex
-      module procedure alloc_x_complex_short
-   end interface alloc_x
-
-   interface alloc_y
-      module procedure alloc_y_real
-      module procedure alloc_y_real_short
-      module procedure alloc_y_complex
-      module procedure alloc_y_complex_short
-   end interface alloc_y
-
-   interface alloc_z
-      module procedure alloc_z_real
-      module procedure alloc_z_real_short
-      module procedure alloc_z_complex
-      module procedure alloc_z_complex_short
-   end interface alloc_z
-
-   interface decomp_2d_abort
-      module procedure decomp_2d_abort_basic
-      module procedure decomp_2d_abort_file_line
+  interface decomp_2d_init
+     module procedure decomp_2d_init_ref
+  end interface decomp_2d_init
+
+  interface decomp_2d_finalize
+     module procedure decomp_2d_finalize_ref
+  end interface decomp_2d_finalize
+  
+  interface transpose_x_to_y
+     module procedure transpose_x_to_y_real
+     module procedure transpose_x_to_y_real_short
+     module procedure transpose_x_to_y_complex
+     module procedure transpose_x_to_y_complex_short
+  end interface transpose_x_to_y
+
+  interface transpose_y_to_z
+     module procedure transpose_y_to_z_real
+     module procedure transpose_y_to_z_real_short
+     module procedure transpose_y_to_z_complex
+     module procedure transpose_y_to_z_complex_short
+  end interface transpose_y_to_z
+
+  interface transpose_z_to_y
+     module procedure transpose_z_to_y_real
+     module procedure transpose_z_to_y_real_short
+     module procedure transpose_z_to_y_complex
+     module procedure transpose_z_to_y_complex_short
+  end interface transpose_z_to_y
+
+  interface transpose_y_to_x
+     module procedure transpose_y_to_x_real
+     module procedure transpose_y_to_x_real_short
+     module procedure transpose_y_to_x_complex
+     module procedure transpose_y_to_x_complex_short
+  end interface transpose_y_to_x
+
+  interface update_halo
+     module procedure update_halo_real
+     module procedure update_halo_real_short
+     module procedure update_halo_complex
+     module procedure update_halo_complex_short
+  end interface update_halo
+
+  interface alloc_x
+     module procedure alloc_x_real
+     module procedure alloc_x_real_short
+     module procedure alloc_x_complex
+     module procedure alloc_x_complex_short
+  end interface alloc_x
+
+  interface alloc_y
+     module procedure alloc_y_real
+     module procedure alloc_y_real_short
+     module procedure alloc_y_complex
+     module procedure alloc_y_complex_short
+  end interface alloc_y
+
+  interface alloc_z
+     module procedure alloc_z_real
+     module procedure alloc_z_real_short
+     module procedure alloc_z_complex
+     module procedure alloc_z_complex_short
+  end interface alloc_z
+
+  interface decomp_2d_abort
+     module procedure decomp_2d_abort_basic
+     module procedure decomp_2d_abort_file_line
 #if defined(_GPU) && defined(_NCCL)
-      module procedure decomp_2d_abort_nccl_basic
-      module procedure decomp_2d_abort_nccl_file_line
+     module procedure decomp_2d_abort_nccl_basic
+     module procedure decomp_2d_abort_nccl_file_line
 #endif
-   end interface decomp_2d_abort
+  end interface decomp_2d_abort
 
-   interface decomp_2d_warning
-      module procedure decomp_2d_warning_basic
-      module procedure decomp_2d_warning_file_line
-   end interface decomp_2d_warning
+  interface decomp_2d_warning
+     module procedure decomp_2d_warning_basic
+     module procedure decomp_2d_warning_file_line
+  end interface decomp_2d_warning
 
-   interface
+  interface
 
-      module subroutine d2d_listing(given_io_unit)
-         integer, intent(in), optional :: given_io_unit
-      end subroutine d2d_listing
+     module subroutine d2d_listing(given_io_unit)
+        integer, intent(in), optional :: given_io_unit
+     end subroutine d2d_listing
 
-      module subroutine decomp_info_print(d2d, io_unit, d2dname)
-         type(decomp_info), intent(in) :: d2d
-         integer, intent(in) :: io_unit
-         character(len=*), intent(in) :: d2dname
-      end subroutine decomp_info_print
+     module subroutine decomp_info_print(d2d, io_unit, d2dname)
+        type(decomp_info), intent(in) :: d2d
+        integer, intent(in) :: io_unit
+        character(len=*), intent(in) :: d2dname
+     end subroutine decomp_info_print
 
-   end interface
+  end interface
 
    ! Generic interface to initialize the profiler
    interface decomp_profiler_init
@@ -412,1220 +412,1228 @@ end subroutine decomp_profiler_end_char
    end interface decomp_profiler_end
 
 contains
-
+        
 #include "decomp_2d_init_fin.f90"
 
-   !
-   ! Small wrapper to free a MPI communicator
-   !
-   subroutine decomp_mpi_comm_free(mpi_comm)
+  !
+  ! Small wrapper to free a MPI communicator
+  !
+  subroutine decomp_mpi_comm_free(mpi_comm)
 
-      implicit none
+    implicit none
 
-      integer, intent(inout) :: mpi_comm
-      integer :: ierror
+    integer, intent(inout) :: mpi_comm
+    integer :: ierror
 
-      ! Return if no MPI comm to free
-      if (mpi_comm == MPI_COMM_NULL) return
+    ! Return if no MPI comm to free
+    if (mpi_comm == MPI_COMM_NULL) return
 
-      ! Free the provided MPI communicator
-      call MPI_COMM_FREE(mpi_comm, ierror)
-      if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
-      mpi_comm = MPI_COMM_NULL
+    ! Free the provided MPI communicator
+    call MPI_COMM_FREE(mpi_comm, ierror)
+    if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
+    mpi_comm = MPI_COMM_NULL
 
-   end subroutine decomp_mpi_comm_free
+  end subroutine decomp_mpi_comm_free
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Return the default decomposition object
+  ! Return the default decomposition object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-   ! FIXME avoid a copy and return a pointer to decomp_main
-   ! TODO list the external codes using this subroutine
-   subroutine get_decomp_info(decomp)
+  ! FIXME avoid a copy and return a pointer to decomp_main
+  ! TODO list the external codes using this subroutine
+  subroutine get_decomp_info(decomp)
 
-      implicit none
+    implicit none
 
-      ! FIXME TYPE(DECOMP_INFO), pointer :: decomp
-      TYPE(DECOMP_INFO), intent(OUT) :: decomp
+    ! FIXME TYPE(DECOMP_INFO), pointer :: decomp
+    TYPE(DECOMP_INFO), intent(OUT) :: decomp
 
-      ! FIXME decomp => decomp_main
-      decomp = decomp_main
+    ! FIXME decomp => decomp_main
+    decomp = decomp_main
 
-      return
-   end subroutine get_decomp_info
+    return
+  end subroutine get_decomp_info
 
-   !
-   ! Return the 2D processor grid
-   !
-   function get_decomp_dims()
+  !
+  ! Return the 2D processor grid
+  !
+  function get_decomp_dims()
 
-      implicit none
+    implicit none
 
-      integer, dimension(2) :: get_decomp_dims
+    integer, dimension(2) :: get_decomp_dims
 
-      get_decomp_dims = dims
+    get_decomp_dims = dims
 
-   end function get_decomp_dims
+  end function get_decomp_dims
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Advanced Interface allowing applications to define globle domain of
-   ! any size, distribute it, and then transpose data among pencils.
-   !  - generate 2D decomposition details as defined in DECOMP_INFO
-   !  - the default global data size is nx*ny*nz
-   !  - a different global size nx/2+1,ny,nz is used in FFT r2c/c2r
-   !  - multiple global sizes can co-exist in one application, each
-   !    using its own DECOMP_INFO object
+  ! Advanced Interface allowing applications to define globle domain of
+  ! any size, distribute it, and then transpose data among pencils.
+  !  - generate 2D decomposition details as defined in DECOMP_INFO
+  !  - the default global data size is nx*ny*nz
+  !  - a different global size nx/2+1,ny,nz is used in FFT r2c/c2r
+  !  - multiple global sizes can co-exist in one application, each
+  !    using its own DECOMP_INFO object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine decomp_info_init(nx, ny, nz, decomp)
-
-      implicit none
-
-      integer, intent(IN) :: nx, ny, nz
-      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-
-      integer :: buf_size, status, errorcode
-
-      ! verify the global size can actually be distributed as pencils
-      if (nx_global < dims(1) .or. ny_global < dims(1) .or. ny_global < dims(2) .or. nz_global < dims(2)) then
-         errorcode = 6
-         call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                              'Invalid 2D processor grid. '// &
-                              'Make sure that min(nx,ny) >= p_row and '// &
-                              'min(ny,nz) >= p_col')
-      end if
-
-      if (mod(nx, dims(1)) == 0 .and. mod(ny, dims(1)) == 0 .and. &
-          mod(ny, dims(2)) == 0 .and. mod(nz, dims(2)) == 0) then
-         decomp%even = .true.
-      else
-         decomp%even = .false.
-      end if
-
-      ! distribute mesh points
-      allocate (decomp%x1dist(0:dims(1) - 1), decomp%y1dist(0:dims(1) - 1), &
-                decomp%y2dist(0:dims(2) - 1), decomp%z2dist(0:dims(2) - 1))
-      call get_dist(nx, ny, nz, decomp)
-
-      ! generate partition information - starting/ending index etc.
-      call partition(nx, ny, nz, (/1, 2, 3/), &
-                     decomp%xst, decomp%xen, decomp%xsz)
-      call partition(nx, ny, nz, (/2, 1, 3/), &
-                     decomp%yst, decomp%yen, decomp%ysz)
-      call partition(nx, ny, nz, (/2, 3, 1/), &
-                     decomp%zst, decomp%zen, decomp%zsz)
-
-      ! prepare send/receive buffer displacement and count for ALLTOALL(V)
-      allocate (decomp%x1cnts(0:dims(1) - 1), decomp%y1cnts(0:dims(1) - 1), &
-                decomp%y2cnts(0:dims(2) - 1), decomp%z2cnts(0:dims(2) - 1))
-      allocate (decomp%x1disp(0:dims(1) - 1), decomp%y1disp(0:dims(1) - 1), &
-                decomp%y2disp(0:dims(2) - 1), decomp%z2disp(0:dims(2) - 1))
-      call prepare_buffer(decomp)
+  subroutine decomp_info_init(nx,ny,nz,decomp)
+
+    implicit none
+
+    integer, intent(IN) :: nx,ny,nz
+    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+
+    integer :: buf_size, status, errorcode
+
+    ! verify the global size can actually be distributed as pencils
+    if (nx_global<dims(1) .or. ny_global<dims(1) .or. ny_global<dims(2) .or. nz_global<dims(2)) then
+       errorcode = 6
+       call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+            'Invalid 2D processor grid. ' // &
+            'Make sure that min(nx,ny) >= p_row and ' // &
+            'min(ny,nz) >= p_col')
+    end if
+
+    if (mod(nx,dims(1))==0 .and. mod(ny,dims(1))==0 .and. &
+         mod(ny,dims(2))==0 .and. mod(nz,dims(2))==0) then
+       decomp%even = .true.
+    else
+       decomp%even = .false.
+    end if
+
+    ! distribute mesh points
+    allocate(decomp%x1dist(0:dims(1)-1),decomp%y1dist(0:dims(1)-1), &
+         decomp%y2dist(0:dims(2)-1),decomp%z2dist(0:dims(2)-1))
+    call get_dist(nx,ny,nz,decomp)
+
+    ! generate partition information - starting/ending index etc.
+    call partition(nx, ny, nz, (/ 1,2,3 /), &
+         decomp%xst, decomp%xen, decomp%xsz)
+    call partition(nx, ny, nz, (/ 2,1,3 /), &
+         decomp%yst, decomp%yen, decomp%ysz)
+    call partition(nx, ny, nz, (/ 2,3,1 /), &
+         decomp%zst, decomp%zen, decomp%zsz)
+
+    ! prepare send/receive buffer displacement and count for ALLTOALL(V)
+    allocate(decomp%x1cnts(0:dims(1)-1),decomp%y1cnts(0:dims(1)-1), &
+         decomp%y2cnts(0:dims(2)-1),decomp%z2cnts(0:dims(2)-1))
+    allocate(decomp%x1disp(0:dims(1)-1),decomp%y1disp(0:dims(1)-1), &
+         decomp%y2disp(0:dims(2)-1),decomp%z2disp(0:dims(2)-1))
+    call prepare_buffer(decomp)
 
 #ifdef SHM
-      ! prepare shared-memory information if required
-      call decomp_info_init_shm(decomp)
+    ! prepare shared-memory information if required
+    call decomp_info_init_shm(decomp)
 #endif
 
-      ! allocate memory for the MPI_ALLTOALL(V) buffers
-      ! define the buffers globally for performance reason
+    ! allocate memory for the MPI_ALLTOALL(V) buffers
+    ! define the buffers globally for performance reason
 
-      buf_size = max(decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3), &
-                     max(decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3), &
-                         decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)))
+    buf_size = max(decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3), &
+         max(decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3), &
+         decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)) )
 #ifdef EVEN
-      ! padded alltoall optimisation may need larger buffer space
-      buf_size = max(buf_size, &
-                     max(decomp%x1count*dims(1), decomp%y2count*dims(2)))
+    ! padded alltoall optimisation may need larger buffer space
+    buf_size = max(buf_size, &
+         max(decomp%x1count*dims(1),decomp%y2count*dims(2)) ) 
 #endif
 
-      ! check if additional memory is required
-      ! *** TODO: consider how to share the real/complex buffers
-      if (buf_size > decomp_buf_size) then
-         decomp_buf_size = buf_size
+    ! check if additional memory is required
+    ! *** TODO: consider how to share the real/complex buffers 
+    if (buf_size > decomp_buf_size) then
+       decomp_buf_size = buf_size
 #if defined(_GPU)
-         if (allocated(work1_r_d)) deallocate (work1_r_d)
-         if (allocated(work2_r_d)) deallocate (work2_r_d)
-         if (allocated(work1_c_d)) deallocate (work1_c_d)
-         if (allocated(work2_c_d)) deallocate (work2_c_d)
-         allocate (work1_r_d(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work1_c_d(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work2_r_d(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work2_c_d(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
+       if (allocated(work1_r_d)) deallocate(work1_r_d)
+       if (allocated(work2_r_d)) deallocate(work2_r_d)
+       if (allocated(work1_c_d)) deallocate(work1_c_d)
+       if (allocated(work2_c_d)) deallocate(work2_c_d)
+       allocate(work1_r_d(buf_size), STAT=status)
+       if (status /= 0) then
+          errorcode = 2
+          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+               'Out of memory when allocating 2DECOMP workspace')
+       end if
+       allocate(work1_c_d(buf_size), STAT=status)
+       if (status /= 0) then
+          errorcode = 2
+          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+               'Out of memory when allocating 2DECOMP workspace')
+       end if
+       allocate(work2_r_d(buf_size), STAT=status)
+       if (status /= 0) then
+          errorcode = 2
+          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+               'Out of memory when allocating 2DECOMP workspace')
+       end if
+       allocate(work2_c_d(buf_size), STAT=status)
+       if (status /= 0) then
+          errorcode = 2
+          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+               'Out of memory when allocating 2DECOMP workspace')
+       end if
 #endif
-         if (allocated(work1_r)) deallocate (work1_r)
-         if (allocated(work2_r)) deallocate (work2_r)
-         if (allocated(work1_c)) deallocate (work1_c)
-         if (allocated(work2_c)) deallocate (work2_c)
-         allocate (work1_r(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work2_r(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work1_c(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work2_c(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-      end if
-
-      return
-   end subroutine decomp_info_init
+       if (allocated(work1_r)) deallocate(work1_r)
+       if (allocated(work2_r)) deallocate(work2_r)
+       if (allocated(work1_c)) deallocate(work1_c)
+       if (allocated(work2_c)) deallocate(work2_c)
+       allocate(work1_r(buf_size), STAT=status)
+       if (status /= 0) then
+          errorcode = 2
+          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+               'Out of memory when allocating 2DECOMP workspace')
+       end if
+       allocate(work2_r(buf_size), STAT=status)
+       if (status /= 0) then
+          errorcode = 2
+          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+               'Out of memory when allocating 2DECOMP workspace')
+       end if
+       allocate(work1_c(buf_size), STAT=status)
+       if (status /= 0) then
+          errorcode = 2
+          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+               'Out of memory when allocating 2DECOMP workspace')
+       end if
+       allocate(work2_c(buf_size), STAT=status)
+       if (status /= 0) then
+          errorcode = 2
+          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+               'Out of memory when allocating 2DECOMP workspace')
+       end if
+    end if
+
+    return
+  end subroutine decomp_info_init
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Release memory associated with a DECOMP_INFO object
+  ! Release memory associated with a DECOMP_INFO object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine decomp_info_finalize(decomp)
+  subroutine decomp_info_finalize(decomp)
 
-      implicit none
+    implicit none
 
-      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
 
-      if (allocated(decomp%x1dist)) deallocate (decomp%x1dist)
-      if (allocated(decomp%y1dist)) deallocate (decomp%y1dist)
-      if (allocated(decomp%y2dist)) deallocate (decomp%y2dist)
-      if (allocated(decomp%z2dist)) deallocate (decomp%z2dist)
-      if (allocated(decomp%x1cnts)) deallocate (decomp%x1cnts)
-      if (allocated(decomp%y1cnts)) deallocate (decomp%y1cnts)
-      if (allocated(decomp%y2cnts)) deallocate (decomp%y2cnts)
-      if (allocated(decomp%z2cnts)) deallocate (decomp%z2cnts)
-      if (allocated(decomp%x1disp)) deallocate (decomp%x1disp)
-      if (allocated(decomp%y1disp)) deallocate (decomp%y1disp)
-      if (allocated(decomp%y2disp)) deallocate (decomp%y2disp)
-      if (allocated(decomp%z2disp)) deallocate (decomp%z2disp)
+    if (allocated(decomp%x1dist)) deallocate(decomp%x1dist)
+    if (allocated(decomp%y1dist)) deallocate(decomp%y1dist)
+    if (allocated(decomp%y2dist)) deallocate(decomp%y2dist)
+    if (allocated(decomp%z2dist)) deallocate(decomp%z2dist)
+    if (allocated(decomp%x1cnts)) deallocate(decomp%x1cnts)
+    if (allocated(decomp%y1cnts)) deallocate(decomp%y1cnts)
+    if (allocated(decomp%y2cnts)) deallocate(decomp%y2cnts)
+    if (allocated(decomp%z2cnts)) deallocate(decomp%z2cnts)
+    if (allocated(decomp%x1disp)) deallocate(decomp%x1disp)
+    if (allocated(decomp%y1disp)) deallocate(decomp%y1disp)
+    if (allocated(decomp%y2disp)) deallocate(decomp%y2disp)
+    if (allocated(decomp%z2disp)) deallocate(decomp%z2disp)
 
 #ifdef SHM
-      if (allocated(decomp%x1disp_o)) deallocate (decomp%x1disp_o)
-      if (allocated(decomp%y1disp_o)) deallocate (decomp%y1disp_o)
-      if (allocated(decomp%y2disp_o)) deallocate (decomp%y2disp_o)
-      if (allocated(decomp%z2disp_o)) deallocate (decomp%z2disp_o)
-      if (allocated(decomp%x1cnts_s)) deallocate (decomp%x1cnts_s)
-      if (allocated(decomp%y1cnts_s)) deallocate (decomp%y1cnts_s)
-      if (allocated(decomp%y2cnts_s)) deallocate (decomp%y2cnts_s)
-      if (allocated(decomp%z2cnts_s)) deallocate (decomp%z2cnts_s)
-      if (allocated(decomp%x1disp_s)) deallocate (decomp%x1disp_s)
-      if (allocated(decomp%y1disp_s)) deallocate (decomp%y1disp_s)
-      if (allocated(decomp%y2disp_s)) deallocate (decomp%y2disp_s)
-      if (allocated(decomp%z2disp_s)) deallocate (decomp%z2disp_s)
+    if (allocated(decomp%x1disp_o)) deallocate(decomp%x1disp_o)
+    if (allocated(decomp%y1disp_o)) deallocate(decomp%y1disp_o)
+    if (allocated(decomp%y2disp_o)) deallocate(decomp%y2disp_o)
+    if (allocated(decomp%z2disp_o)) deallocate(decomp%z2disp_o)
+    if (allocated(decomp%x1cnts_s)) deallocate(decomp%x1cnts_s)
+    if (allocated(decomp%y1cnts_s)) deallocate(decomp%y1cnts_s)
+    if (allocated(decomp%y2cnts_s)) deallocate(decomp%y2cnts_s)
+    if (allocated(decomp%z2cnts_s)) deallocate(decomp%z2cnts_s)
+    if (allocated(decomp%x1disp_s)) deallocate(decomp%x1disp_s)
+    if (allocated(decomp%y1disp_s)) deallocate(decomp%y1disp_s)
+    if (allocated(decomp%y2disp_s)) deallocate(decomp%y2disp_s)
+    if (allocated(decomp%z2disp_s)) deallocate(decomp%z2disp_s)
 #endif
 
-      return
-   end subroutine decomp_info_finalize
+    return
+  end subroutine decomp_info_finalize
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Coarser mesh support for statistic
+  ! Coarser mesh support for statistic
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine init_coarser_mesh_statS(i_skip, j_skip, k_skip, from1)
-
-      implicit none
-
-      integer, intent(IN) :: i_skip, j_skip, k_skip
-      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-      ! .false. - save n,2n,3n...
-
-      integer, dimension(3) :: skip
-      integer :: i
-
-      coarse_mesh_starts_from_1 = from1
-      iskipS = i_skip
-      jskipS = j_skip
-      kskipS = k_skip
-
-      skip(1) = iskipS
-      skip(2) = jskipS
-      skip(3) = kskipS
-
-      do i = 1, 3
-         if (from1) then
-            xstS(i) = (xstart(i) + skip(i) - 1)/skip(i)
-            if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstS(i) = xstS(i) + 1
-            xenS(i) = (xend(i) + skip(i) - 1)/skip(i)
-         else
-            xstS(i) = xstart(i)/skip(i)
-            if (mod(xstart(i), skip(i)) /= 0) xstS(i) = xstS(i) + 1
-            xenS(i) = xend(i)/skip(i)
-         end if
-         xszS(i) = xenS(i) - xstS(i) + 1
-      end do
-
-      do i = 1, 3
-         if (from1) then
-            ystS(i) = (ystart(i) + skip(i) - 1)/skip(i)
-            if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystS(i) = ystS(i) + 1
-            yenS(i) = (yend(i) + skip(i) - 1)/skip(i)
-         else
-            ystS(i) = ystart(i)/skip(i)
-            if (mod(ystart(i), skip(i)) /= 0) ystS(i) = ystS(i) + 1
-            yenS(i) = yend(i)/skip(i)
-         end if
-         yszS(i) = yenS(i) - ystS(i) + 1
-      end do
-
-      do i = 1, 3
-         if (from1) then
-            zstS(i) = (zstart(i) + skip(i) - 1)/skip(i)
-            if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstS(i) = zstS(i) + 1
-            zenS(i) = (zend(i) + skip(i) - 1)/skip(i)
-         else
-            zstS(i) = zstart(i)/skip(i)
-            if (mod(zstart(i), skip(i)) /= 0) zstS(i) = zstS(i) + 1
-            zenS(i) = zend(i)/skip(i)
-         end if
-         zszS(i) = zenS(i) - zstS(i) + 1
-      end do
-
-      return
-   end subroutine init_coarser_mesh_statS
+  subroutine init_coarser_mesh_statS(i_skip,j_skip,k_skip,from1)
+
+    implicit none
+
+    integer, intent(IN) :: i_skip,j_skip,k_skip
+    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+    ! .false. - save n,2n,3n...
+
+    integer, dimension(3) :: skip
+    integer :: i
+
+    coarse_mesh_starts_from_1 = from1
+    iskipS = i_skip
+    jskipS = j_skip
+    kskipS = k_skip
+
+    skip(1)=iskipS
+    skip(2)=jskipS
+    skip(3)=kskipS
+
+    do i=1,3
+       if (from1) then
+          xstS(i) = (xstart(i)+skip(i)-1)/skip(i)
+          if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xstS(i)=xstS(i)+1
+          xenS(i) = (xend(i)+skip(i)-1)/skip(i)
+       else
+          xstS(i) = xstart(i)/skip(i)
+          if (mod(xstart(i),skip(i))/=0) xstS(i)=xstS(i)+1
+          xenS(i) = xend(i)/skip(i)
+       end if
+       xszS(i) = xenS(i)-xstS(i)+1
+    end do
+
+    do i=1,3
+       if (from1) then
+          ystS(i) = (ystart(i)+skip(i)-1)/skip(i)
+          if (mod(ystart(i)+skip(i)-1,skip(i))/=0) ystS(i)=ystS(i)+1
+          yenS(i) = (yend(i)+skip(i)-1)/skip(i)
+       else
+          ystS(i) = ystart(i)/skip(i)
+          if (mod(ystart(i),skip(i))/=0) ystS(i)=ystS(i)+1
+          yenS(i) = yend(i)/skip(i)
+       end if
+       yszS(i) = yenS(i)-ystS(i)+1
+    end do
+
+    do i=1,3
+       if (from1) then
+          zstS(i) = (zstart(i)+skip(i)-1)/skip(i)
+          if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zstS(i)=zstS(i)+1
+          zenS(i) = (zend(i)+skip(i)-1)/skip(i)
+       else
+          zstS(i) = zstart(i)/skip(i)
+          if (mod(zstart(i),skip(i))/=0) zstS(i)=zstS(i)+1
+          zenS(i) = zend(i)/skip(i)
+       end if
+       zszS(i) = zenS(i)-zstS(i)+1
+    end do
+
+    return
+  end subroutine init_coarser_mesh_statS
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Coarser mesh support for visualization
+  ! Coarser mesh support for visualization
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine init_coarser_mesh_statV(i_skip, j_skip, k_skip, from1)
-
-      implicit none
-
-      integer, intent(IN) :: i_skip, j_skip, k_skip
-      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-      ! .false. - save n,2n,3n...
-
-      integer, dimension(3) :: skip
-      integer :: i
-
-      coarse_mesh_starts_from_1 = from1
-      iskipV = i_skip
-      jskipV = j_skip
-      kskipV = k_skip
-
-      skip(1) = iskipV
-      skip(2) = jskipV
-      skip(3) = kskipV
-
-      do i = 1, 3
-         if (from1) then
-            xstV(i) = (xstart(i) + skip(i) - 1)/skip(i)
-            if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstV(i) = xstV(i) + 1
-            xenV(i) = (xend(i) + skip(i) - 1)/skip(i)
-         else
-            xstV(i) = xstart(i)/skip(i)
-            if (mod(xstart(i), skip(i)) /= 0) xstV(i) = xstV(i) + 1
-            xenV(i) = xend(i)/skip(i)
-         end if
-         xszV(i) = xenV(i) - xstV(i) + 1
-      end do
-
-      do i = 1, 3
-         if (from1) then
-            ystV(i) = (ystart(i) + skip(i) - 1)/skip(i)
-            if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystV(i) = ystV(i) + 1
-            yenV(i) = (yend(i) + skip(i) - 1)/skip(i)
-         else
-            ystV(i) = ystart(i)/skip(i)
-            if (mod(ystart(i), skip(i)) /= 0) ystV(i) = ystV(i) + 1
-            yenV(i) = yend(i)/skip(i)
-         end if
-         yszV(i) = yenV(i) - ystV(i) + 1
-      end do
-
-      do i = 1, 3
-         if (from1) then
-            zstV(i) = (zstart(i) + skip(i) - 1)/skip(i)
-            if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstV(i) = zstV(i) + 1
-            zenV(i) = (zend(i) + skip(i) - 1)/skip(i)
-         else
-            zstV(i) = zstart(i)/skip(i)
-            if (mod(zstart(i), skip(i)) /= 0) zstV(i) = zstV(i) + 1
-            zenV(i) = zend(i)/skip(i)
-         end if
-         zszV(i) = zenV(i) - zstV(i) + 1
-      end do
-
-      return
-   end subroutine init_coarser_mesh_statV
+  subroutine init_coarser_mesh_statV(i_skip,j_skip,k_skip,from1)
+
+    implicit none
+
+    integer, intent(IN) :: i_skip,j_skip,k_skip
+    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+    ! .false. - save n,2n,3n...
+
+    integer, dimension(3) :: skip
+    integer :: i
+
+    coarse_mesh_starts_from_1 = from1
+    iskipV = i_skip
+    jskipV = j_skip
+    kskipV = k_skip
+
+    skip(1)=iskipV
+    skip(2)=jskipV
+    skip(3)=kskipV
+
+    do i=1,3
+       if (from1) then
+          xstV(i) = (xstart(i)+skip(i)-1)/skip(i)
+          if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xstV(i)=xstV(i)+1
+          xenV(i) = (xend(i)+skip(i)-1)/skip(i)
+       else
+          xstV(i) = xstart(i)/skip(i)
+          if (mod(xstart(i),skip(i))/=0) xstV(i)=xstV(i)+1
+          xenV(i) = xend(i)/skip(i)
+       end if
+       xszV(i) = xenV(i)-xstV(i)+1
+    end do
+
+    do i=1,3
+       if (from1) then
+          ystV(i) = (ystart(i)+skip(i)-1)/skip(i)
+          if (mod(ystart(i)+skip(i)-1,skip(i))/=0) ystV(i)=ystV(i)+1
+          yenV(i) = (yend(i)+skip(i)-1)/skip(i)
+       else
+          ystV(i) = ystart(i)/skip(i)
+          if (mod(ystart(i),skip(i))/=0) ystV(i)=ystV(i)+1
+          yenV(i) = yend(i)/skip(i)
+       end if
+       yszV(i) = yenV(i)-ystV(i)+1
+    end do
+
+    do i=1,3
+       if (from1) then
+          zstV(i) = (zstart(i)+skip(i)-1)/skip(i)
+          if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zstV(i)=zstV(i)+1
+          zenV(i) = (zend(i)+skip(i)-1)/skip(i)
+       else
+          zstV(i) = zstart(i)/skip(i)
+          if (mod(zstart(i),skip(i))/=0) zstV(i)=zstV(i)+1
+          zenV(i) = zend(i)/skip(i)
+       end if
+       zszV(i) = zenV(i)-zstV(i)+1
+    end do
+
+    return
+  end subroutine init_coarser_mesh_statV
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Coarser mesh support for probe
+  ! Coarser mesh support for probe
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine init_coarser_mesh_statP(i_skip, j_skip, k_skip, from1)
-
-      implicit none
-
-      integer, intent(IN) :: i_skip, j_skip, k_skip
-      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-      ! .false. - save n,2n,3n...
-
-      integer, dimension(3) :: skip
-      integer :: i
-
-      coarse_mesh_starts_from_1 = from1
-      iskipP = i_skip
-      jskipP = j_skip
-      kskipP = k_skip
-
-      skip(1) = iskipP
-      skip(2) = jskipP
-      skip(3) = kskipP
-
-      do i = 1, 3
-         if (from1) then
-            xstP(i) = (xstart(i) + skip(i) - 1)/skip(i)
-            if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstP(i) = xstP(i) + 1
-            xenP(i) = (xend(i) + skip(i) - 1)/skip(i)
-         else
-            xstP(i) = xstart(i)/skip(i)
-            if (mod(xstart(i), skip(i)) /= 0) xstP(i) = xstP(i) + 1
-            xenP(i) = xend(i)/skip(i)
-         end if
-         xszP(i) = xenP(i) - xstP(i) + 1
-      end do
-
-      do i = 1, 3
-         if (from1) then
-            ystP(i) = (ystart(i) + skip(i) - 1)/skip(i)
-            if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystP(i) = ystP(i) + 1
-            yenP(i) = (yend(i) + skip(i) - 1)/skip(i)
-         else
-            ystP(i) = ystart(i)/skip(i)
-            if (mod(ystart(i), skip(i)) /= 0) ystP(i) = ystP(i) + 1
-            yenP(i) = yend(i)/skip(i)
-         end if
-         yszP(i) = yenP(i) - ystP(i) + 1
-      end do
-
-      do i = 1, 3
-         if (from1) then
-            zstP(i) = (zstart(i) + skip(i) - 1)/skip(i)
-            if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstP(i) = zstP(i) + 1
-            zenP(i) = (zend(i) + skip(i) - 1)/skip(i)
-         else
-            zstP(i) = zstart(i)/skip(i)
-            if (mod(zstart(i), skip(i)) /= 0) zstP(i) = zstP(i) + 1
-            zenP(i) = zend(i)/skip(i)
-         end if
-         zszP(i) = zenP(i) - zstP(i) + 1
-      end do
-
-      return
-   end subroutine init_coarser_mesh_statP
-
-   ! Copy data from a fine-resolution array to a coarse one for statistic
-   subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
-
-      implicit none
-
-      real(mytype), dimension(:, :, :) :: var_fine
-      real(mytype), dimension(:, :, :) :: var_coarse
-      integer, intent(IN) :: ipencil
-
-      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
-      integer :: i, j, k
-
-      if (ipencil == 1) then
-         allocate (wk(xstS(1):xenS(1), xstS(2):xenS(2), xstS(3):xenS(3)))
-         allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-         wk2 = var_fine
-         if (coarse_mesh_starts_from_1) then
-            do k = xstS(3), xenS(3)
-               do j = xstS(2), xenS(2)
-                  do i = xstS(1), xenS(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
-                  end do
-               end do
-            end do
-         else
-            do k = xstS(3), xenS(3)
-               do j = xstS(2), xenS(2)
-                  do i = xstS(1), xenS(1)
-                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
-                  end do
-               end do
-            end do
-         end if
-         var_coarse = wk
-      else if (ipencil == 2) then
-         allocate (wk(ystS(1):yenS(1), ystS(2):yenS(2), ystS(3):yenS(3)))
-         allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-         wk2 = var_fine
-         if (coarse_mesh_starts_from_1) then
-            do k = ystS(3), yenS(3)
-               do j = ystS(2), yenS(2)
-                  do i = ystS(1), yenS(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
-                  end do
-               end do
-            end do
-         else
-            do k = ystS(3), yenS(3)
-               do j = ystS(2), yenS(2)
-                  do i = ystS(1), yenS(1)
-                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
-                  end do
-               end do
-            end do
-         end if
-         var_coarse = wk
-      else if (ipencil == 3) then
-         allocate (wk(zstS(1):zenS(1), zstS(2):zenS(2), zstS(3):zenS(3)))
-         allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-         wk2 = var_fine
-         if (coarse_mesh_starts_from_1) then
-            do k = zstS(3), zenS(3)
-               do j = zstS(2), zenS(2)
-                  do i = zstS(1), zenS(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
-                  end do
-               end do
-            end do
-         else
-            do k = zstS(3), zenS(3)
-               do j = zstS(2), zenS(2)
-                  do i = zstS(1), zenS(1)
-                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
-                  end do
-               end do
-            end do
-         end if
-         var_coarse = wk
-      end if
-
-      deallocate (wk, wk2)
-
-      return
-   end subroutine fine_to_coarseS
-
-   ! Copy data from a fine-resolution array to a coarse one for visualization
-   subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
-
-      implicit none
-
-      real(mytype), dimension(:, :, :) :: var_fine
-      real(mytype), dimension(:, :, :) :: var_coarse
-      integer, intent(IN) :: ipencil
-
-      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
-      integer :: i, j, k
-
-      if (ipencil == 1) then
-         allocate (wk(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
-         allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-         wk2 = var_fine
-         if (coarse_mesh_starts_from_1) then
-            do k = xstV(3), xenV(3)
-               do j = xstV(2), xenV(2)
-                  do i = xstV(1), xenV(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
-                  end do
-               end do
-            end do
-         else
-            do k = xstV(3), xenV(3)
-               do j = xstV(2), xenV(2)
-                  do i = xstV(1), xenV(1)
-                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
-                  end do
-               end do
-            end do
-         end if
-         var_coarse = wk
-      else if (ipencil == 2) then
-         allocate (wk(ystV(1):yenV(1), ystV(2):yenV(2), ystV(3):yenV(3)))
-         allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-         wk2 = var_fine
-         if (coarse_mesh_starts_from_1) then
-            do k = ystV(3), yenV(3)
-               do j = ystV(2), yenV(2)
-                  do i = ystV(1), yenV(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
-                  end do
-               end do
-            end do
-         else
-            do k = ystV(3), yenV(3)
-               do j = ystV(2), yenV(2)
-                  do i = ystV(1), yenV(1)
-                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
-                  end do
-               end do
-            end do
-         end if
-         var_coarse = wk
-      else if (ipencil == 3) then
-         allocate (wk(zstV(1):zenV(1), zstV(2):zenV(2), zstV(3):zenV(3)))
-         allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-         wk2 = var_fine
-         if (coarse_mesh_starts_from_1) then
-            do k = zstV(3), zenV(3)
-               do j = zstV(2), zenV(2)
-                  do i = zstV(1), zenV(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
-                  end do
-               end do
-            end do
-         else
-            do k = zstV(3), zenV(3)
-               do j = zstV(2), zenV(2)
-                  do i = zstV(1), zenV(1)
-                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
-                  end do
-               end do
-            end do
-         end if
-         var_coarse = wk
-      end if
-
-      deallocate (wk, wk2)
-
-      return
-   end subroutine fine_to_coarseV
-
-   ! Copy data from a fine-resolution array to a coarse one for probe
-   subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
-
-      implicit none
-
-      real(mytype), dimension(:, :, :) :: var_fine
-      real(mytype), dimension(:, :, :) :: var_coarse
-      integer, intent(IN) :: ipencil
-
-      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
-      integer :: i, j, k
-
-      if (ipencil == 1) then
-         allocate (wk(xstP(1):xenP(1), xstP(2):xenP(2), xstP(3):xenP(3)))
-         allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-         wk2 = var_fine
-         if (coarse_mesh_starts_from_1) then
-            do k = xstP(3), xenP(3)
-               do j = xstP(2), xenP(2)
-                  do i = xstP(1), xenP(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
-                  end do
-               end do
-            end do
-         else
-            do k = xstP(3), xenP(3)
-               do j = xstP(2), xenP(2)
-                  do i = xstP(1), xenP(1)
-                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
-                  end do
-               end do
-            end do
-         end if
-         var_coarse = wk
-      else if (ipencil == 2) then
-         allocate (wk(ystP(1):yenP(1), ystP(2):yenP(2), ystP(3):yenP(3)))
-         allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-         wk2 = var_fine
-         if (coarse_mesh_starts_from_1) then
-            do k = ystP(3), yenP(3)
-               do j = ystP(2), yenP(2)
-                  do i = ystP(1), yenP(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
-                  end do
-               end do
-            end do
-         else
-            do k = ystP(3), yenP(3)
-               do j = ystP(2), yenP(2)
-                  do i = ystP(1), yenP(1)
-                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
-                  end do
-               end do
-            end do
-         end if
-         var_coarse = wk
-      else if (ipencil == 3) then
-         allocate (wk(zstP(1):zenP(1), zstP(2):zenP(2), zstP(3):zenP(3)))
-         allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-         wk2 = var_fine
-         if (coarse_mesh_starts_from_1) then
-            do k = zstP(3), zenP(3)
-               do j = zstP(2), zenP(2)
-                  do i = zstP(1), zenP(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
-                  end do
-               end do
-            end do
-         else
-            do k = zstP(3), zenP(3)
-               do j = zstP(2), zenP(2)
-                  do i = zstP(1), zenP(1)
-                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
-                  end do
-               end do
-            end do
-         end if
-         var_coarse = wk
-      end if
-
-      deallocate (wk, wk2)
-
-      return
-   end subroutine fine_to_coarseP
+  subroutine init_coarser_mesh_statP(i_skip,j_skip,k_skip,from1)
+
+    implicit none
+
+    integer, intent(IN) :: i_skip,j_skip,k_skip
+    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+    ! .false. - save n,2n,3n...
+
+    integer, dimension(3) :: skip
+    integer :: i
+
+    coarse_mesh_starts_from_1 = from1
+    iskipP = i_skip
+    jskipP = j_skip
+    kskipP = k_skip
+
+    skip(1)=iskipP
+    skip(2)=jskipP
+    skip(3)=kskipP
+
+    do i=1,3
+       if (from1) then
+          xstP(i) = (xstart(i)+skip(i)-1)/skip(i)
+          if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xstP(i)=xstP(i)+1
+          xenP(i) = (xend(i)+skip(i)-1)/skip(i)
+       else
+          xstP(i) = xstart(i)/skip(i)
+          if (mod(xstart(i),skip(i))/=0) xstP(i)=xstP(i)+1
+          xenP(i) = xend(i)/skip(i)
+       end if
+       xszP(i) = xenP(i)-xstP(i)+1
+    end do
+
+    do i=1,3
+       if (from1) then
+          ystP(i) = (ystart(i)+skip(i)-1)/skip(i)
+          if (mod(ystart(i)+skip(i)-1,skip(i))/=0) ystP(i)=ystP(i)+1
+          yenP(i) = (yend(i)+skip(i)-1)/skip(i)
+       else
+          ystP(i) = ystart(i)/skip(i)
+          if (mod(ystart(i),skip(i))/=0) ystP(i)=ystP(i)+1
+          yenP(i) = yend(i)/skip(i)
+       end if
+       yszP(i) = yenP(i)-ystP(i)+1
+    end do
+
+    do i=1,3
+       if (from1) then
+          zstP(i) = (zstart(i)+skip(i)-1)/skip(i)
+          if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zstP(i)=zstP(i)+1
+          zenP(i) = (zend(i)+skip(i)-1)/skip(i)
+       else
+          zstP(i) = zstart(i)/skip(i)
+          if (mod(zstart(i),skip(i))/=0) zstP(i)=zstP(i)+1
+          zenP(i) = zend(i)/skip(i)
+       end if
+       zszP(i) = zenP(i)-zstP(i)+1
+    end do
+
+    return
+  end subroutine init_coarser_mesh_statP
+
+  ! Copy data from a fine-resolution array to a coarse one for statistic
+  subroutine fine_to_coarseS(ipencil,var_fine,var_coarse)
+
+    implicit none
+
+    real(mytype), dimension(:,:,:) :: var_fine
+    real(mytype), dimension(:,:,:) :: var_coarse
+    integer, intent(IN) :: ipencil
+
+    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
+    integer :: i,j,k
+
+    if (ipencil==1) then
+       allocate(wk(xstS(1):xenS(1),xstS(2):xenS(2),xstS(3):xenS(3)))
+       allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
+       wk2=var_fine
+       if (coarse_mesh_starts_from_1) then
+          do k=xstS(3),xenS(3)
+             do j=xstS(2),xenS(2)
+                do i=xstS(1),xenS(1)
+                   wk(i,j,k) = wk2((i-1)*iskipS+1,(j-1)*jskipS+1,(k-1)*kskipS+1)
+                end do
+             end do
+          end do
+       else
+          do k=xstS(3),xenS(3)
+             do j=xstS(2),xenS(2)
+                do i=xstS(1),xenS(1)
+                   wk(i,j,k) = wk2(i*iskipS,j*jskipS,k*kskipS)
+                end do
+             end do
+          end do
+       end if
+       var_coarse=wk
+    else if (ipencil==2) then
+       allocate(wk(ystS(1):yenS(1),ystS(2):yenS(2),ystS(3):yenS(3)))
+       allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
+       wk2=var_fine
+       if (coarse_mesh_starts_from_1) then
+          do k=ystS(3),yenS(3)
+             do j=ystS(2),yenS(2)
+                do i=ystS(1),yenS(1)
+                   wk(i,j,k) = wk2((i-1)*iskipS+1,(j-1)*jskipS+1,(k-1)*kskipS+1)
+                end do
+             end do
+          end do
+       else
+          do k=ystS(3),yenS(3)
+             do j=ystS(2),yenS(2)
+                do i=ystS(1),yenS(1)
+                   wk(i,j,k) = wk2(i*iskipS,j*jskipS,k*kskipS)
+                end do
+             end do
+          end do
+       end if
+       var_coarse=wk
+    else if (ipencil==3) then
+       allocate(wk(zstS(1):zenS(1),zstS(2):zenS(2),zstS(3):zenS(3)))
+       allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
+       wk2=var_fine
+       if (coarse_mesh_starts_from_1) then
+          do k=zstS(3),zenS(3)
+             do j=zstS(2),zenS(2)
+                do i=zstS(1),zenS(1)
+                   wk(i,j,k) = wk2((i-1)*iskipS+1,(j-1)*jskipS+1,(k-1)*kskipS+1)
+                end do
+             end do
+          end do
+       else
+          do k=zstS(3),zenS(3)
+             do j=zstS(2),zenS(2)
+                do i=zstS(1),zenS(1)
+                   wk(i,j,k) = wk2(i*iskipS,j*jskipS,k*kskipS)
+                end do
+             end do
+          end do
+       end if
+       var_coarse=wk
+    end if
+
+    deallocate(wk,wk2)
+
+    return
+  end subroutine fine_to_coarseS
+
+  ! Copy data from a fine-resolution array to a coarse one for visualization
+  subroutine fine_to_coarseV(ipencil,var_fine,var_coarse)
+
+    implicit none
+
+    real(mytype), dimension(:,:,:) :: var_fine
+    real(mytype), dimension(:,:,:) :: var_coarse
+    integer, intent(IN) :: ipencil
+
+    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
+    integer :: i,j,k
+
+    if (ipencil==1) then
+       allocate(wk(xstV(1):xenV(1),xstV(2):xenV(2),xstV(3):xenV(3)))
+       allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
+       wk2=var_fine
+       if (coarse_mesh_starts_from_1) then
+          do k=xstV(3),xenV(3)
+             do j=xstV(2),xenV(2)
+                do i=xstV(1),xenV(1)
+                   wk(i,j,k) = wk2((i-1)*iskipV+1,(j-1)*jskipV+1,(k-1)*kskipV+1)
+                end do
+             end do
+          end do
+       else
+          do k=xstV(3),xenV(3)
+             do j=xstV(2),xenV(2)
+                do i=xstV(1),xenV(1)
+                   wk(i,j,k) = wk2(i*iskipV,j*jskipV,k*kskipV)
+                end do
+             end do
+          end do
+       end if
+       var_coarse=wk
+    else if (ipencil==2) then
+       allocate(wk(ystV(1):yenV(1),ystV(2):yenV(2),ystV(3):yenV(3)))
+       allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
+       wk2=var_fine
+       if (coarse_mesh_starts_from_1) then
+          do k=ystV(3),yenV(3)
+             do j=ystV(2),yenV(2)
+                do i=ystV(1),yenV(1)
+                   wk(i,j,k) = wk2((i-1)*iskipV+1,(j-1)*jskipV+1,(k-1)*kskipV+1)
+                end do
+             end do
+          end do
+       else
+          do k=ystV(3),yenV(3)
+             do j=ystV(2),yenV(2)
+                do i=ystV(1),yenV(1)
+                   wk(i,j,k) = wk2(i*iskipV,j*jskipV,k*kskipV)
+                end do
+             end do
+          end do
+       end if
+       var_coarse=wk
+    else if (ipencil==3) then
+       allocate(wk(zstV(1):zenV(1),zstV(2):zenV(2),zstV(3):zenV(3)))
+       allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
+       wk2=var_fine
+       if (coarse_mesh_starts_from_1) then
+          do k=zstV(3),zenV(3)
+             do j=zstV(2),zenV(2)
+                do i=zstV(1),zenV(1)
+                   wk(i,j,k) = wk2((i-1)*iskipV+1,(j-1)*jskipV+1,(k-1)*kskipV+1)
+                end do
+             end do
+          end do
+       else
+          do k=zstV(3),zenV(3)
+             do j=zstV(2),zenV(2)
+                do i=zstV(1),zenV(1)
+                   wk(i,j,k) = wk2(i*iskipV,j*jskipV,k*kskipV)
+                end do
+             end do
+          end do
+       end if
+       var_coarse=wk
+    end if
+
+    deallocate(wk,wk2)
+
+    return
+  end subroutine fine_to_coarseV
+
+  ! Copy data from a fine-resolution array to a coarse one for probe
+  subroutine fine_to_coarseP(ipencil,var_fine,var_coarse)
+
+    implicit none
+
+    real(mytype), dimension(:,:,:) :: var_fine
+    real(mytype), dimension(:,:,:) :: var_coarse
+    integer, intent(IN) :: ipencil
+
+    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
+    integer :: i,j,k
+
+    if (ipencil==1) then
+       allocate(wk(xstP(1):xenP(1),xstP(2):xenP(2),xstP(3):xenP(3)))
+       allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
+       wk2=var_fine
+       if (coarse_mesh_starts_from_1) then
+          do k=xstP(3),xenP(3)
+             do j=xstP(2),xenP(2)
+                do i=xstP(1),xenP(1)
+                   wk(i,j,k) = wk2((i-1)*iskipP+1,(j-1)*jskipP+1,(k-1)*kskipP+1)
+                end do
+             end do
+          end do
+       else
+          do k=xstP(3),xenP(3)
+             do j=xstP(2),xenP(2)
+                do i=xstP(1),xenP(1)
+                   wk(i,j,k) = wk2(i*iskipP,j*jskipP,k*kskipP)
+                end do
+             end do
+          end do
+       end if
+       var_coarse=wk
+    else if (ipencil==2) then
+       allocate(wk(ystP(1):yenP(1),ystP(2):yenP(2),ystP(3):yenP(3)))
+       allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
+       wk2=var_fine
+       if (coarse_mesh_starts_from_1) then
+          do k=ystP(3),yenP(3)
+             do j=ystP(2),yenP(2)
+                do i=ystP(1),yenP(1)
+                   wk(i,j,k) = wk2((i-1)*iskipP+1,(j-1)*jskipP+1,(k-1)*kskipP+1)
+                end do
+             end do
+          end do
+       else
+          do k=ystP(3),yenP(3)
+             do j=ystP(2),yenP(2)
+                do i=ystP(1),yenP(1)
+                   wk(i,j,k) = wk2(i*iskipP,j*jskipP,k*kskipP)
+                end do
+             end do
+          end do
+       end if
+       var_coarse=wk
+    else if (ipencil==3) then
+       allocate(wk(zstP(1):zenP(1),zstP(2):zenP(2),zstP(3):zenP(3)))
+       allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
+       wk2=var_fine
+       if (coarse_mesh_starts_from_1) then
+          do k=zstP(3),zenP(3)
+             do j=zstP(2),zenP(2)
+                do i=zstP(1),zenP(1)
+                   wk(i,j,k) = wk2((i-1)*iskipP+1,(j-1)*jskipP+1,(k-1)*kskipP+1)
+                end do
+             end do
+          end do
+       else
+          do k=zstP(3),zenP(3)
+             do j=zstP(2),zenP(2)
+                do i=zstP(1),zenP(1)
+                   wk(i,j,k) = wk2(i*iskipP,j*jskipP,k*kskipP)
+                end do
+             end do
+          end do
+       end if
+       var_coarse=wk
+    end if
+
+    deallocate(wk,wk2)
+
+    return
+  end subroutine fine_to_coarseP
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Find sub-domain information held by current processor
-   !   INPUT:
-   !     nx, ny, nz - global data dimension
-   !     pdim(3)    - number of processor grid in each dimension,
-   !                  valid values: 1 - distibute locally;
-   !                                2 - distribute across p_row;
-   !                                3 - distribute across p_col
-   !   OUTPUT:
-   !     lstart(3)  - starting index
-   !     lend(3)    - ending index
-   !     lsize(3)   - size of the sub-block (redundant)
+  ! Find sub-domain information held by current processor
+  !   INPUT: 
+  !     nx, ny, nz - global data dimension
+  !     pdim(3)    - number of processor grid in each dimension, 
+  !                  valid values: 1 - distibute locally; 
+  !                                2 - distribute across p_row; 
+  !                                3 - distribute across p_col
+  !   OUTPUT:
+  !     lstart(3)  - starting index
+  !     lend(3)    - ending index
+  !     lsize(3)   - size of the sub-block (redundant) 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine partition(nx, ny, nz, pdim, lstart, lend, lsize)
-
-      implicit none
-
-      integer, intent(IN) :: nx, ny, nz
-      integer, dimension(3), intent(IN) :: pdim
-      integer, dimension(3), intent(OUT) :: lstart, lend, lsize
-
-      integer, allocatable, dimension(:) :: st, en, sz
-      integer :: i, gsize
-
-      do i = 1, 3
-
-         if (i == 1) then
-            gsize = nx
-         else if (i == 2) then
-            gsize = ny
-         else if (i == 3) then
-            gsize = nz
-         end if
-
-         if (pdim(i) == 1) then        ! all local
-            lstart(i) = 1
-            lend(i) = gsize
-            lsize(i) = gsize
-         elseif (pdim(i) == 2) then    ! distribute across dims(1)
-            allocate (st(0:dims(1) - 1))
-            allocate (en(0:dims(1) - 1))
-            allocate (sz(0:dims(1) - 1))
-            call distribute(gsize, dims(1), st, en, sz)
-            lstart(i) = st(coord(1))
-            lend(i) = en(coord(1))
-            lsize(i) = sz(coord(1))
-            deallocate (st, en, sz)
-         elseif (pdim(i) == 3) then    ! distribute across dims(2)
-            allocate (st(0:dims(2) - 1))
-            allocate (en(0:dims(2) - 1))
-            allocate (sz(0:dims(2) - 1))
-            call distribute(gsize, dims(2), st, en, sz)
-            lstart(i) = st(coord(2))
-            lend(i) = en(coord(2))
-            lsize(i) = sz(coord(2))
-            deallocate (st, en, sz)
-         end if
-
-      end do
-      return
-
-   end subroutine partition
+  subroutine partition(nx, ny, nz, pdim, lstart, lend, lsize)
+
+    implicit none
+
+    integer, intent(IN) :: nx, ny, nz
+    integer, dimension(3), intent(IN) :: pdim
+    integer, dimension(3), intent(OUT) :: lstart, lend, lsize
+
+    integer, allocatable, dimension(:) :: st,en,sz
+    integer :: i, gsize
+
+    do i = 1, 3
+
+       if (i==1) then
+          gsize = nx
+       else if (i==2) then
+          gsize = ny
+       else if (i==3) then
+          gsize = nz
+       end if
+
+       if (pdim(i) == 1) then        ! all local
+          lstart(i) = 1
+          lend(i)   = gsize
+          lsize(i)  = gsize
+       elseif (pdim(i) == 2) then    ! distribute across dims(1)
+          allocate(st(0:dims(1)-1))
+          allocate(en(0:dims(1)-1))
+          allocate(sz(0:dims(1)-1))
+          call distribute(gsize,dims(1),st,en,sz)
+          lstart(i) = st(coord(1))
+          lend(i)   = en(coord(1))
+          lsize(i)  = sz(coord(1))
+          deallocate(st,en,sz)
+       elseif (pdim(i) == 3) then    ! distribute across dims(2)
+          allocate(st(0:dims(2)-1))
+          allocate(en(0:dims(2)-1))
+          allocate(sz(0:dims(2)-1))
+          call distribute(gsize,dims(2),st,en,sz)
+          lstart(i) = st(coord(2))
+          lend(i)   = en(coord(2))
+          lsize(i)  = sz(coord(2))
+          deallocate(st,en,sz)
+       end if
+
+    end do
+    return   
+
+  end subroutine partition
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !   - distibutes grid points in one dimension
-   !   - handles uneven distribution properly
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine distribute(data1, proc, st, en, sz)
-
-      implicit none
-      ! data1 -- data size in any dimension to be partitioned
-      ! proc  -- number of processors in that dimension
-      ! st    -- array of starting index
-      ! en    -- array of ending index
-      ! sz    -- array of local size  (redundent)
-      integer data1, proc, st(0:proc - 1), en(0:proc - 1), sz(0:proc - 1)
-      integer i, size1, nl, nu
-
-      size1 = data1/proc
-      nu = data1 - size1*proc
-      nl = proc - nu
-      st(0) = 1
-      sz(0) = size1
-      en(0) = size1
-      do i = 1, nl - 1
-         st(i) = st(i - 1) + size1
-         sz(i) = size1
-         en(i) = en(i - 1) + size1
-      end do
-      size1 = size1 + 1
-      do i = nl, proc - 1
-         st(i) = en(i - 1) + 1
-         sz(i) = size1
-         en(i) = en(i - 1) + size1
-      end do
-      en(proc - 1) = data1
-      sz(proc - 1) = data1 - st(proc - 1) + 1
-
-      return
-   end subroutine distribute
+  !   - distibutes grid points in one dimension
+  !   - handles uneven distribution properly 
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 
+  subroutine distribute(data1,proc,st,en,sz)
+
+    implicit none
+    ! data1 -- data size in any dimension to be partitioned
+    ! proc  -- number of processors in that dimension
+    ! st    -- array of starting index
+    ! en    -- array of ending index
+    ! sz    -- array of local size  (redundent)
+    integer data1,proc,st(0:proc-1),en(0:proc-1),sz(0:proc-1)
+    integer i,size1,nl,nu
+
+    size1=data1/proc
+    nu = data1 - size1 * proc
+    nl = proc - nu
+    st(0) = 1
+    sz(0) = size1
+    en(0) = size1
+    do i=1,nl-1
+       st(i) = st(i-1) + size1
+       sz(i) = size1
+       en(i) = en(i-1) + size1
+    end do
+    size1 = size1 + 1
+    do i=nl,proc-1
+       st(i) = en(i-1) + 1
+       sz(i) = size1
+       en(i) = en(i-1) + size1
+    end do
+    en(proc-1)= data1 
+    sz(proc-1)= data1-st(proc-1)+1
+
+    return
+  end subroutine distribute
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  Define how each dimension is distributed across processors
-   !    e.g. 17 meshes across 4 processor would be distibuted as (4,4,4,5)
-   !    such global information is required locally at MPI_ALLTOALLV time
+  !  Define how each dimension is distributed across processors
+  !    e.g. 17 meshes across 4 processor would be distibuted as (4,4,4,5)
+  !    such global information is required locally at MPI_ALLTOALLV time
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine get_dist(nx, ny, nz, decomp)
+  subroutine get_dist(nx,ny,nz,decomp)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: nx, ny, nz
-      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-      integer, allocatable, dimension(:) :: st, en
+    integer, intent(IN) :: nx, ny, nz
+    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+    integer, allocatable, dimension(:) :: st,en
 
-      allocate (st(0:dims(1) - 1))
-      allocate (en(0:dims(1) - 1))
-      call distribute(nx, dims(1), st, en, decomp%x1dist)
-      call distribute(ny, dims(1), st, en, decomp%y1dist)
-      deallocate (st, en)
+    allocate(st(0:dims(1)-1))
+    allocate(en(0:dims(1)-1))
+    call distribute(nx,dims(1),st,en,decomp%x1dist)
+    call distribute(ny,dims(1),st,en,decomp%y1dist)
+    deallocate(st,en)
 
-      allocate (st(0:dims(2) - 1))
-      allocate (en(0:dims(2) - 1))
-      call distribute(ny, dims(2), st, en, decomp%y2dist)
-      call distribute(nz, dims(2), st, en, decomp%z2dist)
-      deallocate (st, en)
+    allocate(st(0:dims(2)-1))
+    allocate(en(0:dims(2)-1))
+    call distribute(ny,dims(2),st,en,decomp%y2dist)
+    call distribute(nz,dims(2),st,en,decomp%z2dist)
+    deallocate(st,en)
 
-      return
-   end subroutine get_dist
+    return
+  end subroutine get_dist
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Prepare the send / receive buffers for MPI_ALLTOALLV communications
+  ! Prepare the send / receive buffers for MPI_ALLTOALLV communications
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine prepare_buffer(decomp)
-
-      implicit none
-
-      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-
-      integer :: i
-
-      !LG : AJOUTS "bidons" pour eviter un plantage en -O3 avec gcc9.3
-      !       * la fonction sortait des valeurs 'aleatoires'
-      !         et le calcul plantait dans MPI_ALLTOALLV
-      !       * pas de plantage en O2
-
-      if (nrank == 0) then
-         open (newunit=i, file='temp.dat', form='unformatted')
-         write (i) decomp%x1dist, decomp%y1dist, decomp%y2dist, decomp%z2dist, &
-            decomp%xsz, decomp%ysz, decomp%zsz
-         close (i, status='delete')
-      end if
-
-      ! MPI_ALLTOALLV buffer information
-
-      do i = 0, dims(1) - 1
-         decomp%x1cnts(i) = decomp%x1dist(i)*decomp%xsz(2)*decomp%xsz(3)
-         decomp%y1cnts(i) = decomp%ysz(1)*decomp%y1dist(i)*decomp%ysz(3)
-         if (i == 0) then
-            decomp%x1disp(i) = 0  ! displacement is 0-based index
-            decomp%y1disp(i) = 0
-         else
-            decomp%x1disp(i) = decomp%x1disp(i - 1) + decomp%x1cnts(i - 1)
-            decomp%y1disp(i) = decomp%y1disp(i - 1) + decomp%y1cnts(i - 1)
-         end if
-      end do
-
-      do i = 0, dims(2) - 1
-         decomp%y2cnts(i) = decomp%ysz(1)*decomp%y2dist(i)*decomp%ysz(3)
-         decomp%z2cnts(i) = decomp%zsz(1)*decomp%zsz(2)*decomp%z2dist(i)
-         if (i == 0) then
-            decomp%y2disp(i) = 0  ! displacement is 0-based index
-            decomp%z2disp(i) = 0
-         else
-            decomp%y2disp(i) = decomp%y2disp(i - 1) + decomp%y2cnts(i - 1)
-            decomp%z2disp(i) = decomp%z2disp(i - 1) + decomp%z2cnts(i - 1)
-         end if
-      end do
-
-      ! MPI_ALLTOALL buffer information
-
-      ! For evenly distributed data, following is an easier implementation.
-      ! But it should be covered by the more general formulation below.
-      !decomp%x1count = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)/dims(1)
-      !decomp%y1count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(1)
-      !decomp%y2count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(2)
-      !decomp%z2count = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)/dims(2)
-
-      ! For unevenly distributed data, pad smaller messages. Note the
-      ! last blocks along pencils always get assigned more mesh points
-      ! for X <=> Y transposes
-      decomp%x1count = decomp%x1dist(dims(1) - 1)* &
-                       decomp%y1dist(dims(1) - 1)*decomp%xsz(3)
-      decomp%y1count = decomp%x1count
-      ! for Y <=> Z transposes
-      decomp%y2count = decomp%y2dist(dims(2) - 1)* &
-                       decomp%z2dist(dims(2) - 1)*decomp%zsz(1)
-      decomp%z2count = decomp%y2count
-
-      return
-   end subroutine prepare_buffer
+  subroutine prepare_buffer(decomp)
+
+    implicit none
+
+    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+
+    integer :: i
+
+    !LG : AJOUTS "bidons" pour eviter un plantage en -O3 avec gcc9.3
+    !       * la fonction sortait des valeurs 'aleatoires'
+    !         et le calcul plantait dans MPI_ALLTOALLV
+    !       * pas de plantage en O2
+    
+    if (nrank==0) then
+       open(newunit=i, file='temp.dat', form='unformatted')
+       write(i) decomp%x1dist,decomp%y1dist,decomp%y2dist,decomp%z2dist, &
+                decomp%xsz,decomp%ysz,decomp%zsz
+       close(i, status='delete')
+    endif
+
+    ! MPI_ALLTOALLV buffer information
+
+    do i=0, dims(1)-1
+       decomp%x1cnts(i) = decomp%x1dist(i)*decomp%xsz(2)*decomp%xsz(3)
+       decomp%y1cnts(i) = decomp%ysz(1)*decomp%y1dist(i)*decomp%ysz(3)
+       if (i==0) then
+          decomp%x1disp(i) = 0  ! displacement is 0-based index
+          decomp%y1disp(i) = 0
+       else
+          decomp%x1disp(i) = decomp%x1disp(i-1) + decomp%x1cnts(i-1)
+          decomp%y1disp(i) = decomp%y1disp(i-1) + decomp%y1cnts(i-1)
+       end if
+    end do
+
+    do i=0, dims(2)-1
+       decomp%y2cnts(i) = decomp%ysz(1)*decomp%y2dist(i)*decomp%ysz(3)
+       decomp%z2cnts(i) = decomp%zsz(1)*decomp%zsz(2)*decomp%z2dist(i)
+       if (i==0) then
+          decomp%y2disp(i) = 0  ! displacement is 0-based index
+          decomp%z2disp(i) = 0
+       else
+          decomp%y2disp(i) = decomp%y2disp(i-1) + decomp%y2cnts(i-1)
+          decomp%z2disp(i) = decomp%z2disp(i-1) + decomp%z2cnts(i-1)
+       end if
+    end do
+
+    ! MPI_ALLTOALL buffer information
+
+    ! For evenly distributed data, following is an easier implementation.
+    ! But it should be covered by the more general formulation below.
+    !decomp%x1count = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)/dims(1)
+    !decomp%y1count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(1) 
+    !decomp%y2count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(2)
+    !decomp%z2count = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)/dims(2)
+
+    ! For unevenly distributed data, pad smaller messages. Note the 
+    ! last blocks along pencils always get assigned more mesh points
+    ! for X <=> Y transposes
+    decomp%x1count = decomp%x1dist(dims(1)-1) * &
+         decomp%y1dist(dims(1)-1) * decomp%xsz(3)
+    decomp%y1count = decomp%x1count
+    ! for Y <=> Z transposes
+    decomp%y2count = decomp%y2dist(dims(2)-1) * &
+         decomp%z2dist(dims(2)-1) * decomp%zsz(1)
+    decomp%z2count = decomp%y2count
+
+    return
+  end subroutine prepare_buffer
 
 #ifdef SHM
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  Generate shared-memory information
+  !  Generate shared-memory information 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine decomp_info_init_shm(decomp)
+  subroutine decomp_info_init_shm(decomp)
 
-      implicit none
+    implicit none
 
-      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
 
-      ! a copy of old displacement array (will be overwritten by shm code)
-      allocate (decomp%x1disp_o(0:dims(1) - 1), decomp%y1disp_o(0:dims(1) - 1), &
-                decomp%y2disp_o(0:dims(2) - 1), decomp%z2disp_o(0:dims(2) - 1))
-      decomp%x1disp_o = decomp%x1disp
-      decomp%y1disp_o = decomp%y1disp
-      decomp%y2disp_o = decomp%y2disp
-      decomp%z2disp_o = decomp%z2disp
+    ! a copy of old displacement array (will be overwritten by shm code)
+    allocate(decomp%x1disp_o(0:dims(1)-1),decomp%y1disp_o(0:dims(1)-1), &
+         decomp%y2disp_o(0:dims(2)-1),decomp%z2disp_o(0:dims(2)-1))
+    decomp%x1disp_o = decomp%x1disp
+    decomp%y1disp_o = decomp%y1disp
+    decomp%y2disp_o = decomp%y2disp
+    decomp%z2disp_o = decomp%z2disp
 
-      call prepare_shared_buffer(decomp%ROW_INFO, DECOMP_2D_COMM_ROW, decomp)
-      call prepare_shared_buffer(decomp%COL_INFO, DECOMP_2D_COMM_COL, decomp)
+    call prepare_shared_buffer(decomp%ROW_INFO,DECOMP_2D_COMM_ROW,decomp)
+    call prepare_shared_buffer(decomp%COL_INFO,DECOMP_2D_COMM_COL,decomp)
+
+    return
+  end subroutine decomp_info_init_shm
 
-      return
-   end subroutine decomp_info_init_shm
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! For shared-memory implementation, prepare send/recv shared buffer
+  ! For shared-memory implementation, prepare send/recv shared buffer
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine prepare_shared_buffer(C, MPI_COMM, decomp)
-
-      implicit none
-
-      TYPE(SMP_INFO) :: C
-      INTEGER :: MPI_COMM
-      TYPE(DECOMP_INFO) :: decomp
-
-      INTEGER, ALLOCATABLE :: KTBL(:, :), NARY(:, :), KTBLALL(:, :)
-      INTEGER MYSMP, MYCORE, COLOR
-
-      integer :: ierror
-
-      C%MPI_COMM = MPI_COMM
-      CALL MPI_COMM_SIZE(MPI_COMM, C%NCPU, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-      CALL MPI_COMM_RANK(MPI_COMM, C%NODE_ME, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_RANK")
-      C%SMP_COMM = MPI_COMM_NULL
-      C%CORE_COMM = MPI_COMM_NULL
-      C%SMP_ME = 0
-      C%NCORE = 0
-      C%CORE_ME = 0
-      C%MAXCORE = 0
-      C%NSMP = 0
-      C%N_SND = 0
-      C%N_RCV = 0
-      C%SND_P = 0
-      C%RCV_P = 0
-      C%SND_P_c = 0
-      C%RCV_P_c = 0
-
-      ! get smp-node map for this communicator and set up smp communicators
-      CALL GET_SMP_MAP(C%MPI_COMM, C%NSMP, MYSMP, &
-                       C%NCORE, MYCORE, C%MAXCORE)
-      C%SMP_ME = MYSMP + 1
-      C%CORE_ME = MYCORE + 1
-      ! - set up inter/intra smp-node communicators
-      COLOR = MYCORE
-      IF (COLOR .GT. 0) COLOR = MPI_UNDEFINED
-      CALL MPI_Comm_split(C%MPI_COMM, COLOR, MYSMP, C%SMP_COMM, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-      CALL MPI_Comm_split(C%MPI_COMM, MYSMP, MYCORE, C%CORE_COMM, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-      ! - allocate work space
-      ALLOCATE (KTBL(C%MAXCORE, C%NSMP), NARY(C%NCPU, C%NCORE))
-      ALLOCATE (KTBLALL(C%MAXCORE, C%NSMP))
-      ! - set up smp-node/core to node_me lookup table
-      KTBL = 0
-      KTBL(C%CORE_ME, C%SMP_ME) = C%NODE_ME + 1
-      CALL MPI_ALLREDUCE(KTBL, KTBLALL, C%NSMP*C%MAXCORE, MPI_INTEGER, &
-                         MPI_SUM, MPI_COMM, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-      KTBL = KTBLALL
-      !  IF (SUM(KTBL) /= C%NCPU*(C%NCPU+1)/2) &
-      !       CALL MPI_ABORT(...
-
-      ! compute offsets in shared SNDBUF and RCVBUF
-      CALL MAPSET_SMPSHM(C, KTBL, NARY, decomp)
-
-      DEALLOCATE (KTBL, NARY)
-
-      return
-   end subroutine prepare_shared_buffer
+  subroutine prepare_shared_buffer(C,MPI_COMM,decomp)
+
+    implicit none
+
+    TYPE(SMP_INFO) :: C
+    INTEGER :: MPI_COMM
+    TYPE(DECOMP_INFO) :: decomp
+
+    INTEGER, ALLOCATABLE :: KTBL(:,:),NARY(:,:),KTBLALL(:,:)
+    INTEGER MYSMP, MYCORE, COLOR
+
+    integer :: ierror
+
+    C%MPI_COMM = MPI_COMM
+    CALL MPI_COMM_SIZE(MPI_COMM,C%NCPU,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+    CALL MPI_COMM_RANK(MPI_COMM,C%NODE_ME,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_RANK")
+    C%SMP_COMM  = MPI_COMM_NULL
+    C%CORE_COMM = MPI_COMM_NULL
+    C%SMP_ME= 0
+    C%NCORE = 0
+    C%CORE_ME = 0
+    C%MAXCORE = 0
+    C%NSMP  = 0
+    C%N_SND = 0
+    C%N_RCV = 0
+    C%SND_P = 0
+    C%RCV_P = 0
+    C%SND_P_c = 0
+    C%RCV_P_c = 0
+
+    ! get smp-node map for this communicator and set up smp communicators
+    CALL GET_SMP_MAP(C%MPI_COMM, C%NSMP, MYSMP, &
+         C%NCORE, MYCORE, C%MAXCORE)
+    C%SMP_ME = MYSMP + 1
+    C%CORE_ME = MYCORE + 1
+    ! - set up inter/intra smp-node communicators
+    COLOR = MYCORE
+    IF (COLOR.GT.0) COLOR = MPI_UNDEFINED
+    CALL MPI_Comm_split(C%MPI_COMM, COLOR, MYSMP, C%SMP_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+    CALL MPI_Comm_split(C%MPI_COMM, MYSMP, MYCORE, C%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+    ! - allocate work space
+    ALLOCATE(KTBL(C%MAXCORE,C%NSMP),NARY(C%NCPU,C%NCORE))
+    ALLOCATE(KTBLALL(C%MAXCORE,C%NSMP))
+    ! - set up smp-node/core to node_me lookup table
+    KTBL = 0
+    KTBL(C%CORE_ME,C%SMP_ME) = C%NODE_ME + 1
+    CALL MPI_ALLREDUCE(KTBL,KTBLALL,C%NSMP*C%MAXCORE,MPI_INTEGER, &
+         MPI_SUM,MPI_COMM,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+    KTBL=KTBLALL
+    !  IF (SUM(KTBL) /= C%NCPU*(C%NCPU+1)/2) &
+    !       CALL MPI_ABORT(...
+
+    ! compute offsets in shared SNDBUF and RCVBUF
+    CALL MAPSET_SMPSHM(C, KTBL, NARY, decomp)
+
+    DEALLOCATE(KTBL,NARY)
+
+    return
+  end subroutine prepare_shared_buffer
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Use Ian Bush's FreeIPC to generate shared-memory information
-   !  - system independent solution
-   !  - replacing David Tanqueray's implementation in alloc_shm.c
-   !    (old C code renamed to get_smp_map2)
+  ! Use Ian Bush's FreeIPC to generate shared-memory information
+  !  - system independent solution
+  !  - replacing David Tanqueray's implementation in alloc_shm.c
+  !    (old C code renamed to get_smp_map2)
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine get_smp_map(comm, nnodes, my_node, ncores, my_core, maxcor)
+  subroutine get_smp_map(comm, nnodes, my_node, ncores, my_core, maxcor)
+
+    use FIPC_module
 
-      use FIPC_module
+    implicit none
 
-      implicit none
+    integer, intent(IN) :: comm
+    integer, intent(OUT) :: nnodes, my_node, ncores, my_core, maxcor
 
-      integer, intent(IN) :: comm
-      integer, intent(OUT) :: nnodes, my_node, ncores, my_core, maxcor
+    integer :: intra_comm, extra_comm
+    integer :: ierror
 
-      integer :: intra_comm, extra_comm
-      integer :: ierror
+    call FIPC_init(comm, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_init")
 
-      call FIPC_init(comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_init")
+    ! intra_comm: communicator for processes on this shared memory node
+    ! extra_comm: communicator for all rank 0 on each shared memory node
+    call FIPC_ctxt_intra_comm(FIPC_ctxt_world, intra_comm, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_intra_comm")
+    call FIPC_ctxt_extra_comm(FIPC_ctxt_world, extra_comm, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_extra_comm")
 
-      ! intra_comm: communicator for processes on this shared memory node
-      ! extra_comm: communicator for all rank 0 on each shared memory node
-      call FIPC_ctxt_intra_comm(FIPC_ctxt_world, intra_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_intra_comm")
-      call FIPC_ctxt_extra_comm(FIPC_ctxt_world, extra_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_extra_comm")
+    call MPI_COMM_SIZE(intra_comm,  ncores, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+    call MPI_COMM_RANK(intra_comm, my_core, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
 
-      call MPI_COMM_SIZE(intra_comm, ncores, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-      call MPI_COMM_RANK(intra_comm, my_core, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+    ! only rank 0 on each shared memory node member of extra_comm
+    ! for others extra_comm = MPI_COMM_NULL
+    if (extra_comm /= MPI_COMM_NULL) then
+       call MPI_COMM_SIZE(extra_comm,  nnodes, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+       call MPI_COMM_RANK(extra_comm, my_node, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+    end if
 
-      ! only rank 0 on each shared memory node member of extra_comm
-      ! for others extra_comm = MPI_COMM_NULL
-      if (extra_comm /= MPI_COMM_NULL) then
-         call MPI_COMM_SIZE(extra_comm, nnodes, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-         call MPI_COMM_RANK(extra_comm, my_node, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-      end if
+    ! other ranks share the same information as their leaders
+    call MPI_BCAST( nnodes, 1, MPI_INTEGER, 0, intra_comm, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+    call MPI_BCAST(my_node, 1, MPI_INTEGER, 0, intra_comm, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
 
-      ! other ranks share the same information as their leaders
-      call MPI_BCAST(nnodes, 1, MPI_INTEGER, 0, intra_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
-      call MPI_BCAST(my_node, 1, MPI_INTEGER, 0, intra_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+    ! maxcor
+    call MPI_ALLREDUCE(ncores, maxcor, 1, MPI_INTEGER, MPI_MAX, &
+         decomp_2d_comm, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
 
-      ! maxcor
-      call MPI_ALLREDUCE(ncores, maxcor, 1, MPI_INTEGER, MPI_MAX, &
-                         decomp_2d_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+    call FIPC_finalize(ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_finalize")
 
-      call FIPC_finalize(ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_finalize")
+    return
 
-      return
+  end subroutine get_smp_map
 
-   end subroutine get_smp_map
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Set up smp-node based shared memory maps
+  ! Set up smp-node based shared memory maps
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   SUBROUTINE MAPSET_SMPSHM(C, KTBL, NARY, decomp)
-
-      IMPLICIT NONE
-
-      TYPE(SMP_INFO) C
-      INTEGER KTBL(C%MAXCORE, C%NSMP)
-      INTEGER NARY(C%NCPU, C%NCORE)
-      TYPE(DECOMP_INFO) :: decomp
-
-      INTEGER i, j, k, l, N, PTR, BSIZ, ierror, status, seed
-      character*16 s
-
-      BSIZ = C%N_SND
-
-      ! a - SNDBUF
-      IF (C%MPI_COMM == DECOMP_2D_COMM_COL) THEN
-         ALLOCATE (decomp%x1cnts_s(C%NSMP), decomp%x1disp_s(C%NSMP + 1), &
-                   stat=status)
-         CALL MPI_Allgather(decomp%x1cnts, C%NCPU, MPI_INTEGER, &
-                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-         PTR = 0
-         DO i = 1, C%NSMP
-            decomp%x1disp_s(i) = PTR
-            N = 0
-            DO j = 1, C%MAXCORE
-               k = KTBL(j, i)
-               IF (k > 0) then
-                  DO l = 1, C%NCORE
-                     IF (l == C%CORE_ME) decomp%x1disp_o(k - 1) = PTR
-                     N = N + NARY(k, l)
-                     PTR = PTR + NARY(k, l)
-                  END DO
-               END IF
-            END DO
-            decomp%x1cnts_s(i) = N
-         END DO
-         decomp%x1disp_s(C%NSMP + 1) = PTR
-         IF (PTR > BSIZ) BSIZ = PTR
-
-      ELSE IF (C%MPI_COMM == DECOMP_2D_COMM_ROW) THEN
-         ALLOCATE (decomp%y2cnts_s(C%NSMP), decomp%y2disp_s(C%NSMP + 1), &
-                   stat=status)
-         CALL MPI_Allgather(decomp%y2cnts, C%NCPU, MPI_INTEGER, &
-                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-         PTR = 0
-         DO i = 1, C%NSMP
-            decomp%y2disp_s(i) = PTR
-            N = 0
-            DO j = 1, C%MAXCORE
-               k = KTBL(j, i)
-               IF (k > 0) then
-                  DO l = 1, C%NCORE
-                     IF (l == C%CORE_ME) decomp%y2disp_o(k - 1) = PTR
-                     N = N + NARY(k, l)
-                     PTR = PTR + NARY(k, l)
-                  END DO
-               END IF
-            END DO
-            decomp%y2cnts_s(i) = N
-         END DO
-         decomp%y2disp_s(C%NSMP + 1) = PTR
-         IF (PTR > BSIZ) BSIZ = PTR
-      END IF
-
-      ! b - RCVBUF
-
-      IF (C%MPI_COMM == DECOMP_2D_COMM_COL) THEN
-         ALLOCATE (decomp%y1cnts_s(C%NSMP), decomp%y1disp_s(C%NSMP + 1), &
-                   stat=status)
-         CALL MPI_Allgather(decomp%y1cnts, C%NCPU, MPI_INTEGER, &
-                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-         PTR = 0
-         DO i = 1, C%NSMP
-            decomp%y1disp_s(i) = PTR
-            N = 0
-            DO j = 1, C%NCORE
-               DO l = 1, C%MAXCORE
-                  k = KTBL(l, i)
-                  IF (k > 0) then
-                     IF (j == C%CORE_ME) decomp%y1disp_o(k - 1) = PTR
-                     N = N + NARY(k, j)
-                     PTR = PTR + NARY(k, j)
-                  END IF
-               END DO
-            END DO
-            decomp%y1cnts_s(i) = N
-         END DO
-         decomp%y1disp_s(C%NSMP + 1) = PTR
-         IF (PTR > BSIZ) BSIZ = PTR
-
-      ELSE IF (C%MPI_COMM == DECOMP_2D_COMM_ROW) THEN
-         ALLOCATE (decomp%z2cnts_s(C%NSMP), decomp%z2disp_s(C%NSMP + 1), &
-                   stat=status)
-         CALL MPI_Allgather(decomp%z2cnts, C%NCPU, MPI_INTEGER, &
-                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-         PTR = 0
-         DO i = 1, C%NSMP
-            decomp%z2disp_s(i) = PTR
-            N = 0
-            DO j = 1, C%NCORE
-               DO l = 1, C%MAXCORE
-                  k = KTBL(l, i)
-                  IF (k > 0) then
-                     IF (j == C%CORE_ME) decomp%z2disp_o(k - 1) = PTR
-                     N = N + NARY(k, j)
-                     PTR = PTR + NARY(k, j)
-                  END IF
-               END DO
-            END DO
-            decomp%z2cnts_s(i) = N
-         END DO
-         decomp%z2disp_s(C%NSMP + 1) = PTR
-         IF (PTR > BSIZ) BSIZ = PTR
-
-      END IF
-
-      ! check buffer size and (re)-allocate buffer space if necessary
-      IF (BSIZ > C%N_SND) then
-         IF (C%SND_P /= 0) CALL DEALLOC_SHM(C%SND_P, C%CORE_COMM)
-         ! make sure each rank has unique keys to get shared memory
-         !IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
-         !   seed = nrank+nproc*0+1 ! has to be non-zero
-         !ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
-         !   seed = nrank+nproc*1+1
-         !END IF
-         status = 1
-         !CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status, &
-         !     seed)
-         CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status)
-         C%N_SND = BSIZ
-
-         IF (C%RCV_P /= 0) CALL DEALLOC_SHM(C%RCV_P, C%CORE_COMM)
-         status = 1
-         CALL ALLOC_SHM(C%RCV_P, BSIZ, real_type, C%CORE_COMM, status)
-         C%N_RCV = BSIZ
-
-         IF (C%SND_P_c /= 0) CALL DEALLOC_SHM(C%SND_P_c, C%CORE_COMM)
-         status = 1
-         CALL ALLOC_SHM(C%SND_P_c, BSIZ, complex_type, C%CORE_COMM, status)
-         C%N_SND = BSIZ
-
-         IF (C%RCV_P_c /= 0) CALL DEALLOC_SHM(C%RCV_P_c, C%CORE_COMM)
-         status = 1
-         CALL ALLOC_SHM(C%RCV_P_c, BSIZ, complex_type, C%CORE_COMM, status)
-         C%N_RCV = BSIZ
-
-      END IF
-
-      RETURN
-   END SUBROUTINE MAPSET_SMPSHM
+  SUBROUTINE MAPSET_SMPSHM(C, KTBL, NARY, decomp)
+
+    IMPLICIT NONE
+
+    TYPE (SMP_INFO) C
+    INTEGER KTBL(C%MAXCORE,C%NSMP)
+    INTEGER NARY(C%NCPU,C%NCORE)
+    TYPE (DECOMP_INFO) :: decomp
+
+    INTEGER i, j, k, l, N, PTR, BSIZ, ierror, status, seed
+    character*16 s
+
+    BSIZ = C%N_SND
+
+    ! a - SNDBUF
+    IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
+       ALLOCATE(decomp%x1cnts_s(C%NSMP),decomp%x1disp_s(C%NSMP+1), &
+            stat=status)
+       CALL MPI_Allgather(decomp%x1cnts, C%NCPU, MPI_INTEGER, &
+            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+       PTR = 0
+       DO i=1,C%NSMP
+          decomp%x1disp_s(i) = PTR
+          N = 0
+          DO j=1,C%MAXCORE
+             k = KTBL(j,i)
+             IF (k > 0) then
+                DO l=1,C%NCORE
+                   IF (l == C%CORE_ME) decomp%x1disp_o(k-1) = PTR
+                   N = N + NARY(k,l)
+                   PTR = PTR + NARY(k,l)
+                END DO
+             END IF
+          END DO
+          decomp%x1cnts_s(i) = N
+       END DO
+       decomp%x1disp_s(C%NSMP+1) = PTR
+       IF (PTR > BSIZ) BSIZ = PTR
+
+    ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
+       ALLOCATE(decomp%y2cnts_s(C%NSMP),decomp%y2disp_s(C%NSMP+1), &
+            stat=status)
+       CALL MPI_Allgather(decomp%y2cnts, C%NCPU, MPI_INTEGER, &
+            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+       PTR = 0
+       DO i=1,C%NSMP
+          decomp%y2disp_s(i) = PTR
+          N = 0
+          DO j=1,C%MAXCORE
+             k = KTBL(j,i)
+             IF (k > 0) then
+                DO l=1,C%NCORE
+                   IF (l == C%CORE_ME) decomp%y2disp_o(k-1) = PTR
+                   N = N + NARY(k,l)
+                   PTR = PTR + NARY(k,l)
+                END DO
+             END IF
+          END DO
+          decomp%y2cnts_s(i) = N
+       END DO
+       decomp%y2disp_s(C%NSMP+1) = PTR
+       IF (PTR > BSIZ) BSIZ = PTR
+    END IF
+
+    ! b - RCVBUF
+
+    IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
+       ALLOCATE(decomp%y1cnts_s(C%NSMP),decomp%y1disp_s(C%NSMP+1), &
+            stat=status)
+       CALL MPI_Allgather(decomp%y1cnts, C%NCPU, MPI_INTEGER, &
+            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+       PTR = 0
+       DO i=1,C%NSMP
+          decomp%y1disp_s(i) = PTR
+          N=0
+          DO j=1,C%NCORE
+             DO l=1,C%MAXCORE
+                k = KTBL(l,i)
+                IF (k > 0) then
+                   IF (j == C%CORE_ME) decomp%y1disp_o(k-1) = PTR
+                   N = N + NARY(k,j)
+                   PTR = PTR + NARY(k,j)
+                END IF
+             END DO
+          END DO
+          decomp%y1cnts_s(i) = N
+       END DO
+       decomp%y1disp_s(C%NSMP+1) = PTR
+       IF (PTR > BSIZ) BSIZ = PTR
+
+    ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
+       ALLOCATE(decomp%z2cnts_s(C%NSMP),decomp%z2disp_s(C%NSMP+1), &
+            stat=status)
+       CALL MPI_Allgather(decomp%z2cnts, C%NCPU, MPI_INTEGER, &
+            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+       PTR = 0
+       DO i=1,C%NSMP
+          decomp%z2disp_s(i) = PTR
+          N=0
+          DO j=1,C%NCORE
+             DO l=1,C%MAXCORE
+                k = KTBL(l,i)
+                IF (k > 0) then
+                   IF (j == C%CORE_ME) decomp%z2disp_o(k-1) = PTR
+                   N = N + NARY(k,j)
+                   PTR = PTR + NARY(k,j)
+                END IF
+             END DO
+          END DO
+          decomp%z2cnts_s(i) = N
+       END DO
+       decomp%z2disp_s(C%NSMP+1) = PTR
+       IF (PTR > BSIZ) BSIZ = PTR
+
+    END IF
+
+    ! check buffer size and (re)-allocate buffer space if necessary
+    IF (BSIZ > C%N_SND) then
+       IF (C%SND_P /= 0) CALL DEALLOC_SHM(C%SND_P, C%CORE_COMM)
+       ! make sure each rank has unique keys to get shared memory
+       !IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
+       !   seed = nrank+nproc*0+1 ! has to be non-zero
+       !ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
+       !   seed = nrank+nproc*1+1
+       !END IF
+       status = 1
+       !CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status, &
+       !     seed)
+       CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status)
+       C%N_SND = BSIZ
+
+       IF (C%RCV_P /= 0) CALL DEALLOC_SHM(C%RCV_P, C%CORE_COMM)
+       status = 1
+       CALL ALLOC_SHM(C%RCV_P, BSIZ, real_type, C%CORE_COMM, status)
+       C%N_RCV = BSIZ
+
+       IF (C%SND_P_c /= 0) CALL DEALLOC_SHM(C%SND_P_c, C%CORE_COMM)
+       status = 1
+       CALL ALLOC_SHM(C%SND_P_c, BSIZ, complex_type, C%CORE_COMM, status)
+       C%N_SND = BSIZ
+
+       IF (C%RCV_P_c /= 0) CALL DEALLOC_SHM(C%RCV_P_c, C%CORE_COMM)
+       status = 1
+       CALL ALLOC_SHM(C%RCV_P_c, BSIZ, complex_type, C%CORE_COMM, status)
+       C%N_RCV = BSIZ
+
+
+    END IF
+
+    RETURN
+  END SUBROUTINE MAPSET_SMPSHM
 
 #endif
 
+
 #ifdef OCC
-   ! For non-blocking communication code, progress the comminication stack
-   subroutine transpose_test(handle)
+  ! For non-blocking communication code, progress the comminication stack
+  subroutine transpose_test(handle)
 
-      implicit none
+    implicit none
 
-      integer :: handle, ierror
+    integer :: handle, ierror
 
-      call NBC_TEST(handle, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "NBC_TEST")
+    call NBC_TEST(handle,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "NBC_TEST")
 
-      return
-   end subroutine transpose_test
+    return
+  end subroutine transpose_test
 #endif
 
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Transposition routines
+  ! Transposition routines 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "transpose_x_to_y.f90"
 #include "transpose_y_to_z.f90"
@@ -1633,174 +1641,175 @@ end subroutine transpose_test
 #include "transpose_y_to_x.f90"
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Halo cell support
+  ! Halo cell support
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "halo.f90"
 
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Error handling
+  ! Error handling
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine decomp_2d_abort_basic(errorcode, msg)
+  subroutine decomp_2d_abort_basic(errorcode, msg)
 
-      use iso_fortran_env, only: error_unit
+    use iso_fortran_env, only : error_unit
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: errorcode
-      character(len=*), intent(IN) :: msg
+    integer, intent(IN) :: errorcode
+    character(len=*), intent(IN) :: msg
 
-      integer :: ierror
+    integer :: ierror
 
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-         write (*, *) 'ERROR MESSAGE: '//msg
-         write (error_unit, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-         write (error_unit, *) 'ERROR MESSAGE: '//msg
-      end if
-      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
+    if (nrank==0) then
+       write(*,*) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+       write(*,*) 'ERROR MESSAGE: ' // msg
+       write(error_unit,*) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+       write(error_unit,*) 'ERROR MESSAGE: ' // msg
+    end if
+    call MPI_ABORT(decomp_2d_comm,errorcode,ierror)
 
-   end subroutine decomp_2d_abort_basic
+  end subroutine decomp_2d_abort_basic
 
-   subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
+  subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
 
-      use iso_fortran_env, only: error_unit
+    use iso_fortran_env, only : error_unit
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: errorcode, line
-      character(len=*), intent(IN) :: msg, file
+    integer, intent(IN) :: errorcode, line
+    character(len=*), intent(IN) :: msg, file
 
-      integer :: ierror
+    integer :: ierror
 
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT ERROR'
-         write (*, *) '  errorcode:     ', errorcode
-         write (*, *) '  error in file  '//file
-         write (*, *) '           line  ', line
-         write (*, *) '  error message: '//msg
-         write (error_unit, *) '2DECOMP&FFT ERROR'
-         write (error_unit, *) '  errorcode:     ', errorcode
-         write (error_unit, *) '  error in file  '//file
-         write (error_unit, *) '           line  ', line
-         write (error_unit, *) '  error message: '//msg
-      end if
-      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
+    if (nrank==0) then
+       write(*,*) '2DECOMP&FFT ERROR'
+       write(*,*) '  errorcode:     ', errorcode
+       write(*,*) '  error in file  ' // file
+       write(*,*) '           line  ', line
+       write(*,*) '  error message: ' // msg
+       write(error_unit,*) '2DECOMP&FFT ERROR'
+       write(error_unit,*) '  errorcode:     ', errorcode
+       write(error_unit,*) '  error in file  ' // file
+       write(error_unit,*) '           line  ', line
+       write(error_unit,*) '  error message: ' // msg
+    end if
+    call MPI_ABORT(decomp_2d_comm,errorcode,ierror)
 
-   end subroutine decomp_2d_abort_file_line
+  end subroutine decomp_2d_abort_file_line
 
 #if defined(_GPU) && defined(_NCCL)
-   !
-   ! This is based on the file "nccl.h" in nvhpc 22.1
-   !
-   function _ncclresult_to_integer(errorcode)
-
-      implicit none
-
-      type(ncclresult), intent(IN) :: errorcode
-      integer :: _ncclresult_to_integer
-
-      if (errorcode == ncclSuccess) then
-         _ncclresult_to_integer = 0
-      elseif (errorcode == ncclUnhandledCudaError) then
-         _ncclresult_to_integer = 1
-      elseif (errorcode == ncclSystemError) then
-         _ncclresult_to_integer = 2
-      elseif (errorcode == ncclInternalError) then
-         _ncclresult_to_integer = 3
-      elseif (errorcode == ncclInvalidArgument) then
-         _ncclresult_to_integer = 4
-      elseif (errorcode == ncclInvalidUsage) then
-         _ncclresult_to_integer = 5
-      elseif (errorcode == ncclNumResults) then
-         _ncclresult_to_integer = 6
-      else
-         _ncclresult_to_integer = -1
-         call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
-                                "NCCL error handling needs some update")
-      end if
-
-   end function _ncclresult_to_integer
-
-   !
-   ! Small wrapper for basic NCCL errors
-   !
-   subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
-
-      implicit none
-
-      type(ncclresult), intent(IN) :: errorcode
-      character(len=*), intent(IN) :: msg
-
-      call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
-                           msg//" "//ncclGetErrorString(errorcode))
-
-   end subroutine decomp_2d_abort_nccl_basic
-
-   !
-   ! Small wrapper for NCCL errors
-   !
-   subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
-
-      implicit none
-
-      type(ncclresult), intent(IN) :: errorcode
-      integer, intent(in) :: line
-      character(len=*), intent(IN) :: msg, file
-
-      call decomp_2d_abort(file, &
-                           line, &
-                           _ncclresult_to_integer(errorcode), &
-                           msg//" "//ncclGetErrorString(errorcode))
-
-   end subroutine decomp_2d_abort_nccl_file_line
+  !
+  ! This is based on the file "nccl.h" in nvhpc 22.1
+  !
+  function _ncclresult_to_integer(errorcode)
+
+    implicit none
+
+    type(ncclresult), intent(IN) :: errorcode
+    integer :: _ncclresult_to_integer
+
+    if (errorcode == ncclSuccess) then
+        _ncclresult_to_integer = 0
+    elseif (errorcode == ncclUnhandledCudaError) then
+        _ncclresult_to_integer = 1
+    elseif (errorcode == ncclSystemError) then
+        _ncclresult_to_integer = 2
+    elseif (errorcode == ncclInternalError) then
+        _ncclresult_to_integer = 3
+    elseif (errorcode == ncclInvalidArgument) then
+        _ncclresult_to_integer = 4
+    elseif (errorcode == ncclInvalidUsage) then
+        _ncclresult_to_integer = 5
+    elseif (errorcode == ncclNumResults) then
+        _ncclresult_to_integer = 6
+    else
+      _ncclresult_to_integer = -1
+      call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
+                             "NCCL error handling needs some update")
+    end if
+
+  end function _ncclresult_to_integer
+
+  !
+  ! Small wrapper for basic NCCL errors
+  !
+  subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
+
+    implicit none
+
+    type(ncclresult), intent(IN) :: errorcode
+    character(len=*), intent(IN) :: msg
+
+    call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
+                         msg // " " // ncclGetErrorString(errorcode))
+
+  end subroutine decomp_2d_abort_nccl_basic
+
+  !
+  ! Small wrapper for NCCL errors
+  !
+  subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
+
+    implicit none
+
+    type(ncclresult), intent(IN) :: errorcode
+    integer, intent(in) :: line
+    character(len=*), intent(IN) :: msg, file
+
+    call decomp_2d_abort(file, &
+                         line, &
+                         _ncclresult_to_integer(errorcode), &
+                         msg // " " // ncclGetErrorString(errorcode))
+
+  end subroutine decomp_2d_abort_nccl_file_line
 #endif
 
-   subroutine decomp_2d_warning_basic(errorcode, msg)
+  subroutine decomp_2d_warning_basic(errorcode, msg)
 
-      use iso_fortran_env, only: error_unit
+    use iso_fortran_env, only : error_unit
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: errorcode
-      character(len=*), intent(IN) :: msg
+    integer, intent(IN) :: errorcode
+    character(len=*), intent(IN) :: msg
 
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-         write (*, *) 'ERROR MESSAGE: '//msg
-         write (error_unit, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-         write (error_unit, *) 'ERROR MESSAGE: '//msg
-      end if
+    if (nrank==0) then
+       write(*,*) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+       write(*,*) 'ERROR MESSAGE: ' // msg
+       write(error_unit,*) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+       write(error_unit,*) 'ERROR MESSAGE: ' // msg
+    end if
 
-   end subroutine decomp_2d_warning_basic
+  end subroutine decomp_2d_warning_basic
 
-   subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
+  subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
 
-      use iso_fortran_env, only: error_unit
+    use iso_fortran_env, only : error_unit
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: errorcode, line
-      character(len=*), intent(IN) :: msg, file
+    integer, intent(IN) :: errorcode, line
+    character(len=*), intent(IN) :: msg, file
 
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT WARNING'
-         write (*, *) '  errorcode:     ', errorcode
-         write (*, *) '  error in file  '//file
-         write (*, *) '           line  ', line
-         write (*, *) '  error message: '//msg
-         write (error_unit, *) '2DECOMP&FFT WARNING'
-         write (error_unit, *) '  errorcode:     ', errorcode
-         write (error_unit, *) '  error in file  '//file
-         write (error_unit, *) '           line  ', line
-         write (error_unit, *) '  error message: '//msg
-      end if
+    if (nrank==0) then
+       write(*,*) '2DECOMP&FFT WARNING'
+       write(*,*) '  errorcode:     ', errorcode
+       write(*,*) '  error in file  ' // file
+       write(*,*) '           line  ', line
+       write(*,*) '  error message: ' // msg
+       write(error_unit,*) '2DECOMP&FFT WARNING'
+       write(error_unit,*) '  errorcode:     ', errorcode
+       write(error_unit,*) '  error in file  ' // file
+       write(error_unit,*) '           line  ', line
+       write(error_unit,*) '  error message: ' // msg
+    end if
 
-   end subroutine decomp_2d_warning_file_line
+  end subroutine decomp_2d_warning_file_line
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Utility routines to help allocate 3D arrays
+  ! Utility routines to help allocate 3D arrays
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "alloc.f90"
-
+  
 end module decomp_2d
 
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index e4545543..df0c4668 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -1,7 +1,7 @@
 !!!=======================================================================
 !!! This is part of the 2DECOMP&FFT library
-!!!
-!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+!!! 
+!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 !!! decomposition. It also implements a highly scalable distributed
 !!! three-dimensional Fast Fourier Transform (FFT).
 !!!
@@ -19,272 +19,273 @@
   !     all internal data structures initialised properly
   !     library ready to use
   !======================================================================
-  subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
+  subroutine decomp_2d_init_ref(nx,ny,nz,p_row,p_col,periodic_bc,comm)
 
-     use mpi
-     use iso_fortran_env, only: output_unit
+    use mpi
+    use iso_fortran_env, only : output_unit
 
-     implicit none
+    implicit none
 
-     integer, intent(IN) :: nx, ny, nz
-     integer, intent(INOUT) :: p_row, p_col
-     logical, dimension(3), intent(IN), optional :: periodic_bc
-     integer, intent(in), optional :: comm
+    integer, intent(IN) :: nx,ny,nz
+    integer, intent(INOUT) :: p_row,p_col
+    logical, dimension(3), intent(IN), optional :: periodic_bc
+    integer, intent(in), optional :: comm
 
-     integer :: errorcode, ierror, row, col, iounit
-     logical, dimension(2) :: periodic
+    integer :: errorcode, ierror, row, col, iounit
+    logical, dimension(2) :: periodic
 #if defined(_GPU) && defined(_NCCL)
-     integer :: cuda_stat
-     type(ncclResult) :: nccl_stat
+    integer :: cuda_stat
+    type(ncclResult) :: nccl_stat
 #endif
-     character(len=7) fname ! Sufficient for up to O(1M) ranks
+    character(len=7) fname ! Sufficient for up to O(1M) ranks
 
 #ifdef PROFILER
-     ! Prepare the profiler if it was not already prepared
-     if (decomp_profiler .eq. decomp_profiler_none) call decomp_profiler_prep()
-     ! Start the profiler
-     call decomp_profiler_init()
-     ! Start the timer for decomp_2d_init
-     if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_init")
+    ! Prepare the profiler if it was not already prepared
+    if (decomp_profiler.eq.decomp_profiler_none) call decomp_profiler_prep()
+    ! Start the profiler
+    call decomp_profiler_init()
+    ! Start the timer for decomp_2d_init
+    if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_init")
 #endif
 
-     ! Use the provided MPI communicator if present
-     if (present(comm)) then
-        decomp_2d_comm = comm
-     else
-        decomp_2d_comm = MPI_COMM_WORLD
-     end if
-
-     ! Safety check
-     if (MPI_SUCCESS /= 0) call decomp_2d_abort(__FILE__, &
-                                                __LINE__, &
-                                                MPI_SUCCESS, &
-                                                "MPI error check is broken")
-
-     ! If the external code has not set nrank and nproc
-     if (nrank == -1) then
-        call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                              __LINE__, &
-                                              ierror, &
-                                              "MPI_COMM_RANK")
-     end if
-     if (nproc == -1) then
-        call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                              __LINE__, &
-                                              ierror, &
-                                              "MPI_COMM_SIZE")
-     end if
+    ! Use the provided MPI communicator if present
+    if (present(comm)) then
+       decomp_2d_comm = comm
+    else
+       decomp_2d_comm = MPI_COMM_WORLD
+    endif
+
+    ! Safety check
+    if (MPI_SUCCESS /= 0) call decomp_2d_abort(__FILE__, &
+                                               __LINE__, &
+                                               MPI_SUCCESS, &
+                                               "MPI error check is broken")
+
+    ! If the external code has not set nrank and nproc
+    if (nrank == -1) then
+       call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, &
+                                             __LINE__, &
+                                             ierror, &
+                                             "MPI_COMM_RANK")
+    endif
+    if (nproc == -1) then
+       call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, &
+                                             __LINE__, &
+                                             ierror, &
+                                             "MPI_COMM_SIZE")
+    endif
 #ifdef DEBUG
-     ! Check if a modification of the debug level is needed
-     call decomp_2d_debug()
+    ! Check if a modification of the debug level is needed
+    call decomp_2d_debug()
 #endif
 
-     nx_global = nx
-     ny_global = ny
-     nz_global = nz
-
-     if (present(periodic_bc)) then
-        periodic_x = periodic_bc(1)
-        periodic_y = periodic_bc(2)
-        periodic_z = periodic_bc(3)
-     else
-        periodic_x = .false.
-        periodic_y = .false.
-        periodic_z = .false.
-     end if
-
-     if (p_row == 0 .and. p_col == 0) then
-        ! determine the best 2D processor grid
-        call best_2d_grid(nproc, row, col)
-        p_row = row
-        p_col = col
-     else
-        if (nproc /= p_row*p_col) then
-           errorcode = 1
-           call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                'Invalid 2D processor grid - nproc /= p_row*p_col')
-        else
-           row = p_row
-           col = p_col
-        end if
-     end if
-
-     ! Create 2D Catersian topology
-     ! Note that in order to support periodic B.C. in the halo-cell code,
-     ! need to create multiple topology objects: DECOMP_2D_COMM_CART_?,
-     ! corresponding to three pencil orientations. They contain almost
-     ! identical topological information but allow different combinations
-     ! of periodic conditions.
-     dims(1) = row
-     dims(2) = col
-     periodic(1) = periodic_y
-     periodic(2) = periodic_z
-     call MPI_CART_CREATE(decomp_2d_comm, 2, dims, periodic, &
-                          .false., &  ! do not reorder rank
-                          DECOMP_2D_COMM_CART_X, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
-     periodic(1) = periodic_x
-     periodic(2) = periodic_z
-     call MPI_CART_CREATE(decomp_2d_comm, 2, dims, periodic, &
-                          .false., DECOMP_2D_COMM_CART_Y, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
-     periodic(1) = periodic_x
-     periodic(2) = periodic_y
-     call MPI_CART_CREATE(decomp_2d_comm, 2, dims, periodic, &
-                          .false., DECOMP_2D_COMM_CART_Z, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
-
-     call MPI_CART_COORDS(DECOMP_2D_COMM_CART_X, nrank, 2, coord, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_COORDS")
-
-     ! derive communicators defining sub-groups for ALLTOALL(V)
-     call MPI_CART_SUB(DECOMP_2D_COMM_CART_X, (/.true., .false./), &
-                       DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
-     call MPI_CART_SUB(DECOMP_2D_COMM_CART_X, (/.false., .true./), &
-                       DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
-
-     ! gather information for halo-cell support code
-     call init_neighbour
-
-     ! actually generate all 2D decomposition information
-     call decomp_info_init(nx, ny, nz, decomp_main)
-
-     ! make a copy of the decomposition information associated with the
-     ! default global size in these global variables so applications can
-     ! use them to create data structures
-     xstart = decomp_main%xst
-     ystart = decomp_main%yst
-     zstart = decomp_main%zst
-     xend = decomp_main%xen
-     yend = decomp_main%yen
-     zend = decomp_main%zen
-     xsize = decomp_main%xsz
-     ysize = decomp_main%ysz
-     zsize = decomp_main%zsz
-
-     ! determine the number of bytes per float number
-     ! do not use 'mytype' which is compiler dependent
-     ! also possible to use inquire(iolength=...)
-     call MPI_TYPE_SIZE(real_type, mytype_bytes, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+    nx_global = nx
+    ny_global = ny
+    nz_global = nz
+
+    if (present(periodic_bc)) then
+       periodic_x = periodic_bc(1)
+       periodic_y = periodic_bc(2)
+       periodic_z = periodic_bc(3)
+    else
+       periodic_x = .false.
+       periodic_y = .false.
+       periodic_z = .false.
+    end if
+
+    if (p_row==0 .and. p_col==0) then
+       ! determine the best 2D processor grid
+       call best_2d_grid(nproc, row, col)
+       p_row = row
+       p_col = col
+    else
+       if (nproc /= p_row*p_col) then
+          errorcode = 1
+          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+               'Invalid 2D processor grid - nproc /= p_row*p_col')
+       else
+          row = p_row
+          col = p_col
+       end if
+    end if
+
+    ! Create 2D Catersian topology
+    ! Note that in order to support periodic B.C. in the halo-cell code,
+    ! need to create multiple topology objects: DECOMP_2D_COMM_CART_?,
+    ! corresponding to three pencil orientations. They contain almost
+    ! identical topological information but allow different combinations
+    ! of periodic conditions.
+    dims(1) = row
+    dims(2) = col
+    periodic(1) = periodic_y
+    periodic(2) = periodic_z
+    call MPI_CART_CREATE(decomp_2d_comm,2,dims,periodic, &
+         .false., &  ! do not reorder rank
+         DECOMP_2D_COMM_CART_X, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
+    periodic(1) = periodic_x
+    periodic(2) = periodic_z
+    call MPI_CART_CREATE(decomp_2d_comm,2,dims,periodic, &
+         .false., DECOMP_2D_COMM_CART_Y, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
+    periodic(1) = periodic_x
+    periodic(2) = periodic_y
+    call MPI_CART_CREATE(decomp_2d_comm,2,dims,periodic, &
+         .false., DECOMP_2D_COMM_CART_Z, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
+
+    call MPI_CART_COORDS(DECOMP_2D_COMM_CART_X,nrank,2,coord,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_COORDS")
+
+    ! derive communicators defining sub-groups for ALLTOALL(V)
+    call MPI_CART_SUB(DECOMP_2D_COMM_CART_X,(/.true.,.false./), &
+         DECOMP_2D_COMM_COL,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
+    call MPI_CART_SUB(DECOMP_2D_COMM_CART_X,(/.false.,.true./), &
+         DECOMP_2D_COMM_ROW,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
+
+    ! gather information for halo-cell support code
+    call init_neighbour
+
+    ! actually generate all 2D decomposition information
+    call decomp_info_init(nx,ny,nz,decomp_main)
+
+    ! make a copy of the decomposition information associated with the
+    ! default global size in these global variables so applications can
+    ! use them to create data structures 
+    xstart = decomp_main%xst
+    ystart = decomp_main%yst
+    zstart = decomp_main%zst
+    xend   = decomp_main%xen
+    yend   = decomp_main%yen
+    zend   = decomp_main%zen
+    xsize  = decomp_main%xsz
+    ysize  = decomp_main%ysz
+    zsize  = decomp_main%zsz
+
+    ! determine the number of bytes per float number
+    ! do not use 'mytype' which is compiler dependent
+    ! also possible to use inquire(iolength=...) 
+    call MPI_TYPE_SIZE(real_type,mytype_bytes,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
 
 #ifdef EVEN
-     if (nrank == 0) write (*, *) 'Padded ALLTOALL optimisation on'
-#endif
+    if (nrank==0) write(*,*) 'Padded ALLTOALL optimisation on'
+#endif 
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call MPI_COMM_RANK(DECOMP_2D_COMM_COL, col_rank, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-     call MPI_COMM_RANK(DECOMP_2D_COMM_ROW, row_rank, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-     call MPI_COMM_SIZE(DECOMP_2D_COMM_COL, col_comm_size, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-     call MPI_COMM_SIZE(DECOMP_2D_COMM_ROW, row_comm_size, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-
-     allocate (local_to_global_col(col_comm_size), local_to_global_row(row_comm_size))
-
-     local_to_global_col(:) = 0
-     local_to_global_row(:) = 0
-     local_to_global_col(col_rank + 1) = nrank
-     local_to_global_row(row_rank + 1) = nrank
-
-     call mpi_allreduce(MPI_IN_PLACE, local_to_global_col, col_comm_size, MPI_INTEGER, MPI_SUM, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-     call mpi_allreduce(MPI_IN_PLACE, local_to_global_row, row_comm_size, MPI_INTEGER, MPI_SUM, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-
-     if (nrank .eq. 0) then
-        nccl_stat = ncclGetUniqueId(nccl_uid_2decomp)
-     end if
-     call MPI_Bcast(nccl_uid_2decomp, int(sizeof(ncclUniqueId)), MPI_BYTE, 0, decomp_2d_comm, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
-
-     nccl_stat = ncclCommInitRank(nccl_comm_2decomp, nproc, nccl_uid_2decomp, nrank)
-     cuda_stat = cudaStreamCreate(cuda_stream_2decomp)
+    call MPI_COMM_RANK(DECOMP_2D_COMM_COL,col_rank,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+    call MPI_COMM_RANK(DECOMP_2D_COMM_ROW,row_rank,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+    call MPI_COMM_SIZE(DECOMP_2D_COMM_COL,col_comm_size,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+    call MPI_COMM_SIZE(DECOMP_2D_COMM_ROW,row_comm_size,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+
+    allocate(local_to_global_col(col_comm_size), local_to_global_row(row_comm_size))
+    
+    local_to_global_col(:) = 0
+    local_to_global_row(:) = 0
+    local_to_global_col(col_rank+1) = nrank
+    local_to_global_row(row_rank+1) = nrank
+    
+    call mpi_allreduce(MPI_IN_PLACE,local_to_global_col,col_comm_size,MPI_INTEGER,MPI_SUM,DECOMP_2D_COMM_COL,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+    call mpi_allreduce(MPI_IN_PLACE,local_to_global_row,row_comm_size,MPI_INTEGER,MPI_SUM,DECOMP_2D_COMM_ROW,ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+
+    if (nrank .eq. 0) then
+       nccl_stat = ncclGetUniqueId(nccl_uid_2decomp)
+    end if
+    call MPI_Bcast(nccl_uid_2decomp, int(sizeof(ncclUniqueId)), MPI_BYTE, 0, decomp_2d_comm, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+
+    nccl_stat = ncclCommInitRank(nccl_comm_2decomp, nproc, nccl_uid_2decomp, nrank)
+    cuda_stat = cudaStreamCreate(cuda_stream_2decomp)
 #endif
 #endif
 
-     !
-     ! Select the IO unit for decomp_2d setup
-     !
-     if (decomp_log == D2D_LOG_TOFILE_FULL) then
-        write (fname, "(I0)") nrank ! Adapt to magnitude of nrank
-        open (newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
-     elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
-        open (newunit=iounit, file="decomp_2d_setup.log", iostat=ierror)
-     else
-        iounit = output_unit
-        ierror = 0
-     end if
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
-     !
-     ! Print the decomp_2d setup
-     !
-     call d2d_listing(iounit)
-     !
-     ! Close the IO unit if it was not stdout
-     !
-     if (iounit /= output_unit) then
-        close (iounit, iostat=ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-     end if
+    !
+    ! Select the IO unit for decomp_2d setup
+    !
+    if (decomp_log == D2D_LOG_TOFILE_FULL) then
+       write(fname, "(I0)") nrank ! Adapt to magnitude of nrank
+       open(newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
+    elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
+       open(newunit=iounit, file="decomp_2d_setup.log", iostat=ierror)
+    else
+       iounit = output_unit
+       ierror = 0
+    endif
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
+    !
+    ! Print the decomp_2d setup
+    !
+    call d2d_listing(iounit)
+    !
+    ! Close the IO unit if it was not stdout
+    !
+    if (iounit /= output_unit) then
+       close(iounit, iostat=ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+    endif
 
 #ifdef PROFILER
-     ! Stop the timer for decomp_2d_init
-     if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_init")
+    ! Stop the timer for decomp_2d_init
+    if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_init")
 #endif
 
-     return
+    return
   end subroutine decomp_2d_init_ref
 
   !======================================================================
   ! Routine to be called by applications to clean things up
   !======================================================================
   subroutine decomp_2d_finalize_ref
-
-     implicit none
+    
+    implicit none
 #if defined(_GPU) && defined(_NCCL)
-     type(ncclResult) :: nccl_stat
+    type(ncclResult) :: nccl_stat
 #endif
-
+    
+ 
 #ifdef PROFILER
-     if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_fin")
+    if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_fin")
 #endif
 
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_ROW)
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_COL)
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_X)
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Y)
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Z)
+    call decomp_mpi_comm_free(DECOMP_2D_COMM_ROW)
+    call decomp_mpi_comm_free(DECOMP_2D_COMM_COL)
+    call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_X)
+    call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Y)
+    call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Z)
 
-     call decomp_info_finalize(decomp_main)
+    call decomp_info_finalize(decomp_main)
 
-     decomp_buf_size = 0
-     deallocate (work1_r, work2_r, work1_c, work2_c)
+    decomp_buf_size = 0
+    deallocate(work1_r, work2_r, work1_c, work2_c)
 #if defined(_GPU)
-     deallocate (work1_r_d, work2_r_d, work1_c_d, work2_c_d)
+    deallocate(work1_r_d, work2_r_d, work1_c_d, work2_c_d)
 
 #if defined(_NCCL)
-     nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
+    nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
 #endif
 #endif
 
-     nrank = -1
-     nproc = -1
+    nrank = -1
+    nproc = -1
 
 #ifdef PROFILER
-     if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_fin")
-     ! Finalize the profiler
-     call decomp_profiler_fin()
+    if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_fin")
+    ! Finalize the profiler
+    call decomp_profiler_fin()
 #endif
 
-     return
+    return
   end subroutine decomp_2d_finalize_ref
 
 #ifdef DEBUG
@@ -295,33 +296,33 @@ end subroutine decomp_2d_finalize_ref
   !
   subroutine decomp_2d_debug
 
-     implicit none
+    implicit none
 
-     integer :: ierror
-     character(len=4) :: val
-     character(len=*), parameter :: varname = "DECOMP_2D_DEBUG"
+    integer :: ierror
+    character(len=4) :: val
+    character(len=*), parameter :: varname = "DECOMP_2D_DEBUG"
 
-     ! Read the variable
-     call get_environment_variable(varname, value=val, status=ierror)
+    ! Read the variable
+    call get_environment_variable(varname, value=val, status=ierror)
 
-     ! Return if no variable, or no support for env. variable
-     if (ierror >= 1) return
+    ! Return if no variable, or no support for env. variable
+    if (ierror >= 1) return
 
-     ! Minor error, print warning and return
-     if (ierror /= 0) then
-        call decomp_2d_warning(__FILE__, &
-                               __LINE__, &
-                               ierror, &
-                               "Error when reading DECOMP_2D_DEBUG : "//val)
-        return
-     end if
+    ! Minor error, print warning and return
+    if (ierror /= 0) then
+       call decomp_2d_warning(__FILE__, &
+                              __LINE__, &
+                              ierror, &
+                              "Error when reading DECOMP_2D_DEBUG : "//val)
+       return
+    endif
 
-     ! Conversion to integer if possible
-     read (val, '(i4)', iostat=ierror) decomp_debug
-     if (ierror /= 0) call decomp_2d_warning(__FILE__, &
-                                             __LINE__, &
-                                             ierror, &
-                                             "Error when reading DECOMP_2D_DEBUG : "//val)
+    ! Conversion to integer if possible
+    read(val, '(i4)', iostat=ierror) decomp_debug
+    if (ierror /= 0) call decomp_2d_warning(__FILE__, &
+                                            __LINE__, &
+                                            ierror, &
+                                            "Error when reading DECOMP_2D_DEBUG : "//val)
 
   end subroutine decomp_2d_debug
 #endif
@@ -331,34 +332,34 @@ end subroutine decomp_2d_debug
   !---------------------------------------------------------------------
   subroutine best_2d_grid(iproc, best_p_row, best_p_col)
 
-     implicit none
-
-     integer, intent(IN) :: iproc
-     integer, intent(OUT) :: best_p_row, best_p_col
+    implicit none
 
-     integer, allocatable, dimension(:) :: factors
-     integer :: nfact, i, col, i_best
+    integer, intent(IN) :: iproc
+    integer, intent(OUT) :: best_p_row, best_p_col
 
-     if (nrank == 0) write (*, *) 'In auto-tuning mode......'
+    integer, allocatable, dimension(:) :: factors
+    integer :: nfact, i, col, i_best
 
-     i = int(sqrt(real(iproc))) + 10  ! enough space to save all factors
-     allocate (factors(i))
-     call findfactor(iproc, factors, nfact)
-     if (nrank == 0) write (*, *) 'factors: ', (factors(i), i=1, nfact)
+    if (nrank==0) write(*,*) 'In auto-tuning mode......'
 
-     i_best = nfact/2 + 1
-     col = factors(i_best)
+    i = int(sqrt(real(iproc))) + 10  ! enough space to save all factors 
+    allocate(factors(i))
+    call findfactor(iproc, factors, nfact)
+    if (nrank==0) write(*,*) 'factors: ', (factors(i), i=1,nfact)
 
-     best_p_col = col
-     best_p_row = iproc/col
-     if (nrank == 0) print *, 'p_row x p_col', best_p_row, best_p_col
-     if ((best_p_col == 1) .and. (nrank == 0)) then
-        print *, 'WARNING: current 2D DECOMP set-up might not work'
-     end if
+    i_best=nfact/2+1
+    col=factors(i_best)
 
-     deallocate (factors)
+    best_p_col = col
+    best_p_row=iproc/col
+    if (nrank==0) print *,'p_row x p_col', best_p_row, best_p_col
+    if ((best_p_col==1).and.(nrank==0)) then
+       print *,'WARNING: current 2D DECOMP set-up might not work'
+    endif
+    
+    deallocate(factors)
 
-     return
+    return
   end subroutine best_2d_grid
 
   !---------------------------------------------------------------------
@@ -368,36 +369,36 @@ end subroutine best_2d_grid
   !---------------------------------------------------------------------
   subroutine init_neighbour
 
-     integer :: ierror
-
-     ! For X-pencil
-     neighbour(1, 1) = MPI_PROC_NULL               ! east
-     neighbour(1, 2) = MPI_PROC_NULL               ! west
-     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 0, 1, &
-                         neighbour(1, 4), neighbour(1, 3), ierror) ! north & south
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 1, 1, &
-                         neighbour(1, 6), neighbour(1, 5), ierror) ! top & bottom
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-
-     ! For Y-pencil
-     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 0, 1, &
-                         neighbour(2, 2), neighbour(2, 1), ierror) ! east & west
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-     neighbour(2, 3) = MPI_PROC_NULL               ! north
-     neighbour(2, 4) = MPI_PROC_NULL               ! south
-     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 1, 1, &
-                         neighbour(2, 6), neighbour(2, 5), ierror) ! top & bottom
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-
-     ! For Z-pencil
-     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 0, 1, &
-                         neighbour(3, 2), neighbour(3, 1), ierror) ! east & west
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 1, 1, &
-                         neighbour(3, 4), neighbour(3, 3), ierror) ! north & south
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-     neighbour(3, 5) = MPI_PROC_NULL               ! top
-     neighbour(3, 6) = MPI_PROC_NULL               ! bottom
-     return
+    integer :: ierror
+
+    ! For X-pencil
+    neighbour(1,1) = MPI_PROC_NULL               ! east
+    neighbour(1,2) = MPI_PROC_NULL               ! west
+    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 0, 1, &
+         neighbour(1,4), neighbour(1,3), ierror) ! north & south
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 1, 1, &
+         neighbour(1,6), neighbour(1,5), ierror) ! top & bottom
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+
+    ! For Y-pencil
+    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 0, 1, &
+         neighbour(2,2), neighbour(2,1), ierror) ! east & west
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+    neighbour(2,3) = MPI_PROC_NULL               ! north
+    neighbour(2,4) = MPI_PROC_NULL               ! south
+    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 1, 1, &
+         neighbour(2,6), neighbour(2,5), ierror) ! top & bottom
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+
+    ! For Z-pencil
+    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 0, 1, &
+         neighbour(3,2), neighbour(3,1), ierror) ! east & west
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 1, 1, &
+         neighbour(3,4), neighbour(3,3), ierror) ! north & south
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+    neighbour(3,5) = MPI_PROC_NULL               ! top
+    neighbour(3,6) = MPI_PROC_NULL               ! bottom
+    return
   end subroutine init_neighbour
diff --git a/src/factor.f90 b/src/factor.f90
index 5f7e530a..73d96e15 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -1,7 +1,7 @@
 !!!=======================================================================
 !!! This is part of the 2DECOMP&FFT library
-!!!
-!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+!!! 
+!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 !!! decomposition. It also implements a highly scalable distributed
 !!! three-dimensional Fast Fourier Transform (FFT).
 !!!
@@ -13,80 +13,80 @@
 !!! A few utility routines to find factors of integer numbers
 module factor
 
-   implicit none
-
-   private
+  implicit none
 
-   public :: findfactor
-   public :: primefactors
+  private
 
+  public :: findfactor
+  public :: primefactors
+  
 contains
-
-   subroutine findfactor(num, factors, nfact)
-
-      implicit none
-
-      integer, intent(IN) :: num
-      integer, intent(OUT), dimension(*) :: factors
-      integer, intent(OUT) :: nfact
-      integer :: i, m
-
-      ! find the factors <= sqrt(num)
-      m = int(sqrt(real(num)))
-      nfact = 1
-      do i = 1, m
-         if (num/i*i == num) then
-            factors(nfact) = i
-            nfact = nfact + 1
-         end if
-      end do
-      nfact = nfact - 1
-
-      ! derive those > sqrt(num)
-      if (factors(nfact)**2 /= num) then
-         do i = nfact + 1, 2*nfact
-            factors(i) = num/factors(2*nfact - i + 1)
-         end do
-         nfact = nfact*2
-      else
-         do i = nfact + 1, 2*nfact - 1
-            factors(i) = num/factors(2*nfact - i)
-         end do
-         nfact = nfact*2 - 1
-      end if
-
-      return
-
-   end subroutine findfactor
-
-   subroutine primefactors(num, factors, nfact)
-
-      implicit none
-
-      integer, intent(IN) :: num
-      integer, intent(OUT), dimension(*) :: factors
-      integer, intent(INOUT) :: nfact
-
-      integer :: i, n
-
-      i = 2
-      nfact = 1
-      n = num
-      do
-         if (mod(n, i) == 0) then
-            factors(nfact) = i
-            nfact = nfact + 1
-            n = n/i
-         else
-            i = i + 1
-         end if
-         if (n == 1) then
-            nfact = nfact - 1
-            exit
-         end if
-      end do
-
-      return
-
-   end subroutine primefactors
+  
+  subroutine findfactor(num, factors, nfact)
+    
+    implicit none
+
+    integer, intent(IN) :: num
+    integer, intent(OUT), dimension(*) :: factors
+    integer, intent(OUT) :: nfact
+    integer :: i, m
+
+    ! find the factors <= sqrt(num)
+    m = int(sqrt(real(num)))
+    nfact = 1
+    do i=1,m
+       if (num/i*i == num) then
+          factors(nfact) = i
+          nfact = nfact + 1
+       end if
+    end do
+    nfact = nfact - 1
+
+    ! derive those > sqrt(num)
+    if (factors(nfact)**2/=num) then
+       do i=nfact+1, 2*nfact
+          factors(i) = num / factors(2*nfact-i+1)
+       end do
+       nfact = nfact * 2
+    else
+       do i=nfact+1, 2*nfact-1
+          factors(i) = num / factors(2*nfact-i)
+       end do
+       nfact = nfact * 2 - 1
+    endif
+       
+    return
+
+  end subroutine findfactor
+
+  subroutine primefactors(num, factors, nfact)
+
+    implicit none
+  
+    integer, intent(IN) :: num
+    integer, intent(OUT), dimension(*) :: factors
+    integer, intent(INOUT) :: nfact
+
+    integer :: i, n
+    
+    i = 2  
+    nfact = 1
+    n = num 
+    do
+       if (mod(n,i) == 0) then
+          factors(nfact) = i
+          nfact = nfact + 1
+          n = n / i
+       else
+          i = i + 1
+       end if
+       if (n == 1) then
+          nfact = nfact - 1
+          exit
+       end if
+    end do
+    
+    return
+
+  end subroutine primefactors
 end module
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index e0cd9cac..8c17df27 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -16,12 +16,12 @@
 
 ! Physical space data can be stored in either X-pencil or Z-pencil
 integer, parameter, public :: PHYSICAL_IN_X = 1
-integer, parameter, public :: PHYSICAL_IN_Z = 3
+integer, parameter, public :: PHYSICAL_IN_Z = 3 
 
 integer, save :: format                 ! input X-pencil or Z-pencil
 
 ! The libary can only be initialised once
-logical, save :: initialised = .false.
+logical, save :: initialised = .false. 
 
 ! Global size of the FFT
 integer, save :: nx_fft, ny_fft, nz_fft
@@ -31,84 +31,85 @@
 integer, save, dimension(2) :: dims
 
 ! Decomposition objects
-TYPE(DECOMP_INFO), pointer, save :: ph => null()  ! physical space
+TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
 TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
 
 ! Workspace to store the intermediate Y-pencil data
 ! *** TODO: investigate how to use only one workspace array
-complex(mytype), allocatable, dimension(:, :, :) :: wk2_c2c, wk2_r2c
-complex(mytype), allocatable, dimension(:, :, :) :: wk13
+complex(mytype), allocatable, dimension(:,:,:) :: wk2_c2c, wk2_r2c
+complex(mytype), allocatable, dimension(:,:,:) :: wk13
 
 public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-          decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
-          decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
+decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
 
 ! Declare generic interfaces to handle different inputs
 
 interface decomp_2d_fft_init
-   module procedure fft_init_noarg
-   module procedure fft_init_arg
-   module procedure fft_init_general
+module procedure fft_init_noarg
+module procedure fft_init_arg
+module procedure fft_init_general
 end interface
 
 interface decomp_2d_fft_3d
-   module procedure fft_3d_c2c
-   module procedure fft_3d_r2c
-   module procedure fft_3d_c2r
+module procedure fft_3d_c2c
+module procedure fft_3d_r2c
+module procedure fft_3d_c2r
 end interface
 
 contains
 
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Initialise the FFT module
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_init_noarg
 
-   implicit none
+implicit none
 
-   call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
+call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
 
-   return
+return
 end subroutine fft_init_noarg
 
 subroutine fft_init_arg(pencil)     ! allow to handle Z-pencil input
 
-   implicit none
+implicit none
 
-   integer, intent(IN) :: pencil
+integer, intent(IN) :: pencil
 
-   call fft_init_general(pencil, nx_global, ny_global, nz_global)
+call fft_init_general(pencil, nx_global, ny_global, nz_global)
 
-   return
+return
 end subroutine fft_init_arg
 
 ! Initialise the FFT library to perform arbitrary size transforms
 subroutine fft_init_general(pencil, nx, ny, nz)
 
-   implicit none
+implicit none
 
-   integer, intent(IN) :: pencil
-   integer, intent(IN) :: nx, ny, nz
+integer, intent(IN) :: pencil
+integer, intent(IN) :: nx, ny, nz
 
-   integer :: status, errorcode
+integer :: status, errorcode
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
+if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
 #endif
 
-   if (initialised) then
-      errorcode = 4
-      call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                           'FFT library should only be initialised once')
-   end if
+if (initialised) then
+errorcode = 4
+call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+'FFT library should only be initialised once')
+end if
 
-   format = pencil
-   nx_fft = nx
-   ny_fft = ny
-   nz_fft = nz
+format = pencil
+nx_fft = nx
+ny_fft = ny
+nz_fft = nz
 
 ! determine the processor grid in use
-   dims = get_decomp_dims()
+dims = get_decomp_dims()
 
 ! for c2r/r2c interface:
 ! if in physical space, a real array is of size: nx*ny*nz
@@ -116,96 +117,98 @@ subroutine fft_init_general(pencil, nx, ny, nz)
 !         (nx/2+1)*ny*nz, if PHYSICAL_IN_X
 !      or nx*ny*(nz/2+1), if PHYSICAL_IN_Z
 
-   if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
-      ph => decomp_main
-   else
-      if (.not. associated(ph)) allocate (ph)
-      call decomp_info_init(nx, ny, nz, ph)
-   end if
-   if (format == PHYSICAL_IN_X) then
-      call decomp_info_init(nx/2 + 1, ny, nz, sp)
-   else if (format == PHYSICAL_IN_Z) then
-      call decomp_info_init(nx, ny, nz/2 + 1, sp)
-   end if
-
-   allocate (wk2_c2c(ph%ysz(1), ph%ysz(2), ph%ysz(3)), STAT=status)
-   allocate (wk2_r2c(sp%ysz(1), sp%ysz(2), sp%ysz(3)), STAT=status)
-   if (format == PHYSICAL_IN_X) then
-      allocate (wk13(sp%xsz(1), sp%xsz(2), sp%xsz(3)), STAT=status)
-   else if (format == PHYSICAL_IN_Z) then
-      allocate (wk13(sp%zsz(1), sp%zsz(2), sp%zsz(3)), STAT=status)
-   end if
-   if (status /= 0) then
-      errorcode = 3
-      call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                           'Out of memory when initialising FFT')
-   end if
-
-   call init_fft_engine
-
-   initialised = .true.
+if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
+ph=>decomp_main
+else
+if (.not.associated(ph)) allocate(ph)
+call decomp_info_init(nx, ny, nz, ph)
+endif
+if (format==PHYSICAL_IN_X) then
+call decomp_info_init(nx/2+1, ny, nz, sp)
+else if (format==PHYSICAL_IN_Z) then
+call decomp_info_init(nx, ny, nz/2+1, sp)
+end if
+
+allocate(wk2_c2c(ph%ysz(1),ph%ysz(2),ph%ysz(3)), STAT=status)
+allocate(wk2_r2c(sp%ysz(1),sp%ysz(2),sp%ysz(3)), STAT=status)
+if (format==PHYSICAL_IN_X) then
+allocate(wk13(sp%xsz(1),sp%xsz(2),sp%xsz(3)), STAT=status)
+else if (format==PHYSICAL_IN_Z) then
+allocate(wk13(sp%zsz(1),sp%zsz(2),sp%zsz(3)), STAT=status)
+end if
+if (status /= 0) then
+errorcode = 3
+call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+'Out of memory when initialising FFT')
+end if
+
+call init_fft_engine
+
+initialised = .true.
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
+if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
 #endif
 
-   return
+return
 end subroutine fft_init_general
 
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Final clean up
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine decomp_2d_fft_finalize
 
-   implicit none
+implicit none
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
+if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-   if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
-      call decomp_info_finalize(ph)
-      deallocate (ph)
-   end if
-   nullify (ph)
-   call decomp_info_finalize(sp)
+if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
+call decomp_info_finalize(ph)
+deallocate(ph)
+endif
+nullify(ph)
+call decomp_info_finalize(sp)
 
-   if (allocated(wk2_c2c)) deallocate (wk2_c2c)
-   if (allocated(wk2_r2c)) deallocate (wk2_r2c)
-   if (allocated(wk13)) deallocate (wk13)
+if (allocated(wk2_c2c)) deallocate(wk2_c2c)
+if (allocated(wk2_r2c)) deallocate(wk2_r2c)
+if (allocated(wk13)) deallocate(wk13)
 
-   call finalize_fft_engine
+call finalize_fft_engine
 
-   initialised = .false.
+initialised = .false.
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
+if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
 #endif
 
-   return
+return
 end subroutine decomp_2d_fft_finalize
 
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-! Return the size, starting/ending index of the distributed array
+! Return the size, starting/ending index of the distributed array 
 !  whose global size is (nx/2+1)*ny*nz, for defining data structures
 !  in r2c and c2r interfaces
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine decomp_2d_fft_get_size(istart, iend, isize)
 
-   implicit none
-   integer, dimension(3), intent(OUT) :: istart, iend, isize
+implicit none
+integer, dimension(3), intent(OUT) :: istart, iend, isize
 
-   if (format == PHYSICAL_IN_X) then
-      istart = sp%zst
-      iend = sp%zen
-      isize = sp%zsz
-   else if (format == PHYSICAL_IN_Z) then
-      istart = sp%xst
-      iend = sp%xen
-      isize = sp%xsz
-   end if
+if (format==PHYSICAL_IN_X) then
+istart = sp%zst
+iend   = sp%zen
+isize  = sp%zsz
+else if (format==PHYSICAL_IN_Z) then
+istart = sp%xst
+iend   = sp%xen
+isize  = sp%xsz
+end if
 
-   return
+return
 end subroutine decomp_2d_fft_get_size
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -215,15 +218,15 @@ end subroutine decomp_2d_fft_get_size
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 function decomp_2d_fft_get_ph()
 
-   implicit none
+implicit none
 
-   type(decomp_info), pointer :: decomp_2d_fft_get_ph
+type(decomp_info), pointer :: decomp_2d_fft_get_ph
 
-   if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  -1, &
-                                                  'FFT library must be initialised first')
-   decomp_2d_fft_get_ph => ph
+if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
+                                              __LINE__, &
+                                             -1, &
+                                             'FFT library must be initialised first')
+decomp_2d_fft_get_ph => ph
 
 end function decomp_2d_fft_get_ph
 
@@ -234,14 +237,14 @@ end function decomp_2d_fft_get_ph
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 function decomp_2d_fft_get_sp()
 
-   implicit none
+implicit none
 
-   type(decomp_info), pointer :: decomp_2d_fft_get_sp
+type(decomp_info), pointer :: decomp_2d_fft_get_sp
 
-   if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  -1, &
-                                                  'FFT library must be initialised first')
-   decomp_2d_fft_get_sp => sp
+if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
+                                              __LINE__, &
+                                             -1, &
+                                             'FFT library must be initialised first')
+decomp_2d_fft_get_sp => sp
 
 end function decomp_2d_fft_get_sp
diff --git a/src/fft_common_3d.f90 b/src/fft_common_3d.f90
index eb809b37..3c0abc1b 100644
--- a/src/fft_common_3d.f90
+++ b/src/fft_common_3d.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,288 +10,290 @@
 !=======================================================================
 
 ! This file contains 3D c2c/r2c/c2r transform subroutines which are
-! identical for several FFT engines
+! identical for several FFT engines 
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_3d_c2c(in, out, isign)
 
-   implicit none
+implicit none
 
-   complex(mytype), dimension(:, :, :), intent(INOUT) :: in
-   complex(mytype), dimension(:, :, :), intent(OUT) :: out
-   integer, intent(IN) :: isign
+complex(mytype), dimension(:,:,:), intent(INOUT) :: in
+complex(mytype), dimension(:,:,:), intent(OUT) :: out
+integer, intent(IN) :: isign
 
-   integer :: i, j, k
+integer :: i, j, k
 
 #ifndef OVERWRITE
-   complex(mytype), allocatable, dimension(:, :, :) :: wk1
+complex(mytype), allocatable, dimension(:,:,:) :: wk1
 #endif
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-   if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
-       format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
+if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
+format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
 
 ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-      call c2c_1m_x(in, isign, ph)
+call c2c_1m_x(in,isign,ph)
 #else
-      allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
-      do concurrent(k=1:ph%xsz(3), j=1:ph%xsz(2), i=1:ph%xsz(1))
-         wk1(i, j, k) = in(i, j, k)
-      end do
-      call c2c_1m_x(wk1, isign, ph)
+allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
+do concurrent (k=1:ph%xsz(3), j=1:ph%xsz(2), i=1:ph%xsz(1))
+wk1(i,j,k) = in(i,j,k)
+end do
+call c2c_1m_x(wk1,isign,ph)
 #endif
 
 ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-      if (dims(1) > 1) then
+if (dims(1)>1) then
 #ifdef OVERWRITE
-         call transpose_x_to_y(in, wk2_c2c, ph)
+call transpose_x_to_y(in,wk2_c2c,ph)
 #else
-         call transpose_x_to_y(wk1, wk2_c2c, ph)
+call transpose_x_to_y(wk1,wk2_c2c,ph)
 #endif
-         call c2c_1m_y(wk2_c2c, isign, ph)
-      else
+call c2c_1m_y(wk2_c2c,isign,ph)
+else
 #ifdef OVERWRITE
-         call c2c_1m_y(in, isign, ph)
+call c2c_1m_y(in,isign,ph)
 #else
-         call c2c_1m_y(wk1, isign, ph)
+call c2c_1m_y(wk1,isign,ph)
 #endif
-      end if
+end if
 
 ! ===== Swap Y --> Z; 1D FFTs in Z =====
-      if (dims(1) > 1) then
-         call transpose_y_to_z(wk2_c2c, out, ph)
-      else
+if (dims(1)>1) then
+call transpose_y_to_z(wk2_c2c,out,ph)
+else
 #ifdef OVERWRITE
-         call transpose_y_to_z(in, out, ph)
+call transpose_y_to_z(in,out,ph)
 #else
-         call transpose_y_to_z(wk1, out, ph)
+call transpose_y_to_z(wk1,out,ph)
 #endif
-      end if
-      call c2c_1m_z(out, isign, ph)
+end if
+call c2c_1m_z(out,isign,ph)
 
-   else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
-            .OR. &
-            format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
+else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
+.OR. & 
+format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
 
 ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-      call c2c_1m_z(in, isign, ph)
+call c2c_1m_z(in,isign,ph)
 #else
-      allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
-      do concurrent(k=1:ph%zsz(3), j=1:ph%zsz(2), i=1:ph%zsz(1))
-         wk1(i, j, k) = in(i, j, k)
-      end do
-      call c2c_1m_z(wk1, isign, ph)
+allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
+do concurrent (k=1:ph%zsz(3), j=1:ph%zsz(2), i=1:ph%zsz(1))
+wk1(i,j,k) = in(i,j,k)
+end do
+call c2c_1m_z(wk1,isign,ph)
 #endif
 
 ! ===== Swap Z --> Y; 1D FFTs in Y =====
-      if (dims(1) > 1) then
+if (dims(1)>1) then
 #ifdef OVERWRITE
-         call transpose_z_to_y(in, wk2_c2c, ph)
+call transpose_z_to_y(in,wk2_c2c,ph)
 #else
-         call transpose_z_to_y(wk1, wk2_c2c, ph)
+call transpose_z_to_y(wk1,wk2_c2c,ph)
 #endif
-         call c2c_1m_y(wk2_c2c, isign, ph)
-      else  ! out==wk2_c2c if 1D decomposition
+call c2c_1m_y(wk2_c2c,isign,ph)
+else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-         call transpose_z_to_y(in, out, ph)
+call transpose_z_to_y(in,out,ph)
 #else
-         call transpose_z_to_y(wk1, out, ph)
+call transpose_z_to_y(wk1,out,ph)
 #endif
-         call c2c_1m_y(out, isign, ph)
-      end if
+call c2c_1m_y(out,isign,ph)
+end if
 
 ! ===== Swap Y --> X; 1D FFTs in X =====
-      if (dims(1) > 1) then
-         call transpose_y_to_x(wk2_c2c, out, ph)
-      end if
-      call c2c_1m_x(out, isign, ph)
+if (dims(1)>1) then
+call transpose_y_to_x(wk2_c2c,out,ph)
+end if
+call c2c_1m_x(out,isign,ph)
 
-   end if
+end if
 
 #ifndef OVERWRITE
 ! Free memory
-   if (allocated(wk1)) deallocate (wk1)
+if (allocated(wk1)) deallocate(wk1)
 #endif
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-   return
+return
 end subroutine fft_3d_c2c
 
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_3d_r2c(in_r, out_c)
 
-   implicit none
+implicit none
 
-   real(mytype), dimension(:, :, :), intent(IN) :: in_r
-   complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
+real(mytype), dimension(:,:,:), intent(IN) :: in_r
+complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-   if (format == PHYSICAL_IN_X) then
+if (format==PHYSICAL_IN_X) then
 
 ! ===== 1D FFTs in X =====
-      call r2c_1m_x(in_r, wk13)
+call r2c_1m_x(in_r,wk13)
 
 ! ===== Swap X --> Y; 1D FFTs in Y =====
-      if (dims(1) > 1) then
-         call transpose_x_to_y(wk13, wk2_r2c, sp)
-         call c2c_1m_y(wk2_r2c, -1, sp)
-      else
-         call c2c_1m_y(wk13, -1, sp)
-      end if
+if (dims(1)>1) then
+call transpose_x_to_y(wk13,wk2_r2c,sp)
+call c2c_1m_y(wk2_r2c,-1,sp)
+else
+call c2c_1m_y(wk13,-1,sp)
+end if
 
 ! ===== Swap Y --> Z; 1D FFTs in Z =====
-      if (dims(1) > 1) then
-         call transpose_y_to_z(wk2_r2c, out_c, sp)
-      else
-         call transpose_y_to_z(wk13, out_c, sp)
-      end if
-      call c2c_1m_z(out_c, -1, sp)
+if (dims(1)>1) then
+call transpose_y_to_z(wk2_r2c,out_c,sp)
+else
+call transpose_y_to_z(wk13,out_c,sp)
+end if
+call c2c_1m_z(out_c,-1,sp)
 
-   else if (format == PHYSICAL_IN_Z) then
+else if (format==PHYSICAL_IN_Z) then
 
 ! ===== 1D FFTs in Z =====
-      call r2c_1m_z(in_r, wk13)
+call r2c_1m_z(in_r,wk13)
 
 ! ===== Swap Z --> Y; 1D FFTs in Y =====
-      if (dims(1) > 1) then
-         call transpose_z_to_y(wk13, wk2_r2c, sp)
-         call c2c_1m_y(wk2_r2c, -1, sp)
-      else  ! out_c==wk2_r2c if 1D decomposition
-         call transpose_z_to_y(wk13, out_c, sp)
-         call c2c_1m_y(out_c, -1, sp)
-      end if
+if (dims(1)>1) then
+call transpose_z_to_y(wk13,wk2_r2c,sp)
+call c2c_1m_y(wk2_r2c,-1,sp)
+else  ! out_c==wk2_r2c if 1D decomposition
+call transpose_z_to_y(wk13,out_c,sp)
+call c2c_1m_y(out_c,-1,sp)
+end if
 
 ! ===== Swap Y --> X; 1D FFTs in X =====
-      if (dims(1) > 1) then
-         call transpose_y_to_x(wk2_r2c, out_c, sp)
-      end if
-      call c2c_1m_x(out_c, -1, sp)
+if (dims(1)>1) then
+call transpose_y_to_x(wk2_r2c,out_c,sp)
+end if
+call c2c_1m_x(out_c,-1,sp)
 
-   end if
+end if
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-   return
+return
 end subroutine fft_3d_r2c
 
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_3d_c2r(in_c, out_r)
 
-   implicit none
+implicit none
 
-   complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
-   real(mytype), dimension(:, :, :), intent(OUT) :: out_r
+complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
+real(mytype), dimension(:,:,:), intent(OUT) :: out_r
 
-   integer :: i, j, k
+integer :: i, j, k
 
 #ifndef OVERWRITE
-   complex(mytype), allocatable, dimension(:, :, :) :: wk1
+complex(mytype), allocatable, dimension(:,:,:) :: wk1
 #endif
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-   if (format == PHYSICAL_IN_X) then
+if (format==PHYSICAL_IN_X) then
 
 ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-      call c2c_1m_z(in_c, 1, sp)
+call c2c_1m_z(in_c,1,sp)       
 #else
-      allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
-      do concurrent(k=1:sp%zsz(3), j=1:sp%zsz(2), i=1:sp%zsz(1))
-         wk1(i, j, k) = in_c(i, j, k)
-      end do
-      call c2c_1m_z(wk1, 1, sp)
+allocate(wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
+do concurrent (k=1:sp%zsz(3), j=1:sp%zsz(2), i=1:sp%zsz(1))
+wk1(i,j,k) = in_c(i,j,k)
+end do
+call c2c_1m_z(wk1,1,sp)
 #endif
 
 ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-      call transpose_z_to_y(in_c, wk2_r2c, sp)
+call transpose_z_to_y(in_c,wk2_r2c,sp)
 #else
-      call transpose_z_to_y(wk1, wk2_r2c, sp)
+call transpose_z_to_y(wk1,wk2_r2c,sp)
 #endif
-      call c2c_1m_y(wk2_r2c, 1, sp)
+call c2c_1m_y(wk2_r2c,1,sp)
 
 ! ===== Swap Y --> X; 1D FFTs in X =====
-      if (dims(1) > 1) then
-         call transpose_y_to_x(wk2_r2c, wk13, sp)
-         call c2r_1m_x(wk13, out_r)
-      else
-         call c2r_1m_x(wk2_r2c, out_r)
-      end if
+if (dims(1)>1) then
+call transpose_y_to_x(wk2_r2c,wk13,sp)
+call c2r_1m_x(wk13,out_r)
+else
+call c2r_1m_x(wk2_r2c,out_r)
+end if
 
-   else if (format == PHYSICAL_IN_Z) then
+else if (format==PHYSICAL_IN_Z) then
 
 ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-      call c2c_1m_x(in_c, 1, sp)
+call c2c_1m_x(in_c,1,sp)
 #else
-      allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
-      do concurrent(k=1:sp%xsz(3), j=1:sp%xsz(2), i=1:sp%xsz(1))
-         wk1(i, j, k) = in_c(i, j, k)
-      end do
-      call c2c_1m_x(wk1, 1, sp)
+allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
+do concurrent (k=1:sp%xsz(3), j=1:sp%xsz(2), i=1:sp%xsz(1))
+wk1(i,j,k) = in_c(i,j,k)
+end do
+call c2c_1m_x(wk1,1,sp)
 #endif
 
 ! ===== Swap X --> Y; 1D FFTs in Y =====
-      if (dims(1) > 1) then
+if (dims(1)>1) then
 #ifdef OVERWRITE
-         call transpose_x_to_y(in_c, wk2_r2c, sp)
+call transpose_x_to_y(in_c,wk2_r2c,sp)
 #else
-         call transpose_x_to_y(wk1, wk2_r2c, sp)
+call transpose_x_to_y(wk1,wk2_r2c,sp)
 #endif
-         call c2c_1m_y(wk2_r2c, 1, sp)
-      else  ! in_c==wk2_r2c if 1D decomposition
+call c2c_1m_y(wk2_r2c,1,sp)
+else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-         call c2c_1m_y(in_c, 1, sp)
+call c2c_1m_y(in_c,1,sp)
 #else
-         call c2c_1m_y(wk1, 1, sp)
+call c2c_1m_y(wk1,1,sp)
 #endif
-      end if
+end if
 
 ! ===== Swap Y --> Z; 1D FFTs in Z =====
-      if (dims(1) > 1) then
-         call transpose_y_to_z(wk2_r2c, wk13, sp)
-      else
+if (dims(1)>1) then
+call transpose_y_to_z(wk2_r2c,wk13,sp)
+else
 #ifdef OVERWRITE
-         call transpose_y_to_z(in_c, wk13, sp)
+call transpose_y_to_z(in_c,wk13,sp)
 #else
-         call transpose_y_to_z(wk1, wk13, sp)
+call transpose_y_to_z(wk1,wk13,sp)
 #endif
-      end if
-      call c2r_1m_z(wk13, out_r)
+end if
+call c2r_1m_z(wk13,out_r)
 
-   end if
+end if
 
 #ifndef OVERWRITE
 ! Free memory
-   if (allocated(wk1)) deallocate (wk1)
+if (allocated(wk1)) deallocate(wk1)
 #endif
 
 #ifdef PROFILER
-   if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-   return
+return
 end subroutine fft_3d_c2r
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 5f7575df..0622f958 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,1022 +13,1032 @@
 
 module decomp_2d_fft
 
-   use decomp_2d  ! 2D decomposition module
-   use iso_c_binding
-   use cudafor
-   use cufft
+  use decomp_2d  ! 2D decomposition module
+  use iso_c_binding
+  use cudafor
+  use cufft
 
-   implicit none
+  implicit none
 
-   private        ! Make everything private unless declared public
+  private        ! Make everything private unless declared public
 
-   ! engine-specific global variables
-   ! integer, save :: plan_type = FFTW_MEASURE
+  ! engine-specific global variables
+  ! integer, save :: plan_type = FFTW_MEASURE
 
-   ! FFTW plans
-   ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
-   ! For c2c transforms:
-   !     use plan(-1,j) for  forward transform;
-   !     use plan( 1,j) for backward transform;
-   ! For r2c/c2r transforms:
-   !     use plan(0,j) for r2c transforms;
-   !     use plan(2,j) for c2r transforms;
-   integer*4, save :: plan(-1:2, 3)
-   complex*8, device, allocatable, dimension(:) :: cufft_workspace
+  ! FFTW plans
+  ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
+  ! For c2c transforms: 
+  !     use plan(-1,j) for  forward transform; 
+  !     use plan( 1,j) for backward transform;
+  ! For r2c/c2r transforms:
+  !     use plan(0,j) for r2c transforms;
+  !     use plan(2,j) for c2r transforms;
+  integer*4, save :: plan(-1:2,3)
+  complex*8, device, allocatable, dimension(:) :: cufft_workspace
 
-   ! common code used for all engines, including global variables,
-   ! generic interface definitions and several subroutines
+  ! common code used for all engines, including global variables, 
+  ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
-   ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2C case
-   subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
-
-      implicit none
-
-      integer*4, intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-      integer, intent(IN) :: cufft_type
-
-      integer :: istat
-      integer(int_ptr_kind()), intent(out) :: worksize
-      integer, pointer :: null_fptr
-      call c_f_pointer(c_null_ptr, null_fptr)
-
-      istat = cufftCreate(plan1)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-      istat = cufftSetAutoAllocation(plan1, 0)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-      istat = cufftMakePlanMany(plan1, 1, decomp%xsz(1), &
-                                decomp%xsz(1), 1, decomp%xsz(1), &
-                                decomp%xsz(1), 1, decomp%xsz(1), &
-                                cufft_type, decomp%xsz(2)*decomp%xsz(3), worksize)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-   end subroutine c2c_1m_x_plan
-
-   ! Return a cuFFT plan for multiple 1D FFTs in Y direction: C2C case
-   subroutine c2c_1m_y_plan(plan1, decomp, cufft_type, worksize)
-
-      implicit none
-
-      integer*4, intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-      integer, intent(IN) :: cufft_type
-
-      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-      ! done one Z-plane at a time. So plan for 2D data sets here.
-      integer :: istat
-      integer(int_ptr_kind()), intent(out) :: worksize
-      integer, pointer :: null_fptr
-      call c_f_pointer(c_null_ptr, null_fptr)
-
-      istat = cufftCreate(plan1)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-      istat = cufftSetAutoAllocation(plan1, 0)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-      istat = cufftMakePlanMany(plan1, 1, decomp%ysz(2), &
-                                decomp%ysz(2), decomp%ysz(1), 1, &
-                                decomp%ysz(2), decomp%ysz(1), 1, &
-                                cufft_type, decomp%ysz(1), worksize)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-   end subroutine c2c_1m_y_plan
-
-   ! Return a cuFFT plan for multiple 1D FFTs in Z direction: C2C case
-   subroutine c2c_1m_z_plan(plan1, decomp, cufft_type, worksize)
-
-      implicit none
-
-      integer*4, intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-      integer, intent(IN) :: cufft_type
-
-      integer :: istat
-      integer(int_ptr_kind()), intent(out) :: worksize
-      integer, pointer :: null_fptr
-      call c_f_pointer(c_null_ptr, null_fptr)
-
-      istat = cufftCreate(plan1)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-      istat = cufftSetAutoAllocation(plan1, 0)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-      istat = cufftMakePlanMany(plan1, 1, decomp%zsz(3), &
-                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
-                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
-                                cufft_type, decomp%zsz(1)*decomp%zsz(2), worksize)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-   end subroutine c2c_1m_z_plan
-
-   ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
-   subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
-
-      implicit none
-
-      integer*4, intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-      integer, intent(IN) :: cufft_type
-
-      integer :: istat
-      integer(int_ptr_kind()), intent(out) :: worksize
-      integer, pointer :: null_fptr
-      call c_f_pointer(c_null_ptr, null_fptr)
-
-      istat = cufftCreate(plan1)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-      istat = cufftSetAutoAllocation(plan1, 0)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-      istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
-                                decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-                                decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-   end subroutine r2c_1m_x_plan
-
-   ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
-   subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
-
-      implicit none
-
-      integer*4, intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-      integer, intent(IN) :: cufft_type
-
-      integer :: istat
-      integer(int_ptr_kind()), intent(out) :: worksize
-      integer, pointer :: null_fptr
-      call c_f_pointer(c_null_ptr, null_fptr)
-
-      istat = cufftCreate(plan1)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-      istat = cufftSetAutoAllocation(plan1, 0)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-      istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
-                                decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-                                decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-   end subroutine c2r_1m_x_plan
-
-   ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
-   subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
-
-      implicit none
-
-      integer*4, intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-      integer, intent(IN) :: cufft_type
-
-      integer :: istat
-      integer(int_ptr_kind()), intent(out) :: worksize
-      integer, pointer :: null_fptr
-      call c_f_pointer(c_null_ptr, null_fptr)
-
-      istat = cufftCreate(plan1)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-      istat = cufftSetAutoAllocation(plan1, 0)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-      istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
-                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
-                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
-                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-   end subroutine r2c_1m_z_plan
-
-   ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
-   subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
-
-      implicit none
-
-      integer*4, intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-      integer, intent(IN) :: cufft_type
-
-      integer :: istat
-      integer(int_ptr_kind()), intent(out) :: worksize
-      integer, pointer :: null_fptr
-      call c_f_pointer(c_null_ptr, null_fptr)
-
-      istat = cufftCreate(plan1)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-      istat = cufftSetAutoAllocation(plan1, 0)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-      istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
-                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
-                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
-                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-   end subroutine c2r_1m_z_plan
+  ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2C case
+  subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
+    
+    implicit none
+
+    integer*4, intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: cufft_type
+
+    integer :: istat
+    integer(int_ptr_kind()), intent(out) :: worksize
+    integer, pointer :: null_fptr
+    call c_f_pointer( c_null_ptr, null_fptr )
+   
+    istat = cufftCreate(plan1)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+    istat = cufftSetAutoAllocation(plan1,0)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+    istat = cufftMakePlanMany(plan1,1,decomp%xsz(1),     &
+                                      decomp%xsz(1),1,decomp%xsz(1),     &
+                                      decomp%xsz(1),1,decomp%xsz(1),     &
+                                      cufft_type,decomp%xsz(2)*decomp%xsz(3),worksize)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+  end subroutine c2c_1m_x_plan
+
+  ! Return a cuFFT plan for multiple 1D FFTs in Y direction: C2C case
+  subroutine c2c_1m_y_plan(plan1, decomp, cufft_type,worksize)
+
+    implicit none
+
+    integer*4, intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: cufft_type
+
+    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+    ! done one Z-plane at a time. So plan for 2D data sets here.
+    integer :: istat
+    integer(int_ptr_kind()), intent(out) :: worksize
+    integer, pointer :: null_fptr
+    call c_f_pointer( c_null_ptr, null_fptr )
+   
+    istat = cufftCreate(plan1)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+    istat = cufftSetAutoAllocation(plan1,0)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+    istat = cufftMakePlanMany(plan1,1,decomp%ysz(2),     &
+                                      decomp%ysz(2),decomp%ysz(1),1,     &
+                                      decomp%ysz(2),decomp%ysz(1),1,     &
+                                      cufft_type,decomp%ysz(1),worksize)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+  end subroutine c2c_1m_y_plan
+
+  ! Return a cuFFT plan for multiple 1D FFTs in Z direction: C2C case
+  subroutine c2c_1m_z_plan(plan1, decomp, cufft_type,worksize)
+
+    implicit none
+
+    integer*4, intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: cufft_type
+
+    integer :: istat
+    integer(int_ptr_kind()), intent(out) :: worksize
+    integer, pointer :: null_fptr
+    call c_f_pointer( c_null_ptr, null_fptr )
+
+    istat = cufftCreate(plan1)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+    istat = cufftSetAutoAllocation(plan1,0)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+    istat = cufftMakePlanMany(plan1,1,decomp%zsz(3),                           &
+                                      decomp%zsz(3),decomp%zsz(1)*decomp%zsz(2),1,     &
+                                      decomp%zsz(3),decomp%zsz(1)*decomp%zsz(2),1,     &
+                                      cufft_type,decomp%zsz(1)*decomp%zsz(2),worksize)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+  end subroutine c2c_1m_z_plan
+
+  ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
+  subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
+
+    implicit none
+
+    integer*4, intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    integer, intent(IN) :: cufft_type
+
+    integer :: istat
+    integer(int_ptr_kind()), intent(out) :: worksize
+    integer, pointer :: null_fptr
+    call c_f_pointer( c_null_ptr, null_fptr )
+
+    istat = cufftCreate(plan1)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+    istat = cufftSetAutoAllocation(plan1,0)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+    istat = cufftMakePlanMany(plan1,1,decomp_ph%xsz(1),     &
+                                      decomp_ph%xsz(1),1,decomp_ph%xsz(1),     &
+                                      decomp_sp%xsz(1),1,decomp_sp%xsz(1),     &
+                                      cufft_type,decomp_ph%xsz(2)*decomp_ph%xsz(3),worksize)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+  end subroutine r2c_1m_x_plan
+
+  ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
+  subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
+
+    implicit none
+
+    integer*4, intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    integer, intent(IN) :: cufft_type
+
+    integer :: istat
+    integer(int_ptr_kind()), intent(out) :: worksize
+    integer, pointer :: null_fptr
+    call c_f_pointer( c_null_ptr, null_fptr )
+
+    istat = cufftCreate(plan1)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+    istat = cufftSetAutoAllocation(plan1,0)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+    istat = cufftMakePlanMany(plan1,1,decomp_ph%xsz(1),     &
+                                      decomp_sp%xsz(1),1,decomp_sp%xsz(1),     &
+                                      decomp_ph%xsz(1),1,decomp_ph%xsz(1),     &
+                                      cufft_type,decomp_ph%xsz(2)*decomp_ph%xsz(3),worksize)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+  end subroutine c2r_1m_x_plan
+
+  ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
+  subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
+
+    implicit none
+
+    integer*4, intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    integer, intent(IN) :: cufft_type
+
+    integer :: istat
+    integer(int_ptr_kind()), intent(out) :: worksize
+    integer, pointer :: null_fptr
+    call c_f_pointer( c_null_ptr, null_fptr )
+
+    istat = cufftCreate(plan1)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+    istat = cufftSetAutoAllocation(plan1,0)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+    istat = cufftMakePlanMany(plan1,1,decomp_ph%zsz(3),                           &
+                                      decomp_ph%zsz(3),decomp_ph%zsz(1)*decomp_ph%zsz(2),1,     &
+                                      decomp_sp%zsz(3),decomp_sp%zsz(1)*decomp_sp%zsz(2),1,     &
+                                      cufft_type,decomp_ph%zsz(1)*decomp_ph%zsz(2),worksize)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+  end subroutine r2c_1m_z_plan
+
+  ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
+  subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
+
+    implicit none
+
+    integer*4, intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    integer, intent(IN) :: cufft_type
+
+    integer :: istat
+    integer(int_ptr_kind()), intent(out) :: worksize
+    integer, pointer :: null_fptr
+    call c_f_pointer( c_null_ptr, null_fptr )
+
+    istat = cufftCreate(plan1)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+    istat = cufftSetAutoAllocation(plan1,0)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+    istat = cufftMakePlanMany(plan1,1,decomp_ph%zsz(3),                           &
+                                      decomp_sp%zsz(3),decomp_sp%zsz(1)*decomp_sp%zsz(2),1,     &
+                                      decomp_ph%zsz(3),decomp_ph%zsz(1)*decomp_ph%zsz(2),1,     &
+                                      cufft_type,decomp_ph%zsz(1)*decomp_ph%zsz(2),worksize)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+  end subroutine c2r_1m_z_plan
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time initialisations for the FFT engine
+  !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine init_fft_engine
-
-      implicit none
+  subroutine init_fft_engine
 
-      !integer*4 :: cufft_ws, ws
-      integer(int_ptr_kind()) :: cufft_ws, ws
-      integer   :: i, j, istat
+    implicit none
 
-      if (nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the New cuFFT engine *****'
-         write (*, *) ' '
-      end if
+    !integer*4 :: cufft_ws, ws
+    integer(int_ptr_kind()) :: cufft_ws, ws
+    integer   :: i, j, istat
 
-      cufft_ws = 0
+    if (nrank==0) then
+       write(*,*) ' '
+       write(*,*) '***** Using the New cuFFT engine *****'
+       write(*,*) ' '
+    end if
+    
+     cufft_ws = 0 
 #ifdef DOUBLE_PREC
-      if (format == PHYSICAL_IN_X) then
-         ! For C2C transforms
-         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         ! For R2C/C2R tranforms
-         call r2c_1m_x_plan(plan(0, 1), ph, sp, CUFFT_D2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(0, 3), sp, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(2, 3), sp, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2r_1m_x_plan(plan(2, 1), sp, ph, CUFFT_Z2D, ws)
-         cufft_ws = max(cufft_ws, ws)
-
-      else if (format == PHYSICAL_IN_Z) then
-
-         ! For C2C transforms
-         write (*, *) 'Create the plans'
-         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-
-         ! For R2C/C2R tranforms
-         call r2c_1m_z_plan(plan(0, 3), ph, sp, CUFFT_D2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(0, 1), sp, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(2, 1), sp, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_Z2Z, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2r_1m_z_plan(plan(2, 3), sp, ph, CUFFT_Z2D, ws)
-         cufft_ws = max(cufft_ws, ws)
-
-      end if
+    if (format == PHYSICAL_IN_X) then
+       ! For C2C transforms
+       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       ! For R2C/C2R tranforms
+       call r2c_1m_x_plan(plan(0,1), ph, sp, CUFFT_D2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan(0,3), sp, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan(2,3), sp, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2r_1m_x_plan(plan(2,1), sp, ph, CUFFT_Z2D,ws)
+       cufft_ws = max (cufft_ws,ws)
+
+    else if (format == PHYSICAL_IN_Z) then
+
+       ! For C2C transforms
+       write(*,*) 'Create the plans'
+       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+
+       ! For R2C/C2R tranforms
+       call r2c_1m_z_plan(plan(0,3), ph, sp, CUFFT_D2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan(0,1), sp, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan(2,1), sp, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_Z2Z,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2r_1m_z_plan(plan(2,3), sp, ph, CUFFT_Z2D,ws)
+       cufft_ws = max (cufft_ws,ws)
+
+    end if
 #else
-      if (format == PHYSICAL_IN_X) then
-         ! For C2C transforms
-         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         ! For R2C/C2R tranforms
-         call r2c_1m_x_plan(plan(0, 1), ph, sp, CUFFT_R2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(0, 3), sp, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(2, 3), sp, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2r_1m_x_plan(plan(2, 1), sp, ph, CUFFT_C2R, ws)
-         cufft_ws = max(cufft_ws, ws)
-
-      else if (format == PHYSICAL_IN_Z) then
-
-         ! For C2C transforms
-         write (*, *) 'Create the plans on rank ', nproc
-         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-
-         ! For R2C/C2R tranforms
-         call r2c_1m_z_plan(plan(0, 3), ph, sp, CUFFT_R2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(0, 1), sp, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_x_plan(plan(2, 1), sp, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_C2C, ws)
-         cufft_ws = max(cufft_ws, ws)
-         call c2r_1m_z_plan(plan(2, 3), sp, ph, CUFFT_C2R, ws)
-         cufft_ws = max(cufft_ws, ws)
-
-      end if
+    if (format == PHYSICAL_IN_X) then
+       ! For C2C transforms
+       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       ! For R2C/C2R tranforms
+       call r2c_1m_x_plan(plan(0,1), ph, sp, CUFFT_R2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan(0,3), sp, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan(2,3), sp, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2r_1m_x_plan(plan(2,1), sp, ph, CUFFT_C2R,ws)
+       cufft_ws = max (cufft_ws,ws)
+
+    else if (format == PHYSICAL_IN_Z) then
+
+       ! For C2C transforms
+       write(*,*) 'Create the plans on rank ', nproc
+       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+
+       ! For R2C/C2R tranforms
+       call r2c_1m_z_plan(plan(0,3), ph, sp, CUFFT_R2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan(0,1), sp, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_x_plan(plan(2,1), sp, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_C2C,ws)
+       cufft_ws = max (cufft_ws,ws)
+       call c2r_1m_z_plan(plan(2,3), sp, ph, CUFFT_C2R,ws)
+       cufft_ws = max (cufft_ws,ws)
+
+    end if
 #endif
-      allocate (cufft_workspace(cufft_ws))
-      do j = 1, 3
-         do i = -1, 2
-            istat = cufftSetWorkArea(plan(i, j), cufft_workspace)
-            if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetWorkArea")
-         end do
-      end do
+    allocate(cufft_workspace(cufft_ws))
+    do j=1,3
+       do i=-1,2
+         istat = cufftSetWorkArea(plan(i,j),cufft_workspace)
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetWorkArea")
+      enddo
+    enddo
+
+  end subroutine init_fft_engine
 
-   end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time finalisations for the FFT engine
+  !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine finalize_fft_engine
+  subroutine finalize_fft_engine
 
-      implicit none
+    implicit none
 
-      integer :: i, j, istat
+    integer :: i,j, istat
 
-      do j = 1, 3
-         do i = -1, 2
-            istat = cufftDestroy(plan(i, j))
-            if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftDestroy")
-         end do
-      end do
+    do j=1,3
+       do i=-1,2
+          istat = cufftDestroy(plan(i,j))
+          if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftDestroy")
+       end do
+    end do
 
-   end subroutine finalize_fft_engine
+  end subroutine finalize_fft_engine
 
-   ! Following routines calculate multiple one-dimensional FFTs to form
-   ! the basis of three-dimensional FFTs.
 
-   ! c2c transform, multiple 1D FFTs in x direction
-   subroutine c2c_1m_x(inout, isign, plan1)
+  ! Following routines calculate multiple one-dimensional FFTs to form 
+  ! the basis of three-dimensional FFTs.
 
-      implicit none
+  ! c2c transform, multiple 1D FFTs in x direction
+  subroutine c2c_1m_x(inout, isign, plan1)
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      integer*4, intent(IN) :: plan1
+    implicit none
 
-      complex(mytype), dimension(:, :, :), allocatable :: output
-      integer :: istat
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    integer*4, intent(IN) :: plan1
 
-      allocate (output, mold=inout)
+    complex(mytype), dimension(:,:,:), allocatable :: output
+    integer :: istat
+
+    allocate(output,mold=inout)
 #ifdef DOUBLE_PREC
-      !$acc host_data use_device(inout,output)
-      istat = cufftExecZ2Z(plan1, inout, output, isign)
-      !$acc end host_data
+    !$acc host_data use_device(inout,output)
+    istat = cufftExecZ2Z(plan1, inout, output,isign)
+    !$acc end host_data
 #else
-      !$acc host_data use_device(inout,output)
-      istat = cufftExecC2C(plan1, inout, output, isign)
-      !$acc end host_data
+    !$acc host_data use_device(inout,output)
+    istat = cufftExecC2C(plan1, inout, output,isign)
+    !$acc end host_data
 #endif
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-      !$acc kernels
-      inout = output
-      !$acc end kernels
-      deallocate (output)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
+    !$acc kernels
+    inout = output
+    !$acc end kernels 
+    deallocate(output)
 
-   end subroutine c2c_1m_x
+  end subroutine c2c_1m_x
 
-   ! c2c transform, multiple 1D FFTs in y direction
-   subroutine c2c_1m_y(inout, isign, plan1)
 
-      implicit none
+  ! c2c transform, multiple 1D FFTs in y direction
+  subroutine c2c_1m_y(inout, isign, plan1)
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      integer*4, intent(IN) :: plan1
+    implicit none
 
-      complex(mytype), dimension(:, :, :), allocatable :: output
-      integer :: s3, k, istat
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    integer*4, intent(IN) :: plan1
 
-      allocate (output, mold=inout)
-      ! transform on one Z-plane at a time
-      s3 = size(inout, 3)
-      do k = 1, s3
+    complex(mytype), dimension(:,:,:), allocatable :: output
+    integer :: s3, k, istat
+    
+    allocate(output,mold=inout)
+    ! transform on one Z-plane at a time
+    s3 = size(inout,3)
+    do k=1,s3
 #ifdef DOUBLE_PREC
-         !$acc host_data use_device(inout,output)
-         istat = cufftExecZ2Z(plan1, inout(:, :, k), output(:, :, k), isign)
-         !$acc end host_data
+       !$acc host_data use_device(inout,output)
+       istat = cufftExecZ2Z(plan1, inout(:,:,k), output(:,:,k),isign)
+       !$acc end host_data
 #else
-         !$acc host_data use_device(inout,output)
-         istat = cufftExecC2C(plan1, inout(:, :, k), output(:, :, k), isign)
-         !$acc end host_data
+       !$acc host_data use_device(inout,output)
+       istat = cufftExecC2C(plan1, inout(:,:,k), output(:,:,k),isign)
+       !$acc end host_data
 #endif
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-      end do
-      !$acc kernels
-      inout = output
-      !$acc end kernels
-      deallocate (output)
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
+    enddo
+    !$acc kernels
+    inout = output
+    !$acc end kernels 
+    deallocate(output)
 
-   end subroutine c2c_1m_y
+  end subroutine c2c_1m_y
 
-   ! c2c transform, multiple 1D FFTs in z direction
-   subroutine c2c_1m_z(inout, isign, plan1)
+  ! c2c transform, multiple 1D FFTs in z direction
+  subroutine c2c_1m_z(inout, isign, plan1)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      integer*4, intent(IN) :: plan1
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    integer*4, intent(IN) :: plan1
+    
+    complex(mytype), dimension(:,:,:), allocatable :: output
+    integer :: istat
 
-      complex(mytype), dimension(:, :, :), allocatable :: output
-      integer :: istat
+    allocate(output,mold=inout)
 
-      allocate (output, mold=inout)
 
 #ifdef DOUBLE_PREC
-      !$acc host_data use_device(inout,output)
-      istat = cufftExecZ2Z(plan1, inout, output, isign)
-      !$acc end host_data
+    !$acc host_data use_device(inout,output)
+    istat = cufftExecZ2Z(plan1, inout, output,isign)
+    !$acc end host_data
 #else
-      !$acc host_data use_device(inout,output)
-      istat = cufftExecC2C(plan1, inout, output, isign)
-      !$acc end host_data
+    !$acc host_data use_device(inout,output)
+    istat = cufftExecC2C(plan1, inout, output,isign)
+    !$acc end host_data
 #endif
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-      !$acc kernels
-      inout = output
-      !$acc end kernels
-      deallocate (output)
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
+    !$acc kernels
+    inout = output
+    !$acc end kernels
+    deallocate(output)
+
+  end subroutine c2c_1m_z
 
-   end subroutine c2c_1m_z
+  ! r2c transform, multiple 1D FFTs in x direction
+  subroutine r2c_1m_x(input, output)
 
-   ! r2c transform, multiple 1D FFTs in x direction
-   subroutine r2c_1m_x(input, output)
+    implicit none
 
-      implicit none
+    real(mytype), dimension(:,:,:), intent(IN)  ::  input
+    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+    integer :: istat
 
-      real(mytype), dimension(:, :, :), intent(IN)  ::  input
-      complex(mytype), dimension(:, :, :), intent(OUT) :: output
-      integer :: istat
 
 #ifdef DOUBLE_PREC
-      !$acc host_data use_device(input,output)
-      istat = cufftExecD2Z(plan(0, 1), input, output)
-      !$acc end host_data
+    !$acc host_data use_device(input,output)
+    istat =  cufftExecD2Z(plan(0,1), input, output)
+    !$acc end host_data
 #else
-      !$acc host_data use_device(input,output)
-      istat = cufftExecR2C(plan(0, 1), input, output)
-      !$acc end host_data
-#endif
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
+    !$acc host_data use_device(input,output)
+    istat =  cufftExecR2C(plan(0,1), input, output)
+    !$acc end host_data
+#endif    
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
 
-   end subroutine r2c_1m_x
+  end subroutine r2c_1m_x
 
-   ! r2c transform, multiple 1D FFTs in z direction
-   subroutine r2c_1m_z(input, output)
+  ! r2c transform, multiple 1D FFTs in z direction
+  subroutine r2c_1m_z(input, output)
 
-      implicit none
+    implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN)     ::  input
-      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+    real(mytype), dimension(:,:,:), intent(IN)     ::  input
+    complex(mytype), dimension(:,:,:), intent(OUT) :: output
 
-      integer :: istat
+    integer :: istat
 
 #ifdef DOUBLE_PREC
-      !$acc host_data use_device(input,output)
-      istat = cufftExecD2Z(plan(0, 3), input, output)
-      !$acc end host_data
+    !$acc host_data use_device(input,output)
+    istat = cufftExecD2Z(plan(0,3), input, output)
+    !$acc end host_data
 #else
-      !$acc host_data use_device(input,output)
-      istat = cufftExecR2C(plan(0, 3), input, output)
-      !$acc end host_data
+    !$acc host_data use_device(input,output)
+    istat = cufftExecR2C(plan(0,3), input, output)
+    !$acc end host_data
 #endif
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
 
-   end subroutine r2c_1m_z
+  end subroutine r2c_1m_z
 
-   ! c2r transform, multiple 1D FFTs in x direction
-   subroutine c2r_1m_x(input, output)
+  ! c2r transform, multiple 1D FFTs in x direction
+  subroutine c2r_1m_x(input, output)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
-      real(mytype), dimension(:, :, :), intent(OUT)    :: output
+    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
+    real(mytype), dimension(:,:,:), intent(OUT)    :: output
 
-      integer :: istat
+    integer :: istat
 
 #ifdef DOUBLE_PREC
-      !$acc host_data use_device(input,output)
-      istat = cufftExecZ2D(plan(2, 1), input, output)
-      !$acc end host_data
+    !$acc host_data use_device(input,output)
+    istat = cufftExecZ2D(plan(2,1), input, output)
+    !$acc end host_data
 #else
-      !$acc host_data use_device(input,output)
-      istat = cufftExecC2R(plan(2, 1), input, output)
-      !$acc end host_data
+    !$acc host_data use_device(input,output)
+    istat = cufftExecC2R(plan(2,1), input, output)
+    !$acc end host_data
 #endif
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
 
-   end subroutine c2r_1m_x
+  end subroutine c2r_1m_x
 
-   ! c2r transform, multiple 1D FFTs in z direction
-   subroutine c2r_1m_z(input, output)
+  ! c2r transform, multiple 1D FFTs in z direction
+  subroutine c2r_1m_z(input, output)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: input
-      real(mytype), dimension(:, :, :), intent(OUT) :: output
+    complex(mytype), dimension(:,:,:), intent(IN) :: input
+    real(mytype), dimension(:,:,:), intent(OUT) :: output
 
-      integer :: istat
+    integer :: istat
 
 #ifdef DOUBLE_PREC
-      !$acc host_data use_device(input,output)
-      istat = cufftExecZ2D(plan(2, 3), input, output)
-      !$acc end host_data
+    !$acc host_data use_device(input,output)
+    istat = cufftExecZ2D(plan(2,3), input, output)
+    !$acc end host_data
 #else
-      !$acc host_data use_device(input,output)
-      istat = cufftExecC2R(plan(2, 3), input, output)
-      !$acc end host_data
+    !$acc host_data use_device(input,output)
+    istat = cufftExecC2R(plan(2,3), input, output)
+    !$acc end host_data
 #endif
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
+
+  end subroutine c2r_1m_z
+
 
-   end subroutine c2r_1m_z
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D FFT - complex to complex
+  ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_c2c(in, out, isign)
+  subroutine fft_3d_c2c(in, out, isign)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: in
-      complex(mytype), dimension(:, :, :), intent(OUT) :: out
-      integer, intent(IN) :: isign
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: in
+    complex(mytype), dimension(:,:,:), intent(OUT) :: out
+    integer, intent(IN) :: isign
 
 #ifndef OVERWRITE
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1
+    complex(mytype), allocatable, dimension(:,:,:) :: wk1
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
-          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
+    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
+         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
 
-         ! ===== 1D FFTs in X =====
+       ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in, isign, plan(isign, 1))
+       call c2c_1m_x(in,isign,plan(isign,1))
 #else
-         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
-         wk1 = in
-         call c2c_1m_x(wk1, isign, plan(isign, 1))
+       allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
+       wk1 = in
+       call c2c_1m_x(wk1,isign,plan(isign,1))
 #endif
 
-         ! ===== Swap X --> Y; 1D FFTs in Y =====
+       ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-         if (dims(1) > 1) then
+       if (dims(1)>1) then
 #ifdef OVERWRITE
-            call transpose_x_to_y(in, wk2_c2c, ph)
+          call transpose_x_to_y(in,wk2_c2c,ph)
 #else
-            call transpose_x_to_y(wk1, wk2_c2c, ph)
+          call transpose_x_to_y(wk1,wk2_c2c,ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
-         else
+          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
+       else
 #ifdef OVERWRITE
-            call c2c_1m_y(in, isign, plan(isign, 2))
+          call c2c_1m_y(in,isign,plan(isign,2))
 #else
-            call c2c_1m_y(wk1, isign, plan(isign, 2))
+          call c2c_1m_y(wk1,isign,plan(isign,2))
 #endif
-         end if
+       end if
 
-         ! ===== Swap Y --> Z; 1D FFTs in Z =====
-         if (dims(1) > 1) then
-            call transpose_y_to_z(wk2_c2c, out, ph)
-         else
+       ! ===== Swap Y --> Z; 1D FFTs in Z =====
+       if (dims(1)>1) then
+          call transpose_y_to_z(wk2_c2c,out,ph)
+       else
 #ifdef OVERWRITE
-            call transpose_y_to_z(in, out, ph)
+          call transpose_y_to_z(in,out,ph)
 #else
-            call transpose_y_to_z(wk1, out, ph)
+          call transpose_y_to_z(wk1,out,ph)
 #endif
-         end if
-         call c2c_1m_z(out, isign, plan(isign, 3))
+       end if
+       call c2c_1m_z(out,isign,plan(isign,3))
 
-      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
-               .OR. &
-               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
+    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
+         .OR. & 
+         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
 
-         ! ===== 1D FFTs in Z =====
+       ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in, isign, plan(isign, 3))
+       call c2c_1m_z(in,isign,plan(isign,3))
 #else
-         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
-         wk1 = in
-         call c2c_1m_z(wk1, isign, plan(isign, 3))
+       allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
+       wk1 = in
+       call c2c_1m_z(wk1,isign,plan(isign,3))
 #endif
 
-         ! ===== Swap Z --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
+       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
 #ifdef OVERWRITE
-            call transpose_z_to_y(in, wk2_c2c, ph)
+          call transpose_z_to_y(in,wk2_c2c,ph)
 #else
-            call transpose_z_to_y(wk1, wk2_c2c, ph)
+          call transpose_z_to_y(wk1,wk2_c2c,ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
-         else  ! out==wk2_c2c if 1D decomposition
+          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
+       else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-            call transpose_z_to_y(in, out, ph)
+          call transpose_z_to_y(in,out,ph)
 #else
-            call transpose_z_to_y(wk1, out, ph)
+          call transpose_z_to_y(wk1,out,ph)
 #endif
-            call c2c_1m_y(out, isign, plan(isign, 2))
-         end if
+          call c2c_1m_y(out,isign,plan(isign,2))
+       end if
 
-         ! ===== Swap Y --> X; 1D FFTs in X =====
-         if (dims(1) > 1) then
-            call transpose_y_to_x(wk2_c2c, out, ph)
-         end if
-         call c2c_1m_x(out, isign, plan(isign, 1))
+       ! ===== Swap Y --> X; 1D FFTs in X =====
+       if (dims(1)>1) then
+          call transpose_y_to_x(wk2_c2c,out,ph)
+       end if
+       call c2c_1m_x(out,isign,plan(isign,1))
 
-      end if
+    end if
 
 #ifndef OVERWRITE
-      deallocate (wk1)
+    deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-   end subroutine fft_3d_c2c
+  end subroutine fft_3d_c2c
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D forward FFT - real to complex
+  ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_r2c(in_r, out_c)
-      !use nvtx
-      implicit none
+  subroutine fft_3d_r2c(in_r, out_c)
+    !use nvtx
+    implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN) :: in_r
-      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
-      integer :: i, j, k
-      integer, dimension(3) :: dim3d
+    real(mytype), dimension(:,:,:), intent(IN) :: in_r
+    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
+    integer :: i, j ,k
+    integer, dimension(3) :: dim3d
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+    if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-      if (format == PHYSICAL_IN_X) then
+    if (format==PHYSICAL_IN_X) then
 
-         ! ===== 1D FFTs in X =====
-         call r2c_1m_x(in_r, wk13)
+       ! ===== 1D FFTs in X =====
+       call r2c_1m_x(in_r,wk13)
 
-         ! ===== Swap X --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
-            call transpose_x_to_y(wk13, wk2_r2c, sp)
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
-         else
-            call c2c_1m_y(wk13, -1, plan(0, 2))
-         end if
+       ! ===== Swap X --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
+          call transpose_x_to_y(wk13,wk2_r2c,sp)
+          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
+       else
+          call c2c_1m_y(wk13,-1,plan(0,2))
+       end if
 
-         ! ===== Swap Y --> Z; 1D FFTs in Z =====
-         if (dims(1) > 1) then
-            call transpose_y_to_z(wk2_r2c, out_c, sp)
-         else
-            call transpose_y_to_z(wk13, out_c, sp)
-         end if
-         call c2c_1m_z(out_c, -1, plan(0, 3))
+       ! ===== Swap Y --> Z; 1D FFTs in Z =====
+       if (dims(1)>1) then
+          call transpose_y_to_z(wk2_r2c,out_c,sp)
+       else
+          call transpose_y_to_z(wk13,out_c,sp)
+       end if
+       call c2c_1m_z(out_c,-1,plan(0,3))
 
-      else if (format == PHYSICAL_IN_Z) then
+    else if (format==PHYSICAL_IN_Z) then
 
-         ! ===== 1D FFTs in Z =====
-         !call nvtxStartRange("Z r2c_1m_z")
+       ! ===== 1D FFTs in Z =====
+       !call nvtxStartRange("Z r2c_1m_z")
 #ifdef DEBUG
-         dim3d = shape(in_r)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_r(i, j, k))
-               end do
+       dim3d = shape(in_r)
+       do k = 1, dim3d(3),dim3d(3)/8
+         do j = 1, dim3d(2),dim3d(2)/8
+            do i = 1, dim3d(1),dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_r(i,j,k))
             end do
          end do
+       end do
 #endif
-         call r2c_1m_z(in_r, wk13)
-
-         ! ===== Swap Z --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
-            !call nvtxStartRange("Z1 transpose_z_to_y")
-            call transpose_z_to_y(wk13, wk2_r2c, sp)
-            !call nvtxEndRange
-            !call nvtxStartRange("Z1 c2c_1m_y")
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
-            !call nvtxEndRange
+       call r2c_1m_z(in_r,wk13)
+
+       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
+          !call nvtxStartRange("Z1 transpose_z_to_y")
+          call transpose_z_to_y(wk13,wk2_r2c,sp)
+          !call nvtxEndRange
+          !call nvtxStartRange("Z1 c2c_1m_y")
+          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
+          !call nvtxEndRange
 #ifdef DEBUG
-            write (*, *) 'c2c_1m_y'
-            dim3d = shape(wk2_r2c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
-                        aimag(wk2_r2c(i, j, k))
-                  end do
+          write(*,*) 'c2c_1m_y'
+          dim3d = shape(wk2_r2c)
+          do k = 1, dim3d(3),dim3d(3)/8
+            do j = 1, dim3d(2),dim3d(2)/8
+               do i = 1, dim3d(1),dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i,j,k)),&
+                                aimag(wk2_r2c(i,j,k))
                end do
             end do
-            write (*, *)
-            write (*, *)
+          end do
+          write(*,*)
+          write(*,*)
 #endif
-         else  ! out_c==wk2_r2c if 1D decomposition
-            !call nvtxStartRange("Z transpose_z_to_y")
-            call transpose_z_to_y(wk13, out_c, sp)
-            !call nvtxEndRange
-            !call nvtxStartRange("Z c2c_1m_y")
-            call c2c_1m_y(out_c, -1, plan(0, 2))
-            !call nvtxEndRange
+       else  ! out_c==wk2_r2c if 1D decomposition
+          !call nvtxStartRange("Z transpose_z_to_y")
+          call transpose_z_to_y(wk13,out_c,sp)
+          !call nvtxEndRange
+          !call nvtxStartRange("Z c2c_1m_y")
+          call c2c_1m_y(out_c,-1,plan(0,2))
+          !call nvtxEndRange
 #ifdef DEBUG
-            write (*, *) 'c2c_1m_y2'
-            dim3d = shape(out_c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
-                        aimag(out_c(i, j, k))
-                  end do
+          write(*,*) 'c2c_1m_y2'
+          dim3d = shape(out_c)
+          do k = 1, dim3d(3),dim3d(3)/8
+            do j = 1, dim3d(2),dim3d(2)/8
+               do i = 1, dim3d(1),dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i,j,k)),&
+                                aimag(out_c(i,j,k))
                end do
             end do
-            write (*, *)
-            write (*, *)
+          end do
+          write(*,*)
+          write(*,*)
 #endif
-         end if
-
-         ! ===== Swap Y --> X; 1D FFTs in X =====
-         if (dims(1) > 1) then
-            !call nvtxStartRange("Z1 transpose_y_to_x")
-            call transpose_y_to_x(wk2_r2c, out_c, sp)
-            !call nvtxEndRange
-         end if
-         !call nvtxStartRange("c2c_1m_x")
-         call c2c_1m_x(out_c, -1, plan(0, 1))
-         !call nvtxEndRange
+       end if
+
+       ! ===== Swap Y --> X; 1D FFTs in X =====
+       if (dims(1)>1) then
+          !call nvtxStartRange("Z1 transpose_y_to_x")
+          call transpose_y_to_x(wk2_r2c,out_c,sp)
+          !call nvtxEndRange
+       end if
+       !call nvtxStartRange("c2c_1m_x")
+       call c2c_1m_x(out_c,-1,plan(0,1))
+       !call nvtxEndRange
 #ifdef DEBUG
-         write (*, *) 'c2c_1m_x'
-         dim3d = shape(out_c)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
-                     aimag(out_c(i, j, k))
-               end do
+       write(*,*) 'c2c_1m_x'
+       dim3d = shape(out_c)
+       do k = 1, dim3d(3),dim3d(3)/8
+         do j = 1, dim3d(2),dim3d(2)/8
+            do i = 1, dim3d(1),dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i,j,k)),&
+                             aimag(out_c(i,j,k))
             end do
          end do
-         write (*, *)
-         write (*, *)
+       end do
+       write(*,*)
+       write(*,*)
 #endif
-      end if
+    end if
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+    if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-   end subroutine fft_3d_r2c
+  end subroutine fft_3d_r2c
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D inverse FFT - complex to real
+  ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_c2r(in_c, out_r)
+  subroutine fft_3d_c2r(in_c, out_r)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
-      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
-      integer :: i, j, k
-      integer, dimension(3) :: dim3d
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
+    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
+    integer :: i,j,k
+    integer, dimension(3) :: dim3d
 #ifndef OVERWRITE
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1
+    complex(mytype), allocatable, dimension(:,:,:) :: wk1
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-      if (format == PHYSICAL_IN_X) then
+    if (format==PHYSICAL_IN_X) then
 
-         ! ===== 1D FFTs in Z =====
+       ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in_c, 1, plan(2, 3))
+       call c2c_1m_z(in_c,1,plan(2,3))       
 #else
-         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
-         wk1 = in_c
-         call c2c_1m_z(wk1, 1, plan(2, 3))
+       allocate (wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
+       wk1 = in_c
+       call c2c_1m_z(wk1,1,plan(2,3))
 #endif
 
-         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+       ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-         call transpose_z_to_y(in_c, wk2_r2c, sp)
+       call transpose_z_to_y(in_c,wk2_r2c,sp)
 #else
-         call transpose_z_to_y(wk1, wk2_r2c, sp)
+       call transpose_z_to_y(wk1,wk2_r2c,sp)
 #endif
-         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+       call c2c_1m_y(wk2_r2c,1,plan(2,2))
 
-         ! ===== Swap Y --> X; 1D FFTs in X =====
-         if (dims(1) > 1) then
-            call transpose_y_to_x(wk2_r2c, wk13, sp)
-            call c2r_1m_x(wk13, out_r)
-         else
-            call c2r_1m_x(wk2_r2c, out_r)
-         end if
+       ! ===== Swap Y --> X; 1D FFTs in X =====
+       if (dims(1)>1) then
+          call transpose_y_to_x(wk2_r2c,wk13,sp)
+          call c2r_1m_x(wk13,out_r)
+       else
+          call c2r_1m_x(wk2_r2c,out_r)
+       end if
 
-      else if (format == PHYSICAL_IN_Z) then
+    else if (format==PHYSICAL_IN_Z) then
 #ifdef DEBUG
-         write (*, *) 'Back Init c2c_1m_x line 788'
-         dim3d = shape(in_c)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
-                     aimag(in_c(i, j, k))
-               end do
+       write(*,*) 'Back Init c2c_1m_x line 788'
+       dim3d = shape(in_c)
+       do k = 1, dim3d(3),dim3d(3)/8
+         do j = 1, dim3d(2),dim3d(2)/8
+            do i = 1, dim3d(1),dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i,j,k)),&
+                             aimag(in_c(i,j,k))
             end do
          end do
-         write (*, *)
-         write (*, *)
+       end do
+       write(*,*)
+       write(*,*)
 #endif
-         ! ===== 1D FFTs in X =====
+       ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in_c, 1, plan(2, 1))
+       call c2c_1m_x(in_c,1,plan(2,1))
 #ifdef DEBUG
-         write (*, *) 'Back c2c_1m_x overwrite line 804'
-         dim3d = shape(in_c)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
-                     aimag(in_c(i, j, k))
-               end do
+       write(*,*) 'Back c2c_1m_x overwrite line 804'
+       dim3d = shape(in_c)
+       do k = 1, dim3d(3),dim3d(3)/8
+         do j = 1, dim3d(2),dim3d(2)/8
+            do i = 1, dim3d(1),dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i,j,k)),&
+                             aimag(in_c(i,j,k))
             end do
          end do
-         write (*, *)
-         write (*, *)
+       end do
+       write(*,*)
+       write(*,*)
 #endif
 #else
-         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
-         wk1 = in_c
-         call c2c_1m_x(wk1, 1, plan(2, 1))
+       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
+       wk1 = in_c
+       call c2c_1m_x(wk1,1,plan(2,1))
 #ifdef DEBUG
-         write (*, *) 'Back2 c2c_1m_x line 821'
-         dim3d = shape(wk1)
-         do k = 1, dim3d(3), dim3d(1)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
-                     aimag(wk1(i, j, k))
-               end do
+       write(*,*) 'Back2 c2c_1m_x line 821'
+       dim3d = shape(wk1)
+       do k = 1, dim3d(3),dim3d(1)/8
+         do j = 1, dim3d(2),dim3d(2)/8
+            do i = 1, dim3d(1),dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i,j,k)),&
+                             aimag(wk1(i,j,k))
             end do
          end do
-         write (*, *)
-         write (*, *)
+       end do
+       write(*,*)
+       write(*,*)
 #endif
 #endif
 
-         ! ===== Swap X --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
+       ! ===== Swap X --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
 #ifdef OVERWRITE
-            call transpose_x_to_y(in_c, wk2_r2c, sp)
+          call transpose_x_to_y(in_c,wk2_r2c,sp)
 #else
-            call transpose_x_to_y(wk1, wk2_r2c, sp)
+          call transpose_x_to_y(wk1,wk2_r2c,sp)
 #endif
-            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+          call c2c_1m_y(wk2_r2c,1,plan(2,2))
 #ifdef DEBUG
-            write (*, *) 'Back c2c_1m_y line 844'
-            dim3d = shape(wk2_r2c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
-                        aimag(wk2_r2c(i, j, k))
-                  end do
+          write(*,*) 'Back c2c_1m_y line 844'
+          dim3d = shape(wk2_r2c)
+          do k = 1, dim3d(3),dim3d(3)/8
+            do j = 1, dim3d(2),dim3d(2)/8
+               do i = 1, dim3d(1),dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i,j,k)),&
+                                aimag(wk2_r2c(i,j,k))
                end do
             end do
-            write (*, *)
-            write (*, *)
+          end do
+          write(*,*)
+          write(*,*)
 #endif
-         else  ! in_c==wk2_r2c if 1D decomposition
+       else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-            call c2c_1m_y(in_c, 1, plan(2, 2))
+          call c2c_1m_y(in_c,1,plan(2,2))
 #ifdef DEBUG
-            write (*, *) 'Back2 c2c_1m_y line 860'
-            dim3d = shape(in_c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
-                        aimag(in_c(i, j, k))
-                  end do
+          write(*,*) 'Back2 c2c_1m_y line 860'
+          dim3d = shape(in_c)
+          do k = 1, dim3d(3),dim3d(3)/8
+            do j = 1, dim3d(2),dim3d(2)/8
+               do i = 1, dim3d(1),dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i,j,k)),&
+                                aimag(in_c(i,j,k))
                end do
             end do
-            write (*, *)
-            write (*, *)
+          end do
+          write(*,*)
+          write(*,*)
 #endif
 #else
-            call c2c_1m_y(wk1, 1, plan(2, 2))
+          call c2c_1m_y(wk1,1,plan(2,2))
 #ifdef DEBUG
-            write (*, *) 'Back3 c2c_1m_y line 875'
-            dim3d = shape(wk1)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
-                        aimag(wk1(i, j, k))
-                  end do
+          write(*,*) 'Back3 c2c_1m_y line 875'
+          dim3d = shape(wk1)
+          do k = 1, dim3d(3),dim3d(3)/8
+            do j = 1, dim3d(2),dim3d(2)/8
+               do i = 1, dim3d(1),dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i,j,k)),&
+                                aimag(wk1(i,j,k))
                end do
             end do
-            write (*, *)
-            write (*, *)
+          end do
+          write(*,*)
+          write(*,*)
 #endif
 #endif
-         end if
+       end if
 
-         ! ===== Swap Y --> Z; 1D FFTs in Z =====
-         if (dims(1) > 1) then
-            call transpose_y_to_z(wk2_r2c, wk13, sp)
-         else
+       ! ===== Swap Y --> Z; 1D FFTs in Z =====
+       if (dims(1)>1) then
+          call transpose_y_to_z(wk2_r2c,wk13,sp)
+       else
 #ifdef OVERWRITE
-            call transpose_y_to_z(in_c, wk13, sp)
+          call transpose_y_to_z(in_c,wk13,sp)
 #else
-            call transpose_y_to_z(wk1, wk13, sp)
+          call transpose_y_to_z(wk1,wk13,sp)
 #endif
-         end if
+       end if
 #ifdef DEBUG
-         write (*, *) 'Back2 after tr_y2z'
-         dim3d = shape(wk13)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk13(i, j, k)), &
-                     aimag(wk13(i, j, k))
-               end do
+       write(*,*) 'Back2 after tr_y2z'
+       dim3d = shape(wk13)
+       do k = 1, dim3d(3),dim3d(3)/8
+         do j = 1, dim3d(2),dim3d(2)/8
+            do i = 1, dim3d(1),dim3d(1)/8
+              print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk13(i,j,k)),&
+                               aimag(wk13(i,j,k))
             end do
          end do
-         write (*, *)
-         write (*, *)
+       end do
+       write(*,*)
+       write(*,*)
 #endif
-         call c2r_1m_z(wk13, out_r)
+       call c2r_1m_z(wk13,out_r)
 #ifdef DEBUG
-         write (*, *) 'Back2 c2r_1m_z out_r line 902'
-         dim3d = shape(out_r)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_r(i, j, k))
-               end do
+       write(*,*) 'Back2 c2r_1m_z out_r line 902'
+       dim3d = shape(out_r)
+       do k = 1, dim3d(3),dim3d(3)/8
+         do j = 1, dim3d(2),dim3d(2)/8
+            do i = 1, dim3d(1),dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_r(i,j,k))
             end do
          end do
-         write (*, *)
-         write (*, *)
+       end do
+       write(*,*)
+       write(*,*)
 #endif
 
-      end if
+    end if
 
 #ifndef OVERWRITE
-      deallocate (wk1)
+    deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-   end subroutine fft_3d_c2r
+  end subroutine fft_3d_c2r
+
 
 end module decomp_2d_fft
 
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index fb7646ba..91cc0670 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,720 +13,733 @@
 
 module decomp_2d_fft
 
-   use decomp_2d  ! 2D decomposition module
-   use iso_c_binding
+  use decomp_2d  ! 2D decomposition module
+  use iso_c_binding
 
-   implicit none
+  implicit none
 
-   include "fftw3.f"
+  include "fftw3.f"
 
-   private        ! Make everything private unless declared public
+  private        ! Make everything private unless declared public
 
-   ! engine-specific global variables
-   integer, save :: plan_type = FFTW_MEASURE
+  ! engine-specific global variables
+  integer, save :: plan_type = FFTW_MEASURE
 
-   ! FFTW plans
-   ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
-   ! For c2c transforms:
-   !     use plan(-1,j) for  forward transform;
-   !     use plan( 1,j) for backward transform;
-   ! For r2c/c2r transforms:
-   !     use plan(0,j) for r2c transforms;
-   !     use plan(2,j) for c2r transforms;
-   type(C_PTR), save :: plan(-1:2, 3)
+  ! FFTW plans
+  ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
+  ! For c2c transforms: 
+  !     use plan(-1,j) for  forward transform; 
+  !     use plan( 1,j) for backward transform;
+  ! For r2c/c2r transforms:
+  !     use plan(0,j) for r2c transforms;
+  !     use plan(2,j) for c2r transforms;
+  type(C_PTR), save :: plan(-1:2,3)
 
-   ! This is defined in fftw3.f03 but not in fftw3.f
-   interface
-      subroutine fftw_cleanup() bind(C, name='fftw_cleanup')
-         import
-      end subroutine fftw_cleanup
-   end interface
+  ! This is defined in fftw3.f03 but not in fftw3.f
+  interface
+    subroutine fftw_cleanup() bind(C, name='fftw_cleanup')
+      import
+    end subroutine fftw_cleanup
+  end interface
 
-   ! common code used for all engines, including global variables,
-   ! generic interface definitions and several subroutines
+  ! common code used for all engines, including global variables, 
+  ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
-   ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
-   subroutine c2c_1m_x_plan(plan1, decomp, isign)
+  ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
+  subroutine c2c_1m_x_plan(plan1, decomp, isign)
 
-      implicit none
+    implicit none
 
-      type(C_PTR), intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-      integer, intent(IN) :: isign
+    type(C_PTR), intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: isign
 
-      complex(mytype), allocatable, dimension(:, :, :) :: a1
+    complex(mytype), allocatable, dimension(:,:,:) :: a1
 
-      allocate (a1(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)))
+    allocate(a1(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)))
 
 #ifdef DOUBLE_PREC
-      call dfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-                               decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
-                               isign, plan_type)
+    call dfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
+         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+         decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
+         isign, plan_type)
 #else
-      call sfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-                               decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
-                               isign, plan_type)
+    call sfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
+         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+         decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
+         isign, plan_type)
 #endif
 
-      deallocate (a1)
+    deallocate(a1)
 
-      return
-   end subroutine c2c_1m_x_plan
+    return
+  end subroutine c2c_1m_x_plan
 
-   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
-   subroutine c2c_1m_y_plan(plan1, decomp, isign)
+  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
+  subroutine c2c_1m_y_plan(plan1, decomp, isign)
 
-      implicit none
+    implicit none
 
-      type(C_PTR), intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-      integer, intent(IN) :: isign
+    type(C_PTR), intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: isign
 
-      complex(mytype), allocatable, dimension(:, :) :: a1
+    complex(mytype), allocatable, dimension(:,:) :: a1
 
-      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-      ! done one Z-plane at a time. So plan for 2D data sets here.
+    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+    ! done one Z-plane at a time. So plan for 2D data sets here.
 
-      allocate (a1(decomp%ysz(1), decomp%ysz(2)))
+    allocate(a1(decomp%ysz(1),decomp%ysz(2)))
 
 #ifdef DOUBLE_PREC
-      call dfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
-                               a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
-                               decomp%ysz(1), 1, isign, plan_type)
+    call dfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
+         a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
+         decomp%ysz(1), 1, isign, plan_type)
 #else
-      call sfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
-                               a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
-                               decomp%ysz(1), 1, isign, plan_type)
+    call sfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
+         a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
+         decomp%ysz(1), 1, isign, plan_type)
 #endif
 
-      deallocate (a1)
+    deallocate(a1)
+
+    return
+  end subroutine c2c_1m_y_plan
 
-      return
-   end subroutine c2c_1m_y_plan
 
-   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
-   subroutine c2c_1m_z_plan(plan1, decomp, isign)
+  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
+  subroutine c2c_1m_z_plan(plan1, decomp, isign)
 
-      implicit none
+    implicit none
 
-      type(C_PTR), intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-      integer, intent(IN) :: isign
+    type(C_PTR), intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: isign
 
-      complex(mytype), allocatable, dimension(:, :, :) :: a1
+    complex(mytype), allocatable, dimension(:,:,:) :: a1
 
-      allocate (a1(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)))
+    allocate(a1(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)))
 
 #ifdef DOUBLE_PREC
-      call dfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+    call dfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #else
-      call sfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+    call sfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #endif
 
-      deallocate (a1)
+    deallocate(a1)
 
-      return
-   end subroutine c2c_1m_z_plan
+    return
+  end subroutine c2c_1m_z_plan
 
-   ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
-   subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
 
-      implicit none
+  ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
+  subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
 
-      type(C_PTR), intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    implicit none
 
-      real(mytype), allocatable, dimension(:, :, :) :: a1
-      complex(mytype), allocatable, dimension(:, :, :) :: a2
+    type(C_PTR), intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-      allocate (a1(decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)))
-      allocate (a2(decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)))
+    real(mytype), allocatable, dimension(:,:,:) :: a1
+    complex(mytype), allocatable, dimension(:,:,:) :: a2
+
+    allocate(a1(decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)))
+    allocate(a2(decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)))
 #ifdef DOUBLE_PREC
-      call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-                                   decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-                                   plan_type)
+    call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
+         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+         plan_type)
 #else
-      call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-                                   decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-                                   plan_type)
+    call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
+         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+         plan_type)
 #endif
-      deallocate (a1, a2)
+    deallocate(a1,a2)    
+
+    return
+  end subroutine r2c_1m_x_plan
 
-      return
-   end subroutine r2c_1m_x_plan
 
-   ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
-   subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
+  ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
+  subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
 
-      implicit none
+    implicit none
 
-      type(C_PTR), intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    type(C_PTR), intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-      complex(mytype), allocatable, dimension(:, :, :) :: a1
-      real(mytype), allocatable, dimension(:, :, :) :: a2
+    complex(mytype), allocatable, dimension(:,:,:) :: a1
+    real(mytype), allocatable, dimension(:,:,:) :: a2
 
-      allocate (a1(decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)))
-      allocate (a2(decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)))
+    allocate(a1(decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)))
+    allocate(a2(decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)))
 #ifdef DOUBLE_PREC
-      call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-                                   decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-                                   plan_type)
+    call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
+         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+         plan_type)
 #else
-      call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-                                   decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-                                   plan_type)
+    call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
+         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+         plan_type)
 #endif
-      deallocate (a1, a2)
+    deallocate(a1,a2)
 
-      return
-   end subroutine c2r_1m_x_plan
+    return
+  end subroutine c2r_1m_x_plan
 
-   ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
-   subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
 
-      implicit none
+  ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
+  subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
 
-      type(C_PTR), intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    implicit none
 
-      real(mytype), allocatable, dimension(:, :, :) :: a1
-      complex(mytype), allocatable, dimension(:, :, :) :: a2
+    type(C_PTR), intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-      allocate (a1(decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)))
-      allocate (a2(decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)))
+    real(mytype), allocatable, dimension(:,:,:) :: a1
+    complex(mytype), allocatable, dimension(:,:,:) :: a2
+
+    allocate(a1(decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)))
+    allocate(a2(decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)))
 #ifdef DOUBLE_PREC
-      call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+    call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #else
-      call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+    call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #endif
-      deallocate (a1, a2)
+    deallocate(a1,a2)
+
+    return
+  end subroutine r2c_1m_z_plan
 
-      return
-   end subroutine r2c_1m_z_plan
 
-   ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
-   subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
+  ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
+  subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
 
-      implicit none
+    implicit none
 
-      type(C_PTR), intent(OUT) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    type(C_PTR), intent(OUT) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-      complex(mytype), allocatable, dimension(:, :, :) :: a1
-      real(mytype), allocatable, dimension(:, :, :) :: a2
+    complex(mytype), allocatable, dimension(:,:,:) :: a1
+    real(mytype), allocatable, dimension(:,:,:) :: a2
 
-      allocate (a1(decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)))
-      allocate (a2(decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)))
+    allocate(a1(decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)))
+    allocate(a2(decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)))
 
 #ifdef DOUBLE_PREC
-      call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+    call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #else
-      call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+    call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #endif
-      deallocate (a1, a2)
+    deallocate(a1,a2)     
+
+    return
+  end subroutine c2r_1m_z_plan
 
-      return
-   end subroutine c2r_1m_z_plan
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time initialisations for the FFT engine
+  !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine init_fft_engine
-
-      implicit none
-
-      if (nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the FFTW (version 3.x) engine *****'
-         write (*, *) ' '
-      end if
-
-      if (format == PHYSICAL_IN_X) then
-
-         ! For C2C transforms
-         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
-         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
-         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
-         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
-         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
-         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
-
-         ! For R2C/C2R tranforms
-         call r2c_1m_x_plan(plan(0, 1), ph, sp)
-         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
-         call c2c_1m_z_plan(plan(0, 3), sp, FFTW_FORWARD)
-         call c2c_1m_z_plan(plan(2, 3), sp, FFTW_BACKWARD)
-         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
-         call c2r_1m_x_plan(plan(2, 1), sp, ph)
-
-      else if (format == PHYSICAL_IN_Z) then
-
-         ! For C2C transforms
-         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
-         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
-         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
-         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
-         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
-         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
-
-         ! For R2C/C2R tranforms
-         call r2c_1m_z_plan(plan(0, 3), ph, sp)
-         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
-         call c2c_1m_x_plan(plan(0, 1), sp, FFTW_FORWARD)
-         call c2c_1m_x_plan(plan(2, 1), sp, FFTW_BACKWARD)
-         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
-         call c2r_1m_z_plan(plan(2, 3), sp, ph)
-
-      end if
-
-      return
-   end subroutine init_fft_engine
+  subroutine init_fft_engine
+
+    implicit none
+
+    if (nrank==0) then
+       write(*,*) ' '
+       write(*,*) '***** Using the FFTW (version 3.x) engine *****'
+       write(*,*) ' '
+    end if
+
+    if (format == PHYSICAL_IN_X) then
+
+       ! For C2C transforms
+       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
+       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD )
+       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
+       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
+       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
+       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
+
+       ! For R2C/C2R tranforms
+       call r2c_1m_x_plan(plan(0,1), ph, sp)
+       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
+       call c2c_1m_z_plan(plan(0,3), sp, FFTW_FORWARD )
+       call c2c_1m_z_plan(plan(2,3), sp, FFTW_BACKWARD)
+       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
+       call c2r_1m_x_plan(plan(2,1), sp, ph)
+
+    else if (format == PHYSICAL_IN_Z) then
+
+       ! For C2C transforms
+       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
+       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD ) 
+       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
+       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
+       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
+       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
+
+       ! For R2C/C2R tranforms
+       call r2c_1m_z_plan(plan(0,3), ph, sp)
+       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
+       call c2c_1m_x_plan(plan(0,1), sp, FFTW_FORWARD )
+       call c2c_1m_x_plan(plan(2,1), sp, FFTW_BACKWARD)
+       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
+       call c2r_1m_z_plan(plan(2,3), sp, ph)
+
+    end if
+
+    return
+  end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time finalisations for the FFT engine
+  !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine finalize_fft_engine
+  subroutine finalize_fft_engine
 
-      implicit none
+    implicit none
 
-      integer :: i, j
+    integer :: i,j
 
-      do j = 1, 3
-         do i = -1, 2
+    do j=1,3
+       do i=-1,2
 #ifdef DOUBLE_PREC
-            call dfftw_destroy_plan(plan(i, j))
+          call dfftw_destroy_plan(plan(i,j))
 #else
-            call sfftw_destroy_plan(plan(i, j))
+          call sfftw_destroy_plan(plan(i,j))
 #endif
-         end do
-      end do
+       end do
+    end do
 
-      call fftw_cleanup()
+    call fftw_cleanup()
 
-      return
-   end subroutine finalize_fft_engine
+    return
+  end subroutine finalize_fft_engine
 
-   ! Following routines calculate multiple one-dimensional FFTs to form
-   ! the basis of three-dimensional FFTs.
 
-   ! c2c transform, multiple 1D FFTs in x direction
-   subroutine c2c_1m_x(inout, isign, plan1)
+  ! Following routines calculate multiple one-dimensional FFTs to form 
+  ! the basis of three-dimensional FFTs.
 
-      implicit none
+  ! c2c transform, multiple 1D FFTs in x direction
+  subroutine c2c_1m_x(inout, isign, plan1)
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      type(C_PTR), intent(IN) :: plan1
+    implicit none
 
-      integer :: foo
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    type(C_PTR), intent(IN) :: plan1
 
-      foo = isign ! Silence unused dummy argument
+    integer :: foo
 
+    foo = isign ! Silence unused dummy argument
+    
 #ifdef DOUBLE_PREC
-      call dfftw_execute_dft(plan1, inout, inout)
+    call dfftw_execute_dft(plan1, inout, inout)
 #else
-      call sfftw_execute_dft(plan1, inout, inout)
+    call sfftw_execute_dft(plan1, inout, inout)
 #endif
 
-      return
-   end subroutine c2c_1m_x
+    return
+  end subroutine c2c_1m_x
 
-   ! c2c transform, multiple 1D FFTs in y direction
-   subroutine c2c_1m_y(inout, isign, plan1)
 
-      implicit none
+  ! c2c transform, multiple 1D FFTs in y direction
+  subroutine c2c_1m_y(inout, isign, plan1)
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      type(C_PTR), intent(IN) :: plan1
+    implicit none
 
-      integer :: k, s3
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    type(C_PTR), intent(IN) :: plan1
 
-      integer :: foo
+    integer :: k, s3
 
-      foo = isign ! Silence unused dummy argument
+    integer :: foo
 
-      ! transform on one Z-plane at a time
-      s3 = size(inout, 3)
-      do k = 1, s3
+    foo = isign ! Silence unused dummy argument
+    
+    ! transform on one Z-plane at a time
+    s3 = size(inout,3)
+    do k=1,s3
 #ifdef DOUBLE_PREC
-         call dfftw_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
+       call dfftw_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
 #else
-         call sfftw_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
+       call sfftw_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
 #endif
-      end do
-
-      return
-   end subroutine c2c_1m_y
+    end do
 
-   ! c2c transform, multiple 1D FFTs in z direction
-   subroutine c2c_1m_z(inout, isign, plan1)
+    return
+  end subroutine c2c_1m_y
 
-      implicit none
+  ! c2c transform, multiple 1D FFTs in z direction
+  subroutine c2c_1m_z(inout, isign, plan1)
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      type(C_PTR), intent(IN) :: plan1
+    implicit none
 
-      integer :: foo
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    type(C_PTR), intent(IN) :: plan1
 
-      foo = isign ! Silence unused dummy argument
+    integer :: foo
 
+    foo = isign ! Silence unused dummy argument
+    
 #ifdef DOUBLE_PREC
-      call dfftw_execute_dft(plan1, inout, inout)
+    call dfftw_execute_dft(plan1, inout, inout)
 #else
-      call sfftw_execute_dft(plan1, inout, inout)
+    call sfftw_execute_dft(plan1, inout, inout)
 #endif
 
-      return
-   end subroutine c2c_1m_z
+    return
+  end subroutine c2c_1m_z
 
-   ! r2c transform, multiple 1D FFTs in x direction
-   subroutine r2c_1m_x(input, output)
+  ! r2c transform, multiple 1D FFTs in x direction
+  subroutine r2c_1m_x(input, output)
 
-      implicit none
+    implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN)  ::  input
-      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+    real(mytype), dimension(:,:,:), intent(IN)  ::  input
+    complex(mytype), dimension(:,:,:), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-      call dfftw_execute_dft_r2c(plan(0, 1), input, output)
+    call dfftw_execute_dft_r2c(plan(0,1), input, output)
 #else
-      call sfftw_execute_dft_r2c(plan(0, 1), input, output)
-#endif
+    call sfftw_execute_dft_r2c(plan(0,1), input, output)
+#endif    
 
-      return
+    return
 
-   end subroutine r2c_1m_x
+  end subroutine r2c_1m_x
 
-   ! r2c transform, multiple 1D FFTs in z direction
-   subroutine r2c_1m_z(input, output)
+  ! r2c transform, multiple 1D FFTs in z direction
+  subroutine r2c_1m_z(input, output)
 
-      implicit none
+    implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN)  ::  input
-      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+    real(mytype), dimension(:,:,:), intent(IN)  ::  input
+    complex(mytype), dimension(:,:,:), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-      call dfftw_execute_dft_r2c(plan(0, 3), input, output)
+    call dfftw_execute_dft_r2c(plan(0,3), input, output)
 #else
-      call sfftw_execute_dft_r2c(plan(0, 3), input, output)
+    call sfftw_execute_dft_r2c(plan(0,3), input, output)
 #endif
 
-      return
+    return
 
-   end subroutine r2c_1m_z
+  end subroutine r2c_1m_z
 
-   ! c2r transform, multiple 1D FFTs in x direction
-   subroutine c2r_1m_x(input, output)
+  ! c2r transform, multiple 1D FFTs in x direction
+  subroutine c2r_1m_x(input, output)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
-      real(mytype), dimension(:, :, :), intent(OUT) :: output
+    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
+    real(mytype), dimension(:,:,:), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-      call dfftw_execute_dft_c2r(plan(2, 1), input, output)
+    call dfftw_execute_dft_c2r(plan(2,1), input, output)
 #else
-      call sfftw_execute_dft_c2r(plan(2, 1), input, output)
+    call sfftw_execute_dft_c2r(plan(2,1), input, output)
 #endif
 
-      return
+    return
 
-   end subroutine c2r_1m_x
+  end subroutine c2r_1m_x
 
-   ! c2r transform, multiple 1D FFTs in z direction
-   subroutine c2r_1m_z(input, output)
+  ! c2r transform, multiple 1D FFTs in z direction
+  subroutine c2r_1m_z(input, output)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: input
-      real(mytype), dimension(:, :, :), intent(OUT) :: output
+    complex(mytype), dimension(:,:,:), intent(IN) :: input
+    real(mytype), dimension(:,:,:), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-      call dfftw_execute_dft_c2r(plan(2, 3), input, output)
+    call dfftw_execute_dft_c2r(plan(2,3), input, output)
 #else
-      call sfftw_execute_dft_c2r(plan(2, 3), input, output)
-#endif
+    call sfftw_execute_dft_c2r(plan(2,3), input, output)
+#endif    
+
+    return
+
+  end subroutine c2r_1m_z
 
-      return
 
-   end subroutine c2r_1m_z
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D FFT - complex to complex
+  ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_c2c(in, out, isign)
+  subroutine fft_3d_c2c(in, out, isign)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: in
-      complex(mytype), dimension(:, :, :), intent(OUT) :: out
-      integer, intent(IN) :: isign
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: in
+    complex(mytype), dimension(:,:,:), intent(OUT) :: out
+    integer, intent(IN) :: isign
 
 #ifndef OVERWRITE
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1
+    complex(mytype), allocatable, dimension(:,:,:) :: wk1
 #endif
 
-      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
-          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
+    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
+         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
 
-         ! ===== 1D FFTs in X =====
+       ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in, isign, plan(isign, 1))
+       call c2c_1m_x(in,isign,plan(isign,1))
 #else
-         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
-         wk1 = in
-         call c2c_1m_x(wk1, isign, plan(isign, 1))
+       allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
+       wk1 = in
+       call c2c_1m_x(wk1,isign,plan(isign,1))
 #endif
 
-         ! ===== Swap X --> Y; 1D FFTs in Y =====
+       ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-         if (dims(1) > 1) then
+       if (dims(1)>1) then
 #ifdef OVERWRITE
-            call transpose_x_to_y(in, wk2_c2c, ph)
+          call transpose_x_to_y(in,wk2_c2c,ph)
 #else
-            call transpose_x_to_y(wk1, wk2_c2c, ph)
+          call transpose_x_to_y(wk1,wk2_c2c,ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
-         else
+          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
+       else
 #ifdef OVERWRITE
-            call c2c_1m_y(in, isign, plan(isign, 2))
+          call c2c_1m_y(in,isign,plan(isign,2))
 #else
-            call c2c_1m_y(wk1, isign, plan(isign, 2))
+          call c2c_1m_y(wk1,isign,plan(isign,2))
 #endif
-         end if
+       end if
 
-         ! ===== Swap Y --> Z; 1D FFTs in Z =====
-         if (dims(1) > 1) then
-            call transpose_y_to_z(wk2_c2c, out, ph)
-         else
+       ! ===== Swap Y --> Z; 1D FFTs in Z =====
+       if (dims(1)>1) then
+          call transpose_y_to_z(wk2_c2c,out,ph)
+       else
 #ifdef OVERWRITE
-            call transpose_y_to_z(in, out, ph)
+          call transpose_y_to_z(in,out,ph)
 #else
-            call transpose_y_to_z(wk1, out, ph)
+          call transpose_y_to_z(wk1,out,ph)
 #endif
-         end if
-         call c2c_1m_z(out, isign, plan(isign, 3))
+       end if
+       call c2c_1m_z(out,isign,plan(isign,3))
 
-      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
-               .OR. &
-               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
+    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
+         .OR. & 
+         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
 
-         ! ===== 1D FFTs in Z =====
+       ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in, isign, plan(isign, 3))
+       call c2c_1m_z(in,isign,plan(isign,3))
 #else
-         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
-         wk1 = in
-         call c2c_1m_z(wk1, isign, plan(isign, 3))
+       allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
+       wk1 = in
+       call c2c_1m_z(wk1,isign,plan(isign,3))
 #endif
 
-         ! ===== Swap Z --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
+       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
 #ifdef OVERWRITE
-            call transpose_z_to_y(in, wk2_c2c, ph)
+          call transpose_z_to_y(in,wk2_c2c,ph)
 #else
-            call transpose_z_to_y(wk1, wk2_c2c, ph)
+          call transpose_z_to_y(wk1,wk2_c2c,ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
-         else  ! out==wk2_c2c if 1D decomposition
+          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
+       else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-            call transpose_z_to_y(in, out, ph)
+          call transpose_z_to_y(in,out,ph)
 #else
-            call transpose_z_to_y(wk1, out, ph)
+          call transpose_z_to_y(wk1,out,ph)
 #endif
-            call c2c_1m_y(out, isign, plan(isign, 2))
-         end if
+          call c2c_1m_y(out,isign,plan(isign,2))
+       end if
 
-         ! ===== Swap Y --> X; 1D FFTs in X =====
-         if (dims(1) > 1) then
-            call transpose_y_to_x(wk2_c2c, out, ph)
-         end if
-         call c2c_1m_x(out, isign, plan(isign, 1))
+       ! ===== Swap Y --> X; 1D FFTs in X =====
+       if (dims(1)>1) then
+          call transpose_y_to_x(wk2_c2c,out,ph)
+       end if
+       call c2c_1m_x(out,isign,plan(isign,1))
 
-      end if
+    end if
 
 #ifndef OVERWRITE
-      deallocate (wk1)
+    deallocate (wk1)
 #endif
 
-      return
-   end subroutine fft_3d_c2c
+    return
+  end subroutine fft_3d_c2c
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D forward FFT - real to complex
+  ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_r2c(in_r, out_c)
+  subroutine fft_3d_r2c(in_r, out_c)
+
+    implicit none
 
-      implicit none
+    real(mytype), dimension(:,:,:), intent(IN) :: in_r
+    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
 
-      real(mytype), dimension(:, :, :), intent(IN) :: in_r
-      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
+    if (format==PHYSICAL_IN_X) then
 
-      if (format == PHYSICAL_IN_X) then
+       ! ===== 1D FFTs in X =====
+       call r2c_1m_x(in_r,wk13)
 
-         ! ===== 1D FFTs in X =====
-         call r2c_1m_x(in_r, wk13)
+       ! ===== Swap X --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
+          call transpose_x_to_y(wk13,wk2_r2c,sp)
+          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
+       else
+          call c2c_1m_y(wk13,-1,plan(0,2))
+       end if
 
-         ! ===== Swap X --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
-            call transpose_x_to_y(wk13, wk2_r2c, sp)
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
-         else
-            call c2c_1m_y(wk13, -1, plan(0, 2))
-         end if
+       ! ===== Swap Y --> Z; 1D FFTs in Z =====
+       if (dims(1)>1) then
+          call transpose_y_to_z(wk2_r2c,out_c,sp)
+       else
+          call transpose_y_to_z(wk13,out_c,sp)
+       end if
+       call c2c_1m_z(out_c,-1,plan(0,3))
 
-         ! ===== Swap Y --> Z; 1D FFTs in Z =====
-         if (dims(1) > 1) then
-            call transpose_y_to_z(wk2_r2c, out_c, sp)
-         else
-            call transpose_y_to_z(wk13, out_c, sp)
-         end if
-         call c2c_1m_z(out_c, -1, plan(0, 3))
+    else if (format==PHYSICAL_IN_Z) then
 
-      else if (format == PHYSICAL_IN_Z) then
+       ! ===== 1D FFTs in Z =====
+       call r2c_1m_z(in_r,wk13)
 
-         ! ===== 1D FFTs in Z =====
-         call r2c_1m_z(in_r, wk13)
+       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
+          call transpose_z_to_y(wk13,wk2_r2c,sp)
+          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
+       else  ! out_c==wk2_r2c if 1D decomposition
+          call transpose_z_to_y(wk13,out_c,sp)
+          call c2c_1m_y(out_c,-1,plan(0,2))
+       end if
 
-         ! ===== Swap Z --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
-            call transpose_z_to_y(wk13, wk2_r2c, sp)
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
-         else  ! out_c==wk2_r2c if 1D decomposition
-            call transpose_z_to_y(wk13, out_c, sp)
-            call c2c_1m_y(out_c, -1, plan(0, 2))
-         end if
+       ! ===== Swap Y --> X; 1D FFTs in X =====
+       if (dims(1)>1) then
+          call transpose_y_to_x(wk2_r2c,out_c,sp)
+       end if
+       call c2c_1m_x(out_c,-1,plan(0,1))
 
-         ! ===== Swap Y --> X; 1D FFTs in X =====
-         if (dims(1) > 1) then
-            call transpose_y_to_x(wk2_r2c, out_c, sp)
-         end if
-         call c2c_1m_x(out_c, -1, plan(0, 1))
+    end if
 
-      end if
+    return
+  end subroutine fft_3d_r2c
 
-      return
-   end subroutine fft_3d_r2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D inverse FFT - complex to real
+  ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_c2r(in_c, out_r)
+  subroutine fft_3d_c2r(in_c, out_r)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
-      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
+    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
 
 #ifndef OVERWRITE
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1
+    complex(mytype), allocatable, dimension(:,:,:) :: wk1
 #endif
 
-      if (format == PHYSICAL_IN_X) then
+    if (format==PHYSICAL_IN_X) then
 
-         ! ===== 1D FFTs in Z =====
+       ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in_c, 1, plan(2, 3))
+       call c2c_1m_z(in_c,1,plan(2,3))       
 #else
-         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
-         wk1 = in_c
-         call c2c_1m_z(wk1, 1, plan(2, 3))
+       allocate (wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
+       wk1 = in_c
+       call c2c_1m_z(wk1,1,plan(2,3))
 #endif
 
-         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+       ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-         call transpose_z_to_y(in_c, wk2_r2c, sp)
+       call transpose_z_to_y(in_c,wk2_r2c,sp)
 #else
-         call transpose_z_to_y(wk1, wk2_r2c, sp)
+       call transpose_z_to_y(wk1,wk2_r2c,sp)
 #endif
-         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+       call c2c_1m_y(wk2_r2c,1,plan(2,2))
 
-         ! ===== Swap Y --> X; 1D FFTs in X =====
-         if (dims(1) > 1) then
-            call transpose_y_to_x(wk2_r2c, wk13, sp)
-            call c2r_1m_x(wk13, out_r)
-         else
-            call c2r_1m_x(wk2_r2c, out_r)
-         end if
+       ! ===== Swap Y --> X; 1D FFTs in X =====
+       if (dims(1)>1) then
+          call transpose_y_to_x(wk2_r2c,wk13,sp)
+          call c2r_1m_x(wk13,out_r)
+       else
+          call c2r_1m_x(wk2_r2c,out_r)
+       end if
 
-      else if (format == PHYSICAL_IN_Z) then
+    else if (format==PHYSICAL_IN_Z) then
 
-         ! ===== 1D FFTs in X =====
+       ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in_c, 1, plan(2, 1))
+       call c2c_1m_x(in_c,1,plan(2,1))
 #else
-         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
-         wk1 = in_c
-         call c2c_1m_x(wk1, 1, plan(2, 1))
+       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
+       wk1 = in_c
+       call c2c_1m_x(wk1,1,plan(2,1))
 #endif
 
-         ! ===== Swap X --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
+       ! ===== Swap X --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
 #ifdef OVERWRITE
-            call transpose_x_to_y(in_c, wk2_r2c, sp)
+          call transpose_x_to_y(in_c,wk2_r2c,sp)
 #else
-            call transpose_x_to_y(wk1, wk2_r2c, sp)
+          call transpose_x_to_y(wk1,wk2_r2c,sp)
 #endif
-            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
-         else  ! in_c==wk2_r2c if 1D decomposition
+          call c2c_1m_y(wk2_r2c,1,plan(2,2))
+       else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-            call c2c_1m_y(in_c, 1, plan(2, 2))
+          call c2c_1m_y(in_c,1,plan(2,2))
 #else
-            call c2c_1m_y(wk1, 1, plan(2, 2))
+          call c2c_1m_y(wk1,1,plan(2,2))
 #endif
-         end if
+       end if
 
-         ! ===== Swap Y --> Z; 1D FFTs in Z =====
-         if (dims(1) > 1) then
-            call transpose_y_to_z(wk2_r2c, wk13, sp)
-         else
+       ! ===== Swap Y --> Z; 1D FFTs in Z =====
+       if (dims(1)>1) then
+          call transpose_y_to_z(wk2_r2c,wk13,sp)
+       else
 #ifdef OVERWRITE
-            call transpose_y_to_z(in_c, wk13, sp)
+          call transpose_y_to_z(in_c,wk13,sp)
 #else
-            call transpose_y_to_z(wk1, wk13, sp)
+          call transpose_y_to_z(wk1,wk13,sp)
 #endif
-         end if
-         call c2r_1m_z(wk13, out_r)
+       end if
+       call c2r_1m_z(wk13,out_r)
 
-      end if
+    end if
 
 #ifndef OVERWRITE
-      deallocate (wk1)
+    deallocate (wk1)
 #endif
 
-      return
-   end subroutine fft_3d_c2r
+    return
+  end subroutine fft_3d_c2r
+
 
 end module decomp_2d_fft
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index e8c97081..5765ab66 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,1070 +9,1083 @@
 !
 !=======================================================================
 
-! This is the FFTW implementation of the FFT library using
+! This is the FFTW implementation of the FFT library using 
 ! the Fortran 2003 interface introduced in FFTW 3.3-beta1
 
 module decomp_2d_fft
 
-   use decomp_2d  ! 2D decomposition module
-   use, intrinsic :: iso_c_binding
-
-   implicit none
-
-   include "fftw3.f03"
-
-   private        ! Make everything private unless declared public
-
-   ! engine-specific global variables
-   integer, save :: plan_type = FFTW_MEASURE
-
-   ! FFTW plans
-   ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
-   ! For c2c transforms:
-   !     use plan(-1,j) for  forward transform;
-   !     use plan( 1,j) for backward transform;
-   ! For r2c/c2r transforms:
-   !     use plan(0,j) for r2c transforms;
-   !     use plan(2,j) for c2r transforms;
-   type(C_PTR), save :: plan(-1:2, 3)
-
-   integer, parameter, public :: DECOMP_2D_FFT_FORWARD = -1
-   integer, parameter, public :: DECOMP_2D_FFT_BACKWARD = 1
-
-   ! Physical space data can be stored in either X-pencil or Z-pencil
-   integer, parameter, public :: PHYSICAL_IN_X = 1
-   integer, parameter, public :: PHYSICAL_IN_Z = 3
-
-   integer, save :: format                 ! input X-pencil or Z-pencil
-
-   ! The libary can only be initialised once
-   logical, save :: initialised = .false.
-
-   ! Global size of the FFT
-   integer, save :: nx_fft, ny_fft, nz_fft
-
-   ! 2D processor grid
-   ! FIXME this is already available in the module decomp_2d
-   integer, save, dimension(2) :: dims
-
-   ! Decomposition objects
-   TYPE(DECOMP_INFO), pointer, save :: ph => null()  ! physical space
-   TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
-
-   ! Workspace to store the intermediate Y-pencil data
-   ! *** TODO: investigate how to use only one workspace array
-   complex(mytype), pointer :: wk2_c2c(:, :, :), wk2_r2c(:, :, :), wk13(:, :, :)
-   type(C_PTR) :: wk2_c2c_p, wk2_r2c_p, wk13_p
-
-   public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-             decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
-             decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
-
-   ! Declare generic interfaces to handle different inputs
-
-   interface decomp_2d_fft_init
-      module procedure fft_init_noarg
-      module procedure fft_init_arg
-      module procedure fft_init_general
-   end interface
-
-   interface decomp_2d_fft_3d
-      module procedure fft_3d_c2c
-      module procedure fft_3d_r2c
-      module procedure fft_3d_c2r
-   end interface
-
+  use decomp_2d  ! 2D decomposition module
+  use, intrinsic :: iso_c_binding
+  
+  implicit none
+
+  include "fftw3.f03"
+  
+  private        ! Make everything private unless declared public
+
+  ! engine-specific global variables
+  integer, save :: plan_type = FFTW_MEASURE
+
+  ! FFTW plans
+  ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
+  ! For c2c transforms: 
+  !     use plan(-1,j) for  forward transform; 
+  !     use plan( 1,j) for backward transform;
+  ! For r2c/c2r transforms:
+  !     use plan(0,j) for r2c transforms;
+  !     use plan(2,j) for c2r transforms;
+  type(C_PTR), save :: plan(-1:2,3)
+
+  integer, parameter, public :: DECOMP_2D_FFT_FORWARD = -1
+  integer, parameter, public :: DECOMP_2D_FFT_BACKWARD = 1
+  
+  ! Physical space data can be stored in either X-pencil or Z-pencil
+  integer, parameter, public :: PHYSICAL_IN_X = 1
+  integer, parameter, public :: PHYSICAL_IN_Z = 3 
+
+  integer, save :: format                 ! input X-pencil or Z-pencil
+  
+  ! The libary can only be initialised once
+  logical, save :: initialised = .false. 
+
+  ! Global size of the FFT
+  integer, save :: nx_fft, ny_fft, nz_fft
+
+  ! 2D processor grid
+  ! FIXME this is already available in the module decomp_2d
+  integer, save, dimension(2) :: dims
+
+  ! Decomposition objects
+  TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
+  TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
+
+  ! Workspace to store the intermediate Y-pencil data
+  ! *** TODO: investigate how to use only one workspace array
+  complex(mytype), pointer :: wk2_c2c(:,:,:), wk2_r2c(:,:,:), wk13(:,:,:)
+  type(C_PTR) :: wk2_c2c_p, wk2_r2c_p, wk13_p
+
+  public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
+       decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+       decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
+  
+  ! Declare generic interfaces to handle different inputs
+  
+  interface decomp_2d_fft_init
+     module procedure fft_init_noarg
+     module procedure fft_init_arg
+     module procedure fft_init_general
+  end interface
+  
+  interface decomp_2d_fft_3d
+     module procedure fft_3d_c2c
+     module procedure fft_3d_r2c
+     module procedure fft_3d_c2r
+  end interface
+
+  
 contains
-
+  
+  
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Initialise the FFT module
+  ! Initialise the FFT module
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_init_noarg
-
-      implicit none
-
-      call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
+  subroutine fft_init_noarg
+    
+    implicit none
+    
+    call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
+    
+    return
+  end subroutine fft_init_noarg
 
-      return
-   end subroutine fft_init_noarg
+  subroutine fft_init_arg(pencil)     ! allow to handle Z-pencil input
 
-   subroutine fft_init_arg(pencil)     ! allow to handle Z-pencil input
+    implicit none
 
-      implicit none
+    integer, intent(IN) :: pencil
 
-      integer, intent(IN) :: pencil
+    call fft_init_general(pencil, nx_global, ny_global, nz_global)
 
-      call fft_init_general(pencil, nx_global, ny_global, nz_global)
+    return
+  end subroutine fft_init_arg
 
-      return
-   end subroutine fft_init_arg
+  ! Initialise the FFT library to perform arbitrary size transforms
+  subroutine fft_init_general(pencil, nx, ny, nz)
 
-   ! Initialise the FFT library to perform arbitrary size transforms
-   subroutine fft_init_general(pencil, nx, ny, nz)
+    implicit none
 
-      implicit none
+    integer, intent(IN) :: pencil
+    integer, intent(IN) :: nx, ny, nz
 
-      integer, intent(IN) :: pencil
-      integer, intent(IN) :: nx, ny, nz
-
-      integer :: errorcode
-      integer(C_SIZE_T) :: sz
+    integer :: errorcode
+    integer(C_SIZE_T) :: sz
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
+    if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
 #endif
 
-      if (initialised) then
-         errorcode = 4
-         call decomp_2d_abort(errorcode, &
-                              'FFT library should only be initialised once')
-      end if
-
-      format = pencil
-      nx_fft = nx
-      ny_fft = ny
-      nz_fft = nz
-
-      ! determine the processor grid in use
-      dims = get_decomp_dims()
-
-      ! for c2r/r2c interface:
-      ! if in physical space, a real array is of size: nx*ny*nz
-      ! in spectral space, the complex array is of size:
-      !         (nx/2+1)*ny*nz, if PHYSICAL_IN_X
-      !      or nx*ny*(nz/2+1), if PHYSICAL_IN_Z
-
-      if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
-         ph => decomp_main
-      else
-         if (.not. associated(ph)) allocate (ph)
-         call decomp_info_init(nx, ny, nz, ph)
-      end if
-      if (format == PHYSICAL_IN_X) then
-         call decomp_info_init(nx/2 + 1, ny, nz, sp)
-      else if (format == PHYSICAL_IN_Z) then
-         call decomp_info_init(nx, ny, nz/2 + 1, sp)
-      end if
-
-      sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)
-      wk2_c2c_p = fftw_alloc_complex(sz)
-      call c_f_pointer(wk2_c2c_p, wk2_c2c, [ph%ysz(1), ph%ysz(2), ph%ysz(3)])
-
-      sz = sp%ysz(1)*sp%ysz(2)*sp%ysz(3)
-      wk2_r2c_p = fftw_alloc_complex(sz)
-      call c_f_pointer(wk2_r2c_p, wk2_r2c, [sp%ysz(1), sp%ysz(2), sp%ysz(3)])
-
-      if (format == PHYSICAL_IN_X) then
-         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
-         wk13_p = fftw_alloc_complex(sz)
-         call c_f_pointer(wk13_p, wk13, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
-      else if (format == PHYSICAL_IN_Z) then
-         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
-         wk13_p = fftw_alloc_complex(sz)
-         call c_f_pointer(wk13_p, wk13, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
-      end if
-
-      call init_fft_engine
-
-      initialised = .true.
+    if (initialised) then
+       errorcode = 4
+       call decomp_2d_abort(errorcode, &
+            'FFT library should only be initialised once')
+    end if
+    
+    format = pencil
+    nx_fft = nx
+    ny_fft = ny
+    nz_fft = nz
+
+    ! determine the processor grid in use
+    dims = get_decomp_dims()
+
+    ! for c2r/r2c interface:
+    ! if in physical space, a real array is of size: nx*ny*nz
+    ! in spectral space, the complex array is of size:
+    !         (nx/2+1)*ny*nz, if PHYSICAL_IN_X
+    !      or nx*ny*(nz/2+1), if PHYSICAL_IN_Z
+
+    if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
+       ph => decomp_main
+    else
+       if (.not.associated(ph)) allocate(ph)
+       call decomp_info_init(nx, ny, nz, ph)
+    endif
+    if (format==PHYSICAL_IN_X) then
+       call decomp_info_init(nx/2+1, ny, nz, sp)
+    else if (format==PHYSICAL_IN_Z) then
+       call decomp_info_init(nx, ny, nz/2+1, sp)
+    end if
+
+    sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)
+    wk2_c2c_p = fftw_alloc_complex(sz)
+    call c_f_pointer(wk2_c2c_p,wk2_c2c,[ph%ysz(1),ph%ysz(2),ph%ysz(3)])
+
+    sz = sp%ysz(1)*sp%ysz(2)*sp%ysz(3)
+    wk2_r2c_p = fftw_alloc_complex(sz)
+    call c_f_pointer(wk2_r2c_p,wk2_r2c,[sp%ysz(1),sp%ysz(2),sp%ysz(3)])
+
+
+    if (format==PHYSICAL_IN_X) then
+       sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+       wk13_p = fftw_alloc_complex(sz)
+       call c_f_pointer(wk13_p,wk13,[sp%xsz(1),sp%xsz(2),sp%xsz(3)])
+    else if (format==PHYSICAL_IN_Z) then
+       sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+       wk13_p = fftw_alloc_complex(sz)
+       call c_f_pointer(wk13_p,wk13,[sp%zsz(1),sp%zsz(2),sp%zsz(3)])
+    end if
+
+    call init_fft_engine
+    
+    initialised = .true.
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
+    if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
 #endif
 
-      return
-   end subroutine fft_init_general
+    return
+  end subroutine fft_init_general
 
+  
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Final clean up
+  ! Final clean up
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine decomp_2d_fft_finalize
-
-      implicit none
+  subroutine decomp_2d_fft_finalize
+    
+    implicit none
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
+    if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-      if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
-         call decomp_info_finalize(ph)
-         deallocate (ph)
-      end if
-      nullify (ph)
-      call decomp_info_finalize(sp)
+    if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
+      call decomp_info_finalize(ph)
+      deallocate(ph)
+    endif
+    nullify(ph)
+    call decomp_info_finalize(sp)
 
-      call fftw_free(wk2_c2c_p)
-      call fftw_free(wk2_r2c_p)
-      call fftw_free(wk13_p)
+    call fftw_free(wk2_c2c_p)
+    call fftw_free(wk2_r2c_p)
+    call fftw_free(wk13_p)
 
-      call finalize_fft_engine
+    call finalize_fft_engine
 
-      initialised = .false.
+    initialised = .false.
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
+    if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
 #endif
 
-      return
-   end subroutine decomp_2d_fft_finalize
+    return
+  end subroutine decomp_2d_fft_finalize
+
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Return the size, starting/ending index of the distributed array
-   !  whose global size is (nx/2+1)*ny*nz, for defining data structures
-   !  in r2c and c2r interfaces
+  ! Return the size, starting/ending index of the distributed array 
+  !  whose global size is (nx/2+1)*ny*nz, for defining data structures
+  !  in r2c and c2r interfaces
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine decomp_2d_fft_get_size(istart, iend, isize)
-
-      implicit none
-      integer, dimension(3), intent(OUT) :: istart, iend, isize
-
-      if (format == PHYSICAL_IN_X) then
-         istart = sp%zst
-         iend = sp%zen
-         isize = sp%zsz
-      else if (format == PHYSICAL_IN_Z) then
-         istart = sp%xst
-         iend = sp%xen
-         isize = sp%xsz
-      end if
-
-      return
-   end subroutine decomp_2d_fft_get_size
+  subroutine decomp_2d_fft_get_size(istart, iend, isize)
+    
+    implicit none
+    integer, dimension(3), intent(OUT) :: istart, iend, isize
+    
+    if (format==PHYSICAL_IN_X) then
+       istart = sp%zst
+       iend   = sp%zen
+       isize  = sp%zsz
+    else if (format==PHYSICAL_IN_Z) then
+       istart = sp%xst
+       iend   = sp%xen
+       isize  = sp%xsz
+    end if
+    
+    return
+  end subroutine decomp_2d_fft_get_size
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Return a pointer to the decomp_info object ph
-   !
-   ! The caller should not apply decomp_info_finalize on the pointer
+  ! Return a pointer to the decomp_info object ph
+  !
+  ! The caller should not apply decomp_info_finalize on the pointer
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   function decomp_2d_fft_get_ph()
+  function decomp_2d_fft_get_ph()
 
-      implicit none
+    implicit none
 
-      type(decomp_info), pointer :: decomp_2d_fft_get_ph
+    type(decomp_info), pointer :: decomp_2d_fft_get_ph
 
-      if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                     __LINE__, &
-                                                     -1, &
-                                                     'FFT library must be initialised first')
-      decomp_2d_fft_get_ph => ph
+    if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  -1, &
+                                                  'FFT library must be initialised first')
+    decomp_2d_fft_get_ph => ph
 
-   end function decomp_2d_fft_get_ph
+  end function decomp_2d_fft_get_ph
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Return a pointer to the decomp_info object sp
-   !
-   ! The caller should not apply decomp_info_finalize on the pointer
+  ! Return a pointer to the decomp_info object sp
+  !
+  ! The caller should not apply decomp_info_finalize on the pointer
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   function decomp_2d_fft_get_sp()
+  function decomp_2d_fft_get_sp()
 
-      implicit none
+    implicit none
 
-      type(decomp_info), pointer :: decomp_2d_fft_get_sp
+    type(decomp_info), pointer :: decomp_2d_fft_get_sp
 
-      if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                     __LINE__, &
-                                                     -1, &
-                                                     'FFT library must be initialised first')
-      decomp_2d_fft_get_sp => sp
+    if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  -1, &
+                                                  'FFT library must be initialised first')
+    decomp_2d_fft_get_sp => sp
 
-   end function decomp_2d_fft_get_sp
+  end function decomp_2d_fft_get_sp
 
-   ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
-   subroutine c2c_1m_x_plan(plan1, decomp, isign)
+  ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
+  subroutine c2c_1m_x_plan(plan1, decomp, isign)
 
-      implicit none
+    implicit none
 
-      type(C_PTR) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-      integer, intent(IN) :: isign
+    type(C_PTR) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: isign
 
 #ifdef DOUBLE_PREC
-      complex(C_DOUBLE_COMPLEX), pointer :: a1(:, :, :)
-      complex(C_DOUBLE_COMPLEX), pointer :: a1o(:, :, :)
+    complex(C_DOUBLE_COMPLEX), pointer :: a1(:,:,:)
+    complex(C_DOUBLE_COMPLEX), pointer :: a1o(:,:,:)
 #else
-      complex(C_FLOAT_COMPLEX), pointer :: a1(:, :, :)
-      complex(C_FLOAT_COMPLEX), pointer :: a1o(:, :, :)
+    complex(C_FLOAT_COMPLEX), pointer :: a1(:,:,:)
+    complex(C_FLOAT_COMPLEX), pointer :: a1o(:,:,:)
 #endif
-      type(C_PTR) :: a1_p
-      integer(C_SIZE_T) :: sz
+    type(C_PTR) :: a1_p
+    integer(C_SIZE_T) :: sz
 
-      sz = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)
-      a1_p = fftw_alloc_complex(sz)
-      call c_f_pointer(a1_p, a1, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
-      call c_f_pointer(a1_p, a1o, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
+    sz = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)
+    a1_p = fftw_alloc_complex(sz)
+    call c_f_pointer(a1_p,a1,[decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)])
+    call c_f_pointer(a1_p,a1o,[decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)])
 
 #ifdef DOUBLE_PREC
-      plan1 = fftw_plan_many_dft(1, decomp%xsz(1), &
-                                 decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-                                 decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
-                                 isign, plan_type)
+    plan1 = fftw_plan_many_dft(1, decomp%xsz(1), &
+         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+         decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
+         isign, plan_type)
 #else
-      plan1 = fftwf_plan_many_dft(1, decomp%xsz(1), &
-                                  decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-                                  decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
-                                  isign, plan_type)
+    plan1 = fftwf_plan_many_dft(1, decomp%xsz(1), &
+         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+         decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
+         isign, plan_type)
 #endif
 
-      call fftw_free(a1_p)
+    call fftw_free(a1_p)
 
-      return
-   end subroutine c2c_1m_x_plan
+    return
+  end subroutine c2c_1m_x_plan
 
-   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
-   subroutine c2c_1m_y_plan(plan1, decomp, isign)
+  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
+  subroutine c2c_1m_y_plan(plan1, decomp, isign)
 
-      implicit none
+    implicit none
 
-      type(C_PTR) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-      integer, intent(IN) :: isign
+    type(C_PTR) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: isign
 
 #ifdef DOUBLE_PREC
-      complex(C_DOUBLE_COMPLEX), pointer :: a1(:, :)
-      complex(C_DOUBLE_COMPLEX), pointer :: a1o(:, :)
+    complex(C_DOUBLE_COMPLEX), pointer :: a1(:,:)
+    complex(C_DOUBLE_COMPLEX), pointer :: a1o(:,:)
 #else
-      complex(C_FLOAT_COMPLEX), pointer :: a1(:, :)
-      complex(C_FLOAT_COMPLEX), pointer :: a1o(:, :)
+    complex(C_FLOAT_COMPLEX), pointer :: a1(:,:)
+    complex(C_FLOAT_COMPLEX), pointer :: a1o(:,:)
 #endif
-      type(C_PTR) :: a1_p
-      integer(C_SIZE_T) :: sz
+    type(C_PTR) :: a1_p
+    integer(C_SIZE_T) :: sz
 
-      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-      ! done one Z-plane at a time. So plan for 2D data sets here.
-      sz = decomp%ysz(1)*decomp%ysz(2)
-      a1_p = fftw_alloc_complex(sz)
-      call c_f_pointer(a1_p, a1, [decomp%ysz(1), decomp%ysz(2)])
-      call c_f_pointer(a1_p, a1o, [decomp%ysz(1), decomp%ysz(2)])
+    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+    ! done one Z-plane at a time. So plan for 2D data sets here.
+    sz = decomp%ysz(1)*decomp%ysz(2)
+    a1_p = fftw_alloc_complex(sz)
+    call c_f_pointer(a1_p,a1,[decomp%ysz(1),decomp%ysz(2)])
+    call c_f_pointer(a1_p,a1o,[decomp%ysz(1),decomp%ysz(2)])
 
 #ifdef DOUBLE_PREC
-      plan1 = fftw_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
-                                 a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
-                                 decomp%ysz(1), 1, isign, plan_type)
+    plan1 =  fftw_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
+         a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
+         decomp%ysz(1), 1, isign, plan_type)
 #else
-      plan1 = fftwf_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
-                                  a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
-                                  decomp%ysz(1), 1, isign, plan_type)
+    plan1 = fftwf_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
+         a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
+         decomp%ysz(1), 1, isign, plan_type)
 #endif
 
-      call fftw_free(a1_p)
+    call fftw_free(a1_p)
 
-      return
-   end subroutine c2c_1m_y_plan
+    return
+  end subroutine c2c_1m_y_plan
 
-   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
-   subroutine c2c_1m_z_plan(plan1, decomp, isign)
+  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
+  subroutine c2c_1m_z_plan(plan1, decomp, isign)
 
-      implicit none
+    implicit none
 
-      type(C_PTR) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-      integer, intent(IN) :: isign
+    type(C_PTR) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: isign
 
 #ifdef DOUBLE_PREC
-      complex(C_DOUBLE_COMPLEX), pointer :: a1(:, :, :)
-      complex(C_DOUBLE_COMPLEX), pointer :: a1o(:, :, :)
+    complex(C_DOUBLE_COMPLEX), pointer :: a1(:,:,:)
+    complex(C_DOUBLE_COMPLEX), pointer :: a1o(:,:,:)
 #else
-      complex(C_FLOAT_COMPLEX), pointer :: a1(:, :, :)
-      complex(C_FLOAT_COMPLEX), pointer :: a1o(:, :, :)
+    complex(C_FLOAT_COMPLEX), pointer :: a1(:,:,:)
+    complex(C_FLOAT_COMPLEX), pointer :: a1o(:,:,:)
 #endif
-      type(C_PTR) :: a1_p
-      integer(C_SIZE_T) :: sz
+    type(C_PTR) :: a1_p
+    integer(C_SIZE_T) :: sz
 
-      sz = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)
-      a1_p = fftw_alloc_complex(sz)
-      call c_f_pointer(a1_p, a1, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
-      call c_f_pointer(a1_p, a1o, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
+    sz = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)
+    a1_p = fftw_alloc_complex(sz)
+    call c_f_pointer(a1_p,a1,[decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)])
+    call c_f_pointer(a1_p,a1o,[decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)])
 
 #ifdef DOUBLE_PREC
-      plan1 = fftw_plan_many_dft(1, decomp%zsz(3), &
-                                 decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                                 decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-                                 decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+    plan1 =  fftw_plan_many_dft(1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #else
-      plan1 = fftwf_plan_many_dft(1, decomp%zsz(3), &
-                                  decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                                  decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-                                  decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+    plan1 = fftwf_plan_many_dft(1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #endif
 
-      call fftw_free(a1_p)
+    call fftw_free(a1_p)
 
-      return
-   end subroutine c2c_1m_z_plan
+    return
+  end subroutine c2c_1m_z_plan
 
-   ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
-   subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
+  ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
+  subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
 
-      implicit none
+    implicit none
 
-      type(C_PTR) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    type(C_PTR) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-      real(mytype), pointer :: a1(:, :, :)
-      complex(mytype), pointer :: a2(:, :, :)
-      type(C_PTR) :: a1_p, a2_p
-      integer(C_SIZE_T) :: sz
+    real(mytype), pointer :: a1(:,:,:)
+    complex(mytype), pointer :: a2(:,:,:)
+    type(C_PTR) :: a1_p, a2_p
+    integer(C_SIZE_T) :: sz
 
-      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
-      a1_p = fftw_alloc_real(sz)
-      call c_f_pointer(a1_p, a1, &
-                       [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
-      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
-      a2_p = fftw_alloc_complex(sz)
-      call c_f_pointer(a2_p, a2, &
-                       [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
+    sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+    a1_p = fftw_alloc_real(sz)
+    call c_f_pointer(a1_p,a1, &
+         [decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)])
+    sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+    a2_p = fftw_alloc_complex(sz)
+    call c_f_pointer(a2_p,a2, &
+         [decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)])
 
 #ifdef DOUBLE_PREC
-      plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-                                     decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-                                     plan_type)
+    plan1 =  fftw_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
+         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+         plan_type)
 #else
-      plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-                                      decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-                                      plan_type)
+    plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
+         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+         plan_type)
 #endif
 
-      call fftw_free(a1_p)
-      call fftw_free(a2_p)
+    call fftw_free(a1_p)
+    call fftw_free(a2_p)
 
-      return
-   end subroutine r2c_1m_x_plan
+    return
+  end subroutine r2c_1m_x_plan
 
-   ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
-   subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
+  ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
+  subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
 
-      implicit none
+    implicit none
 
-      type(C_PTR) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    type(C_PTR) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-      complex(mytype), pointer :: a1(:, :, :)
-      real(mytype), pointer :: a2(:, :, :)
-      type(C_PTR) :: a1_p, a2_p
-      integer(C_SIZE_T) :: sz
+    complex(mytype), pointer :: a1(:,:,:)
+    real(mytype), pointer :: a2(:,:,:)
+    type(C_PTR) :: a1_p, a2_p
+    integer(C_SIZE_T) :: sz
 
-      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
-      a1_p = fftw_alloc_complex(sz)
-      call c_f_pointer(a1_p, a1, &
-                       [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
-      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
-      a2_p = fftw_alloc_real(sz)
-      call c_f_pointer(a2_p, a2, &
-                       [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
+    sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+    a1_p = fftw_alloc_complex(sz)
+    call c_f_pointer(a1_p,a1, &
+         [decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)])
+    sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+    a2_p = fftw_alloc_real(sz)
+    call c_f_pointer(a2_p,a2, &
+         [decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)])
 
 #ifdef DOUBLE_PREC
-      plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-                                     decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-                                     plan_type)
+    plan1 =  fftw_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
+         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+         plan_type)
 #else
-      plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-                                      decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-                                      plan_type)
+    plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
+         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+         plan_type)
 #endif
 
-      call fftw_free(a1_p)
-      call fftw_free(a2_p)
+    call fftw_free(a1_p)
+    call fftw_free(a2_p)
 
-      return
-   end subroutine c2r_1m_x_plan
+    return
+  end subroutine c2r_1m_x_plan
 
-   ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
-   subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
+  ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
+  subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
 
-      implicit none
+    implicit none
 
-      type(C_PTR) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    type(C_PTR) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-      real(mytype), pointer :: a1(:, :, :)
-      complex(mytype), pointer :: a2(:, :, :)
-      type(C_PTR) :: a1_p, a2_p
-      integer(C_SIZE_T) :: sz
+    real(mytype), pointer :: a1(:,:,:)
+    complex(mytype), pointer :: a2(:,:,:)
+    type(C_PTR) :: a1_p, a2_p
+    integer(C_SIZE_T) :: sz
 
-      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
-      a1_p = fftw_alloc_real(sz)
-      call c_f_pointer(a1_p, a1, &
-                       [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
-      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
-      a2_p = fftw_alloc_complex(sz)
-      call c_f_pointer(a2_p, a2, &
-                       [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
+    sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+    a1_p = fftw_alloc_real(sz)
+    call c_f_pointer(a1_p,a1, &
+         [decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)])
+    sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+    a2_p = fftw_alloc_complex(sz)
+    call c_f_pointer(a2_p,a2, &
+         [decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)])
 
 #ifdef DOUBLE_PREC
-      plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+    plan1 =  fftw_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #else
-      plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+    plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #endif
 
-      call fftw_free(a1_p)
-      call fftw_free(a2_p)
+    call fftw_free(a1_p)
+    call fftw_free(a2_p)
 
-      return
-   end subroutine r2c_1m_z_plan
+    return
+  end subroutine r2c_1m_z_plan
 
-   ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
-   subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
+  ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
+  subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
 
-      implicit none
+    implicit none
 
-      type(C_PTR) :: plan1
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+    type(C_PTR) :: plan1
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-      complex(mytype), pointer :: a1(:, :, :)
-      real(mytype), pointer :: a2(:, :, :)
-      type(C_PTR) :: a1_p, a2_p
-      integer(C_SIZE_T) :: sz
+    complex(mytype), pointer :: a1(:,:,:)
+    real(mytype), pointer :: a2(:,:,:)
+    type(C_PTR) :: a1_p, a2_p
+    integer(C_SIZE_T) :: sz
 
-      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
-      a1_p = fftw_alloc_complex(sz)
-      call c_f_pointer(a1_p, a1, &
-                       [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
-      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
-      a2_p = fftw_alloc_real(sz)
-      call c_f_pointer(a2_p, a2, &
-                       [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
+    sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+    a1_p = fftw_alloc_complex(sz)
+    call c_f_pointer(a1_p,a1, &
+         [decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)])
+    sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+    a2_p = fftw_alloc_real(sz)
+    call c_f_pointer(a2_p,a2, &
+         [decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)])
 
 #ifdef DOUBLE_PREC
-      plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+    plan1 =  fftw_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #else
-      plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+    plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #endif
 
-      call fftw_free(a1_p)
-      call fftw_free(a2_p)
+    call fftw_free(a1_p)
+    call fftw_free(a2_p)
+
+    return
+  end subroutine c2r_1m_z_plan
 
-      return
-   end subroutine c2r_1m_z_plan
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time initialisations for the FFT engine
+  !  This routine performs one-time initialisations for the FFT engine
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine init_fft_engine
-
-      implicit none
-
-      if (nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the FFTW (F2003 interface) engine *****'
-         write (*, *) ' '
-      end if
-
-      if (format == PHYSICAL_IN_X) then
-
-         ! For C2C transforms
-         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
-         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
-         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
-         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
-         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
-         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
-
-         ! For R2C/C2R tranforms
-         call r2c_1m_x_plan(plan(0, 1), ph, sp)
-         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
-         call c2c_1m_z_plan(plan(0, 3), sp, FFTW_FORWARD)
-         call c2c_1m_z_plan(plan(2, 3), sp, FFTW_BACKWARD)
-         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
-         call c2r_1m_x_plan(plan(2, 1), sp, ph)
-
-      else if (format == PHYSICAL_IN_Z) then
-
-         ! For C2C transforms
-         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
-         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
-         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
-         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
-         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
-         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
-
-         ! For R2C/C2R tranforms
-         call r2c_1m_z_plan(plan(0, 3), ph, sp)
-         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
-         call c2c_1m_x_plan(plan(0, 1), sp, FFTW_FORWARD)
-         call c2c_1m_x_plan(plan(2, 1), sp, FFTW_BACKWARD)
-         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
-         call c2r_1m_z_plan(plan(2, 3), sp, ph)
-
-      end if
-
-      return
-   end subroutine init_fft_engine
+  subroutine init_fft_engine
+
+    implicit none
+
+    if (nrank==0) then
+       write(*,*) ' '
+       write(*,*) '***** Using the FFTW (F2003 interface) engine *****'
+       write(*,*) ' '
+    end if
+
+    if (format == PHYSICAL_IN_X) then
+
+       ! For C2C transforms
+       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
+       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD )
+       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
+       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
+       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
+       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
+       
+       ! For R2C/C2R tranforms
+       call r2c_1m_x_plan(plan(0,1), ph, sp)
+       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
+       call c2c_1m_z_plan(plan(0,3), sp, FFTW_FORWARD )
+       call c2c_1m_z_plan(plan(2,3), sp, FFTW_BACKWARD)
+       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
+       call c2r_1m_x_plan(plan(2,1), sp, ph)
+
+    else if (format == PHYSICAL_IN_Z) then
+
+       ! For C2C transforms
+       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
+       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD ) 
+       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
+       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
+       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
+       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
+       
+       ! For R2C/C2R tranforms
+       call r2c_1m_z_plan(plan(0,3), ph, sp)
+       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
+       call c2c_1m_x_plan(plan(0,1), sp, FFTW_FORWARD )
+       call c2c_1m_x_plan(plan(2,1), sp, FFTW_BACKWARD)
+       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
+       call c2r_1m_z_plan(plan(2,3), sp, ph)
+       
+    end if
+
+    return
+  end subroutine init_fft_engine
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time finalisations for the FFT engine
+  !  This routine performs one-time finalisations for the FFT engine
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine finalize_fft_engine
-
-      implicit none
+  subroutine finalize_fft_engine
 
-      integer :: i, j
+    implicit none
 
-      do j = 1, 3
-         do i = -1, 2
+    integer :: i,j
+    
+    do j=1,3
+       do i=-1,2
 #ifdef DOUBLE_PREC
-            call fftw_destroy_plan(plan(i, j))
+          call fftw_destroy_plan(plan(i,j))
 #else
-            call fftwf_destroy_plan(plan(i, j))
+          call fftwf_destroy_plan(plan(i,j))
 #endif
-         end do
-      end do
+       end do
+    end do
 
-      call fftw_cleanup()
+    call fftw_cleanup()
 
-      return
-   end subroutine finalize_fft_engine
+    return
+  end subroutine finalize_fft_engine
 
-   ! Following routines calculate multiple one-dimensional FFTs to form
-   ! the basis of three-dimensional FFTs.
 
-   ! c2c transform, multiple 1D FFTs in x direction
-   subroutine c2c_1m_x(inout, isign, plan1)
+  ! Following routines calculate multiple one-dimensional FFTs to form 
+  ! the basis of three-dimensional FFTs.
 
-      implicit none
+  ! c2c transform, multiple 1D FFTs in x direction
+  subroutine c2c_1m_x(inout, isign, plan1)
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      type(C_PTR) :: plan1
+    implicit none
 
-      integer :: foo
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    type(C_PTR) :: plan1
 
-      foo = isign ! Silence unused dummy argument
+    integer :: foo
 
+    foo = isign ! Silence unused dummy argument
+    
 #ifdef DOUBLE_PREC
-      call fftw_execute_dft(plan1, inout, inout)
+    call fftw_execute_dft(plan1, inout, inout)
 #else
-      call fftwf_execute_dft(plan1, inout, inout)
+    call fftwf_execute_dft(plan1, inout, inout)
 #endif
 
-      return
-   end subroutine c2c_1m_x
+    return
+  end subroutine c2c_1m_x
 
-   ! c2c transform, multiple 1D FFTs in y direction
-   subroutine c2c_1m_y(inout, isign, plan1)
 
-      implicit none
+  ! c2c transform, multiple 1D FFTs in y direction
+  subroutine c2c_1m_y(inout, isign, plan1)
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      type(C_PTR) :: plan1
+    implicit none
 
-      integer :: k, s3
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    type(C_PTR) :: plan1
 
-      integer :: foo
+    integer :: k, s3
 
-      foo = isign ! Silence unused dummy argument
+    integer :: foo
 
-      s3 = size(inout, 3)
+    foo = isign ! Silence unused dummy argument
+    
+    s3 = size(inout,3)
 
-      do k = 1, s3  ! transform on one Z-plane at a time
+    do k=1,s3  ! transform on one Z-plane at a time
 #ifdef DOUBLE_PREC
-         call fftw_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
+       call fftw_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
 #else
-         call fftwf_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
+       call fftwf_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
 #endif
-      end do
-
-      return
-   end subroutine c2c_1m_y
+    end do
 
-   ! c2c transform, multiple 1D FFTs in z direction
-   subroutine c2c_1m_z(inout, isign, plan1)
+    return
+  end subroutine c2c_1m_y
 
-      implicit none
+  ! c2c transform, multiple 1D FFTs in z direction
+  subroutine c2c_1m_z(inout, isign, plan1)
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      type(C_PTR) :: plan1
+    implicit none
 
-      integer :: foo
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    type(C_PTR) :: plan1
 
-      foo = isign ! Silence unused dummy argument
+    integer :: foo
 
+    foo = isign ! Silence unused dummy argument
+    
 #ifdef DOUBLE_PREC
-      call fftw_execute_dft(plan1, inout, inout)
+       call fftw_execute_dft(plan1, inout, inout)
 #else
-      call fftwf_execute_dft(plan1, inout, inout)
+       call fftwf_execute_dft(plan1, inout, inout)
 #endif
 
-      return
-   end subroutine c2c_1m_z
+    return
+  end subroutine c2c_1m_z
 
-   ! r2c transform, multiple 1D FFTs in x direction
-   subroutine r2c_1m_x(input, output)
+  ! r2c transform, multiple 1D FFTs in x direction
+  subroutine r2c_1m_x(input, output)
 
-      implicit none
+    implicit none
 
-      real(mytype), dimension(:, :, :), intent(INOUT)  ::  input
-      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+    real(mytype), dimension(:,:,:), intent(INOUT)  ::  input
+    complex(mytype), dimension(:,:,:), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-      call fftw_execute_dft_r2c(plan(0, 1), input, output)
+    call fftw_execute_dft_r2c(plan(0,1), input, output)
 #else
-      call fftwf_execute_dft_r2c(plan(0, 1), input, output)
-#endif
+    call fftwf_execute_dft_r2c(plan(0,1), input, output)
+#endif    
 
-      return
+    return
 
-   end subroutine r2c_1m_x
+  end subroutine r2c_1m_x
 
-   ! r2c transform, multiple 1D FFTs in z direction
-   subroutine r2c_1m_z(input, output)
+  ! r2c transform, multiple 1D FFTs in z direction
+  subroutine r2c_1m_z(input, output)
 
-      implicit none
+    implicit none
 
-      real(mytype), dimension(:, :, :), intent(INOUT)  ::  input
-      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+    real(mytype), dimension(:,:,:), intent(INOUT)  ::  input
+    complex(mytype), dimension(:,:,:), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-      call fftw_execute_dft_r2c(plan(0, 3), input, output)
+    call fftw_execute_dft_r2c(plan(0,3), input, output)
 #else
-      call fftwf_execute_dft_r2c(plan(0, 3), input, output)
+    call fftwf_execute_dft_r2c(plan(0,3), input, output)
 #endif
 
-      return
+    return
 
-   end subroutine r2c_1m_z
+  end subroutine r2c_1m_z
 
-   ! c2r transform, multiple 1D FFTs in x direction
-   subroutine c2r_1m_x(input, output)
+  ! c2r transform, multiple 1D FFTs in x direction
+  subroutine c2r_1m_x(input, output)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT)  ::  input
-      real(mytype), dimension(:, :, :), intent(OUT) :: output
+    complex(mytype), dimension(:,:,:), intent(INOUT)  ::  input
+    real(mytype), dimension(:,:,:), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-      call fftw_execute_dft_c2r(plan(2, 1), input, output)
+    call fftw_execute_dft_c2r(plan(2,1), input, output)
 #else
-      call fftwf_execute_dft_c2r(plan(2, 1), input, output)
+    call fftwf_execute_dft_c2r(plan(2,1), input, output)
 #endif
 
-      return
+    return
 
-   end subroutine c2r_1m_x
+  end subroutine c2r_1m_x
 
-   ! c2r transform, multiple 1D FFTs in z direction
-   subroutine c2r_1m_z(input, output)
+  ! c2r transform, multiple 1D FFTs in z direction
+  subroutine c2r_1m_z(input, output)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: input
-      real(mytype), dimension(:, :, :), intent(OUT) :: output
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: input
+    real(mytype), dimension(:,:,:), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-      call fftw_execute_dft_c2r(plan(2, 3), input, output)
+    call fftw_execute_dft_c2r(plan(2,3), input, output)
 #else
-      call fftwf_execute_dft_c2r(plan(2, 3), input, output)
-#endif
+    call fftwf_execute_dft_c2r(plan(2,3), input, output)
+#endif    
 
-      return
+    return
 
-   end subroutine c2r_1m_z
+  end subroutine c2r_1m_z
 
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D FFT - complex to complex
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_c2c(in, out, isign)
 
-      implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: in
-      complex(mytype), dimension(:, :, :), intent(OUT) :: out
-      integer, intent(IN) :: isign
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  ! 3D FFT - complex to complex
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  subroutine fft_3d_c2c(in, out, isign)
+    
+    implicit none
+    
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: in
+    complex(mytype), dimension(:,:,:), intent(OUT) :: out
+    integer, intent(IN) :: isign
 
 #ifndef OVERWRITE
-      complex(mytype), pointer :: wk1(:, :, :)
-      integer(C_SIZE_T) :: sz
-      type(C_PTR) :: wk1_p
+    complex(mytype), pointer :: wk1(:,:,:)
+    integer(C_SIZE_T) :: sz
+    type(C_PTR) :: wk1_p
 
-      wk1_p = c_null_ptr ! Initialise to NULL pointer
+    wk1_p = c_null_ptr ! Initialise to NULL pointer
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
-          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
-
-         ! ===== 1D FFTs in X =====
+    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
+         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
+       
+       ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in, isign, plan(isign, 1))
+       call c2c_1m_x(in,isign,plan(isign,1))
 #else
-         sz = ph%xsz(1)*ph%xsz(2)*ph%xsz(3)
-         wk1_p = fftw_alloc_complex(sz)
-         call c_f_pointer(wk1_p, wk1, [ph%xsz(1), ph%xsz(2), ph%xsz(3)])
-         wk1 = in
-         call c2c_1m_x(wk1, isign, plan(isign, 1))
+       sz = ph%xsz(1)*ph%xsz(2)*ph%xsz(3)
+       wk1_p = fftw_alloc_complex(sz)
+       call c_f_pointer(wk1_p, wk1, [ph%xsz(1),ph%xsz(2),ph%xsz(3)])
+       wk1 = in
+       call c2c_1m_x(wk1,isign,plan(isign,1))
 #endif
 
-         ! ===== Swap X --> Y; 1D FFTs in Y =====
+       ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-         if (dims(1) > 1) then
+       if (dims(1)>1) then
 #ifdef OVERWRITE
-            call transpose_x_to_y(in, wk2_c2c, ph)
+          call transpose_x_to_y(in,wk2_c2c,ph)
 #else
-            call transpose_x_to_y(wk1, wk2_c2c, ph)
+          call transpose_x_to_y(wk1,wk2_c2c,ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
-         else
+          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
+       else
 #ifdef OVERWRITE
-            call c2c_1m_y(in, isign, plan(isign, 2))
+          call c2c_1m_y(in,isign,plan(isign,2))
 #else
-            call c2c_1m_y(wk1, isign, plan(isign, 2))
+          call c2c_1m_y(wk1,isign,plan(isign,2))
 #endif
-         end if
+       end if
 
-         ! ===== Swap Y --> Z; 1D FFTs in Z =====
-         if (dims(1) > 1) then
-            call transpose_y_to_z(wk2_c2c, out, ph)
-         else
+       ! ===== Swap Y --> Z; 1D FFTs in Z =====
+       if (dims(1)>1) then
+          call transpose_y_to_z(wk2_c2c,out,ph)
+       else
 #ifdef OVERWRITE
-            call transpose_y_to_z(in, out, ph)
+          call transpose_y_to_z(in,out,ph)
 #else
-            call transpose_y_to_z(wk1, out, ph)
+          call transpose_y_to_z(wk1,out,ph)
 #endif
-         end if
-         call c2c_1m_z(out, isign, plan(isign, 3))
+       end if
+       call c2c_1m_z(out,isign,plan(isign,3))
 
-      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
-               .OR. &
-               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
+    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
+         .OR. & 
+         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
 
-         ! ===== 1D FFTs in Z =====
+       ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in, isign, plan(isign, 3))
+       call c2c_1m_z(in,isign,plan(isign,3))
 #else
-         sz = ph%zsz(1)*ph%zsz(2)*ph%zsz(3)
-         wk1_p = fftw_alloc_complex(sz)
-         call c_f_pointer(wk1_p, wk1, [ph%zsz(1), ph%zsz(2), ph%zsz(3)])
-         wk1 = in
-         call c2c_1m_z(wk1, isign, plan(isign, 3))
+       sz = ph%zsz(1)*ph%zsz(2)*ph%zsz(3)
+       wk1_p = fftw_alloc_complex(sz)
+       call c_f_pointer(wk1_p, wk1, [ph%zsz(1),ph%zsz(2),ph%zsz(3)])
+       wk1 = in
+       call c2c_1m_z(wk1,isign,plan(isign,3))
 #endif
 
-         ! ===== Swap Z --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
+       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
 #ifdef OVERWRITE
-            call transpose_z_to_y(in, wk2_c2c, ph)
+          call transpose_z_to_y(in,wk2_c2c,ph)
 #else
-            call transpose_z_to_y(wk1, wk2_c2c, ph)
+          call transpose_z_to_y(wk1,wk2_c2c,ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
-         else  ! out==wk2_c2c if 1D decomposition
+          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
+       else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-            call transpose_z_to_y(in, out, ph)
+          call transpose_z_to_y(in,out,ph)
 #else
-            call transpose_z_to_y(wk1, out, ph)
+          call transpose_z_to_y(wk1,out,ph)
 #endif
-            call c2c_1m_y(out, isign, plan(isign, 2))
-         end if
-
-         ! ===== Swap Y --> X; 1D FFTs in X =====
-         if (dims(1) > 1) then
-            call transpose_y_to_x(wk2_c2c, out, ph)
-         end if
-         call c2c_1m_x(out, isign, plan(isign, 1))
+          call c2c_1m_y(out,isign,plan(isign,2))
+       end if
 
-      end if
+       ! ===== Swap Y --> X; 1D FFTs in X =====
+       if (dims(1)>1) then
+          call transpose_y_to_x(wk2_c2c,out,ph)
+       end if
+       call c2c_1m_x(out,isign,plan(isign,1))
+       
+    end if
 
 #ifndef OVERWRITE
-      call fftw_free(wk1_p)
+    call fftw_free(wk1_p)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-      return
-   end subroutine fft_3d_c2c
+    return
+  end subroutine fft_3d_c2c
 
+  
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D forward FFT - real to complex
+  ! 3D forward FFT - real to complex
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_r2c(in_r, out_c)
-
-      implicit none
-
-      real(mytype), dimension(:, :, :), intent(INOUT) :: in_r
-      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
+  subroutine fft_3d_r2c(in_r, out_c)
+    
+    implicit none
+    
+    real(mytype), dimension(:,:,:), intent(INOUT) :: in_r
+    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+    if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-      if (format == PHYSICAL_IN_X) then
-
-         ! ===== 1D FFTs in X =====
-         call r2c_1m_x(in_r, wk13)
-
-         ! ===== Swap X --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
-            call transpose_x_to_y(wk13, wk2_r2c, sp)
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
-         else
-            call c2c_1m_y(wk13, -1, plan(0, 2))
-         end if
-
-         ! ===== Swap Y --> Z; 1D FFTs in Z =====
-         if (dims(1) > 1) then
-            call transpose_y_to_z(wk2_r2c, out_c, sp)
-         else
-            call transpose_y_to_z(wk13, out_c, sp)
-         end if
-         call c2c_1m_z(out_c, -1, plan(0, 3))
-
-      else if (format == PHYSICAL_IN_Z) then
-
-         ! ===== 1D FFTs in Z =====
-         call r2c_1m_z(in_r, wk13)
-
-         ! ===== Swap Z --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
-            call transpose_z_to_y(wk13, wk2_r2c, sp)
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
-         else  ! out_c==wk2_r2c if 1D decomposition
-            call transpose_z_to_y(wk13, out_c, sp)
-            call c2c_1m_y(out_c, -1, plan(0, 2))
-         end if
-
-         ! ===== Swap Y --> X; 1D FFTs in X =====
-         if (dims(1) > 1) then
-            call transpose_y_to_x(wk2_r2c, out_c, sp)
-         end if
-         call c2c_1m_x(out_c, -1, plan(0, 1))
-
-      end if
+    if (format==PHYSICAL_IN_X) then
+
+       ! ===== 1D FFTs in X =====
+       call r2c_1m_x(in_r,wk13)
+
+       ! ===== Swap X --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
+          call transpose_x_to_y(wk13,wk2_r2c,sp)
+          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
+       else
+          call c2c_1m_y(wk13,-1,plan(0,2))
+       end if
+
+       ! ===== Swap Y --> Z; 1D FFTs in Z =====
+       if (dims(1)>1) then
+          call transpose_y_to_z(wk2_r2c,out_c,sp)
+       else
+          call transpose_y_to_z(wk13,out_c,sp)
+       end if
+       call c2c_1m_z(out_c,-1,plan(0,3))
+                
+    else if (format==PHYSICAL_IN_Z) then
+
+       ! ===== 1D FFTs in Z =====
+       call r2c_1m_z(in_r,wk13)
+
+       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
+          call transpose_z_to_y(wk13,wk2_r2c,sp)
+          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
+       else  ! out_c==wk2_r2c if 1D decomposition
+          call transpose_z_to_y(wk13,out_c,sp)
+          call c2c_1m_y(out_c,-1,plan(0,2))
+       end if
+
+       ! ===== Swap Y --> X; 1D FFTs in X =====
+       if (dims(1)>1) then
+          call transpose_y_to_x(wk2_r2c,out_c,sp)
+       end if
+       call c2c_1m_x(out_c,-1,plan(0,1))
+
+    end if
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+    if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-      return
-   end subroutine fft_3d_r2c
-
+    return
+  end subroutine fft_3d_r2c
+  
+  
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D inverse FFT - complex to real
+  ! 3D inverse FFT - complex to real
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_c2r(in_c, out_r)
-
-      implicit none
-
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
-      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
+  subroutine fft_3d_c2r(in_c, out_r)
+    
+    implicit none
+    
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
+    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
 
 #ifndef OVERWRITE
-      complex(mytype), pointer :: wk1(:, :, :)
-      integer(C_SIZE_T) :: sz
-      type(C_PTR) :: wk1_p
+    complex(mytype), pointer :: wk1(:,:,:)
+    integer(C_SIZE_T) :: sz
+    type(C_PTR) :: wk1_p
 
-      wk1_p = c_null_ptr ! Initialise to NULL pointer
+    wk1_p = c_null_ptr ! Initialise to NULL pointer
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-      if (format == PHYSICAL_IN_X) then
+    if (format==PHYSICAL_IN_X) then
 
-         ! ===== 1D FFTs in Z =====
+       ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in_c, 1, plan(2, 3))
+       call c2c_1m_z(in_c,1,plan(2,3))       
 #else
-         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
-         wk1_p = fftw_alloc_complex(sz)
-         call c_f_pointer(wk1_p, wk1, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
-         wk1 = in_c
-         call c2c_1m_z(wk1, 1, plan(2, 3))
+       sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+       wk1_p = fftw_alloc_complex(sz)
+       call c_f_pointer(wk1_p, wk1, [sp%zsz(1),sp%zsz(2),sp%zsz(3)])
+       wk1 = in_c
+       call c2c_1m_z(wk1,1,plan(2,3))
 #endif
 
-         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+       ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-         call transpose_z_to_y(in_c, wk2_r2c, sp)
+       call transpose_z_to_y(in_c,wk2_r2c,sp)
 #else
-         call transpose_z_to_y(wk1, wk2_r2c, sp)
+       call transpose_z_to_y(wk1,wk2_r2c,sp)
 #endif
-         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+       call c2c_1m_y(wk2_r2c,1,plan(2,2))
 
-         ! ===== Swap Y --> X; 1D FFTs in X =====
-         if (dims(1) > 1) then
-            call transpose_y_to_x(wk2_r2c, wk13, sp)
-            call c2r_1m_x(wk13, out_r)
-         else
-            call c2r_1m_x(wk2_r2c, out_r)
-         end if
+       ! ===== Swap Y --> X; 1D FFTs in X =====
+       if (dims(1)>1) then
+          call transpose_y_to_x(wk2_r2c,wk13,sp)
+          call c2r_1m_x(wk13,out_r)
+       else
+          call c2r_1m_x(wk2_r2c,out_r)
+       end if
 
-      else if (format == PHYSICAL_IN_Z) then
+    else if (format==PHYSICAL_IN_Z) then
 
-         ! ===== 1D FFTs in X =====
+       ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in_c, 1, plan(2, 1))
+       call c2c_1m_x(in_c,1,plan(2,1))
 #else
-         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
-         wk1_p = fftw_alloc_complex(sz)
-         call c_f_pointer(wk1_p, wk1, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
-         wk1 = in_c
-         call c2c_1m_x(wk1, 1, plan(2, 1))
+       sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+       wk1_p = fftw_alloc_complex(sz)
+       call c_f_pointer(wk1_p, wk1, [sp%xsz(1),sp%xsz(2),sp%xsz(3)])
+       wk1 = in_c
+       call c2c_1m_x(wk1,1,plan(2,1))
 #endif
 
-         ! ===== Swap X --> Y; 1D FFTs in Y =====
-         if (dims(1) > 1) then
+       ! ===== Swap X --> Y; 1D FFTs in Y =====
+       if (dims(1)>1) then
 #ifdef OVERWRITE
-            call transpose_x_to_y(in_c, wk2_r2c, sp)
+          call transpose_x_to_y(in_c,wk2_r2c,sp)
 #else
-            call transpose_x_to_y(wk1, wk2_r2c, sp)
+          call transpose_x_to_y(wk1,wk2_r2c,sp)
 #endif
-            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
-         else  ! in_c==wk2_r2c if 1D decomposition
+          call c2c_1m_y(wk2_r2c,1,plan(2,2))
+       else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-            call c2c_1m_y(in_c, 1, plan(2, 2))
+          call c2c_1m_y(in_c,1,plan(2,2))
 #else
-            call c2c_1m_y(wk1, 1, plan(2, 2))
+          call c2c_1m_y(wk1,1,plan(2,2))
 #endif
-         end if
+       end if
 
-         ! ===== Swap Y --> Z; 1D FFTs in Z =====
-         if (dims(1) > 1) then
-            call transpose_y_to_z(wk2_r2c, wk13, sp)
-         else
+       ! ===== Swap Y --> Z; 1D FFTs in Z =====
+       if (dims(1)>1) then
+          call transpose_y_to_z(wk2_r2c,wk13,sp)
+       else
 #ifdef OVERWRITE
-            call transpose_y_to_z(in_c, wk13, sp)
+          call transpose_y_to_z(in_c,wk13,sp)
 #else
-            call transpose_y_to_z(wk1, wk13, sp)
+          call transpose_y_to_z(wk1,wk13,sp)
 #endif
-         end if
-         call c2r_1m_z(wk13, out_r)
+       end if
+       call c2r_1m_z(wk13,out_r)
 
-      end if
+    end if
 
 #ifndef OVERWRITE
-      call fftw_free(wk1_p)
+    call fftw_free(wk1_p)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-      return
-   end subroutine fft_3d_c2r
+    return
+  end subroutine fft_3d_c2r
 
+  
 end module decomp_2d_fft
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index 8bab0ae3..d66425bd 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,316 +13,320 @@
 
 module decomp_2d_fft
 
-   use decomp_2d  ! 2D decomposition module
-   use glassman
+  use decomp_2d  ! 2D decomposition module
+  use glassman
 
-   implicit none
+  implicit none
 
-   private        ! Make everything private unless declared public
+  private        ! Make everything private unless declared public
 
-   ! engine-specific global variables
-   complex(mytype), allocatable, dimension(:) :: buf, scratch
+  ! engine-specific global variables
+  complex(mytype), allocatable, dimension(:) :: buf, scratch
 
-   ! common code used for all engines, including global variables,
-   ! generic interface definitions and several subroutines
+  ! common code used for all engines, including global variables, 
+  ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time initialisations for the FFT engine
+  !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine init_fft_engine
+  subroutine init_fft_engine
+
+    implicit none
 
-      implicit none
+    integer :: cbuf_size
 
-      integer :: cbuf_size
+    if (nrank==0) then
+       write(*,*) ' '
+       write(*,*) '***** Using the generic FFT engine *****'
+       write(*,*) ' '
+    end if
 
-      if (nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the generic FFT engine *****'
-         write (*, *) ' '
-      end if
+    cbuf_size = max(ph%xsz(1), ph%ysz(2))
+    cbuf_size = max(cbuf_size, ph%zsz(3))
+    allocate(buf(cbuf_size))
+    allocate(scratch(cbuf_size))
 
-      cbuf_size = max(ph%xsz(1), ph%ysz(2))
-      cbuf_size = max(cbuf_size, ph%zsz(3))
-      allocate (buf(cbuf_size))
-      allocate (scratch(cbuf_size))
+    return
+  end subroutine init_fft_engine
 
-      return
-   end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time finalisations for the FFT engine
+  !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine finalize_fft_engine
+  subroutine finalize_fft_engine
+
+    implicit none
 
-      implicit none
+    if (allocated(buf)) deallocate(buf)
+    if (allocated(scratch)) deallocate(scratch)
 
-      if (allocated(buf)) deallocate (buf)
-      if (allocated(scratch)) deallocate (scratch)
+    return
+  end subroutine finalize_fft_engine
 
-      return
-   end subroutine finalize_fft_engine
 
-   ! Following routines calculate multiple one-dimensional FFTs to form
-   ! the basis of three-dimensional FFTs.
+  ! Following routines calculate multiple one-dimensional FFTs to form 
+  ! the basis of three-dimensional FFTs.
 
-   ! c2c transform, multiple 1D FFTs in x direction
-   subroutine c2c_1m_x(inout, isign, decomp)
+  ! c2c transform, multiple 1D FFTs in x direction
+  subroutine c2c_1m_x(inout, isign, decomp)
 
-      !$acc routine(spcfft) seq
+    !$acc routine(spcfft) seq
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: i, j, k
+    integer :: i,j,k
 
-      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-      do k = 1, decomp%xsz(3)
-         do j = 1, decomp%xsz(2)
-            do i = 1, decomp%xsz(1)
-               buf(i) = inout(i, j, k)
-            end do
-            call spcfft(buf, decomp%xsz(1), isign, scratch)
-            do i = 1, decomp%xsz(1)
-               inout(i, j, k) = buf(i)
-            end do
-         end do
-      end do
-      !$acc end parallel loop
+    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+    do k=1,decomp%xsz(3)
+       do j=1,decomp%xsz(2)
+          do i=1,decomp%xsz(1)
+             buf(i) = inout(i,j,k)
+          end do
+          call spcfft(buf,decomp%xsz(1),isign,scratch)
+          do i=1,decomp%xsz(1)
+             inout(i,j,k) = buf(i)
+          end do
+       end do
+    end do
+    !$acc end parallel loop
 
-      return
+    return
 
-   end subroutine c2c_1m_x
+  end subroutine c2c_1m_x
 
-   ! c2c transform, multiple 1D FFTs in y direction
-   subroutine c2c_1m_y(inout, isign, decomp)
+  ! c2c transform, multiple 1D FFTs in y direction
+  subroutine c2c_1m_y(inout, isign, decomp)
 
-      !$acc routine(spcfft) seq
+    !$acc routine(spcfft) seq
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: i, j, k
+    integer :: i,j,k
 
-      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-      do k = 1, decomp%ysz(3)
-         do i = 1, decomp%ysz(1)
-            do j = 1, decomp%ysz(2)
-               buf(j) = inout(i, j, k)
-            end do
-            call spcfft(buf, decomp%ysz(2), isign, scratch)
-            do j = 1, decomp%ysz(2)
-               inout(i, j, k) = buf(j)
-            end do
-         end do
-      end do
-      !$acc end parallel loop
+    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+    do k=1,decomp%ysz(3)
+       do i=1,decomp%ysz(1)
+          do j=1,decomp%ysz(2)
+             buf(j) = inout(i,j,k)
+          end do
+          call spcfft(buf,decomp%ysz(2),isign,scratch)
+          do j=1,decomp%ysz(2)
+             inout(i,j,k) = buf(j)
+          end do
+       end do
+    end do
+    !$acc end parallel loop
 
-      return
+    return
 
-   end subroutine c2c_1m_y
+  end subroutine c2c_1m_y
 
-   ! c2c transform, multiple 1D FFTs in z direction
-   subroutine c2c_1m_z(inout, isign, decomp)
+  ! c2c transform, multiple 1D FFTs in z direction
+  subroutine c2c_1m_z(inout, isign, decomp)
 
-      !$acc routine(spcfft) seq
-
-      implicit none
+    !$acc routine(spcfft) seq
 
-      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
 
-      integer :: i, j, k
+    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
+    integer, intent(IN) :: isign
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-      do j = 1, decomp%zsz(2)
-         do i = 1, decomp%zsz(1)
-            do k = 1, decomp%zsz(3)
-               buf(k) = inout(i, j, k)
-            end do
-            call spcfft(buf, decomp%zsz(3), isign, scratch)
-            do k = 1, decomp%zsz(3)
-               inout(i, j, k) = buf(k)
-            end do
-         end do
-      end do
-      !$acc end parallel loop
+    integer :: i,j,k
 
-      return
+    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+    do j=1,decomp%zsz(2)
+       do i=1,decomp%zsz(1)
+          do k=1,decomp%zsz(3)
+             buf(k) = inout(i,j,k)
+          end do
+          call spcfft(buf,decomp%zsz(3),isign,scratch)
+          do k=1,decomp%zsz(3)
+             inout(i,j,k) = buf(k)
+          end do
+       end do
+    end do
+    !$acc end parallel loop
 
-   end subroutine c2c_1m_z
+    return
 
-   ! r2c transform, multiple 1D FFTs in x direction
-   subroutine r2c_1m_x(input, output)
+  end subroutine c2c_1m_z
+
+  ! r2c transform, multiple 1D FFTs in x direction
+  subroutine r2c_1m_x(input, output)
+
+    !$acc routine(spcfft) seq
+
+    implicit none
+
+    real(mytype), dimension(:,:,:), intent(IN)  ::  input
+    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+
+    integer :: i,j,k, s1,s2,s3, d1
+
+    s1 = size(input,1)
+    s2 = size(input,2)
+    s3 = size(input,3)
+    d1 = size(output,1)
+
+    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+    do k=1,s3
+       do j=1,s2
+          ! Glassman's FFT is c2c only, 
+          ! needing some pre- and post-processing for r2c
+          ! pack real input in complex storage
+          do i=1,s1
+             buf(i) = cmplx(input(i,j,k),0._mytype, kind=mytype)
+          end do
+          call spcfft(buf,s1,-1,scratch)
+          ! note d1 ~ s1/2+1
+          ! simply drop the redundant part of the complex output
+          do i=1,d1
+             output(i,j,k) = buf(i)
+          end do
+       end do
+    end do
+    !$acc end parallel loop
+
+    return
+
+  end subroutine r2c_1m_x
+
+  ! r2c transform, multiple 1D FFTs in z direction
+  subroutine r2c_1m_z(input, output)
+
+    !$acc routine(spcfft) seq
+
+    implicit none
+
+    real(mytype), dimension(:,:,:), intent(IN)  ::  input
+    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+
+    integer :: i,j,k, s1,s2,s3, d3
+
+    s1 = size(input,1)
+    s2 = size(input,2)
+    s3 = size(input,3)
+    d3 = size(output,3)
+
+    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+    do j=1,s2
+       do i=1,s1
+          ! Glassman's FFT is c2c only, 
+          ! needing some pre- and post-processing for r2c
+          ! pack real input in complex storage
+          do k=1,s3
+             buf(k) = cmplx(input(i,j,k),0._mytype, kind=mytype)
+          end do
+          call spcfft(buf,s3,-1,scratch)
+          ! note d3 ~ s3/2+1
+          ! simply drop the redundant part of the complex output
+          do k=1,d3
+             output(i,j,k) = buf(k)
+          end do
+       end do
+    end do
+    !$acc end parallel loop
+
+    return
+
+  end subroutine r2c_1m_z
+
+  ! c2r transform, multiple 1D FFTs in x direction
+  subroutine c2r_1m_x(input, output)
+
+    !$acc routine(spcfft) seq
+
+    implicit none
+
+    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
+    real(mytype), dimension(:,:,:), intent(OUT) :: output
+
+    integer :: i,j,k, d1,d2,d3
+
+    d1 = size(output,1)
+    d2 = size(output,2)
+    d3 = size(output,3)
+
+    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+    do k=1,d3
+       do j=1,d2
+          ! Glassman's FFT is c2c only, 
+          ! needing some pre- and post-processing for c2r
+          do i=1,d1/2+1
+             buf(i) = input(i,j,k)
+          end do
+          ! expanding to a full-size complex array
+          ! For odd N, the storage is:
+          !  1, 2, ...... N/2+1   integer division rounded down
+          !     N, ...... N/2+2   => a(i) is conjugate of a(N+2-i)
+          ! For even N, the storage is:
+          !  1, 2, ...... N/2  , N/2+1
+          !     N, ...... N/2+2  again a(i) conjugate of a(N+2-i)
+          do i=d1/2+2,d1
+             buf(i) =  conjg(buf(d1+2-i))
+          end do
+          call spcfft(buf,d1,1,scratch)
+          do i=1,d1
+             ! simply drop imaginary part
+             output(i,j,k) = real(buf(i), kind=mytype)
+          end do
+       end do
+    end do
+    !$acc end parallel loop
+
+    return
+
+  end subroutine c2r_1m_x
+
+  ! c2r transform, multiple 1D FFTs in z direction
+  subroutine c2r_1m_z(input, output)
+
+    !$acc routine(spcfft) seq
+
+    implicit none
+
+    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
+    real(mytype), dimension(:,:,:), intent(OUT) :: output
+
+    integer :: i,j,k, d1,d2,d3
+
+    d1 = size(output,1)
+    d2 = size(output,2)
+    d3 = size(output,3)
+
+    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+    do j=1,d2
+       do i=1,d1
+          do k=1,d3/2+1
+             buf(k) = input(i,j,k)
+          end do
+          do k=d3/2+2,d3
+             buf(k) =  conjg(buf(d3+2-k))
+          end do
+          call spcfft(buf,d3,1,scratch)
+          do k=1,d3
+             output(i,j,k) = real(buf(k), kind=mytype)
+          end do
+       end do
+    end do
+    !$acc end parallel loop
+
+    return
+
+  end subroutine c2r_1m_z
 
-      !$acc routine(spcfft) seq
-
-      implicit none
-
-      real(mytype), dimension(:, :, :), intent(IN)  ::  input
-      complex(mytype), dimension(:, :, :), intent(OUT) :: output
-
-      integer :: i, j, k, s1, s2, s3, d1
-
-      s1 = size(input, 1)
-      s2 = size(input, 2)
-      s3 = size(input, 3)
-      d1 = size(output, 1)
-
-      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-      do k = 1, s3
-         do j = 1, s2
-            ! Glassman's FFT is c2c only,
-            ! needing some pre- and post-processing for r2c
-            ! pack real input in complex storage
-            do i = 1, s1
-               buf(i) = cmplx(input(i, j, k), 0._mytype, kind=mytype)
-            end do
-            call spcfft(buf, s1, -1, scratch)
-            ! note d1 ~ s1/2+1
-            ! simply drop the redundant part of the complex output
-            do i = 1, d1
-               output(i, j, k) = buf(i)
-            end do
-         end do
-      end do
-      !$acc end parallel loop
-
-      return
-
-   end subroutine r2c_1m_x
-
-   ! r2c transform, multiple 1D FFTs in z direction
-   subroutine r2c_1m_z(input, output)
-
-      !$acc routine(spcfft) seq
-
-      implicit none
-
-      real(mytype), dimension(:, :, :), intent(IN)  ::  input
-      complex(mytype), dimension(:, :, :), intent(OUT) :: output
-
-      integer :: i, j, k, s1, s2, s3, d3
-
-      s1 = size(input, 1)
-      s2 = size(input, 2)
-      s3 = size(input, 3)
-      d3 = size(output, 3)
-
-      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-      do j = 1, s2
-         do i = 1, s1
-            ! Glassman's FFT is c2c only,
-            ! needing some pre- and post-processing for r2c
-            ! pack real input in complex storage
-            do k = 1, s3
-               buf(k) = cmplx(input(i, j, k), 0._mytype, kind=mytype)
-            end do
-            call spcfft(buf, s3, -1, scratch)
-            ! note d3 ~ s3/2+1
-            ! simply drop the redundant part of the complex output
-            do k = 1, d3
-               output(i, j, k) = buf(k)
-            end do
-         end do
-      end do
-      !$acc end parallel loop
-
-      return
-
-   end subroutine r2c_1m_z
-
-   ! c2r transform, multiple 1D FFTs in x direction
-   subroutine c2r_1m_x(input, output)
-
-      !$acc routine(spcfft) seq
-
-      implicit none
-
-      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
-      real(mytype), dimension(:, :, :), intent(OUT) :: output
-
-      integer :: i, j, k, d1, d2, d3
-
-      d1 = size(output, 1)
-      d2 = size(output, 2)
-      d3 = size(output, 3)
-
-      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-      do k = 1, d3
-         do j = 1, d2
-            ! Glassman's FFT is c2c only,
-            ! needing some pre- and post-processing for c2r
-            do i = 1, d1/2 + 1
-               buf(i) = input(i, j, k)
-            end do
-            ! expanding to a full-size complex array
-            ! For odd N, the storage is:
-            !  1, 2, ...... N/2+1   integer division rounded down
-            !     N, ...... N/2+2   => a(i) is conjugate of a(N+2-i)
-            ! For even N, the storage is:
-            !  1, 2, ...... N/2  , N/2+1
-            !     N, ...... N/2+2  again a(i) conjugate of a(N+2-i)
-            do i = d1/2 + 2, d1
-               buf(i) = conjg(buf(d1 + 2 - i))
-            end do
-            call spcfft(buf, d1, 1, scratch)
-            do i = 1, d1
-               ! simply drop imaginary part
-               output(i, j, k) = real(buf(i), kind=mytype)
-            end do
-         end do
-      end do
-      !$acc end parallel loop
-
-      return
-
-   end subroutine c2r_1m_x
-
-   ! c2r transform, multiple 1D FFTs in z direction
-   subroutine c2r_1m_z(input, output)
-
-      !$acc routine(spcfft) seq
-
-      implicit none
-
-      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
-      real(mytype), dimension(:, :, :), intent(OUT) :: output
-
-      integer :: i, j, k, d1, d2, d3
-
-      d1 = size(output, 1)
-      d2 = size(output, 2)
-      d3 = size(output, 3)
-
-      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-      do j = 1, d2
-         do i = 1, d1
-            do k = 1, d3/2 + 1
-               buf(k) = input(i, j, k)
-            end do
-            do k = d3/2 + 2, d3
-               buf(k) = conjg(buf(d3 + 2 - k))
-            end do
-            call spcfft(buf, d3, 1, scratch)
-            do k = 1, d3
-               output(i, j, k) = real(buf(k), kind=mytype)
-            end do
-         end do
-      end do
-      !$acc end parallel loop
-
-      return
-
-   end subroutine c2r_1m_z
 
 #include "fft_common_3d.f90"
 
+
 end module decomp_2d_fft
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index cebed05c..b50120aa 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,660 +13,666 @@
 
 module decomp_2d_fft
 
-   use decomp_2d  ! 2D decomposition module
-   use MKL_DFTI   ! MKL FFT module
+  use decomp_2d  ! 2D decomposition module
+  use MKL_DFTI   ! MKL FFT module
 
-   implicit none
+  implicit none
 
-   private        ! Make everything private unless declared public
+  private        ! Make everything private unless declared public
 
-   ! engine-specific global variables
+  ! engine-specific global variables
 
-   ! Descriptors for MKL FFT, one for each set of 1D FFTs
-   !  for c2c transforms
-   type(DFTI_DESCRIPTOR), pointer :: c2c_x, c2c_y, c2c_z
-   !  for r2c/c2r transforms, PHYSICAL_IN_X
-   type(DFTI_DESCRIPTOR), pointer :: r2c_x, c2c_y2, c2c_z2, c2r_x
-   !  for r2c/c2r transforms, PHYSICAL_IN_Z
-   type(DFTI_DESCRIPTOR), pointer :: r2c_z, c2c_x2, c2r_z
+  ! Descriptors for MKL FFT, one for each set of 1D FFTs
+  !  for c2c transforms
+  type(DFTI_DESCRIPTOR), pointer :: c2c_x, c2c_y, c2c_z     
+  !  for r2c/c2r transforms, PHYSICAL_IN_X
+  type(DFTI_DESCRIPTOR), pointer :: r2c_x, c2c_y2, c2c_z2, c2r_x
+  !  for r2c/c2r transforms, PHYSICAL_IN_Z
+  type(DFTI_DESCRIPTOR), pointer :: r2c_z, c2c_x2, c2r_z
 
-   ! common code used for all engines, including global variables,
-   ! generic interface definitions and several subroutines
+  ! common code used for all engines, including global variables, 
+  ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time initialisations for the FFT engine
+  !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine init_fft_engine
+  subroutine init_fft_engine
 
-      implicit none
+    implicit none
 
-      if (nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the MKL engine *****'
-         write (*, *) ' '
-      end if
+    if (nrank==0) then
+       write(*,*) ' '
+       write(*,*) '***** Using the MKL engine *****'
+       write(*,*) ' '
+    end if
 #ifdef OVERWRITE
-      call decomp_2d_warning(0, "MKL FFT engine does not support the OVERWRITE flag.")
+    call decomp_2d_warning(0, "MKL FFT engine does not support the OVERWRITE flag.")
 #endif
 
-      ! For C2C transforms
-      call c2c_1m_x_plan(c2c_x, ph)
-      call c2c_1m_y_plan(c2c_y, ph)
-      call c2c_1m_z_plan(c2c_z, ph)
+    ! For C2C transforms
+    call c2c_1m_x_plan(c2c_x, ph)
+    call c2c_1m_y_plan(c2c_y, ph)
+    call c2c_1m_z_plan(c2c_z, ph)
+
+    ! For R2C/C2R tranfroms with physical space in X-pencil
+    if (format == PHYSICAL_IN_X) then
+       call r2c_1m_x_plan(r2c_x, ph, sp, -1)
+       call c2c_1m_y_plan(c2c_y2, sp)
+       call c2c_1m_z_plan(c2c_z2, sp)
+       call r2c_1m_x_plan(c2r_x, ph, sp,  1)
 
-      ! For R2C/C2R tranfroms with physical space in X-pencil
-      if (format == PHYSICAL_IN_X) then
-         call r2c_1m_x_plan(r2c_x, ph, sp, -1)
-         call c2c_1m_y_plan(c2c_y2, sp)
-         call c2c_1m_z_plan(c2c_z2, sp)
-         call r2c_1m_x_plan(c2r_x, ph, sp, 1)
+       ! For R2C/C2R tranfroms with physical space in Z-pencil
+    else if (format == PHYSICAL_IN_Z) then
+       call r2c_1m_z_plan(r2c_z, ph, sp, -1)
+       call c2c_1m_y_plan(c2c_y2, sp)
+       call c2c_1m_x_plan(c2c_x2, sp)
+       call r2c_1m_z_plan(c2r_z, ph, sp,  1)
+    end if
 
-         ! For R2C/C2R tranfroms with physical space in Z-pencil
-      else if (format == PHYSICAL_IN_Z) then
-         call r2c_1m_z_plan(r2c_z, ph, sp, -1)
-         call c2c_1m_y_plan(c2c_y2, sp)
-         call c2c_1m_x_plan(c2c_x2, sp)
-         call r2c_1m_z_plan(c2r_z, ph, sp, 1)
-      end if
+    return
+  end subroutine init_fft_engine
 
-      return
-   end subroutine init_fft_engine
 
-   ! Return an MKL plan for multiple 1D c2c FFTs in X direction
-   subroutine c2c_1m_x_plan(desc, decomp)
+  ! Return an MKL plan for multiple 1D c2c FFTs in X direction
+  subroutine c2c_1m_x_plan(desc, decomp)
 
-      implicit none
+    implicit none
 
-      type(DFTI_DESCRIPTOR), pointer :: desc
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+    type(DFTI_DESCRIPTOR), pointer :: desc
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: status
+    integer :: status
 
 #ifdef DOUBLE_PREC
-      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-                                    DFTI_COMPLEX, 1, decomp%xsz(1))
-#else
-      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-                                    DFTI_COMPLEX, 1, decomp%xsz(1))
+    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+         DFTI_COMPLEX, 1, decomp%xsz(1))
+#else       
+    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+         DFTI_COMPLEX, 1, decomp%xsz(1))
 #endif
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp%xsz(2)*decomp%xsz(3))
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, decomp%xsz(1))
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, decomp%xsz(1))
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiCommitDescriptor(desc)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+         decomp%xsz(2)*decomp%xsz(3))
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, decomp%xsz(1))
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, decomp%xsz(1))
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiCommitDescriptor(desc)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
 
-      return
-   end subroutine c2c_1m_x_plan
+    return
+  end subroutine c2c_1m_x_plan
 
-   ! Return an MKL plan for multiple 1D c2c FFTs in Y direction
-   subroutine c2c_1m_y_plan(desc, decomp)
+  ! Return an MKL plan for multiple 1D c2c FFTs in Y direction
+  subroutine c2c_1m_y_plan(desc, decomp)
 
-      implicit none
+    implicit none
 
-      type(DFTI_DESCRIPTOR), pointer :: desc
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+    type(DFTI_DESCRIPTOR), pointer :: desc
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: status, strides(2)
+    integer :: status, strides(2)
 
-      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-      ! done one Z-plane at a time. So plan for 2D data sets here.
+    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+    ! done one Z-plane at a time. So plan for 2D data sets here.
 
 #ifdef DOUBLE_PREC
-      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-                                    DFTI_COMPLEX, 1, decomp%ysz(2))
+    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+         DFTI_COMPLEX, 1, decomp%ysz(2))
 #else
-      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-                                    DFTI_COMPLEX, 1, decomp%ysz(2))
+    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+         DFTI_COMPLEX, 1, decomp%ysz(2))
 #endif
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, decomp%ysz(1))
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      strides(1) = 0
-      strides(2) = decomp%ysz(1)
-      status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiCommitDescriptor(desc)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-      return
-   end subroutine c2c_1m_y_plan
-
-   ! Return an MKL plan for multiple 1D c2c FFTs in Z direction
-   subroutine c2c_1m_z_plan(desc, decomp)
-
-      implicit none
-
-      type(DFTI_DESCRIPTOR), pointer :: desc
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-
-      integer :: status, strides(2)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, decomp%ysz(1))
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    strides(1) = 0
+    strides(2) = decomp%ysz(1)
+    status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiCommitDescriptor(desc)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+    return
+  end subroutine c2c_1m_y_plan
+
+  ! Return an MKL plan for multiple 1D c2c FFTs in Z direction
+  subroutine c2c_1m_z_plan(desc, decomp)
+
+    implicit none
+
+    type(DFTI_DESCRIPTOR), pointer :: desc
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+
+    integer :: status, strides(2)
 
 #ifdef DOUBLE_PREC
-      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-                                    DFTI_COMPLEX, 1, decomp%zsz(3))
+    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+         DFTI_COMPLEX, 1, decomp%zsz(3))
 #else
-      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-                                    DFTI_COMPLEX, 1, decomp%zsz(3))
+    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+         DFTI_COMPLEX, 1, decomp%zsz(3))
 #endif
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp%zsz(1)*decomp%zsz(2))
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      strides(1) = 0
-      strides(2) = decomp%zsz(1)*decomp%zsz(2)
-      status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiCommitDescriptor(desc)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-      return
-   end subroutine c2c_1m_z_plan
-
-   ! Return an MKL plan for multiple 1D r2c FFTs in X direction
-   subroutine r2c_1m_x_plan(desc, decomp_ph, decomp_sp, direction)
-
-      implicit none
-
-      type(DFTI_DESCRIPTOR), pointer :: desc
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
-      integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
-
-      integer :: status
-
-      ! c2r and r2c plans are almost the same, just swap input/output
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+         decomp%zsz(1)*decomp%zsz(2))
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    strides(1) = 0
+    strides(2) = decomp%zsz(1)*decomp%zsz(2)
+    status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiCommitDescriptor(desc)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+    return
+  end subroutine c2c_1m_z_plan
+
+  ! Return an MKL plan for multiple 1D r2c FFTs in X direction
+  subroutine r2c_1m_x_plan(desc, decomp_ph, decomp_sp, direction)
+
+    implicit none
+
+    type(DFTI_DESCRIPTOR), pointer :: desc
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
+    integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
+
+    integer :: status
+
+    ! c2r and r2c plans are almost the same, just swap input/output
 
 #ifdef DOUBLE_PREC
-      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-                                    DFTI_REAL, 1, decomp_ph%xsz(1))
+    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+         DFTI_REAL, 1, decomp_ph%xsz(1))
 #else
-      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-                                    DFTI_REAL, 1, decomp_ph%xsz(1))
+    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+         DFTI_REAL, 1, decomp_ph%xsz(1))
 #endif
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp_ph%xsz(2)*decomp_ph%xsz(3))
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
-                            DFTI_COMPLEX_COMPLEX)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      if (direction == -1) then  ! r2c
-         status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
-                               decomp_ph%xsz(1))
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-         status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
-                               decomp_sp%xsz(1))
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      else if (direction == 1) then  ! c2r
-         status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
-                               decomp_sp%xsz(1))
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-         status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
-                               decomp_ph%xsz(1))
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      end if
-      status = DftiCommitDescriptor(desc)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-      return
-   end subroutine r2c_1m_x_plan
-
-   ! Return an MKL plan for multiple 1D r2c FFTs in Z direction
-   subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
-
-      implicit none
-
-      type(DFTI_DESCRIPTOR), pointer :: desc
-      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
-      integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
-
-      integer :: status, strides(2)
-
-      ! c2r and r2c plans are almost the same, just swap input/output
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+         decomp_ph%xsz(2)*decomp_ph%xsz(3))
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
+         DFTI_COMPLEX_COMPLEX)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    if (direction == -1) then  ! r2c
+       status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
+            decomp_ph%xsz(1))
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
+            decomp_sp%xsz(1))
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    else if (direction == 1) then  ! c2r
+       status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
+            decomp_sp%xsz(1))
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
+            decomp_ph%xsz(1))
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    end if
+    status = DftiCommitDescriptor(desc)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+    return
+  end subroutine r2c_1m_x_plan
+
+  ! Return an MKL plan for multiple 1D r2c FFTs in Z direction
+  subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
+
+    implicit none
+
+    type(DFTI_DESCRIPTOR), pointer :: desc
+    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
+    integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
+
+    integer :: status, strides(2)
+
+    ! c2r and r2c plans are almost the same, just swap input/output
 
 #ifdef DOUBLE_PREC
-      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-                                    DFTI_REAL, 1, decomp_ph%zsz(3))
+    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+         DFTI_REAL, 1, decomp_ph%zsz(3))
 #else
-      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-                                    DFTI_REAL, 1, decomp_ph%zsz(3))
+    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+         DFTI_REAL, 1, decomp_ph%zsz(3))
 #endif
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp_ph%zsz(1)*decomp_ph%zsz(2))
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
-                            DFTI_COMPLEX_COMPLEX)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      strides(1) = 0
-      strides(2) = decomp_ph%zsz(1)*decomp_ph%zsz(2)
-      if (direction == -1) then
-         status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-      else if (direction == 1) then
-         status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-      end if
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      strides(2) = decomp_sp%zsz(1)*decomp_sp%zsz(2)
-      if (direction == -1) then
-         status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-      else if (direction == 1) then
-         status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-      end if
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      status = DftiCommitDescriptor(desc)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-      return
-   end subroutine r2c_1m_z_plan
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+         decomp_ph%zsz(1)*decomp_ph%zsz(2))
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
+         DFTI_COMPLEX_COMPLEX)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    strides(1) = 0
+    strides(2) = decomp_ph%zsz(1)*decomp_ph%zsz(2)
+    if (direction == -1) then
+       status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+    else if (direction == 1) then
+       status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+    end if
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    strides(2) = decomp_sp%zsz(1)*decomp_sp%zsz(2)
+    if (direction == -1) then
+       status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+    else if (direction == 1) then
+       status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+    end if
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+    status = DftiCommitDescriptor(desc)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+    return
+  end subroutine r2c_1m_z_plan
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  This routine performs one-time finalisations for the FFT engine
+  !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine finalize_fft_engine
-
-      implicit none
-
-      integer :: status
-
-      status = DftiFreeDescriptor(c2c_x)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-      status = DftiFreeDescriptor(c2c_y)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-      status = DftiFreeDescriptor(c2c_z)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-      if (format == PHYSICAL_IN_X) then
-         status = DftiFreeDescriptor(r2c_x)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-         status = DftiFreeDescriptor(c2c_z2)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-         status = DftiFreeDescriptor(c2r_x)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-      else if (format == PHYSICAL_IN_Z) then
-         status = DftiFreeDescriptor(r2c_z)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-         status = DftiFreeDescriptor(c2c_x2)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-         status = DftiFreeDescriptor(c2r_z)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-      end if
-      status = DftiFreeDescriptor(c2c_y2)
-      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-
-      return
-   end subroutine finalize_fft_engine
+  subroutine finalize_fft_engine
+
+    implicit none
+
+    integer :: status
+
+    status = DftiFreeDescriptor(c2c_x)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+    status = DftiFreeDescriptor(c2c_y)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+    status = DftiFreeDescriptor(c2c_z)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+    if (format==PHYSICAL_IN_X) then
+       status = DftiFreeDescriptor(r2c_x)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+       status = DftiFreeDescriptor(c2c_z2)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+       status = DftiFreeDescriptor(c2r_x)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+    else if (format==PHYSICAL_IN_Z) then
+       status = DftiFreeDescriptor(r2c_z)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+       status = DftiFreeDescriptor(c2c_x2)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+       status = DftiFreeDescriptor(c2r_z)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+    end if
+    status = DftiFreeDescriptor(c2c_y2)
+    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+
+    return
+  end subroutine finalize_fft_engine
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D FFT - complex to complex
+  ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_c2c(in, out, isign)
+  subroutine fft_3d_c2c(in, out, isign)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: in
-      complex(mytype), dimension(:, :, :), intent(OUT) :: out
-      integer, intent(IN) :: isign
+    complex(mytype), dimension(:,:,:), intent(IN) :: in
+    complex(mytype), dimension(:,:,:), intent(OUT) :: out
+    integer, intent(IN) :: isign   
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
-      integer :: k, status
+    complex(mytype), allocatable, dimension(:,:,:) :: wk1,wk2,wk2b,wk3
+    integer :: k, status
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
-          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
-
-         ! ===== 1D FFTs in X =====
-         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
-         !       if (isign==DECOMP_2D_FFT_FORWARD) then
-         !          status = DftiComputeForward(c2c_x, in(:,1,1), wk1(:,1,1))
-         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-         !          status = DftiComputeBackward(c2c_x, in(:,1,1), wk1(:,1,1))
-         !       end if
-         status = wrapper_c2c(c2c_x, in, wk1, isign)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-         ! ===== Swap X --> Y =====
-         allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
-         call transpose_x_to_y(wk1, wk2, ph)
-
-         ! ===== 1D FFTs in Y =====
-         allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
-         do k = 1, ph%xsz(3) ! one Z-plane at a time
-            !          if (isign==DECOMP_2D_FFT_FORWARD) then
-            !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-            !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
-            !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-            !          end if
-            status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
-            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-         end do
-
-         ! ===== Swap Y --> Z =====
-         allocate (wk3(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
-         call transpose_y_to_z(wk2b, wk3, ph)
-
-         ! ===== 1D FFTs in Z =====
-         !       if (isign==DECOMP_2D_FFT_FORWARD) then
-         !          status = DftiComputeForward(c2c_z, wk3(:,1,1), out(:,1,1))
-         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-         !          status = DftiComputeBackward(c2c_z, wk3(:,1,1), out(:,1,1))
-         !       end if
-         status = wrapper_c2c(c2c_z, wk3, out, isign)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
-               .OR. &
-               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
-
-         ! ===== 1D FFTs in Z =====
-         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
-         !       if (isign==DECOMP_2D_FFT_FORWARD) then
-         !          status = DftiComputeForward(c2c_z, in(:,1,1), wk1(:,1,1))
-         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-         !          status = DftiComputeBackward(c2c_z, in(:,1,1), wk1(:,1,1))
-         !       end if
-         status = wrapper_c2c(c2c_z, in, wk1, isign)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-         ! ===== Swap Z --> Y =====
-         allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
-         call transpose_z_to_y(wk1, wk2, ph)
-
-         ! ===== 1D FFTs in Y =====
-         allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
-         do k = 1, ph%xsz(3) ! one Z-plane at a time
-            !          if (isign==DECOMP_2D_FFT_FORWARD) then
-            !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-            !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
-            !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-            !          end if
-            status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
-            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-         end do
-
-         ! ===== Swap Y --> X =====
-         allocate (wk3(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
-         call transpose_y_to_x(wk2b, wk3, ph)
-
-         ! ===== 1D FFTs in X =====
-         !       if (isign==DECOMP_2D_FFT_FORWARD) then
-         !          status = DftiComputeForward(c2c_x, wk3(:,1,1), out(:,1,1))
-         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-         !          status = DftiComputeBackward(c2c_x, wk3(:,1,1), out(:,1,1))
-         !       end if
-         status = wrapper_c2c(c2c_x, wk3, out, isign)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-      end if
+    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
+         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
+
+       ! ===== 1D FFTs in X =====
+       allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
+       !       if (isign==DECOMP_2D_FFT_FORWARD) then
+       !          status = DftiComputeForward(c2c_x, in(:,1,1), wk1(:,1,1))
+       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+       !          status = DftiComputeBackward(c2c_x, in(:,1,1), wk1(:,1,1))
+       !       end if
+       status = wrapper_c2c(c2c_x, in, wk1, isign)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+       ! ===== Swap X --> Y =====
+       allocate (wk2(ph%ysz(1),ph%ysz(2),ph%ysz(3)))
+       call transpose_x_to_y(wk1,wk2,ph)
+
+       ! ===== 1D FFTs in Y =====
+       allocate (wk2b(ph%ysz(1),ph%ysz(2),ph%ysz(3))) 
+       do k=1,ph%xsz(3) ! one Z-plane at a time
+          !          if (isign==DECOMP_2D_FFT_FORWARD) then
+          !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+          !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
+          !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+          !          end if
+          status = wrapper_c2c(c2c_y, wk2(1,1,k), wk2b(1,1,k), isign)
+          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       end do
+
+       ! ===== Swap Y --> Z =====
+       allocate (wk3(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
+       call transpose_y_to_z(wk2b,wk3,ph)
+
+       ! ===== 1D FFTs in Z =====
+       !       if (isign==DECOMP_2D_FFT_FORWARD) then
+       !          status = DftiComputeForward(c2c_z, wk3(:,1,1), out(:,1,1))
+       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+       !          status = DftiComputeBackward(c2c_z, wk3(:,1,1), out(:,1,1))
+       !       end if
+       status = wrapper_c2c(c2c_z, wk3, out, isign)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
+         .OR. & 
+         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
+
+       ! ===== 1D FFTs in Z =====
+       allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
+       !       if (isign==DECOMP_2D_FFT_FORWARD) then
+       !          status = DftiComputeForward(c2c_z, in(:,1,1), wk1(:,1,1))
+       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+       !          status = DftiComputeBackward(c2c_z, in(:,1,1), wk1(:,1,1))
+       !       end if
+       status = wrapper_c2c(c2c_z, in, wk1, isign)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+       ! ===== Swap Z --> Y =====
+       allocate (wk2(ph%ysz(1),ph%ysz(2),ph%ysz(3)))
+       call transpose_z_to_y(wk1,wk2,ph)
+
+       ! ===== 1D FFTs in Y =====
+       allocate (wk2b(ph%ysz(1),ph%ysz(2),ph%ysz(3)))
+       do k=1,ph%xsz(3) ! one Z-plane at a time
+          !          if (isign==DECOMP_2D_FFT_FORWARD) then
+          !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+          !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
+          !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+          !          end if
+          status = wrapper_c2c(c2c_y, wk2(1,1,k), wk2b(1,1,k), isign)
+          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       end do
+
+       ! ===== Swap Y --> X =====
+       allocate (wk3(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
+       call transpose_y_to_x(wk2b,wk3,ph)
+
+       ! ===== 1D FFTs in X =====
+       !       if (isign==DECOMP_2D_FFT_FORWARD) then
+       !          status = DftiComputeForward(c2c_x, wk3(:,1,1), out(:,1,1))
+       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+       !          status = DftiComputeBackward(c2c_x, wk3(:,1,1), out(:,1,1))
+       !       end if
+       status = wrapper_c2c(c2c_x, wk3, out, isign)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+    end if
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-      return
-   end subroutine fft_3d_c2c
+    return
+  end subroutine fft_3d_c2c
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D forward FFT - real to complex
+  ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_r2c(in_r, out_c)
+  subroutine fft_3d_r2c(in_r, out_c)
 
-      implicit none
+    implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN) :: in_r
-      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
+    real(mytype), dimension(:,:,:), intent(IN) :: in_r
+    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
-      integer :: k, status, isign
+    complex(mytype), allocatable, dimension(:,:,:) :: wk1,wk2,wk2b,wk3
+    integer :: k, status, isign
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-      isign = DECOMP_2D_FFT_FORWARD
+    isign = DECOMP_2D_FFT_FORWARD
 
-      if (format == PHYSICAL_IN_X) then
+    if (format==PHYSICAL_IN_X) then
 
-         ! ===== 1D FFTs in X =====
-         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
-         !       status = DftiComputeForward(r2c_x, in_r(:,1,1), wk1(:,1,1))
-         status = wrapper_r2c(r2c_x, in_r, wk1)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
+       ! ===== 1D FFTs in X =====
+       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
+       !       status = DftiComputeForward(r2c_x, in_r(:,1,1), wk1(:,1,1))       
+       status = wrapper_r2c(r2c_x, in_r, wk1)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
-         ! ===== Swap X --> Y =====
-         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         call transpose_x_to_y(wk1, wk2, sp)
+       ! ===== Swap X --> Y =====
+       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
+       call transpose_x_to_y(wk1,wk2,sp)
 
-         ! ===== 1D FFTs in Y =====
-         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         do k = 1, sp%ysz(3)
-            !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
-            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-         end do
+       ! ===== 1D FFTs in Y =====
+       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
+       do k=1,sp%ysz(3)
+          !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
+          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       end do
 
-         ! ===== Swap Y --> Z =====
-         allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
-         call transpose_y_to_z(wk2b, wk3, sp)
+       ! ===== Swap Y --> Z =====
+       allocate (wk3(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
+       call transpose_y_to_z(wk2b,wk3,sp)
 
-         ! ===== 1D FFTs in Z =====
-         !       status = DftiComputeForward(c2c_z2, wk3(:,1,1), out_c(:,1,1))
-         status = wrapper_c2c(c2c_z2, wk3, out_c, isign)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       ! ===== 1D FFTs in Z =====
+       !       status = DftiComputeForward(c2c_z2, wk3(:,1,1), out_c(:,1,1))
+       status = wrapper_c2c(c2c_z2, wk3, out_c, isign)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-      else if (format == PHYSICAL_IN_Z) then
+    else if (format==PHYSICAL_IN_Z) then
 
-         ! ===== 1D FFTs in Z =====
-         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
-         !       status = DftiComputeForward(r2c_z, in_r(:,1,1), wk1(:,1,1))
-         status = wrapper_r2c(r2c_z, in_r, wk1)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
+       ! ===== 1D FFTs in Z =====
+       allocate(wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
+       !       status = DftiComputeForward(r2c_z, in_r(:,1,1), wk1(:,1,1))
+       status = wrapper_r2c(r2c_z, in_r, wk1)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
-         ! ===== Swap Z --> Y =====
-         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         call transpose_z_to_y(wk1, wk2, sp)
+       ! ===== Swap Z --> Y =====
+       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
+       call transpose_z_to_y(wk1,wk2,sp)
 
-         ! ===== 1D FFTs in Y =====
-         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         do k = 1, sp%ysz(3)
-            !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
-            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-         end do
+       ! ===== 1D FFTs in Y =====
+       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
+       do k=1,sp%ysz(3)
+          !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
+          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       end do
 
-         ! ===== Swap Y --> X =====
-         allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
-         call transpose_y_to_x(wk2b, wk3, sp)
+       ! ===== Swap Y --> X =====
+       allocate (wk3(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
+       call transpose_y_to_x(wk2b,wk3,sp)
 
-         ! ===== 1D FFTs in X =====
-         !       status = DftiComputeForward(c2c_x2, wk3(:,1,1), out_c(:,1,1))
-         status = wrapper_c2c(c2c_x2, wk3, out_c, isign)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       ! ===== 1D FFTs in X =====
+       !       status = DftiComputeForward(c2c_x2, wk3(:,1,1), out_c(:,1,1))
+       status = wrapper_c2c(c2c_x2, wk3, out_c, isign)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-      end if
+    end if
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-      return
-   end subroutine fft_3d_r2c
+    return
+  end subroutine fft_3d_r2c
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! 3D inverse FFT - complex to real
+  ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine fft_3d_c2r(in_c, out_r)
+  subroutine fft_3d_c2r(in_c, out_r)
 
-      implicit none
+    implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: in_c
-      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
+    complex(mytype), dimension(:,:,:), intent(IN) :: in_c
+    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
-      integer :: k, status, isign
+    complex(mytype), allocatable, dimension(:,:,:) :: wk1,wk2,wk2b,wk3
+    integer :: k, status, isign
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-      isign = DECOMP_2D_FFT_BACKWARD
+    isign = DECOMP_2D_FFT_BACKWARD
 
-      if (format == PHYSICAL_IN_X) then
+    if (format==PHYSICAL_IN_X) then
 
-         ! ===== 1D FFTs in Z =====
-         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
-         !       status = DftiComputeBackward(c2c_z2, in_c(:,1,1), wk1(:,1,1))
-         status = wrapper_c2c(c2c_z2, in_c, wk1, isign)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       ! ===== 1D FFTs in Z =====
+       allocate (wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
+       !       status = DftiComputeBackward(c2c_z2, in_c(:,1,1), wk1(:,1,1))
+       status = wrapper_c2c(c2c_z2, in_c, wk1, isign)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-         ! ===== Swap Z --> Y =====
-         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         call transpose_z_to_y(wk1, wk2, sp)
+       ! ===== Swap Z --> Y =====
+       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
+       call transpose_z_to_y(wk1,wk2,sp)
 
-         ! ===== 1D FFTs in Y =====
-         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         do k = 1, sp%ysz(3)
-            !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
-            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-         end do
+       ! ===== 1D FFTs in Y =====
+       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
+       do k=1,sp%ysz(3)
+          !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
+          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       end do
 
-         ! ===== Swap Y --> X =====
-         allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
-         call transpose_y_to_x(wk2b, wk3, sp)
+       ! ===== Swap Y --> X =====
+       allocate (wk3(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
+       call transpose_y_to_x(wk2b,wk3,sp)
 
-         ! ===== 1D FFTs in X =====
-         !       status = DftiComputeBackward(c2r_x, wk3(:,1,1), out_r(:,1,1))
-         status = wrapper_c2r(c2r_x, wk3, out_r)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
+       ! ===== 1D FFTs in X =====
+       !       status = DftiComputeBackward(c2r_x, wk3(:,1,1), out_r(:,1,1))
+       status =  wrapper_c2r(c2r_x, wk3, out_r)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
 
-      else if (format == PHYSICAL_IN_Z) then
+    else if (format==PHYSICAL_IN_Z) then
 
-         ! ===== 1D FFTs in X =====
-         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
-         !       status = DftiComputeBackward(c2c_x2, in_c(:,1,1), wk1(:,1,1))
-         status = wrapper_c2c(c2c_x2, in_c, wk1, isign)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       ! ===== 1D FFTs in X =====
+       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
+       !       status = DftiComputeBackward(c2c_x2, in_c(:,1,1), wk1(:,1,1))
+       status = wrapper_c2c(c2c_x2, in_c, wk1, isign)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-         ! ===== Swap X --> Y =====
-         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         call transpose_x_to_y(wk1, wk2, sp)
+       ! ===== Swap X --> Y =====
+       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
+       call transpose_x_to_y(wk1,wk2,sp)
 
-         ! ===== 1D FFTs in Y =====
-         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         do k = 1, sp%ysz(3)
-            !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
-            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-         end do
+       ! ===== 1D FFTs in Y =====
+       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
+       do k=1,sp%ysz(3)
+          !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
+          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+       end do
 
-         ! ===== Swap Y --> Z =====
-         allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
-         call transpose_y_to_z(wk2b, wk3, sp)
+       ! ===== Swap Y --> Z =====
+       allocate (wk3(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
+       call transpose_y_to_z(wk2b,wk3,sp)
 
-         ! ===== 1D FFTs in Z =====
-         !       status = DftiComputeBackward(c2r_z, wk3(:,1,1), out_r(:,1,1))
-         status = wrapper_c2r(c2r_z, wk3, out_r)
-         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
+       ! ===== 1D FFTs in Z =====
+       !       status = DftiComputeBackward(c2r_z, wk3(:,1,1), out_r(:,1,1))
+       status =  wrapper_c2r(c2r_z, wk3, out_r)
+       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
 
-      end if
+    end if
 
 #ifdef PROFILER
-      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-      return
-   end subroutine fft_3d_c2r
+    return
+  end subroutine fft_3d_c2r
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Wrapper functions so that one can pass 3D arrays to DftiCompute
-   !  -- MKL accepts only 1D arrays as input/output for its multi-
-   !     dimensional FFTs.
-   !  -- Using EQUIVALENCE as suggested by MKL documents is impossible
-   !     for allocated arrays, not to mention bad coding style
-   !  -- All code commented out above may well work but not safe. There
-   !     is no guarantee that compiler wouldn't make copies of 1D arrays
-   !     (which would contain only one slice of the original 3D data)
-   !     rather than referring to the same memory address, i.e. 3D array
-   !     A and 1D array A(:,1,1) may refer to different memory location.
-   !  -- Using the following wrappers is safe and standard conforming.
+  ! Wrapper functions so that one can pass 3D arrays to DftiCompute
+  !  -- MKL accepts only 1D arrays as input/output for its multi-
+  !     dimensional FFTs.
+  !  -- Using EQUIVALENCE as suggested by MKL documents is impossible 
+  !     for allocated arrays, not to mention bad coding style
+  !  -- All code commented out above may well work but not safe. There 
+  !     is no guarantee that compiler wouldn't make copies of 1D arrays
+  !     (which would contain only one slice of the original 3D data) 
+  !     rather than referring to the same memory address, i.e. 3D array
+  !     A and 1D array A(:,1,1) may refer to different memory location.
+  !  -- Using the following wrappers is safe and standard conforming.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-   integer function wrapper_c2c(desc, in, out, isign)
+  integer function wrapper_c2c(desc, in, out, isign)
 
-      implicit none
+    implicit none
 
-      type(DFTI_DESCRIPTOR), pointer :: desc
-      complex(mytype), dimension(*) :: in, out
-      integer :: isign, status
+    type(DFTI_DESCRIPTOR), pointer :: desc
+    complex(mytype), dimension(*) :: in, out
+    integer :: isign, status
 
-      if (isign == DECOMP_2D_FFT_FORWARD) then
-         status = DftiComputeForward(desc, in, out)
-      else if (isign == DECOMP_2D_FFT_BACKWARD) then
-         status = DftiComputeBackward(desc, in, out)
-      end if
+    if (isign == DECOMP_2D_FFT_FORWARD) then
+       status = DftiComputeForward(desc, in, out)
+    else if (isign == DECOMP_2D_FFT_BACKWARD) then
+       status = DftiComputeBackward(desc, in, out)
+    end if
 
-      wrapper_c2c = status
+    wrapper_c2c = status
 
-      return
-   end function wrapper_c2c
+    return
+  end function wrapper_c2c
 
-   integer function wrapper_r2c(desc, in, out)
+  integer function wrapper_r2c(desc, in, out)
 
-      implicit none
+    implicit none
 
-      type(DFTI_DESCRIPTOR), pointer :: desc
-      real(mytype), dimension(*) :: in
-      complex(mytype), dimension(*) :: out
+    type(DFTI_DESCRIPTOR), pointer :: desc
+    real(mytype), dimension(*) :: in 
+    complex(mytype), dimension(*) :: out
 
-      wrapper_r2c = DftiComputeForward(desc, in, out)
+    wrapper_r2c = DftiComputeForward(desc, in, out)
 
-      return
-   end function wrapper_r2c
+    return
+  end function wrapper_r2c
 
-   integer function wrapper_c2r(desc, in, out)
+  integer function wrapper_c2r(desc, in, out)
 
-      implicit none
+    implicit none
 
-      type(DFTI_DESCRIPTOR), pointer :: desc
-      complex(mytype), dimension(*) :: in
-      real(mytype), dimension(*) :: out
+    type(DFTI_DESCRIPTOR), pointer :: desc
+    complex(mytype), dimension(*) :: in 
+    real(mytype), dimension(*) :: out
 
-      wrapper_c2r = DftiComputeBackward(desc, in, out)
+    wrapper_c2r = DftiComputeBackward(desc, in, out)
 
-      return
-   end function wrapper_c2r
+    return
+  end function wrapper_c2r
 
 end module decomp_2d_fft
diff --git a/src/glassman.f90 b/src/glassman.f90
index 170fb905..89fcf41e 100644
--- a/src/glassman.f90
+++ b/src/glassman.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,181 +9,184 @@
 !
 !=======================================================================
 
-! This module contains a few 'generic' FFT routines, making the
+! This module contains a few 'generic' FFT routines, making the 
 ! 2DECOMP&FFT library not dependent on any external libraries
 
 module glassman
 
-   use decomp_2d, only: mytype
+  use decomp_2d, only : mytype
 
-   implicit none
+  implicit none
 
 contains
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Following is a FFT implementation based on algorithm proposed by
-   ! Glassman, a general FFT algorithm supporting arbitrary input length.
-   !
-   ! W. E. Ferguson, Jr., "A simple derivation of Glassman general-n fast
-   ! Fourier transform," Comput. and Math. with Appls., vol. 8, no. 6, pp.
-   ! 401-411, 1982.
-   !
-   ! Original implemtation online at http://www.jjj.de/fft/fftpage.html
-   !
-   ! Updated
-   !  -  to handle double-precision as well
-   !  -  unnecessary scaling code removed
+  ! Following is a FFT implementation based on algorithm proposed by
+  ! Glassman, a general FFT algorithm supporting arbitrary input length.
+  !
+  ! W. E. Ferguson, Jr., "A simple derivation of Glassman general-n fast
+  ! Fourier transform," Comput. and Math. with Appls., vol. 8, no. 6, pp.
+  ! 401-411, 1982.
+  !
+  ! Original implemtation online at http://www.jjj.de/fft/fftpage.html
+  !
+  ! Updated  
+  !  -  to handle double-precision as well
+  !  -  unnecessary scaling code removed
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-   PURE SUBROUTINE SPCFFT(U, N, ISIGN, WORK)
-
-      !$acc routine seq
-
-      IMPLICIT NONE
-
-      LOGICAL :: INU
-      INTEGER :: A, B, C, I
-      INTEGER, INTENT(IN) :: N, ISIGN
-      COMPLEX(mytype), INTENT(INOUT) :: U(*), WORK(*)
-
-      A = 1
-      B = N
-      C = 1
-      INU = .TRUE.
-
-      DO WHILE (B .GT. 1)
-         A = C*A
-         C = 2
-         DO WHILE (MOD(B, C) .NE. 0)
-            C = C + 1
-         END DO
-         B = B/C
-         IF (INU) THEN
-            CALL SPCPFT(A, B, C, U, WORK, ISIGN)
-         ELSE
-            CALL SPCPFT(A, B, C, WORK, U, ISIGN)
-         END IF
-         INU = (.NOT. INU)
-      END DO
-
-      IF (.NOT. INU) THEN
-         DO I = 1, N
-            U(I) = WORK(I)
-         END DO
-      END IF
-
-      RETURN
-   END SUBROUTINE SPCFFT
-
-   PURE SUBROUTINE SPCPFT(A, B, C, UIN, UOUT, ISIGN)
-
-      !$acc routine seq
-
-      IMPLICIT NONE
-
-      INTEGER, INTENT(IN), VALUE :: A, B, C, ISIGN
-      INTEGER :: IA, IB, IC, JCR, JC
-
-      DOUBLE PRECISION :: ANGLE
-
-      COMPLEX(mytype), INTENT(IN) :: UIN(B, C, A)
-      COMPLEX(mytype), INTENT(OUT) :: UOUT(B, A, C)
-      COMPLEX(mytype) :: DELTA, OMEGA, SUM
-
-      ANGLE = 6.28318530717958_mytype/REAL(A*C, kind=mytype)
-      OMEGA = CMPLX(1.0, 0.0, kind=mytype)
-
-      IF (ISIGN .EQ. 1) THEN
-         DELTA = CMPLX(COS(ANGLE), SIN(ANGLE), kind=mytype)
-      ELSE
-         DELTA = CMPLX(COS(ANGLE), -SIN(ANGLE), kind=mytype)
-      END IF
-
-      DO IC = 1, C
-         DO IA = 1, A
-            DO IB = 1, B
-               SUM = UIN(IB, C, IA)
-               DO JCR = 2, C
-                  JC = C + 1 - JCR
-                  SUM = UIN(IB, JC, IA) + OMEGA*SUM
-               END DO
-               UOUT(IB, IA, IC) = SUM
-            END DO
-            OMEGA = DELTA*OMEGA
-         END DO
-      END DO
-
-      RETURN
-   END SUBROUTINE SPCPFT
+  PURE SUBROUTINE SPCFFT(U,N,ISIGN,WORK)
+
+    !$acc routine seq
+
+    IMPLICIT NONE
+
+    LOGICAL :: INU
+    INTEGER :: A,B,C,I
+    INTEGER, INTENT(IN) :: N, ISIGN
+    COMPLEX(mytype), INTENT(INOUT) :: U(*),WORK(*)
+
+    A = 1
+    B = N
+    C = 1
+    INU = .TRUE.
+
+    DO WHILE ( B .GT. 1 )
+       A = C * A
+       C = 2
+       DO WHILE ( MOD(B,C) .NE. 0 )
+          C = C + 1
+       END DO
+       B = B / C
+       IF ( INU ) THEN
+          CALL SPCPFT (A,B,C,U,WORK,ISIGN)
+       ELSE
+          CALL SPCPFT (A,B,C,WORK,U,ISIGN)
+       END IF
+       INU = ( .NOT. INU )
+    END DO
+
+    IF ( .NOT. INU ) THEN
+       DO I = 1, N
+          U(I) = WORK(I)
+       END DO
+    END IF
+
+    RETURN
+  END SUBROUTINE SPCFFT
+
+
+  PURE SUBROUTINE SPCPFT( A, B, C, UIN, UOUT, ISIGN )
+
+    !$acc routine seq
+
+    IMPLICIT NONE
+
+    INTEGER, INTENT(IN), VALUE :: A, B, C, ISIGN
+    INTEGER :: IA,IB,IC,JCR,JC
+
+    DOUBLE PRECISION :: ANGLE
+
+    COMPLEX(mytype), INTENT(IN) :: UIN(B,C,A)
+    COMPLEX(mytype), INTENT(OUT) :: UOUT(B,A,C)
+    COMPLEX(mytype) :: DELTA,OMEGA,SUM
+
+    ANGLE = 6.28318530717958_mytype / REAL( A * C, kind=mytype )
+    OMEGA = CMPLX( 1.0, 0.0, kind=mytype )
+
+    IF( ISIGN .EQ. 1 ) THEN
+       DELTA = CMPLX( COS(ANGLE), SIN(ANGLE), kind=mytype )
+    ELSE
+       DELTA = CMPLX( COS(ANGLE), -SIN(ANGLE), kind=mytype )
+    END IF
+
+    DO IC = 1, C
+       DO IA = 1, A
+          DO IB = 1, B
+             SUM = UIN( IB, C, IA )
+             DO JCR = 2, C
+                JC = C + 1 - JCR
+                SUM = UIN( IB, JC, IA ) + OMEGA * SUM
+             END DO
+             UOUT( IB, IA, IC ) = SUM
+          END DO
+          OMEGA = DELTA * OMEGA
+       END DO
+    END DO
+
+    RETURN
+  END SUBROUTINE SPCPFT
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! A 3D real-to-complex routine implemented using the 1D FFT above
-   !   Input:   nx*ny*nz real numbers
-   !   Output:  (nx/2+1)*ny*nz complex numbers
-   ! Just like big FFT libraries (such as FFTW) do
+  ! A 3D real-to-complex routine implemented using the 1D FFT above
+  !   Input:   nx*ny*nz real numbers
+  !   Output:  (nx/2+1)*ny*nz complex numbers
+  ! Just like big FFT libraries (such as FFTW) do
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine glassman_3d_r2c(in_r, nx, ny, nz, out_c)
-
-      implicit none
-
-      integer, intent(IN) :: nx, ny, nz
-      real(mytype), dimension(nx, ny, nz) :: in_r
-      complex(mytype), dimension(nx/2 + 1, ny, nz) :: out_c
-
-      complex(mytype), allocatable, dimension(:) :: buf, scratch
-      integer :: maxsize, i, j, k
-
-      maxsize = max(nx, max(ny, nz))
-      allocate (buf(maxsize))
-      allocate (scratch(maxsize))
-
-      ! ===== 1D FFTs in X =====
-      do k = 1, nz
-         do j = 1, ny
-            ! Glassman's 1D FFT is c2c only,
-            ! needing some pre- and post-processing for r2c
-            ! pack real input in complex storage
-            do i = 1, nx
-               buf(i) = cmplx(in_r(i, j, k), 0._mytype, kind=mytype)
-            end do
-            call spcfft(buf, nx, -1, scratch)
-            ! simply drop the redundant part of the complex output
-            do i = 1, nx/2 + 1
-               out_c(i, j, k) = buf(i)
-            end do
-         end do
-      end do
-
-      ! ===== 1D FFTs in Y =====
-      do k = 1, nz
-         do i = 1, nx/2 + 1
-            do j = 1, ny
-               buf(j) = out_c(i, j, k)
-            end do
-            call spcfft(buf, ny, -1, scratch)
-            do j = 1, ny
-               out_c(i, j, k) = buf(j)
-            end do
-         end do
-      end do
-
-      ! ===== 1D FFTs in Z =====
-      do j = 1, ny
-         do i = 1, nx/2 + 1
-            do k = 1, nz
-               buf(k) = out_c(i, j, k)
-            end do
-            call spcfft(buf, nz, -1, scratch)
-            do k = 1, nz
-               out_c(i, j, k) = buf(k)
-            end do
-         end do
-      end do
-
-      deallocate (buf, scratch)
-
-      return
-   end subroutine glassman_3d_r2c
+  subroutine glassman_3d_r2c(in_r,nx,ny,nz,out_c)
+
+    implicit none
+
+    integer, intent(IN) :: nx,ny,nz
+    real(mytype), dimension(nx,ny,nz) :: in_r
+    complex(mytype), dimension(nx/2+1,ny,nz) :: out_c
+
+    complex(mytype), allocatable, dimension(:) :: buf, scratch
+    integer :: maxsize, i,j,k
+
+    maxsize = max(nx, max(ny,nz))
+    allocate(buf(maxsize))
+    allocate(scratch(maxsize))
+
+    ! ===== 1D FFTs in X =====
+    do k=1,nz
+       do j=1,ny
+          ! Glassman's 1D FFT is c2c only, 
+          ! needing some pre- and post-processing for r2c
+          ! pack real input in complex storage
+          do i=1,nx
+             buf(i) = cmplx(in_r(i,j,k),0._mytype, kind=mytype)
+          end do
+          call spcfft(buf,nx,-1,scratch)
+          ! simply drop the redundant part of the complex output
+          do i=1,nx/2+1
+             out_c(i,j,k) = buf(i)
+          end do
+       end do
+    end do
+
+    ! ===== 1D FFTs in Y =====
+    do k=1,nz
+       do i=1,nx/2+1
+          do j=1,ny
+             buf(j) = out_c(i,j,k)
+          end do
+          call spcfft(buf,ny,-1,scratch)
+          do j=1,ny
+             out_c(i,j,k) = buf(j)
+          end do
+       end do
+    end do
+
+    ! ===== 1D FFTs in Z =====
+    do j=1,ny
+       do i=1,nx/2+1
+          do k=1,nz
+             buf(k) = out_c(i,j,k)
+          end do
+          call spcfft(buf,nz,-1,scratch)
+          do k=1,nz
+             out_c(i,j,k) = buf(k)
+          end do
+       end do
+    end do
+
+    deallocate(buf,scratch)
+
+    return
+  end subroutine glassman_3d_r2c
+
 
 end module glassman
 
diff --git a/src/halo.f90 b/src/halo.f90
index d0082f5c..ef324bf5 100644
--- a/src/halo.f90
+++ b/src/halo.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -14,98 +14,98 @@
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   subroutine update_halo_real_short(in, out, level, opt_global, opt_pencil)
 
-     implicit none
+    implicit none
 
-     integer, intent(IN) :: level      ! levels of halo cells required
-     real(mytype), dimension(:, :, :), intent(IN) :: in
-     real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
-     logical, optional :: opt_global
-     integer, intent(in), optional :: opt_pencil
+    integer, intent(IN) :: level      ! levels of halo cells required
+    real(mytype), dimension(:,:,:), intent(IN) :: in
+    real(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
+    logical, optional :: opt_global
+    integer, intent(in), optional :: opt_pencil
 
-     call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
+    call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
 
   end subroutine update_halo_real_short
 
   subroutine update_halo_real(in, out, level, decomp, opt_global, opt_pencil)
 
-     implicit none
-
-     integer, intent(IN) :: level      ! levels of halo cells required
-     real(mytype), dimension(:, :, :), intent(IN) :: in
-     real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
-     TYPE(DECOMP_INFO), intent(in) :: decomp
-     logical, optional :: opt_global
-     integer, intent(in), optional :: opt_pencil
+    implicit none
 
-     logical :: global
+    integer, intent(IN) :: level      ! levels of halo cells required
+    real(mytype), dimension(:,:,:), intent(IN) :: in    
+    real(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
+    TYPE(DECOMP_INFO), intent(in) :: decomp
+    logical, optional :: opt_global
+    integer, intent(in), optional :: opt_pencil
 
-     ! starting/ending index of array with halo cells
-     integer :: xs, ys, zs, xe, ye, ze
+    logical :: global
 
-     integer :: i, j, k, s1, s2, s3, ierror
-     integer :: data_type
+    ! starting/ending index of array with halo cells
+    integer :: xs, ys, zs, xe, ye, ze
 
-     integer :: icount, ilength, ijump
-     integer :: halo12, halo21, halo31, halo32
-     integer, dimension(4) :: requests
-     integer, dimension(MPI_STATUS_SIZE, 4) :: status
-     integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
+    integer :: i, j, k, s1, s2, s3, ierror
+    integer :: data_type
 
-     integer :: ipencil
-     logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
+    integer :: icount, ilength, ijump 
+    integer :: halo12, halo21, halo31, halo32                 
+    integer, dimension(4) :: requests
+    integer, dimension(MPI_STATUS_SIZE,4) :: status
+    integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
 
-     data_type = real_type
+    integer :: ipencil
+    logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
+    
+    data_type = real_type
 
 #include "halo_common.f90"
 
-     return
+    return
   end subroutine update_halo_real
 
   subroutine update_halo_complex_short(in, out, level, opt_global, opt_pencil)
 
-     implicit none
+    implicit none
 
-     integer, intent(IN) :: level      ! levels of halo cells required
-     complex(mytype), dimension(:, :, :), intent(IN) :: in
-     complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
-     logical, optional :: opt_global
-     integer, intent(in), optional :: opt_pencil
+    integer, intent(IN) :: level      ! levels of halo cells required
+    complex(mytype), dimension(:,:,:), intent(IN) :: in
+    complex(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
+    logical, optional :: opt_global
+    integer, intent(in), optional :: opt_pencil
 
-     call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
+    call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
 
   end subroutine update_halo_complex_short
 
   subroutine update_halo_complex(in, out, level, decomp, opt_global, opt_pencil)
 
-     implicit none
-
-     integer, intent(IN) :: level      ! levels of halo cells required
-     complex(mytype), dimension(:, :, :), intent(IN) :: in
-     complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
-     TYPE(DECOMP_INFO), intent(in) :: decomp
-     logical, optional :: opt_global
-     integer, intent(in), optional :: opt_pencil
+    implicit none
 
-     logical :: global
+    integer, intent(IN) :: level      ! levels of halo cells required
+    complex(mytype), dimension(:,:,:), intent(IN) :: in    
+    complex(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
+    TYPE(DECOMP_INFO), intent(in) :: decomp
+    logical, optional :: opt_global
+    integer, intent(in), optional :: opt_pencil
 
-     ! starting/ending index of array with halo cells
-     integer :: xs, ys, zs, xe, ye, ze
+    logical :: global
 
-     integer :: i, j, k, s1, s2, s3, ierror
-     integer :: data_type
+    ! starting/ending index of array with halo cells
+    integer :: xs, ys, zs, xe, ye, ze
 
-     integer :: icount, ilength, ijump
-     integer :: halo12, halo21, halo31, halo32
-     integer, dimension(4) :: requests
-     integer, dimension(MPI_STATUS_SIZE, 4) :: status
-     integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
+    integer :: i, j, k, s1, s2, s3, ierror
+    integer :: data_type
 
-     integer :: ipencil
-     logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
+    integer :: icount, ilength, ijump 
+    integer :: halo12, halo21, halo31, halo32                 
+    integer, dimension(4) :: requests
+    integer, dimension(MPI_STATUS_SIZE,4) :: status
+    integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
 
-     data_type = complex_type
+    integer :: ipencil
+    logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
+    
+    data_type = complex_type
 
 #include "halo_common.f90"
 
-     return
+    return
   end subroutine update_halo_complex
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index 13fa5d43..11859eae 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,7 +9,7 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines
+! This file contain common code to be included by subroutines 
 ! 'update_halo_...' in halo.f90
 
     if (present(opt_global)) then
@@ -18,9 +18,9 @@
        global = .false.
     end if
 
-    s1 = size(in, 1)
-    s2 = size(in, 2)
-    s3 = size(in, 3)
+    s1 = size(in,1)
+    s2 = size(in,2)
+    s3 = size(in,3)
 
     if (present(opt_pencil)) then
        ipencil = opt_pencil
@@ -31,21 +31,21 @@
           if (first_call_x) then
              first_call_x = .false.
              call decomp_2d_warning(__FILE__, __LINE__, &
-                                    0, "Deprecated interface - calling halo in X without explicit pencil")
+                  0, "Deprecated interface - calling halo in X without explicit pencil")
           end if
        else if (s2 == decomp%ysz(2)) then
           ipencil = 2
           if (first_call_y) then
              first_call_y = .false.
              call decomp_2d_warning(__FILE__, __LINE__, &
-                                    0, "Deprecated interface - calling halo in Y without explicit pencil")
+                  0, "Deprecated interface - calling halo in Y without explicit pencil")
           end if
        else if (s3 == decomp%zsz(3)) then
           ipencil = 3
           if (first_call_z) then
              first_call_z = .false.
              call decomp_2d_warning(__FILE__, __LINE__, &
-                                    0, "Deprecated interface - calling halo in Z without explicit pencil")
+                  0, "Deprecated interface - calling halo in Z without explicit pencil")
           end if
        else
           ipencil = 0
@@ -56,23 +56,23 @@
     ! Calculate the starting index and ending index of output
     if (ipencil == 1) then  ! X-pencil input
        if (global) then
-          xs = decomp%xst(1)
+          xs = decomp%xst(1) 
           xe = decomp%xen(1)
           ys = decomp%xst(2) - level
           ye = decomp%xen(2) + level
           zs = decomp%xst(3) - level
           ze = decomp%xen(3) + level
        else
-          xs = 1
+          xs = 1 
           xe = s1
           ys = 1 - level
-          ye = s2 + level
+          ye = s2 + level 
           zs = 1 - level
           ze = s3 + level
        end if
     else if (ipencil == 2) then  ! Y-pencil input
        if (global) then
-          xs = decomp%yst(1) - level
+          xs = decomp%yst(1) - level 
           xe = decomp%yen(1) + level
           ys = decomp%yst(2)
           ye = decomp%yen(2)
@@ -88,7 +88,7 @@
        end if
     else if (ipencil == 3) then  ! Z-pencil input
        if (global) then
-          xs = decomp%zst(1) - level
+          xs = decomp%zst(1) - level 
           xe = decomp%zen(1) + level
           ys = decomp%zst(2) - level
           ye = decomp%zen(2) + level
@@ -109,49 +109,49 @@
        xs = 1; xe = 1
        ys = 1; ye = 1
        zs = 1; ze = 1
-
+       
        call decomp_2d_abort(__FILE__, __LINE__, 10, &
-                            'Invalid data passed to update_halo')
+            'Invalid data passed to update_halo')
     end if
-    allocate (out(xs:xe, ys:ye, zs:ze))
+    allocate(out(xs:xe, ys:ye, zs:ze))
 !    out = -1.0_mytype ! fill the halo for debugging
 
     ! copy input data to output
     if (global) then
        ! using global coordinate
-       ! note the input array passed in always has index starting from 1
+       ! note the input array passed in always has index starting from 1 
        ! need to work out the corresponding global index
        if (ipencil == 1) then
-          do k = decomp%xst(3), decomp%xen(3)
-             do j = decomp%xst(2), decomp%xen(2)
-                do i = 1, s1  ! x all local
-                   out(i, j, k) = in(i, j - decomp%xst(2) + 1, k - decomp%xst(3) + 1)
+          do k=decomp%xst(3),decomp%xen(3)
+             do j=decomp%xst(2),decomp%xen(2)
+                do i=1,s1  ! x all local
+                   out(i,j,k) = in(i,j-decomp%xst(2)+1,k-decomp%xst(3)+1)
                 end do
              end do
           end do
        else if (ipencil == 2) then
-          do k = decomp%yst(3), decomp%yen(3)
-             do j = 1, s2  ! y all local
-                do i = decomp%yst(1), decomp%yen(1)
-                   out(i, j, k) = in(i - decomp%yst(1) + 1, j, k - decomp%yst(3) + 1)
+          do k=decomp%yst(3),decomp%yen(3)
+             do j=1,s2  ! y all local
+                do i=decomp%yst(1),decomp%yen(1)
+                   out(i,j,k) = in(i-decomp%yst(1)+1,j,k-decomp%yst(3)+1)
                 end do
              end do
           end do
        else if (ipencil == 3) then
-          do k = 1, s3  ! z all local
-             do j = decomp%zst(2), decomp%zen(2)
-                do i = decomp%zst(1), decomp%zen(1)
-                   out(i, j, k) = in(i - decomp%zst(1) + 1, j - decomp%zst(2) + 1, k)
+          do k=1,s3  ! z all local
+             do j=decomp%zst(2),decomp%zen(2)
+                do i=decomp%zst(1),decomp%zen(1)
+                   out(i,j,k) = in(i-decomp%zst(1)+1,j-decomp%zst(2)+1,k)
                 end do
              end do
           end do
        end if
     else
        ! not using global coordinate
-       do k = 1, s3
-          do j = 1, s2
-             do i = 1, s1
-                out(i, j, k) = in(i, j, k)
+       do k=1,s3
+          do j=1,s2
+             do i=1,s1
+                out(i,j,k) = in(i,j,k)
              end do
           end do
        end do
@@ -159,19 +159,19 @@
 
     ! If needed, define MPI derived data type to pack halo data,
     ! then call MPI send/receive to exchange halo data
-
+    
     !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     ! X-pencil
     if (ipencil == 1) then
 
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
-          write (*, *) 'X-pencil input'
-          write (*, *) '=============='
-          write (*, *) 'Data on a y-z plane is shown'
-          write (*, *) 'Before halo exchange'
-          do j = ye, ys, -1
-             write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
+       if (nrank==4) then
+          write(*,*) 'X-pencil input'
+          write(*,*) '=============='
+          write(*,*) 'Data on a y-z plane is shown'
+          write(*,*) 'Before halo exchange'
+          do j=ye,ys,-1
+             write(*,'(10F4.0)') (out(1,j,k),k=zs,ze)
           end do
        end if
 #endif
@@ -179,50 +179,50 @@
        ! *** east/west ***
        ! all data in local memory already, no halo exchange
 
-       ! *** north/south ***
+       ! *** north/south *** 
        tag_s = coord(1)
-       if (coord(1) == dims(1) - 1 .AND. periodic_y) then
+       if (coord(1)==dims(1)-1 .AND. periodic_y) then
           tag_n = 0
        else
           tag_n = coord(1) + 1
        end if
        icount = s3 + 2*level
-       ilength = level*s1
-       ijump = s1*(s2 + 2*level)
+       ilength = level * s1
+       ijump = s1*(s2+2*level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-                            data_type, halo12, ierror)
+            data_type, halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from south
-       call MPI_IRECV(out(xs, ys, zs), 1, halo12, &
-                      neighbour(1, 4), tag_s, DECOMP_2D_COMM_CART_X, &
-                      requests(1), ierror)
+       call MPI_IRECV(out(xs,ys,zs), 1, halo12, &
+            neighbour(1,4), tag_s, DECOMP_2D_COMM_CART_X, &
+            requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from north
-       call MPI_IRECV(out(xs, ye - level + 1, zs), 1, halo12, &
-                      neighbour(1, 3), tag_n, DECOMP_2D_COMM_CART_X, &
-                      requests(2), ierror)
+       call MPI_IRECV(out(xs,ye-level+1,zs), 1, halo12, &
+            neighbour(1,3), tag_n, DECOMP_2D_COMM_CART_X, &
+            requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to south
-       call MPI_ISSEND(out(xs, ys + level, zs), 1, halo12, &
-                       neighbour(1, 4), tag_s, DECOMP_2D_COMM_CART_X, &
-                       requests(3), ierror)
+       call MPI_ISSEND(out(xs,ys+level,zs), 1, halo12, &
+            neighbour(1,4), tag_s, DECOMP_2D_COMM_CART_X, &
+            requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to north
-       call MPI_ISSEND(out(xs, ye - level - level + 1, zs), 1, halo12, &
-                       neighbour(1, 3), tag_n, DECOMP_2D_COMM_CART_X, &
-                       requests(4), ierror)
+       call MPI_ISSEND(out(xs,ye-level-level+1,zs), 1, halo12, &
+            neighbour(1,3), tag_n, DECOMP_2D_COMM_CART_X, &
+            requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
        call MPI_TYPE_FREE(halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
-          write (*, *) 'After exchange in Y'
-          do j = ye, ys, -1
-             write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
+       if (nrank==4) then
+          write(*,*) 'After exchange in Y'
+          do j=ye,ys,-1
+             write(*,'(10F4.0)') (out(1,j,k),k=zs,ze)
           end do
        end if
 #endif
@@ -232,103 +232,103 @@
        ! all contiguous in memory, which can be sent/received using
        ! MPI directly
        tag_b = coord(2)
-       if (coord(2) == dims(2) - 1 .AND. periodic_z) then
+       if (coord(2)==dims(2)-1 .AND. periodic_z) then
           tag_t = 0
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s1*(s2 + 2*level))*level
+       icount = (s1 * (s2+2*level)) * level
        ! receive from bottom
-       call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
-                      neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
-                      requests(1), ierror)
+       call MPI_IRECV(out(xs,ys,zs), icount, data_type, &
+            neighbour(1,6), tag_b, DECOMP_2D_COMM_CART_X, &
+            requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from top
-       call MPI_IRECV(out(xs, ys, ze - level + 1), icount, data_type, &
-                      neighbour(1, 5), tag_t, DECOMP_2D_COMM_CART_X, &
-                      requests(2), ierror)
+       call MPI_IRECV(out(xs,ys,ze-level+1), icount, data_type, &
+            neighbour(1,5), tag_t, DECOMP_2D_COMM_CART_X, &
+            requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to bottom
-       call MPI_ISSEND(out(xs, ys, zs + level), icount, data_type, &
-                       neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
-                       requests(3), ierror)
+       call MPI_ISSEND(out(xs,ys,zs+level), icount, data_type, &
+            neighbour(1,6), tag_b, DECOMP_2D_COMM_CART_X, &
+            requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to top
-       call MPI_ISSEND(out(xs, ys, ze - level - level + 1), icount, data_type, &
-                       neighbour(1, 5), tag_t, DECOMP_2D_COMM_CART_X, &
-                       requests(4), ierror)
+       call MPI_ISSEND(out(xs,ys,ze-level-level+1), icount, data_type, &
+            neighbour(1,5), tag_t, DECOMP_2D_COMM_CART_X, &
+            requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
-#ifdef HALO_DEBUG
-       if (nrank == 4) then
-          write (*, *) 'After exchange in Z'
-          do j = ye, ys, -1
-             write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
+#ifdef HALO_DEBUG       
+       if (nrank==4) then
+          write(*,*) 'After exchange in Z'
+          do j=ye,ys,-1
+             write(*,'(10F4.0)') (out(1,j,k),k=zs,ze)
           end do
        end if
-#endif
+#endif       
 
-    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-       ! Y-pencil
+    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   
+    ! Y-pencil   
     else if (ipencil == 2) then
 
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
-          write (*, *) 'Y-pencil input'
-          write (*, *) '=============='
-          write (*, *) 'Data on a x-z plane is shown'
-          write (*, *) 'Before halo exchange'
-          do i = xe, xs, -1
-             write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
+       if (nrank==4) then
+          write(*,*) 'Y-pencil input'
+          write(*,*) '=============='
+          write(*,*) 'Data on a x-z plane is shown'
+          write(*,*) 'Before halo exchange'
+          do i=xe,xs,-1
+             write(*,'(10F4.0)') (out(i,1,k),k=zs,ze)
           end do
        end if
 #endif
 
        ! *** east/west ***
        tag_w = coord(1)
-       if (coord(1) == dims(1) - 1 .AND. periodic_x) then
+       if (coord(1)==dims(1)-1 .AND. periodic_x) then
           tag_e = 0
        else
           tag_e = coord(1) + 1
        end if
-       icount = s2*(s3 + 2*level)
+       icount = s2*(s3+2*level)
        ilength = level
-       ijump = s1 + 2*level
+       ijump = s1+2*level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-                            data_type, halo21, ierror)
+            data_type, halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from west
-       call MPI_IRECV(out(xs, ys, zs), 1, halo21, &
-                      neighbour(2, 2), tag_w, DECOMP_2D_COMM_CART_Y, &
-                      requests(1), ierror)
+       call MPI_IRECV(out(xs,ys,zs), 1, halo21, &
+            neighbour(2,2), tag_w, DECOMP_2D_COMM_CART_Y, &
+            requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from east
-       call MPI_IRECV(out(xe - level + 1, ys, zs), 1, halo21, &
-                      neighbour(2, 1), tag_e, DECOMP_2D_COMM_CART_Y, &
-                      requests(2), ierror)
+       call MPI_IRECV(out(xe-level+1,ys,zs), 1, halo21, &
+            neighbour(2,1), tag_e, DECOMP_2D_COMM_CART_Y, &
+            requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to west
-       call MPI_ISSEND(out(xs + level, ys, zs), 1, halo21, &
-                       neighbour(2, 2), tag_w, DECOMP_2D_COMM_CART_Y, &
-                       requests(3), ierror)
+       call MPI_ISSEND(out(xs+level,ys,zs), 1, halo21, &
+            neighbour(2,2), tag_w, DECOMP_2D_COMM_CART_Y, &
+            requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to east
-       call MPI_ISSEND(out(xe - level - level + 1, ys, zs), 1, halo21, &
-                       neighbour(2, 1), tag_e, DECOMP_2D_COMM_CART_Y, &
-                       requests(4), ierror)
+       call MPI_ISSEND(out(xe-level-level+1,ys,zs), 1, halo21, &
+            neighbour(2,1), tag_e, DECOMP_2D_COMM_CART_Y, &
+            requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
        call MPI_TYPE_FREE(halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-#ifdef HALO_DEBUG
-       if (nrank == 4) then
-          write (*, *) 'After exchange in X'
-          do i = xe, xs, -1
-             write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
+#ifdef HALO_DEBUG       
+       if (nrank==4) then
+          write(*,*) 'After exchange in X'
+          do i=xe,xs,-1
+             write(*,'(10F4.0)') (out(i,1,k),k=zs,ze)
           end do
        end if
 #endif
@@ -341,150 +341,150 @@
        ! all contiguous in memory, which can be sent/received using
        ! MPI directly
        tag_b = coord(2)
-       if (coord(2) == dims(2) - 1 .AND. periodic_z) then
+       if (coord(2)==dims(2)-1 .AND. periodic_z) then
           tag_t = 0
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s2*(s1 + 2*level))*level
+       icount = (s2 * (s1+2*level)) * level
        ! receive from bottom
-       call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
-                      neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
-                      requests(1), ierror)
+       call MPI_IRECV(out(xs,ys,zs), icount, data_type, &
+            neighbour(2,6), tag_b, DECOMP_2D_COMM_CART_Y, &
+            requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from top
-       call MPI_IRECV(out(xs, ys, ze - level + 1), icount, data_type, &
-                      neighbour(2, 5), tag_t, DECOMP_2D_COMM_CART_Y, &
-                      requests(2), ierror)
+       call MPI_IRECV(out(xs,ys,ze-level+1), icount, data_type, &
+            neighbour(2,5), tag_t, DECOMP_2D_COMM_CART_Y, &
+            requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to bottom
-       call MPI_ISSEND(out(xs, ys, zs + level), icount, data_type, &
-                       neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
-                       requests(3), ierror)
+       call MPI_ISSEND(out(xs,ys,zs+level), icount, data_type, &
+            neighbour(2,6), tag_b, DECOMP_2D_COMM_CART_Y, &
+            requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to top
-       call MPI_ISSEND(out(xs, ys, ze - level - level + 1), icount, data_type, &
-                       neighbour(2, 5), tag_t, DECOMP_2D_COMM_CART_Y, &
-                       requests(4), ierror)
+       call MPI_ISSEND(out(xs,ys,ze-level-level+1), icount, data_type, &
+            neighbour(2,5), tag_t, DECOMP_2D_COMM_CART_Y, &
+            requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
-          write (*, *) 'After exchange in Z'
-          do i = xe, xs, -1
-             write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
+       if (nrank==4) then
+          write(*,*) 'After exchange in Z'
+          do i=xe,xs,-1
+             write(*,'(10F4.0)') (out(i,1,k),k=zs,ze)
           end do
        end if
 #endif
 
-    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-       ! Z-pencil
-    else if (ipencil == 3) then
+    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   
+    ! Z-pencil
+    else if (ipencil == 3) then   
 
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
-          write (*, *) 'Z-pencil input'
-          write (*, *) '=============='
-          write (*, *) 'Data on a x-y plane is shown'
-          write (*, *) 'Before halo exchange'
-          do i = xe, xs, -1
-             write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
+       if (nrank==4) then
+          write(*,*) 'Z-pencil input'
+          write(*,*) '=============='
+          write(*,*) 'Data on a x-y plane is shown'
+          write(*,*) 'Before halo exchange'
+          do i=xe,xs,-1
+             write(*,'(10F4.0)') (out(i,j,1),j=ys,ye)
           end do
        end if
 #endif
        ! *** east/west ***
        tag_w = coord(1)
-       if (coord(1) == dims(1) - 1 .AND. periodic_x) then
+       if (coord(1)==dims(1)-1 .AND. periodic_x) then
           tag_e = 0
        else
           tag_e = coord(1) + 1
        end if
-       icount = (s2 + 2*level)*s3
+       icount = (s2+2*level)*s3
        ilength = level
-       ijump = s1 + 2*level
+       ijump = s1+2*level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-                            data_type, halo31, ierror)
+            data_type, halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from west
-       call MPI_IRECV(out(xs, ys, zs), 1, halo31, &
-                      neighbour(3, 2), tag_w, DECOMP_2D_COMM_CART_Z, &
-                      requests(1), ierror)
+       call MPI_IRECV(out(xs,ys,zs), 1, halo31, &
+            neighbour(3,2), tag_w, DECOMP_2D_COMM_CART_Z, &
+            requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from east
-       call MPI_IRECV(out(xe - level + 1, ys, zs), 1, halo31, &
-                      neighbour(3, 1), tag_e, DECOMP_2D_COMM_CART_Z, &
-                      requests(2), ierror)
+       call MPI_IRECV(out(xe-level+1,ys,zs), 1, halo31, &
+            neighbour(3,1), tag_e, DECOMP_2D_COMM_CART_Z, &
+            requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to west
-       call MPI_ISSEND(out(xs + level, ys, zs), 1, halo31, &
-                       neighbour(3, 2), tag_w, DECOMP_2D_COMM_CART_Z, &
-                       requests(3), ierror)
+       call MPI_ISSEND(out(xs+level,ys,zs), 1, halo31, &
+            neighbour(3,2), tag_w, DECOMP_2D_COMM_CART_Z, &
+            requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to east
-       call MPI_ISSEND(out(xe - level - level + 1, ys, zs), 1, halo31, &
-                       neighbour(3, 1), tag_e, DECOMP_2D_COMM_CART_Z, &
-                       requests(4), ierror)
+       call MPI_ISSEND(out(xe-level-level+1,ys,zs), 1, halo31, &
+            neighbour(3,1), tag_e, DECOMP_2D_COMM_CART_Z, &
+            requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
-       call MPI_TYPE_FREE(halo31, ierror)
+       call MPI_TYPE_FREE(halo31, ierror)       
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
-          write (*, *) 'After exchange in X'
-          do i = xe, xs, -1
-             write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
+       if (nrank==4) then
+          write(*,*) 'After exchange in X'
+          do i=xe,xs,-1
+             write(*,'(10F4.0)') (out(i,j,1),j=ys,ye)
           end do
        end if
 #endif
 
-       ! *** north/south ***
+       ! *** north/south *** 
        tag_s = coord(2)
-       if (coord(2) == dims(2) - 1 .AND. periodic_y) then
+       if (coord(2)==dims(2)-1 .AND. periodic_y) then
           tag_n = 0
        else
           tag_n = coord(2) + 1
        end if
        icount = s3
-       ilength = level*(s1 + 2*level)
-       ijump = (s1 + 2*level)*(s2 + 2*level)
+       ilength = level * (s1+2*level)
+       ijump = (s1+2*level) * (s2+2*level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-                            data_type, halo32, ierror)
+            data_type, halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from south
-       call MPI_IRECV(out(xs, ys, zs), 1, halo32, &
-                      neighbour(3, 4), tag_s, DECOMP_2D_COMM_CART_Z, &
-                      requests(1), ierror)
+       call MPI_IRECV(out(xs,ys,zs), 1, halo32, &
+            neighbour(3,4), tag_s, DECOMP_2D_COMM_CART_Z, &
+            requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from north
-       call MPI_IRECV(out(xs, ye - level + 1, zs), 1, halo32, &
-                      neighbour(3, 3), tag_n, DECOMP_2D_COMM_CART_Z, &
-                      requests(2), ierror)
+       call MPI_IRECV(out(xs,ye-level+1,zs), 1, halo32, &
+            neighbour(3,3), tag_n, DECOMP_2D_COMM_CART_Z, &
+            requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to south
-       call MPI_ISSEND(out(xs, ys + level, zs), 1, halo32, &
-                       neighbour(3, 4), tag_s, DECOMP_2D_COMM_CART_Z, &
-                       requests(3), ierror)
+       call MPI_ISSEND(out(xs,ys+level,zs), 1, halo32, &
+            neighbour(3,4), tag_s, DECOMP_2D_COMM_CART_Z, &
+            requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to north
-       call MPI_ISSEND(out(xs, ye - level - level + 1, zs), 1, halo32, &
-                       neighbour(3, 3), tag_n, DECOMP_2D_COMM_CART_Z, &
-                       requests(4), ierror)
+       call MPI_ISSEND(out(xs,ye-level-level+1,zs), 1, halo32, &
+            neighbour(3,3), tag_n, DECOMP_2D_COMM_CART_Z, &
+            requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
        call MPI_TYPE_FREE(halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
-          write (*, *) 'After exchange in Y'
-          do i = xe, xs, -1
-             write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
+       if (nrank==4) then
+          write(*,*) 'After exchange in Y'
+          do i=xe,xs,-1
+             write(*,'(10F4.0)') (out(i,j,1),j=ys,ye)
           end do
        end if
 #endif
diff --git a/src/io.f90 b/src/io.f90
index f547a840..2869145d 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -15,2028 +15,2047 @@
 
 module decomp_2d_io
 
-   use decomp_2d
-   use MPI
+  use decomp_2d
+  use MPI
 #ifdef T3PIO
-   use t3pio
+  use t3pio
 #endif
 
 #ifdef ADIOS2
-   use adios2
+  use adios2
 #endif
 
-   implicit none
+  implicit none
 
-   integer, parameter, public :: decomp_2d_write_mode = 1, decomp_2d_read_mode = 2, &
-                                 decomp_2d_append_mode = 3
-   integer, parameter :: MAX_IOH = 10 ! How many live IO things should we handle?
-   character(len=*), parameter :: io_sep = "::"
-   integer, save :: nreg_io = 0
+  integer, parameter, public :: decomp_2d_write_mode = 1, decomp_2d_read_mode = 2, &
+       decomp_2d_append_mode = 3
+  integer, parameter :: MAX_IOH = 10 ! How many live IO things should we handle?
+  character(len=*), parameter :: io_sep = "::"
+  integer, save :: nreg_io = 0
 #ifndef ADIOS2
-   integer, dimension(MAX_IOH), save :: fh_registry
-   logical, dimension(MAX_IOH), target, save :: fh_live
-   character(len=1024), dimension(MAX_IOH), target, save :: fh_names
-   integer(kind=MPI_OFFSET_KIND), dimension(MAX_IOH), save :: fh_disp
+  integer, dimension(MAX_IOH), save :: fh_registry
+  logical, dimension(MAX_IOH), target, save :: fh_live
+  character(len=1024), dimension(MAX_IOH), target, save :: fh_names
+  integer(kind=MPI_OFFSET_KIND), dimension(MAX_IOH), save :: fh_disp
 #else
-   type(adios2_adios) :: adios
-   character(len=1024), dimension(MAX_IOH), target, save :: engine_names
-   logical, dimension(MAX_IOH), target, save :: engine_live
-   type(adios2_engine), dimension(MAX_IOH), save :: engine_registry
-#endif
-
-   private        ! Make everything private unless declared public
-
-   public :: decomp_2d_write_one, decomp_2d_read_one, &
-             decomp_2d_write_var, decomp_2d_read_var, &
-             decomp_2d_write_scalar, decomp_2d_read_scalar, &
-             decomp_2d_write_plane, decomp_2d_write_every, &
-             decomp_2d_write_subdomain, &
-             decomp_2d_write_outflow, decomp_2d_read_inflow, &
-             decomp_2d_io_init, decomp_2d_io_finalise, & ! XXX: initialise/finalise 2decomp&fft IO module
-             decomp_2d_init_io, & ! XXX: initialise an io process - awful naming
-             decomp_2d_register_variable, &
-             decomp_2d_open_io, decomp_2d_close_io, &
-             decomp_2d_start_io, decomp_2d_end_io, &
-             gen_iodir_name
-
-   ! Generic interface to handle multiple data types
-
-   interface decomp_2d_write_one
-      module procedure write_one_real
-      module procedure write_one_complex
-      module procedure mpiio_write_real_coarse
-      module procedure mpiio_write_real_probe
-   end interface decomp_2d_write_one
-
-   interface decomp_2d_read_one
-      module procedure read_one_real
-      module procedure read_one_complex
-   end interface decomp_2d_read_one
-
-   interface decomp_2d_write_var
-      module procedure write_var_real
-      module procedure write_var_complex
-   end interface decomp_2d_write_var
-
-   interface decomp_2d_read_var
-      module procedure read_var_real
-      module procedure read_var_complex
-   end interface decomp_2d_read_var
-
-   interface decomp_2d_write_scalar
-      module procedure write_scalar_real
-      module procedure write_scalar_complex
-      module procedure write_scalar_integer
-      module procedure write_scalar_logical
-   end interface decomp_2d_write_scalar
-
-   interface decomp_2d_read_scalar
-      module procedure read_scalar_real
-      module procedure read_scalar_complex
-      module procedure read_scalar_integer
-      module procedure read_scalar_logical
-   end interface decomp_2d_read_scalar
-
-   interface decomp_2d_write_plane
-      module procedure write_plane_3d_real
-      module procedure write_plane_3d_complex
-      !     module procedure write_plane_2d
-   end interface decomp_2d_write_plane
-
-   interface decomp_2d_write_every
-      module procedure write_every_real
-      module procedure write_every_complex
-   end interface decomp_2d_write_every
-
-   interface decomp_2d_write_subdomain
-      module procedure write_subdomain
-   end interface decomp_2d_write_subdomain
-
-   interface decomp_2d_write_outflow
-      module procedure write_outflow
-   end interface decomp_2d_write_outflow
-
-   interface decomp_2d_read_inflow
-      module procedure read_inflow
-   end interface decomp_2d_read_inflow
+  type(adios2_adios) :: adios
+  character(len=1024), dimension(MAX_IOH), target, save :: engine_names
+  logical, dimension(MAX_IOH), target, save :: engine_live
+  type(adios2_engine), dimension(MAX_IOH), save :: engine_registry
+#endif
+  
+  private        ! Make everything private unless declared public
+
+  public :: decomp_2d_write_one, decomp_2d_read_one, &
+       decomp_2d_write_var, decomp_2d_read_var, &
+       decomp_2d_write_scalar, decomp_2d_read_scalar, &
+       decomp_2d_write_plane, decomp_2d_write_every, &
+       decomp_2d_write_subdomain, &
+       decomp_2d_write_outflow, decomp_2d_read_inflow, &
+       decomp_2d_io_init, decomp_2d_io_finalise, & ! XXX: initialise/finalise 2decomp&fft IO module
+       decomp_2d_init_io, & ! XXX: initialise an io process - awful naming
+       decomp_2d_register_variable, &
+       decomp_2d_open_io, decomp_2d_close_io, &
+       decomp_2d_start_io, decomp_2d_end_io, &
+       gen_iodir_name
+
+  ! Generic interface to handle multiple data types
+
+  interface decomp_2d_write_one
+     module procedure write_one_real
+     module procedure write_one_complex
+     module procedure mpiio_write_real_coarse
+     module procedure mpiio_write_real_probe
+  end interface decomp_2d_write_one
+
+  interface decomp_2d_read_one
+     module procedure read_one_real
+     module procedure read_one_complex
+  end interface decomp_2d_read_one
+
+  interface decomp_2d_write_var
+     module procedure write_var_real
+     module procedure write_var_complex
+  end interface decomp_2d_write_var
+
+  interface decomp_2d_read_var
+     module procedure read_var_real
+     module procedure read_var_complex
+  end interface decomp_2d_read_var
+
+  interface decomp_2d_write_scalar
+     module procedure write_scalar_real
+     module procedure write_scalar_complex
+     module procedure write_scalar_integer
+     module procedure write_scalar_logical
+  end interface decomp_2d_write_scalar
+
+  interface decomp_2d_read_scalar
+     module procedure read_scalar_real
+     module procedure read_scalar_complex
+     module procedure read_scalar_integer
+     module procedure read_scalar_logical
+  end interface decomp_2d_read_scalar
+
+  interface decomp_2d_write_plane
+     module procedure write_plane_3d_real
+     module procedure write_plane_3d_complex
+     !     module procedure write_plane_2d
+  end interface decomp_2d_write_plane
+
+  interface decomp_2d_write_every
+     module procedure write_every_real
+     module procedure write_every_complex
+  end interface decomp_2d_write_every
+
+  interface decomp_2d_write_subdomain
+     module procedure write_subdomain
+  end interface decomp_2d_write_subdomain
+
+  interface decomp_2d_write_outflow
+     module procedure write_outflow
+  end interface decomp_2d_write_outflow
+
+  interface decomp_2d_read_inflow
+     module procedure read_inflow
+  end interface decomp_2d_read_inflow
 
 contains
 
-   subroutine decomp_2d_io_init()
+  subroutine decomp_2d_io_init()
 
 #ifdef ADIOS2
-      integer :: ierror
-      logical :: adios2_debug_mode
-      character(len=80) :: config_file = "adios2_config.xml"
+  integer :: ierror
+  logical :: adios2_debug_mode
+  character(len=80) :: config_file="adios2_config.xml"
 #endif
 
 #ifdef ADIOS2
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_init")
+  if (decomp_profiler_io) call decomp_profiler_start("io_init")
 #endif
 
   !! TODO: make this a runtime-option
-      adios2_debug_mode = .true.
+  adios2_debug_mode = .true.
 
-      call adios2_init(adios, trim(config_file), decomp_2d_comm, adios2_debug_mode, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-                                              "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
+  call adios2_init(adios, trim(config_file), decomp_2d_comm, adios2_debug_mode, ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+     "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
 
-      engine_live(:) = .false.
+  engine_live(:) = .false.
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_init")
+  if (decomp_profiler_io) call decomp_profiler_end("io_init")
 #endif
 #endif
-
-   end subroutine decomp_2d_io_init
-   subroutine decomp_2d_io_finalise()
+  
+  end subroutine decomp_2d_io_init
+  subroutine decomp_2d_io_finalise()
 
 #ifdef ADIOS2
-      use adios2
+    use adios2
 #endif
 
-      implicit none
+    implicit none
 
 #ifdef ADIOS2
-      integer :: ierror
+    integer :: ierror
 #endif
-
+    
 #ifdef ADIOS2
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_fin")
+    if (decomp_profiler_io) call decomp_profiler_start("io_fin")
 #endif
-      call adios2_finalize(adios, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-                                              "adios2_finalize")
+    call adios2_finalize(adios, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+       "adios2_finalize")
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_fin")
+    if (decomp_profiler_io) call decomp_profiler_end("io_fin")
 #endif
 #endif
 
-   end subroutine decomp_2d_io_finalise
-
+  end subroutine decomp_2d_io_finalise
+  
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Using MPI-IO library to write a single 3D array to a file
+  ! Using MPI-IO library to write a single 3D array to a file
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine write_one_real(ipencil, var, filename, opt_decomp)
+  subroutine write_one_real(ipencil,var,filename,opt_decomp)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: ipencil
-      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      character(len=*), intent(IN) :: filename
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    integer, intent(IN) :: ipencil
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    character(len=*), intent(IN) :: filename
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-      TYPE(DECOMP_INFO) :: decomp
-      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: ierror, newtype, fh, data_type, info, gs
+    TYPE(DECOMP_INFO) :: decomp
+    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: ierror, newtype, fh, data_type, info, gs
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_write_one_real")
+  if (decomp_profiler_io) call decomp_profiler_start("io_write_one_real")
 #endif
 
-      data_type = real_type
+    data_type = real_type
 
 #include "io_write_one.inc"
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_write_one_real")
+  if (decomp_profiler_io) call decomp_profiler_end("io_write_one_real")
 #endif
 
-      return
-   end subroutine write_one_real
+    return
+  end subroutine write_one_real
 
-   subroutine write_one_complex(ipencil, var, filename, opt_decomp)
+  subroutine write_one_complex(ipencil,var,filename,opt_decomp)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: ipencil
-      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      character(len=*), intent(IN) :: filename
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    integer, intent(IN) :: ipencil
+    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    character(len=*), intent(IN) :: filename
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-      TYPE(DECOMP_INFO) :: decomp
-      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: ierror, newtype, fh, data_type, info, gs
+    TYPE(DECOMP_INFO) :: decomp
+    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: ierror, newtype, fh, data_type, info, gs
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_write_one_cplx")
+  if (decomp_profiler_io) call decomp_profiler_start("io_write_one_cplx")
 #endif
 
-      data_type = complex_type
+    data_type = complex_type
 
 #include "io_write_one.inc"
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_write_one_cplx")
+  if (decomp_profiler_io) call decomp_profiler_end("io_write_one_cplx")
 #endif
 
-      return
-   end subroutine write_one_complex
+    return
+  end subroutine write_one_complex
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Using MPI-IO library to read from a file a single 3D array
+  ! Using MPI-IO library to read from a file a single 3D array
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine read_one_real(ipencil, var, dirname, varname, io_name, opt_decomp, reduce_prec)
-
-      implicit none
+  subroutine read_one_real(ipencil,var,dirname,varname,io_name,opt_decomp,reduce_prec)
 
-      integer, intent(IN) :: ipencil
-      real(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
-      character(len=*), intent(IN) :: varname, dirname, io_name
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
-      logical, intent(in), optional :: reduce_prec
+    implicit none
 
-      logical :: read_reduce_prec
+    integer, intent(IN) :: ipencil
+    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
+    character(len=*), intent(IN) :: varname, dirname, io_name
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    logical, intent(in), optional :: reduce_prec
 
-      integer :: idx
+    logical :: read_reduce_prec
+    
+    integer :: idx
 #ifndef ADIOS2
-      TYPE(DECOMP_INFO) :: decomp
-      integer, dimension(3) :: sizes, subsizes, starts
-      real(mytype_single), allocatable, dimension(:, :, :) :: varsingle
-      integer :: data_type
-      logical :: dir_exists
-      integer :: disp_bytes
-      integer :: ierror, newtype
-      character(len=:), allocatable :: full_io_name
-      logical :: opened_new
+    TYPE(DECOMP_INFO) :: decomp
+    integer, dimension(3) :: sizes, subsizes, starts
+    real(mytype_single), allocatable, dimension(:,:,:) :: varsingle
+    integer :: data_type
+    logical :: dir_exists
+    integer :: disp_bytes
+    integer :: ierror, newtype
+    character(len=:), allocatable :: full_io_name
+    logical :: opened_new
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_read_one_real")
+    if (decomp_profiler_io) call decomp_profiler_start("io_read_one_real")
 #endif
-
-      read_reduce_prec = .true.
-
-      idx = get_io_idx(io_name, dirname)
+    
+    read_reduce_prec = .true.
+    
+    idx = get_io_idx(io_name, dirname)
 #ifndef ADIOS2
-      opened_new = .false.
-      if (idx .lt. 1) then
-         ! Check file exists
-         full_io_name = trim(dirname)//"/"//trim(varname)
-         if (nrank == 0) then
-            inquire (file=full_io_name, exist=dir_exists)
-            if (.not. dir_exists) then
-               print *, "ERROR: cannot read from", full_io_name, " directory doesn't exist!"
-               stop
-            end if
-         end if
-
-         call decomp_2d_open_io(io_name, full_io_name, decomp_2d_read_mode)
-         idx = get_io_idx(io_name, full_io_name)
-         opened_new = .true.
-      else
-         full_io_name = "" ! Ensure string is not unset
-      end if
-
-      if (present(reduce_prec)) then
-         if (.not. reduce_prec) then
-            read_reduce_prec = .false.
-         end if
-      end if
+    opened_new = .false.
+    if (idx .lt. 1) then
+       ! Check file exists
+       full_io_name = trim(dirname)//"/"//trim(varname)
+       if (nrank==0) then
+          inquire(file=full_io_name, exist=dir_exists)
+          if (.not.dir_exists) then
+             print *, "ERROR: cannot read from", full_io_name, " directory doesn't exist!"
+             stop
+          end if
+       end if
+       
+       call decomp_2d_open_io(io_name, full_io_name, decomp_2d_read_mode)
+       idx = get_io_idx(io_name, full_io_name)
+       opened_new = .true.
+    else
+       full_io_name = "" ! Ensure string is not unset
+    end if
+    
+    if (present(reduce_prec)) then
+       if (.not. reduce_prec) then
+          read_reduce_prec = .false.
+       end if
+    end if
+    if (read_reduce_prec) then
+       data_type = real_type_single
+    else
+       data_type = real_type
+    end if
+    call MPI_TYPE_SIZE(data_type,disp_bytes,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+    
+    if (present(opt_decomp)) then
+       decomp = opt_decomp
+    else
+       call get_decomp_info(decomp)
+    end if
+
+    ! determine subarray parameters
+    sizes(1) = decomp%xsz(1)
+    sizes(2) = decomp%ysz(2)
+    sizes(3) = decomp%zsz(3)
+
+    if (ipencil == 1) then
+       subsizes(1) = decomp%xsz(1)
+       subsizes(2) = decomp%xsz(2)
+       subsizes(3) = decomp%xsz(3)
+       starts(1) = decomp%xst(1)-1  ! 0-based index
+       starts(2) = decomp%xst(2)-1
+       starts(3) = decomp%xst(3)-1
+    else if (ipencil == 2) then
+       subsizes(1) = decomp%ysz(1)
+       subsizes(2) = decomp%ysz(2)
+       subsizes(3) = decomp%ysz(3)
+       starts(1) = decomp%yst(1)-1
+       starts(2) = decomp%yst(2)-1
+       starts(3) = decomp%yst(3)-1
+    else if (ipencil == 3) then
+       subsizes(1) = decomp%zsz(1)
+       subsizes(2) = decomp%zsz(2)
+       subsizes(3) = decomp%zsz(3)
+       starts(1) = decomp%zst(1)-1
+       starts(2) = decomp%zst(2)-1
+       starts(3) = decomp%zst(3)-1
+    else
+       call decomp_2d_abort(-1, "IO/read_one_real : Wrong value for ipencil")
+    endif
+
+    associate(fh => fh_registry(idx), &
+         disp => fh_disp(idx))
+      call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+           MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+      call MPI_TYPE_COMMIT(newtype,ierror)
+      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+      call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+           newtype,'native',MPI_INFO_NULL,ierror)
+      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
       if (read_reduce_prec) then
-         data_type = real_type_single
-      else
-         data_type = real_type
-      end if
-      call MPI_TYPE_SIZE(data_type, disp_bytes, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-
-      if (present(opt_decomp)) then
-         decomp = opt_decomp
+         allocate (varsingle(xstV(1):xenV(1),xstV(2):xenV(2),xstV(3):xenV(3)))
+         call MPI_FILE_READ_ALL(fh, varsingle, &
+              subsizes(1)*subsizes(2)*subsizes(3), &
+              data_type, MPI_STATUS_IGNORE, ierror)
+         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+         var = real(varsingle,mytype)
+         deallocate(varsingle)
       else
-         call get_decomp_info(decomp)
-      end if
-
-      ! determine subarray parameters
-      sizes(1) = decomp%xsz(1)
-      sizes(2) = decomp%ysz(2)
-      sizes(3) = decomp%zsz(3)
-
-      if (ipencil == 1) then
-         subsizes(1) = decomp%xsz(1)
-         subsizes(2) = decomp%xsz(2)
-         subsizes(3) = decomp%xsz(3)
-         starts(1) = decomp%xst(1) - 1  ! 0-based index
-         starts(2) = decomp%xst(2) - 1
-         starts(3) = decomp%xst(3) - 1
-      else if (ipencil == 2) then
-         subsizes(1) = decomp%ysz(1)
-         subsizes(2) = decomp%ysz(2)
-         subsizes(3) = decomp%ysz(3)
-         starts(1) = decomp%yst(1) - 1
-         starts(2) = decomp%yst(2) - 1
-         starts(3) = decomp%yst(3) - 1
-      else if (ipencil == 3) then
-         subsizes(1) = decomp%zsz(1)
-         subsizes(2) = decomp%zsz(2)
-         subsizes(3) = decomp%zsz(3)
-         starts(1) = decomp%zst(1) - 1
-         starts(2) = decomp%zst(2) - 1
-         starts(3) = decomp%zst(3) - 1
-      else
-         call decomp_2d_abort(-1, "IO/read_one_real : Wrong value for ipencil")
-      end if
-
-      associate (fh => fh_registry(idx), &
-                 disp => fh_disp(idx))
-         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                                       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-         call MPI_TYPE_COMMIT(newtype, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-         call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                                newtype, 'native', MPI_INFO_NULL, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-         if (read_reduce_prec) then
-            allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
-            call MPI_FILE_READ_ALL(fh, varsingle, &
-                                   subsizes(1)*subsizes(2)*subsizes(3), &
-                                   data_type, MPI_STATUS_IGNORE, ierror)
-            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-            var = real(varsingle, mytype)
-            deallocate (varsingle)
-         else
-            call MPI_FILE_READ_ALL(fh, var, &
-                                   subsizes(1)*subsizes(2)*subsizes(3), &
-                                   data_type, MPI_STATUS_IGNORE, ierror)
-            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-         end if
-         call MPI_TYPE_FREE(newtype, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-
-         disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                *int(disp_bytes, kind=MPI_OFFSET_KIND)
-      end associate
-
-      if (opened_new) then
-         call decomp_2d_close_io(io_name, full_io_name)
-         deallocate (full_io_name)
-      end if
+         call MPI_FILE_READ_ALL(fh, var, &
+              subsizes(1)*subsizes(2)*subsizes(3), &
+              data_type, MPI_STATUS_IGNORE, ierror)
+         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+      endif
+      call MPI_TYPE_FREE(newtype,ierror)
+      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+
+      disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                  * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                  * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                  * int(disp_bytes, kind=MPI_OFFSET_KIND)
+    end associate
+
+    if (opened_new) then
+       call decomp_2d_close_io(io_name, full_io_name)
+       deallocate(full_io_name)
+    end if
 #else
-      call adios2_read_one_real(var, dirname, varname, io_name)
+    call adios2_read_one_real(var, dirname, varname, io_name)
 
-      associate (pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
-      end associate
+    associate(pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
+    end associate
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_read_one_real")
+    if (decomp_profiler_io) call decomp_profiler_end("io_read_one_real")
 #endif
 
-   end subroutine read_one_real
+  end subroutine read_one_real
+
 
-   subroutine read_one_complex(ipencil, var, filename, opt_decomp)
+  subroutine read_one_complex(ipencil,var,filename,opt_decomp)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: ipencil
-      complex(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
-      character(len=*), intent(IN) :: filename
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    integer, intent(IN) :: ipencil
+    complex(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
+    character(len=*), intent(IN) :: filename
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-      TYPE(DECOMP_INFO) :: decomp
-      integer(kind=MPI_OFFSET_KIND) :: disp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: ierror, newtype, fh, data_type
+    TYPE(DECOMP_INFO) :: decomp
+    integer(kind=MPI_OFFSET_KIND) :: disp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: ierror, newtype, fh, data_type
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_read_one_cplx")
+    if (decomp_profiler_io) call decomp_profiler_start("io_read_one_cplx")
 #endif
 
-      data_type = complex_type
+    data_type = complex_type
 
 #include "io_read_one.inc"
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_read_one_cplx")
+    if (decomp_profiler_io) call decomp_profiler_end("io_read_one_cplx")
 #endif
 
-   end subroutine read_one_complex
+  end subroutine read_one_complex
 
 #ifdef ADIOS2
-   subroutine adios2_read_one_real(var, engine_name, varname, io_name)
+  subroutine adios2_read_one_real(var,engine_name,varname,io_name)
 
-      implicit none
+    implicit none
 
-      character(len=*), intent(in) :: engine_name
-      character(len=*), intent(in) :: io_name
-      character*(*), intent(in) :: varname
-      real(mytype), contiguous, dimension(:, :, :), intent(out) :: var
+    character(len=*), intent(in) :: engine_name
+    character(len=*), intent(in) :: io_name
+    character*(*), intent(in) :: varname
+    real(mytype), contiguous, dimension(:,:,:), intent(out) :: var
 
-      integer :: ierror
-      type(adios2_io) :: io_handle
-      type(adios2_variable) :: var_handle
-      integer :: idx
+    integer :: ierror
+    type(adios2_io) :: io_handle
+    type(adios2_variable) :: var_handle
+    integer :: idx
 
-      integer(kind=8) :: nsteps
-      integer(kind=8) :: curstep
+    integer(kind=8) :: nsteps
+    integer(kind=8) :: curstep
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("adios2_read_one_real")
-#endif
-
-      call adios2_at_io(io_handle, adios, io_name, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-      call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-      if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-                                                       "ERROR: trying to read variable without registering first! "//trim(varname))
-
-      call adios2_variable_steps(nsteps, var_handle, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_variable_steps")
-      print *, "AVAILABLE steps for ", nsteps
-
-      print *, "IO_NAME: ", io_name
-      print *, "ENGINE_NAME: ", engine_name
-      print *, "VAR_NAME: ", varname
-      idx = get_io_idx(io_name, engine_name)
-      print *, idx
-      call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_deferred, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
-
-      print *, "MAX: ", maxval(var)
-
-      call adios2_current_step(curstep, engine_registry(idx), ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_current_step")
-      print *, "Current step: ", curstep
+    if (decomp_profiler_io) call decomp_profiler_start("adios2_read_one_real")
+#endif
+
+    call adios2_at_io(io_handle, adios, io_name, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+    call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+    if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+       "ERROR: trying to read variable without registering first! "//trim(varname))
+
+    call adios2_variable_steps(nsteps, var_handle, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_variable_steps")
+    print *, "AVAILABLE steps for ", nsteps
+    
+    print *, "IO_NAME: ", io_name
+    print *, "ENGINE_NAME: ", engine_name
+    print *, "VAR_NAME: ", varname
+    idx = get_io_idx(io_name, engine_name)
+    print *, idx
+    call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_deferred, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
+
+    print *, "MAX: ", maxval(var)
+    
+    call adios2_current_step(curstep, engine_registry(idx), ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_current_step")
+    print *, "Current step: ", curstep
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("adios2_read_one_real")
+    if (decomp_profiler_io) call decomp_profiler_end("adios2_read_one_real")
 #endif
-
-   end subroutine adios2_read_one_real
+    
+  end subroutine adios2_read_one_real
 #endif
-
+  
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Write a 3D array as part of a big MPI-IO file, starting from
-   !  displacement 'disp'; 'disp' will be updated after the writing
-   !  operation to prepare the writing of next chunk of data.
+  ! Write a 3D array as part of a big MPI-IO file, starting from 
+  !  displacement 'disp'; 'disp' will be updated after the writing
+  !  operation to prepare the writing of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine write_var_real(fh, disp, ipencil, var, opt_decomp)
+  subroutine write_var_real(fh,disp,ipencil,var,opt_decomp)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: ipencil
-      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: ipencil
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-      TYPE(DECOMP_INFO) :: decomp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: ierror, newtype, data_type
+    TYPE(DECOMP_INFO) :: decomp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: ierror, newtype, data_type
 
-      data_type = real_type
+    data_type = real_type
 
 #include "io_write_var.inc"
 
-      return
-   end subroutine write_var_real
+    return
+  end subroutine write_var_real
 
-   subroutine write_var_complex(fh, disp, ipencil, var, opt_decomp)
+  subroutine write_var_complex(fh,disp,ipencil,var,opt_decomp)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: ipencil
-      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: ipencil
+    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-      TYPE(DECOMP_INFO) :: decomp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: ierror, newtype, data_type
+    TYPE(DECOMP_INFO) :: decomp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: ierror, newtype, data_type
 
-      data_type = complex_type
+    data_type = complex_type
 
 #include "io_write_var.inc"
 
-      return
-   end subroutine write_var_complex
+    return
+  end subroutine write_var_complex
+
 
-   subroutine write_outflow(dirname, varname, ntimesteps, var, io_name, opt_decomp)
+  subroutine write_outflow(dirname,varname,ntimesteps,var,io_name,opt_decomp)
 
-      implicit none
+    implicit none
 
-      character(len=*), intent(in) :: dirname, varname, io_name
-      integer, intent(IN) :: ntimesteps
-      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    character(len=*), intent(in) :: dirname, varname, io_name
+    integer, intent(IN) :: ntimesteps 
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-      TYPE(DECOMP_INFO) :: decomp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: ierror, data_type
-      integer :: idx
+    TYPE(DECOMP_INFO) :: decomp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: ierror, data_type
+    integer :: idx
 #ifdef ADIOS2
-      type(adios2_io) :: io_handle
-      type(adios2_variable) :: var_handle
+    type(adios2_io) :: io_handle
+    type(adios2_variable) :: var_handle
 #else
-      integer :: newtype
+    integer :: newtype
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_write_outflow")
+    if (decomp_profiler_io) call decomp_profiler_start("io_write_outflow")
 #endif
 
-      data_type = real_type
+    data_type = real_type
 
 #include "io_write_outflow.f90"
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_write_outflow")
+    if (decomp_profiler_io) call decomp_profiler_end("io_write_outflow")
 #endif
 
-   end subroutine write_outflow
+  end subroutine write_outflow
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Read a 3D array as part of a big MPI-IO file, starting from
-   !  displacement 'disp'; 'disp' will be updated after the reading
-   !  operation to prepare the reading of next chunk of data.
+  ! Read a 3D array as part of a big MPI-IO file, starting from 
+  !  displacement 'disp'; 'disp' will be updated after the reading
+  !  operation to prepare the reading of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine read_var_real(fh, disp, ipencil, var, opt_decomp)
+  subroutine read_var_real(fh,disp,ipencil,var,opt_decomp)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: ipencil
-      real(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: ipencil
+    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-      TYPE(DECOMP_INFO) :: decomp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: ierror, newtype, data_type
+    TYPE(DECOMP_INFO) :: decomp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: ierror, newtype, data_type
 
-      data_type = real_type
+    data_type = real_type
 
 #include "io_read_var.inc"
 
-      return
-   end subroutine read_var_real
+    return
+  end subroutine read_var_real
+
 
-   subroutine read_var_complex(fh, disp, ipencil, var, opt_decomp)
+  subroutine read_var_complex(fh,disp,ipencil,var,opt_decomp)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: ipencil
-      complex(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: ipencil
+    complex(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-      TYPE(DECOMP_INFO) :: decomp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: ierror, newtype, data_type
+    TYPE(DECOMP_INFO) :: decomp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: ierror, newtype, data_type
 
-      data_type = complex_type
+    data_type = complex_type
 
 #include "io_read_var.inc"
 
-      return
-   end subroutine read_var_complex
+    return
+  end subroutine read_var_complex
 
-   subroutine read_inflow(dirname, varname, ntimesteps, var, io_name, opt_decomp)
 
-      implicit none
+  subroutine read_inflow(dirname,varname,ntimesteps,var,io_name,opt_decomp)
 
-      character(len=*), intent(in) :: dirname, varname, io_name
-      integer, intent(IN) :: ntimesteps
-      real(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    implicit none
 
-      TYPE(DECOMP_INFO) :: decomp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: ierror, data_type
-      integer :: idx
+    character(len=*), intent(in) :: dirname, varname, io_name
+    integer, intent(IN) :: ntimesteps
+    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+
+    TYPE(DECOMP_INFO) :: decomp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: ierror, data_type
+    integer :: idx
 #ifdef ADIOS2
-      type(adios2_io) :: io_handle
-      type(adios2_variable) :: var_handle
+    type(adios2_io) :: io_handle
+    type(adios2_variable) :: var_handle
 #else
-      integer :: newtype
+    integer :: newtype
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_read_inflow")
+    if (decomp_profiler_io) call decomp_profiler_start("io_read_inflow")
 #endif
 
-      data_type = real_type
+    data_type = real_type
 
 #include "io_read_inflow.f90"
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_read_inflow")
+    if (decomp_profiler_io) call decomp_profiler_end("io_read_inflow")
 #endif
 
-   end subroutine read_inflow
+  end subroutine read_inflow
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Write scalar variables as part of a big MPI-IO file, starting from
-   !  displacement 'disp'; 'disp' will be updated after the reading
-   !  operation to prepare the reading of next chunk of data.
+  ! Write scalar variables as part of a big MPI-IO file, starting from 
+  !  displacement 'disp'; 'disp' will be updated after the reading
+  !  operation to prepare the reading of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine write_scalar_real(fh, disp, n, var)
+  subroutine write_scalar_real(fh,disp,n,var)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: fh             ! file handle
-      integer(KIND=MPI_OFFSET_KIND), &
+    integer, intent(IN) :: fh             ! file handle
+    integer(KIND=MPI_OFFSET_KIND), &
          intent(INOUT) :: disp            ! displacement
-      integer, intent(IN) :: n              ! number of scalars
-      real(mytype), dimension(n), &
+    integer, intent(IN) :: n              ! number of scalars
+    real(mytype), dimension(n), &
          intent(IN) :: var                ! array of scalars
 
-      integer :: m, ierror
-
-      call MPI_FILE_SET_VIEW(fh, disp, real_type, &
-                             real_type, 'native', MPI_INFO_NULL, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-      if (nrank == 0) then
-         m = n ! only one rank needs to write
-      else
-         m = 0
-      end if
-      call MPI_FILE_WRITE_ALL(fh, var, m, real_type, &
-                              MPI_STATUS_IGNORE, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
-
-      return
-   end subroutine write_scalar_real
-
-   subroutine write_scalar_complex(fh, disp, n, var)
-
-      implicit none
-
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: n
-      complex(mytype), dimension(n), intent(IN) :: var
-
-      integer :: m, ierror
-
-      call MPI_FILE_SET_VIEW(fh, disp, complex_type, &
-                             complex_type, 'native', MPI_INFO_NULL, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-      if (nrank == 0) then
-         m = n
-      else
-         m = 0
-      end if
-      call MPI_FILE_WRITE_ALL(fh, var, m, complex_type, &
-                              MPI_STATUS_IGNORE, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-             *2_MPI_OFFSET_KIND
-
-      return
-   end subroutine write_scalar_complex
-
-   subroutine write_scalar_integer(fh, disp, n, var)
-
-      implicit none
-
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: n
-      integer, dimension(n), intent(IN) :: var
-
-      integer :: m, ierror
-
-      call MPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, &
-                             MPI_INTEGER, 'native', MPI_INFO_NULL, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-      if (nrank == 0) then
-         m = n
-      else
-         m = 0
-      end if
-      call MPI_FILE_WRITE_ALL(fh, var, m, MPI_INTEGER, &
-                              MPI_STATUS_IGNORE, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-      call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
-
-      return
-   end subroutine write_scalar_integer
-
-   subroutine write_scalar_logical(fh, disp, n, var)
+    integer :: m, ierror
+
+    call MPI_FILE_SET_VIEW(fh,disp,real_type, &
+         real_type,'native',MPI_INFO_NULL,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+    if (nrank==0) then
+       m = n ! only one rank needs to write
+    else
+       m = 0
+    end if
+    call MPI_FILE_WRITE_ALL(fh, var, m, real_type, &
+         MPI_STATUS_IGNORE, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+
+    return
+  end subroutine write_scalar_real
+
+
+  subroutine write_scalar_complex(fh,disp,n,var)
+
+    implicit none
+
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: n
+    complex(mytype), dimension(n), intent(IN) :: var
+
+    integer :: m, ierror
+
+    call MPI_FILE_SET_VIEW(fh,disp,complex_type, &
+         complex_type,'native',MPI_INFO_NULL,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+    if (nrank==0) then
+       m = n
+    else
+       m = 0
+    end if
+    call MPI_FILE_WRITE_ALL(fh, var, m, complex_type, &
+         MPI_STATUS_IGNORE, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+                * 2_MPI_OFFSET_KIND
+
+    return
+  end subroutine write_scalar_complex
+
+
+  subroutine write_scalar_integer(fh,disp,n,var)
+
+    implicit none
+
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: n
+    integer, dimension(n), intent(IN) :: var
+
+    integer :: m, ierror
+
+    call MPI_FILE_SET_VIEW(fh,disp,MPI_INTEGER, &
+         MPI_INTEGER,'native',MPI_INFO_NULL,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+    if (nrank==0) then
+       m = n
+    else
+       m = 0
+    end if
+    call MPI_FILE_WRITE_ALL(fh, var, m, MPI_INTEGER, &
+         MPI_STATUS_IGNORE, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+    call MPI_TYPE_SIZE(MPI_INTEGER,m,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(m, kind=MPI_OFFSET_KIND)
+
+    return
+  end subroutine write_scalar_integer
+
+
+  subroutine write_scalar_logical(fh,disp,n,var)
+
+    implicit none
+
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: n
+    logical, dimension(n), intent(IN) :: var
+
+    integer :: m, ierror
+
+    call MPI_FILE_SET_VIEW(fh,disp,MPI_LOGICAL, &
+         MPI_LOGICAL,'native',MPI_INFO_NULL,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+    if (nrank==0) then
+       m = n
+    else
+       m = 0
+    end if
+    call MPI_FILE_WRITE_ALL(fh, var, m, MPI_LOGICAL, &
+         MPI_STATUS_IGNORE, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+    call MPI_TYPE_SIZE(MPI_LOGICAL,m,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(m, kind=MPI_OFFSET_KIND)
+
+    return
+  end subroutine write_scalar_logical
 
-      implicit none
-
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: n
-      logical, dimension(n), intent(IN) :: var
-
-      integer :: m, ierror
-
-      call MPI_FILE_SET_VIEW(fh, disp, MPI_LOGICAL, &
-                             MPI_LOGICAL, 'native', MPI_INFO_NULL, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-      if (nrank == 0) then
-         m = n
-      else
-         m = 0
-      end if
-      call MPI_FILE_WRITE_ALL(fh, var, m, MPI_LOGICAL, &
-                              MPI_STATUS_IGNORE, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-      call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
-
-      return
-   end subroutine write_scalar_logical
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Read scalar variables as part of a big MPI-IO file, starting from
-   !  displacement 'disp'; 'disp' will be updated after the reading
-   !  operation to prepare the reading of next chunk of data.
+  ! Read scalar variables as part of a big MPI-IO file, starting from 
+  !  displacement 'disp'; 'disp' will be updated after the reading
+  !  operation to prepare the reading of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine read_scalar_real(fh, disp, n, var)
+  subroutine read_scalar_real(fh,disp,n,var)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: fh             ! file handle
-      integer(KIND=MPI_OFFSET_KIND), &
+    integer, intent(IN) :: fh             ! file handle
+    integer(KIND=MPI_OFFSET_KIND), &
          intent(INOUT) :: disp            ! displacement
-      integer, intent(IN) :: n              ! number of scalars
-      real(mytype), dimension(n), &
+    integer, intent(IN) :: n              ! number of scalars
+    real(mytype), dimension(n), &
          intent(INOUT) :: var             ! array of scalars
 
-      integer :: ierror
-
-      call MPI_FILE_SET_VIEW(fh, disp, real_type, &
-                             real_type, 'native', MPI_INFO_NULL, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-      call MPI_FILE_READ_ALL(fh, var, n, real_type, &
-                             MPI_STATUS_IGNORE, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
-
-      return
-   end subroutine read_scalar_real
-
-   subroutine read_scalar_complex(fh, disp, n, var)
-
-      implicit none
-
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: n
-      complex(mytype), dimension(n), intent(INOUT) :: var
+    integer :: ierror
 
-      integer :: ierror
+    call MPI_FILE_SET_VIEW(fh,disp,real_type, &
+         real_type,'native',MPI_INFO_NULL,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+    call MPI_FILE_READ_ALL(fh, var, n, real_type, &
+         MPI_STATUS_IGNORE, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 
-      call MPI_FILE_SET_VIEW(fh, disp, complex_type, &
-                             complex_type, 'native', MPI_INFO_NULL, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-      call MPI_FILE_READ_ALL(fh, var, n, complex_type, &
-                             MPI_STATUS_IGNORE, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-             *2_MPI_OFFSET_KIND
+    return
+  end subroutine read_scalar_real
 
-      return
-   end subroutine read_scalar_complex
 
-   subroutine read_scalar_integer(fh, disp, n, var)
+  subroutine read_scalar_complex(fh,disp,n,var)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: n
-      integer, dimension(n), intent(INOUT) :: var
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: n
+    complex(mytype), dimension(n), intent(INOUT) :: var
 
-      integer :: m, ierror
+    integer :: ierror
 
-      call MPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, &
-                             MPI_INTEGER, 'native', MPI_INFO_NULL, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-      call MPI_FILE_READ_ALL(fh, var, n, MPI_INTEGER, &
-                             MPI_STATUS_IGNORE, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-      call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+    call MPI_FILE_SET_VIEW(fh,disp,complex_type, &
+         complex_type,'native',MPI_INFO_NULL,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+    call MPI_FILE_READ_ALL(fh, var, n, complex_type, &
+         MPI_STATUS_IGNORE, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+                * 2_MPI_OFFSET_KIND
 
-      return
-   end subroutine read_scalar_integer
+    return
+  end subroutine read_scalar_complex
 
-   subroutine read_scalar_logical(fh, disp, n, var)
 
-      implicit none
+  subroutine read_scalar_integer(fh,disp,n,var)
 
-      integer, intent(IN) :: fh
-      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-      integer, intent(IN) :: n
-      logical, dimension(n), intent(INOUT) :: var
+    implicit none
 
-      integer :: m, ierror
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: n
+    integer, dimension(n), intent(INOUT) :: var
 
-      call MPI_FILE_SET_VIEW(fh, disp, MPI_LOGICAL, &
-                             MPI_LOGICAL, 'native', MPI_INFO_NULL, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-      call MPI_FILE_READ_ALL(fh, var, n, MPI_LOGICAL, &
-                             MPI_STATUS_IGNORE, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-      call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+    integer :: m, ierror
 
-      return
-   end subroutine read_scalar_logical
+    call MPI_FILE_SET_VIEW(fh,disp,MPI_INTEGER, &
+         MPI_INTEGER,'native',MPI_INFO_NULL,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+    call MPI_FILE_READ_ALL(fh, var, n, MPI_INTEGER, &
+         MPI_STATUS_IGNORE, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+    call MPI_TYPE_SIZE(MPI_INTEGER,m,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(m, kind=MPI_OFFSET_KIND)
 
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Write a 2D slice of the 3D data to a file
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine plane_extents(sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes)
-
-      integer, intent(in) :: iplane
-      type(decomp_info), intent(in), optional :: opt_decomp
-      integer, intent(in), optional :: opt_nplanes
+    return
+  end subroutine read_scalar_integer
 
-      integer, dimension(3), intent(out) :: sizes, subsizes, starts
 
-      integer :: nplanes
-      type(decomp_info) :: decomp
-
-      if (present(opt_decomp)) then
-         decomp = opt_decomp
-      else
-         call get_decomp_info(decomp)
-      end if
+  subroutine read_scalar_logical(fh,disp,n,var)
 
-      if (present(opt_nplanes)) then
-         nplanes = opt_nplanes
-      else
-         nplanes = 1
-      end if
-
-      if (iplane == 1) then
-         sizes(1) = nplanes
-         sizes(2) = decomp%ysz(2)
-         sizes(3) = decomp%zsz(3)
-         subsizes(1) = nplanes
-         subsizes(2) = decomp%xsz(2)
-         subsizes(3) = decomp%xsz(3)
-         starts(1) = 0
-         starts(2) = decomp%xst(2) - 1
-         starts(3) = decomp%xst(3) - 1
-      else if (iplane == 2) then
-         sizes(1) = decomp%xsz(1)
-         sizes(2) = nplanes
-         sizes(3) = decomp%zsz(3)
-         subsizes(1) = decomp%ysz(1)
-         subsizes(2) = nplanes
-         subsizes(3) = decomp%ysz(3)
-         starts(1) = decomp%yst(1) - 1
-         starts(2) = 0
-         starts(3) = decomp%yst(3) - 1
-      else if (iplane == 3) then
-         sizes(1) = decomp%xsz(1)
-         sizes(2) = decomp%ysz(2)
-         sizes(3) = nplanes
-         subsizes(1) = decomp%zsz(1)
-         subsizes(2) = decomp%zsz(2)
-         subsizes(3) = nplanes
-         starts(1) = decomp%zst(1) - 1
-         starts(2) = decomp%zst(2) - 1
-         starts(3) = 0
-      else
-         print *, "Can't work with plane ", iplane
-         stop
-      end if
+    implicit none
 
-   end subroutine plane_extents
+    integer, intent(IN) :: fh
+    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+    integer, intent(IN) :: n
+    logical, dimension(n), intent(INOUT) :: var
 
-   subroutine write_plane_3d_real(ipencil, var, iplane, n, dirname, varname, io_name, &
-                                  opt_decomp)
+    integer :: m, ierror
 
-      implicit none
+    call MPI_FILE_SET_VIEW(fh,disp,MPI_LOGICAL, &
+         MPI_LOGICAL,'native',MPI_INFO_NULL,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+    call MPI_FILE_READ_ALL(fh, var, n, MPI_LOGICAL, &
+         MPI_STATUS_IGNORE, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+    call MPI_TYPE_SIZE(MPI_LOGICAL,m,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+                * int(m, kind=MPI_OFFSET_KIND)
 
-      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
-      integer, intent(IN) :: n ! which plane to write (global coordinate)
-      character(len=*), intent(IN) :: dirname, varname, io_name
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    return
+  end subroutine read_scalar_logical
 
-      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
-      real(mytype), allocatable, dimension(:, :, :) :: wk2d
-      TYPE(DECOMP_INFO) :: decomp
-      integer :: i, j, k, ierror, data_type
 
-      logical :: opened_new
-      integer :: idx
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  ! Write a 2D slice of the 3D data to a file
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+  subroutine plane_extents (sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes)
+
+    integer, intent(in) :: iplane
+    type(decomp_info), intent(in), optional :: opt_decomp
+    integer, intent(in), optional :: opt_nplanes
+    
+    integer, dimension(3), intent(out) :: sizes, subsizes, starts
+
+    integer :: nplanes
+    type(decomp_info) :: decomp
+
+    if (present(opt_decomp)) then
+       decomp = opt_decomp
+    else
+       call get_decomp_info(decomp)
+    end if
+
+    if (present(opt_nplanes)) then
+       nplanes = opt_nplanes
+    else
+       nplanes = 1
+    end if
+    
+    if (iplane == 1) then
+       sizes(1) = nplanes
+       sizes(2) = decomp%ysz(2)
+       sizes(3) = decomp%zsz(3)
+       subsizes(1) = nplanes
+       subsizes(2) = decomp%xsz(2)
+       subsizes(3) = decomp%xsz(3)
+       starts(1) = 0
+       starts(2) = decomp%xst(2)-1
+       starts(3) = decomp%xst(3)-1
+    else if (iplane == 2) then
+       sizes(1) = decomp%xsz(1)
+       sizes(2) = nplanes
+       sizes(3) = decomp%zsz(3)
+       subsizes(1) = decomp%ysz(1)
+       subsizes(2) = nplanes
+       subsizes(3) = decomp%ysz(3)
+       starts(1) = decomp%yst(1)-1
+       starts(2) = 0
+       starts(3) = decomp%yst(3)-1
+    else if (iplane == 3) then
+       sizes(1) = decomp%xsz(1)
+       sizes(2) = decomp%ysz(2)
+       sizes(3) = nplanes
+       subsizes(1) = decomp%zsz(1)
+       subsizes(2) = decomp%zsz(2)
+       subsizes(3) = nplanes
+       starts(1) = decomp%zst(1)-1
+       starts(2) = decomp%zst(2)-1
+       starts(3) = 0
+    else
+       print *, "Can't work with plane ", iplane
+       stop
+    endif
+    
+  end subroutine plane_extents
+
+  subroutine write_plane_3d_real(ipencil,var,iplane,n,dirname,varname,io_name, &
+       opt_decomp)
+
+    implicit none
+
+    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
+    integer, intent(IN) :: n ! which plane to write (global coordinate)
+    character(len=*), intent(IN) :: dirname,varname,io_name
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+
+    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
+    real(mytype), allocatable, dimension(:,:,:) :: wk2d
+    TYPE(DECOMP_INFO) :: decomp
+    integer :: i,j,k, ierror, data_type
+
+    logical :: opened_new
+    integer :: idx
 #ifdef ADIOS2
-      type(adios2_io) :: io_handle
-      type(adios2_variable) :: var_handle
+    type(adios2_io) :: io_handle
+    type(adios2_variable) :: var_handle
 #else
-      integer, dimension(3) :: sizes, subsizes, starts
-      logical :: dir_exists
-      character(len=:), allocatable :: full_io_name
-      integer :: newtype
+    integer, dimension(3) :: sizes, subsizes, starts
+    logical :: dir_exists
+    character(len=:), allocatable :: full_io_name
+    integer :: newtype
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_real")
+    if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_real")
 #endif
 
-      data_type = real_type
+    data_type = real_type
 
 #include "io_write_plane.inc"
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_real")
+    if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_real")
 #endif
 
-   end subroutine write_plane_3d_real
+  end subroutine write_plane_3d_real
+
 
-   subroutine write_plane_3d_complex(ipencil, var, iplane, n, &
-                                     dirname, varname, io_name, opt_decomp)
+  subroutine write_plane_3d_complex(ipencil,var,iplane,n, &
+       dirname,varname,io_name,opt_decomp)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
-      integer, intent(IN) :: n ! which plane to write (global coordinate)
-      character(len=*), intent(IN) :: dirname, varname, io_name
-      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
+    integer, intent(IN) :: n ! which plane to write (global coordinate)
+    character(len=*), intent(IN) :: dirname,varname,io_name
+    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk, wk2
-      complex(mytype), allocatable, dimension(:, :, :) :: wk2d
-      TYPE(DECOMP_INFO) :: decomp
-      integer :: i, j, k, ierror, data_type
-      logical :: opened_new
-      integer :: idx
+    complex(mytype), allocatable, dimension(:,:,:) :: wk, wk2
+    complex(mytype), allocatable, dimension(:,:,:) :: wk2d
+    TYPE(DECOMP_INFO) :: decomp
+    integer :: i,j,k, ierror, data_type
+    logical :: opened_new
+    integer :: idx
 #ifdef ADIOS2
-      type(adios2_io) :: io_handle
-      type(adios2_variable) :: var_handle
+    type(adios2_io) :: io_handle
+    type(adios2_variable) :: var_handle
 #else
-      integer, dimension(3) :: sizes, subsizes, starts
-      logical :: dir_exists
-      character(len=:), allocatable :: full_io_name
-      integer :: newtype
+    integer, dimension(3) :: sizes, subsizes, starts
+    logical :: dir_exists
+    character(len=:), allocatable :: full_io_name
+    integer :: newtype
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_cplx")
+    if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_cplx")
 #endif
 
-      data_type = complex_type
+    data_type = complex_type
 
 #include "io_write_plane.inc"
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_cplx")
+    if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_cplx")
 #endif
 
-   end subroutine write_plane_3d_complex
+  end subroutine write_plane_3d_complex
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Write a 2D array to a file
+  ! Write a 2D array to a file
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !************** TO DO ***************
-   !* Consider handling distributed 2D data set
-   !  subroutine write_plane_2d(ipencil,var,filename)
-   !    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-   !    real(mytype), dimension(:,:), intent(IN) :: var ! 2D array
-   !    character(len=*), intent(IN) :: filename
-   !
-   !    if (ipencil==1) then
-   !       ! var should be defined as var(xsize(2)
-   !
-   !    else if (ipencil==2) then
-   !
-   !    else if (ipencil==3) then
-   !
-   !    end if
-   !
-   !    return
-   !  end subroutine write_plane_2d
+  !************** TO DO ***************
+  !* Consider handling distributed 2D data set
+  !  subroutine write_plane_2d(ipencil,var,filename)
+  !    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+  !    real(mytype), dimension(:,:), intent(IN) :: var ! 2D array
+  !    character(len=*), intent(IN) :: filename
+  !
+  !    if (ipencil==1) then
+  !       ! var should be defined as var(xsize(2)
+  !
+  !    else if (ipencil==2) then
+  !
+  !    else if (ipencil==3) then
+  !
+  !    end if
+  !
+  !    return
+  !  end subroutine write_plane_2d
+
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Write 3D array data for every specified mesh point
+  ! Write 3D array data for every specified mesh point
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine write_every_real(ipencil, var, iskip, jskip, kskip, &
-                               filename, from1)
+  subroutine write_every_real(ipencil,var,iskip,jskip,kskip, &
+       filename, from1)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      integer, intent(IN) :: iskip, jskip, kskip
-      character(len=*), intent(IN) :: filename
-      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-      ! .false. - save n,2n,3n...
+    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    integer, intent(IN) :: iskip,jskip,kskip 
+    character(len=*), intent(IN) :: filename
+    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+    ! .false. - save n,2n,3n...
 
-      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
-      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: i, j, k, ierror, newtype, fh, key, color, newcomm, data_type
-      integer, dimension(3) :: xsz, ysz, zsz, xst, yst, zst, xen, yen, zen, skip
+    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
+    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: i,j,k, ierror, newtype, fh, key,color,newcomm, data_type
+    integer, dimension(3) :: xsz,ysz,zsz,xst,yst,zst,xen,yen,zen,skip
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_write_every_real")
+    if (decomp_profiler_io) call decomp_profiler_start("io_write_every_real")
 #endif
 
-      data_type = real_type
+    data_type = real_type
 
 #include "io_write_every.inc"
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_write_every_real")
+    if (decomp_profiler_io) call decomp_profiler_end("io_write_every_real")
 #endif
 
-   end subroutine write_every_real
+  end subroutine write_every_real
+
 
-   subroutine write_every_complex(ipencil, var, iskip, jskip, kskip, &
-                                  filename, from1)
+  subroutine write_every_complex(ipencil,var,iskip,jskip,kskip, &
+       filename, from1)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      integer, intent(IN) :: iskip, jskip, kskip
-      character(len=*), intent(IN) :: filename
-      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-      ! .false. - save n,2n,3n...
+    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    integer, intent(IN) :: iskip,jskip,kskip 
+    character(len=*), intent(IN) :: filename
+    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+    ! .false. - save n,2n,3n...
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk, wk2
-      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: i, j, k, ierror, newtype, fh, key, color, newcomm, data_type
-      integer, dimension(3) :: xsz, ysz, zsz, xst, yst, zst, xen, yen, zen, skip
+    complex(mytype), allocatable, dimension(:,:,:) :: wk, wk2
+    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: i,j,k, ierror, newtype, fh, key,color,newcomm, data_type
+    integer, dimension(3) :: xsz,ysz,zsz,xst,yst,zst,xen,yen,zen,skip
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_write_every_cplx")
+    if (decomp_profiler_io) call decomp_profiler_start("io_write_every_cplx")
 #endif
 
-      data_type = complex_type
+    data_type = complex_type
 
 #include "io_write_every.inc"
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_write_every_cplx")
+    if (decomp_profiler_io) call decomp_profiler_end("io_write_every_cplx")
 #endif
 
-   end subroutine write_every_complex
+  end subroutine write_every_complex
 
-   subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
+  subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
 
-      implicit none
+    implicit none
+    
+    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+    integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
+    type(decomp_info), intent(in), optional :: opt_decomp
 
-      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-      integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-      type(decomp_info), intent(in), optional :: opt_decomp
+    integer, dimension(3) :: sizes, subsizes, starts
+    type(decomp_info) :: decomp
 
-      integer, dimension(3) :: sizes, subsizes, starts
-      type(decomp_info) :: decomp
+    if ((icoarse.lt.0).or.(icoarse.gt.2)) then
+       call decomp_2d_abort(__FILE__, __LINE__, icoarse, "Error invalid value of icoarse")
+    endif
+    if ((ipencil.lt.1).or.(ipencil.gt.3)) then
+       call decomp_2d_abort(__FILE__, __LINE__, ipencil, "Error invalid value of ipencil ")
+    endif
 
-      if ((icoarse .lt. 0) .or. (icoarse .gt. 2)) then
-         call decomp_2d_abort(__FILE__, __LINE__, icoarse, "Error invalid value of icoarse")
-      end if
-      if ((ipencil .lt. 1) .or. (ipencil .gt. 3)) then
-         call decomp_2d_abort(__FILE__, __LINE__, ipencil, "Error invalid value of ipencil ")
-      end if
-
-      if (icoarse == 0) then
+    if (icoarse==0) then
        !! Use full fields
-
-         if (present(opt_decomp)) then
-            decomp = opt_decomp
-         else
-            call get_decomp_info(decomp)
-         end if
-
-         sizes(1) = decomp%xsz(1)
-         sizes(2) = decomp%ysz(2)
-         sizes(3) = decomp%zsz(3)
-
-         if (ipencil == 1) then
-            subsizes(1:3) = decomp%xsz(1:3)
-            starts(1:3) = decomp%xst(1:3) - 1
-         elseif (ipencil == 2) then
-            subsizes(1:3) = decomp%ysz(1:3)
-            starts(1:3) = decomp%yst(1:3) - 1
-         elseif (ipencil == 3) then
-            subsizes(1:3) = decomp%zsz(1:3)
-            starts(1:3) = decomp%zst(1:3) - 1
-         else
-            call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
-         end if
-      elseif (icoarse == 1) then
-         sizes(1) = xszS(1)
-         sizes(2) = yszS(2)
-         sizes(3) = zszS(3)
-
-         if (ipencil == 1) then
-            subsizes(1) = xszS(1)
-            subsizes(2) = xszS(2)
-            subsizes(3) = xszS(3)
-            starts(1) = xstS(1) - 1  ! 0-based index
-            starts(2) = xstS(2) - 1
-            starts(3) = xstS(3) - 1
-         else if (ipencil == 2) then
-            subsizes(1) = yszS(1)
-            subsizes(2) = yszS(2)
-            subsizes(3) = yszS(3)
-            starts(1) = ystS(1) - 1
-            starts(2) = ystS(2) - 1
-            starts(3) = ystS(3) - 1
-         else if (ipencil == 3) then
-            subsizes(1) = zszS(1)
-            subsizes(2) = zszS(2)
-            subsizes(3) = zszS(3)
-            starts(1) = zstS(1) - 1
-            starts(2) = zstS(2) - 1
-            starts(3) = zstS(3) - 1
-         else
-            call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
-         end if
-      elseif (icoarse == 2) then
-         sizes(1) = xszV(1)
-         sizes(2) = yszV(2)
-         sizes(3) = zszV(3)
-
-         if (ipencil == 1) then
-            subsizes(1) = xszV(1)
-            subsizes(2) = xszV(2)
-            subsizes(3) = xszV(3)
-            starts(1) = xstV(1) - 1  ! 0-based index
-            starts(2) = xstV(2) - 1
-            starts(3) = xstV(3) - 1
-         else if (ipencil == 2) then
-            subsizes(1) = yszV(1)
-            subsizes(2) = yszV(2)
-            subsizes(3) = yszV(3)
-            starts(1) = ystV(1) - 1
-            starts(2) = ystV(2) - 1
-            starts(3) = ystV(3) - 1
-         else if (ipencil == 3) then
-            subsizes(1) = zszV(1)
-            subsizes(2) = zszV(2)
-            subsizes(3) = zszV(3)
-            starts(1) = zstV(1) - 1
-            starts(2) = zstV(2) - 1
-            starts(3) = zstV(3) - 1
-         else
-            call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
-         end if
-      end if
-
-   end subroutine coarse_extents
-
-  subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_name, opt_decomp, reduce_prec, opt_deferred_writes)
-
-      ! USE param
-      ! USE variables
-
-      implicit none
-
-      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-      integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      character(len=*), intent(in) :: dirname, varname, io_name
-      type(decomp_info), intent(in), optional :: opt_decomp
-      logical, intent(in), optional :: reduce_prec
-      logical, intent(in), optional :: opt_deferred_writes
-
-      logical :: write_reduce_prec
-      logical :: deferred_writes
-
-      integer :: ierror
-      integer :: idx
-      logical :: opened_new
+       
+       if (present(opt_decomp)) then
+          decomp = opt_decomp
+       else
+          call get_decomp_info(decomp)
+       endif
+
+       sizes(1) = decomp%xsz(1)
+       sizes(2) = decomp%ysz(2)
+       sizes(3) = decomp%zsz(3)
+
+       if (ipencil == 1) then
+          subsizes(1:3) = decomp%xsz(1:3)
+          starts(1:3) = decomp%xst(1:3) - 1
+       elseif (ipencil == 2) then
+          subsizes(1:3) = decomp%ysz(1:3)
+          starts(1:3) = decomp%yst(1:3) - 1
+       elseif (ipencil == 3) then
+          subsizes(1:3) = decomp%zsz(1:3)
+          starts(1:3) = decomp%zst(1:3) - 1
+       else
+          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
+       endif
+    elseif (icoarse==1) then
+       sizes(1) = xszS(1)
+       sizes(2) = yszS(2)
+       sizes(3) = zszS(3)
+
+       if (ipencil == 1) then
+          subsizes(1) = xszS(1)
+          subsizes(2) = xszS(2)
+          subsizes(3) = xszS(3)
+          starts(1) = xstS(1)-1  ! 0-based index
+          starts(2) = xstS(2)-1
+          starts(3) = xstS(3)-1
+       else if (ipencil == 2) then
+          subsizes(1) = yszS(1)
+          subsizes(2) = yszS(2)
+          subsizes(3) = yszS(3)
+          starts(1) = ystS(1)-1
+          starts(2) = ystS(2)-1
+          starts(3) = ystS(3)-1
+       else if (ipencil == 3) then
+          subsizes(1) = zszS(1)
+          subsizes(2) = zszS(2)
+          subsizes(3) = zszS(3)
+          starts(1) = zstS(1)-1
+          starts(2) = zstS(2)-1
+          starts(3) = zstS(3)-1
+       else
+          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
+       endif
+    elseif (icoarse==2) then
+       sizes(1) = xszV(1)
+       sizes(2) = yszV(2)
+       sizes(3) = zszV(3)
+
+       if (ipencil == 1) then
+          subsizes(1) = xszV(1)
+          subsizes(2) = xszV(2)
+          subsizes(3) = xszV(3)
+          starts(1) = xstV(1)-1  ! 0-based index
+          starts(2) = xstV(2)-1
+          starts(3) = xstV(3)-1
+       else if (ipencil == 2) then
+          subsizes(1) = yszV(1)
+          subsizes(2) = yszV(2)
+          subsizes(3) = yszV(3)
+          starts(1) = ystV(1)-1
+          starts(2) = ystV(2)-1
+          starts(3) = ystV(3)-1
+       else if (ipencil == 3) then
+          subsizes(1) = zszV(1)
+          subsizes(2) = zszV(2)
+          subsizes(3) = zszV(3)
+          starts(1) = zstV(1)-1
+          starts(2) = zstV(2)-1
+          starts(3) = zstV(3)-1
+       else
+          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
+       endif
+    endif
+    
+  end subroutine coarse_extents
+
+  subroutine mpiio_write_real_coarse(ipencil,var,dirname,varname,icoarse,io_name,opt_decomp,reduce_prec,opt_deferred_writes)
+
+    ! USE param
+    ! USE variables
+
+    implicit none
+
+    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+    integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    character(len=*), intent(in) :: dirname, varname, io_name
+    type(decomp_info), intent(in), optional :: opt_decomp
+    logical, intent(in), optional :: reduce_prec
+    logical, intent(in), optional :: opt_deferred_writes
+
+    logical :: write_reduce_prec
+    logical :: deferred_writes
+    
+    integer :: ierror
+    integer :: idx
+    logical :: opened_new
 #ifdef ADIOS2
-      type(adios2_io) :: io_handle
-      type(adios2_variable) :: var_handle
-      integer :: write_mode
+    type(adios2_io) :: io_handle
+    type(adios2_variable) :: var_handle
+    integer :: write_mode
 #else
-      real(mytype_single), allocatable, dimension(:, :, :) :: varsingle
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: newtype
-      logical :: dir_exists
-      integer :: disp_bytes
-      character(len=:), allocatable :: full_io_name
+    real(mytype_single), allocatable, dimension(:,:,:) :: varsingle
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: newtype
+    logical :: dir_exists
+    integer :: disp_bytes
+    character(len=:), allocatable :: full_io_name
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_coarse")
+    if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_coarse")
 #endif
 
     !! Set defaults
-      write_reduce_prec = .true.
-      if (present(opt_deferred_writes)) then
-         deferred_writes = opt_deferred_writes
-      else
-         deferred_writes = .true.
-      end if
-
-      opened_new = .false.
-      idx = get_io_idx(io_name, dirname)
+    write_reduce_prec = .true.
+    if (present(opt_deferred_writes)) then
+       deferred_writes = opt_deferred_writes
+    else
+       deferred_writes = .true.
+    end if
+    
+    opened_new = .false.
+    idx = get_io_idx(io_name, dirname)
 #ifndef ADIOS2
-      if (present(reduce_prec)) then
-         if (.not. reduce_prec) then
-            write_reduce_prec = .false.
-         end if
-      end if
-      if (write_reduce_prec) then
-         call MPI_TYPE_SIZE(real_type_single, disp_bytes, ierror)
-      else
-         call MPI_TYPE_SIZE(real_type, disp_bytes, ierror)
-      end if
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-
+    if (present(reduce_prec)) then
+       if (.not. reduce_prec) then
+          write_reduce_prec = .false.
+       end if
+    end if
+    if (write_reduce_prec) then
+       call MPI_TYPE_SIZE(real_type_single,disp_bytes,ierror)
+    else
+       call MPI_TYPE_SIZE(real_type,disp_bytes,ierror)
+    end if
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+    
     !! Use original MPIIO writers
-      if (present(opt_decomp)) then
-         call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
-      else
-         call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
-      end if
-      if (write_reduce_prec) then
-         allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
-         varsingle = real(var, mytype_single)
-      end if
-
-      if (write_reduce_prec) then
-         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                                       MPI_ORDER_FORTRAN, real_type_single, newtype, ierror)
-      else
-         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                                       MPI_ORDER_FORTRAN, real_type, newtype, ierror)
-      end if
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-      call MPI_TYPE_COMMIT(newtype, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-
-      if (idx .lt. 1) then
-         ! Create folder if needed
-         if (nrank == 0) then
-            inquire (file=dirname, exist=dir_exists)
-            if (.not. dir_exists) then
-               call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
-            end if
-         end if
-         full_io_name = trim(dirname)//"/"//trim(varname)
-         call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
-         idx = get_io_idx(io_name, full_io_name)
-         opened_new = .true.
-      else
-         full_io_name = "" ! Ensure string is set
-      end if
-
-      if (write_reduce_prec) then
-         call MPI_FILE_SET_VIEW(fh_registry(idx), fh_disp(idx), real_type_single, &
-                                newtype, 'native', MPI_INFO_NULL, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-         call MPI_FILE_WRITE_ALL(fh_registry(idx), varsingle, &
-                                 subsizes(1)*subsizes(2)*subsizes(3), &
-                                 real_type_single, MPI_STATUS_IGNORE, ierror)
-      else
-         call MPI_FILE_SET_VIEW(fh_registry(idx), fh_disp(idx), real_type, &
-                                newtype, 'native', MPI_INFO_NULL, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-         call MPI_FILE_WRITE_ALL(fh_registry(idx), var, &
-                                 subsizes(1)*subsizes(2)*subsizes(3), &
-                                 real_type, MPI_STATUS_IGNORE, ierror)
-      end if
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-
-      fh_disp(idx) = fh_disp(idx) + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                     *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                     *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                     *int(disp_bytes, kind=MPI_OFFSET_KIND)
-
-      if (opened_new) then
-         call decomp_2d_close_io(io_name, full_io_name)
-         deallocate (full_io_name)
-      end if
-
-      call MPI_TYPE_FREE(newtype, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-      if (write_reduce_prec) then
-         deallocate (varsingle)
-      end if
+    if (present(opt_decomp)) then
+       call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
+    else
+       call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
+    end if
+    if (write_reduce_prec) then
+       allocate (varsingle(xstV(1):xenV(1),xstV(2):xenV(2),xstV(3):xenV(3)))
+       varsingle=real(var, mytype_single)
+    end if
+
+    if (write_reduce_prec) then
+       call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+            MPI_ORDER_FORTRAN, real_type_single, newtype, ierror)
+    else
+       call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+            MPI_ORDER_FORTRAN, real_type, newtype, ierror)
+    end if
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+    call MPI_TYPE_COMMIT(newtype,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+
+    if (idx .lt. 1) then
+       ! Create folder if needed
+       if (nrank==0) then
+          inquire(file=dirname, exist=dir_exists)
+          if (.not.dir_exists) then
+             call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
+          end if
+       end if
+       full_io_name = trim(dirname)//"/"//trim(varname)
+       call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
+       idx = get_io_idx(io_name, full_io_name)
+       opened_new = .true.
+    else
+       full_io_name = "" ! Ensure string is set
+    end if
+
+    if (write_reduce_prec) then
+       call MPI_FILE_SET_VIEW(fh_registry(idx),fh_disp(idx),real_type_single, &
+            newtype,'native',MPI_INFO_NULL,ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+       call MPI_FILE_WRITE_ALL(fh_registry(idx), varsingle, &
+            subsizes(1)*subsizes(2)*subsizes(3), &
+            real_type_single, MPI_STATUS_IGNORE, ierror)
+    else
+       call MPI_FILE_SET_VIEW(fh_registry(idx),fh_disp(idx),real_type, &
+            newtype,'native',MPI_INFO_NULL,ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+       call MPI_FILE_WRITE_ALL(fh_registry(idx), var, &
+            subsizes(1)*subsizes(2)*subsizes(3), &
+            real_type, MPI_STATUS_IGNORE, ierror)
+    end if
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+    
+    fh_disp(idx) = fh_disp(idx) + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                * int(disp_bytes, kind=MPI_OFFSET_KIND)
+    
+    if (opened_new) then
+       call decomp_2d_close_io(io_name, full_io_name)
+       deallocate(full_io_name)
+    end if
+
+    call MPI_TYPE_FREE(newtype,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+    if (write_reduce_prec) then
+       deallocate(varsingle)
+    end if
 #else
-      if (.not. engine_live(idx)) then
-         call decomp_2d_abort(__FILE__, __LINE__, -1, "ERROR: Engine is not live!")
-      end if
-
-      call adios2_at_io(io_handle, adios, io_name, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-      call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-      if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-                                                       "ERROR: trying to write variable before registering! "//trim(varname))
-
-      if (idx .lt. 1) call decomp_2d_abort(__FILE__, __LINE__, idx, &
-                                           "You haven't opened "//trim(io_name)//":"//trim(dirname))
-
-      if (deferred_writes) then
-         write_mode = adios2_mode_deferred
-      else
-         write_mode = adios2_mode_sync
-      end if
-
-      if (engine_registry(idx)%valid) then
-         call adios2_put(engine_registry(idx), var_handle, var, write_mode, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
-      else
-         call decomp_2d_abort(__FILE__, __LINE__, -1, &
-                              "ERROR: decomp2d thinks engine is live, but adios2 engine object is not valid")
-      end if
-
-      associate (crs => icoarse, pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
-      end associate
+    if (.not. engine_live(idx)) then
+       call decomp_2d_abort(__FILE__, __LINE__, -1, "ERROR: Engine is not live!")
+    end if
+    
+    call adios2_at_io(io_handle, adios, io_name, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+    call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+    if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+       "ERROR: trying to write variable before registering! "//trim(varname))
+
+    if (idx .lt. 1) call decomp_2d_abort(__FILE__, __LINE__, idx, &
+       "You haven't opened "//trim(io_name)//":"//trim(dirname))
+
+    if (deferred_writes) then
+       write_mode = adios2_mode_deferred
+    else
+       write_mode = adios2_mode_sync
+    end if
+
+    if (engine_registry(idx)%valid) then
+       call adios2_put(engine_registry(idx), var_handle, var, write_mode, ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
+    else
+       call decomp_2d_abort(__FILE__, __LINE__, -1, &
+          "ERROR: decomp2d thinks engine is live, but adios2 engine object is not valid")
+    end if
+
+    associate(crs => icoarse, pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
+    end associate
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_coarse")
+    if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_coarse")
 #endif
 
-   end subroutine mpiio_write_real_coarse
+  end subroutine mpiio_write_real_coarse
 
-   subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplane, type, opt_decomp, opt_nplanes)
+  subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplane, type, opt_decomp, opt_nplanes)
 
-      use, intrinsic :: iso_fortran_env, only: real32, real64
+    use, intrinsic :: iso_fortran_env, only : real32, real64
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-      integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-      character(len=*), intent(in) :: io_name
-      integer, intent(in) :: type
-      integer, intent(in) :: iplane
-      type(decomp_info), intent(in), optional :: opt_decomp
-      integer, intent(in), optional :: opt_nplanes
+    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+    integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
+    character(len=*), intent(in) :: io_name
+    integer, intent(in) :: type
+    integer, intent(in) :: iplane
+    type(decomp_info), intent(in), optional :: opt_decomp
+    integer, intent(in), optional :: opt_nplanes
 
-      integer :: nplanes
-      character*(*), intent(in) :: varname
+    integer :: nplanes
+    character*(*), intent(in) :: varname
 #ifdef ADIOS2
-      integer, dimension(3) :: sizes, subsizes, starts
-      type(adios2_io) :: io_handle
-      type(adios2_variable) :: var_handle
-      integer, parameter :: ndims = 3
-      logical, parameter :: adios2_constant_dims = .true.
-      integer :: data_type
-      integer :: ierror
-
-      if (iplane .eq. 0) then
-         if (present(opt_decomp)) then
-            call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
-         else
-            call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
-         end if
-      else
-         if (present(opt_nplanes)) then
-            nplanes = opt_nplanes
-         else
-            nplanes = 1
-         end if
-         if (present(opt_decomp)) then
-            call plane_extents(sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes=nplanes)
-         else
-            call plane_extents(sizes, subsizes, starts, iplane, opt_nplanes=nplanes)
-         end if
-      end if
-
-      ! Check if variable already exists, if not create it
-      call adios2_at_io(io_handle, adios, io_name, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-      if (io_handle%valid) then
-         call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-         if (.not. var_handle%valid) then
+    integer, dimension(3) :: sizes, subsizes, starts
+    type(adios2_io) :: io_handle
+    type(adios2_variable) :: var_handle
+    integer, parameter :: ndims = 3
+    logical, parameter :: adios2_constant_dims = .true.
+    integer :: data_type
+    integer :: ierror
+
+    if (iplane .eq. 0) then
+       if (present(opt_decomp)) then
+          call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
+       else
+          call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
+       endif
+    else
+       if (present(opt_nplanes)) then
+          nplanes = opt_nplanes
+       else
+          nplanes = 1
+       end if
+       if (present(opt_decomp)) then
+          call plane_extents(sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes=nplanes)
+       else
+          call plane_extents(sizes, subsizes, starts, iplane, opt_nplanes=nplanes)
+       endif
+    end if
+    
+    ! Check if variable already exists, if not create it
+    call adios2_at_io(io_handle, adios, io_name, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+    if (io_handle%valid) then
+       call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+       if (.not.var_handle % valid) then
           !! New variable
-            if (nrank .eq. 0) then
-               print *, "Registering variable for IO: ", varname
-            end if
+          if (nrank .eq. 0) then
+             print *, "Registering variable for IO: ", varname
+          endif
 
-            ! Need to set the ADIOS2 data type
-            if (type .eq. kind(0._real64)) then
+          ! Need to set the ADIOS2 data type
+          if (type.eq.kind(0._real64)) then
              !! Double
-               data_type = adios2_type_dp
-            else if (type .eq. kind(0._real32)) then
+             data_type = adios2_type_dp
+          else if (type.eq.kind(0._real32)) then
              !! Single
-               data_type = adios2_type_real
-            else
-               call decomp_2d_abort(__FILE__, __LINE__, -1, "Trying to write unknown data type!")
-            end if
-
-            call adios2_define_variable(var_handle, io_handle, varname, data_type, &
-                                        ndims, int(sizes, kind=8), int(starts, kind=8), int(subsizes, kind=8), &
-                                        adios2_constant_dims, ierror)
-            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-                                                    "adios2_define_variable, ERROR registering variable "//trim(varname))
-         end if
-      else
-         call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to register variable with invalid IO!")
-      end if
+             data_type = adios2_type_real
+          else
+             call decomp_2d_abort(__FILE__, __LINE__, -1, "Trying to write unknown data type!")
+          endif
+
+          call adios2_define_variable(var_handle, io_handle, varname, data_type, &
+               ndims, int(sizes, kind=8), int(starts, kind=8), int(subsizes, kind=8), &
+               adios2_constant_dims, ierror)
+          if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+             "adios2_define_variable, ERROR registering variable "//trim(varname))
+       endif
+    else
+       call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to register variable with invalid IO!")
+    end if
 #else
-      nplanes = 1 ! Silence unused variable
-      associate (crs => icoarse, nm => io_name, pncl => ipencil, pln => iplane, &
-                 opdcmp => opt_decomp, opnpl => opt_nplanes, tp => type, &
-                 vnm => varname) ! Silence unused dummy argument
-      end associate
+    nplanes = 1 ! Silence unused variable
+    associate(crs => icoarse, nm =>io_name, pncl => ipencil, pln => iplane, &
+         opdcmp => opt_decomp, opnpl => opt_nplanes, tp => type, &
+         vnm => varname) ! Silence unused dummy argument
+    end associate
 #endif
+    
+  end subroutine decomp_2d_register_variable
+  
+  subroutine mpiio_write_real_probe(ipencil,var,filename,nlength)
 
-   end subroutine decomp_2d_register_variable
-
-   subroutine mpiio_write_real_probe(ipencil, var, filename, nlength)
-
-      ! USE param
-      ! USE variables
+    ! USE param
+    ! USE variables
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-      integer, intent(in) :: nlength
-      real(mytype), contiguous, dimension(:, :, :, :), intent(IN) :: var
+    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+    integer, intent(in) :: nlength
+    real(mytype), contiguous, dimension(:,:,:,:), intent(IN) :: var
 
-      character(len=*) :: filename
+    character(len=*) :: filename
 
-      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-      integer, dimension(4) :: sizes, subsizes, starts
-      integer :: ierror, newtype, fh
+    integer (kind=MPI_OFFSET_KIND) :: filesize, disp
+    integer, dimension(4) :: sizes, subsizes, starts
+    integer :: ierror, newtype, fh
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_probe")
-#endif
-
-      sizes(1) = xszP(1)
-      sizes(2) = yszP(2)
-      sizes(3) = zszP(3)
-      sizes(4) = nlength
-      if (ipencil == 1) then
-         subsizes(1) = xszP(1)
-         subsizes(2) = xszP(2)
-         subsizes(3) = xszP(3)
-         subsizes(4) = nlength
-         starts(1) = xstP(1) - 1  ! 0-based index
-         starts(2) = xstP(2) - 1
-         starts(3) = xstP(3) - 1
-         starts(4) = 0
-      else if (ipencil == 2) then
-         subsizes(1) = yszP(1)
-         subsizes(2) = yszP(2)
-         subsizes(3) = yszP(3)
-         starts(1) = ystP(1) - 1
-         starts(2) = ystP(2) - 1
-         starts(3) = ystP(3) - 1
-      else if (ipencil == 3) then
-         subsizes(1) = zszP(1)
-         subsizes(2) = zszP(2)
-         subsizes(3) = zszP(3)
-         starts(1) = zstP(1) - 1
-         starts(2) = zstP(2) - 1
-         starts(3) = zstP(3) - 1
-      end if
-      !   print *,nrank,starts(1),starts(2),starts(3),starts(4)
-      call MPI_TYPE_CREATE_SUBARRAY(4, sizes, subsizes, starts, &
-                                    MPI_ORDER_FORTRAN, real_type, newtype, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-      call MPI_TYPE_COMMIT(newtype, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-      call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-                         MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
-                         fh, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
-      filesize = 0_MPI_OFFSET_KIND
-      call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
-      disp = 0_MPI_OFFSET_KIND
-      call MPI_FILE_SET_VIEW(fh, disp, real_type, &
-                             newtype, 'native', MPI_INFO_NULL, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-      call MPI_FILE_WRITE_ALL(fh, var, &
-                              subsizes(1)*subsizes(2)*subsizes(3)*subsizes(4), &
-                              real_type, MPI_STATUS_IGNORE, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-      call MPI_FILE_CLOSE(fh, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-      call MPI_TYPE_FREE(newtype, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+    if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_probe")
+#endif
+
+    sizes(1) = xszP(1)
+    sizes(2) = yszP(2)
+    sizes(3) = zszP(3)
+    sizes(4) = nlength
+    if (ipencil == 1) then
+       subsizes(1) = xszP(1)
+       subsizes(2) = xszP(2)
+       subsizes(3) = xszP(3)
+       subsizes(4) = nlength
+       starts(1) = xstP(1)-1  ! 0-based index
+       starts(2) = xstP(2)-1
+       starts(3) = xstP(3)-1
+       starts(4) = 0
+    else if (ipencil == 2) then
+       subsizes(1) = yszP(1)
+       subsizes(2) = yszP(2)
+       subsizes(3) = yszP(3)
+       starts(1) = ystP(1)-1
+       starts(2) = ystP(2)-1
+       starts(3) = ystP(3)-1
+    else if (ipencil == 3) then
+       subsizes(1) = zszP(1)
+       subsizes(2) = zszP(2)
+       subsizes(3) = zszP(3)
+       starts(1) = zstP(1)-1
+       starts(2) = zstP(2)-1
+       starts(3) = zstP(3)-1
+    endif
+    !   print *,nrank,starts(1),starts(2),starts(3),starts(4)
+    call MPI_TYPE_CREATE_SUBARRAY(4, sizes, subsizes, starts,  &
+         MPI_ORDER_FORTRAN, real_type, newtype, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+    call MPI_TYPE_COMMIT(newtype,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+    call MPI_FILE_OPEN(decomp_2d_comm, filename, &
+         MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
+         fh, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+    filesize = 0_MPI_OFFSET_KIND
+    call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+    disp = 0_MPI_OFFSET_KIND
+    call MPI_FILE_SET_VIEW(fh,disp,real_type, &
+         newtype,'native',MPI_INFO_NULL,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+    call MPI_FILE_WRITE_ALL(fh, var, &
+         subsizes(1)*subsizes(2)*subsizes(3)*subsizes(4), &
+         real_type, MPI_STATUS_IGNORE, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+    call MPI_FILE_CLOSE(fh,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+    call MPI_TYPE_FREE(newtype,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_probe")
+    if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_probe")
 #endif
 
-   end subroutine mpiio_write_real_probe
+  end subroutine mpiio_write_real_probe
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Write a 3D data set covering a smaller sub-domain only
+  ! Write a 3D data set covering a smaller sub-domain only
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine write_subdomain(ipencil, var, is, ie, js, je, ks, ke, filename)
+  subroutine write_subdomain(ipencil,var,is,ie,js,je,ks,ke,filename)
 
-      implicit none
+    implicit none
 
-      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
-      integer, intent(IN) :: is, ie, js, je, ks, ke
-      character(len=*), intent(IN) :: filename
+    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
+    integer, intent(IN) :: is, ie, js, je, ks, ke
+    character(len=*), intent(IN) :: filename
 
-      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
-      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-      integer, dimension(3) :: sizes, subsizes, starts
-      integer :: color, key, errorcode, newcomm, ierror
-      integer :: newtype, fh, data_type, i, j, k
-      integer :: i1, i2, j1, j2, k1, k2
+    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
+    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+    integer, dimension(3) :: sizes, subsizes, starts
+    integer :: color, key, errorcode, newcomm, ierror
+    integer :: newtype, fh, data_type, i, j, k
+    integer :: i1, i2, j1, j2, k1, k2
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_write_subdomain")
-#endif
-
-      data_type = real_type
-
-      ! validate the input paramters
-      if (is < 1 .OR. ie > nx_global .OR. js < 1 .OR. je > ny_global .OR. &
-          ks < 1 .OR. ke > nz_global) then
-         errorcode = 10
-         call decomp_2d_abort(errorcode, &
-                              'Invalid subdomain specified in I/O')
-      end if
-
-      ! create a communicator for all those MPI ranks containing the subdomain
-      color = 1
-      key = 1
-      if (ipencil == 1) then
-         if (xstart(1) > ie .OR. xend(1) < is .OR. xstart(2) > je .OR. xend(2) < js &
-             .OR. xstart(3) > ke .OR. xend(3) < ks) then
-            color = 2
-         end if
-      else if (ipencil == 2) then
-         if (ystart(1) > ie .OR. yend(1) < is .OR. ystart(2) > je .OR. yend(2) < js &
-             .OR. ystart(3) > ke .OR. yend(3) < ks) then
-            color = 2
-         end if
-      else if (ipencil == 3) then
-         if (zstart(1) > ie .OR. zend(1) < is .OR. zstart(2) > je .OR. zend(2) < js &
-             .OR. zstart(3) > ke .OR. zend(3) < ks) then
-            color = 2
-         end if
-      end if
-      call MPI_COMM_SPLIT(decomp_2d_comm, color, key, newcomm, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-
-      if (color == 1) then ! only ranks in this group do IO collectively
-
-         ! generate MPI-IO subarray information
-
-         ! global size of the sub-domain to write
-         sizes(1) = ie - is + 1
-         sizes(2) = je - js + 1
-         sizes(3) = ke - ks + 1
-
-         ! 'subsizes' & 'starts' as required by MPI_TYPE_CREATE_SUBARRAY
-         ! note the special code whe subdomain only occupy part of the pencil
-         if (ipencil == 1) then
-
-            subsizes(1) = xsize(1)
-            starts(1) = xstart(1) - is
-            if (xend(1) > ie .AND. xstart(1) < is) then
-               subsizes(1) = ie - is + 1
-               starts(1) = 0
-            else if (xstart(1) < is) then
-               subsizes(1) = xend(1) - is + 1
-               starts(1) = 0
-            else if (xend(1) > ie) then
-               subsizes(1) = ie - xstart(1) + 1
-            end if
-            subsizes(2) = xsize(2)
-            starts(2) = xstart(2) - js
-            if (xend(2) > je .AND. xstart(2) < js) then
-               subsizes(2) = je - js + 1
-               starts(2) = 0
-            else if (xstart(2) < js) then
-               subsizes(2) = xend(2) - js + 1
-               starts(2) = 0
-            else if (xend(2) > je) then
-               subsizes(2) = je - xstart(2) + 1
-            end if
-            subsizes(3) = xsize(3)
-            starts(3) = xstart(3) - ks
-            if (xend(3) > ke .AND. xstart(3) < ks) then
-               subsizes(3) = ke - ks + 1
-               starts(3) = 0
-            else if (xstart(3) < ks) then
-               subsizes(3) = xend(3) - ks + 1
-               starts(3) = 0
-            else if (xend(3) > ke) then
-               subsizes(3) = ke - xstart(3) + 1
-            end if
-
-         else if (ipencil == 2) then
-
-            ! TODO
-
-         else if (ipencil == 3) then
-
-            ! TODO
-
-         end if
-
-         ! copy data from orginal to a temp array
-         ! pay attention to blocks only partially cover the sub-domain
-         if (ipencil == 1) then
-
-            if (xend(1) > ie .AND. xstart(1) < is) then
-               i1 = is
-               i2 = ie
-            else if (xend(1) > ie) then
-               i1 = xstart(1)
-               i2 = ie
-            else if (xstart(1) < is) then
-               i1 = is
-               i2 = xend(1)
-            else
-               i1 = xstart(1)
-               i2 = xend(1)
-            end if
-
-            if (xend(2) > je .AND. xstart(2) < js) then
-               j1 = js
-               j2 = je
-            else if (xend(2) > je) then
-               j1 = xstart(2)
-               j2 = je
-            else if (xstart(2) < js) then
-               j1 = js
-               j2 = xend(2)
-            else
-               j1 = xstart(2)
-               j2 = xend(2)
-            end if
-
-            if (xend(3) > ke .AND. xstart(3) < ks) then
-               k1 = ks
-               k2 = ke
-            else if (xend(3) > ke) then
-               k1 = xstart(3)
-               k2 = ke
-            else if (xstart(3) < ks) then
-               k1 = ks
-               k2 = xend(3)
-            else
-               k1 = xstart(3)
-               k2 = xend(3)
-            end if
-
-            allocate (wk(i1:i2, j1:j2, k1:k2))
-            allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-            wk2 = var
-            do k = k1, k2
-               do j = j1, j2
-                  do i = i1, i2
-                     wk(i, j, k) = wk2(i, j, k)
-                  end do
-               end do
-            end do
-
-         else if (ipencil == 2) then
-
-            ! TODO
-
-         else if (ipencil == 3) then
-
-            ! TODO
-
-         end if
-
-         deallocate (wk2)
-
-         ! MPI-IO
-         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                                       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-         call MPI_TYPE_COMMIT(newtype, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-         call MPI_FILE_OPEN(newcomm, filename, &
-                            MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
-                            fh, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
-         filesize = 0_MPI_OFFSET_KIND
-         call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
-         disp = 0_MPI_OFFSET_KIND
-         call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                                newtype, 'native', MPI_INFO_NULL, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-         call MPI_FILE_WRITE_ALL(fh, wk, &
-                                 subsizes(1)*subsizes(2)*subsizes(3), &
-                                 data_type, MPI_STATUS_IGNORE, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-         call MPI_FILE_CLOSE(fh, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-         call MPI_TYPE_FREE(newtype, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-
-         deallocate (wk)
-
-      end if
-
-      call decomp_mpi_comm_free(newcomm)
+    if (decomp_profiler_io) call decomp_profiler_start("io_write_subdomain")
+#endif
+
+    data_type = real_type
+
+    ! validate the input paramters
+    if (is<1 .OR. ie>nx_global .OR. js<1 .OR. je>ny_global .OR. &
+         ks<1 .OR. ke>nz_global) then
+       errorcode = 10
+       call decomp_2d_abort(errorcode, &
+            'Invalid subdomain specified in I/O')
+    end if
+
+    ! create a communicator for all those MPI ranks containing the subdomain
+    color = 1
+    key = 1
+    if (ipencil==1) then
+       if (xstart(1)>ie .OR. xend(1)<is .OR. xstart(2)>je .OR. xend(2)<js &
+            .OR. xstart(3)>ke .OR. xend(3)<ks) then
+          color = 2
+       end if
+    else if (ipencil==2) then
+       if (ystart(1)>ie .OR. yend(1)<is .OR. ystart(2)>je .OR. yend(2)<js &
+            .OR. ystart(3)>ke .OR. yend(3)<ks) then
+          color = 2
+       end if
+    else if (ipencil==3) then
+       if (zstart(1)>ie .OR. zend(1)<is .OR. zstart(2)>je .OR. zend(2)<js &
+            .OR. zstart(3)>ke .OR. zend(3)<ks) then
+          color = 2
+       end if
+    end if
+    call MPI_COMM_SPLIT(decomp_2d_comm,color,key,newcomm,ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+
+    if (color==1) then ! only ranks in this group do IO collectively
+
+       ! generate MPI-IO subarray information
+
+       ! global size of the sub-domain to write
+       sizes(1) = ie - is + 1
+       sizes(2) = je - js + 1
+       sizes(3) = ke - ks + 1
+
+       ! 'subsizes' & 'starts' as required by MPI_TYPE_CREATE_SUBARRAY
+       ! note the special code whe subdomain only occupy part of the pencil
+       if (ipencil==1) then
+
+          subsizes(1) = xsize(1)
+          starts(1) = xstart(1) - is
+          if (xend(1)>ie .AND. xstart(1)<is) then
+             subsizes(1) = ie - is + 1
+             starts(1) = 0
+          else if (xstart(1)<is) then
+             subsizes(1) = xend(1) - is + 1
+             starts(1) = 0
+          else if (xend(1)>ie) then
+             subsizes(1) = ie - xstart(1) + 1
+          end if
+          subsizes(2) = xsize(2)
+          starts(2) = xstart(2) - js
+          if (xend(2)>je .AND. xstart(2)<js) then
+             subsizes(2) = je - js + 1
+             starts(2) = 0
+          else if (xstart(2)<js) then
+             subsizes(2) = xend(2) - js + 1
+             starts(2) = 0
+          else if (xend(2)>je) then
+             subsizes(2) = je - xstart(2) + 1
+          end if
+          subsizes(3) = xsize(3)
+          starts(3) = xstart(3) - ks
+          if (xend(3)>ke .AND. xstart(3)<ks) then
+             subsizes(3) = ke - ks + 1
+             starts(3) = 0
+          else if (xstart(3)<ks) then
+             subsizes(3) = xend(3) - ks + 1
+             starts(3) = 0
+          else if (xend(3)>ke) then
+             subsizes(3) = ke - xstart(3) + 1
+          end if
+
+       else if (ipencil==2) then
+
+          ! TODO 
+
+       else if (ipencil==3) then
+
+          ! TODO
+
+       end if
+
+
+       ! copy data from orginal to a temp array
+       ! pay attention to blocks only partially cover the sub-domain
+       if (ipencil==1) then
+
+          if (xend(1)>ie .AND. xstart(1)<is) then
+             i1 = is
+             i2 = ie
+          else if (xend(1)>ie) then
+             i1 = xstart(1)
+             i2 = ie
+          else if (xstart(1)<is) then
+             i1 = is
+             i2 = xend(1)
+          else
+             i1 = xstart(1)
+             i2 = xend(1)
+          end if
+
+          if (xend(2)>je .AND. xstart(2)<js) then
+             j1 = js
+             j2 = je
+          else if (xend(2)>je) then
+             j1 = xstart(2)
+             j2 = je
+          else if (xstart(2)<js) then
+             j1 = js
+             j2 = xend(2)
+          else
+             j1 = xstart(2)
+             j2 = xend(2)
+          end if
+
+          if (xend(3)>ke .AND. xstart(3)<ks) then
+             k1 = ks
+             k2 = ke
+          else if (xend(3)>ke) then
+             k1 = xstart(3)
+             k2 = ke
+          else if (xstart(3)<ks) then
+             k1 = ks
+             k2 = xend(3)
+          else
+             k1 = xstart(3)
+             k2 = xend(3)
+          end if
+
+          allocate(wk(i1:i2, j1:j2, k1:k2))
+          allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
+          wk2 = var
+          do k=k1,k2
+             do j=j1,j2
+                do i=i1,i2
+                   wk(i,j,k) = wk2(i,j,k)
+                end do
+             end do
+          end do
+
+       else if (ipencil==2) then
+
+          ! TODO
+
+       else if (ipencil==3) then
+
+          ! TODO
+
+       end if
+
+       deallocate(wk2)
+
+       ! MPI-IO
+       call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+            MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+       call MPI_TYPE_COMMIT(newtype,ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+       call MPI_FILE_OPEN(newcomm, filename, &
+            MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
+            fh, ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+       filesize = 0_MPI_OFFSET_KIND
+       call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+       disp = 0_MPI_OFFSET_KIND
+       call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+            newtype,'native',MPI_INFO_NULL,ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+       call MPI_FILE_WRITE_ALL(fh, wk, &
+            subsizes(1)*subsizes(2)*subsizes(3), &
+            data_type, MPI_STATUS_IGNORE, ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+       call MPI_FILE_CLOSE(fh,ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+       call MPI_TYPE_FREE(newtype,ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+
+       deallocate(wk)
+
+    end if
+
+    call decomp_mpi_comm_free(newcomm)
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_write_subdomain")
+    if (decomp_profiler_io) call decomp_profiler_end("io_write_subdomain")
 #endif
 
-   end subroutine write_subdomain
+  end subroutine write_subdomain
 
-   subroutine decomp_2d_init_io(io_name)
+  subroutine decomp_2d_init_io(io_name)
 
-      implicit none
+    implicit none
 
-      character(len=*), intent(in) :: io_name
+    character(len=*), intent(in) :: io_name
 #ifdef ADIOS2
-      integer :: ierror
-      type(adios2_io) :: io
+    integer :: ierror
+    type(adios2_io) :: io
 #endif
-
-      if (nrank .eq. 0) then
-         print *, "Initialising IO for ", io_name
-      end if
+    
+    if (nrank .eq. 0) then
+       print *, "Initialising IO for ", io_name
+    end if
 
 #ifdef ADIOS2
-      if (adios%valid) then
-         call adios2_declare_io(io, adios, io_name, ierror)
-         if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_declare_io "//trim(io_name))
-      else
-         call decomp_2d_abort(__FILE__, __LINE__, -1, "couldn't declare IO - adios object not valid")
-      end if
-#endif
-
-   end subroutine decomp_2d_init_io
-
-   subroutine decomp_2d_open_io(io_name, io_dir, mode)
-
-      implicit none
-
-      character(len=*), intent(in) :: io_name, io_dir
-      integer, intent(in) :: mode
-
-      logical, dimension(:), pointer :: live_ptrh
-      character(len=1024), dimension(:), pointer :: names_ptr
-      character(len=(len(io_name) + len(io_sep) + len(io_dir))) :: full_name
-
-      integer :: idx, ierror
-      integer :: access_mode
+    if (adios%valid) then
+       call adios2_declare_io(io, adios, io_name, ierror)
+       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_declare_io "//trim(io_name))
+    else
+       call decomp_2d_abort(__FILE__, __LINE__, -1, "couldn't declare IO - adios object not valid")
+    end if
+#endif
+    
+  end subroutine decomp_2d_init_io
+  
+  subroutine decomp_2d_open_io(io_name, io_dir, mode)
+
+    implicit none
+
+    character(len=*), intent(in) :: io_name, io_dir
+    integer, intent(in) :: mode
+
+    logical, dimension(:), pointer :: live_ptrh
+    character(len=1024), dimension(:), pointer :: names_ptr
+    character(len=(len(io_name)+len(io_sep)+len(io_dir))) :: full_name
+    
+    integer :: idx, ierror
+    integer :: access_mode
 #ifndef ADIOS2
-      integer(MPI_OFFSET_KIND) :: filesize
+    integer(MPI_OFFSET_KIND) :: filesize
 #else
-      type(adios2_io) :: io
+    type(adios2_io) :: io
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_start("io_open_close")
+    if (decomp_profiler_io) call decomp_profiler_start("io_open_close")
 #endif
 
 #ifndef ADIOS2
-      live_ptrh => fh_live
-      names_ptr => fh_names
+    live_ptrh => fh_live
+    names_ptr => fh_names
 #else
-      live_ptrh => engine_live
-      names_ptr => engine_names
+    live_ptrh => engine_live
+    names_ptr => engine_names
 #endif
-
-      idx = get_io_idx(io_name, io_dir)
-      if (idx .lt. 1) then
+    
+    idx = get_io_idx(io_name, io_dir)
+    if (idx .lt. 1) then
        !! New io destination
-         if (nreg_io .lt. MAX_IOH) then
-            nreg_io = nreg_io + 1
-            do idx = 1, MAX_IOH
-               if (.not. live_ptrh(idx)) then
-                  live_ptrh(idx) = .true.
-                  exit
-               end if
-            end do
-
-            full_name = io_name//io_sep//io_dir
-            names_ptr(idx) = full_name
-
-            if (mode .eq. decomp_2d_write_mode) then
+       if (nreg_io .lt. MAX_IOH) then
+          nreg_io = nreg_io + 1
+          do idx = 1, MAX_IOH
+             if (.not. live_ptrh(idx)) then
+                live_ptrh(idx) = .true.
+                exit
+             end if
+          end do
+          
+          full_name = io_name//io_sep//io_dir
+          names_ptr(idx) = full_name
+
+          if (mode .eq. decomp_2d_write_mode) then
              !! Setup writers
 #ifndef ADIOS2
-               filesize = 0_MPI_OFFSET_KIND
-               fh_disp(idx) = 0_MPI_OFFSET_KIND
-               access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
+             filesize = 0_MPI_OFFSET_KIND
+             fh_disp(idx) = 0_MPI_OFFSET_KIND
+             access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
 #else
-               access_mode = adios2_mode_write
+             access_mode = adios2_mode_write
 #endif
-            else if (mode .eq. decomp_2d_read_mode) then
+          else if (mode .eq. decomp_2d_read_mode) then
              !! Setup readers
 #ifndef ADIOS2
-               fh_disp(idx) = 0_MPI_OFFSET_KIND
-               access_mode = MPI_MODE_RDONLY
+             fh_disp(idx) = 0_MPI_OFFSET_KIND
+             access_mode = MPI_MODE_RDONLY
 #else
-               access_mode = adios2_mode_read
+             access_mode = adios2_mode_read
 #endif
-            else if (mode .eq. decomp_2d_append_mode) then
+          else if (mode .eq. decomp_2d_append_mode) then
 #ifndef ADIOS2
-               filesize = 0_MPI_OFFSET_KIND
-               fh_disp(idx) = 0_MPI_OFFSET_KIND
-               access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
+             filesize = 0_MPI_OFFSET_KIND
+             fh_disp(idx) = 0_MPI_OFFSET_KIND
+             access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
 #else
-               access_mode = adios2_mode_append
+             access_mode = adios2_mode_append
 #endif
-            else
-               print *, "ERROR: Unknown mode!"
-               stop
-            end if
+          else
+             print *, "ERROR: Unknown mode!"
+             stop
+          endif
 
           !! Open IO
 #ifndef ADIOS2
-            call MPI_FILE_OPEN(decomp_2d_comm, io_dir, &
-                               access_mode, MPI_INFO_NULL, &
-                               fh_registry(idx), ierror)
-            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
-            if (mode .eq. decomp_2d_write_mode) then
+          call MPI_FILE_OPEN(decomp_2d_comm, io_dir, &
+               access_mode, MPI_INFO_NULL, &
+               fh_registry(idx), ierror)
+          if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+          if (mode .eq. decomp_2d_write_mode) then
              !! Guarantee overwriting
-               call MPI_FILE_SET_SIZE(fh_registry(idx), filesize, ierror)
-               if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
-            end if
+             call MPI_FILE_SET_SIZE(fh_registry(idx), filesize, ierror)
+             if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+          end if
 #else
-            call adios2_at_io(io, adios, io_name, ierror)
-            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-            if (io%valid) then
-               call adios2_open(engine_registry(idx), io, trim(gen_iodir_name(io_dir, io_name)), access_mode, ierror)
-               if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "ERROR opening engine!")
-            else
-               call decomp_2d_abort(__FILE__, __LINE__, -1, "Couldn't find IO handle")
-            end if
-#endif
-         end if
-      end if
-
-   end subroutine decomp_2d_open_io
+          call adios2_at_io(io, adios, io_name, ierror)
+          if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+          if (io%valid) then
+             call adios2_open(engine_registry(idx), io, trim(gen_iodir_name(io_dir, io_name)), access_mode, ierror)
+             if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "ERROR opening engine!")
+          else
+             call decomp_2d_abort(__FILE__, __LINE__, -1, "Couldn't find IO handle")
+          end if
+#endif
+       end if
+    end if
 
-   subroutine decomp_2d_close_io(io_name, io_dir)
+  end subroutine decomp_2d_open_io
 
-      implicit none
+  subroutine decomp_2d_close_io(io_name, io_dir)
 
-      character(len=*), intent(in) :: io_name, io_dir
+    implicit none
 
-      character(len=1024), dimension(:), pointer :: names_ptr
-      logical, dimension(:), pointer :: live_ptrh
-      integer :: idx, ierror
+    character(len=*), intent(in) :: io_name, io_dir
+    
+    character(len=1024), dimension(:), pointer :: names_ptr
+    logical, dimension(:), pointer :: live_ptrh
+    integer :: idx, ierror
 
-      idx = get_io_idx(io_name, io_dir)
+    idx = get_io_idx(io_name, io_dir)
 #ifndef ADIOS2
-      names_ptr => fh_names
-      live_ptrh => fh_live
-      call MPI_FILE_CLOSE(fh_registry(idx), ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+    names_ptr => fh_names
+    live_ptrh => fh_live
+    call MPI_FILE_CLOSE(fh_registry(idx), ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
 #else
-      names_ptr => engine_names
-      live_ptrh => engine_live
-      call adios2_close(engine_registry(idx), ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_close")
+    names_ptr => engine_names
+    live_ptrh => engine_live
+    call adios2_close(engine_registry(idx), ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_close")
 #endif
-      names_ptr(idx) = ""
-      live_ptrh(idx) = .false.
-      nreg_io = nreg_io - 1
+    names_ptr(idx) = ""
+    live_ptrh(idx) = .false.
+    nreg_io = nreg_io - 1
 
 #ifdef PROFILER
-      if (decomp_profiler_io) call decomp_profiler_end("io_open_close")
+    if (decomp_profiler_io) call decomp_profiler_end("io_open_close")
 #endif
 
-   end subroutine decomp_2d_close_io
+  end subroutine decomp_2d_close_io
 
-   subroutine decomp_2d_start_io(io_name, io_dir)
+  subroutine decomp_2d_start_io(io_name, io_dir)
 
-      implicit none
+    implicit none
 
-      character(len=*), intent(in) :: io_name, io_dir
+    character(len=*), intent(in) :: io_name, io_dir
 #ifdef ADIOS2
-      integer :: idx, ierror
-
-      idx = get_io_idx(io_name, io_dir)
-      associate (engine => engine_registry(idx))
-         if (engine%valid) then
-            call adios2_begin_step(engine, ierror)
-            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_begin_step")
-         else
-            call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to begin step with invalid engine")
-         end if
-      end associate
+    integer :: idx, ierror
+
+    idx = get_io_idx(io_name, io_dir)
+    associate(engine => engine_registry(idx))
+      if (engine%valid) then
+         call adios2_begin_step(engine, ierror)
+         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_begin_step")
+      else
+         call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to begin step with invalid engine")
+      end if
+    end associate
 #else
-      associate (nm => io_name, dr => io_dir) ! Silence unused dummy argument
-      end associate
+    associate(nm => io_name, dr => io_dir) ! Silence unused dummy argument
+    end associate
 #endif
+    
+  end subroutine decomp_2d_start_io
 
-   end subroutine decomp_2d_start_io
+  subroutine decomp_2d_end_io(io_name, io_dir)
 
-   subroutine decomp_2d_end_io(io_name, io_dir)
+    implicit none
 
-      implicit none
-
-      character(len=*), intent(in) :: io_name, io_dir
+    character(len=*), intent(in) :: io_name, io_dir
 #ifdef ADIOS2
-      integer :: idx, ierror
-
-      idx = get_io_idx(io_name, io_dir)
-      associate (engine => engine_registry(idx))
-         if (engine%valid) then
-            call adios2_end_step(engine, ierror)
-            if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_end_step")
-         else
-            call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to end step with invalid engine")
-         end if
-      end associate
+    integer :: idx, ierror
+
+    idx = get_io_idx(io_name, io_dir)
+    associate(engine => engine_registry(idx))
+      if (engine%valid) then
+         call adios2_end_step(engine, ierror)
+         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_end_step")
+      else
+         call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to end step with invalid engine")
+      end if
+    end associate
 #else
-      associate (nm => io_name, dr => io_dir) ! Silence unused dummy argument
-      end associate
+    associate(nm => io_name, dr => io_dir) ! Silence unused dummy argument
+    end associate
 #endif
 
-   end subroutine decomp_2d_end_io
-
-   integer function get_io_idx(io_name, engine_name)
+  end subroutine decomp_2d_end_io
+  
+  integer function get_io_idx(io_name, engine_name)
 
-      implicit none
+    implicit none
 
-      character(len=*), intent(in) :: io_name
-      character(len=*), intent(in) :: engine_name
+    character(len=*), intent(in) :: io_name
+    character(len=*), intent(in) :: engine_name
 
-      character(len=(len(io_name) + len(io_sep) + len(engine_name))) :: full_name
-      integer :: idx
-      logical :: found
+    character(len=(len(io_name)+len(io_sep)+len(engine_name))) :: full_name
+    integer :: idx
+    logical :: found
 
-      character(len=1024), dimension(:), pointer :: names_ptr
+    character(len=1024), dimension(:), pointer :: names_ptr
 
 #ifndef ADIOS2
-      names_ptr => fh_names
+    names_ptr => fh_names
 #else
-      names_ptr => engine_names
+    names_ptr => engine_names
 #endif
 
-      full_name = io_name//io_sep//engine_name
-
-      found = .false.
-      do idx = 1, MAX_IOH
-         if (names_ptr(idx) .eq. full_name) then
-            found = .true.
-            exit
-         end if
-      end do
-
-      if (.not. found) then
-         idx = -1
-      end if
+    full_name = io_name//io_sep//engine_name
+    
+    found = .false.
+    do idx = 1, MAX_IOH
+       if (names_ptr(idx) .eq. full_name) then
+          found = .true.
+          exit
+       end if
+    end do
 
-      get_io_idx = idx
+    if (.not. found) then
+       idx = -1
+    end if
 
-   end function get_io_idx
+    get_io_idx = idx
+    
+  end function get_io_idx
 
-   function gen_iodir_name(io_dir, io_name)
+  function gen_iodir_name(io_dir, io_name)
 
-      character(len=*), intent(in) :: io_dir, io_name
-      character(len=(len(io_dir) + 5)) :: gen_iodir_name
+    character(len=*), intent(in) :: io_dir, io_name
+    character(len=(len(io_dir) + 5)) :: gen_iodir_name
 #ifdef ADIOS2
-      integer :: ierror
-      type(adios2_io) :: io
-      character(len=5) :: ext
+    integer :: ierror
+    type(adios2_io) :: io
+    character(len=5) :: ext
 #endif
 
 #ifndef ADIOS2
-      associate (nm => io_name) ! Silence unused dummy argument
-      end associate
-      write (gen_iodir_name, "(A)") io_dir
+    associate(nm => io_name) ! Silence unused dummy argument
+    end associate
+    write(gen_iodir_name, "(A)") io_dir
 #else
-      call adios2_at_io(io, adios, io_name, ierror)
-      if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-      if (io%engine_type .eq. "BP4") then
-         ext = ".bp4"
-      else if (io%engine_type .eq. "HDF5") then
-         ext = ".hdf5"
-      else if (io%engine_type .eq. "SST") then
-         ext = ""
-      else
-         print *, "ERROR: Unkown engine type! ", io%engine_type
-         print *, "-  IO: ", io_name
-         print *, "- DIR:", io_dir
-         stop
-      end if
-      write (gen_iodir_name, "(A,A)") io_dir, trim(ext)
-#endif
-
-   end function gen_iodir_name
+    call adios2_at_io(io, adios, io_name, ierror)
+    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+    if (io%engine_type .eq. "BP4") then
+       ext = ".bp4"
+    else if (io%engine_type .eq. "HDF5") then
+       ext = ".hdf5"
+    else if (io%engine_type .eq. "SST") then
+       ext = ""
+    else
+       print *, "ERROR: Unkown engine type! ", io%engine_type
+       print *, "-  IO: ", io_name
+       print *, "- DIR:", io_dir
+       stop
+    endif
+    write(gen_iodir_name, "(A,A)") io_dir, trim(ext)
+#endif
+    
+  end function gen_iodir_name
 
 end module decomp_2d_io
diff --git a/src/io_read_inflow.f90 b/src/io_read_inflow.f90
index 4d2df417..3e5a9fb1 100644
--- a/src/io_read_inflow.f90
+++ b/src/io_read_inflow.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,12 +9,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines
+! This file contain common code to be included by subroutines 
 ! 'read_var_...' in io.f90
 
-! Using MPI-IO to read a distributed 3D variable from a file. File
-! operations (open/close) need to be done in calling application. This
-! allows multiple variables to be read from a single file. Together
+! Using MPI-IO to read a distributed 3D variable from a file. File 
+! operations (open/close) need to be done in calling application. This 
+! allows multiple variables to be read from a single file. Together 
 ! with the corresponding write operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -25,62 +25,62 @@
 end if
 
 ! Create file type and set file view
-sizes(1) = ntimesteps
+sizes(1) = ntimesteps 
 sizes(2) = decomp%ysz(2)
 sizes(3) = decomp%zsz(3)
 subsizes(1) = ntimesteps
 subsizes(2) = decomp%xsz(2)
 subsizes(3) = decomp%xsz(3)
 starts(1) = 0  ! 0-based index
-starts(2) = decomp%xst(2) - 1
-starts(3) = decomp%xst(3) - 1
+starts(2) = decomp%xst(2)-1
+starts(3) = decomp%xst(3)-1
 
 idx = get_io_idx(io_name, dirname)
 
 #ifndef ADIOS2
 !! Use default MPIIO
-associate (fh => fh_registry(idx), &
-           disp => fh_disp(idx))
-   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-   call MPI_TYPE_COMMIT(newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                          newtype, 'native', MPI_INFO_NULL, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-   call MPI_FILE_READ_ALL(fh, var, &
-                          subsizes(1)*subsizes(2)*subsizes(3), &
-                          data_type, MPI_STATUS_IGNORE, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-   call MPI_TYPE_FREE(newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+associate(fh=>fh_registry(idx), &
+  disp => fh_disp(idx))
+  call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+  call MPI_TYPE_COMMIT(newtype,ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+  call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+       newtype,'native',MPI_INFO_NULL,ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+  call MPI_FILE_READ_ALL(fh, var, &
+       subsizes(1)*subsizes(2)*subsizes(3), &
+       data_type, MPI_STATUS_IGNORE, ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+  call MPI_TYPE_FREE(newtype,ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-   ! update displacement for the next read operation
-   disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-          *int(sizes(2), kind=MPI_OFFSET_KIND) &
-          *int(sizes(3), kind=MPI_OFFSET_KIND) &
-          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
-   if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+  ! update displacement for the next read operation
+  disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+              * int(sizes(2), kind=MPI_OFFSET_KIND) &
+              * int(sizes(3), kind=MPI_OFFSET_KIND) &
+              * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+  if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                             * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                             * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
-associate (vnm => varname) ! Silence unused argument
+associate(vnm => varname) ! Silence unused argument
 end associate
 
 #else
 !! Use ADIOS2
 call adios2_at_io(io_handle, adios, io_name, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-                                                 "trying to write variable before registering! "//trim(varname))
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+   "trying to write variable before registering! "//trim(varname))
 
 !! Note - need to use sync mode as we are using a view into the array - unsure how this works with deferred writes
 ! call adios2_set_step_selection(var_handle, int(0, kind=8), int(1, kind=8), ierror)
 call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_sync, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
 #endif
diff --git a/src/io_read_one.inc b/src/io_read_one.inc
index 8b8c2f5c..211cbc2c 100644
--- a/src/io_read_one.inc
+++ b/src/io_read_one.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-
+!                          -*- mode: f90 -*-                            
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,7 +10,7 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines
+! This file contain common code to be included by subroutines 
 ! 'mpiio_read_one_...' in io.f90
 
 ! Using MPI-IO to write a distributed 3D array into a file
@@ -30,43 +30,43 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1) - 1  ! 0-based index
-   starts(2) = decomp%xst(2) - 1
-   starts(3) = decomp%xst(3) - 1
+   starts(1) = decomp%xst(1)-1  ! 0-based index
+   starts(2) = decomp%xst(2)-1
+   starts(3) = decomp%xst(3)-1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1) - 1
-   starts(2) = decomp%yst(2) - 1
-   starts(3) = decomp%yst(3) - 1
+   starts(1) = decomp%yst(1)-1
+   starts(2) = decomp%yst(2)-1
+   starts(3) = decomp%yst(3)-1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1) - 1
-   starts(2) = decomp%zst(2) - 1
-   starts(3) = decomp%zst(3) - 1
-end if
+   starts(1) = decomp%zst(1)-1
+   starts(2) = decomp%zst(2)-1
+   starts(3) = decomp%zst(3)-1
+endif
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
 call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-                   MPI_MODE_RDONLY, MPI_INFO_NULL, &
-                   fh, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+     MPI_MODE_RDONLY, MPI_INFO_NULL, &
+     fh, ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
 disp = 0_MPI_OFFSET_KIND
-call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                       newtype, 'native', MPI_INFO_NULL, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+     newtype,'native',MPI_INFO_NULL,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_READ_ALL(fh, var, &
-                       subsizes(1)*subsizes(2)*subsizes(3), &
-                       data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-call MPI_FILE_CLOSE(fh, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-call MPI_TYPE_FREE(newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+     subsizes(1)*subsizes(2)*subsizes(3), &
+     data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+call MPI_FILE_CLOSE(fh,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+call MPI_TYPE_FREE(newtype,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
diff --git a/src/io_read_var.inc b/src/io_read_var.inc
index 816dd6eb..489247cb 100644
--- a/src/io_read_var.inc
+++ b/src/io_read_var.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-
+!                          -*- mode: f90 -*-                            
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,12 +10,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines
+! This file contain common code to be included by subroutines 
 ! 'read_var_...' in io.f90
 
-! Using MPI-IO to read a distributed 3D variable from a file. File
-! operations (open/close) need to be done in calling application. This
-! allows multiple variables to be read from a single file. Together
+! Using MPI-IO to read a distributed 3D variable from a file. File 
+! operations (open/close) need to be done in calling application. This 
+! allows multiple variables to be read from a single file. Together 
 ! with the corresponding write operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -33,46 +33,46 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1) - 1  ! 0-based index
-   starts(2) = decomp%xst(2) - 1
-   starts(3) = decomp%xst(3) - 1
+   starts(1) = decomp%xst(1)-1  ! 0-based index
+   starts(2) = decomp%xst(2)-1
+   starts(3) = decomp%xst(3)-1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1) - 1
-   starts(2) = decomp%yst(2) - 1
-   starts(3) = decomp%yst(3) - 1
+   starts(1) = decomp%yst(1)-1
+   starts(2) = decomp%yst(2)-1
+   starts(3) = decomp%yst(3)-1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1) - 1
-   starts(2) = decomp%zst(2) - 1
-   starts(3) = decomp%zst(3) - 1
-end if
+   starts(1) = decomp%zst(1)-1
+   starts(2) = decomp%zst(2)-1
+   starts(3) = decomp%zst(3)-1
+endif
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                       newtype, 'native', MPI_INFO_NULL, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+     newtype,'native',MPI_INFO_NULL,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_READ_ALL(fh, var, &
-                       subsizes(1)*subsizes(2)*subsizes(3), &
-                       data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-call MPI_TYPE_FREE(newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+     subsizes(1)*subsizes(2)*subsizes(3), &
+     data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+call MPI_TYPE_FREE(newtype,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 ! update displacement for the next read operation
 disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-       *int(sizes(2), kind=MPI_OFFSET_KIND) &
-       *int(sizes(3), kind=MPI_OFFSET_KIND) &
-       *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+            * int(sizes(2), kind=MPI_OFFSET_KIND) &
+            * int(sizes(3), kind=MPI_OFFSET_KIND) &
+            * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                      *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                      *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                      *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                           * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                           * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                           * int(mytype_bytes, kind=MPI_OFFSET_KIND)
diff --git a/src/io_write_every.inc b/src/io_write_every.inc
index a0fe74d3..cbf98b7a 100644
--- a/src/io_write_every.inc
+++ b/src/io_write_every.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-
+!                          -*- mode: f90 -*-                            
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,12 +10,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines
+! This file contain common code to be included by subroutines 
 ! 'write_every_...' in io.f90
 
 ! To write every few points of a 3D array to a file
 
-! work out the distribution parameters, which may be different from
+! work out the distribution parameters, which may be different from 
 ! the default distribution used by the decomposition library
 !  For exmample if nx=17 and p_row=4
 !    distribution is: 4 4 4 5
@@ -38,199 +38,199 @@
 !             4th block (13-17) contains then 12th & 15th point
 !    giving a 1 2 2 1 distribution
 
-skip(1) = iskip
-skip(2) = jskip
-skip(3) = kskip
+skip(1)=iskip
+skip(2)=jskip
+skip(3)=kskip
 
-do i = 1, 3
+do i=1,3
    if (from1) then
-      xst(i) = (xstart(i) + skip(i) - 1)/skip(i)
-      if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xst(i) = xst(i) + 1
-      xen(i) = (xend(i) + skip(i) - 1)/skip(i)
+      xst(i) = (xstart(i)+skip(i)-1)/skip(i)
+      if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xst(i)=xst(i)+1
+      xen(i) = (xend(i)+skip(i)-1)/skip(i)
    else
       xst(i) = xstart(i)/skip(i)
-      if (mod(xstart(i), skip(i)) /= 0) xst(i) = xst(i) + 1
+      if (mod(xstart(i),skip(i))/=0) xst(i)=xst(i)+1
       xen(i) = xend(i)/skip(i)
    end if
-   xsz(i) = xen(i) - xst(i) + 1
+   xsz(i) = xen(i)-xst(i)+1
 end do
 
-do i = 1, 3
+do i=1,3
    if (from1) then
-      yst(i) = (ystart(i) + skip(i) - 1)/skip(i)
-      if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) yst(i) = yst(i) + 1
-      yen(i) = (yend(i) + skip(i) - 1)/skip(i)
+      yst(i) = (ystart(i)+skip(i)-1)/skip(i)
+      if (mod(ystart(i)+skip(i)-1,skip(i))/=0) yst(i)=yst(i)+1
+      yen(i) = (yend(i)+skip(i)-1)/skip(i)
    else
       yst(i) = ystart(i)/skip(i)
-      if (mod(ystart(i), skip(i)) /= 0) yst(i) = yst(i) + 1
+      if (mod(ystart(i),skip(i))/=0) yst(i)=yst(i)+1
       yen(i) = yend(i)/skip(i)
    end if
-   ysz(i) = yen(i) - yst(i) + 1
+   ysz(i) = yen(i)-yst(i)+1
 end do
 
-do i = 1, 3
+do i=1,3
    if (from1) then
-      zst(i) = (zstart(i) + skip(i) - 1)/skip(i)
-      if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zst(i) = zst(i) + 1
-      zen(i) = (zend(i) + skip(i) - 1)/skip(i)
+      zst(i) = (zstart(i)+skip(i)-1)/skip(i)
+      if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zst(i)=zst(i)+1
+      zen(i) = (zend(i)+skip(i)-1)/skip(i)
    else
       zst(i) = zstart(i)/skip(i)
-      if (mod(zstart(i), skip(i)) /= 0) zst(i) = zst(i) + 1
+      if (mod(zstart(i),skip(i))/=0) zst(i)=zst(i)+1
       zen(i) = zend(i)/skip(i)
    end if
-   zsz(i) = zen(i) - zst(i) + 1
+   zsz(i) = zen(i)-zst(i)+1
 end do
 
-! if 'skip' value is large it is possible that some ranks do not
-! contain any points to be written. Subarray constructor requires
+! if 'skip' value is large it is possible that some ranks do not 
+! contain any points to be written. Subarray constructor requires 
 ! nonzero size so it is not possible to use decomp_2d_comm for IO.
 ! Create a sub communicator for this...
 color = 1
 key = 0  ! rank order doesn't matter
-if (ipencil == 1) then
-   if (xsz(1) == 0 .or. xsz(2) == 0 .or. xsz(3) == 0) then
+if (ipencil==1) then
+   if (xsz(1)==0 .or. xsz(2)==0 .or. xsz(3)==0) then
       color = 2
    end if
-else if (ipencil == 2) then
-   if (ysz(1) == 0 .or. ysz(2) == 0 .or. ysz(3) == 0) then
+else if (ipencil==2) then
+   if (ysz(1)==0 .or. ysz(2)==0 .or. ysz(3)==0) then
       color = 2
    end if
-else if (ipencil == 3) then
-   if (zsz(1) == 0 .or. zsz(2) == 0 .or. zsz(3) == 0) then
+else if (ipencil==3) then
+   if (zsz(1)==0 .or. zsz(2)==0 .or. zsz(3)==0) then
       color = 2
    end if
 end if
-call MPI_COMM_SPLIT(decomp_2d_comm, color, key, newcomm, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+call MPI_COMM_SPLIT(decomp_2d_comm,color,key,newcomm,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
 
-if (color == 1) then ! only ranks in this group do IO collectively
+if (color==1) then ! only ranks in this group do IO collectively
 
    ! generate subarray information
    sizes(1) = xsz(1)
    sizes(2) = ysz(2)
    sizes(3) = zsz(3)
-   if (ipencil == 1) then
+   if (ipencil==1) then
       subsizes(1) = xsz(1)
       subsizes(2) = xsz(2)
       subsizes(3) = xsz(3)
-      starts(1) = xst(1) - 1
-      starts(2) = xst(2) - 1
-      starts(3) = xst(3) - 1
-   else if (ipencil == 2) then
+      starts(1) = xst(1)-1
+      starts(2) = xst(2)-1
+      starts(3) = xst(3)-1
+   else if (ipencil==2) then
       subsizes(1) = ysz(1)
       subsizes(2) = ysz(2)
       subsizes(3) = ysz(3)
-      starts(1) = yst(1) - 1
-      starts(2) = yst(2) - 1
-      starts(3) = yst(3) - 1
-   else if (ipencil == 3) then
+      starts(1) = yst(1)-1
+      starts(2) = yst(2)-1
+      starts(3) = yst(3)-1
+   else if (ipencil==3) then
       subsizes(1) = zsz(1)
       subsizes(2) = zsz(2)
       subsizes(3) = zsz(3)
-      starts(1) = zst(1) - 1
-      starts(2) = zst(2) - 1
-      starts(3) = zst(3) - 1
+      starts(1) = zst(1)-1
+      starts(2) = zst(2)-1
+      starts(3) = zst(3)-1
    end if
 
    ! copy data from original array
-   ! needs a copy of original array in global coordinate
-   if (ipencil == 1) then
-      allocate (wk(xst(1):xen(1), xst(2):xen(2), xst(3):xen(3)))
-      allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      wk2 = var
+   ! needs a copy of original array in global coordinate 
+   if (ipencil==1) then
+      allocate(wk(xst(1):xen(1),xst(2):xen(2),xst(3):xen(3)))
+      allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
+      wk2=var
       if (from1) then
-         do k = xst(3), xen(3)
-            do j = xst(2), xen(2)
-               do i = xst(1), xen(1)
-                  wk(i, j, k) = wk2((i - 1)*iskip + 1, (j - 1)*jskip + 1, (k - 1)*kskip + 1)
+         do k=xst(3),xen(3)
+            do j=xst(2),xen(2)
+               do i=xst(1),xen(1)
+                  wk(i,j,k) = wk2((i-1)*iskip+1,(j-1)*jskip+1,(k-1)*kskip+1)
                end do
             end do
          end do
       else
-         do k = xst(3), xen(3)
-            do j = xst(2), xen(2)
-               do i = xst(1), xen(1)
-                  wk(i, j, k) = wk2(i*iskip, j*jskip, k*kskip)
+         do k=xst(3),xen(3)
+            do j=xst(2),xen(2)
+               do i=xst(1),xen(1)
+                  wk(i,j,k) = wk2(i*iskip,j*jskip,k*kskip)
                end do
             end do
          end do
       end if
-   else if (ipencil == 2) then
-      allocate (wk(yst(1):yen(1), yst(2):yen(2), yst(3):yen(3)))
-      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      wk2 = var
+   else if (ipencil==2) then
+      allocate(wk(yst(1):yen(1),yst(2):yen(2),yst(3):yen(3)))
+      allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
+      wk2=var
       if (from1) then
-         do k = yst(3), yen(3)
-            do j = yst(2), yen(2)
-               do i = yst(1), yen(1)
-                  wk(i, j, k) = wk2((i - 1)*iskip + 1, (j - 1)*jskip + 1, (k - 1)*kskip + 1)
+         do k=yst(3),yen(3)
+            do j=yst(2),yen(2)
+               do i=yst(1),yen(1)
+                  wk(i,j,k) = wk2((i-1)*iskip+1,(j-1)*jskip+1,(k-1)*kskip+1)
                end do
             end do
          end do
       else
-         do k = yst(3), yen(3)
-            do j = yst(2), yen(2)
-               do i = yst(1), yen(1)
-                  wk(i, j, k) = wk2(i*iskip, j*jskip, k*kskip)
+         do k=yst(3),yen(3)
+            do j=yst(2),yen(2)
+               do i=yst(1),yen(1)
+                  wk(i,j,k) = wk2(i*iskip,j*jskip,k*kskip)
                end do
             end do
          end do
       end if
-   else if (ipencil == 3) then
-      allocate (wk(zst(1):zen(1), zst(2):zen(2), zst(3):zen(3)))
-      allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-      wk2 = var
+   else if (ipencil==3) then
+      allocate(wk(zst(1):zen(1),zst(2):zen(2),zst(3):zen(3)))
+      allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
+      wk2=var
       if (from1) then
-         do k = zst(3), zen(3)
-            do j = zst(2), zen(2)
-               do i = zst(1), zen(1)
-                  wk(i, j, k) = wk2((i - 1)*iskip + 1, (j - 1)*jskip + 1, (k - 1)*kskip + 1)
+         do k=zst(3),zen(3)
+            do j=zst(2),zen(2)
+               do i=zst(1),zen(1)
+                  wk(i,j,k) = wk2((i-1)*iskip+1,(j-1)*jskip+1,(k-1)*kskip+1)
                end do
             end do
          end do
       else
-         do k = zst(3), zen(3)
-            do j = zst(2), zen(2)
-               do i = zst(1), zen(1)
-                  wk(i, j, k) = wk2(i*iskip, j*jskip, k*kskip)
+         do k=zst(3),zen(3)
+            do j=zst(2),zen(2)
+               do i=zst(1),zen(1)
+                  wk(i,j,k) = wk2(i*iskip,j*jskip,k*kskip)
                end do
             end do
          end do
       end if
    end if
-   deallocate (wk2)
+   deallocate(wk2)
 
    ! MPI-IO
-   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-   call MPI_TYPE_COMMIT(newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+        MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+   call MPI_TYPE_COMMIT(newtype,ierror)
+   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
    call MPI_FILE_OPEN(newcomm, filename, &
-                      MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
-                      fh, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+        MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
+        fh, ierror)
+   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
    filesize = 0_MPI_OFFSET_KIND
-   call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+   call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
+   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
    disp = 0_MPI_OFFSET_KIND
-   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                          newtype, 'native', MPI_INFO_NULL, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+   call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+        newtype,'native',MPI_INFO_NULL,ierror)
+   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
    call MPI_FILE_WRITE_ALL(fh, wk, &
-                           subsizes(1)*subsizes(2)*subsizes(3), &
-                           data_type, MPI_STATUS_IGNORE, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-   call MPI_FILE_CLOSE(fh, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-   call MPI_TYPE_FREE(newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+        subsizes(1)*subsizes(2)*subsizes(3), &
+        data_type, MPI_STATUS_IGNORE, ierror)
+   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+   call MPI_FILE_CLOSE(fh,ierror)
+   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+   call MPI_TYPE_FREE(newtype,ierror)
+   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-   deallocate (wk)
+   deallocate(wk)
 
 end if ! color==1
 
 call decomp_mpi_comm_free(newcomm)
 
 call MPI_BARRIER(decomp_2d_comm, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
diff --git a/src/io_write_one.inc b/src/io_write_one.inc
index e3972fb3..29facac5 100644
--- a/src/io_write_one.inc
+++ b/src/io_write_one.inc
@@ -1,17 +1,17 @@
-!                          -*- mode: f90 -*-
+!                          -*- mode: f90 -*-                            
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
+! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG) 
 ! Copyright (C) 2021               the University of Edinburgh (UoE)
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines
+! This file contain common code to be included by subroutines 
 ! 'mpiio_write_one_...' in io.f90
 
 ! Using MPI-IO to write a distributed 3D array into a file
@@ -31,67 +31,67 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1) - 1  ! 0-based index
-   starts(2) = decomp%xst(2) - 1
-   starts(3) = decomp%xst(3) - 1
+   starts(1) = decomp%xst(1)-1  ! 0-based index
+   starts(2) = decomp%xst(2)-1
+   starts(3) = decomp%xst(3)-1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1) - 1
-   starts(2) = decomp%yst(2) - 1
-   starts(3) = decomp%yst(3) - 1
+   starts(1) = decomp%yst(1)-1
+   starts(2) = decomp%yst(2)-1
+   starts(3) = decomp%yst(3)-1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1) - 1
-   starts(2) = decomp%zst(2) - 1
-   starts(3) = decomp%zst(3) - 1
-end if
+   starts(1) = decomp%zst(1)-1
+   starts(2) = decomp%zst(2)-1
+   starts(3) = decomp%zst(3)-1
+endif
 
 #ifdef T3PIO
 call MPI_INFO_CREATE(info, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_CREATE")
-gs = ceiling(real(sizes(1), mytype)*real(sizes(2), mytype)* &
-             real(sizes(3), mytype)/1024./1024.)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_CREATE")
+gs = ceiling(real(sizes(1),mytype)*real(sizes(2),mytype)* &
+     real(sizes(3),mytype)/1024./1024.)
 call t3pio_set_info(decomp_2d_comm, info, "./", ierror, &
-                    GLOBAL_SIZE=gs, factor=1)
+     GLOBAL_SIZE=gs, factor=1)
 #else
 gs = 1 ! Silence unused variable
 info = 1 ! Silence unused variable
 #endif
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
 #ifdef T3PIO
 call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-                   MPI_MODE_CREATE + MPI_MODE_WRONLY, info, fh, ierror)
+     MPI_MODE_CREATE+MPI_MODE_WRONLY, info, fh, ierror)
 #else
 call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-                   MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
-                   fh, ierror)
+     MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
+     fh, ierror)
 #endif
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
 filesize = 0_MPI_OFFSET_KIND
-call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
 disp = 0_MPI_OFFSET_KIND
-call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                       newtype, 'native', MPI_INFO_NULL, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+     newtype,'native',MPI_INFO_NULL,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_WRITE_ALL(fh, var, &
-                        subsizes(1)*subsizes(2)*subsizes(3), &
-                        data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-call MPI_FILE_CLOSE(fh, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-call MPI_TYPE_FREE(newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+     subsizes(1)*subsizes(2)*subsizes(3), &
+     data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+call MPI_FILE_CLOSE(fh,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+call MPI_TYPE_FREE(newtype,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef T3PIO
-call MPI_INFO_FREE(info, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_FREE")
+call MPI_INFO_FREE(info,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_INFO_FREE")
 #endif
diff --git a/src/io_write_outflow.f90 b/src/io_write_outflow.f90
index 4f6f3845..ccbd0314 100644
--- a/src/io_write_outflow.f90
+++ b/src/io_write_outflow.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,12 +9,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines
+! This file contain common code to be included by subroutines 
 ! 'write_var_...' in io.f90
 
-! Using MPI-IO to write a distributed 3D variable to a file. File
+! Using MPI-IO to write a distributed 3D variable to a file. File 
 ! operations (open/close) need to be done in calling application. This
-! allows multiple variables to be written to a single file. Together
+! allows multiple variables to be written to a single file. Together 
 ! with the corresponding read operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -25,62 +25,62 @@
 end if
 
 ! Create file type and set file view
-sizes(1) = ntimesteps
+sizes(1) = ntimesteps 
 sizes(2) = decomp%ysz(2)
 sizes(3) = decomp%zsz(3)
-subsizes(1) = ntimesteps
+subsizes(1) = ntimesteps 
 subsizes(2) = decomp%xsz(2)
 subsizes(3) = decomp%xsz(3)
 starts(1) = 0  ! 0-based index
-starts(2) = decomp%xst(2) - 1
-starts(3) = decomp%xst(3) - 1
+starts(2) = decomp%xst(2)-1
+starts(3) = decomp%xst(3)-1
 
 idx = get_io_idx(io_name, dirname)
 
 #ifndef ADIOS2
 !! Use default MPIIO
-associate (fh => fh_registry(idx), &
-           disp => fh_disp(idx))
-   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-   call MPI_TYPE_COMMIT(newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                          newtype, 'native', MPI_INFO_NULL, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-   call MPI_FILE_WRITE_ALL(fh, var, &
-                           subsizes(1)*subsizes(2)*subsizes(3), &
-                           data_type, MPI_STATUS_IGNORE, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-   call MPI_TYPE_FREE(newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+associate(fh=>fh_registry(idx), &
+     disp=>fh_disp(idx))
+  call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+  call MPI_TYPE_COMMIT(newtype,ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+  call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+       newtype,'native',MPI_INFO_NULL,ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+  call MPI_FILE_WRITE_ALL(fh, var, &
+       subsizes(1)*subsizes(2)*subsizes(3), &
+       data_type, MPI_STATUS_IGNORE, ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+  call MPI_TYPE_FREE(newtype,ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-   ! update displacement for the next write operation
-   disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-          *int(sizes(2), kind=MPI_OFFSET_KIND) &
-          *int(sizes(3), kind=MPI_OFFSET_KIND) &
-          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
-   if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+  ! update displacement for the next write operation
+  disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+              * int(sizes(2), kind=MPI_OFFSET_KIND) &
+              * int(sizes(3), kind=MPI_OFFSET_KIND) &
+              * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+  if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                             * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                             * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
-associate (vnm => varname) ! Silence unused dummy argument
+associate(vnm => varname) ! Silence unused dummy argument
 end associate
 
 #else
 !! Use ADIOS2
 call adios2_at_io(io_handle, adios, io_name, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not. var_handle%valid) then
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+if (.not.var_handle % valid) then
    call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to write variable before registering!"//trim(varname))
-end if
+endif
 
 !! Note - need to use sync mode as we are using a view into the array - unsure how this works with deferred writes
 call adios2_put(engine_registry(idx), var_handle, var, adios2_mode_sync, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
 #endif
diff --git a/src/io_write_plane.inc b/src/io_write_plane.inc
index b48db97d..088fe097 100644
--- a/src/io_write_plane.inc
+++ b/src/io_write_plane.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-
+!                          -*- mode: f90 -*-                            
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,7 +10,7 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines
+! This file contain common code to be included by subroutines 
 ! 'mpiio_write_plane_3d_...' in io.f90
 
 ! It is much easier to implement if all mpi ranks participate I/O.
@@ -24,81 +24,81 @@ end if
 
 opened_new = .false.
 
-if (iplane == 1) then
-   allocate (wk(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)))
-   if (ipencil == 1) then
+if (iplane==1) then
+   allocate(wk(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)))
+   if (ipencil==1) then
       wk = var
-   else if (ipencil == 2) then
-      call transpose_y_to_x(var, wk, decomp)
-   else if (ipencil == 3) then
-      allocate (wk2(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)))
-      call transpose_z_to_y(var, wk2, decomp)
-      call transpose_y_to_x(wk2, wk, decomp)
-      deallocate (wk2)
+   else if (ipencil==2) then
+      call transpose_y_to_x(var,wk,decomp)
+   else if (ipencil==3) then
+      allocate(wk2(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)))
+      call transpose_z_to_y(var,wk2,decomp)
+      call transpose_y_to_x(wk2,wk,decomp)
+      deallocate(wk2)
    end if
-   allocate (wk2d(1, decomp%xsz(2), decomp%xsz(3)))
-   if (n .ge. 1) then
-      do k = 1, decomp%xsz(3)
-         do j = 1, decomp%xsz(2)
-            wk2d(1, j, k) = wk(n, j, k)
+   allocate(wk2d(1,decomp%xsz(2),decomp%xsz(3)))
+   if (n.ge.1) then
+      do k=1,decomp%xsz(3)
+         do j=1,decomp%xsz(2)
+            wk2d(1,j,k)=wk(n,j,k)
          end do
       end do
    else
-      do k = 1, decomp%xsz(3)
-         do j = 1, decomp%xsz(2)
-            wk2d(1, j, k) = sum(wk(:, j, k))/real(decomp%xsz(1), kind=mytype)
+      do k=1,decomp%xsz(3)
+         do j=1,decomp%xsz(2)
+            wk2d(1,j,k)=sum(wk(:,j,k))/real(decomp%xsz(1),kind=mytype)
          end do
       end do
-   end if
-else if (iplane == 2) then
-   allocate (wk(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)))
-   if (ipencil == 1) then
-      call transpose_x_to_y(var, wk, decomp)
-   else if (ipencil == 2) then
+   endif
+else if (iplane==2) then
+   allocate(wk(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)))
+   if (ipencil==1) then
+      call transpose_x_to_y(var,wk,decomp)
+   else if (ipencil==2) then
       wk = var
-   else if (ipencil == 3) then
-      call transpose_z_to_y(var, wk, decomp)
+   else if (ipencil==3) then
+      call transpose_z_to_y(var,wk,decomp)
    end if
-   allocate (wk2d(decomp%ysz(1), 1, decomp%ysz(3)))
-   if (n .ge. 1) then
-      do k = 1, decomp%ysz(3)
-         do i = 1, decomp%ysz(1)
-            wk2d(i, 1, k) = wk(i, n, k)
+   allocate(wk2d(decomp%ysz(1),1,decomp%ysz(3)))
+   if (n.ge.1) then
+      do k=1,decomp%ysz(3)
+         do i=1,decomp%ysz(1)
+            wk2d(i,1,k)=wk(i,n,k)
          end do
       end do
    else
-      do k = 1, decomp%ysz(3)
-         do i = 1, decomp%ysz(1)
-            wk2d(i, 1, k) = sum(wk(i, :, k))/real(decomp%ysz(2), kind=mytype)
+      do k=1,decomp%ysz(3)
+         do i=1,decomp%ysz(1)
+            wk2d(i,1,k)=sum(wk(i,:,k))/real(decomp%ysz(2),kind=mytype)
          end do
       end do
-   end if
-else if (iplane == 3) then
-   allocate (wk(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)))
-   if (ipencil == 1) then
-      allocate (wk2(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)))
-      call transpose_x_to_y(var, wk2, decomp)
-      call transpose_y_to_z(wk2, wk, decomp)
-      deallocate (wk2)
-   else if (ipencil == 2) then
-      call transpose_y_to_z(var, wk, decomp)
-   else if (ipencil == 3) then
+   endif
+else if (iplane==3) then
+   allocate(wk(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)))
+   if (ipencil==1) then
+      allocate(wk2(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)))
+      call transpose_x_to_y(var,wk2,decomp)
+      call transpose_y_to_z(wk2,wk,decomp)
+      deallocate(wk2)
+   else if (ipencil==2) then
+      call transpose_y_to_z(var,wk,decomp)
+   else if (ipencil==3) then
       wk = var
    end if
-   allocate (wk2d(decomp%zsz(1), decomp%zsz(2), 1))
-   if (n .ge. 1) then
-      do j = 1, decomp%zsz(2)
-         do i = 1, decomp%zsz(1)
-            wk2d(i, j, 1) = wk(i, j, n)
+   allocate(wk2d(decomp%zsz(1),decomp%zsz(2),1))
+   if (n.ge.1) then
+      do j=1,decomp%zsz(2)
+         do i=1,decomp%zsz(1) 
+            wk2d(i,j,1)=wk(i,j,n)
          end do
       end do
    else
-      do j = 1, decomp%zsz(2)
-         do i = 1, decomp%zsz(1)
-            wk2d(i, j, 1) = sum(wk(i, j, :))/real(decomp%zsz(3), kind=mytype)
+      do j=1,decomp%zsz(2)
+         do i=1,decomp%zsz(1)
+            wk2d(i,j,1)=sum(wk(i,j,:))/real(decomp%zsz(3),kind=mytype)
          end do
       end do
-   end if
+   endif
 end if
 
 idx = get_io_idx(io_name, dirname)
@@ -108,13 +108,13 @@ idx = get_io_idx(io_name, dirname)
 
 if (idx .lt. 1) then
    ! Create folder if needed
-   if (nrank == 0) then
-      inquire (file=dirname, exist=dir_exists)
-      if (.not. dir_exists) then
+   if (nrank==0) then
+      inquire(file=dirname, exist=dir_exists)
+      if (.not.dir_exists) then
          call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
       end if
    end if
-   allocate (character(len(trim(dirname)) + 1 + len(trim(varname))) :: full_io_name)
+   allocate(character(len(trim(dirname)) + 1 + len(trim(varname))) :: full_io_name)
    full_io_name = dirname//"/"//varname
    call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
    idx = get_io_idx(io_name, full_io_name)
@@ -122,40 +122,40 @@ if (idx .lt. 1) then
 end if
 
 call plane_extents(sizes, subsizes, starts, iplane, decomp)
-associate (fh => fh_registry(idx), &
-           disp => fh_disp(idx))
-   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-   call MPI_TYPE_COMMIT(newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                          newtype, 'native', MPI_INFO_NULL, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-   call MPI_FILE_WRITE_ALL(fh, wk2d, &
-                           subsizes(1)*subsizes(2)*subsizes(3), &
-                           data_type, MPI_STATUS_IGNORE, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-   call MPI_TYPE_FREE(newtype, ierror)
-   if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+associate(fh=>fh_registry(idx), &
+     disp=>fh_disp(idx))
+  call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+  call MPI_TYPE_COMMIT(newtype,ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+  call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+       newtype,'native',MPI_INFO_NULL,ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+  call MPI_FILE_WRITE_ALL(fh, wk2d, &
+       subsizes(1)*subsizes(2)*subsizes(3), &
+       data_type, MPI_STATUS_IGNORE, ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+  call MPI_TYPE_FREE(newtype,ierror)
+  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 end associate
-
+    
 if (opened_new) then
    call decomp_2d_close_io(io_name, full_io_name)
-   deallocate (full_io_name)
+   deallocate(full_io_name)
 end if
 #else
 !! Write using ADIOS2
 call adios2_at_io(io_handle, adios, io_name, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-                                                 "trying to write variable before registering! "//trim(varname))
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+   "trying to write variable before registering! "//trim(varname))
 
 !! Note - need to use sync mode as the array for the output plane gets reused.
 call adios2_put(engine_registry(idx), var_handle, wk2d, adios2_mode_sync, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
 #endif
 
-deallocate (wk, wk2d)
+deallocate(wk,wk2d)
diff --git a/src/io_write_var.inc b/src/io_write_var.inc
index 338f42fe..23c8b617 100644
--- a/src/io_write_var.inc
+++ b/src/io_write_var.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-
+!                          -*- mode: f90 -*-                            
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,12 +10,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines
+! This file contain common code to be included by subroutines 
 ! 'write_var_...' in io.f90
 
-! Using MPI-IO to write a distributed 3D variable to a file. File
+! Using MPI-IO to write a distributed 3D variable to a file. File 
 ! operations (open/close) need to be done in calling application. This
-! allows multiple variables to be written to a single file. Together
+! allows multiple variables to be written to a single file. Together 
 ! with the corresponding read operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -33,46 +33,46 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1) - 1  ! 0-based index
-   starts(2) = decomp%xst(2) - 1
-   starts(3) = decomp%xst(3) - 1
+   starts(1) = decomp%xst(1)-1  ! 0-based index
+   starts(2) = decomp%xst(2)-1
+   starts(3) = decomp%xst(3)-1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1) - 1
-   starts(2) = decomp%yst(2) - 1
-   starts(3) = decomp%yst(3) - 1
+   starts(1) = decomp%yst(1)-1
+   starts(2) = decomp%yst(2)-1
+   starts(3) = decomp%yst(3)-1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1) - 1
-   starts(2) = decomp%zst(2) - 1
-   starts(3) = decomp%zst(3) - 1
-end if
+   starts(1) = decomp%zst(1)-1
+   starts(2) = decomp%zst(2)-1
+   starts(3) = decomp%zst(3)-1
+endif
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
-                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-call MPI_FILE_SET_VIEW(fh, disp, data_type, &
-                       newtype, 'native', MPI_INFO_NULL, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
+     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_FILE_SET_VIEW(fh,disp,data_type, &
+     newtype,'native',MPI_INFO_NULL,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_WRITE_ALL(fh, var, &
-                        subsizes(1)*subsizes(2)*subsizes(3), &
-                        data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-call MPI_TYPE_FREE(newtype, ierror)
-if (ierror .ne. 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+     subsizes(1)*subsizes(2)*subsizes(3), &
+     data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+call MPI_TYPE_FREE(newtype,ierror)
+if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 ! update displacement for the next write operation
 disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-       *int(sizes(2), kind=MPI_OFFSET_KIND) &
-       *int(sizes(3), kind=MPI_OFFSET_KIND) &
-       *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+            * int(sizes(2), kind=MPI_OFFSET_KIND) &
+            * int(sizes(3), kind=MPI_OFFSET_KIND) &
+            * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                      *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                      *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                      *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                           * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                           * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                           * int(mytype_bytes, kind=MPI_OFFSET_KIND)
diff --git a/src/log.f90 b/src/log.f90
index 53e3165a..866cd97e 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,296 +10,296 @@
 !
 !=======================================================================
 
-submodule(decomp_2d) d2d_log
+submodule (decomp_2d) d2d_log
 
-   implicit none
+  implicit none
 
-contains
+  contains
 
-   !
-   ! Print some information about decomp_2d
-   !
-   module subroutine d2d_listing(given_io_unit)
+  !
+  ! Print some information about decomp_2d
+  !
+  module subroutine d2d_listing(given_io_unit)
 
-      use iso_fortran_env, only: output_unit, compiler_version, compiler_options
+    use iso_fortran_env, only : output_unit, compiler_version, compiler_options
 
-      implicit none
+    implicit none
 
-      ! Argument
-      integer, intent(in), optional :: given_io_unit
+    ! Argument
+    integer, intent(in), optional :: given_io_unit
 
-      ! Local variable
-      integer :: io_unit
-      integer :: version, subversion, ierror
+    ! Local variable
+    integer :: io_unit
+    integer :: version, subversion, ierror
 #ifdef DEBUG
-      character(len=64) :: fname
+    character(len=64) :: fname
 #endif
 
-      ! Output log if needed
-      if (decomp_log == D2D_LOG_QUIET) return
-      if (decomp_log == D2D_LOG_STDOUT .and. nrank /= 0) return
-      if (decomp_log == D2D_LOG_TOFILE .and. nrank /= 0) return
+    ! Output log if needed
+    if (decomp_log == D2D_LOG_QUIET) return
+    if (decomp_log == D2D_LOG_STDOUT .and. nrank /= 0) return
+    if (decomp_log == D2D_LOG_TOFILE .and. nrank /= 0) return
 
-      ! If no IO unit provided, use stdout
-      if (present(given_io_unit)) then
-         io_unit = given_io_unit
-      else
-         io_unit = output_unit
-      end if
+    ! If no IO unit provided, use stdout
+    if (present(given_io_unit)) then
+       io_unit = given_io_unit
+    else
+       io_unit = output_unit
+    endif
 
-      ! Header
-      write (io_unit, *) '==========================================================='
-      write (io_unit, *) '=================== Decomp2D - log ========================'
-      write (io_unit, *) '==========================================================='
+    ! Header
+    write (io_unit, *) '==========================================================='
+    write (io_unit, *) '=================== Decomp2D - log ========================'
+    write (io_unit, *) '==========================================================='
 
-      ! Git hash if available
+    ! Git hash if available
 #if defined(VERSION)
-      write (io_unit, *) 'Git version        : ', VERSION
+    write (io_unit, *) 'Git version        : ', VERSION
 #else
-      write (io_unit, *) 'Git version        : unknown'
+    write (io_unit, *) 'Git version        : unknown'
 #endif
 
-      ! Basic info
+    ! Basic info
 #ifdef DEBUG
-      if (decomp_debug >= D2D_DEBUG_LEVEL_INFO) &
-         write (io_unit, *) 'I am mpi rank ', nrank
+    if (decomp_debug >= D2D_DEBUG_LEVEL_INFO) &
+       write (io_unit, *) 'I am mpi rank ', nrank
 #endif
-      write (io_unit, *) 'Total ranks ', nproc
-      write (io_unit, *) 'Global data size : ', nx_global, ny_global, nz_global
-      write (io_unit, *) 'p_row, p_col : ', dims(1), dims(2)
-      write (io_unit, *) 'Periodicity : ', periodic_x, periodic_y, periodic_z
-      write (io_unit, *) 'Number of bytes / float number : ', mytype_bytes
-      write (io_unit, *) '==========================================================='
-
-      ! Show detected flags, compiler options, version of the MPI library
-      write (io_unit, *) 'Compile flags detected :'
+    write (io_unit, *) 'Total ranks ', nproc
+    write (io_unit, *) 'Global data size : ', nx_global, ny_global, nz_global
+    write (io_unit, *) 'p_row, p_col : ', dims(1), dims(2)
+    write (io_unit, *) 'Periodicity : ', periodic_x, periodic_y, periodic_z
+    write (io_unit, *) 'Number of bytes / float number : ', mytype_bytes
+    write (io_unit, *) '==========================================================='
+
+    ! Show detected flags, compiler options, version of the MPI library
+    write (io_unit, *) 'Compile flags detected :'
 #ifdef DOUBLE_PREC
 #ifdef SAVE_SINGLE
-      write (io_unit, *) 'Numerical precision: Double, saving in single'
+    write (io_unit, *) 'Numerical precision: Double, saving in single'
 #else
-      write (io_unit, *) 'Numerical precision: Double'
+    write (io_unit, *) 'Numerical precision: Double'
 #endif
 #else
-      write (io_unit, *) 'Numerical precision: Single'
+    write (io_unit, *) 'Numerical precision: Single'
 #endif
-      write (io_unit, *) 'Compiled with ', compiler_version()
-      write (io_unit, *) 'Compiler options : ', compiler_options()
-      call MPI_Get_version(version, subversion, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_Get_version")
-      write (io_unit, '(" Version of the MPI library : ",I0,".",I0)') version, subversion
+    write (io_unit, *) 'Compiled with ', compiler_version()
+    write (io_unit, *) 'Compiler options : ', compiler_options()
+    call MPI_Get_version(version, subversion, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_Get_version")
+    write (io_unit, '(" Version of the MPI library : ",I0,".",I0)') version, subversion
 #ifdef DEBUG
-      write (io_unit, *) 'Compile flag DEBUG detected'
-      write (io_unit, *) '   debug level : ', decomp_debug
+    write (io_unit, *) 'Compile flag DEBUG detected'
+    write (io_unit, *) '   debug level : ', decomp_debug
 #endif
 #ifdef PROFILER
-      write (io_unit, *) 'Compile flag PROFILER detected'
+    write (io_unit, *) 'Compile flag PROFILER detected'
 #endif
 #ifdef SHM
-      write (io_unit, *) 'Compile flag SHM detected'
+    write (io_unit, *) 'Compile flag SHM detected'
 #endif
 #ifdef EVEN
-      write (io_unit, *) 'Compile flag EVEN detected'
+    write (io_unit, *) 'Compile flag EVEN detected'
 #endif
 #ifdef OCC
-      write (io_unit, *) 'Compile flag OCC detected'
+    write (io_unit, *) 'Compile flag OCC detected'
 #endif
 #ifdef OVERWRITE
-      write (io_unit, *) 'Compile flag OVERWRITE detected'
+    write (io_unit, *) 'Compile flag OVERWRITE detected'
 #endif
 #ifdef HALO_DEBUG
-      write (io_unit, *) 'Compile flag HALO_DEBUG detected'
+    write (io_unit, *) 'Compile flag HALO_DEBUG detected'
 #endif
 #ifdef SHM_DEBUG
-      write (io_unit, *) 'Compile flag SHM_DEBUG detected'
+    write (io_unit, *) 'Compile flag SHM_DEBUG detected'
 #endif
 #ifdef _GPU
-      write (io_unit, *) 'Compile flag _GPU detected'
+    write (io_unit, *) 'Compile flag _GPU detected'
 #endif
 #ifdef _NCCL
-      write (io_unit, *) 'Compile flag _NCCL detected'
+    write (io_unit, *) 'Compile flag _NCCL detected'
 #endif
-      write (io_unit, *) '==========================================================='
-      write (io_unit, *) 'Profiler id : ', decomp_profiler
+    write (io_unit, *) '==========================================================='
+    write (io_unit, *) 'Profiler id : ', decomp_profiler
 #ifdef PROFILER
-      call decomp_profiler_log(io_unit)
-      write (io_unit, *) "   Profiling transpose : ", decomp_profiler_transpose
-      write (io_unit, *) "   Profiling IO : ", decomp_profiler_io
-      write (io_unit, *) "   Profiling FFT : ", decomp_profiler_fft
-      write (io_unit, *) "   Profiling decomp_2d : ", decomp_profiler_d2d
+    call decomp_profiler_log(io_unit)
+    write(io_unit, *) "   Profiling transpose : ", decomp_profiler_transpose
+    write(io_unit, *) "   Profiling IO : ", decomp_profiler_io
+    write(io_unit, *) "   Profiling FFT : ", decomp_profiler_fft
+    write(io_unit, *) "   Profiling decomp_2d : ", decomp_profiler_d2d
 #endif
-      write (io_unit, *) '==========================================================='
-      ! Info about each decomp_info object
-      call decomp_info_print(decomp_main, io_unit, "decomp_main")
-      call decomp_info_print(phG, io_unit, "phG")
-      call decomp_info_print(ph1, io_unit, "ph1")
-      call decomp_info_print(ph2, io_unit, "ph2")
-      call decomp_info_print(ph3, io_unit, "ph3")
-      call decomp_info_print(ph4, io_unit, "ph4")
+    write (io_unit, *) '==========================================================='
+    ! Info about each decomp_info object
+    call decomp_info_print(decomp_main, io_unit, "decomp_main")
+    call decomp_info_print(phG, io_unit, "phG")
+    call decomp_info_print(ph1, io_unit, "ph1")
+    call decomp_info_print(ph2, io_unit, "ph2")
+    call decomp_info_print(ph3, io_unit, "ph3")
+    call decomp_info_print(ph4, io_unit, "ph4")
 #ifdef SHM_DEBUG
-      write (io_unit, *) '==========================================================='
-      call print_smp(io_unit)
+    write (io_unit, *) '==========================================================='
+    call print_smp(io_unit)
 #endif
-      write (io_unit, *) '==========================================================='
-      write (io_unit, *) '==========================================================='
+    write (io_unit, *) '==========================================================='
+    write (io_unit, *) '==========================================================='
 #ifdef DEBUG
-      !
-      ! In DEBUG mode, rank 0 will also print environment variables
-      !
-      ! At high debug level, all ranks will print env. variables
-      !
-      ! The system call, if writing to a file, is not blocking if supported
-      !
-      if (nrank == 0 .or. decomp_debug >= D2D_DEBUG_LEVEL_INFO) then
-         write (io_unit, *) '============== Environment variables ======================'
-         write (io_unit, *) '==========================================================='
-         write (io_unit, *) '==========================================================='
-         if (io_unit == output_unit) then
-            call execute_command_line("env", wait=.true.)
-         else
-            inquire (unit=io_unit, name=fname, iostat=ierror)
-            if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  ierror, &
-                                                  "No name for the log file")
-            call execute_command_line("env >> "//trim(fname), wait=.false.)
-         end if
-      end if
+    !
+    ! In DEBUG mode, rank 0 will also print environment variables
+    !
+    ! At high debug level, all ranks will print env. variables
+    !
+    ! The system call, if writing to a file, is not blocking if supported
+    !
+    if (nrank == 0 .or. decomp_debug >= D2D_DEBUG_LEVEL_INFO) then
+       write (io_unit, *) '============== Environment variables ======================'
+       write (io_unit, *) '==========================================================='
+       write (io_unit, *) '==========================================================='
+       if (io_unit == output_unit ) then
+          call execute_command_line("env", wait = .true.)
+       else
+          inquire(unit = io_unit, name = fname, iostat = ierror)
+          if (ierror /= 0) call decomp_2d_abort(__FILE__, &
+                                                __LINE__, &
+                                                ierror, &
+                                                "No name for the log file")
+          call execute_command_line("env >> "//trim(fname), wait = .false.)
+       endif
+    endif
 #endif
 
-   end subroutine d2d_listing
+  end subroutine d2d_listing
 
-   !
-   ! Print some information about given decomp_info object
-   !
-   module subroutine decomp_info_print(d2d, io_unit, d2dname)
+  !
+  ! Print some information about given decomp_info object
+  !
+  module subroutine decomp_info_print(d2d, io_unit, d2dname)
 
-      implicit none
+    implicit none
 
-      ! Arguments
-      type(decomp_info), intent(in) :: d2d
-      integer, intent(in) :: io_unit
-      character(len=*), intent(in) :: d2dname
+    ! Arguments
+    type(decomp_info), intent(in) :: d2d
+    integer, intent(in) :: io_unit
+    character(len=*), intent(in) :: d2dname
 
-      ! Nothing to print if not initialized
-      if (.not. allocated(d2d%x1dist)) then
-         write (io_unit, *) 'Uninitialized decomp_info ', d2dname
-         return
-      end if
+    ! Nothing to print if not initialized
+    if (.not.allocated(d2d%x1dist)) then
+      write (io_unit, *) 'Uninitialized decomp_info ', d2dname
+      return
+    endif
 
-      !
-      ! If DEBUG mode, print everything
-      ! Otherwise, print only global size
-      !
-      write (io_unit, *) 'Decomp_info : ', d2dname
-      write (io_unit, *) '   Global size : ', d2d%xsz(1), d2d%ysz(2), d2d%zsz(3)
+    !
+    ! If DEBUG mode, print everything
+    ! Otherwise, print only global size
+    !
+    write (io_unit, *) 'Decomp_info : ', d2dname
+    write (io_unit, *) '   Global size : ', d2d%xsz(1), d2d%ysz(2), d2d%zsz(3)
 #ifdef DEBUG
-      write (io_unit, *) '   xsz, xst, xen : ', d2d%xsz, d2d%xst, d2d%xen
-      write (io_unit, *) '   ysz, yst, yen : ', d2d%ysz, d2d%yst, d2d%yen
-      write (io_unit, *) '   zsz, zst, zen : ', d2d%zsz, d2d%zst, d2d%zen
-      write (io_unit, *) '   x1dist : ', d2d%x1dist
-      write (io_unit, *) '   y1dist : ', d2d%y1dist
-      write (io_unit, *) '   y2dist : ', d2d%y2dist
-      write (io_unit, *) '   z2dist : ', d2d%z2dist
-      write (io_unit, *) '   x1cnts : ', d2d%x1cnts
-      write (io_unit, *) '   y1cnts : ', d2d%y1cnts
-      write (io_unit, *) '   y2cnts : ', d2d%y2cnts
-      write (io_unit, *) '   z2cnts : ', d2d%z2cnts
-      write (io_unit, *) '   x1disp : ', d2d%x1disp
-      write (io_unit, *) '   y1disp : ', d2d%y1disp
-      write (io_unit, *) '   y2disp : ', d2d%y2disp
-      write (io_unit, *) '   z2disp : ', d2d%z2disp
-      write (io_unit, *) '   x1count : ', d2d%x1count
-      write (io_unit, *) '   y1count : ', d2d%y1count
-      write (io_unit, *) '   y2count : ', d2d%y2count
-      write (io_unit, *) '   z2count : ', d2d%z2count
-      write (io_unit, *) '   even : ', d2d%even
+    write (io_unit, *) '   xsz, xst, xen : ', d2d%xsz, d2d%xst, d2d%xen
+    write (io_unit, *) '   ysz, yst, yen : ', d2d%ysz, d2d%yst, d2d%yen
+    write (io_unit, *) '   zsz, zst, zen : ', d2d%zsz, d2d%zst, d2d%zen
+    write (io_unit, *) '   x1dist : ', d2d%x1dist
+    write (io_unit, *) '   y1dist : ', d2d%y1dist
+    write (io_unit, *) '   y2dist : ', d2d%y2dist
+    write (io_unit, *) '   z2dist : ', d2d%z2dist
+    write (io_unit, *) '   x1cnts : ', d2d%x1cnts
+    write (io_unit, *) '   y1cnts : ', d2d%y1cnts
+    write (io_unit, *) '   y2cnts : ', d2d%y2cnts
+    write (io_unit, *) '   z2cnts : ', d2d%z2cnts
+    write (io_unit, *) '   x1disp : ', d2d%x1disp
+    write (io_unit, *) '   y1disp : ', d2d%y1disp
+    write (io_unit, *) '   y2disp : ', d2d%y2disp
+    write (io_unit, *) '   z2disp : ', d2d%z2disp
+    write (io_unit, *) '   x1count : ', d2d%x1count
+    write (io_unit, *) '   y1count : ', d2d%y1count
+    write (io_unit, *) '   y2count : ', d2d%y2count
+    write (io_unit, *) '   z2count : ', d2d%z2count
+    write (io_unit, *) '   even : ', d2d%even
 #ifdef SHM
-      write (io_unit, *) '   listing of the SHM part is not yet implemented'
+    write (io_unit, *) '   listing of the SHM part is not yet implemented'
 #endif
 #endif
 
-   end subroutine decomp_info_print
+  end subroutine decomp_info_print
 
 #ifdef SHM_DEBUG
 
-   ! For debugging, print the shared-memory structure
-   module subroutine print_smp(io_unit)
-
-      implicit none
-
-      ! Argument
-      integer, intent(in) :: io_unit
-
-      ! print out shared-memory information
-      write (io_unit, *) 'I am mpi rank ', nrank, 'Total ranks ', nproc
-      write (io_unit, *) ' '
-      write (io_unit, *) 'Global data size:'
-      write (io_unit, *) 'nx*ny*nz', nx, ny, nz
-      write (io_unit, *) ' '
-      write (io_unit, *) '2D processor grid:'
-      write (io_unit, *) 'p_row*p_col:', dims(1), dims(2)
-      write (io_unit, *) ' '
-      write (io_unit, *) 'Portion of global data held locally:'
-      write (io_unit, *) 'xsize:', xsize
-      write (io_unit, *) 'ysize:', ysize
-      write (io_unit, *) 'zsize:', zsize
-      write (io_unit, *) ' '
-      write (io_unit, *) 'How pensils are to be divided and sent in alltoallv:'
-      write (io_unit, *) 'x1dist:', decomp_main%x1dist
-      write (io_unit, *) 'y1dist:', decomp_main%y1dist
-      write (io_unit, *) 'y2dist:', decomp_main%y2dist
-      write (io_unit, *) 'z2dist:', decomp_main%z2dist
-      write (io_unit, *) ' '
-      write (io_unit, *) '######Shared buffer set up after this point######'
-      write (io_unit, *) ' '
-      write (io_unit, *) 'col communicator detais:'
-      call print_smp_info(decomp_main%COL_INFO, io_unit)
-      write (io_unit, *) ' '
-      write (io_unit, *) 'row communicator detais:'
-      call print_smp_info(decomp_main%ROW_INFO; io_unit)
-      write (io_unit, *) ' '
-      write (io_unit, *) 'Buffer count and displacement of per-core buffers'
-      write (io_unit, *) 'x1cnts:', decomp_main%x1cnts
-      write (io_unit, *) 'y1cnts:', decomp_main%y1cnts
-      write (io_unit, *) 'y2cnts:', decomp_main%y2cnts
-      write (io_unit, *) 'z2cnts:', decomp_main%z2cnts
-      write (io_unit, *) 'x1disp:', decomp_main%x1disp
-      write (io_unit, *) 'y1disp:', decomp_main%y1disp
-      write (io_unit, *) 'y2disp:', decomp_main%y2disp
-      write (io_unit, *) 'z2disp:', decomp_main%z2disp
-      write (io_unit, *) ' '
-      write (io_unit, *) 'Buffer count and displacement of shared buffers'
-      write (io_unit, *) 'x1cnts:', decomp_main%x1cnts_s
-      write (io_unit, *) 'y1cnts:', decomp_main%y1cnts_s
-      write (io_unit, *) 'y2cnts:', decomp_main%y2cnts_s
-      write (io_unit, *) 'z2cnts:', decomp_main%z2cnts_s
-      write (io_unit, *) 'x1disp:', decomp_main%x1disp_s
-      write (io_unit, *) 'y1disp:', decomp_main%y1disp_s
-      write (io_unit, *) 'y2disp:', decomp_main%y2disp_s
-      write (io_unit, *) 'z2disp:', decomp_main%z2disp_s
-
-   end subroutine print_smp
-
-   ! For debugging, print the shared-memory structure
-   module subroutine print_smp_info(s, io_unit)
-
-      implicit none
-
-      ! Argument
-      TYPE(SMP_INFO), intent(in) :: s
-      integer, intent(in) :: io_unit
-
-      write (io_unit, *) 'size of current communicator:', s%NCPU
-      write (io_unit, *) 'rank in current communicator:', s%NODE_ME
-      write (io_unit, *) 'number of SMP-nodes in this communicator:', s%NSMP
-      write (io_unit, *) 'SMP-node id (1 ~ NSMP):', s%SMP_ME
-      write (io_unit, *) 'NCORE - number of cores on this SMP-node', s%NCORE
-      write (io_unit, *) 'core id (1 ~ NCORE):', s%CORE_ME
-      write (io_unit, *) 'maximum no. cores on any SMP-node:', s%MAXCORE
-      write (io_unit, *) 'size of SMP shared memory SND buffer:', s%N_SND
-      write (io_unit, *) 'size of SMP shared memory RCV buffer:', s%N_RCV
-
-   end subroutine print_smp_info
+  ! For debugging, print the shared-memory structure
+  module subroutine print_smp(io_unit)
+
+    implicit none
+    
+    ! Argument
+    integer, intent(in) :: io_unit
+
+    ! print out shared-memory information
+    write(io_unit,*)'I am mpi rank ', nrank, 'Total ranks ', nproc
+    write(io_unit,*)' '
+    write(io_unit,*)'Global data size:'
+    write(io_unit,*)'nx*ny*nz', nx,ny,nz
+    write(io_unit,*)' '
+    write(io_unit,*)'2D processor grid:'
+    write(io_unit,*)'p_row*p_col:', dims(1), dims(2)
+    write(io_unit,*)' '
+    write(io_unit,*)'Portion of global data held locally:'
+    write(io_unit,*)'xsize:',xsize
+    write(io_unit,*)'ysize:',ysize
+    write(io_unit,*)'zsize:',zsize
+    write(io_unit,*)' '
+    write(io_unit,*)'How pensils are to be divided and sent in alltoallv:'
+    write(io_unit,*)'x1dist:',decomp_main%x1dist
+    write(io_unit,*)'y1dist:',decomp_main%y1dist
+    write(io_unit,*)'y2dist:',decomp_main%y2dist
+    write(io_unit,*)'z2dist:',decomp_main%z2dist
+    write(io_unit,*)' '
+    write(io_unit,*)'######Shared buffer set up after this point######'
+    write(io_unit,*)' '
+    write(io_unit,*) 'col communicator detais:'
+    call print_smp_info(decomp_main%COL_INFO, io_unit)
+    write(io_unit,*)' '
+    write(io_unit,*) 'row communicator detais:'
+    call print_smp_info(decomp_main%ROW_INFO; io_unit)
+    write(io_unit,*)' '
+    write(io_unit,*)'Buffer count and displacement of per-core buffers'
+    write(io_unit,*)'x1cnts:',decomp_main%x1cnts
+    write(io_unit,*)'y1cnts:',decomp_main%y1cnts
+    write(io_unit,*)'y2cnts:',decomp_main%y2cnts
+    write(io_unit,*)'z2cnts:',decomp_main%z2cnts
+    write(io_unit,*)'x1disp:',decomp_main%x1disp
+    write(io_unit,*)'y1disp:',decomp_main%y1disp
+    write(io_unit,*)'y2disp:',decomp_main%y2disp
+    write(io_unit,*)'z2disp:',decomp_main%z2disp
+    write(io_unit,*)' '
+    write(io_unit,*)'Buffer count and displacement of shared buffers'
+    write(io_unit,*)'x1cnts:',decomp_main%x1cnts_s
+    write(io_unit,*)'y1cnts:',decomp_main%y1cnts_s
+    write(io_unit,*)'y2cnts:',decomp_main%y2cnts_s
+    write(io_unit,*)'z2cnts:',decomp_main%z2cnts_s
+    write(io_unit,*)'x1disp:',decomp_main%x1disp_s
+    write(io_unit,*)'y1disp:',decomp_main%y1disp_s
+    write(io_unit,*)'y2disp:',decomp_main%y2disp_s
+    write(io_unit,*)'z2disp:',decomp_main%z2disp_s
+
+  end subroutine print_smp
+
+  ! For debugging, print the shared-memory structure
+  module subroutine print_smp_info(s, io_unit)
+
+    implicit none
+
+    ! Argument
+    TYPE(SMP_INFO), intent(in) :: s
+    integer, intent(in) :: io_unit
+
+    write(io_unit,*) 'size of current communicator:', s%NCPU
+    write(io_unit,*) 'rank in current communicator:', s%NODE_ME
+    write(io_unit,*) 'number of SMP-nodes in this communicator:', s%NSMP
+    write(io_unit,*) 'SMP-node id (1 ~ NSMP):', s%SMP_ME
+    write(io_unit,*) 'NCORE - number of cores on this SMP-node', s%NCORE
+    write(io_unit,*) 'core id (1 ~ NCORE):', s%CORE_ME
+    write(io_unit,*) 'maximum no. cores on any SMP-node:', s%MAXCORE
+    write(io_unit,*) 'size of SMP shared memory SND buffer:', s%N_SND
+    write(io_unit,*) 'size of SMP shared memory RCV buffer:', s%N_RCV
+
+  end subroutine print_smp_info
 #endif
 
 end submodule d2d_log
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index bd783d7e..c5d83fa4 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,474 +13,480 @@
 
   subroutine transpose_x_to_y_real_short(src, dst)
 
-     implicit none
+    implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+    real(mytype), dimension(:,:,:), intent(IN) :: src
+    real(mytype), dimension(:,:,:), intent(OUT) :: dst
 
-     call transpose_x_to_y(src, dst, decomp_main)
+    call transpose_x_to_y(src, dst, decomp_main)
 
   end subroutine transpose_x_to_y_real_short
 
   subroutine transpose_x_to_y_real(src, dst, decomp)
 
-     implicit none
+    implicit none
+    
+    Real(mytype), dimension(:,:,:), intent(IN) :: src
+    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-     Real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU) && defined(_NCCL)
-     integer :: col_rank_id, cuda_stat
-     type(ncclResult) :: nccl_stat
+    integer :: col_rank_id, cuda_stat
+    type(ncclResult) :: nccl_stat
 #endif
 
 #ifdef SHM
-     real(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
+    real(mytype) :: work1(*), work2(*)
+    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+    
+    integer :: s1,s2,s3,d1,d2,d3
+    integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
+    if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+    s1 = SIZE(src,1)
+    s2 = SIZE(src,2)
+    s3 = SIZE(src,3)
+    d1 = SIZE(dst,1)
+    d2 = SIZE(dst,2)
+    d3 = SIZE(dst,3)
 
-     ! rearrange source array as send buffer
+    ! rearrange source array as send buffer
 #ifdef SHM
-     work1_p = decomp%COL_INFO%SND_P
-     call mem_split_xy_real(src, s1, s2, s3, work1, dims(1), &
-                            decomp%x1dist, decomp)
+    work1_p = decomp%COL_INFO%SND_P
+    call mem_split_xy_real(src, s1, s2, s3, work1, dims(1), &
+         decomp%x1dist, decomp)
 #else
-
+  
 #if defined(_GPU)
-     call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
-                            decomp%x1dist, decomp)
+    call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
+         decomp%x1dist, decomp)
 #else
-     call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
-                            decomp%x1dist, decomp)
+    call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
+         decomp%x1dist, decomp)
 #endif
 
 #endif
 
-     ! define receive buffer
+    ! define receive buffer
 #ifdef SHM
-     work2_p = decomp%COL_INFO%RCV_P
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    work2_p = decomp%COL_INFO%RCV_P
+    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-
-     ! transpose using MPI_ALLTOALL(V)
+    
+    ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-     if (decomp%COL_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
-                           real_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
-                           real_type, decomp%COL_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
+    if (decomp%COL_INFO%CORE_ME==1) THEN
+       call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
+            real_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
+            real_type, decomp%COL_INFO%SMP_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    end if
 #else
 #ifdef EVEN
-     call MPI_ALLTOALL(work1_r, decomp%x1count, &
-                       real_type, work2_r, decomp%y1count, &
-                       real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+    call MPI_ALLTOALL(work1_r, decomp%x1count, &
+         real_type, work2_r, decomp%y1count, &
+         real_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     nccl_stat = ncclGroupStart()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-     do col_rank_id = 0, (col_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
-                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+    nccl_stat = ncclGroupStart()
+    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+    do col_rank_id = 0, (col_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d( decomp%x1disp(col_rank_id)+1 ), decomp%x1cnts(col_rank_id), &
+          ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
-                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+        nccl_stat = ncclRecv(work2_r_d( decomp%y1disp(col_rank_id)+1 ), decomp%y1cnts(col_rank_id), &
+          ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-     end do
-     nccl_stat = ncclGroupEnd()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+    end do
+    nccl_stat = ncclGroupEnd()
+    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-     call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
-                        real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
+         real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
+         real_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
-                        real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
+         real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
+         real_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-     ! rearrange receive buffer
+    ! rearrange receive buffer
 #ifdef SHM
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_xy_real(work2, d1, d2, d3, dst, dims(1), &
-                            decomp%y1dist, decomp)
+    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    call mem_merge_xy_real(work2, d1, d2, d3, dst, dims(1), &
+         decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-                            decomp%y1dist, decomp)
+    call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+         decomp%y1dist, decomp)
 #else
-     call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
-                            decomp%y1dist, decomp)
+    call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
+         decomp%y1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
+    if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
 #endif
 
-     return
+    return
   end subroutine transpose_x_to_y_real
 
+
   subroutine transpose_x_to_y_complex_short(src, dst)
 
-     implicit none
+    implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+    complex(mytype), dimension(:,:,:), intent(IN) :: src
+    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
 
-     call transpose_x_to_y(src, dst, decomp_main)
+    call transpose_x_to_y(src, dst, decomp_main)
 
   end subroutine transpose_x_to_y_complex_short
 
   subroutine transpose_x_to_y_complex(src, dst, decomp)
 
-     implicit none
-
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    complex(mytype), dimension(:,:,:), intent(IN) :: src
+    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #ifdef SHM
-     complex(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
+    complex(mytype) :: work1(*), work2(*)
+    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+    
+    integer :: s1,s2,s3,d1,d2,d3
+    integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
+    if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
-
-     ! rearrange source array as send buffer
+    s1 = SIZE(src,1)
+    s2 = SIZE(src,2)
+    s3 = SIZE(src,3)
+    d1 = SIZE(dst,1)
+    d2 = SIZE(dst,2)
+    d3 = SIZE(dst,3)
+    
+    ! rearrange source array as send buffer
 #ifdef SHM
-     work1_p = decomp%COL_INFO%SND_P_c
-     call mem_split_xy_complex(src, s1, s2, s3, work1, dims(1), &
-                               decomp%x1dist, decomp)
+    work1_p = decomp%COL_INFO%SND_P_c
+    call mem_split_xy_complex(src, s1, s2, s3, work1, dims(1), &
+         decomp%x1dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-                               decomp%x1dist, decomp)
+    call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+         decomp%x1dist, decomp)
 #else
-     call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
-                               decomp%x1dist, decomp)
+    call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
+         decomp%x1dist, decomp)
 #endif
 
 #endif
-
-     ! define receive buffer
+    
+    ! define receive buffer
 #ifdef SHM
-     work2_p = decomp%COL_INFO%RCV_P_c
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    work2_p = decomp%COL_INFO%RCV_P_c
+    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-
-     ! transpose using MPI_ALLTOALL(V)
+    
+    ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-     if (decomp%COL_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
-                           complex_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
-                           complex_type, decomp%COL_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
+    if (decomp%COL_INFO%CORE_ME==1) THEN
+       call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
+            complex_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
+            complex_type, decomp%COL_INFO%SMP_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    end if
 #else
 #ifdef EVEN
-     call MPI_ALLTOALL(work1_c, decomp%x1count, &
-                       complex_type, work2_c, decomp%y1count, &
-                       complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+    call MPI_ALLTOALL(work1_c, decomp%x1count, &
+         complex_type, work2_c, decomp%y1count, &
+         complex_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-     call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
-                        complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
+         complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
+         complex_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-     call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
-                        complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
+         complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
+         complex_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-     ! rearrange receive buffer
+    ! rearrange receive buffer
 #ifdef SHM
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_xy_complex(work2, d1, d2, d3, dst, dims(1), &
-                               decomp%y1dist, decomp)
+    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    call mem_merge_xy_complex(work2, d1, d2, d3, dst, dims(1), &
+         decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-                               decomp%y1dist, decomp)
+    call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+         decomp%y1dist, decomp)
 #else
-     call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
-                               decomp%y1dist, decomp)
+    call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
+         decomp%y1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
+    if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
 #endif
 
-     return
+    return
   end subroutine transpose_x_to_y_complex
 
-  subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_split_xy_real(in,n1,n2,n3,out,iproc,dist,decomp)
+
+    implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     real(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: n1,n2,n3
+    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
+    real(mytype), dimension(*), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+    attributes(device) :: out
+    integer :: istat
 #endif
+    
+    integer :: i,j,k, m,i1,i2,pos
 
-     integer :: i, j, k, m, i1, i2, pos
-
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then 
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%x1disp_o(m) + 1
+       pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+       pos = m * decomp%x1count + 1
 #else
-        pos = decomp%x1disp(m) + 1
+       pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = 1, n2
-              do i = i1, i2
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=1,n2
+             do i=i1,i2
+                out(pos) = in(i,j,k)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_split_xy_real
 
-  subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_split_xy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+
+    implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     complex(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: n1,n2,n3
+    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
+    complex(mytype), dimension(*), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+    attributes(device) :: out
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2,pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then 
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%x1disp_o(m) + 1
+       pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+       pos = m * decomp%x1count + 1
 #else
-        pos = decomp%x1disp(m) + 1
+       pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = 1, n2
-              do i = i1, i2
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=1,n2
+             do i=i1,i2
+                out(pos) = in(i,j,k)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_split_xy_complex
 
-  subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_merge_xy_real(in,n1,n2,n3,out,iproc,dist,decomp)
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(*), intent(IN) :: in
-     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    integer, intent(IN) :: n1,n2,n3
+    real(mytype), dimension(*), intent(IN) :: in
+    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+    attributes(device) :: in
+    integer :: istat
 #endif
+        
+    integer :: i,j,k, m,i1,i2, pos
 
-     integer :: i, j, k, m, i1, i2, pos
-
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%y1disp_o(m) + 1
+       pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+       pos = m * decomp%y1count + 1
 #else
-        pos = decomp%y1disp(m) + 1
+       pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=i1,i2
+             do i=1,n1
+                out(i,j,k) = in(pos)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_merge_xy_real
 
-  subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_merge_xy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(*), intent(IN) :: in
-     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    integer, intent(IN) :: n1,n2,n3
+    complex(mytype), dimension(*), intent(IN) :: in
+    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+    attributes(device) :: in
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%y1disp_o(m) + 1
+       pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+       pos = m * decomp%y1count + 1
 #else
-        pos = decomp%y1disp(m) + 1
+       pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=i1,i2
+             do i=1,n1
+                out(i,j,k) = in(pos)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_merge_xy_complex
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index a742ead6..1e7c62c2 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,474 +13,479 @@
 
   subroutine transpose_y_to_x_real_short(src, dst)
 
-     implicit none
+    implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+    real(mytype), dimension(:,:,:), intent(IN) :: src
+    real(mytype), dimension(:,:,:), intent(OUT) :: dst
 
-     call transpose_y_to_x(src, dst, decomp_main)
+    call transpose_y_to_x(src, dst, decomp_main)
 
   end subroutine transpose_y_to_x_real_short
 
   subroutine transpose_y_to_x_real(src, dst, decomp)
 
-     implicit none
-
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    real(mytype), dimension(:,:,:), intent(IN) :: src
+    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU) && defined(_NCCL)
-     type(ncclResult) :: nccl_stat
-     integer :: col_rank_id, cuda_stat
+    type(ncclResult) :: nccl_stat
+    integer :: col_rank_id, cuda_stat
 #endif
 
 #ifdef SHM
-     real(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
+    real(mytype) :: work1(*), work2(*)
+    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+    
+    integer :: s1,s2,s3,d1,d2,d3
+    integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
+    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+    s1 = SIZE(src,1)
+    s2 = SIZE(src,2)
+    s3 = SIZE(src,3)
+    d1 = SIZE(dst,1)
+    d2 = SIZE(dst,2)
+    d3 = SIZE(dst,3)
 
-     ! rearrange source array as send buffer
+    ! rearrange source array as send buffer
 #ifdef SHM
-     work1_p = decomp%COL_INFO%SND_P
-     call mem_split_yx_real(src, s1, s2, s3, work1, dims(1), &
-                            decomp%y1dist, decomp)
+    work1_p = decomp%COL_INFO%SND_P
+    call mem_split_yx_real(src, s1, s2, s3, work1, dims(1), &
+         decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
-                            decomp%y1dist, decomp)
+    call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
+         decomp%y1dist, decomp)
 #else
-     call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
-                            decomp%y1dist, decomp)
+    call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
+         decomp%y1dist, decomp)
 #endif
 
 #endif
 
-     ! define receive buffer
+    ! define receive buffer
 #ifdef SHM
-     work2_p = decomp%COL_INFO%RCV_P
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    work2_p = decomp%COL_INFO%RCV_P
+    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-
-     ! transpose using MPI_ALLTOALL(V)
+    
+    ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-     if (decomp%COL_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
-                           real_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
-                           real_type, decomp%COL_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
+    if (decomp%COL_INFO%CORE_ME==1) THEN
+       call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
+            real_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
+            real_type, decomp%COL_INFO%SMP_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    end if
 #else
 #ifdef EVEN
-     call MPI_ALLTOALL(work1_r, decomp%y1count, &
-                       real_type, work2_r, decomp%x1count, &
-                       real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+    call MPI_ALLTOALL(work1_r, decomp%y1count, &
+         real_type, work2_r, decomp%x1count, &
+         real_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     nccl_stat = ncclGroupStart()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-     do col_rank_id = 0, (col_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
-                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
-                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-     end do
-     nccl_stat = ncclGroupEnd()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+    nccl_stat = ncclGroupStart()
+    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+    do col_rank_id = 0, (col_comm_size - 1)
+     nccl_stat = ncclSend(work1_r_d( decomp%y1disp(col_rank_id)+1 ), decomp%y1cnts(col_rank_id), &
+       ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
+     nccl_stat = ncclRecv(work2_r_d( decomp%x1disp(col_rank_id)+1 ), decomp%x1cnts(col_rank_id), &
+       ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
+    end do
+    nccl_stat = ncclGroupEnd()
+    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-     call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
-                        real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
+         real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
+         real_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
-                        real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
+         real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
+         real_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-     ! rearrange receive buffer
+    ! rearrange receive buffer
 #ifdef SHM
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_yx_real(work2, d1, d2, d3, dst, dims(1), &
-                            decomp%x1dist, decomp)
+    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    call mem_merge_yx_real(work2, d1, d2, d3, dst, dims(1), &
+         decomp%x1dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-                            decomp%x1dist, decomp)
+    call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+         decomp%x1dist, decomp)
 #else
-     call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
-                            decomp%x1dist, decomp)
+    call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
+         decomp%x1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
+    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
 #endif
 
-     return
+    return
   end subroutine transpose_y_to_x_real
 
+
   subroutine transpose_y_to_x_complex_short(src, dst)
 
-     implicit none
+    implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+    complex(mytype), dimension(:,:,:), intent(IN) :: src
+    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
 
-     call transpose_y_to_x(src, dst, decomp_main)
+    call transpose_y_to_x(src, dst, decomp_main)
 
   end subroutine transpose_y_to_x_complex_short
 
   subroutine transpose_y_to_x_complex(src, dst, decomp)
 
-     implicit none
-
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    complex(mytype), dimension(:,:,:), intent(IN) :: src
+    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #ifdef SHM
-     complex(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
+    complex(mytype) :: work1(*), work2(*)
+    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+    
+    integer :: s1,s2,s3,d1,d2,d3
+    integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
+    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
-
-     ! rearrange source array as send buffer
+    s1 = SIZE(src,1)
+    s2 = SIZE(src,2)
+    s3 = SIZE(src,3)
+    d1 = SIZE(dst,1)
+    d2 = SIZE(dst,2)
+    d3 = SIZE(dst,3)
+    
+    ! rearrange source array as send buffer
 #ifdef SHM
-     work1_p = decomp%COL_INFO%SND_P_c
-     call mem_split_yx_complex(src, s1, s2, s3, work1, dims(1), &
-                               decomp%y1dist, decomp)
+    work1_p = decomp%COL_INFO%SND_P_c
+    call mem_split_yx_complex(src, s1, s2, s3, work1, dims(1), &
+         decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-                               decomp%y1dist, decomp)
+    call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+         decomp%y1dist, decomp)
 #else
-     call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
-                               decomp%y1dist, decomp)
+    call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
+         decomp%y1dist, decomp)
 #endif
 
 #endif
-
-     ! define receive buffer
+    
+    ! define receive buffer
 #ifdef SHM
-     work2_p = decomp%COL_INFO%RCV_P_c
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    work2_p = decomp%COL_INFO%RCV_P_c
+    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-
-     ! transpose using MPI_ALLTOALL(V)
+    
+    ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-     if (decomp%COL_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
-                           complex_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
-                           complex_type, decomp%COL_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
+    if (decomp%COL_INFO%CORE_ME==1) THEN
+       call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
+            complex_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
+            complex_type, decomp%COL_INFO%SMP_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    end if
 #else
 #ifdef EVEN
-     call MPI_ALLTOALL(work1_c, decomp%y1count, &
-                       complex_type, work2_c, decomp%x1count, &
-                       complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+    call MPI_ALLTOALL(work1_c, decomp%y1count, &
+         complex_type, work2_c, decomp%x1count, &
+         complex_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-     call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
-                        complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
+         complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
+         complex_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-     call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
-                        complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
+         complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
+         complex_type, DECOMP_2D_COMM_COL, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-     ! rearrange receive buffer
+    ! rearrange receive buffer
 #ifdef SHM
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_yx_complex(work2, d1, d2, d3, dst, dims(1), &
-                               decomp%x1dist, decomp)
+    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    call mem_merge_yx_complex(work2, d1, d2, d3, dst, dims(1), &
+         decomp%x1dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-                               decomp%x1dist, decomp)
+    call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+         decomp%x1dist, decomp)
 #else
-     call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
-                               decomp%x1dist, decomp)
+    call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
+         decomp%x1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
+    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
 #endif
 
-     return
+    return
   end subroutine transpose_y_to_x_complex
 
-  subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_split_yx_real(in,n1,n2,n3,out,iproc,dist,decomp)
+
+    implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     real(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: n1,n2,n3
+    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
+    real(mytype), dimension(*), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+    attributes(device) :: out
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2,pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then 
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%y1disp_o(m) + 1
+       pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+       pos = m * decomp%y1count + 1
 #else
-        pos = decomp%y1disp(m) + 1
+       pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=i1,i2
+             do i=1,n1
+                out(pos) = in(i,j,k)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_split_yx_real
 
-  subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_split_yx_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+
+    implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     complex(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: n1,n2,n3
+    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
+    complex(mytype), dimension(*), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+    attributes(device) :: out
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2,pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then 
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%y1disp_o(m) + 1
+       pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+       pos = m * decomp%y1count + 1
 #else
-        pos = decomp%y1disp(m) + 1
+       pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=i1,i2
+             do i=1,n1
+                out(pos) = in(i,j,k)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_split_yx_complex
 
-  subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_merge_yx_real(in,n1,n2,n3,out,iproc,dist,decomp)
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(*), intent(IN) :: in
-     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    integer, intent(IN) :: n1,n2,n3
+    real(mytype), dimension(*), intent(IN) :: in
+    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+    attributes(device) :: in
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%x1disp_o(m) + 1
+       pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+       pos = m * decomp%x1count + 1
 #else
-        pos = decomp%x1disp(m) + 1
+       pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = 1, n2
-              do i = i1, i2
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=1,n2
+             do i=i1,i2
+                out(i,j,k) = in(pos)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_merge_yx_real
 
-  subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_merge_yx_complex(in,n1,n2,n3,out,iproc,dist,decomp)
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(*), intent(IN) :: in
-     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    integer, intent(IN) :: n1,n2,n3
+    complex(mytype), dimension(*), intent(IN) :: in
+    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+    attributes(device) :: in
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%x1disp_o(m) + 1
+       pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+       pos = m * decomp%x1count + 1
 #else
-        pos = decomp%x1disp(m) + 1
+       pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = 1, n2
-              do i = i1, i2
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=1,n2
+             do i=i1,i2
+                out(i,j,k) = in(pos)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_merge_yx_complex
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 62bc9071..70f427cf 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,484 +13,488 @@
 
   subroutine transpose_y_to_z_real_short(src, dst)
 
-     implicit none
+    implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+    real(mytype), dimension(:,:,:), intent(IN) :: src
+    real(mytype), dimension(:,:,:), intent(OUT) :: dst
 
-     call transpose_y_to_z(src, dst, decomp_main)
+    call transpose_y_to_z(src, dst, decomp_main)
 
   end subroutine transpose_y_to_z_real_short
 
   subroutine transpose_y_to_z_real(src, dst, decomp)
 
-     implicit none
-
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    real(mytype), dimension(:,:,:), intent(IN) :: src
+    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     type(ncclResult) :: nccl_stat
-     integer :: row_rank_id, cuda_stat
+    type(ncclResult) :: nccl_stat
+    integer :: row_rank_id, cuda_stat
 #endif
-     integer :: istat
+    integer :: istat
 #endif
 
 #ifdef SHM
-     real(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
+    real(mytype) :: work1(*), work2(*)
+    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+    
+    integer :: s1,s2,s3,d1,d2,d3
+    integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
+    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+    s1 = SIZE(src,1)
+    s2 = SIZE(src,2)
+    s3 = SIZE(src,3)
+    d1 = SIZE(dst,1)
+    d2 = SIZE(dst,2)
+    d3 = SIZE(dst,3)
 
-     ! rearrange source array as send buffer
+    ! rearrange source array as send buffer
 #ifdef SHM
-     work1_p = decomp%ROW_INFO%SND_P
-     call mem_split_yz_real(src, s1, s2, s3, work1, dims(2), &
-                            decomp%y2dist, decomp)
+    work1_p = decomp%ROW_INFO%SND_P
+    call mem_split_yz_real(src, s1, s2, s3, work1, dims(2), &
+         decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
-                            decomp%y2dist, decomp)
+    call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
+         decomp%y2dist, decomp)
 #else
-     call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
-                            decomp%y2dist, decomp)
+    call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
+         decomp%y2dist, decomp)
 #endif
 
 #endif
 
-     ! define receive buffer
+    ! define receive buffer
 #ifdef SHM
-     work2_p = decomp%ROW_INFO%RCV_P
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    work2_p = decomp%ROW_INFO%RCV_P
+    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-
+    
 #ifdef SHM
-     if (decomp%ROW_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
-                           real_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
-                           real_type, decomp%ROW_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
+    if (decomp%ROW_INFO%CORE_ME==1) THEN
+       call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
+            real_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
+            real_type, decomp%ROW_INFO%SMP_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    end if
 #else
 #ifdef EVEN
-     if (decomp%even) then
-        call MPI_ALLTOALL(work1_r, decomp%y2count, &
-                          real_type, dst, decomp%z2count, &
-                          real_type, DECOMP_2D_COMM_ROW, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     else
-        call MPI_ALLTOALL(work1_r, decomp%y2count, &
-                          real_type, work2_r, decomp%z2count, &
-                          real_type, DECOMP_2D_COMM_ROW, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
-     end if
+    if (decomp%even) then
+       call MPI_ALLTOALL(work1_r, decomp%y2count, &
+            real_type, dst, decomp%z2count, &
+            real_type, DECOMP_2D_COMM_ROW, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    else
+       call MPI_ALLTOALL(work1_r, decomp%y2count, &
+            real_type, work2_r, decomp%z2count, &
+            real_type, DECOMP_2D_COMM_ROW, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+    end if
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     nccl_stat = ncclGroupStart()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-     do row_rank_id = 0, (row_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
-                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+    nccl_stat = ncclGroupStart()
+    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+    do row_rank_id = 0, (row_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d( decomp%y2disp(row_rank_id)+1 ), decomp%y2cnts(row_rank_id), &
+          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
-                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+        nccl_stat = ncclRecv(work2_r_d( decomp%z2disp(row_rank_id)+1 ), decomp%z2cnts(row_rank_id), &
+          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-     end do
-     nccl_stat = ncclGroupEnd()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+    end do
+    nccl_stat = ncclGroupEnd()
+    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-     call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
-                        real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
-                        real_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
+         real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
+         real_type, DECOMP_2D_COMM_ROW, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
-                        real_type, dst, decomp%z2cnts, decomp%z2disp, &
-                        real_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
+         real_type, dst, decomp%z2cnts, decomp%z2disp, &
+         real_type, DECOMP_2D_COMM_ROW, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-     ! rearrange receive buffer
+    ! rearrange receive buffer
 #ifdef SHM
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_yz_real(work2, d1, d2, d3, dst, dims(2), &
-                            decomp%z2dist, decomp)
+    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    call mem_merge_yz_real(work2, d1, d2, d3, dst, dims(2), &
+         decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-     if (.not. decomp%even) then
-        call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
-                               decomp%z2dist, decomp)
-     end if
+    if (.not. decomp%even) then
+       call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
+         decomp%z2dist, decomp)
+    end if
 #else
-     ! note the receive buffer is already in natural (i,j,k) order
-     ! so no merge operation needed
+    ! note the receive buffer is already in natural (i,j,k) order
+    ! so no merge operation needed
 
 #if defined(_GPU)
-     istat = cudaMemcpy(dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
+    istat = cudaMemcpy( dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
 #endif
 
 #endif
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
+    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
 #endif
 
-     return
+    return
   end subroutine transpose_y_to_z_real
 
+
   subroutine transpose_y_to_z_complex_short(src, dst)
 
-     implicit none
+    implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+    complex(mytype), dimension(:,:,:), intent(IN) :: src
+    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
 
-     call transpose_y_to_z(src, dst, decomp_main)
+    call transpose_y_to_z(src, dst, decomp_main)
 
   end subroutine transpose_y_to_z_complex_short
 
   subroutine transpose_y_to_z_complex(src, dst, decomp)
 
-     implicit none
-
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    complex(mytype), dimension(:,:,:), intent(IN) :: src
+    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-     integer :: istat
+    integer :: istat
 #endif
 
 #ifdef SHM
-     complex(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
+    complex(mytype) :: work1(*), work2(*)
+    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+    
+    integer :: s1,s2,s3,d1,d2,d3
+    integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
+    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
-
-     ! rearrange source array as send buffer
+    s1 = SIZE(src,1)
+    s2 = SIZE(src,2)
+    s3 = SIZE(src,3)
+    d1 = SIZE(dst,1)
+    d2 = SIZE(dst,2)
+    d3 = SIZE(dst,3)
+    
+    ! rearrange source array as send buffer
 #ifdef SHM
-     work1_p = decomp%ROW_INFO%SND_P_c
-     call mem_split_yz_complex(src, s1, s2, s3, work1, dims(2), &
-                               decomp%y2dist, decomp)
+    work1_p = decomp%ROW_INFO%SND_P_c
+    call mem_split_yz_complex(src, s1, s2, s3, work1, dims(2), &
+         decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
-                               decomp%y2dist, decomp)
+    call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
+         decomp%y2dist, decomp)
 #else
-     call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
-                               decomp%y2dist, decomp)
+    call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
+         decomp%y2dist, decomp)
 #endif
 
 #endif
-
-     ! define receive buffer
+    
+    ! define receive buffer
 #ifdef SHM
-     work2_p = decomp%ROW_INFO%RCV_P_c
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    work2_p = decomp%ROW_INFO%RCV_P_c
+    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-
+    
 #ifdef SHM
-     if (decomp%ROW_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
-                           complex_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
-                           complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
+    if (decomp%ROW_INFO%CORE_ME==1) THEN
+       call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
+            complex_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
+            complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    end if
 #else
 #ifdef EVEN
-     if (decomp%even) then
-        call MPI_ALLTOALL(work1_c, decomp%y2count, &
-                          complex_type, dst, decomp%z2count, &
-                          complex_type, DECOMP_2D_COMM_ROW, ierror)
-     else
-        call MPI_ALLTOALL(work1_c, decomp%y2count, &
-                          complex_type, work2_c, decomp%z2count, &
-                          complex_type, DECOMP_2D_COMM_ROW, ierror)
-     end if
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+    if (decomp%even) then
+       call MPI_ALLTOALL(work1_c, decomp%y2count, &
+            complex_type, dst, decomp%z2count, &
+            complex_type, DECOMP_2D_COMM_ROW, ierror)
+    else
+       call MPI_ALLTOALL(work1_c, decomp%y2count, &
+            complex_type, work2_c, decomp%z2count, &
+            complex_type, DECOMP_2D_COMM_ROW, ierror)
+    end if
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-     call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
-                        complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
-                        complex_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
+         complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
+         complex_type, DECOMP_2D_COMM_ROW, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-     call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
-                        complex_type, dst, decomp%z2cnts, decomp%z2disp, &
-                        complex_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
+         complex_type, dst, decomp%z2cnts, decomp%z2disp, &
+         complex_type, DECOMP_2D_COMM_ROW, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-     ! rearrange receive buffer
+    ! rearrange receive buffer
 #ifdef SHM
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_yz_complex(work2, d1, d2, d3, dst, dims(2), &
-                               decomp%z2dist, decomp)
+    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    call mem_merge_yz_complex(work2, d1, d2, d3, dst, dims(2), &
+         decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-     if (.not. decomp%even) then
-        call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
-                                  decomp%z2dist, decomp)
-     end if
+    if (.not. decomp%even) then
+       call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
+         decomp%z2dist, decomp)
+    end if
 #else
-     ! note the receive buffer is already in natural (i,j,k) order
-     ! so no merge operation needed
+    ! note the receive buffer is already in natural (i,j,k) order
+    ! so no merge operation needed
 
 #if defined(_GPU)
-     istat = cudaMemcpy(dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
+    istat = cudaMemcpy( dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
 #endif
 
 #endif
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
+    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
 #endif
 
-     return
+    return
   end subroutine transpose_y_to_z_complex
 
-  subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_split_yz_real(in,n1,n2,n3,out,iproc,dist,decomp)
 
-     implicit none
+    implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     real(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: n1,n2,n3
+    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
+    real(mytype), dimension(*), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+    attributes(device) :: out
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2,pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then 
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%y2disp_o(m) + 1
+       pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+       pos = m * decomp%y2count + 1
 #else
-        pos = decomp%y2disp(m) + 1
+       pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=i1,i2
+             do i=1,n1
+                out(pos) = in(i,j,k)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_split_yz_real
 
-  subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_split_yz_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+
+    implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     complex(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    integer, intent(IN) :: n1,n2,n3
+    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
+    complex(mytype), dimension(*), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+    attributes(device) :: out
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2,pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then 
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%y2disp_o(m) + 1
+       pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+       pos = m * decomp%y2count + 1
 #else
-        pos = decomp%y2disp(m) + 1
+       pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=i1,i2
+             do i=1,n1
+                out(pos) = in(i,j,k)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_split_yz_complex
 
-  subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
-
-     implicit none
-
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(*), intent(IN) :: in
-     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-     integer :: i, j, k, m, i1, i2, pos
-
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+  subroutine mem_merge_yz_real(in,n1,n2,n3,out,iproc,dist,decomp)
+
+    implicit none
+    
+    integer, intent(IN) :: n1,n2,n3
+    real(mytype), dimension(*), intent(IN) :: in
+    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    
+    integer :: i,j,k, m,i1,i2, pos
+
+    do m=0,iproc-1
+       if (m==0) then
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%z2disp_o(m) + 1
+       pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+       pos = m * decomp%z2count + 1
 #else
-        pos = decomp%z2disp(m) + 1
+       pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-        do k = i1, i2
-           do j = 1, n2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-     end do
+       do k=i1,i2
+          do j=1,n2
+             do i=1,n1
+                out(i,j,k) = in(pos)
+                pos = pos + 1
+             end do
+          end do
+       end do
+    end do
 
-     return
+    return
   end subroutine mem_merge_yz_real
 
-  subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
-
-     implicit none
-
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(*), intent(IN) :: in
-     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
-
-     integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+  subroutine mem_merge_yz_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+    
+    implicit none
+    
+    integer, intent(IN) :: n1,n2,n3
+    complex(mytype), dimension(*), intent(IN) :: in
+    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    
+    integer :: i,j,k, m,i1,i2, pos
+
+    do m=0,iproc-1
+       if (m==0) then
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%z2disp_o(m) + 1
+       pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+       pos = m * decomp%z2count + 1
 #else
-        pos = decomp%z2disp(m) + 1
+       pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-        do k = i1, i2
-           do j = 1, n2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-     end do
+       do k=i1,i2
+          do j=1,n2
+             do i=1,n1
+                out(i,j,k) = in(pos)
+                pos = pos + 1
+             end do
+          end do
+       end do
+    end do
 
-     return
+    return
   end subroutine mem_merge_yz_complex
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index a2f86844..33030a7d 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! 
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,485 +13,490 @@
 
   subroutine transpose_z_to_y_real_short(src, dst)
 
-     implicit none
+    implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+    real(mytype), dimension(:,:,:), intent(IN) :: src
+    real(mytype), dimension(:,:,:), intent(OUT) :: dst
 
-     call transpose_z_to_y(src, dst, decomp_main)
+    call transpose_z_to_y(src, dst, decomp_main)
 
   end subroutine transpose_z_to_y_real_short
 
   subroutine transpose_z_to_y_real(src, dst, decomp)
 
-     implicit none
-
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    real(mytype), dimension(:,:,:), intent(IN) :: src
+    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     type(ncclResult) :: nccl_stat
-     integer :: row_rank_id, cuda_stat
+    type(ncclResult) :: nccl_stat
+    integer :: row_rank_id, cuda_stat
 #endif
-     integer :: istat
+    integer :: istat
 #endif
 
 #ifdef SHM
-     real(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
+    real(mytype) :: work1(*), work2(*)
+    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+    
+    integer :: s1,s2,s3,d1,d2,d3
+    integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
+    if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+    s1 = SIZE(src,1)
+    s2 = SIZE(src,2)
+    s3 = SIZE(src,3)
+    d1 = SIZE(dst,1)
+    d2 = SIZE(dst,2)
+    d3 = SIZE(dst,3)
 
-     ! rearrange source array as send buffer
+    ! rearrange source array as send buffer
 #ifdef SHM
-     work1_p = decomp%ROW_INFO%SND_P
-     call mem_split_zy_real(src, s1, s2, s3, work1, dims(2), &
-                            decomp%z2dist, decomp)
+    work1_p = decomp%ROW_INFO%SND_P
+    call mem_split_zy_real(src, s1, s2, s3, work1, dims(2), &
+         decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-     if (.not. decomp%even) then
-        call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
-                               decomp%z2dist, decomp)
-     end if
+    if (.not. decomp%even) then
+       call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
+            decomp%z2dist, decomp)
+    end if
 #else
-     ! note the src array is suitable to be a send buffer
-     ! so no split operation needed
+    ! note the src array is suitable to be a send buffer
+    ! so no split operation needed
 
 #if defined(_GPU)
-     istat = cudaMemcpy(work1_r_d, src, s1*s2*s3)
+    istat = cudaMemcpy( work1_r_d, src, s1*s2*s3 )
 #endif
 
 #endif
 #endif
-
-     ! define receive buffer
+    
+    ! define receive buffer
 #ifdef SHM
-     work2_p = decomp%ROW_INFO%RCV_P
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    work2_p = decomp%ROW_INFO%RCV_P
+    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-
+    
 #ifdef SHM
-     if (decomp%ROW_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
-                           real_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
-                           real_type, decomp%ROW_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
+    if (decomp%ROW_INFO%CORE_ME==1) THEN
+       call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
+            real_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
+            real_type, decomp%ROW_INFO%SMP_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    end if
 #else
 #ifdef EVEN
-     if (decomp%even) then
-        call MPI_ALLTOALL(src, decomp%z2count, &
-                          real_type, work2_r, decomp%y2count, &
-                          real_type, DECOMP_2D_COMM_ROW, ierror)
-     else
-        call MPI_ALLTOALL(work1_r, decomp%z2count, &
-                          real_type, work2_r, decomp%y2count, &
-                          real_type, DECOMP_2D_COMM_ROW, ierror)
-     end if
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+    if (decomp%even) then
+       call MPI_ALLTOALL(src, decomp%z2count, &
+            real_type, work2_r, decomp%y2count, &
+            real_type, DECOMP_2D_COMM_ROW, ierror)
+    else
+       call MPI_ALLTOALL(work1_r, decomp%z2count, &
+            real_type, work2_r, decomp%y2count, &
+            real_type, DECOMP_2D_COMM_ROW, ierror)
+    end if
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     nccl_stat = ncclGroupStart()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-     do row_rank_id = 0, (row_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
-                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+    nccl_stat = ncclGroupStart()
+    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+    do row_rank_id = 0, (row_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d( decomp%z2disp(row_rank_id)+1 ), decomp%z2cnts(row_rank_id), &
+          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
-                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+        nccl_stat = ncclRecv(work2_r_d( decomp%y2disp(row_rank_id)+1 ), decomp%y2cnts(row_rank_id), &
+          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-     end do
-     nccl_stat = ncclGroupEnd()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+    end do
+    nccl_stat = ncclGroupEnd()
+    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-     call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
-                        real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
-                        real_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
+         real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
+         real_type, DECOMP_2D_COMM_ROW, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-                        real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
-                        real_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+         real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
+         real_type, DECOMP_2D_COMM_ROW, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #endif
 
-     ! rearrange receive buffer
+    ! rearrange receive buffer
 #ifdef SHM
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_zy_real(work2, d1, d2, d3, dst, dims(2), &
-                            decomp%y2dist, decomp)
+    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    call mem_merge_zy_real(work2, d1, d2, d3, dst, dims(2), &
+         decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
-                            decomp%y2dist, decomp)
+    call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
+         decomp%y2dist, decomp)
 #else
-     call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
-                            decomp%y2dist, decomp)
+    call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
+         decomp%y2dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
+    if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
 #endif
 
-     return
+    return
   end subroutine transpose_z_to_y_real
 
+
   subroutine transpose_z_to_y_complex_short(src, dst)
 
-     implicit none
+    implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+    complex(mytype), dimension(:,:,:), intent(IN) :: src
+    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
 
-     call transpose_z_to_y(src, dst, decomp_main)
+    call transpose_z_to_y(src, dst, decomp_main)
 
   end subroutine transpose_z_to_y_complex_short
 
   subroutine transpose_z_to_y_complex(src, dst, decomp)
 
-     implicit none
-
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    complex(mytype), dimension(:,:,:), intent(IN) :: src
+    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-     integer :: istat
+    integer :: istat
 #endif
 
 #ifdef SHM
-     complex(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
+    complex(mytype) :: work1(*), work2(*)
+    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+    
+    integer :: s1,s2,s3,d1,d2,d3
+    integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
+    if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
-
-     ! rearrange source array as send buffer
+    s1 = SIZE(src,1)
+    s2 = SIZE(src,2)
+    s3 = SIZE(src,3)
+    d1 = SIZE(dst,1)
+    d2 = SIZE(dst,2)
+    d3 = SIZE(dst,3)
+    
+    ! rearrange source array as send buffer
 #ifdef SHM
-     work1_p = decomp%ROW_INFO%SND_P_c
-     call mem_split_zy_complex(src, s1, s2, s3, work1, dims(2), &
-                               decomp%z2dist, decomp)
+    work1_p = decomp%ROW_INFO%SND_P_c
+    call mem_split_zy_complex(src, s1, s2, s3, work1, dims(2), &
+         decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-     if (.not. decomp%even) then
-        call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
-                                  decomp%z2dist, decomp)
-     end if
+    if (.not. decomp%even) then
+       call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
+            decomp%z2dist, decomp)
+    end if
 #else
-     ! note the src array is suitable to be a send buffer
-     ! so no split operation needed
+    ! note the src array is suitable to be a send buffer
+    ! so no split operation needed
 
 #if defined(_GPU)
-     istat = cudaMemcpy(work1_c_d, src, s1*s2*s3)
+    istat = cudaMemcpy( work1_c_d, src, s1*s2*s3 )
 #endif
 
 #endif
 #endif
-
-     ! define receive buffer
+    
+    ! define receive buffer
 #ifdef SHM
-     work2_p = decomp%ROW_INFO%RCV_P_c
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    work2_p = decomp%ROW_INFO%RCV_P_c
+    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-
+    
 #ifdef SHM
-     if (decomp%ROW_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
-                           complex_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
-                           complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
+    if (decomp%ROW_INFO%CORE_ME==1) THEN
+       call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
+            complex_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
+            complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
+       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    end if
 #else
 #ifdef EVEN
-     if (decomp%even) then
-        call MPI_ALLTOALL(src, decomp%z2count, &
-                          complex_type, work2_c, decomp%y2count, &
-                          complex_type, DECOMP_2D_COMM_ROW, ierror)
-     else
-        call MPI_ALLTOALL(work1_c, decomp%z2count, &
-                          complex_type, work2_c, decomp%y2count, &
-                          complex_type, DECOMP_2D_COMM_ROW, ierror)
-     end if
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+    if (decomp%even) then
+       call MPI_ALLTOALL(src, decomp%z2count, &
+            complex_type, work2_c, decomp%y2count, &
+            complex_type, DECOMP_2D_COMM_ROW, ierror)
+    else
+       call MPI_ALLTOALL(work1_c, decomp%z2count, &
+            complex_type, work2_c, decomp%y2count, &
+            complex_type, DECOMP_2D_COMM_ROW, ierror)
+    end if
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-     call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
-                        complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
-                        complex_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
+         complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
+         complex_type, DECOMP_2D_COMM_ROW, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-                        complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
-                        complex_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+    call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+         complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
+         complex_type, DECOMP_2D_COMM_ROW, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #endif
 
 #endif
 #endif
 
-     ! rearrange receive buffer
+    ! rearrange receive buffer
 #ifdef SHM
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_zy_complex(work2, d1, d2, d3, dst, dims(2), &
-                               decomp%y2dist, decomp)
+    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+    call mem_merge_zy_complex(work2, d1, d2, d3, dst, dims(2), &
+         decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-     call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
-                               decomp%y2dist, decomp)
+    call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
+         decomp%y2dist, decomp)
 #else
-     call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
-                               decomp%y2dist, decomp)
+    call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
+         decomp%y2dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
+    if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
 #endif
 
-     return
+    return
   end subroutine transpose_z_to_y_complex
 
-  ! pack/unpack ALLTOALL(V) buffers
 
-  subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
+  ! pack/unpack ALLTOALL(V) buffers
 
-     implicit none
+  subroutine mem_split_zy_real(in,n1,n2,n3,out,iproc,dist,decomp)
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     real(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer, intent(IN) :: n1,n2,n3
+    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
+    real(mytype), dimension(*), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    
+    integer :: i,j,k, m,i1,i2,pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then 
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%z2disp_o(m) + 1
+       pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+       pos = m * decomp%z2count + 1
 #else
-        pos = decomp%z2disp(m) + 1
+       pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-        do k = i1, i2
-           do j = 1, n2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-     end do
+       do k=i1,i2
+          do j=1,n2
+             do i=1,n1
+                out(pos) = in(i,j,k)
+                pos = pos + 1
+             end do
+          end do
+       end do
+    end do
 
-     return
+    return
   end subroutine mem_split_zy_real
 
-  subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_split_zy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     complex(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer, intent(IN) :: n1,n2,n3
+    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
+    complex(mytype), dimension(*), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    
+    integer :: i,j,k, m,i1,i2,pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then 
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%z2disp_o(m) + 1
+       pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+       pos = m * decomp%z2count + 1
 #else
-        pos = decomp%z2disp(m) + 1
+       pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-        do k = i1, i2
-           do j = 1, n2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-     end do
+       do k=i1,i2
+          do j=1,n2
+             do i=1,n1
+                out(pos) = in(i,j,k)
+                pos = pos + 1
+             end do
+          end do
+       end do
+    end do
 
-     return
+    return
   end subroutine mem_split_zy_complex
 
-  subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+  subroutine mem_merge_zy_real(in,n1,n2,n3,out,iproc,dist,decomp)
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(*), intent(IN) :: in
-     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+    implicit none
+    
+    integer, intent(IN) :: n1,n2,n3
+    real(mytype), dimension(*), intent(IN) :: in
+    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+    attributes(device) :: in
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%y2disp_o(m) + 1
+       pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+       pos = m * decomp%y2count + 1
 #else
-        pos = decomp%y2disp(m) + 1
+       pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=i1,i2
+             do i=1,n1
+                out(i,j,k) = in(pos)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_merge_zy_real
 
-  subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
-
-     implicit none
-
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(*), intent(IN) :: in
-     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
+  subroutine mem_merge_zy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+    
+    implicit none
+    
+    integer, intent(IN) :: n1,n2,n3
+    complex(mytype), dimension(*), intent(IN) :: in
+    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
+    integer, intent(IN) :: iproc
+    integer, dimension(0:iproc-1), intent(IN) :: dist
+    TYPE(DECOMP_INFO), intent(IN) :: decomp
+    
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+    attributes(device) :: in
+    integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+    integer :: i,j,k, m,i1,i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+    do m=0,iproc-1
+       if (m==0) then
+          i1 = 1
+          i2 = dist(0)
+       else
+          i1 = i2+1
+          i2 = i1+dist(m)-1
+       end if
 
 #ifdef SHM
-        pos = decomp%y2disp_o(m) + 1
+       pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+       pos = m * decomp%y2count + 1
 #else
-        pos = decomp%y2disp(m) + 1
+       pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-#endif
-     end do
-
-     return
+       do k=1,n3
+          do j=i1,i2
+             do i=1,n1
+                out(i,j,k) = in(pos)
+                pos = pos + 1
+             end do
+          end do
+       end do
+#endif
+    end do
+
+    return
   end subroutine mem_merge_zy_complex

From c572a03a674121ab198f903036e5a52bfb8651da Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 8 Dec 2022 09:47:07 +0100
Subject: [PATCH 029/436] Apply fprettify

---
 examples/fft_physical_x/fft_c2c_x.f90  |  230 +-
 examples/fft_physical_x/fft_grid_x.f90 |  234 +-
 examples/fft_physical_x/fft_r2c_x.f90  |  216 +-
 examples/fft_physical_z/fft_c2c_z.f90  |  231 +-
 examples/fft_physical_z/fft_r2c_z.f90  |  218 +-
 examples/halo_test/halo_test.f90       |  772 +++---
 examples/init_test/init_test.f90       |  120 +-
 examples/io_test/io_bench.f90          |   50 +-
 examples/io_test/io_plane_test.f90     |  230 +-
 examples/io_test/io_read.f90           |  128 +-
 examples/io_test/io_test.f90           |  210 +-
 examples/io_test/io_var_test.f90       |  552 ++--
 examples/test2d/test2d.f90             |  260 +-
 src/alloc.f90                          |  424 ++--
 src/decomp_2d.f90                      | 3077 +++++++++++-----------
 src/decomp_2d_init_fin.f90             |  591 +++--
 src/factor.f90                         |  150 +-
 src/fft_common.f90                     |  221 +-
 src/fft_common_3d.f90                  |  284 ++-
 src/fft_cufft.f90                      | 1558 ++++++------
 src/fft_fftw3.f90                      |  937 ++++---
 src/fft_fftw3_f03.f90                  | 1507 ++++++-----
 src/fft_generic.f90                    |  536 ++--
 src/fft_mkl.f90                        | 1048 ++++----
 src/glassman.f90                       |  329 ++-
 src/halo.f90                           |  120 +-
 src/halo_common.f90                    |  366 +--
 src/io.f90                             | 3217 ++++++++++++------------
 src/io_read_inflow.f90                 |   80 +-
 src/io_read_one.inc                    |   64 +-
 src/io_read_var.inc                    |   72 +-
 src/io_write_every.inc                 |  218 +-
 src/io_write_one.inc                   |   70 +-
 src/io_write_outflow.f90               |   80 +-
 src/io_write_plane.inc                 |  174 +-
 src/io_write_var.inc                   |   70 +-
 src/log.f90                            |  454 ++--
 src/transpose_x_to_y.f90               |  590 +++--
 src/transpose_y_to_x.f90               |  589 +++--
 src/transpose_y_to_z.f90               |  610 +++--
 src/transpose_z_to_y.f90               |  605 +++--
 41 files changed, 10694 insertions(+), 10798 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 2c48ddab..0e46d01d 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -1,128 +1,128 @@
 program fft_c2c_x
 
-  use decomp_2d
-  use decomp_2d_fft
-  use MPI
-#if defined(_GPU) 
-  use cudafor
-  use cufft
-  use openacc 
+   use decomp_2d
+   use decomp_2d_fft
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use cufft
+   use openacc
 #endif
 
-  implicit none
-  
-  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
-  integer :: nx, ny, nz
-  integer :: p_row=0, p_col=0
-  integer :: resize_domain
-  integer :: nranks_tot
-  
-  integer, parameter :: ntest = 10  ! repeat test this times
-  
-  complex(mytype), allocatable, dimension(:,:,:) :: in, out
-  
-  real(mytype) :: dr,di, error, err_all, n1,flops
-  integer :: ierror, i,j,k,m
-  real(mytype) :: t1, t2, t3 ,t4
-  
-  call MPI_INIT(ierror)
-  ! To resize the domain we need to know global number of ranks
-  ! This operation is also done as part of decomp_2d_init
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-  resize_domain = int(nranks_tot/4)+1
-  nx = nx_base*resize_domain
-  ny = ny_base*resize_domain
-  nz = nz_base*resize_domain
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   implicit none
+
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+
+   integer, parameter :: ntest = 10  ! repeat test this times
+
+   complex(mytype), allocatable, dimension(:, :, :) :: in, out
+
+   real(mytype) :: dr, di, error, err_all, n1, flops
+   integer :: ierror, i, j, k, m
+   real(mytype) :: t1, t2, t3, t4
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the c2c interface
+   ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  call decomp_2d_fft_init(PHYSICAL_IN_X) ! force the default x pencil
-
-  !  input is X-pencil data
-  ! output is Z-pencil data
-  allocate (in(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-  allocate (out(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-  ! initilise input
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           in(i,j,k) = cmplx(dr,di,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-     
-     ! forward FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
-     t2 = t2 + MPI_WTIME() - t1
-
-     ! inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     ! normalisation - note 2DECOMP&FFT doesn't normalise
-     !$acc kernels
-     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
-     !$acc end kernels
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_X) ! force the default x pencil
+
+   !  input is X-pencil data
+   ! output is Z-pencil data
+   allocate (in(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (out(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   ! initilise input
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            in(i, j, k) = cmplx(dr, di, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! forward FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+      t4 = t4 + MPI_WTIME() - t3
+
+      ! normalisation - note 2DECOMP&FFT doesn't normalise
+      !$acc kernels
+      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
+   ierror = cudaDeviceSynchronize()
 #endif
-  
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           dr = dr - real(in(i,j,k),mytype)
-           di = di - aimag(in(i,j,k))
-           error = error + sqrt(dr*dr + di*di)
-        end do
-     end do
-  end do
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-
-  if (nrank==0) then
-     write(*,*) '===== c2c interface ====='
-     write(*,*) 'error / mesh point: ', err_all 
-     write(*,*) 'time (sec): ', t1,t3
-     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
-     n1 = n1 ** (1._mytype/3._mytype)
-     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
-     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-     flops = flops * 3._mytype * n1**2  
-     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
-     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
-  end if
-  
-  deallocate(in,out)
-  call decomp_2d_fft_finalize
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            dr = dr - real(in(i, j, k), mytype)
+            di = di - aimag(in(i, j, k))
+            error = error + sqrt(dr*dr + di*di)
+         end do
+      end do
+   end do
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== c2c interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
+      n1 = n1**(1._mytype/3._mytype)
+      ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+      flops = flops*3._mytype*n1**2
+      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+   end if
+
+   deallocate (in, out)
+   call decomp_2d_fft_finalize
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
 end program fft_c2c_x
 
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 5b424138..7955a81e 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -1,130 +1,130 @@
 program fft_physical_x
 
-  use decomp_2d
-  use decomp_2d_fft
-  use MPI
-#if defined(_GPU) 
-  use cudafor
-  use cufft
-  use openacc 
+   use decomp_2d
+   use decomp_2d_fft
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use cufft
+   use openacc
 #endif
 
-  implicit none
-  
-  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
-  integer :: nx, ny, nz
-  integer :: p_row=0, p_col=0
-  integer :: resize_domain
-  integer :: nranks_tot
-  
-  integer, parameter :: ntest = 10  ! repeat test this times
- 
-  type(decomp_info), pointer :: ph=>null()
-  complex(mytype), allocatable, dimension(:,:,:) :: in, out
-  
-  real(mytype) :: dr,di, error, err_all, n1,flops
-  integer :: ierror, i,j,k,m
-  real(mytype) :: t1, t2, t3 ,t4
-  
-  call MPI_INIT(ierror)
-  ! To resize the domain we need to know global number of ranks
-  ! This operation is also done as part of decomp_2d_init
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-  resize_domain = int(nranks_tot/4)+1
-  nx = nx_base*resize_domain
-  ny = ny_base*resize_domain
-  nz = nz_base*resize_domain
-  call decomp_2d_init(nx+1,ny+1,nz+1,p_row,p_col)
+   implicit none
+
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+
+   integer, parameter :: ntest = 10  ! repeat test this times
+
+   type(decomp_info), pointer :: ph => null()
+   complex(mytype), allocatable, dimension(:, :, :) :: in, out
+
+   real(mytype) :: dr, di, error, err_all, n1, flops
+   integer :: ierror, i, j, k, m
+   real(mytype) :: t1, t2, t3, t4
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   call decomp_2d_init(nx + 1, ny + 1, nz + 1, p_row, p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the c2c interface
+   ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  call decomp_2d_fft_init(PHYSICAL_IN_X, nx, ny, nz) ! force the default x pencil
-  ph => decomp_2d_fft_get_ph()
-  !  input is X-pencil data
-  ! output is Z-pencil data
-  call alloc_x(in, ph, .true.)
-  call alloc_z(out, ph, .true.)
-  ! initilise input
-  do k=ph%xst(3),ph%xen(3)
-     do j=ph%xst(2),ph%xen(2)
-        do i=ph%xst(1),ph%xen(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           in(i,j,k) = cmplx(dr,di,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-     
-     ! forward FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
-     t2 = t2 + MPI_WTIME() - t1
-
-     ! inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     ! normalisation - note 2DECOMP&FFT doesn't normalise
-     !$acc kernels
-     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
-     !$acc end kernels
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_X, nx, ny, nz) ! force the default x pencil
+   ph => decomp_2d_fft_get_ph()
+   !  input is X-pencil data
+   ! output is Z-pencil data
+   call alloc_x(in, ph, .true.)
+   call alloc_z(out, ph, .true.)
+   ! initilise input
+   do k = ph%xst(3), ph%xen(3)
+      do j = ph%xst(2), ph%xen(2)
+         do i = ph%xst(1), ph%xen(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            in(i, j, k) = cmplx(dr, di, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! forward FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+      t4 = t4 + MPI_WTIME() - t3
+
+      ! normalisation - note 2DECOMP&FFT doesn't normalise
+      !$acc kernels
+      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
+   ierror = cudaDeviceSynchronize()
 #endif
-  
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=ph%xst(3),ph%xen(3) 
-     do j=ph%xst(2),ph%xen(2)
-        do i=ph%xst(1),ph%xen(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           dr = dr - real(in(i,j,k),mytype)
-           di = di - aimag(in(i,j,k))
-           error = error + sqrt(dr*dr + di*di)
-        end do
-     end do
-  end do
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-
-  if (nrank==0) then
-     write(*,*) '===== c2c interface ====='
-     write(*,*) 'error / mesh point: ', err_all 
-     write(*,*) 'time (sec): ', t1,t3
-     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
-     n1 = n1 ** (1._mytype/3._mytype)
-     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
-     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-     flops = flops * 3._mytype * n1**2  
-     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
-     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
-  end if
-  
-  deallocate(in,out)
-  nullify(ph)
-  call decomp_2d_fft_finalize
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = ph%xst(3), ph%xen(3)
+      do j = ph%xst(2), ph%xen(2)
+         do i = ph%xst(1), ph%xen(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            dr = dr - real(in(i, j, k), mytype)
+            di = di - aimag(in(i, j, k))
+            error = error + sqrt(dr*dr + di*di)
+         end do
+      end do
+   end do
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== c2c interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
+      n1 = n1**(1._mytype/3._mytype)
+      ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+      flops = flops*3._mytype*n1**2
+      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+   end if
+
+   deallocate (in, out)
+   nullify (ph)
+   call decomp_2d_fft_finalize
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
 end program fft_physical_x
 
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 045b8264..0603ab37 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -1,119 +1,119 @@
 program fft_r2c_x
 
-  use decomp_2d
-  use decomp_2d_fft
-  use MPI
-#if defined(_GPU) 
-  use cudafor
-  use cufft
-  use openacc 
+   use decomp_2d
+   use decomp_2d_fft
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use cufft
+   use openacc
 #endif
 
-  implicit none
-  
-  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
-  integer :: nx, ny, nz
-  integer :: p_row=0, p_col=0
-  integer :: resize_domain
-  integer :: nranks_tot
-  
-  integer, parameter :: ntest = 10  ! repeat test this times
-  
-  complex(mytype), allocatable, dimension(:,:,:) :: out
-  real(mytype), allocatable, dimension(:,:,:) :: in_r
-  
-  integer, dimension(3) :: fft_start, fft_end, fft_size
-  
-  real(mytype) :: dr, error, err_all
-  integer :: ierror, i,j,k,m
-  real(mytype) :: t1, t2, t3 ,t4
-  
-  call MPI_INIT(ierror)
-  ! To resize the domain we need to know global number of ranks
-  ! This operation is also done as part of decomp_2d_init
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-  resize_domain = int(nranks_tot/4)+1
-  nx = nx_base*resize_domain
-  ny = ny_base*resize_domain
-  nz = nz_base*resize_domain
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   implicit none
+
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+
+   integer, parameter :: ntest = 10  ! repeat test this times
+
+   complex(mytype), allocatable, dimension(:, :, :) :: out
+   real(mytype), allocatable, dimension(:, :, :) :: in_r
+
+   integer, dimension(3) :: fft_start, fft_end, fft_size
+
+   real(mytype) :: dr, error, err_all
+   integer :: ierror, i, j, k, m
+   real(mytype) :: t1, t2, t3, t4
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the r2c/c2r interface
+   ! Test the r2c/c2r interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  call decomp_2d_fft_init(PHYSICAL_IN_X)
-  
-  allocate (in_r(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
-  allocate (out(fft_start(1):fft_end(1), &
-                fft_start(2):fft_end(2), &
-                fft_start(3):fft_end(3)))
-  
-  ! initilise input
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-  
-     ! 3D r2c FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in_r, out)
-     t2 = t2 + MPI_WTIME() - t1
-  
-     ! 3D inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in_r)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     ! normalisation - note 2DECOMP&FFT doesn't normalise
-     !$acc kernels
-     in_r = in_r / (real(nx,mytype)*real(ny,mytype)*real(nz,mytype))
-     !$acc end kernels
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_X)
+
+   allocate (in_r(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   call decomp_2d_fft_get_size(fft_start, fft_end, fft_size)
+   allocate (out(fft_start(1):fft_end(1), &
+                 fft_start(2):fft_end(2), &
+                 fft_start(3):fft_end(3)))
+
+   ! initilise input
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! 3D r2c FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in_r, out)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! 3D inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in_r)
+      t4 = t4 + MPI_WTIME() - t3
+
+      ! normalisation - note 2DECOMP&FFT doesn't normalise
+      !$acc kernels
+      in_r = in_r/(real(nx, mytype)*real(ny, mytype)*real(nz, mytype))
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
-#endif 
-
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           error = error + abs(in_r(i,j,k)-dr)
-        end do
-     end do
-  end do
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-  
-  if (nrank==0) then
-     write(*,*) '===== r2c/c2r interface ====='
-     write(*,*) 'error / mesh point: ', err_all
-     write(*,*) 'time (sec): ', t1,t3
-  end if
-  
-  deallocate(in_r,out)
-  call decomp_2d_fft_finalize
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
+   ierror = cudaDeviceSynchronize()
+#endif
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            error = error + abs(in_r(i, j, k) - dr)
+         end do
+      end do
+   end do
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== r2c/c2r interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+   end if
+
+   deallocate (in_r, out)
+   call decomp_2d_fft_finalize
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
 end program fft_r2c_x
 
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index ed346cc4..e3581bf6 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -1,127 +1,126 @@
 program fft_c2c_z
 
-  use decomp_2d
-  use decomp_2d_fft
-  use MPI
-#if defined(_GPU) 
-  use cudafor
-  use cufft
-  use openacc 
+   use decomp_2d
+   use decomp_2d_fft
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use cufft
+   use openacc
 #endif
 
-  implicit none
-  
-  !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
-  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
-  integer :: nx, ny, nz
-  integer :: p_row=0, p_col=0
-  integer :: resize_domain
-  integer :: nranks_tot
-  
-  integer, parameter :: ntest = 10  ! repeat test this times
-  
-  complex(mytype), allocatable, dimension(:,:,:) :: in, out
-
-  real(mytype) :: dr,di, error, err_all, n1,flops
-  integer :: ierror, i,j,k,m
-  real(mytype) :: t1, t2, t3 ,t4
-  
-  call MPI_INIT(ierror)
-  ! To resize the domain we need to know global number of ranks
-  ! This operation is also done as part of decomp_2d_init
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-  resize_domain = int(nranks_tot/4)+1
-  nx = nx_base*resize_domain
-  ny = ny_base*resize_domain
-  nz = nz_base*resize_domain
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   implicit none
+
+   !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+
+   integer, parameter :: ntest = 10  ! repeat test this times
+
+   complex(mytype), allocatable, dimension(:, :, :) :: in, out
+
+   real(mytype) :: dr, di, error, err_all, n1, flops
+   integer :: ierror, i, j, k, m
+   real(mytype) :: t1, t2, t3, t4
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the c2c interface
+   ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
-  !  input is Z-pencil data
-  ! output is X-pencil data
-  allocate (in(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-  allocate (out(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-  ! initilise input
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           in(i,j,k) = cmplx(dr,di,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-     
-     ! forward FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
-     t2 = t2 + MPI_WTIME() - t1
-
-     ! inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     ! normalisation - note 2DECOMP&FFT doesn't normalise
-     !$acc kernels
-     in = in / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
-     !$acc end kernels
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
+   !  input is Z-pencil data
+   ! output is X-pencil data
+   allocate (in(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   allocate (out(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   ! initilise input
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            in(i, j, k) = cmplx(dr, di, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! forward FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+      t4 = t4 + MPI_WTIME() - t3
+
+      ! normalisation - note 2DECOMP&FFT doesn't normalise
+      !$acc kernels
+      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
+   ierror = cudaDeviceSynchronize()
 #endif
-  
-
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           di = dr
-           dr = dr - real(in(i,j,k),mytype)
-           di = di - aimag(in(i,j,k))
-           error = error + sqrt(dr*dr + di*di)
-        end do
-     end do
-  end do
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-
-  if (nrank==0) then
-     write(*,*) '===== c2c interface ====='
-     write(*,*) 'error / mesh point: ', err_all 
-     write(*,*) 'time (sec): ', t1,t3
-     n1 = real(nx,mytype) * real(ny,mytype) * real(nz,mytype)
-     n1 = n1 ** (1._mytype/3._mytype)
-     ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-     flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
-     ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-     flops = flops * 3._mytype * n1**2  
-     flops = 2._mytype * flops / ((t1+t3)/real(NTEST,mytype))
-     write(*,*) 'GFLOPS : ', flops / 1000._mytype**3
-  end if
-  
-  deallocate(in,out)
-  call decomp_2d_fft_finalize
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            di = dr
+            dr = dr - real(in(i, j, k), mytype)
+            di = di - aimag(in(i, j, k))
+            error = error + sqrt(dr*dr + di*di)
+         end do
+      end do
+   end do
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== c2c interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
+      n1 = n1**(1._mytype/3._mytype)
+      ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
+      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
+      flops = flops*3._mytype*n1**2
+      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+   end if
+
+   deallocate (in, out)
+   call decomp_2d_fft_finalize
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
 end program fft_c2c_z
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 86fbbd76..321f2421 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -1,121 +1,121 @@
 program fft_r2c_z
 
-  use decomp_2d
-  use decomp_2d_fft
-  use MPI
-#if defined(_GPU) 
-  use cudafor
-  use cufft
-  use openacc 
+   use decomp_2d
+   use decomp_2d_fft
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use cufft
+   use openacc
 #endif
 
-  implicit none
-  
-  !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
-  integer, parameter :: nx_base=17, ny_base=13, nz_base=11
-  integer :: nx, ny, nz
-  integer :: p_row=0, p_col=0
-  integer :: resize_domain
-  integer :: nranks_tot
-  
-  integer, parameter :: ntest = 10  ! repeat test this times
-  
-  complex(mytype), allocatable, dimension(:,:,:) :: out
-  real(mytype), allocatable, dimension(:,:,:) :: in_r
-
-  integer, dimension(3) :: fft_start, fft_end, fft_size
-  
-  real(mytype) :: dr, error, err_all
-  integer :: ierror, i,j,k,m
-  real(mytype) :: t1, t2, t3 ,t4
-  
-  call MPI_INIT(ierror)
-  ! To resize the domain we need to know global number of ranks
-  ! This operation is also done as part of decomp_2d_init
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-  resize_domain = int(nranks_tot/4)+1
-  nx = nx_base*resize_domain
-  ny = ny_base*resize_domain
-  nz = nz_base*resize_domain
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   implicit none
+
+   !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+
+   integer, parameter :: ntest = 10  ! repeat test this times
+
+   complex(mytype), allocatable, dimension(:, :, :) :: out
+   real(mytype), allocatable, dimension(:, :, :) :: in_r
+
+   integer, dimension(3) :: fft_start, fft_end, fft_size
+
+   real(mytype) :: dr, error, err_all
+   integer :: ierror, i, j, k, m
+   real(mytype) :: t1, t2, t3, t4
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Test the r2c/c2r interface
+   ! Test the r2c/c2r interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
-  
-  allocate (in_r(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-  call decomp_2d_fft_get_size(fft_start,fft_end,fft_size)
-  allocate (out(fft_start(1):fft_end(1), &
-                fft_start(2):fft_end(2), &
-                fft_start(3):fft_end(3)))
-  
-  ! initilise input
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           in_r(i,j,k) = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-        end do
-     end do
-  end do
-  
-  t2 = 0._mytype
-  t4 = 0._mytype
-  do m=1,ntest
-     
-     ! 3D r2c FFT
-     t1 = MPI_WTIME()
-     call decomp_2d_fft_3d(in_r, out)
-     t2 = t2 + MPI_WTIME() - t1
-  
-     ! 3D inverse FFT
-     t3 = MPI_WTIME()
-     call decomp_2d_fft_3d(out, in_r)
-     t4 = t4 + MPI_WTIME() - t3
-  
-     !$acc kernels
-     in_r = in_r / real(nx,mytype) / real(ny,mytype) /real(nz,mytype)
-     !$acc end kernels     
-
-  end do
+   call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
+
+   allocate (in_r(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   call decomp_2d_fft_get_size(fft_start, fft_end, fft_size)
+   allocate (out(fft_start(1):fft_end(1), &
+                 fft_start(2):fft_end(2), &
+                 fft_start(3):fft_end(3)))
+
+   ! initilise input
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+         end do
+      end do
+   end do
+
+   t2 = 0._mytype
+   t4 = 0._mytype
+   do m = 1, ntest
+
+      ! 3D r2c FFT
+      t1 = MPI_WTIME()
+      call decomp_2d_fft_3d(in_r, out)
+      t2 = t2 + MPI_WTIME() - t1
+
+      ! 3D inverse FFT
+      t3 = MPI_WTIME()
+      call decomp_2d_fft_3d(out, in_r)
+      t4 = t4 + MPI_WTIME() - t3
+
+      !$acc kernels
+      in_r = in_r/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      !$acc end kernels
+
+   end do
 #if defined(_GPU)
-  ierror = cudaDeviceSynchronize()
-#endif 
-
-  call MPI_ALLREDUCE(t2,t1,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t1 = t1 / real(nproc,mytype)
-  call MPI_ALLREDUCE(t4,t3,1,real_type,MPI_SUM, &
-       MPI_COMM_WORLD,ierror)
-  t3 = t3 / real(nproc,mytype)
-  
-  ! checking accuracy
-  error = 0._mytype
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           dr = real(i,mytype)/real(nx,mytype)*real(j,mytype) &
-                /real(ny,mytype)*real(k,mytype)/real(nz,mytype)
-           error = error + abs(in_r(i,j,k)-dr)
-           !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
-        end do
-     end do
-  end do
+   ierror = cudaDeviceSynchronize()
+#endif
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+
+   ! checking accuracy
+   error = 0._mytype
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
+                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            error = error + abs(in_r(i, j, k) - dr)
+            !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
+         end do
+      end do
+   end do
 !10 format('in_r final ', I2,1x,I2,1x,I2,1x,I2,1x,F12.6,1x,F12.6)
 
-  call MPI_ALLREDUCE(error,err_all,1,real_type,MPI_SUM,MPI_COMM_WORLD,ierror)
-  err_all = err_all / real(nx,mytype) / real(ny,mytype) / real(nz,mytype)
-  
-  if (nrank==0) then
-     write(*,*) '===== r2c/c2r interface ====='
-     write(*,*) 'error / mesh point: ', err_all
-     write(*,*) 'time (sec): ', t1,t3
-  end if
-  
-  deallocate(in_r,out)
-  call decomp_2d_fft_finalize
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
+   call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+
+   if (nrank == 0) then
+      write (*, *) '===== r2c/c2r interface ====='
+      write (*, *) 'error / mesh point: ', err_all
+      write (*, *) 'time (sec): ', t1, t3
+   end if
+
+   deallocate (in_r, out)
+   call decomp_2d_fft_finalize
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
 end program fft_r2c_z
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 9f52a2bd..2222d3c4 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -6,495 +6,495 @@
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 program halo_test
 
-  use mpi
+   use mpi
 
-  use decomp_2d
+   use decomp_2d
 
-  implicit none
+   implicit none
 
-  integer, parameter :: nx=171, ny=132, nz=113
-  integer :: p_row=0, p_col=0
+   integer, parameter :: nx = 171, ny = 132, nz = 113
+   integer :: p_row = 0, p_col = 0
 
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  real(mytype), allocatable, dimension(:,:,:) :: v1, v2, v3
-  real(mytype), allocatable, dimension(:,:,:) :: w1, w2, w3
-  real(mytype), allocatable, dimension(:,:,:) :: wk2, wk3
-  real(mytype), allocatable, dimension(:,:,:) :: uh, vh, wh
-  real(mytype), allocatable, dimension(:,:,:) :: div1, div2, div3, div4
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   real(mytype), allocatable, dimension(:, :, :) :: v1, v2, v3
+   real(mytype), allocatable, dimension(:, :, :) :: w1, w2, w3
+   real(mytype), allocatable, dimension(:, :, :) :: wk2, wk3
+   real(mytype), allocatable, dimension(:, :, :) :: uh, vh, wh
+   real(mytype), allocatable, dimension(:, :, :) :: div1, div2, div3, div4
 
-  integer :: i,j,k, ierror, n
+   integer :: i, j, k, ierror, n
 
-  integer, allocatable, dimension(:) :: seed
+   integer, allocatable, dimension(:) :: seed
 
-  real(mytype) :: err
-  integer :: xlast, ylast, zlast
+   real(mytype) :: err
+   integer :: xlast, ylast, zlast
 
-  integer :: nx_expected, ny_expected, nz_expected
-  
-  logical :: passing, all_pass
+   integer :: nx_expected, ny_expected, nz_expected
 
-  call MPI_INIT(ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   logical :: passing, all_pass
 
-  xlast = xsize(1) - 1
-  if (xend(2) == ny) then
-     ylast = xsize(2) - 1
-  else
-     ylast = xsize(2)
-  end if
-  if (xend(3) == nz) then
-     zlast = xsize(3) - 1
-  else
-     zlast = xsize(3)
-  end if
+   call MPI_INIT(ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-  call initialise()
-  call test_div_transpose()
-  call test_div_haloX()
-  call test_div_haloY()
-  call test_div_haloZ()
+   xlast = xsize(1) - 1
+   if (xend(2) == ny) then
+      ylast = xsize(2) - 1
+   else
+      ylast = xsize(2)
+   end if
+   if (xend(3) == nz) then
+      zlast = xsize(3) - 1
+   else
+      zlast = xsize(3)
+   end if
 
-  if (nrank == 0) then
-     write(*,*) '-----------------------------------------------'
-     write(*,*) "All pass: ", all_pass
-     write(*,*) '==============================================='
-  end if
+   call initialise()
+   call test_div_transpose()
+   call test_div_haloX()
+   call test_div_haloY()
+   call test_div_haloZ()
 
-  deallocate(u1,v1,w1,u2,v2,w2,u3,v3,w3)
-  deallocate(div1,div2,div3,div4)
+   if (nrank == 0) then
+      write (*, *) '-----------------------------------------------'
+      write (*, *) "All pass: ", all_pass
+      write (*, *) '==============================================='
+   end if
 
-  call decomp_2d_finalize 
+   deallocate (u1, v1, w1, u2, v2, w2, u3, v3, w3)
+   deallocate (div1, div2, div3, div4)
 
-  if (.not. all_pass) call decomp_2d_abort(1, "Error in halo_test")
+   call decomp_2d_finalize
 
-  call MPI_FINALIZE(ierror)
+   if (.not. all_pass) call decomp_2d_abort(1, "Error in halo_test")
+
+   call MPI_FINALIZE(ierror)
 
 contains
 
-  subroutine initialise()
+   subroutine initialise()
 
-    ! initialise u,v,w with random numbers in X-pencil
+      ! initialise u,v,w with random numbers in X-pencil
 #ifdef HALO_GLOBAL
-    allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    allocate(v1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    allocate(w1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (v1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (w1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
 #else
-    allocate(u1(xsize(1), xsize(2), xsize(3)))
-    allocate(v1(xsize(1), xsize(2), xsize(3)))
-    allocate(w1(xsize(1), xsize(2), xsize(3)))
+      allocate (u1(xsize(1), xsize(2), xsize(3)))
+      allocate (v1(xsize(1), xsize(2), xsize(3)))
+      allocate (w1(xsize(1), xsize(2), xsize(3)))
 #endif
 
-    call random_seed(size = n)
-    allocate(seed(n))
-    seed = nrank+1
-    call random_seed(put=seed)
-    call random_number(u1)
-    call random_number(v1)
-    call random_number(w1)
+      call random_seed(size=n)
+      allocate (seed(n))
+      seed = nrank + 1
+      call random_seed(put=seed)
+      call random_number(u1)
+      call random_number(v1)
+      call random_number(w1)
 
-    all_pass = .true.
+      all_pass = .true.
 
-  end subroutine initialise
+   end subroutine initialise
 
-  !=====================================================================
-  ! Calculate divergence using global transposition
-  !=====================================================================
-  subroutine test_div_transpose()
+   !=====================================================================
+   ! Calculate divergence using global transposition
+   !=====================================================================
+   subroutine test_div_transpose()
 
-    integer :: i1, in ! I loop start/end
-    integer :: j1, jn ! J loop start/end
-    integer :: k1, kn ! K loop start/end
+      integer :: i1, in ! I loop start/end
+      integer :: j1, jn ! J loop start/end
+      integer :: k1, kn ! K loop start/end
 
-    ! du/dx calculated on X-pencil
+      ! du/dx calculated on X-pencil
 #ifdef HALO_GLOBAL
-    allocate(div1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    k1 = xstart(3); kn = xend(3)
-    j1 = xstart(2); jn = xend(2)
+      allocate (div1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      k1 = xstart(3); kn = xend(3)
+      j1 = xstart(2); jn = xend(2)
 #else
-    allocate(div1(xsize(1), xsize(2), xsize(3)))
-    k1 = 1; kn = xsize(3)
-    j1 = 1; jn = xsize(2)
+      allocate (div1(xsize(1), xsize(2), xsize(3)))
+      k1 = 1; kn = xsize(3)
+      j1 = 1; jn = xsize(2)
 #endif
-    i1 = 2; in = xsize(1) - 1
-
-    div1 = 0.0_mytype
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             div1(i,j,k) = u1(i+1,j,k)-u1(i-1,j,k)
-          end do
-       end do
-    end do
-
-    ! dv/dy calculated on Y-pencil
+      i1 = 2; in = xsize(1) - 1
+
+      div1 = 0.0_mytype
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               div1(i, j, k) = u1(i + 1, j, k) - u1(i - 1, j, k)
+            end do
+         end do
+      end do
+
+      ! dv/dy calculated on Y-pencil
 #ifdef HALO_GLOBAL
-    allocate(v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    k1 = ystart(3); kn = yend(3)
-    i1 = ystart(1); in = yend(1)
+      allocate (v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      k1 = ystart(3); kn = yend(3)
+      i1 = ystart(1); in = yend(1)
 #else
-    allocate(v2(ysize(1), ysize(2), ysize(3)))
-    allocate(wk2(ysize(1), ysize(2), ysize(3)))
-    k1 = 1; kn = ysize(3)
-    i1 = 1; in = ysize(1)
+      allocate (v2(ysize(1), ysize(2), ysize(3)))
+      allocate (wk2(ysize(1), ysize(2), ysize(3)))
+      k1 = 1; kn = ysize(3)
+      i1 = 1; in = ysize(1)
 #endif
-    j1 = 2; jn = ysize(2) - 1
+      j1 = 2; jn = ysize(2) - 1
 
-    call transpose_x_to_y(v1,v2)
-    call transpose_x_to_y(div1,wk2)
+      call transpose_x_to_y(v1, v2)
+      call transpose_x_to_y(div1, wk2)
 
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             wk2(i,j,k) = wk2(i,j,k) + v2(i,j+1,k)-v2(i,j-1,k)
-          end do
-       end do
-    end do
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               wk2(i, j, k) = wk2(i, j, k) + v2(i, j + 1, k) - v2(i, j - 1, k)
+            end do
+         end do
+      end do
 
-    ! dw/dz calculated on Z-pencil
+      ! dw/dz calculated on Z-pencil
 #ifdef HALO_GLOBAL
-    allocate(w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-    allocate(wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-    j1 = zstart(2); jn = zend(2)
-    i1 = zstart(1); in = zend(1)
+      allocate (w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      j1 = zstart(2); jn = zend(2)
+      i1 = zstart(1); in = zend(1)
 #else
-    allocate(w2(ysize(1), ysize(2), ysize(3)))
-    allocate(w3(zsize(1), zsize(2), zsize(3)))
-    allocate(wk3(zsize(1), zsize(2), zsize(3)))
-    j1 = 1; jn = zsize(2)
-    i1 = 1; in = zsize(1)
+      allocate (w2(ysize(1), ysize(2), ysize(3)))
+      allocate (w3(zsize(1), zsize(2), zsize(3)))
+      allocate (wk3(zsize(1), zsize(2), zsize(3)))
+      j1 = 1; jn = zsize(2)
+      i1 = 1; in = zsize(1)
 #endif
-    k1 = 2; kn = zsize(3) - 1
-    
-    call transpose_x_to_y(w1,w2)
-    call transpose_y_to_z(w2,w3)
-    call transpose_y_to_z(wk2,wk3)
-
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             wk3(i,j,k) = wk3(i,j,k) + w3(i,j,k+1)-w3(i,j,k-1)
-          end do
-       end do
-    end do
-
-    ! result in X-pencil
-    call transpose_z_to_y(wk3,wk2)
-    call transpose_y_to_x(wk2,div1)
-
-    if (nrank==0) then
-       write(*,*) 'Calculated via global transposition'
+      k1 = 2; kn = zsize(3) - 1
+
+      call transpose_x_to_y(w1, w2)
+      call transpose_y_to_z(w2, w3)
+      call transpose_y_to_z(wk2, wk3)
+
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
+            end do
+         end do
+      end do
+
+      ! result in X-pencil
+      call transpose_z_to_y(wk3, wk2)
+      call transpose_y_to_x(wk2, div1)
+
+      if (nrank == 0) then
+         write (*, *) 'Calculated via global transposition'
 #ifdef DEBUG
-       write(*,*) (div1(i,i,i), i=2,13)
+         write (*, *) (div1(i, i, i), i=2, 13)
 #endif
-    end if
-
-    deallocate(v2,w2,w3,wk2,wk3)
-
-  end subroutine test_div_transpose
-
-  !=====================================================================
-  ! Calculate divergence using halo-cell exchange (data in X-pencil)
-  !=====================================================================
-  subroutine test_div_haloX()
-
-    integer :: i1, in ! I loop start/end
-    integer :: j1, jn ! J loop start/end
-    integer :: k1, kn ! K loop start/end
-    
-    ! Expected sizes
-    nx_expected = nx
-    ny_expected = xsize(2) + 2
-    nz_expected = xsize(3) + 2
-    
+      end if
+
+      deallocate (v2, w2, w3, wk2, wk3)
+
+   end subroutine test_div_transpose
+
+   !=====================================================================
+   ! Calculate divergence using halo-cell exchange (data in X-pencil)
+   !=====================================================================
+   subroutine test_div_haloX()
+
+      integer :: i1, in ! I loop start/end
+      integer :: j1, jn ! J loop start/end
+      integer :: k1, kn ! K loop start/end
+
+      ! Expected sizes
+      nx_expected = nx
+      ny_expected = xsize(2) + 2
+      nz_expected = xsize(3) + 2
+
 #ifdef HALO_GLOBAL
-    allocate(div2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    call update_halo(v1,vh,1,opt_global=.true.,opt_pencil=1)
-    call update_halo(w1,wh,1,opt_global=.true.,opt_pencil=1)
-    
-    k1 = xstart(3); kn = xend(3)
-    j1 = xstart(2); jn = xend(2)
+      allocate (div2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      call update_halo(v1, vh, 1, opt_global=.true., opt_pencil=1)
+      call update_halo(w1, wh, 1, opt_global=.true., opt_pencil=1)
+
+      k1 = xstart(3); kn = xend(3)
+      j1 = xstart(2); jn = xend(2)
 #else
-    allocate(div2(xsize(1), xsize(2), xsize(3)))
-    call update_halo(v1,vh,1,opt_pencil=1)
-    call update_halo(w1,wh,1,opt_pencil=1)
+      allocate (div2(xsize(1), xsize(2), xsize(3)))
+      call update_halo(v1, vh, 1, opt_pencil=1)
+      call update_halo(w1, wh, 1, opt_pencil=1)
 
-    k1 = 1; kn = xsize(3)
-    j1 = 1; jn = xsize(2)
+      k1 = 1; kn = xsize(3)
+      j1 = 1; jn = xsize(2)
 #endif
-    i1 = 2; in = xsize(1) - 1
-
-    call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "X:v")
-    call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
-    
-    div2 = 0.0_mytype
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             div2(i,j,k) = (u1(i+1,j,k)-u1(i-1,j,k)) &
-                  + (vh(i,j+1,k)-vh(i,j-1,k)) &
-                  + (wh(i,j,k+1)-wh(i,j,k-1))
-          end do
-       end do
-    end do
-
-    ! Compute error
-    call check_err(div2, "X")
-           
-    deallocate(vh,wh)
-
-  end subroutine test_div_haloX
-
-  !=====================================================================
-  ! Calculate divergence using halo-cell exchange (data in Y-pencil)
-  !=====================================================================
-  subroutine test_div_haloY()
-
-    integer :: i1, in ! I loop start/end
-    integer :: j1, jn ! J loop start/end
-    integer :: k1, kn ! K loop start/end
-
-    ! Expected sizes
-    nx_expected = ysize(1) + 2
-    ny_expected = ny
-    nz_expected = ysize(3) + 2
-  
+      i1 = 2; in = xsize(1) - 1
+
+      call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "X:v")
+      call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
+
+      div2 = 0.0_mytype
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               div2(i, j, k) = (u1(i + 1, j, k) - u1(i - 1, j, k)) &
+                               + (vh(i, j + 1, k) - vh(i, j - 1, k)) &
+                               + (wh(i, j, k + 1) - wh(i, j, k - 1))
+            end do
+         end do
+      end do
+
+      ! Compute error
+      call check_err(div2, "X")
+
+      deallocate (vh, wh)
+
+   end subroutine test_div_haloX
+
+   !=====================================================================
+   ! Calculate divergence using halo-cell exchange (data in Y-pencil)
+   !=====================================================================
+   subroutine test_div_haloY()
+
+      integer :: i1, in ! I loop start/end
+      integer :: j1, jn ! J loop start/end
+      integer :: k1, kn ! K loop start/end
+
+      ! Expected sizes
+      nx_expected = ysize(1) + 2
+      ny_expected = ny
+      nz_expected = ysize(3) + 2
+
 #ifdef HALO_GLOBAL
-    allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-    allocate(div3(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-    allocate(wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (div3(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
 #else
-    allocate(u2(ysize(1), ysize(2), ysize(3)))
-    allocate(v2(ysize(1), ysize(2), ysize(3)))
-    allocate(w2(ysize(1), ysize(2), ysize(3)))
-    allocate(div3(xsize(1), xsize(2), xsize(3)))
-    allocate(wk2(ysize(1), ysize(2), ysize(3)))
+      allocate (u2(ysize(1), ysize(2), ysize(3)))
+      allocate (v2(ysize(1), ysize(2), ysize(3)))
+      allocate (w2(ysize(1), ysize(2), ysize(3)))
+      allocate (div3(xsize(1), xsize(2), xsize(3)))
+      allocate (wk2(ysize(1), ysize(2), ysize(3)))
 #endif
-    call transpose_x_to_y(u1,u2)
-    call transpose_x_to_y(v1,v2)
-    call transpose_x_to_y(w1,w2)
+      call transpose_x_to_y(u1, u2)
+      call transpose_x_to_y(v1, v2)
+      call transpose_x_to_y(w1, w2)
 
-    ! du/dx
+      ! du/dx
 #ifdef HALO_GLOBAL
-    call update_halo(u2,uh,1,opt_global=.true.,opt_pencil=2)
-    call update_halo(w2,wh,1,opt_global=.true.,opt_pencil=2)
-    k1 = ystart(3); kn = yend(3)
-    i1 = ystart(1); in = yend(1)
+      call update_halo(u2, uh, 1, opt_global=.true., opt_pencil=2)
+      call update_halo(w2, wh, 1, opt_global=.true., opt_pencil=2)
+      k1 = ystart(3); kn = yend(3)
+      i1 = ystart(1); in = yend(1)
 #else
-    call update_halo(u2,uh,1,opt_pencil=2)
-    call update_halo(w2,wh,1,opt_pencil=2)
-    k1 = 1; kn = ysize(3)
-    i1 = 1; in = ysize(1)
+      call update_halo(u2, uh, 1, opt_pencil=2)
+      call update_halo(w2, wh, 1, opt_pencil=2)
+      k1 = 1; kn = ysize(3)
+      i1 = 1; in = ysize(1)
 #endif
-    j1 = 2; jn = ysize(2) - 1
+      j1 = 2; jn = ysize(2) - 1
 
-    call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Y:u")
-    call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "Y:w")
+      call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Y:u")
+      call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "Y:w")
 
-    do k=k1,kn
-       do j=j1,jn
-          do i=i1,in
-             wk2(i,j,k) = (uh(i+1,j,k)-uh(i-1,j,k)) &
-                  + (v2(i,j+1,k)-v2(i,j-1,k)) &
-                  + (wh(i,j,k+1)-wh(i,j,k-1))
-          end do
-       end do
-    end do
+      do k = k1, kn
+         do j = j1, jn
+            do i = i1, in
+               wk2(i, j, k) = (uh(i + 1, j, k) - uh(i - 1, j, k)) &
+                              + (v2(i, j + 1, k) - v2(i, j - 1, k)) &
+                              + (wh(i, j, k + 1) - wh(i, j, k - 1))
+            end do
+         end do
+      end do
 
-    call transpose_y_to_x(wk2,div3)
+      call transpose_y_to_x(wk2, div3)
 
-    ! Compute error
-    call check_err(div3, "Y")
+      ! Compute error
+      call check_err(div3, "Y")
 
-    deallocate(uh,wh,wk2)
+      deallocate (uh, wh, wk2)
 
-  end subroutine test_div_haloY
+   end subroutine test_div_haloY
 
-  !=====================================================================
-  ! Calculate divergence using halo-cell exchange (data in Z-pencil)
-  !=====================================================================
-  subroutine test_div_haloZ()
+   !=====================================================================
+   ! Calculate divergence using halo-cell exchange (data in Z-pencil)
+   !=====================================================================
+   subroutine test_div_haloZ()
 
-  ! Expected sizes
-  nx_expected = zsize(1) + 2
-  ny_expected = zsize(2) + 2
-  nz_expected = nz
+      ! Expected sizes
+      nx_expected = zsize(1) + 2
+      ny_expected = zsize(2) + 2
+      nz_expected = nz
 
 #ifdef HALO_GLOBAL
-  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  allocate(v3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  allocate(w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (v3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
 #else
-  allocate(u3(zsize(1), zsize(2), zsize(3)))
-  allocate(v3(zsize(1), zsize(2), zsize(3)))
-  allocate(w3(zsize(1), zsize(2), zsize(3)))
+      allocate (u3(zsize(1), zsize(2), zsize(3)))
+      allocate (v3(zsize(1), zsize(2), zsize(3)))
+      allocate (w3(zsize(1), zsize(2), zsize(3)))
 #endif
-  call transpose_y_to_z(u2,u3)
-  call transpose_y_to_z(v2,v3)
-  call transpose_y_to_z(w2,w3)
+      call transpose_y_to_z(u2, u3)
+      call transpose_y_to_z(v2, v3)
+      call transpose_y_to_z(w2, w3)
 
 #ifdef HALO_GLOBAL
-  allocate(div4(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      allocate (div4(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      allocate (wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
 #else
-  allocate(div4(xsize(1), xsize(2), xsize(3)))
-  allocate(wk2(ysize(1), ysize(2), ysize(3)))
-  allocate(wk3(zsize(1), zsize(2), zsize(3)))
+      allocate (div4(xsize(1), xsize(2), xsize(3)))
+      allocate (wk2(ysize(1), ysize(2), ysize(3)))
+      allocate (wk3(zsize(1), zsize(2), zsize(3)))
 #endif
 
-  ! du/dx
+      ! du/dx
 #ifdef HALO_GLOBAL
-  call update_halo(u3,uh,1,opt_global=.true.,opt_pencil=3)
+      call update_halo(u3, uh, 1, opt_global=.true., opt_pencil=3)
 #else
-  call update_halo(u3,uh,1,opt_pencil=3)
+      call update_halo(u3, uh, 1, opt_pencil=3)
 #endif
 
-  call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
-  
+      call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
+
 #ifdef HALO_GLOBAL
-  do j=zstart(2),zend(2)
-     do i=zstart(1),zend(1)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
 #else
-  do j=1,zsize(2)
-     do i=1,zsize(1)
+            do j = 1, zsize(2)
+               do i = 1, zsize(1)
 #endif
-        do k=2,zsize(3)-1
-           wk3(i,j,k) = uh(i+1,j,k)-uh(i-1,j,k)
-        end do
-     end do
-  end do
+                  do k = 2, zsize(3) - 1
+                     wk3(i, j, k) = uh(i + 1, j, k) - uh(i - 1, j, k)
+                  end do
+               end do
+            end do
 
-  ! dv/dy
+            ! dv/dy
 #ifdef HALO_GLOBAL
-  call update_halo(v3,vh,1,opt_global=.true.,opt_pencil=3)
+            call update_halo(v3, vh, 1, opt_global=.true., opt_pencil=3)
 #else
-  call update_halo(v3,vh,1,opt_pencil=3)
+            call update_halo(v3, vh, 1, opt_pencil=3)
 #endif
 
-  call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
-  
+            call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
+
 #ifdef HALO_GLOBAL
-  do j=zstart(2),zend(2)
-     do i=zstart(1),zend(1)
+            do j = zstart(2), zend(2)
+               do i = zstart(1), zend(1)
 #else
-  do j=1,zsize(2)
-     do i=1,zsize(1)
+                  do j = 1, zsize(2)
+                     do i = 1, zsize(1)
 #endif
-        do k=2,zsize(3)-1
-           wk3(i,j,k) = wk3(i,j,k) + vh(i,j+1,k)-vh(i,j-1,k)
-        end do
-     end do
-  end do
+                        do k = 2, zsize(3) - 1
+                           wk3(i, j, k) = wk3(i, j, k) + vh(i, j + 1, k) - vh(i, j - 1, k)
+                        end do
+                     end do
+                  end do
 
-  ! dw/dz
+                  ! dw/dz
 #ifdef HALO_GLOBAL
-  do j=zstart(2),zend(2)
-     do i=zstart(1),zend(1)
+                  do j = zstart(2), zend(2)
+                     do i = zstart(1), zend(1)
 #else
-  do j=1,zsize(2)
-     do i=1,zsize(1)
+                        do j = 1, zsize(2)
+                           do i = 1, zsize(1)
 #endif
-        do k=2,zsize(3)-1
-           wk3(i,j,k) = wk3(i,j,k) + w3(i,j,k+1)-w3(i,j,k-1)
-        end do
-     end do
-  end do
-
-  call transpose_z_to_y(wk3,wk2)
-  call transpose_y_to_x(wk2,div4)
-
-  ! Compute error
-  call check_err(div4, "Z")
-
-  deallocate(uh,vh,wk2,wk3)
-  end subroutine test_div_haloZ
-
-  subroutine check_err(divh, pencil)
-
-    real(mytype), dimension(:,:,:), intent(in) :: divh
-    character(len=*), intent(in) :: pencil
-
-    real(mytype), dimension(:,:,:), allocatable :: tmp
-
-    real(mytype) :: divmag
-
-    ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
-    !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
-    !      causes a SEGFAULT. Explicitly computing the difference in a temporary
-    !      array seems to be OK.
-    allocate(tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
-    tmp(2:xlast,2:ylast,2:zlast) = divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast)
-    err = mag(tmp(2:xlast,2:ylast,2:zlast))
-    deallocate(tmp)
-    divmag = mag(div1(2:xlast,2:ylast,2:zlast))
-    if (err < epsilon(divmag) * divmag) then
-       passing = .true.
-    else
-       passing = .false.
-    end if
-    all_pass = all_pass .and. passing
-
-    if (nrank==0) then
-       write(*,*) '-----------------------------------------------'
-       write(*,*) 'Calculated via halo exchange (data in '//pencil//'-pencil)'
+                              do k = 2, zsize(3) - 1
+                                 wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
+                              end do
+                           end do
+                        end do
+
+                        call transpose_z_to_y(wk3, wk2)
+                        call transpose_y_to_x(wk2, div4)
+
+                        ! Compute error
+                        call check_err(div4, "Z")
+
+                        deallocate (uh, vh, wk2, wk3)
+                        end subroutine test_div_haloZ
+
+                        subroutine check_err(divh, pencil)
+
+                           real(mytype), dimension(:, :, :), intent(in) :: divh
+                           character(len=*), intent(in) :: pencil
+
+                           real(mytype), dimension(:, :, :), allocatable :: tmp
+
+                           real(mytype) :: divmag
+
+                           ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
+                           !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
+                           !      causes a SEGFAULT. Explicitly computing the difference in a temporary
+                           !      array seems to be OK.
+                           allocate (tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
+                           tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - div1(2:xlast, 2:ylast, 2:zlast)
+                           err = mag(tmp(2:xlast, 2:ylast, 2:zlast))
+                           deallocate (tmp)
+                           divmag = mag(div1(2:xlast, 2:ylast, 2:zlast))
+                           if (err < epsilon(divmag)*divmag) then
+                              passing = .true.
+                           else
+                              passing = .false.
+                           end if
+                           all_pass = all_pass .and. passing
+
+                           if (nrank == 0) then
+                              write (*, *) '-----------------------------------------------'
+                              write (*, *) 'Calculated via halo exchange (data in '//pencil//'-pencil)'
 #ifdef DEBUG
-       write(*,*) (divh(i,i,i), i=2,13)
+                              write (*, *) (divh(i, i, i), i=2, 13)
 #endif
-       write(*,*) 'Error: ', err, '; Relative: ', err / divmag
-       write(*,*) 'Pass: ', passing
-    end if
+                              write (*, *) 'Error: ', err, '; Relative: ', err/divmag
+                              write (*, *) 'Pass: ', passing
+                           end if
+
+                        end subroutine check_err
+
+                        real(mytype) function mag(a)
+
+                           real(mytype), dimension(:, :, :), intent(in) :: a
 
-  end subroutine check_err
+                           real(mytype) :: lmag, gmag
 
-  real(mytype) function mag(a)
+                           lmag = sum(a(:, :, :)**2)
+                           call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+                           if (ierror /= 0) then
+                              call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                                                   "halo_test::mag::MPI_Allreduce")
+                           end if
 
-    real(mytype), dimension(:,:,:), intent(in) :: a
-    
-    real(mytype) :: lmag, gmag
+                           mag = sqrt(gmag/(nx - 2)/(ny - 2)/(nz - 2))
 
-    lmag = sum(a(:,:,:)**2)
-    call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-    if (ierror /= 0) then
-       call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-            "halo_test::mag::MPI_Allreduce")
-    endif
+                        end function mag
 
-    mag = sqrt(gmag / (nx - 2) / (ny - 2) / (nz - 2))
+                        subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
 
-  end function mag
+                           real(mytype), dimension(:, :, :), intent(in) :: arrh
+                           integer, intent(in) :: nx_expected, ny_expected, nz_expected
+                           character(len=*), intent(in) :: tag
 
-  subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
+                           integer :: nx, ny, nz
 
-    real(mytype), dimension(:,:,:), intent(in) :: arrh
-    integer, intent(in) :: nx_expected, ny_expected, nz_expected
-    character(len=*), intent(in) :: tag
+                           character(len=128) :: rank_lbl
 
-    integer :: nx, ny, nz
+                           nx = size(arrh, 1)
+                           ny = size(arrh, 2)
+                           nz = size(arrh, 3)
 
-    character(len=128) :: rank_lbl
+                           write (rank_lbl, "(A,I0,A)") "Rank", nrank, ":"
 
-    nx = size(arrh, 1)
-    ny = size(arrh, 2)
-    nz = size(arrh, 3)
+                           if ((nx /= nx_expected) .or. &
+                               (ny /= ny_expected) .or. &
+                               (nz /= nz_expected)) then
+                              write (*, *) trim(rank_lbl), " ", tag, ":ERROR: halo size"
+                              write (*, *) trim(rank_lbl), " ", "+ Expected: ", nx_expected, " ", ny_expected, " ", nz_expected, " "
+                              write (*, *) trim(rank_lbl), " ", "+ Got:      ", nx, " ", ny, " ", nz, " "
 
-    write(rank_lbl, "(A,I0,A)") "Rank", nrank, ":"
+                              all_pass = .false.
+                           else
+                              write (*, *) trim(rank_lbl), " ", tag, ":PASS"
+                           end if
 
-    if ((nx /= nx_expected) .or. &
-         (ny /= ny_expected) .or. &
-         (nz /= nz_expected)) then
-       write(*,*) trim(rank_lbl), " ", tag, ":ERROR: halo size"
-       write(*,*) trim(rank_lbl), " ", "+ Expected: ", nx_expected, " ", ny_expected, " ", nz_expected, " "
-       write(*,*) trim(rank_lbl), " ", "+ Got:      ", nx, " ", ny, " ", nz, " "
+                        end subroutine test_halo_size
 
-       all_pass = .false.
-    else
-       write(*,*) trim(rank_lbl), " ", tag, ":PASS"
-    end if
-    
-  end subroutine test_halo_size
-  
-end program halo_test
+                        end program halo_test
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 5f9c5dd1..25953391 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -4,71 +4,71 @@
 
 program init_test
 
-  use MPI
-  use decomp_2d
-  
-  implicit none
+   use MPI
+   use decomp_2d
 
-  integer, parameter :: nx = 5
-  integer, parameter :: ny = 6
-  integer, parameter :: nz = 7
-  integer, parameter :: nexpect = nx * ny * nz
+   implicit none
 
-  integer :: p_row, p_col
+   integer, parameter :: nx = 5
+   integer, parameter :: ny = 6
+   integer, parameter :: nz = 7
+   integer, parameter :: nexpect = nx*ny*nz
 
-  integer :: ierr
+   integer :: p_row, p_col
 
-  call MPI_Init(ierr)
-  
-  p_row = 0; p_col = 0
-  call run(p_row, p_col)
+   integer :: ierr
+
+   call MPI_Init(ierr)
+
+   p_row = 0; p_col = 0
+   call run(p_row, p_col)
+
+   call MPI_Finalize(ierr)
 
-  call MPI_Finalize(ierr)
-  
 contains
 
-  subroutine run(p_row, p_col)
-
-    integer, intent(inout) :: p_row, p_col
-
-    call decomp_2d_init(nx, ny, nz, p_row, p_col)
-
-    call check_axis("X")
-    call check_axis("Y")
-    call check_axis("Z")
-    
-    call decomp_2d_finalize()
-    
-  end subroutine run
-
-  subroutine check_axis(axis)
-
-    character(len=*), intent(in) :: axis
-
-    integer :: suml
-    integer :: sumg
-    integer, dimension(3) :: sizes
-
-    if (axis == "X") then
-       sizes = xsize
-    else if (axis == "Y") then
-       sizes = ysize
-    else if (axis == "Z") then
-       sizes = zsize
-    else
-       sizes = 0
-       print *, "ERROR: unknown axis requested!"
-       stop 1
-    end if
-
-    suml = product(sizes)
-    call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierr)
-    
-    if (sumg /= nexpect) then
-       print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
-       stop 1
-    end if
-    
-  end subroutine check_axis
-  
+   subroutine run(p_row, p_col)
+
+      integer, intent(inout) :: p_row, p_col
+
+      call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+      call check_axis("X")
+      call check_axis("Y")
+      call check_axis("Z")
+
+      call decomp_2d_finalize()
+
+   end subroutine run
+
+   subroutine check_axis(axis)
+
+      character(len=*), intent(in) :: axis
+
+      integer :: suml
+      integer :: sumg
+      integer, dimension(3) :: sizes
+
+      if (axis == "X") then
+         sizes = xsize
+      else if (axis == "Y") then
+         sizes = ysize
+      else if (axis == "Z") then
+         sizes = zsize
+      else
+         sizes = 0
+         print *, "ERROR: unknown axis requested!"
+         stop 1
+      end if
+
+      suml = product(sizes)
+      call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierr)
+
+      if (sumg /= nexpect) then
+         print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
+         stop 1
+      end if
+
+   end subroutine check_axis
+
 end program init_test
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index a8e05390..4e3200ca 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -1,36 +1,36 @@
 program io_bench
 
-  use decomp_2d
-  use decomp_2d_io
-  use MPI
+   use decomp_2d
+   use decomp_2d_io
+   use MPI
 
-  implicit none
-  
-  integer, parameter :: nx=100, ny=100, nz=100
-  integer :: p_row=0, p_col=0
+   implicit none
 
-  real(mytype), allocatable, dimension(:,:,:) :: u1
-  
-  double precision :: t1, t2
-  integer :: ierror
+   integer, parameter :: nx = 100, ny = 100, nz = 100
+   integer :: p_row = 0, p_col = 0
 
-  call MPI_INIT(ierror)
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
+   real(mytype), allocatable, dimension(:, :, :) :: u1
 
-  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  call random_number(u1)
+   double precision :: t1, t2
+   integer :: ierror
 
-  t1 = MPI_WTIME()
-  call decomp_2d_write_one(1,u1,'io.dat')
-  t2 = MPI_WTIME()
+   call MPI_INIT(ierror)
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-  if (nrank==0) write(*,*) 'I/O time: ', t2-t1
+   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   call random_number(u1)
 
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1)
+   t1 = MPI_WTIME()
+   call decomp_2d_write_one(1, u1, 'io.dat')
+   t2 = MPI_WTIME()
+
+   if (nrank == 0) write (*, *) 'I/O time: ', t2 - t1
+
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1)
 
 end program io_bench
-  
+
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 42a6d221..7bebfa03 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -1,128 +1,128 @@
 program io_plane_test
 
-  use mpi
-
-  use decomp_2d
-  use decomp_2d_io
-
-  implicit none
-
-  integer, parameter :: nx=17, ny=13, nz=11
-  integer :: p_row=0, p_col=0
-
-  real(mytype), dimension(nx,ny,nz) :: data1
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-
-  real(mytype), allocatable, dimension(:,:,:) :: work
-  
-  integer :: i,j,k, m, ierror, iol
-
-  call MPI_INIT(ierror)
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-  
-  ! ***** global data *****
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
-           data1(i,j,k) = real(m,mytype)
-           m = m+1
-        end do
-     end do
-  end do
-
-  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  ! original X-pensil based data 
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        u1(i,j,k) = data1(i,j,k)
+   use mpi
+
+   use decomp_2d
+   use decomp_2d_io
+
+   implicit none
+
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer :: p_row = 0, p_col = 0
+
+   real(mytype), dimension(nx, ny, nz) :: data1
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+
+   real(mytype), allocatable, dimension(:, :, :) :: work
+
+   integer :: i, j, k, m, ierror, iol
+
+   call MPI_INIT(ierror)
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
+            data1(i, j, k) = real(m, mytype)
+            m = m + 1
+         end do
+      end do
+   end do
+
+   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   ! original X-pensil based data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            u1(i, j, k) = data1(i, j, k)
+         end do
+      end do
+   end do
+   call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
+
+   ! Y-pencil data
+   call transpose_x_to_y(u1, u2)
+   call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 2, ny/2, '.', 'y_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 3, nz/2, '.', 'y_pencil-z_plane.dat', 'test')
+
+   ! Z-pencil data
+   call transpose_y_to_z(u2, u3)
+   call decomp_2d_write_plane(3, u3, 1, nx/2, '.', 'z_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 2, ny/2, '.', 'z_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 3, nz/2, '.', 'z_pencil-z_plane.dat', 'test')
+
+   ! Attemp to read the files
+   if (nrank == 0) then
+      inquire (iolength=iol) data1(1, 1, 1)
+
+      ! X-plane
+      allocate (work(1, ny, nz))
+      open (10, FILE='x_pencil-x_plane.dat', FORM='unformatted', &
+            ACCESS='DIRECT', RECL=iol)
+      m = 1
+      do k = 1, nz
+         do j = 1, ny
+            read (10, rec=m) work(1, j, k)
+            m = m + 1
+         end do
       end do
-    end do
-  end do
-  call decomp_2d_write_plane(1,u1,1,nx/2,'.','x_pencil-x_plane.dat','test')
-  call decomp_2d_write_plane(1,u1,2,ny/2,'.','x_pencil-y_plane.dat','test')
-  call decomp_2d_write_plane(1,u1,3,nz/2,'.','x_pencil-z_plane.dat','test')
-
-  ! Y-pencil data
-  call transpose_x_to_y(u1,u2)
-  call decomp_2d_write_plane(2,u2,1,nx/2,'.','y_pencil-x_plane.dat','test')
-  call decomp_2d_write_plane(2,u2,2,ny/2,'.','y_pencil-y_plane.dat','test')
-  call decomp_2d_write_plane(2,u2,3,nz/2,'.','y_pencil-z_plane.dat','test')
-
-  ! Z-pencil data
-  call transpose_y_to_z(u2,u3)
-  call decomp_2d_write_plane(3,u3,1,nx/2,'.','z_pencil-x_plane.dat','test')
-  call decomp_2d_write_plane(3,u3,2,ny/2,'.','z_pencil-y_plane.dat','test')
-  call decomp_2d_write_plane(3,u3,3,nz/2,'.','z_pencil-z_plane.dat','test')
-
-  ! Attemp to read the files
-  if (nrank==0) then
-     inquire(iolength=iol) data1(1,1,1)
-
-     ! X-plane
-     allocate(work(1,ny,nz))
-     open(10, FILE='x_pencil-x_plane.dat', FORM='unformatted', &
-          ACCESS='DIRECT', RECL=iol)
-     m=1
-     do k=1,nz
-        do j=1,ny
-           read(10,rec=m) work(1,j,k)
-           m=m+1
-        end do
-     end do
 !     write(*,*) ' '
 !     write(*,'(15I5)') int(work)
-     close(10)
-     deallocate(work)
-
-     write(*,*) 'passed self test x-plane'
-
-     ! Y-plane
-     allocate(work(nx,1,nz))
-     open(10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
-          ACCESS='DIRECT', RECL=iol)
-     m=1
-     do k=1,nz
-        do i=1,nx
-           read(10,rec=m) work(i,1,k)
-           m=m+1
-        end do
-     end do
+      close (10)
+      deallocate (work)
+
+      write (*, *) 'passed self test x-plane'
+
+      ! Y-plane
+      allocate (work(nx, 1, nz))
+      open (10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
+            ACCESS='DIRECT', RECL=iol)
+      m = 1
+      do k = 1, nz
+         do i = 1, nx
+            read (10, rec=m) work(i, 1, k)
+            m = m + 1
+         end do
+      end do
 !     write(*,*) ' '
 !     write(*,'(15I5)') int(work)
-     close(10)
-     deallocate(work)
-
-     write(*,*) 'passed self test y-plane'
-
-     ! Z-plane
-     allocate(work(nx,ny,1))
-     open(10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
-          ACCESS='DIRECT', RECL=iol)
-     m=1
-     do j=1,ny
-        do i=1,nx
-           read(10,rec=m) work(i,j,1)
-           m=m+1
-        end do
-     end do
+      close (10)
+      deallocate (work)
+
+      write (*, *) 'passed self test y-plane'
+
+      ! Z-plane
+      allocate (work(nx, ny, 1))
+      open (10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
+            ACCESS='DIRECT', RECL=iol)
+      m = 1
+      do j = 1, ny
+         do i = 1, nx
+            read (10, rec=m) work(i, j, 1)
+            m = m + 1
+         end do
+      end do
 !     write(*,*) ' '
 !     write(*,'(15I5)') int(work)
-     close(10)
-     deallocate(work)
+      close (10)
+      deallocate (work)
+
+      write (*, *) 'passed self test z-plane'
 
-     write(*,*) 'passed self test z-plane'
+   end if
 
-  end if
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1, u2, u3)
 
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1,u2,u3)
-  
 end program io_plane_test
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 38c9cb85..694f1f21 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -1,86 +1,86 @@
 program io_read
 
-  use mpi
+   use mpi
 
-  use decomp_2d
-  use decomp_2d_io
+   use decomp_2d
+   use decomp_2d_io
 
-  implicit none
+   implicit none
 
-  integer, parameter :: nx=17, ny=13, nz=11
-  ! use different number of processes
-  integer :: p_row=0, p_col=0
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   ! use different number of processes
+   integer :: p_row = 0, p_col = 0
 
 #ifdef COMPLEX_TEST
-  complex(mytype), dimension(nx,ny,nz) :: data1
+   complex(mytype), dimension(nx, ny, nz) :: data1
 
-  complex(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
+   complex(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #else
-  real(mytype), dimension(nx,ny,nz) :: data1
+   real(mytype), dimension(nx, ny, nz) :: data1
 
-  real(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
+   real(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #endif
 
-  real(mytype), parameter :: eps = 1.0E-7_mytype
- 
-  integer :: i,j,k, m, ierror
-  
-  call MPI_INIT(ierror)
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-
-  ! ***** global data *****
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
+   real(mytype), parameter :: eps = 1.0E-7_mytype
+
+   integer :: i, j, k, m, ierror
+
+   call MPI_INIT(ierror)
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
 #ifdef COMPLEX_TEST
-           data1(i,j,k) = cmplx(real(m,mytype), real(nx*ny*nz-m,mytype)) 
+            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
 #else
-           data1(i,j,k) = real(m,mytype)
+            data1(i, j, k) = real(m, mytype)
 #endif
-           m = m+1
-        end do
-     end do
-  end do
-
-  allocate(u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  ! read back to different arrays
-  call decomp_2d_read_one(1,u1b,'.','u1.dat','test',reduce_prec=.false.)
-  call decomp_2d_read_one(2,u2b,'.','u2.dat','test',reduce_prec=.false.)
-  call decomp_2d_read_one(3,u3b,'.','u3.dat','test',reduce_prec=.false.)
-
-  ! Check against the global data array
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        if (abs((data1(i,j,k)-u1b(i,j,k))) > eps) stop 4
+            m = m + 1
+         end do
+      end do
+   end do
+
+   allocate (u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   ! read back to different arrays
+   call decomp_2d_read_one(1, u1b, '.', 'u1.dat', 'test', reduce_prec=.false.)
+   call decomp_2d_read_one(2, u2b, '.', 'u2.dat', 'test', reduce_prec=.false.)
+   call decomp_2d_read_one(3, u3b, '.', 'u3.dat', 'test', reduce_prec=.false.)
+
+   ! Check against the global data array
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs((data1(i, j, k) - u1b(i, j, k))) > eps) stop 4
+         end do
       end do
-    end do
-  end do
+   end do
 
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs((data1(i,j,k)-u2b(i,j,k))) > eps) stop 5
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
+         end do
       end do
-    end do
-  end do
-  
-  do k=zstart(3),zend(3)
-    do j=zstart(2),zend(2)
-      do i=zstart(1),zend(1)
-        if (abs((data1(i,j,k)-u3b(i,j,k))) > eps) stop 6
+   end do
+
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
+         end do
       end do
-    end do
-  end do
+   end do
 
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1b,u2b,u3b)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1b, u2b, u3b)
 
 end program io_read
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index b38aa2a0..eecdcfeb 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -1,133 +1,133 @@
 program io_test
 
-  use mpi
+   use mpi
 
-  use decomp_2d
-  use decomp_2d_io
+   use decomp_2d
+   use decomp_2d_io
 
-  implicit none
+   implicit none
 
-  integer, parameter :: nx=17, ny=13, nz=11
-  integer :: p_row=0, p_col=0
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer :: p_row = 0, p_col = 0
 
 #ifdef COMPLEX_TEST
-  complex(mytype), dimension(nx,ny,nz) :: data1
+   complex(mytype), dimension(nx, ny, nz) :: data1
 
-  complex(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  complex(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
+   complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   complex(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #else
-  real(mytype), dimension(nx,ny,nz) :: data1
+   real(mytype), dimension(nx, ny, nz) :: data1
 
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  real(mytype), allocatable, dimension(:,:,:) :: u1b, u2b, u3b
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   real(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #endif
 
-  real(mytype), parameter :: eps = 1.0E-7_mytype
- 
-  integer :: i,j,k, m, ierror
-  
-  call MPI_INIT(ierror)
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-
-  ! ***** global data *****
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
+   real(mytype), parameter :: eps = 1.0E-7_mytype
+
+   integer :: i, j, k, m, ierror
+
+   call MPI_INIT(ierror)
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
 #ifdef COMPLEX_TEST
-           data1(i,j,k) = cmplx(real(m,mytype), real(nx*ny*nz-m,mytype)) 
+            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
 #else
-           data1(i,j,k) = real(m,mytype)
+            data1(i, j, k) = real(m, mytype)
 #endif
-           m = m+1
-        end do
-     end do
-  end do
-
-  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  allocate(u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  ! original x-pencil based data 
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        u1(i,j,k) = data1(i,j,k)
+            m = m + 1
+         end do
       end do
-    end do
-  end do
-
-  ! transpose
-  call transpose_x_to_y(u1,u2)
-  call transpose_y_to_z(u2,u3)
-
-  ! write to disk
-  call decomp_2d_write_one(1,u1,'.','u1.dat',0,'test')
-  call decomp_2d_write_one(2,u2,'.','u2.dat',0,'test')
-  call decomp_2d_write_one(3,u3,'.','u3.dat',0,'test')
-
-  ! read back to different arrays
-  call decomp_2d_read_one(1,u1b,'.','u1.dat','test',reduce_prec=.false.)
-  call decomp_2d_read_one(2,u2b,'.','u2.dat','test',reduce_prec=.false.)
-  call decomp_2d_read_one(3,u3b,'.','u3.dat','test',reduce_prec=.false.)
-
-  ! compare  
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        if (abs((u1(i,j,k)-u1b(i,j,k))) > eps) stop 1
+   end do
+
+   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   allocate (u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   ! original x-pencil based data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            u1(i, j, k) = data1(i, j, k)
+         end do
       end do
-    end do
-  end do
+   end do
+
+   ! transpose
+   call transpose_x_to_y(u1, u2)
+   call transpose_y_to_z(u2, u3)
+
+   ! write to disk
+   call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
+   call decomp_2d_write_one(2, u2, '.', 'u2.dat', 0, 'test')
+   call decomp_2d_write_one(3, u3, '.', 'u3.dat', 0, 'test')
+
+   ! read back to different arrays
+   call decomp_2d_read_one(1, u1b, '.', 'u1.dat', 'test', reduce_prec=.false.)
+   call decomp_2d_read_one(2, u2b, '.', 'u2.dat', 'test', reduce_prec=.false.)
+   call decomp_2d_read_one(3, u3b, '.', 'u3.dat', 'test', reduce_prec=.false.)
+
+   ! compare
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) stop 1
+         end do
+      end do
+   end do
 
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs((u2(i,j,k)-u2b(i,j,k))) > eps) stop 2
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 2
+         end do
       end do
-    end do
-  end do
+   end do
 
-  do k=zstart(3),zend(3)
-    do j=zstart(2),zend(2)
-      do i=zstart(1),zend(1)
-        if (abs((u3(i,j,k)-u3b(i,j,k))) > eps) stop 3
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs((u3(i, j, k) - u3b(i, j, k))) > eps) stop 3
+         end do
       end do
-    end do
-  end do
-
-  ! Also check against the global data array
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        if (abs(data1(i,j,k)-u1b(i,j,k)) > eps) stop 4
+   end do
+
+   ! Also check against the global data array
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs(data1(i, j, k) - u1b(i, j, k)) > eps) stop 4
+         end do
       end do
-    end do
-  end do
+   end do
 
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs((data1(i,j,k)-u2b(i,j,k))) > eps) stop 5
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
+         end do
       end do
-    end do
-  end do
-  
-  do k=zstart(3),zend(3)
-    do j=zstart(2),zend(2)
-      do i=zstart(1),zend(1)
-        if (abs((data1(i,j,k)-u3b(i,j,k))) > eps) stop 6
+   end do
+
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
+         end do
       end do
-    end do
-  end do
+   end do
 
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1,u2,u3)
-  deallocate(u1b,u2b,u3b)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1, u2, u3)
+   deallocate (u1b, u2b, u3b)
 
 end program io_test
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 2d25dcf9..7f91e01d 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -3,283 +3,283 @@
 
 program io_var_test
 
-  use decomp_2d
-  use decomp_2d_io
-  use MPI
-
-  implicit none
-
-  integer, parameter :: nx=17, ny=13, nz=11
-  integer :: p_row, p_col
-
-  real(mytype), parameter :: eps = 1.0E-7
-
-  ! for global data
-  real(mytype), dimension(nx,ny,nz) :: data1
-  real(mytype), allocatable, dimension(:,:,:) :: data1_large
-  complex(mytype), dimension(nx,ny,nz) :: cdata1
-
-  ! for distributed data
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  real(mytype), allocatable, dimension(:,:,:) :: u1l, u2l, u3l
-  complex(mytype), allocatable, dimension(:,:,:) :: cu1, cu2, cu3
-
-  ! another copy
-  real(mytype), allocatable, dimension(:,:,:) :: u1_b, u2_b, u3_b
-  real(mytype), allocatable, dimension(:,:,:) :: u1l_b, u2l_b, u3l_b
-  complex(mytype), allocatable, dimension(:,:,:) :: cu1_b, cu2_b, cu3_b
-
-  real(mytype), allocatable, dimension(:) :: tmp
-  complex(mytype), allocatable, dimension(:) :: ctmp
-  integer, allocatable, dimension(:) :: itmp
-
-  TYPE(DECOMP_INFO) :: large
-
-  integer :: i,j,k, m, ierror, fh
-  character(len=15) :: filename, arg
-  integer (kind=MPI_OFFSET_KIND) :: filesize, disp
-
-  allocate(data1_large(nx*2,ny*2,nz*2))
-
-  call MPI_INIT(ierror)
-  call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-  call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-
-  ! Defaults
-  p_row = 0
-  p_col = 0
-
-  ! Read commandline input
-  i = command_argument_count()
-  if (i/=2) then
-     call MPI_ABORT(MPI_COMM_WORLD, 1, ierror)
-  else
-     call get_command_argument(1, arg)
-     read(arg, '(I10)') i
-     p_row = i
-     call get_command_argument(2, arg)
-     read(arg, '(I10)') i
-     p_col = i
-  end if
-  
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-
-  ! also create a data set over a large domain
-  call decomp_info_init(nx*2, ny*2, nz*2, large)
-
-  ! initialise global data
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
-           data1(i,j,k) = real(m,mytype)
-           cdata1(i,j,k) = cmplx(real(m,mytype),real(m,mytype), kind=mytype)
-           m = m+1
-        end do
-     end do
-  end do
-
-  m = 1
-  do k=1,nz*2
-     do j=1,ny*2
-        do i=1,nx*2
-           data1_large(i,j,k) = real(m,mytype)
-           m = m+1
-        end do
-     end do
-  end do
-
-  ! allocate memory
-  allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  allocate(u1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(u2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(u3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  allocate(cu1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(cu2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(cu3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  allocate(cu1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  allocate(cu2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  allocate(cu3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-  allocate(u1l(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
-       large%xst(3):large%xen(3)))
-  allocate(u2l(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
-       large%yst(3):large%yen(3)))
-  allocate(u3l(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
-       large%zst(3):large%zen(3)))
-  allocate(u1l_b(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
-       large%xst(3):large%xen(3)))
-  allocate(u2l_b(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
-       large%yst(3):large%yen(3)))
-  allocate(u3l_b(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
-       large%zst(3):large%zen(3)))
-
-  ! distribute the data
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           u1(i,j,k) = data1(i,j,k)
-           cu1(i,j,k) = cdata1(i,j,k)
-        end do
-     end do
-  end do
-  do k=large%xst(3),large%xen(3)
-     do j=large%xst(2),large%xen(2)
-        do i=large%xst(1),large%xen(1)
-           u1l(i,j,k) = data1_large(i,j,k)
-        end do
-     end do
-  end do
-
-  ! transpose
-  call transpose_x_to_y(u1,u2)
-  call transpose_y_to_z(u2,u3)
-  call transpose_x_to_y(u1l,u2l,large)
-  call transpose_y_to_z(u2l,u3l,large)
-  call transpose_x_to_y(cu1,cu2)
-  call transpose_y_to_z(cu2,cu3)
-
-  ! open file for IO
-  write(filename,'(A,I3.3)') 'io_var_data.', nproc
-  call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
-       MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-       fh, ierror)
-  filesize = 0_MPI_OFFSET_KIND
-  call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-  disp = 0_MPI_OFFSET_KIND
-
-  ! test writing scalar data
-  allocate(tmp(2))
-  tmp(1) = 1._mytype
-  tmp(2) = 2._mytype
-  allocate(ctmp(3))
-  ctmp(1) = cmplx(1.0,1.0,mytype)
-  ctmp(2) = cmplx(2.0,2.0,mytype)
-  ctmp(3) = cmplx(3.0,3.0,mytype)
-  allocate(itmp(3))
-  call decomp_2d_write_scalar(fh,disp,2,tmp)
-  call decomp_2d_write_scalar(fh,disp,3,ctmp)
-  call decomp_2d_write_scalar(fh,disp,3,(/nx,ny,nz/))
-  
-  ! test the IO routines by writing all data to disk
-  call decomp_2d_write_var(fh,disp,1,u1)
-  call decomp_2d_write_var(fh,disp,2,u2)
-  call decomp_2d_write_var(fh,disp,3,u3)
-  call decomp_2d_write_var(fh,disp,1,u1l,large)
-  call decomp_2d_write_var(fh,disp,2,u2l,large)
-  call decomp_2d_write_var(fh,disp,3,u3l,large)
-  call decomp_2d_write_var(fh,disp,1,cu1)
-  call decomp_2d_write_var(fh,disp,2,cu2)
-  call decomp_2d_write_var(fh,disp,3,cu3)
-
-  call MPI_FILE_CLOSE(fh,ierror)
-
-  if (nrank==0) write(*,*) 'disp=',disp
-
-  ! read data back in from file
-  call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
-       MPI_MODE_RDONLY, MPI_INFO_NULL, &
-       fh, ierror)
-  ! skip the scalars (2 real, 3 cmplx, 3 int)
+   use decomp_2d
+   use decomp_2d_io
+   use MPI
+
+   implicit none
+
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer :: p_row, p_col
+
+   real(mytype), parameter :: eps = 1.0E-7
+
+   ! for global data
+   real(mytype), dimension(nx, ny, nz) :: data1
+   real(mytype), allocatable, dimension(:, :, :) :: data1_large
+   complex(mytype), dimension(nx, ny, nz) :: cdata1
+
+   ! for distributed data
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   real(mytype), allocatable, dimension(:, :, :) :: u1l, u2l, u3l
+   complex(mytype), allocatable, dimension(:, :, :) :: cu1, cu2, cu3
+
+   ! another copy
+   real(mytype), allocatable, dimension(:, :, :) :: u1_b, u2_b, u3_b
+   real(mytype), allocatable, dimension(:, :, :) :: u1l_b, u2l_b, u3l_b
+   complex(mytype), allocatable, dimension(:, :, :) :: cu1_b, cu2_b, cu3_b
+
+   real(mytype), allocatable, dimension(:) :: tmp
+   complex(mytype), allocatable, dimension(:) :: ctmp
+   integer, allocatable, dimension(:) :: itmp
+
+   TYPE(DECOMP_INFO) :: large
+
+   integer :: i, j, k, m, ierror, fh
+   character(len=15) :: filename, arg
+   integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+
+   allocate (data1_large(nx*2, ny*2, nz*2))
+
+   call MPI_INIT(ierror)
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
+   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+
+   ! Defaults
+   p_row = 0
+   p_col = 0
+
+   ! Read commandline input
+   i = command_argument_count()
+   if (i /= 2) then
+      call MPI_ABORT(MPI_COMM_WORLD, 1, ierror)
+   else
+      call get_command_argument(1, arg)
+      read (arg, '(I10)') i
+      p_row = i
+      call get_command_argument(2, arg)
+      read (arg, '(I10)') i
+      p_col = i
+   end if
+
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! also create a data set over a large domain
+   call decomp_info_init(nx*2, ny*2, nz*2, large)
+
+   ! initialise global data
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
+            data1(i, j, k) = real(m, mytype)
+            cdata1(i, j, k) = cmplx(real(m, mytype), real(m, mytype), kind=mytype)
+            m = m + 1
+         end do
+      end do
+   end do
+
+   m = 1
+   do k = 1, nz*2
+      do j = 1, ny*2
+         do i = 1, nx*2
+            data1_large(i, j, k) = real(m, mytype)
+            m = m + 1
+         end do
+      end do
+   end do
+
+   ! allocate memory
+   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   allocate (u1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (u2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (u3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   allocate (cu1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (cu2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (cu3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   allocate (cu1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   allocate (cu2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   allocate (cu3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+
+   allocate (u1l(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
+                 large%xst(3):large%xen(3)))
+   allocate (u2l(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
+                 large%yst(3):large%yen(3)))
+   allocate (u3l(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
+                 large%zst(3):large%zen(3)))
+   allocate (u1l_b(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
+                   large%xst(3):large%xen(3)))
+   allocate (u2l_b(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
+                   large%yst(3):large%yen(3)))
+   allocate (u3l_b(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
+                   large%zst(3):large%zen(3)))
+
+   ! distribute the data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            u1(i, j, k) = data1(i, j, k)
+            cu1(i, j, k) = cdata1(i, j, k)
+         end do
+      end do
+   end do
+   do k = large%xst(3), large%xen(3)
+      do j = large%xst(2), large%xen(2)
+         do i = large%xst(1), large%xen(1)
+            u1l(i, j, k) = data1_large(i, j, k)
+         end do
+      end do
+   end do
+
+   ! transpose
+   call transpose_x_to_y(u1, u2)
+   call transpose_y_to_z(u2, u3)
+   call transpose_x_to_y(u1l, u2l, large)
+   call transpose_y_to_z(u2l, u3l, large)
+   call transpose_x_to_y(cu1, cu2)
+   call transpose_y_to_z(cu2, cu3)
+
+   ! open file for IO
+   write (filename, '(A,I3.3)') 'io_var_data.', nproc
+   call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
+                      MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                      fh, ierror)
+   filesize = 0_MPI_OFFSET_KIND
+   call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+   disp = 0_MPI_OFFSET_KIND
+
+   ! test writing scalar data
+   allocate (tmp(2))
+   tmp(1) = 1._mytype
+   tmp(2) = 2._mytype
+   allocate (ctmp(3))
+   ctmp(1) = cmplx(1.0, 1.0, mytype)
+   ctmp(2) = cmplx(2.0, 2.0, mytype)
+   ctmp(3) = cmplx(3.0, 3.0, mytype)
+   allocate (itmp(3))
+   call decomp_2d_write_scalar(fh, disp, 2, tmp)
+   call decomp_2d_write_scalar(fh, disp, 3, ctmp)
+   call decomp_2d_write_scalar(fh, disp, 3, (/nx, ny, nz/))
+
+   ! test the IO routines by writing all data to disk
+   call decomp_2d_write_var(fh, disp, 1, u1)
+   call decomp_2d_write_var(fh, disp, 2, u2)
+   call decomp_2d_write_var(fh, disp, 3, u3)
+   call decomp_2d_write_var(fh, disp, 1, u1l, large)
+   call decomp_2d_write_var(fh, disp, 2, u2l, large)
+   call decomp_2d_write_var(fh, disp, 3, u3l, large)
+   call decomp_2d_write_var(fh, disp, 1, cu1)
+   call decomp_2d_write_var(fh, disp, 2, cu2)
+   call decomp_2d_write_var(fh, disp, 3, cu3)
+
+   call MPI_FILE_CLOSE(fh, ierror)
+
+   if (nrank == 0) write (*, *) 'disp=', disp
+
+   ! read data back in from file
+   call MPI_FILE_OPEN(MPI_COMM_WORLD, filename, &
+                      MPI_MODE_RDONLY, MPI_INFO_NULL, &
+                      fh, ierror)
+   ! skip the scalars (2 real, 3 cmplx, 3 int)
 #ifdef DOUBLE_PREC
-  ! if double precision: 2*8+3*8*2+3*4
-  disp = 76_MPI_OFFSET_KIND
+   ! if double precision: 2*8+3*8*2+3*4
+   disp = 76_MPI_OFFSET_KIND
 #else
-  ! if single precision: 2*4+3*4*2+3*4
-  disp = 44_MPI_OFFSET_KIND
+   ! if single precision: 2*4+3*4*2+3*4
+   disp = 44_MPI_OFFSET_KIND
 #endif
-  
-  call decomp_2d_read_var(fh,disp,1,u1_b)
-  call decomp_2d_read_var(fh,disp,2,u2_b)
-  call decomp_2d_read_var(fh,disp,3,u3_b)
-  call decomp_2d_read_var(fh,disp,1,u1l_b,large)
-  call decomp_2d_read_var(fh,disp,2,u2l_b,large)
-  call decomp_2d_read_var(fh,disp,3,u3l_b,large)
-  call decomp_2d_read_var(fh,disp,1,cu1_b)
-  call decomp_2d_read_var(fh,disp,2,cu2_b)
-  call decomp_2d_read_var(fh,disp,3,cu3_b)
-
-  disp = 0_MPI_OFFSET_KIND
-  call decomp_2d_read_scalar(fh,disp,2,tmp)
-  call decomp_2d_read_scalar(fh,disp,3,ctmp)
-  call decomp_2d_read_scalar(fh,disp,3,itmp)
-  if (nrank==0) then
-     write(*,'(2F8.3)') tmp
-     write(*,20) ctmp
-20   format(3(:,'(',F5.2,',',F5.2,')'))
-     write(*,'(A,3I5)') 'nx,ny,nz', itmp
-  end if
-
-  call MPI_FILE_CLOSE(fh,ierror)
-  deallocate(tmp, ctmp, itmp)
-
-  ! validate the data
-  do k=xstart(3),xend(3)
-     do j=xstart(2),xend(2)
-        do i=xstart(1),xend(1)
-           if (abs(u1(i,j,k)-u1_b(i,j,k)) > eps) stop 1
-           if (abs(cu1(i,j,k)-cu1_b(i,j,k)) > eps) stop 2
-        end do
-     end do
-  end do
-
-  do k=ystart(3),yend(3)
-     do j=ystart(2),yend(2)
-        do i=ystart(1),yend(1)
-           if (abs(u2(i,j,k)-u2_b(i,j,k)) > eps) stop 3
-           if (abs(cu2(i,j,k)-cu2_b(i,j,k)) > eps) stop 4
-        end do
-     end do
-  end do
-
-  do k=zstart(3),zend(3)
-     do j=zstart(2),zend(2)
-        do i=zstart(1),zend(1)
-           if (abs(u3(i,j,k)-u3_b(i,j,k)) > eps) stop 5
-           if (abs(cu3(i,j,k)-cu3_b(i,j,k)) > eps) stop 6
-        end do
-     end do
-  end do
-
-  do k=large%xst(3),large%xen(3)
-     do j=large%xst(2),large%xen(2)
-        do i=large%xst(1),large%xen(1)
-           if (abs(u1l(i,j,k)-u1l_b(i,j,k)) > eps) stop 7
-        end do
-     end do
-  end do
-
-  do k=large%yst(3),large%yen(3)
-     do j=large%yst(2),large%yen(2)
-        do i=large%yst(1),large%yen(1)
-           if (abs(u2l(i,j,k)-u2l_b(i,j,k)) > eps) stop 8
-        end do
-     end do
-  end do
-
-  do k=large%zst(3),large%zen(3)
-     do j=large%zst(2),large%zen(2)
-        do i=large%zst(1),large%zen(1)
-           if (abs(u3l(i,j,k)-u3l_b(i,j,k)) > eps) stop 9
-        end do
-     end do
-  end do
-
-  if (nrank==0) write(*,*) 'passed self test'
-
-  ! clean up
-  call decomp_info_finalize(large)
-  call decomp_2d_finalize
-  call MPI_FINALIZE(ierror)
-  deallocate(u1,u2,u3)
-  deallocate(u1l,u2l,u3l)
-  deallocate(cu1,cu2,cu3)
-  deallocate(u1_b,u2_b,u3_b)
-  deallocate(u1l_b,u2l_b,u3l_b)
-  deallocate(cu1_b,cu2_b,cu3_b)
-  deallocate(data1_large)
-  
+
+   call decomp_2d_read_var(fh, disp, 1, u1_b)
+   call decomp_2d_read_var(fh, disp, 2, u2_b)
+   call decomp_2d_read_var(fh, disp, 3, u3_b)
+   call decomp_2d_read_var(fh, disp, 1, u1l_b, large)
+   call decomp_2d_read_var(fh, disp, 2, u2l_b, large)
+   call decomp_2d_read_var(fh, disp, 3, u3l_b, large)
+   call decomp_2d_read_var(fh, disp, 1, cu1_b)
+   call decomp_2d_read_var(fh, disp, 2, cu2_b)
+   call decomp_2d_read_var(fh, disp, 3, cu3_b)
+
+   disp = 0_MPI_OFFSET_KIND
+   call decomp_2d_read_scalar(fh, disp, 2, tmp)
+   call decomp_2d_read_scalar(fh, disp, 3, ctmp)
+   call decomp_2d_read_scalar(fh, disp, 3, itmp)
+   if (nrank == 0) then
+      write (*, '(2F8.3)') tmp
+      write (*, 20) ctmp
+20    format(3(:, '(', F5.2, ',', F5.2, ')'))
+      write (*, '(A,3I5)') 'nx,ny,nz', itmp
+   end if
+
+   call MPI_FILE_CLOSE(fh, ierror)
+   deallocate (tmp, ctmp, itmp)
+
+   ! validate the data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs(u1(i, j, k) - u1_b(i, j, k)) > eps) stop 1
+            if (abs(cu1(i, j, k) - cu1_b(i, j, k)) > eps) stop 2
+         end do
+      end do
+   end do
+
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs(u2(i, j, k) - u2_b(i, j, k)) > eps) stop 3
+            if (abs(cu2(i, j, k) - cu2_b(i, j, k)) > eps) stop 4
+         end do
+      end do
+   end do
+
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs(u3(i, j, k) - u3_b(i, j, k)) > eps) stop 5
+            if (abs(cu3(i, j, k) - cu3_b(i, j, k)) > eps) stop 6
+         end do
+      end do
+   end do
+
+   do k = large%xst(3), large%xen(3)
+      do j = large%xst(2), large%xen(2)
+         do i = large%xst(1), large%xen(1)
+            if (abs(u1l(i, j, k) - u1l_b(i, j, k)) > eps) stop 7
+         end do
+      end do
+   end do
+
+   do k = large%yst(3), large%yen(3)
+      do j = large%yst(2), large%yen(2)
+         do i = large%yst(1), large%yen(1)
+            if (abs(u2l(i, j, k) - u2l_b(i, j, k)) > eps) stop 8
+         end do
+      end do
+   end do
+
+   do k = large%zst(3), large%zen(3)
+      do j = large%zst(2), large%zen(2)
+         do i = large%zst(1), large%zen(1)
+            if (abs(u3l(i, j, k) - u3l_b(i, j, k)) > eps) stop 9
+         end do
+      end do
+   end do
+
+   if (nrank == 0) write (*, *) 'passed self test'
+
+   ! clean up
+   call decomp_info_finalize(large)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1, u2, u3)
+   deallocate (u1l, u2l, u3l)
+   deallocate (cu1, cu2, cu3)
+   deallocate (u1_b, u2_b, u3_b)
+   deallocate (u1l_b, u2l_b, u3l_b)
+   deallocate (cu1_b, cu2_b, cu3_b)
+   deallocate (data1_large)
+
 end program io_var_test
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 9c19841a..cf8a0194 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -1,166 +1,166 @@
 program test2d
- 
-  use mpi
-
-  use decomp_2d
-  ! use decomp_2d_io
-
-  implicit none
-
-  integer, parameter :: nx=17, ny=13, nz=11
-  integer :: p_row=0, p_col=0
-
-  real(mytype), dimension(nx,ny,nz) :: data1
-  
-  real(mytype), allocatable, dimension(:,:,:) :: u1, u2, u3
-  
-  integer :: i,j,k, m, ierror
-  logical :: error_flag
-
-  ! Init
-  error_flag = .false.
-  call MPI_INIT(ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_INIT")
-  call decomp_2d_init(nx,ny,nz,p_row,p_col)
-
-  ! ***** global data *****
-  m = 1
-  do k=1,nz
-     do j=1,ny
-        do i=1,nx
-           data1(i,j,k) = float(m)
-           m = m+1
-        end do
-     end do
-  end do
+
+   use mpi
+
+   use decomp_2d
+   ! use decomp_2d_io
+
+   implicit none
+
+   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer :: p_row = 0, p_col = 0
+
+   real(mytype), dimension(nx, ny, nz) :: data1
+
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+
+   integer :: i, j, k, m, ierror
+   logical :: error_flag
+
+   ! Init
+   error_flag = .false.
+   call MPI_INIT(ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_INIT")
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
+            data1(i, j, k) = float(m)
+            m = m + 1
+         end do
+      end do
+   end do
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Testing the swap routines
+   ! Testing the swap routines
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  
-  !allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-  !allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-  !allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-  call alloc_x(u1, opt_global=.true.)
-  call alloc_y(u2, opt_global=.true.)
-  call alloc_z(u3, opt_global=.true.)
- 
-  ! original x-pensil based data 
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        u1(i,j,k) = data1(i,j,k)
+
+   !allocate(u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   !allocate(u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+   !allocate(u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   call alloc_x(u1, opt_global=.true.)
+   call alloc_y(u2, opt_global=.true.)
+   call alloc_z(u3, opt_global=.true.)
+
+   ! original x-pensil based data
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            u1(i, j, k) = data1(i, j, k)
+         end do
       end do
-    end do
-  end do
+   end do
 
 10 format(15I5)
 
 #ifdef DEBUG
-  if (nrank==0) then 
-     write(*,*) 'Numbers held on Rank 0'
-     write(*,*) ' '
-     write(*,*) 'X-pencil'
-     write(*,10) int(u1)
-  end if
+   if (nrank == 0) then
+      write (*, *) 'Numbers held on Rank 0'
+      write (*, *) ' '
+      write (*, *) 'X-pencil'
+      write (*, 10) int(u1)
+   end if
 #endif
 
-  ! call decomp_2d_write_one(1,u1,'u1.dat')
+   ! call decomp_2d_write_one(1,u1,'u1.dat')
 
   !!!!!!!!!!!!!!!!!!!!!!!
-  ! x-pensil ==> y-pensil
-  call transpose_x_to_y(u1,u2)
+   ! x-pensil ==> y-pensil
+   call transpose_x_to_y(u1, u2)
 
 #ifdef DEBUG
-  if (nrank==0) then
-     write(*,*) ' '
-     write(*,*) 'Y-pencil'
-     write(*,10) int(u2)
-  end if
+   if (nrank == 0) then
+      write (*, *) ' '
+      write (*, *) 'Y-pencil'
+      write (*, 10) int(u2)
+   end if
 #endif
 
-  ! call decomp_2d_write_one(2,u2,'u2.dat')
-  ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte 
+   ! call decomp_2d_write_one(2,u2,'u2.dat')
+   ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
 
-  ! also check the transposition this way
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs(u2(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
+   ! also check the transposition this way
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+         end do
       end do
-    end do
-  end do
-  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-  if (error_flag) call decomp_2d_abort(1, "error swaping x->y")
+   end do
+   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+   if (error_flag) call decomp_2d_abort(1, "error swaping x->y")
 
   !!!!!!!!!!!!!!!!!!!!!!!
-  ! y-pensil ==> z-pensil
-  call transpose_y_to_z(u2,u3)
+   ! y-pensil ==> z-pensil
+   call transpose_y_to_z(u2, u3)
 
 #ifdef DEBUG
-  if (nrank==0) then
-     write(*,*) ' '
-     write(*,*) 'Z-pencil'
-     write(*,10) int(u3)
-  end if
+   if (nrank == 0) then
+      write (*, *) ' '
+      write (*, *) 'Z-pencil'
+      write (*, 10) int(u3)
+   end if
 #endif
 
-  ! call decomp_2d_write_one(3,u3,'u3.dat')
-  ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
+   ! call decomp_2d_write_one(3,u3,'u3.dat')
+   ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
-  do k=zstart(3),zend(3)
-    do j=zstart(2),zend(2)
-      do i=zstart(1),zend(1)
-        if (abs(u3(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
+   do k = zstart(3), zend(3)
+      do j = zstart(2), zend(2)
+         do i = zstart(1), zend(1)
+            if (abs(u3(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+         end do
       end do
-    end do
-  end do
-  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-  if (error_flag) call decomp_2d_abort(2, "error swaping y->z")
+   end do
+   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+   if (error_flag) call decomp_2d_abort(2, "error swaping y->z")
 
   !!!!!!!!!!!!!!!!!!!!!!!
-  ! z-pensil ==> y-pensil
-  call transpose_z_to_y(u3,u2)
-  ! call decomp_2d_write_one(2,u2,'u2b.dat')
-
-  do k=ystart(3),yend(3)
-    do j=ystart(2),yend(2)
-      do i=ystart(1),yend(1)
-        if (abs(u2(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
+   ! z-pensil ==> y-pensil
+   call transpose_z_to_y(u3, u2)
+   ! call decomp_2d_write_one(2,u2,'u2b.dat')
+
+   do k = ystart(3), yend(3)
+      do j = ystart(2), yend(2)
+         do i = ystart(1), yend(1)
+            if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+         end do
       end do
-    end do
-  end do
-  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-  if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
+   end do
+   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+   if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
 
   !!!!!!!!!!!!!!!!!!!!!!!
-  ! y-pensil ==> x-pensil
-  call transpose_y_to_x(u2,u1)
-  ! call decomp_2d_write_one(1,u1,'u1b.dat')
-
-  do k=xstart(3),xend(3)
-    do j=xstart(2),xend(2)
-      do i=xstart(1),xend(1)
-        if (abs(u1(i,j,k)-data1(i,j,k)).gt.0) error_flag = .true.
+   ! y-pensil ==> x-pensil
+   call transpose_y_to_x(u2, u1)
+   ! call decomp_2d_write_one(1,u1,'u1b.dat')
+
+   do k = xstart(3), xend(3)
+      do j = xstart(2), xend(2)
+         do i = xstart(1), xend(1)
+            if (abs(u1(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+         end do
       end do
-    end do
-  end do
-  call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
-  if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
-  if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
-
-  if (nrank == 0) then
-    write(*,*) " "
-    write(*,*) "test2d completed"
-    write(*,*) " "
-  endif
-
-  call decomp_2d_finalize 
-  call MPI_FINALIZE(ierror)
-  deallocate(u1,u2,u3)
+   end do
+   call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+   if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
+
+   if (nrank == 0) then
+      write (*, *) " "
+      write (*, *) "test2d completed"
+      write (*, *) " "
+   end if
+
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   deallocate (u1, u2, u3)
 
 end program test2d
 
diff --git a/src/alloc.f90 b/src/alloc.f90
index ffcbfa14..51c893ef 100644
--- a/src/alloc.f90
+++ b/src/alloc.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -16,281 +16,281 @@
   ! X-pencil real arrays
   subroutine alloc_x_real_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_x(var, decomp_main, opt_global)
+     call alloc_x(var, decomp_main, opt_global)
 
   end subroutine alloc_x_real_short
 
   subroutine alloc_x_real(var, decomp, opt_global)
 
-    implicit none
-
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%xst(1):decomp%xen(1), &
-            decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%xst(1):decomp%xen(1), &
+                      decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_x_real
 
   ! X-pencil complex arrays
   subroutine alloc_x_complex_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_x(var, decomp_main, opt_global)
+     call alloc_x(var, decomp_main, opt_global)
 
   end subroutine alloc_x_complex_short
 
   subroutine alloc_x_complex(var, decomp, opt_global)
 
-    implicit none
-
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%xst(1):decomp%xen(1), &
-            decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%xst(1):decomp%xen(1), &
+                      decomp%xst(2):decomp%xen(2), decomp%xst(3):decomp%xen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_x_complex
 
   ! Y-pencil real arrays
   subroutine alloc_y_real_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_y(var, decomp_main, opt_global)
+     call alloc_y(var, decomp_main, opt_global)
 
   end subroutine alloc_y_real_short
 
   subroutine alloc_y_real(var, decomp, opt_global)
 
-    implicit none
-
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%yst(1):decomp%yen(1), &
-            decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%yst(1):decomp%yen(1), &
+                      decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_y_real
 
   ! Y-pencil complex arrays
   subroutine alloc_y_complex_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_y(var, decomp_main, opt_global)
+     call alloc_y(var, decomp_main, opt_global)
 
   end subroutine alloc_y_complex_short
 
   subroutine alloc_y_complex(var, decomp, opt_global)
 
-    implicit none
-
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%yst(1):decomp%yen(1), &
-            decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%yst(1):decomp%yen(1), &
+                      decomp%yst(2):decomp%yen(2), decomp%yst(3):decomp%yen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_y_complex
 
   ! Z-pencil real arrays
   subroutine alloc_z_real_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_z(var, decomp_main, opt_global)
+     call alloc_z(var, decomp_main, opt_global)
 
   end subroutine alloc_z_real_short
 
   subroutine alloc_z_real(var, decomp, opt_global)
 
-    implicit none
-
-    real(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%zst(1):decomp%zen(1), &
-            decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     real(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%zst(1):decomp%zen(1), &
+                      decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_z_real
 
   ! Z-pencil complex arrays
   subroutine alloc_z_complex_short(var, opt_global)
 
-    implicit none
+     implicit none
 
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
 
-    call alloc_z(var, decomp_main, opt_global)
+     call alloc_z(var, decomp_main, opt_global)
 
   end subroutine alloc_z_complex_short
 
   subroutine alloc_z_complex(var, decomp, opt_global)
 
-    implicit none
-
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    logical, intent(IN), optional :: opt_global
-
-    logical :: global
-    integer :: alloc_stat, errorcode
-
-    if (present(opt_global)) then
-       global = opt_global
-    else
-       global = .false.
-    end if
-
-    if (global) then
-       allocate(var(decomp%zst(1):decomp%zen(1), &
-            decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
-            stat=alloc_stat)
-    else
-       allocate(var(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)), &
-            stat=alloc_stat)
-    end if
-    
-    if (alloc_stat /= 0) then
-       errorcode = 8
-       call decomp_2d_abort(errorcode, &
-            'Memory allocation failed when creating new arrays')
-    end if
-
-    return
+     implicit none
+
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+     logical, intent(IN), optional :: opt_global
+
+     logical :: global
+     integer :: alloc_stat, errorcode
+
+     if (present(opt_global)) then
+        global = opt_global
+     else
+        global = .false.
+     end if
+
+     if (global) then
+        allocate (var(decomp%zst(1):decomp%zen(1), &
+                      decomp%zst(2):decomp%zen(2), decomp%zst(3):decomp%zen(3)), &
+                  stat=alloc_stat)
+     else
+        allocate (var(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)), &
+                  stat=alloc_stat)
+     end if
+
+     if (alloc_stat /= 0) then
+        errorcode = 8
+        call decomp_2d_abort(errorcode, &
+                             'Memory allocation failed when creating new arrays')
+     end if
+
+     return
   end subroutine alloc_z_complex
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 11953900..e4142fd9 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -14,362 +14,362 @@
 
 module decomp_2d
 
-  use MPI
-  use, intrinsic :: iso_fortran_env, only : real32, real64
-  use factor
+   use MPI
+   use, intrinsic :: iso_fortran_env, only: real32, real64
+   use factor
 #if defined(_GPU)
-  use cudafor
+   use cudafor
 #if defined(_NCCL)
-  use nccl
+   use nccl
 #endif
 #endif
 
-  implicit none
+   implicit none
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
 #ifdef DOUBLE_PREC
-  integer, parameter, public :: mytype = KIND(0._real64)
-  integer, parameter, public :: real_type = MPI_DOUBLE_PRECISION
-  integer, parameter, public :: real2_type = MPI_2DOUBLE_PRECISION
-  integer, parameter, public :: complex_type = MPI_DOUBLE_COMPLEX
+   integer, parameter, public :: mytype = KIND(0._real64)
+   integer, parameter, public :: real_type = MPI_DOUBLE_PRECISION
+   integer, parameter, public :: real2_type = MPI_2DOUBLE_PRECISION
+   integer, parameter, public :: complex_type = MPI_DOUBLE_COMPLEX
 #ifdef SAVE_SINGLE
-  integer, parameter, public :: mytype_single = KIND(0._real32)
-  integer, parameter, public :: real_type_single = MPI_REAL
+   integer, parameter, public :: mytype_single = KIND(0._real32)
+   integer, parameter, public :: real_type_single = MPI_REAL
 #else
-  integer, parameter, public :: mytype_single = KIND(0._real64)
-  integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
+   integer, parameter, public :: mytype_single = KIND(0._real64)
+   integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
 #endif
 #else
-  integer, parameter, public :: mytype = KIND(0._real32)
-  integer, parameter, public :: real_type = MPI_REAL
-  integer, parameter, public :: real2_type = MPI_2REAL
-  integer, parameter, public :: complex_type = MPI_COMPLEX
-  integer, parameter, public :: mytype_single = KIND(0._real32)
-  integer, parameter, public :: real_type_single = MPI_REAL
+   integer, parameter, public :: mytype = KIND(0._real32)
+   integer, parameter, public :: real_type = MPI_REAL
+   integer, parameter, public :: real2_type = MPI_2REAL
+   integer, parameter, public :: complex_type = MPI_COMPLEX
+   integer, parameter, public :: mytype_single = KIND(0._real32)
+   integer, parameter, public :: real_type_single = MPI_REAL
 #endif
 
-  integer, save, public :: mytype_bytes
-
-  ! some key global variables
-  integer, save, public :: nx_global, ny_global, nz_global  ! global size
-
-  integer, save, public :: nrank = -1 ! local MPI rank 
-  integer, save, public :: nproc = -1 ! total number of processors
-  integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
-
-  ! parameters for 2D Cartesian topology 
-  integer, save, dimension(2) :: dims, coord
-  integer, save, public :: DECOMP_2D_COMM_CART_X = MPI_COMM_NULL
-  integer, save, public :: DECOMP_2D_COMM_CART_Y = MPI_COMM_NULL
-  integer, save, public :: DECOMP_2D_COMM_CART_Z = MPI_COMM_NULL
-  integer, save :: DECOMP_2D_COMM_ROW = MPI_COMM_NULL
-  integer, save :: DECOMP_2D_COMM_COL = MPI_COMM_NULL
-
-  ! define neighboring blocks (to be used in halo-cell support)
-  !  first dimension 1=X-pencil, 2=Y-pencil, 3=Z-pencil
-  ! second dimension 1=east, 2=west, 3=north, 4=south, 5=top, 6=bottom 
-  integer, save, dimension(3,6) :: neighbour 
-
-  ! flags for periodic condition in three dimensions
-  logical, save :: periodic_x, periodic_y, periodic_z
-
-  !
-  ! Output for the log can be changed by the external code before calling decomp_2d_init
-  !
-  !    0 => No log output
-  !    1 => Master rank log output to stdout
-  !    2 => Master rank log output to the file "decomp_2d_setup.log"
-  !    3 => All ranks log output to a dedicated file
-  !
-  ! The default value is 2 (3 for debug builds)
-  !
-  integer, parameter, public :: D2D_LOG_QUIET = 0
-  integer, parameter, public :: D2D_LOG_STDOUT = 1
-  integer, parameter, public :: D2D_LOG_TOFILE = 2
-  integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
+   integer, save, public :: mytype_bytes
+
+   ! some key global variables
+   integer, save, public :: nx_global, ny_global, nz_global  ! global size
+
+   integer, save, public :: nrank = -1 ! local MPI rank
+   integer, save, public :: nproc = -1 ! total number of processors
+   integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
+
+   ! parameters for 2D Cartesian topology
+   integer, save, dimension(2) :: dims, coord
+   integer, save, public :: DECOMP_2D_COMM_CART_X = MPI_COMM_NULL
+   integer, save, public :: DECOMP_2D_COMM_CART_Y = MPI_COMM_NULL
+   integer, save, public :: DECOMP_2D_COMM_CART_Z = MPI_COMM_NULL
+   integer, save :: DECOMP_2D_COMM_ROW = MPI_COMM_NULL
+   integer, save :: DECOMP_2D_COMM_COL = MPI_COMM_NULL
+
+   ! define neighboring blocks (to be used in halo-cell support)
+   !  first dimension 1=X-pencil, 2=Y-pencil, 3=Z-pencil
+   ! second dimension 1=east, 2=west, 3=north, 4=south, 5=top, 6=bottom
+   integer, save, dimension(3, 6) :: neighbour
+
+   ! flags for periodic condition in three dimensions
+   logical, save :: periodic_x, periodic_y, periodic_z
+
+   !
+   ! Output for the log can be changed by the external code before calling decomp_2d_init
+   !
+   !    0 => No log output
+   !    1 => Master rank log output to stdout
+   !    2 => Master rank log output to the file "decomp_2d_setup.log"
+   !    3 => All ranks log output to a dedicated file
+   !
+   ! The default value is 2 (3 for debug builds)
+   !
+   integer, parameter, public :: D2D_LOG_QUIET = 0
+   integer, parameter, public :: D2D_LOG_STDOUT = 1
+   integer, parameter, public :: D2D_LOG_TOFILE = 2
+   integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
 #ifdef DEBUG
-  integer, public, save :: decomp_log = D2D_LOG_TOFILE_FULL
+   integer, public, save :: decomp_log = D2D_LOG_TOFILE_FULL
 #else
-  integer, public, save :: decomp_log = D2D_LOG_TOFILE
+   integer, public, save :: decomp_log = D2D_LOG_TOFILE
 #endif
 
-  !
-  ! Debug level can be changed by the external code before calling decomp_2d_init
-  !
-  ! The environment variable "DECOMP_2D_DEBUG" can be used to change the debug level
-  !
-  ! Debug checks are performed only when the preprocessor variable DEBUG is defined
-  !
-  enum, bind(c)
-     enumerator :: D2D_DEBUG_LEVEL_OFF = 0
-     enumerator :: D2D_DEBUG_LEVEL_CRITICAL = 1
-     enumerator :: D2D_DEBUG_LEVEL_ERROR = 2
-     enumerator :: D2D_DEBUG_LEVEL_WARN = 3
-     enumerator :: D2D_DEBUG_LEVEL_INFO = 4
-     enumerator :: D2D_DEBUG_LEVEL_DEBUG = 5
-     enumerator :: D2D_DEBUG_LEVEL_TRACE = 6
-  end enum
+   !
+   ! Debug level can be changed by the external code before calling decomp_2d_init
+   !
+   ! The environment variable "DECOMP_2D_DEBUG" can be used to change the debug level
+   !
+   ! Debug checks are performed only when the preprocessor variable DEBUG is defined
+   !
+   enum, bind(c)
+      enumerator :: D2D_DEBUG_LEVEL_OFF = 0
+      enumerator :: D2D_DEBUG_LEVEL_CRITICAL = 1
+      enumerator :: D2D_DEBUG_LEVEL_ERROR = 2
+      enumerator :: D2D_DEBUG_LEVEL_WARN = 3
+      enumerator :: D2D_DEBUG_LEVEL_INFO = 4
+      enumerator :: D2D_DEBUG_LEVEL_DEBUG = 5
+      enumerator :: D2D_DEBUG_LEVEL_TRACE = 6
+   end enum
 #ifdef DEBUG
-  integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_INFO
+   integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_INFO
 #else
-  integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_OFF
+   integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_OFF
 #endif
 
 #if defined(_GPU)
 #if defined(_NCCL)
-  integer, save :: row_rank, col_rank
+   integer, save :: row_rank, col_rank
 #endif
 #endif
 
 #ifdef SHM
-  ! derived type to store shared-memory info
-  TYPE, public :: SMP_INFO
-     integer MPI_COMM          ! SMP associated with this communicator
-     integer NODE_ME           ! rank in this communicator
-     integer NCPU              ! size of this communicator
-     integer SMP_COMM          ! communicator for SMP-node masters
-     integer CORE_COMM         ! communicator for cores on SMP-node
-     integer SMP_ME            ! SMP-node id starting from 1 ... NSMP
-     integer NSMP              ! number of SMP-nodes in this communicator
-     integer CORE_ME           ! core id starting from 1 ... NCORE
-     integer NCORE             ! number of cores on this SMP-node
-     integer MAXCORE           ! maximum no. cores on any SMP-node
-     integer N_SND             ! size of SMP shared memory buffer
-     integer N_RCV             ! size of SMP shared memory buffer
-     integer(8) SND_P          ! SNDBUF address (cray pointer), for real 
-     integer(8) RCV_P          ! RCVBUF address (cray pointer), for real
-     integer(8) SND_P_c        ! for complex
-     integer(8) RCV_P_c        ! for complex
-  END TYPE SMP_INFO
+   ! derived type to store shared-memory info
+   TYPE, public :: SMP_INFO
+      integer MPI_COMM          ! SMP associated with this communicator
+      integer NODE_ME           ! rank in this communicator
+      integer NCPU              ! size of this communicator
+      integer SMP_COMM          ! communicator for SMP-node masters
+      integer CORE_COMM         ! communicator for cores on SMP-node
+      integer SMP_ME            ! SMP-node id starting from 1 ... NSMP
+      integer NSMP              ! number of SMP-nodes in this communicator
+      integer CORE_ME           ! core id starting from 1 ... NCORE
+      integer NCORE             ! number of cores on this SMP-node
+      integer MAXCORE           ! maximum no. cores on any SMP-node
+      integer N_SND             ! size of SMP shared memory buffer
+      integer N_RCV             ! size of SMP shared memory buffer
+      integer(8) SND_P          ! SNDBUF address (cray pointer), for real
+      integer(8) RCV_P          ! RCVBUF address (cray pointer), for real
+      integer(8) SND_P_c        ! for complex
+      integer(8) RCV_P_c        ! for complex
+   END TYPE SMP_INFO
 #endif
 
-  ! derived type to store decomposition info for a given global data size
-  TYPE, public :: DECOMP_INFO
-     ! staring/ending index and size of data held by current processor
-     integer, dimension(3) :: xst, xen, xsz  ! x-pencil
-     integer, dimension(3) :: yst, yen, ysz  ! y-pencil
-     integer, dimension(3) :: zst, zen, zsz  ! z-pencil
+   ! derived type to store decomposition info for a given global data size
+   TYPE, public :: DECOMP_INFO
+      ! staring/ending index and size of data held by current processor
+      integer, dimension(3) :: xst, xen, xsz  ! x-pencil
+      integer, dimension(3) :: yst, yen, ysz  ! y-pencil
+      integer, dimension(3) :: zst, zen, zsz  ! z-pencil
 
-     ! in addition to local information, processors also need to know 
-     ! some global information for global communications to work 
+      ! in addition to local information, processors also need to know
+      ! some global information for global communications to work
 
-     ! how each dimension is distributed along pencils
-     integer, allocatable, dimension(:) :: &
-          x1dist, y1dist, y2dist, z2dist
+      ! how each dimension is distributed along pencils
+      integer, allocatable, dimension(:) :: &
+         x1dist, y1dist, y2dist, z2dist
 
-     ! send/receive buffer counts and displacements for MPI_ALLTOALLV
-     integer, allocatable, dimension(:) :: &
-          x1cnts, y1cnts, y2cnts, z2cnts
-     integer, allocatable, dimension(:) :: &
-          x1disp, y1disp, y2disp, z2disp
+      ! send/receive buffer counts and displacements for MPI_ALLTOALLV
+      integer, allocatable, dimension(:) :: &
+         x1cnts, y1cnts, y2cnts, z2cnts
+      integer, allocatable, dimension(:) :: &
+         x1disp, y1disp, y2disp, z2disp
 
-     ! buffer counts for MPI_ALLTOALL: either for evenly distributed data
-     ! or for padded-alltoall
-     integer :: x1count, y1count, y2count, z2count
+      ! buffer counts for MPI_ALLTOALL: either for evenly distributed data
+      ! or for padded-alltoall
+      integer :: x1count, y1count, y2count, z2count
 
-     ! evenly distributed data
-     logical :: even
+      ! evenly distributed data
+      logical :: even
 
 #ifdef SHM
-     ! For shared-memory implementation
-
-     ! one instance of this derived type for each communicator
-     ! shared moemory info, such as which MPI rank belongs to which node
-     TYPE(SMP_INFO) :: ROW_INFO, COL_INFO
-
-     ! shared send/recv buffers for ALLTOALLV
-     integer, allocatable, dimension(:) :: x1cnts_s, y1cnts_s, &
-          y2cnts_s, z2cnts_s
-     integer, allocatable, dimension(:) :: x1disp_s, y1disp_s, &
-          y2disp_s, z2disp_s
-     ! A copy of original buffer displacement (will be overwriten)
-     integer, allocatable, dimension(:) :: x1disp_o, y1disp_o, &
-          y2disp_o, z2disp_o
+      ! For shared-memory implementation
+
+      ! one instance of this derived type for each communicator
+      ! shared moemory info, such as which MPI rank belongs to which node
+      TYPE(SMP_INFO) :: ROW_INFO, COL_INFO
+
+      ! shared send/recv buffers for ALLTOALLV
+      integer, allocatable, dimension(:) :: x1cnts_s, y1cnts_s, &
+                                            y2cnts_s, z2cnts_s
+      integer, allocatable, dimension(:) :: x1disp_s, y1disp_s, &
+                                            y2disp_s, z2disp_s
+      ! A copy of original buffer displacement (will be overwriten)
+      integer, allocatable, dimension(:) :: x1disp_o, y1disp_o, &
+                                            y2disp_o, z2disp_o
 #endif
-  END TYPE DECOMP_INFO
+   END TYPE DECOMP_INFO
 
-  ! main (default) decomposition information for global size nx*ny*nz
-  TYPE(DECOMP_INFO), target, save, public :: decomp_main
-  ! FIXME The extra decomp_info objects should be defined in the external code, not here
-  !       Currently keeping them to avoid breaking external codes
-  TYPE(DECOMP_INFO), save, public :: phG,ph1,ph2,ph3,ph4
+   ! main (default) decomposition information for global size nx*ny*nz
+   TYPE(DECOMP_INFO), target, save, public :: decomp_main
+   ! FIXME The extra decomp_info objects should be defined in the external code, not here
+   !       Currently keeping them to avoid breaking external codes
+   TYPE(DECOMP_INFO), save, public :: phG, ph1, ph2, ph3, ph4
 
-  ! staring/ending index and size of data held by current processor
-  ! duplicate 'decomp_main', needed by apps to define data structure 
-  integer, save, dimension(3), public :: xstart, xend, xsize  ! x-pencil
-  integer, save, dimension(3), public :: ystart, yend, ysize  ! y-pencil
-  integer, save, dimension(3), public :: zstart, zend, zsize  ! z-pencil
+   ! staring/ending index and size of data held by current processor
+   ! duplicate 'decomp_main', needed by apps to define data structure
+   integer, save, dimension(3), public :: xstart, xend, xsize  ! x-pencil
+   integer, save, dimension(3), public :: ystart, yend, ysize  ! y-pencil
+   integer, save, dimension(3), public :: zstart, zend, zsize  ! z-pencil
 
-  ! These are the buffers used by MPI_ALLTOALL(V) calls
-  integer, save :: decomp_buf_size = 0
-  real(mytype),    allocatable, dimension(:) :: work1_r, work2_r
-  complex(mytype), allocatable, dimension(:) :: work1_c, work2_c
+   ! These are the buffers used by MPI_ALLTOALL(V) calls
+   integer, save :: decomp_buf_size = 0
+   real(mytype), allocatable, dimension(:) :: work1_r, work2_r
+   complex(mytype), allocatable, dimension(:) :: work1_c, work2_c
 
 #if defined(_GPU)
-  real(mytype), allocatable, dimension(:), device :: work1_r_d, work2_r_d
-  complex(mytype), allocatable, dimension(:), device :: work1_c_d, work2_c_d
+   real(mytype), allocatable, dimension(:), device :: work1_r_d, work2_r_d
+   complex(mytype), allocatable, dimension(:), device :: work1_c_d, work2_c_d
 
 #if defined(_NCCL)
-  integer col_comm_size, row_comm_size
-  integer, allocatable, dimension(:) :: local_to_global_col, local_to_global_row
-  type(ncclUniqueId) :: nccl_uid_2decomp
-  type(ncclComm) :: nccl_comm_2decomp
-  integer(kind=cuda_stream_kind) :: cuda_stream_2decomp
+   integer col_comm_size, row_comm_size
+   integer, allocatable, dimension(:) :: local_to_global_col, local_to_global_row
+   type(ncclUniqueId) :: nccl_uid_2decomp
+   type(ncclComm) :: nccl_comm_2decomp
+   integer(kind=cuda_stream_kind) :: cuda_stream_2decomp
 #endif
 #endif
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! To define smaller arrays using every several mesh points
-  integer, save, dimension(3), public :: xszS,yszS,zszS,xstS,ystS,zstS,xenS,yenS,zenS
-  integer, save, dimension(3), public :: xszV,yszV,zszV,xstV,ystV,zstV,xenV,yenV,zenV
-  integer, save, dimension(3), public :: xszP,yszP,zszP,xstP,ystP,zstP,xenP,yenP,zenP
-  logical, save :: coarse_mesh_starts_from_1
-  integer, save :: iskipS, jskipS, kskipS
-  integer, save :: iskipV, jskipV, kskipV
-  integer, save :: iskipP, jskipP, kskipP
-
-  !
-  ! Profiler section
-  !
-  ! Integer to select the profiling tool
-  !    0 => no profiling, default
-  !    1 => Caliper (https://github.com/LLNL/Caliper)
-  !
-  enum, bind(c)
-    enumerator :: decomp_profiler_none = 0
-    enumerator :: decomp_profiler_caliper = 1
-  end enum
-  integer(kind(decomp_profiler_none)), save, public :: decomp_profiler = decomp_profiler_none
-  ! Default : profile everything
-  logical, save, public :: decomp_profiler_transpose = .true.
-  logical, save, public :: decomp_profiler_io = .true.
-  logical, save, public :: decomp_profiler_fft = .true.
-  logical, save, public :: decomp_profiler_d2d = .true.
-
-  ! public user routines
-  public :: decomp_2d_init, decomp_2d_finalize, &
-       transpose_x_to_y, transpose_y_to_z, &
-       transpose_z_to_y, transpose_y_to_x, &
-       decomp_info_init, decomp_info_finalize, partition, &
-       decomp_info_print, decomp_profiler_prep, &
-       decomp_profiler_start, decomp_profiler_end, &
-       init_coarser_mesh_statS,fine_to_coarseS,&
-       init_coarser_mesh_statV,fine_to_coarseV,&
-       init_coarser_mesh_statP,fine_to_coarseP,&
-       alloc_x, alloc_y, alloc_z, &
-       update_halo, decomp_2d_abort, &
-       decomp_2d_warning, get_decomp_info, &
-       decomp_mpi_comm_free, get_decomp_dims
+   ! To define smaller arrays using every several mesh points
+   integer, save, dimension(3), public :: xszS, yszS, zszS, xstS, ystS, zstS, xenS, yenS, zenS
+   integer, save, dimension(3), public :: xszV, yszV, zszV, xstV, ystV, zstV, xenV, yenV, zenV
+   integer, save, dimension(3), public :: xszP, yszP, zszP, xstP, ystP, zstP, xenP, yenP, zenP
+   logical, save :: coarse_mesh_starts_from_1
+   integer, save :: iskipS, jskipS, kskipS
+   integer, save :: iskipV, jskipV, kskipV
+   integer, save :: iskipP, jskipP, kskipP
+
+   !
+   ! Profiler section
+   !
+   ! Integer to select the profiling tool
+   !    0 => no profiling, default
+   !    1 => Caliper (https://github.com/LLNL/Caliper)
+   !
+   enum, bind(c)
+      enumerator :: decomp_profiler_none = 0
+      enumerator :: decomp_profiler_caliper = 1
+   end enum
+   integer(kind(decomp_profiler_none)), save, public :: decomp_profiler = decomp_profiler_none
+   ! Default : profile everything
+   logical, save, public :: decomp_profiler_transpose = .true.
+   logical, save, public :: decomp_profiler_io = .true.
+   logical, save, public :: decomp_profiler_fft = .true.
+   logical, save, public :: decomp_profiler_d2d = .true.
+
+   ! public user routines
+   public :: decomp_2d_init, decomp_2d_finalize, &
+             transpose_x_to_y, transpose_y_to_z, &
+             transpose_z_to_y, transpose_y_to_x, &
+             decomp_info_init, decomp_info_finalize, partition, &
+             decomp_info_print, decomp_profiler_prep, &
+             decomp_profiler_start, decomp_profiler_end, &
+             init_coarser_mesh_statS, fine_to_coarseS, &
+             init_coarser_mesh_statV, fine_to_coarseV, &
+             init_coarser_mesh_statP, fine_to_coarseP, &
+             alloc_x, alloc_y, alloc_z, &
+             update_halo, decomp_2d_abort, &
+             decomp_2d_warning, get_decomp_info, &
+             decomp_mpi_comm_free, get_decomp_dims
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! These are routines to perform global data transpositions
-  ! 
-  !   Four combinations are available, enough to cover all situations
-  !    - transpose_x_to_y (X-pencil --> Y-pencil)
-  !    - transpose_y_to_z (Y-pencil --> Z-pencil)
-  !    - transpose_z_to_y (Z-pencil --> Y-pencil)
-  !    - transpose_y_to_x (Y-pencil --> X-pencil)
-  !
-  !   Generic interface provided here to support multiple data types
-  !    - real and complex types supported through generic interface
-  !    - single/double precision supported through pre-processing
-  !       * see 'mytype' variable at the beginning
-  !    - an optional argument can be supplied to transpose data whose 
-  !      global size is not the default nx*ny*nz 
-  !       * as the case in fft r2c/c2r interface 
+   ! These are routines to perform global data transpositions
+   !
+   !   Four combinations are available, enough to cover all situations
+   !    - transpose_x_to_y (X-pencil --> Y-pencil)
+   !    - transpose_y_to_z (Y-pencil --> Z-pencil)
+   !    - transpose_z_to_y (Z-pencil --> Y-pencil)
+   !    - transpose_y_to_x (Y-pencil --> X-pencil)
+   !
+   !   Generic interface provided here to support multiple data types
+   !    - real and complex types supported through generic interface
+   !    - single/double precision supported through pre-processing
+   !       * see 'mytype' variable at the beginning
+   !    - an optional argument can be supplied to transpose data whose
+   !      global size is not the default nx*ny*nz
+   !       * as the case in fft r2c/c2r interface
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  interface decomp_2d_init
-     module procedure decomp_2d_init_ref
-  end interface decomp_2d_init
-
-  interface decomp_2d_finalize
-     module procedure decomp_2d_finalize_ref
-  end interface decomp_2d_finalize
-  
-  interface transpose_x_to_y
-     module procedure transpose_x_to_y_real
-     module procedure transpose_x_to_y_real_short
-     module procedure transpose_x_to_y_complex
-     module procedure transpose_x_to_y_complex_short
-  end interface transpose_x_to_y
-
-  interface transpose_y_to_z
-     module procedure transpose_y_to_z_real
-     module procedure transpose_y_to_z_real_short
-     module procedure transpose_y_to_z_complex
-     module procedure transpose_y_to_z_complex_short
-  end interface transpose_y_to_z
-
-  interface transpose_z_to_y
-     module procedure transpose_z_to_y_real
-     module procedure transpose_z_to_y_real_short
-     module procedure transpose_z_to_y_complex
-     module procedure transpose_z_to_y_complex_short
-  end interface transpose_z_to_y
-
-  interface transpose_y_to_x
-     module procedure transpose_y_to_x_real
-     module procedure transpose_y_to_x_real_short
-     module procedure transpose_y_to_x_complex
-     module procedure transpose_y_to_x_complex_short
-  end interface transpose_y_to_x
-
-  interface update_halo
-     module procedure update_halo_real
-     module procedure update_halo_real_short
-     module procedure update_halo_complex
-     module procedure update_halo_complex_short
-  end interface update_halo
-
-  interface alloc_x
-     module procedure alloc_x_real
-     module procedure alloc_x_real_short
-     module procedure alloc_x_complex
-     module procedure alloc_x_complex_short
-  end interface alloc_x
-
-  interface alloc_y
-     module procedure alloc_y_real
-     module procedure alloc_y_real_short
-     module procedure alloc_y_complex
-     module procedure alloc_y_complex_short
-  end interface alloc_y
-
-  interface alloc_z
-     module procedure alloc_z_real
-     module procedure alloc_z_real_short
-     module procedure alloc_z_complex
-     module procedure alloc_z_complex_short
-  end interface alloc_z
-
-  interface decomp_2d_abort
-     module procedure decomp_2d_abort_basic
-     module procedure decomp_2d_abort_file_line
+   interface decomp_2d_init
+      module procedure decomp_2d_init_ref
+   end interface decomp_2d_init
+
+   interface decomp_2d_finalize
+      module procedure decomp_2d_finalize_ref
+   end interface decomp_2d_finalize
+
+   interface transpose_x_to_y
+      module procedure transpose_x_to_y_real
+      module procedure transpose_x_to_y_real_short
+      module procedure transpose_x_to_y_complex
+      module procedure transpose_x_to_y_complex_short
+   end interface transpose_x_to_y
+
+   interface transpose_y_to_z
+      module procedure transpose_y_to_z_real
+      module procedure transpose_y_to_z_real_short
+      module procedure transpose_y_to_z_complex
+      module procedure transpose_y_to_z_complex_short
+   end interface transpose_y_to_z
+
+   interface transpose_z_to_y
+      module procedure transpose_z_to_y_real
+      module procedure transpose_z_to_y_real_short
+      module procedure transpose_z_to_y_complex
+      module procedure transpose_z_to_y_complex_short
+   end interface transpose_z_to_y
+
+   interface transpose_y_to_x
+      module procedure transpose_y_to_x_real
+      module procedure transpose_y_to_x_real_short
+      module procedure transpose_y_to_x_complex
+      module procedure transpose_y_to_x_complex_short
+   end interface transpose_y_to_x
+
+   interface update_halo
+      module procedure update_halo_real
+      module procedure update_halo_real_short
+      module procedure update_halo_complex
+      module procedure update_halo_complex_short
+   end interface update_halo
+
+   interface alloc_x
+      module procedure alloc_x_real
+      module procedure alloc_x_real_short
+      module procedure alloc_x_complex
+      module procedure alloc_x_complex_short
+   end interface alloc_x
+
+   interface alloc_y
+      module procedure alloc_y_real
+      module procedure alloc_y_real_short
+      module procedure alloc_y_complex
+      module procedure alloc_y_complex_short
+   end interface alloc_y
+
+   interface alloc_z
+      module procedure alloc_z_real
+      module procedure alloc_z_real_short
+      module procedure alloc_z_complex
+      module procedure alloc_z_complex_short
+   end interface alloc_z
+
+   interface decomp_2d_abort
+      module procedure decomp_2d_abort_basic
+      module procedure decomp_2d_abort_file_line
 #if defined(_GPU) && defined(_NCCL)
-     module procedure decomp_2d_abort_nccl_basic
-     module procedure decomp_2d_abort_nccl_file_line
+      module procedure decomp_2d_abort_nccl_basic
+      module procedure decomp_2d_abort_nccl_file_line
 #endif
-  end interface decomp_2d_abort
+   end interface decomp_2d_abort
 
-  interface decomp_2d_warning
-     module procedure decomp_2d_warning_basic
-     module procedure decomp_2d_warning_file_line
-  end interface decomp_2d_warning
+   interface decomp_2d_warning
+      module procedure decomp_2d_warning_basic
+      module procedure decomp_2d_warning_file_line
+   end interface decomp_2d_warning
 
-  interface
+   interface
 
-     module subroutine d2d_listing(given_io_unit)
-        integer, intent(in), optional :: given_io_unit
-     end subroutine d2d_listing
+      module subroutine d2d_listing(given_io_unit)
+         integer, intent(in), optional :: given_io_unit
+      end subroutine d2d_listing
 
-     module subroutine decomp_info_print(d2d, io_unit, d2dname)
-        type(decomp_info), intent(in) :: d2d
-        integer, intent(in) :: io_unit
-        character(len=*), intent(in) :: d2dname
-     end subroutine decomp_info_print
+      module subroutine decomp_info_print(d2d, io_unit, d2dname)
+         type(decomp_info), intent(in) :: d2d
+         integer, intent(in) :: io_unit
+         character(len=*), intent(in) :: d2dname
+      end subroutine decomp_info_print
 
-  end interface
+   end interface
 
    ! Generic interface to initialize the profiler
    interface decomp_profiler_init
@@ -412,1228 +412,1220 @@ end subroutine decomp_profiler_end_char
    end interface decomp_profiler_end
 
 contains
-        
+
 #include "decomp_2d_init_fin.f90"
 
-  !
-  ! Small wrapper to free a MPI communicator
-  !
-  subroutine decomp_mpi_comm_free(mpi_comm)
+   !
+   ! Small wrapper to free a MPI communicator
+   !
+   subroutine decomp_mpi_comm_free(mpi_comm)
 
-    implicit none
+      implicit none
 
-    integer, intent(inout) :: mpi_comm
-    integer :: ierror
+      integer, intent(inout) :: mpi_comm
+      integer :: ierror
 
-    ! Return if no MPI comm to free
-    if (mpi_comm == MPI_COMM_NULL) return
+      ! Return if no MPI comm to free
+      if (mpi_comm == MPI_COMM_NULL) return
 
-    ! Free the provided MPI communicator
-    call MPI_COMM_FREE(mpi_comm, ierror)
-    if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
-    mpi_comm = MPI_COMM_NULL
+      ! Free the provided MPI communicator
+      call MPI_COMM_FREE(mpi_comm, ierror)
+      if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
+      mpi_comm = MPI_COMM_NULL
 
-  end subroutine decomp_mpi_comm_free
+   end subroutine decomp_mpi_comm_free
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Return the default decomposition object
+   ! Return the default decomposition object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  ! FIXME avoid a copy and return a pointer to decomp_main
-  ! TODO list the external codes using this subroutine
-  subroutine get_decomp_info(decomp)
+   ! FIXME avoid a copy and return a pointer to decomp_main
+   ! TODO list the external codes using this subroutine
+   subroutine get_decomp_info(decomp)
 
-    implicit none
+      implicit none
 
-    ! FIXME TYPE(DECOMP_INFO), pointer :: decomp
-    TYPE(DECOMP_INFO), intent(OUT) :: decomp
+      ! FIXME TYPE(DECOMP_INFO), pointer :: decomp
+      TYPE(DECOMP_INFO), intent(OUT) :: decomp
 
-    ! FIXME decomp => decomp_main
-    decomp = decomp_main
+      ! FIXME decomp => decomp_main
+      decomp = decomp_main
 
-    return
-  end subroutine get_decomp_info
+      return
+   end subroutine get_decomp_info
 
-  !
-  ! Return the 2D processor grid
-  !
-  function get_decomp_dims()
+   !
+   ! Return the 2D processor grid
+   !
+   function get_decomp_dims()
 
-    implicit none
+      implicit none
 
-    integer, dimension(2) :: get_decomp_dims
+      integer, dimension(2) :: get_decomp_dims
 
-    get_decomp_dims = dims
+      get_decomp_dims = dims
 
-  end function get_decomp_dims
+   end function get_decomp_dims
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Advanced Interface allowing applications to define globle domain of
-  ! any size, distribute it, and then transpose data among pencils.
-  !  - generate 2D decomposition details as defined in DECOMP_INFO
-  !  - the default global data size is nx*ny*nz
-  !  - a different global size nx/2+1,ny,nz is used in FFT r2c/c2r
-  !  - multiple global sizes can co-exist in one application, each
-  !    using its own DECOMP_INFO object
+   ! Advanced Interface allowing applications to define globle domain of
+   ! any size, distribute it, and then transpose data among pencils.
+   !  - generate 2D decomposition details as defined in DECOMP_INFO
+   !  - the default global data size is nx*ny*nz
+   !  - a different global size nx/2+1,ny,nz is used in FFT r2c/c2r
+   !  - multiple global sizes can co-exist in one application, each
+   !    using its own DECOMP_INFO object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_info_init(nx,ny,nz,decomp)
-
-    implicit none
-
-    integer, intent(IN) :: nx,ny,nz
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-
-    integer :: buf_size, status, errorcode
-
-    ! verify the global size can actually be distributed as pencils
-    if (nx_global<dims(1) .or. ny_global<dims(1) .or. ny_global<dims(2) .or. nz_global<dims(2)) then
-       errorcode = 6
-       call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-            'Invalid 2D processor grid. ' // &
-            'Make sure that min(nx,ny) >= p_row and ' // &
-            'min(ny,nz) >= p_col')
-    end if
-
-    if (mod(nx,dims(1))==0 .and. mod(ny,dims(1))==0 .and. &
-         mod(ny,dims(2))==0 .and. mod(nz,dims(2))==0) then
-       decomp%even = .true.
-    else
-       decomp%even = .false.
-    end if
-
-    ! distribute mesh points
-    allocate(decomp%x1dist(0:dims(1)-1),decomp%y1dist(0:dims(1)-1), &
-         decomp%y2dist(0:dims(2)-1),decomp%z2dist(0:dims(2)-1))
-    call get_dist(nx,ny,nz,decomp)
-
-    ! generate partition information - starting/ending index etc.
-    call partition(nx, ny, nz, (/ 1,2,3 /), &
-         decomp%xst, decomp%xen, decomp%xsz)
-    call partition(nx, ny, nz, (/ 2,1,3 /), &
-         decomp%yst, decomp%yen, decomp%ysz)
-    call partition(nx, ny, nz, (/ 2,3,1 /), &
-         decomp%zst, decomp%zen, decomp%zsz)
-
-    ! prepare send/receive buffer displacement and count for ALLTOALL(V)
-    allocate(decomp%x1cnts(0:dims(1)-1),decomp%y1cnts(0:dims(1)-1), &
-         decomp%y2cnts(0:dims(2)-1),decomp%z2cnts(0:dims(2)-1))
-    allocate(decomp%x1disp(0:dims(1)-1),decomp%y1disp(0:dims(1)-1), &
-         decomp%y2disp(0:dims(2)-1),decomp%z2disp(0:dims(2)-1))
-    call prepare_buffer(decomp)
+   subroutine decomp_info_init(nx, ny, nz, decomp)
+
+      implicit none
+
+      integer, intent(IN) :: nx, ny, nz
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+
+      integer :: buf_size, status, errorcode
+
+      ! verify the global size can actually be distributed as pencils
+      if (nx_global < dims(1) .or. ny_global < dims(1) .or. ny_global < dims(2) .or. nz_global < dims(2)) then
+         errorcode = 6
+         call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                              'Invalid 2D processor grid. '// &
+                              'Make sure that min(nx,ny) >= p_row and '// &
+                              'min(ny,nz) >= p_col')
+      end if
+
+      if (mod(nx, dims(1)) == 0 .and. mod(ny, dims(1)) == 0 .and. &
+          mod(ny, dims(2)) == 0 .and. mod(nz, dims(2)) == 0) then
+         decomp%even = .true.
+      else
+         decomp%even = .false.
+      end if
+
+      ! distribute mesh points
+      allocate (decomp%x1dist(0:dims(1) - 1), decomp%y1dist(0:dims(1) - 1), &
+                decomp%y2dist(0:dims(2) - 1), decomp%z2dist(0:dims(2) - 1))
+      call get_dist(nx, ny, nz, decomp)
+
+      ! generate partition information - starting/ending index etc.
+      call partition(nx, ny, nz, (/1, 2, 3/), &
+                     decomp%xst, decomp%xen, decomp%xsz)
+      call partition(nx, ny, nz, (/2, 1, 3/), &
+                     decomp%yst, decomp%yen, decomp%ysz)
+      call partition(nx, ny, nz, (/2, 3, 1/), &
+                     decomp%zst, decomp%zen, decomp%zsz)
+
+      ! prepare send/receive buffer displacement and count for ALLTOALL(V)
+      allocate (decomp%x1cnts(0:dims(1) - 1), decomp%y1cnts(0:dims(1) - 1), &
+                decomp%y2cnts(0:dims(2) - 1), decomp%z2cnts(0:dims(2) - 1))
+      allocate (decomp%x1disp(0:dims(1) - 1), decomp%y1disp(0:dims(1) - 1), &
+                decomp%y2disp(0:dims(2) - 1), decomp%z2disp(0:dims(2) - 1))
+      call prepare_buffer(decomp)
 
 #ifdef SHM
-    ! prepare shared-memory information if required
-    call decomp_info_init_shm(decomp)
+      ! prepare shared-memory information if required
+      call decomp_info_init_shm(decomp)
 #endif
 
-    ! allocate memory for the MPI_ALLTOALL(V) buffers
-    ! define the buffers globally for performance reason
+      ! allocate memory for the MPI_ALLTOALL(V) buffers
+      ! define the buffers globally for performance reason
 
-    buf_size = max(decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3), &
-         max(decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3), &
-         decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)) )
+      buf_size = max(decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3), &
+                     max(decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3), &
+                         decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)))
 #ifdef EVEN
-    ! padded alltoall optimisation may need larger buffer space
-    buf_size = max(buf_size, &
-         max(decomp%x1count*dims(1),decomp%y2count*dims(2)) ) 
+      ! padded alltoall optimisation may need larger buffer space
+      buf_size = max(buf_size, &
+                     max(decomp%x1count*dims(1), decomp%y2count*dims(2)))
 #endif
 
-    ! check if additional memory is required
-    ! *** TODO: consider how to share the real/complex buffers 
-    if (buf_size > decomp_buf_size) then
-       decomp_buf_size = buf_size
+      ! check if additional memory is required
+      ! *** TODO: consider how to share the real/complex buffers
+      if (buf_size > decomp_buf_size) then
+         decomp_buf_size = buf_size
 #if defined(_GPU)
-       if (allocated(work1_r_d)) deallocate(work1_r_d)
-       if (allocated(work2_r_d)) deallocate(work2_r_d)
-       if (allocated(work1_c_d)) deallocate(work1_c_d)
-       if (allocated(work2_c_d)) deallocate(work2_c_d)
-       allocate(work1_r_d(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work1_c_d(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work2_r_d(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work2_c_d(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
+         if (allocated(work1_r_d)) deallocate (work1_r_d)
+         if (allocated(work2_r_d)) deallocate (work2_r_d)
+         if (allocated(work1_c_d)) deallocate (work1_c_d)
+         if (allocated(work2_c_d)) deallocate (work2_c_d)
+         allocate (work1_r_d(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work1_c_d(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work2_r_d(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work2_c_d(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
 #endif
-       if (allocated(work1_r)) deallocate(work1_r)
-       if (allocated(work2_r)) deallocate(work2_r)
-       if (allocated(work1_c)) deallocate(work1_c)
-       if (allocated(work2_c)) deallocate(work2_c)
-       allocate(work1_r(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work2_r(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work1_c(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-       allocate(work2_c(buf_size), STAT=status)
-       if (status /= 0) then
-          errorcode = 2
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Out of memory when allocating 2DECOMP workspace')
-       end if
-    end if
-
-    return
-  end subroutine decomp_info_init
-
+         if (allocated(work1_r)) deallocate (work1_r)
+         if (allocated(work2_r)) deallocate (work2_r)
+         if (allocated(work1_c)) deallocate (work1_c)
+         if (allocated(work2_c)) deallocate (work2_c)
+         allocate (work1_r(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work2_r(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work1_c(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+         allocate (work2_c(buf_size), STAT=status)
+         if (status /= 0) then
+            errorcode = 2
+            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                 'Out of memory when allocating 2DECOMP workspace')
+         end if
+      end if
+
+      return
+   end subroutine decomp_info_init
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Release memory associated with a DECOMP_INFO object
+   ! Release memory associated with a DECOMP_INFO object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_info_finalize(decomp)
+   subroutine decomp_info_finalize(decomp)
 
-    implicit none
+      implicit none
 
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
 
-    if (allocated(decomp%x1dist)) deallocate(decomp%x1dist)
-    if (allocated(decomp%y1dist)) deallocate(decomp%y1dist)
-    if (allocated(decomp%y2dist)) deallocate(decomp%y2dist)
-    if (allocated(decomp%z2dist)) deallocate(decomp%z2dist)
-    if (allocated(decomp%x1cnts)) deallocate(decomp%x1cnts)
-    if (allocated(decomp%y1cnts)) deallocate(decomp%y1cnts)
-    if (allocated(decomp%y2cnts)) deallocate(decomp%y2cnts)
-    if (allocated(decomp%z2cnts)) deallocate(decomp%z2cnts)
-    if (allocated(decomp%x1disp)) deallocate(decomp%x1disp)
-    if (allocated(decomp%y1disp)) deallocate(decomp%y1disp)
-    if (allocated(decomp%y2disp)) deallocate(decomp%y2disp)
-    if (allocated(decomp%z2disp)) deallocate(decomp%z2disp)
+      if (allocated(decomp%x1dist)) deallocate (decomp%x1dist)
+      if (allocated(decomp%y1dist)) deallocate (decomp%y1dist)
+      if (allocated(decomp%y2dist)) deallocate (decomp%y2dist)
+      if (allocated(decomp%z2dist)) deallocate (decomp%z2dist)
+      if (allocated(decomp%x1cnts)) deallocate (decomp%x1cnts)
+      if (allocated(decomp%y1cnts)) deallocate (decomp%y1cnts)
+      if (allocated(decomp%y2cnts)) deallocate (decomp%y2cnts)
+      if (allocated(decomp%z2cnts)) deallocate (decomp%z2cnts)
+      if (allocated(decomp%x1disp)) deallocate (decomp%x1disp)
+      if (allocated(decomp%y1disp)) deallocate (decomp%y1disp)
+      if (allocated(decomp%y2disp)) deallocate (decomp%y2disp)
+      if (allocated(decomp%z2disp)) deallocate (decomp%z2disp)
 
 #ifdef SHM
-    if (allocated(decomp%x1disp_o)) deallocate(decomp%x1disp_o)
-    if (allocated(decomp%y1disp_o)) deallocate(decomp%y1disp_o)
-    if (allocated(decomp%y2disp_o)) deallocate(decomp%y2disp_o)
-    if (allocated(decomp%z2disp_o)) deallocate(decomp%z2disp_o)
-    if (allocated(decomp%x1cnts_s)) deallocate(decomp%x1cnts_s)
-    if (allocated(decomp%y1cnts_s)) deallocate(decomp%y1cnts_s)
-    if (allocated(decomp%y2cnts_s)) deallocate(decomp%y2cnts_s)
-    if (allocated(decomp%z2cnts_s)) deallocate(decomp%z2cnts_s)
-    if (allocated(decomp%x1disp_s)) deallocate(decomp%x1disp_s)
-    if (allocated(decomp%y1disp_s)) deallocate(decomp%y1disp_s)
-    if (allocated(decomp%y2disp_s)) deallocate(decomp%y2disp_s)
-    if (allocated(decomp%z2disp_s)) deallocate(decomp%z2disp_s)
+      if (allocated(decomp%x1disp_o)) deallocate (decomp%x1disp_o)
+      if (allocated(decomp%y1disp_o)) deallocate (decomp%y1disp_o)
+      if (allocated(decomp%y2disp_o)) deallocate (decomp%y2disp_o)
+      if (allocated(decomp%z2disp_o)) deallocate (decomp%z2disp_o)
+      if (allocated(decomp%x1cnts_s)) deallocate (decomp%x1cnts_s)
+      if (allocated(decomp%y1cnts_s)) deallocate (decomp%y1cnts_s)
+      if (allocated(decomp%y2cnts_s)) deallocate (decomp%y2cnts_s)
+      if (allocated(decomp%z2cnts_s)) deallocate (decomp%z2cnts_s)
+      if (allocated(decomp%x1disp_s)) deallocate (decomp%x1disp_s)
+      if (allocated(decomp%y1disp_s)) deallocate (decomp%y1disp_s)
+      if (allocated(decomp%y2disp_s)) deallocate (decomp%y2disp_s)
+      if (allocated(decomp%z2disp_s)) deallocate (decomp%z2disp_s)
 #endif
 
-    return
-  end subroutine decomp_info_finalize
-
+      return
+   end subroutine decomp_info_finalize
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Coarser mesh support for statistic
+   ! Coarser mesh support for statistic
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_coarser_mesh_statS(i_skip,j_skip,k_skip,from1)
-
-    implicit none
-
-    integer, intent(IN) :: i_skip,j_skip,k_skip
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
-
-    integer, dimension(3) :: skip
-    integer :: i
-
-    coarse_mesh_starts_from_1 = from1
-    iskipS = i_skip
-    jskipS = j_skip
-    kskipS = k_skip
-
-    skip(1)=iskipS
-    skip(2)=jskipS
-    skip(3)=kskipS
-
-    do i=1,3
-       if (from1) then
-          xstS(i) = (xstart(i)+skip(i)-1)/skip(i)
-          if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xstS(i)=xstS(i)+1
-          xenS(i) = (xend(i)+skip(i)-1)/skip(i)
-       else
-          xstS(i) = xstart(i)/skip(i)
-          if (mod(xstart(i),skip(i))/=0) xstS(i)=xstS(i)+1
-          xenS(i) = xend(i)/skip(i)
-       end if
-       xszS(i) = xenS(i)-xstS(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          ystS(i) = (ystart(i)+skip(i)-1)/skip(i)
-          if (mod(ystart(i)+skip(i)-1,skip(i))/=0) ystS(i)=ystS(i)+1
-          yenS(i) = (yend(i)+skip(i)-1)/skip(i)
-       else
-          ystS(i) = ystart(i)/skip(i)
-          if (mod(ystart(i),skip(i))/=0) ystS(i)=ystS(i)+1
-          yenS(i) = yend(i)/skip(i)
-       end if
-       yszS(i) = yenS(i)-ystS(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          zstS(i) = (zstart(i)+skip(i)-1)/skip(i)
-          if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zstS(i)=zstS(i)+1
-          zenS(i) = (zend(i)+skip(i)-1)/skip(i)
-       else
-          zstS(i) = zstart(i)/skip(i)
-          if (mod(zstart(i),skip(i))/=0) zstS(i)=zstS(i)+1
-          zenS(i) = zend(i)/skip(i)
-       end if
-       zszS(i) = zenS(i)-zstS(i)+1
-    end do
-
-    return
-  end subroutine init_coarser_mesh_statS
+   subroutine init_coarser_mesh_statS(i_skip, j_skip, k_skip, from1)
+
+      implicit none
+
+      integer, intent(IN) :: i_skip, j_skip, k_skip
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
+
+      integer, dimension(3) :: skip
+      integer :: i
+
+      coarse_mesh_starts_from_1 = from1
+      iskipS = i_skip
+      jskipS = j_skip
+      kskipS = k_skip
+
+      skip(1) = iskipS
+      skip(2) = jskipS
+      skip(3) = kskipS
+
+      do i = 1, 3
+         if (from1) then
+            xstS(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstS(i) = xstS(i) + 1
+            xenS(i) = (xend(i) + skip(i) - 1)/skip(i)
+         else
+            xstS(i) = xstart(i)/skip(i)
+            if (mod(xstart(i), skip(i)) /= 0) xstS(i) = xstS(i) + 1
+            xenS(i) = xend(i)/skip(i)
+         end if
+         xszS(i) = xenS(i) - xstS(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            ystS(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystS(i) = ystS(i) + 1
+            yenS(i) = (yend(i) + skip(i) - 1)/skip(i)
+         else
+            ystS(i) = ystart(i)/skip(i)
+            if (mod(ystart(i), skip(i)) /= 0) ystS(i) = ystS(i) + 1
+            yenS(i) = yend(i)/skip(i)
+         end if
+         yszS(i) = yenS(i) - ystS(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            zstS(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstS(i) = zstS(i) + 1
+            zenS(i) = (zend(i) + skip(i) - 1)/skip(i)
+         else
+            zstS(i) = zstart(i)/skip(i)
+            if (mod(zstart(i), skip(i)) /= 0) zstS(i) = zstS(i) + 1
+            zenS(i) = zend(i)/skip(i)
+         end if
+         zszS(i) = zenS(i) - zstS(i) + 1
+      end do
+
+      return
+   end subroutine init_coarser_mesh_statS
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Coarser mesh support for visualization
+   ! Coarser mesh support for visualization
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_coarser_mesh_statV(i_skip,j_skip,k_skip,from1)
-
-    implicit none
-
-    integer, intent(IN) :: i_skip,j_skip,k_skip
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
-
-    integer, dimension(3) :: skip
-    integer :: i
-
-    coarse_mesh_starts_from_1 = from1
-    iskipV = i_skip
-    jskipV = j_skip
-    kskipV = k_skip
-
-    skip(1)=iskipV
-    skip(2)=jskipV
-    skip(3)=kskipV
-
-    do i=1,3
-       if (from1) then
-          xstV(i) = (xstart(i)+skip(i)-1)/skip(i)
-          if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xstV(i)=xstV(i)+1
-          xenV(i) = (xend(i)+skip(i)-1)/skip(i)
-       else
-          xstV(i) = xstart(i)/skip(i)
-          if (mod(xstart(i),skip(i))/=0) xstV(i)=xstV(i)+1
-          xenV(i) = xend(i)/skip(i)
-       end if
-       xszV(i) = xenV(i)-xstV(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          ystV(i) = (ystart(i)+skip(i)-1)/skip(i)
-          if (mod(ystart(i)+skip(i)-1,skip(i))/=0) ystV(i)=ystV(i)+1
-          yenV(i) = (yend(i)+skip(i)-1)/skip(i)
-       else
-          ystV(i) = ystart(i)/skip(i)
-          if (mod(ystart(i),skip(i))/=0) ystV(i)=ystV(i)+1
-          yenV(i) = yend(i)/skip(i)
-       end if
-       yszV(i) = yenV(i)-ystV(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          zstV(i) = (zstart(i)+skip(i)-1)/skip(i)
-          if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zstV(i)=zstV(i)+1
-          zenV(i) = (zend(i)+skip(i)-1)/skip(i)
-       else
-          zstV(i) = zstart(i)/skip(i)
-          if (mod(zstart(i),skip(i))/=0) zstV(i)=zstV(i)+1
-          zenV(i) = zend(i)/skip(i)
-       end if
-       zszV(i) = zenV(i)-zstV(i)+1
-    end do
-
-    return
-  end subroutine init_coarser_mesh_statV
+   subroutine init_coarser_mesh_statV(i_skip, j_skip, k_skip, from1)
+
+      implicit none
+
+      integer, intent(IN) :: i_skip, j_skip, k_skip
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
+
+      integer, dimension(3) :: skip
+      integer :: i
+
+      coarse_mesh_starts_from_1 = from1
+      iskipV = i_skip
+      jskipV = j_skip
+      kskipV = k_skip
+
+      skip(1) = iskipV
+      skip(2) = jskipV
+      skip(3) = kskipV
+
+      do i = 1, 3
+         if (from1) then
+            xstV(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstV(i) = xstV(i) + 1
+            xenV(i) = (xend(i) + skip(i) - 1)/skip(i)
+         else
+            xstV(i) = xstart(i)/skip(i)
+            if (mod(xstart(i), skip(i)) /= 0) xstV(i) = xstV(i) + 1
+            xenV(i) = xend(i)/skip(i)
+         end if
+         xszV(i) = xenV(i) - xstV(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            ystV(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystV(i) = ystV(i) + 1
+            yenV(i) = (yend(i) + skip(i) - 1)/skip(i)
+         else
+            ystV(i) = ystart(i)/skip(i)
+            if (mod(ystart(i), skip(i)) /= 0) ystV(i) = ystV(i) + 1
+            yenV(i) = yend(i)/skip(i)
+         end if
+         yszV(i) = yenV(i) - ystV(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            zstV(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstV(i) = zstV(i) + 1
+            zenV(i) = (zend(i) + skip(i) - 1)/skip(i)
+         else
+            zstV(i) = zstart(i)/skip(i)
+            if (mod(zstart(i), skip(i)) /= 0) zstV(i) = zstV(i) + 1
+            zenV(i) = zend(i)/skip(i)
+         end if
+         zszV(i) = zenV(i) - zstV(i) + 1
+      end do
+
+      return
+   end subroutine init_coarser_mesh_statV
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Coarser mesh support for probe
+   ! Coarser mesh support for probe
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_coarser_mesh_statP(i_skip,j_skip,k_skip,from1)
-
-    implicit none
-
-    integer, intent(IN) :: i_skip,j_skip,k_skip
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
-
-    integer, dimension(3) :: skip
-    integer :: i
-
-    coarse_mesh_starts_from_1 = from1
-    iskipP = i_skip
-    jskipP = j_skip
-    kskipP = k_skip
-
-    skip(1)=iskipP
-    skip(2)=jskipP
-    skip(3)=kskipP
-
-    do i=1,3
-       if (from1) then
-          xstP(i) = (xstart(i)+skip(i)-1)/skip(i)
-          if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xstP(i)=xstP(i)+1
-          xenP(i) = (xend(i)+skip(i)-1)/skip(i)
-       else
-          xstP(i) = xstart(i)/skip(i)
-          if (mod(xstart(i),skip(i))/=0) xstP(i)=xstP(i)+1
-          xenP(i) = xend(i)/skip(i)
-       end if
-       xszP(i) = xenP(i)-xstP(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          ystP(i) = (ystart(i)+skip(i)-1)/skip(i)
-          if (mod(ystart(i)+skip(i)-1,skip(i))/=0) ystP(i)=ystP(i)+1
-          yenP(i) = (yend(i)+skip(i)-1)/skip(i)
-       else
-          ystP(i) = ystart(i)/skip(i)
-          if (mod(ystart(i),skip(i))/=0) ystP(i)=ystP(i)+1
-          yenP(i) = yend(i)/skip(i)
-       end if
-       yszP(i) = yenP(i)-ystP(i)+1
-    end do
-
-    do i=1,3
-       if (from1) then
-          zstP(i) = (zstart(i)+skip(i)-1)/skip(i)
-          if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zstP(i)=zstP(i)+1
-          zenP(i) = (zend(i)+skip(i)-1)/skip(i)
-       else
-          zstP(i) = zstart(i)/skip(i)
-          if (mod(zstart(i),skip(i))/=0) zstP(i)=zstP(i)+1
-          zenP(i) = zend(i)/skip(i)
-       end if
-       zszP(i) = zenP(i)-zstP(i)+1
-    end do
-
-    return
-  end subroutine init_coarser_mesh_statP
-
-  ! Copy data from a fine-resolution array to a coarse one for statistic
-  subroutine fine_to_coarseS(ipencil,var_fine,var_coarse)
-
-    implicit none
-
-    real(mytype), dimension(:,:,:) :: var_fine
-    real(mytype), dimension(:,:,:) :: var_coarse
-    integer, intent(IN) :: ipencil
-
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer :: i,j,k
-
-    if (ipencil==1) then
-       allocate(wk(xstS(1):xenS(1),xstS(2):xenS(2),xstS(3):xenS(3)))
-       allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=xstS(3),xenS(3)
-             do j=xstS(2),xenS(2)
-                do i=xstS(1),xenS(1)
-                   wk(i,j,k) = wk2((i-1)*iskipS+1,(j-1)*jskipS+1,(k-1)*kskipS+1)
-                end do
-             end do
-          end do
-       else
-          do k=xstS(3),xenS(3)
-             do j=xstS(2),xenS(2)
-                do i=xstS(1),xenS(1)
-                   wk(i,j,k) = wk2(i*iskipS,j*jskipS,k*kskipS)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==2) then
-       allocate(wk(ystS(1):yenS(1),ystS(2):yenS(2),ystS(3):yenS(3)))
-       allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=ystS(3),yenS(3)
-             do j=ystS(2),yenS(2)
-                do i=ystS(1),yenS(1)
-                   wk(i,j,k) = wk2((i-1)*iskipS+1,(j-1)*jskipS+1,(k-1)*kskipS+1)
-                end do
-             end do
-          end do
-       else
-          do k=ystS(3),yenS(3)
-             do j=ystS(2),yenS(2)
-                do i=ystS(1),yenS(1)
-                   wk(i,j,k) = wk2(i*iskipS,j*jskipS,k*kskipS)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==3) then
-       allocate(wk(zstS(1):zenS(1),zstS(2):zenS(2),zstS(3):zenS(3)))
-       allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=zstS(3),zenS(3)
-             do j=zstS(2),zenS(2)
-                do i=zstS(1),zenS(1)
-                   wk(i,j,k) = wk2((i-1)*iskipS+1,(j-1)*jskipS+1,(k-1)*kskipS+1)
-                end do
-             end do
-          end do
-       else
-          do k=zstS(3),zenS(3)
-             do j=zstS(2),zenS(2)
-                do i=zstS(1),zenS(1)
-                   wk(i,j,k) = wk2(i*iskipS,j*jskipS,k*kskipS)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    end if
-
-    deallocate(wk,wk2)
-
-    return
-  end subroutine fine_to_coarseS
-
-  ! Copy data from a fine-resolution array to a coarse one for visualization
-  subroutine fine_to_coarseV(ipencil,var_fine,var_coarse)
-
-    implicit none
-
-    real(mytype), dimension(:,:,:) :: var_fine
-    real(mytype), dimension(:,:,:) :: var_coarse
-    integer, intent(IN) :: ipencil
-
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer :: i,j,k
-
-    if (ipencil==1) then
-       allocate(wk(xstV(1):xenV(1),xstV(2):xenV(2),xstV(3):xenV(3)))
-       allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=xstV(3),xenV(3)
-             do j=xstV(2),xenV(2)
-                do i=xstV(1),xenV(1)
-                   wk(i,j,k) = wk2((i-1)*iskipV+1,(j-1)*jskipV+1,(k-1)*kskipV+1)
-                end do
-             end do
-          end do
-       else
-          do k=xstV(3),xenV(3)
-             do j=xstV(2),xenV(2)
-                do i=xstV(1),xenV(1)
-                   wk(i,j,k) = wk2(i*iskipV,j*jskipV,k*kskipV)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==2) then
-       allocate(wk(ystV(1):yenV(1),ystV(2):yenV(2),ystV(3):yenV(3)))
-       allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=ystV(3),yenV(3)
-             do j=ystV(2),yenV(2)
-                do i=ystV(1),yenV(1)
-                   wk(i,j,k) = wk2((i-1)*iskipV+1,(j-1)*jskipV+1,(k-1)*kskipV+1)
-                end do
-             end do
-          end do
-       else
-          do k=ystV(3),yenV(3)
-             do j=ystV(2),yenV(2)
-                do i=ystV(1),yenV(1)
-                   wk(i,j,k) = wk2(i*iskipV,j*jskipV,k*kskipV)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==3) then
-       allocate(wk(zstV(1):zenV(1),zstV(2):zenV(2),zstV(3):zenV(3)))
-       allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=zstV(3),zenV(3)
-             do j=zstV(2),zenV(2)
-                do i=zstV(1),zenV(1)
-                   wk(i,j,k) = wk2((i-1)*iskipV+1,(j-1)*jskipV+1,(k-1)*kskipV+1)
-                end do
-             end do
-          end do
-       else
-          do k=zstV(3),zenV(3)
-             do j=zstV(2),zenV(2)
-                do i=zstV(1),zenV(1)
-                   wk(i,j,k) = wk2(i*iskipV,j*jskipV,k*kskipV)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    end if
-
-    deallocate(wk,wk2)
-
-    return
-  end subroutine fine_to_coarseV
-
-  ! Copy data from a fine-resolution array to a coarse one for probe
-  subroutine fine_to_coarseP(ipencil,var_fine,var_coarse)
-
-    implicit none
-
-    real(mytype), dimension(:,:,:) :: var_fine
-    real(mytype), dimension(:,:,:) :: var_coarse
-    integer, intent(IN) :: ipencil
-
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer :: i,j,k
-
-    if (ipencil==1) then
-       allocate(wk(xstP(1):xenP(1),xstP(2):xenP(2),xstP(3):xenP(3)))
-       allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=xstP(3),xenP(3)
-             do j=xstP(2),xenP(2)
-                do i=xstP(1),xenP(1)
-                   wk(i,j,k) = wk2((i-1)*iskipP+1,(j-1)*jskipP+1,(k-1)*kskipP+1)
-                end do
-             end do
-          end do
-       else
-          do k=xstP(3),xenP(3)
-             do j=xstP(2),xenP(2)
-                do i=xstP(1),xenP(1)
-                   wk(i,j,k) = wk2(i*iskipP,j*jskipP,k*kskipP)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==2) then
-       allocate(wk(ystP(1):yenP(1),ystP(2):yenP(2),ystP(3):yenP(3)))
-       allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=ystP(3),yenP(3)
-             do j=ystP(2),yenP(2)
-                do i=ystP(1),yenP(1)
-                   wk(i,j,k) = wk2((i-1)*iskipP+1,(j-1)*jskipP+1,(k-1)*kskipP+1)
-                end do
-             end do
-          end do
-       else
-          do k=ystP(3),yenP(3)
-             do j=ystP(2),yenP(2)
-                do i=ystP(1),yenP(1)
-                   wk(i,j,k) = wk2(i*iskipP,j*jskipP,k*kskipP)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    else if (ipencil==3) then
-       allocate(wk(zstP(1):zenP(1),zstP(2):zenP(2),zstP(3):zenP(3)))
-       allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-       wk2=var_fine
-       if (coarse_mesh_starts_from_1) then
-          do k=zstP(3),zenP(3)
-             do j=zstP(2),zenP(2)
-                do i=zstP(1),zenP(1)
-                   wk(i,j,k) = wk2((i-1)*iskipP+1,(j-1)*jskipP+1,(k-1)*kskipP+1)
-                end do
-             end do
-          end do
-       else
-          do k=zstP(3),zenP(3)
-             do j=zstP(2),zenP(2)
-                do i=zstP(1),zenP(1)
-                   wk(i,j,k) = wk2(i*iskipP,j*jskipP,k*kskipP)
-                end do
-             end do
-          end do
-       end if
-       var_coarse=wk
-    end if
-
-    deallocate(wk,wk2)
-
-    return
-  end subroutine fine_to_coarseP
-
+   subroutine init_coarser_mesh_statP(i_skip, j_skip, k_skip, from1)
+
+      implicit none
+
+      integer, intent(IN) :: i_skip, j_skip, k_skip
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
+
+      integer, dimension(3) :: skip
+      integer :: i
+
+      coarse_mesh_starts_from_1 = from1
+      iskipP = i_skip
+      jskipP = j_skip
+      kskipP = k_skip
+
+      skip(1) = iskipP
+      skip(2) = jskipP
+      skip(3) = kskipP
+
+      do i = 1, 3
+         if (from1) then
+            xstP(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstP(i) = xstP(i) + 1
+            xenP(i) = (xend(i) + skip(i) - 1)/skip(i)
+         else
+            xstP(i) = xstart(i)/skip(i)
+            if (mod(xstart(i), skip(i)) /= 0) xstP(i) = xstP(i) + 1
+            xenP(i) = xend(i)/skip(i)
+         end if
+         xszP(i) = xenP(i) - xstP(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            ystP(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystP(i) = ystP(i) + 1
+            yenP(i) = (yend(i) + skip(i) - 1)/skip(i)
+         else
+            ystP(i) = ystart(i)/skip(i)
+            if (mod(ystart(i), skip(i)) /= 0) ystP(i) = ystP(i) + 1
+            yenP(i) = yend(i)/skip(i)
+         end if
+         yszP(i) = yenP(i) - ystP(i) + 1
+      end do
+
+      do i = 1, 3
+         if (from1) then
+            zstP(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstP(i) = zstP(i) + 1
+            zenP(i) = (zend(i) + skip(i) - 1)/skip(i)
+         else
+            zstP(i) = zstart(i)/skip(i)
+            if (mod(zstart(i), skip(i)) /= 0) zstP(i) = zstP(i) + 1
+            zenP(i) = zend(i)/skip(i)
+         end if
+         zszP(i) = zenP(i) - zstP(i) + 1
+      end do
+
+      return
+   end subroutine init_coarser_mesh_statP
+
+   ! Copy data from a fine-resolution array to a coarse one for statistic
+   subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
+
+      implicit none
+
+      real(mytype), dimension(:, :, :) :: var_fine
+      real(mytype), dimension(:, :, :) :: var_coarse
+      integer, intent(IN) :: ipencil
+
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer :: i, j, k
+
+      if (ipencil == 1) then
+         allocate (wk(xstS(1):xenS(1), xstS(2):xenS(2), xstS(3):xenS(3)))
+         allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = xstS(3), xenS(3)
+               do j = xstS(2), xenS(2)
+                  do i = xstS(1), xenS(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = xstS(3), xenS(3)
+               do j = xstS(2), xenS(2)
+                  do i = xstS(1), xenS(1)
+                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 2) then
+         allocate (wk(ystS(1):yenS(1), ystS(2):yenS(2), ystS(3):yenS(3)))
+         allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = ystS(3), yenS(3)
+               do j = ystS(2), yenS(2)
+                  do i = ystS(1), yenS(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = ystS(3), yenS(3)
+               do j = ystS(2), yenS(2)
+                  do i = ystS(1), yenS(1)
+                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 3) then
+         allocate (wk(zstS(1):zenS(1), zstS(2):zenS(2), zstS(3):zenS(3)))
+         allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = zstS(3), zenS(3)
+               do j = zstS(2), zenS(2)
+                  do i = zstS(1), zenS(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = zstS(3), zenS(3)
+               do j = zstS(2), zenS(2)
+                  do i = zstS(1), zenS(1)
+                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      end if
+
+      deallocate (wk, wk2)
+
+      return
+   end subroutine fine_to_coarseS
+
+   ! Copy data from a fine-resolution array to a coarse one for visualization
+   subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
+
+      implicit none
+
+      real(mytype), dimension(:, :, :) :: var_fine
+      real(mytype), dimension(:, :, :) :: var_coarse
+      integer, intent(IN) :: ipencil
+
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer :: i, j, k
+
+      if (ipencil == 1) then
+         allocate (wk(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
+         allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = xstV(3), xenV(3)
+               do j = xstV(2), xenV(2)
+                  do i = xstV(1), xenV(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = xstV(3), xenV(3)
+               do j = xstV(2), xenV(2)
+                  do i = xstV(1), xenV(1)
+                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 2) then
+         allocate (wk(ystV(1):yenV(1), ystV(2):yenV(2), ystV(3):yenV(3)))
+         allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = ystV(3), yenV(3)
+               do j = ystV(2), yenV(2)
+                  do i = ystV(1), yenV(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = ystV(3), yenV(3)
+               do j = ystV(2), yenV(2)
+                  do i = ystV(1), yenV(1)
+                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 3) then
+         allocate (wk(zstV(1):zenV(1), zstV(2):zenV(2), zstV(3):zenV(3)))
+         allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = zstV(3), zenV(3)
+               do j = zstV(2), zenV(2)
+                  do i = zstV(1), zenV(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = zstV(3), zenV(3)
+               do j = zstV(2), zenV(2)
+                  do i = zstV(1), zenV(1)
+                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      end if
+
+      deallocate (wk, wk2)
+
+      return
+   end subroutine fine_to_coarseV
+
+   ! Copy data from a fine-resolution array to a coarse one for probe
+   subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
+
+      implicit none
+
+      real(mytype), dimension(:, :, :) :: var_fine
+      real(mytype), dimension(:, :, :) :: var_coarse
+      integer, intent(IN) :: ipencil
+
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer :: i, j, k
+
+      if (ipencil == 1) then
+         allocate (wk(xstP(1):xenP(1), xstP(2):xenP(2), xstP(3):xenP(3)))
+         allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = xstP(3), xenP(3)
+               do j = xstP(2), xenP(2)
+                  do i = xstP(1), xenP(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = xstP(3), xenP(3)
+               do j = xstP(2), xenP(2)
+                  do i = xstP(1), xenP(1)
+                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 2) then
+         allocate (wk(ystP(1):yenP(1), ystP(2):yenP(2), ystP(3):yenP(3)))
+         allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = ystP(3), yenP(3)
+               do j = ystP(2), yenP(2)
+                  do i = ystP(1), yenP(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = ystP(3), yenP(3)
+               do j = ystP(2), yenP(2)
+                  do i = ystP(1), yenP(1)
+                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      else if (ipencil == 3) then
+         allocate (wk(zstP(1):zenP(1), zstP(2):zenP(2), zstP(3):zenP(3)))
+         allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+         wk2 = var_fine
+         if (coarse_mesh_starts_from_1) then
+            do k = zstP(3), zenP(3)
+               do j = zstP(2), zenP(2)
+                  do i = zstP(1), zenP(1)
+                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                  end do
+               end do
+            end do
+         else
+            do k = zstP(3), zenP(3)
+               do j = zstP(2), zenP(2)
+                  do i = zstP(1), zenP(1)
+                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                  end do
+               end do
+            end do
+         end if
+         var_coarse = wk
+      end if
+
+      deallocate (wk, wk2)
+
+      return
+   end subroutine fine_to_coarseP
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Find sub-domain information held by current processor
-  !   INPUT: 
-  !     nx, ny, nz - global data dimension
-  !     pdim(3)    - number of processor grid in each dimension, 
-  !                  valid values: 1 - distibute locally; 
-  !                                2 - distribute across p_row; 
-  !                                3 - distribute across p_col
-  !   OUTPUT:
-  !     lstart(3)  - starting index
-  !     lend(3)    - ending index
-  !     lsize(3)   - size of the sub-block (redundant) 
+   ! Find sub-domain information held by current processor
+   !   INPUT:
+   !     nx, ny, nz - global data dimension
+   !     pdim(3)    - number of processor grid in each dimension,
+   !                  valid values: 1 - distibute locally;
+   !                                2 - distribute across p_row;
+   !                                3 - distribute across p_col
+   !   OUTPUT:
+   !     lstart(3)  - starting index
+   !     lend(3)    - ending index
+   !     lsize(3)   - size of the sub-block (redundant)
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine partition(nx, ny, nz, pdim, lstart, lend, lsize)
-
-    implicit none
-
-    integer, intent(IN) :: nx, ny, nz
-    integer, dimension(3), intent(IN) :: pdim
-    integer, dimension(3), intent(OUT) :: lstart, lend, lsize
-
-    integer, allocatable, dimension(:) :: st,en,sz
-    integer :: i, gsize
-
-    do i = 1, 3
-
-       if (i==1) then
-          gsize = nx
-       else if (i==2) then
-          gsize = ny
-       else if (i==3) then
-          gsize = nz
-       end if
-
-       if (pdim(i) == 1) then        ! all local
-          lstart(i) = 1
-          lend(i)   = gsize
-          lsize(i)  = gsize
-       elseif (pdim(i) == 2) then    ! distribute across dims(1)
-          allocate(st(0:dims(1)-1))
-          allocate(en(0:dims(1)-1))
-          allocate(sz(0:dims(1)-1))
-          call distribute(gsize,dims(1),st,en,sz)
-          lstart(i) = st(coord(1))
-          lend(i)   = en(coord(1))
-          lsize(i)  = sz(coord(1))
-          deallocate(st,en,sz)
-       elseif (pdim(i) == 3) then    ! distribute across dims(2)
-          allocate(st(0:dims(2)-1))
-          allocate(en(0:dims(2)-1))
-          allocate(sz(0:dims(2)-1))
-          call distribute(gsize,dims(2),st,en,sz)
-          lstart(i) = st(coord(2))
-          lend(i)   = en(coord(2))
-          lsize(i)  = sz(coord(2))
-          deallocate(st,en,sz)
-       end if
-
-    end do
-    return   
-
-  end subroutine partition
+   subroutine partition(nx, ny, nz, pdim, lstart, lend, lsize)
+
+      implicit none
+
+      integer, intent(IN) :: nx, ny, nz
+      integer, dimension(3), intent(IN) :: pdim
+      integer, dimension(3), intent(OUT) :: lstart, lend, lsize
+
+      integer, allocatable, dimension(:) :: st, en, sz
+      integer :: i, gsize
+
+      do i = 1, 3
+
+         if (i == 1) then
+            gsize = nx
+         else if (i == 2) then
+            gsize = ny
+         else if (i == 3) then
+            gsize = nz
+         end if
+
+         if (pdim(i) == 1) then        ! all local
+            lstart(i) = 1
+            lend(i) = gsize
+            lsize(i) = gsize
+         elseif (pdim(i) == 2) then    ! distribute across dims(1)
+            allocate (st(0:dims(1) - 1))
+            allocate (en(0:dims(1) - 1))
+            allocate (sz(0:dims(1) - 1))
+            call distribute(gsize, dims(1), st, en, sz)
+            lstart(i) = st(coord(1))
+            lend(i) = en(coord(1))
+            lsize(i) = sz(coord(1))
+            deallocate (st, en, sz)
+         elseif (pdim(i) == 3) then    ! distribute across dims(2)
+            allocate (st(0:dims(2) - 1))
+            allocate (en(0:dims(2) - 1))
+            allocate (sz(0:dims(2) - 1))
+            call distribute(gsize, dims(2), st, en, sz)
+            lstart(i) = st(coord(2))
+            lend(i) = en(coord(2))
+            lsize(i) = sz(coord(2))
+            deallocate (st, en, sz)
+         end if
+
+      end do
+      return
+
+   end subroutine partition
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !   - distibutes grid points in one dimension
-  !   - handles uneven distribution properly 
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 
-  subroutine distribute(data1,proc,st,en,sz)
-
-    implicit none
-    ! data1 -- data size in any dimension to be partitioned
-    ! proc  -- number of processors in that dimension
-    ! st    -- array of starting index
-    ! en    -- array of ending index
-    ! sz    -- array of local size  (redundent)
-    integer data1,proc,st(0:proc-1),en(0:proc-1),sz(0:proc-1)
-    integer i,size1,nl,nu
-
-    size1=data1/proc
-    nu = data1 - size1 * proc
-    nl = proc - nu
-    st(0) = 1
-    sz(0) = size1
-    en(0) = size1
-    do i=1,nl-1
-       st(i) = st(i-1) + size1
-       sz(i) = size1
-       en(i) = en(i-1) + size1
-    end do
-    size1 = size1 + 1
-    do i=nl,proc-1
-       st(i) = en(i-1) + 1
-       sz(i) = size1
-       en(i) = en(i-1) + size1
-    end do
-    en(proc-1)= data1 
-    sz(proc-1)= data1-st(proc-1)+1
-
-    return
-  end subroutine distribute
+   !   - distibutes grid points in one dimension
+   !   - handles uneven distribution properly
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   subroutine distribute(data1, proc, st, en, sz)
+
+      implicit none
+      ! data1 -- data size in any dimension to be partitioned
+      ! proc  -- number of processors in that dimension
+      ! st    -- array of starting index
+      ! en    -- array of ending index
+      ! sz    -- array of local size  (redundent)
+      integer data1, proc, st(0:proc - 1), en(0:proc - 1), sz(0:proc - 1)
+      integer i, size1, nl, nu
+
+      size1 = data1/proc
+      nu = data1 - size1*proc
+      nl = proc - nu
+      st(0) = 1
+      sz(0) = size1
+      en(0) = size1
+      do i = 1, nl - 1
+         st(i) = st(i - 1) + size1
+         sz(i) = size1
+         en(i) = en(i - 1) + size1
+      end do
+      size1 = size1 + 1
+      do i = nl, proc - 1
+         st(i) = en(i - 1) + 1
+         sz(i) = size1
+         en(i) = en(i - 1) + size1
+      end do
+      en(proc - 1) = data1
+      sz(proc - 1) = data1 - st(proc - 1) + 1
+
+      return
+   end subroutine distribute
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  Define how each dimension is distributed across processors
-  !    e.g. 17 meshes across 4 processor would be distibuted as (4,4,4,5)
-  !    such global information is required locally at MPI_ALLTOALLV time
+   !  Define how each dimension is distributed across processors
+   !    e.g. 17 meshes across 4 processor would be distibuted as (4,4,4,5)
+   !    such global information is required locally at MPI_ALLTOALLV time
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine get_dist(nx,ny,nz,decomp)
+   subroutine get_dist(nx, ny, nz, decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: nx, ny, nz
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-    integer, allocatable, dimension(:) :: st,en
+      integer, intent(IN) :: nx, ny, nz
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+      integer, allocatable, dimension(:) :: st, en
 
-    allocate(st(0:dims(1)-1))
-    allocate(en(0:dims(1)-1))
-    call distribute(nx,dims(1),st,en,decomp%x1dist)
-    call distribute(ny,dims(1),st,en,decomp%y1dist)
-    deallocate(st,en)
+      allocate (st(0:dims(1) - 1))
+      allocate (en(0:dims(1) - 1))
+      call distribute(nx, dims(1), st, en, decomp%x1dist)
+      call distribute(ny, dims(1), st, en, decomp%y1dist)
+      deallocate (st, en)
 
-    allocate(st(0:dims(2)-1))
-    allocate(en(0:dims(2)-1))
-    call distribute(ny,dims(2),st,en,decomp%y2dist)
-    call distribute(nz,dims(2),st,en,decomp%z2dist)
-    deallocate(st,en)
+      allocate (st(0:dims(2) - 1))
+      allocate (en(0:dims(2) - 1))
+      call distribute(ny, dims(2), st, en, decomp%y2dist)
+      call distribute(nz, dims(2), st, en, decomp%z2dist)
+      deallocate (st, en)
 
-    return
-  end subroutine get_dist
+      return
+   end subroutine get_dist
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Prepare the send / receive buffers for MPI_ALLTOALLV communications
+   ! Prepare the send / receive buffers for MPI_ALLTOALLV communications
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine prepare_buffer(decomp)
-
-    implicit none
-
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-
-    integer :: i
-
-    !LG : AJOUTS "bidons" pour eviter un plantage en -O3 avec gcc9.3
-    !       * la fonction sortait des valeurs 'aleatoires'
-    !         et le calcul plantait dans MPI_ALLTOALLV
-    !       * pas de plantage en O2
-    
-    if (nrank==0) then
-       open(newunit=i, file='temp.dat', form='unformatted')
-       write(i) decomp%x1dist,decomp%y1dist,decomp%y2dist,decomp%z2dist, &
-                decomp%xsz,decomp%ysz,decomp%zsz
-       close(i, status='delete')
-    endif
-
-    ! MPI_ALLTOALLV buffer information
-
-    do i=0, dims(1)-1
-       decomp%x1cnts(i) = decomp%x1dist(i)*decomp%xsz(2)*decomp%xsz(3)
-       decomp%y1cnts(i) = decomp%ysz(1)*decomp%y1dist(i)*decomp%ysz(3)
-       if (i==0) then
-          decomp%x1disp(i) = 0  ! displacement is 0-based index
-          decomp%y1disp(i) = 0
-       else
-          decomp%x1disp(i) = decomp%x1disp(i-1) + decomp%x1cnts(i-1)
-          decomp%y1disp(i) = decomp%y1disp(i-1) + decomp%y1cnts(i-1)
-       end if
-    end do
-
-    do i=0, dims(2)-1
-       decomp%y2cnts(i) = decomp%ysz(1)*decomp%y2dist(i)*decomp%ysz(3)
-       decomp%z2cnts(i) = decomp%zsz(1)*decomp%zsz(2)*decomp%z2dist(i)
-       if (i==0) then
-          decomp%y2disp(i) = 0  ! displacement is 0-based index
-          decomp%z2disp(i) = 0
-       else
-          decomp%y2disp(i) = decomp%y2disp(i-1) + decomp%y2cnts(i-1)
-          decomp%z2disp(i) = decomp%z2disp(i-1) + decomp%z2cnts(i-1)
-       end if
-    end do
-
-    ! MPI_ALLTOALL buffer information
-
-    ! For evenly distributed data, following is an easier implementation.
-    ! But it should be covered by the more general formulation below.
-    !decomp%x1count = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)/dims(1)
-    !decomp%y1count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(1) 
-    !decomp%y2count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(2)
-    !decomp%z2count = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)/dims(2)
-
-    ! For unevenly distributed data, pad smaller messages. Note the 
-    ! last blocks along pencils always get assigned more mesh points
-    ! for X <=> Y transposes
-    decomp%x1count = decomp%x1dist(dims(1)-1) * &
-         decomp%y1dist(dims(1)-1) * decomp%xsz(3)
-    decomp%y1count = decomp%x1count
-    ! for Y <=> Z transposes
-    decomp%y2count = decomp%y2dist(dims(2)-1) * &
-         decomp%z2dist(dims(2)-1) * decomp%zsz(1)
-    decomp%z2count = decomp%y2count
-
-    return
-  end subroutine prepare_buffer
+   subroutine prepare_buffer(decomp)
+
+      implicit none
+
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+
+      integer :: i
+
+      !LG : AJOUTS "bidons" pour eviter un plantage en -O3 avec gcc9.3
+      !       * la fonction sortait des valeurs 'aleatoires'
+      !         et le calcul plantait dans MPI_ALLTOALLV
+      !       * pas de plantage en O2
+
+      if (nrank == 0) then
+         open (newunit=i, file='temp.dat', form='unformatted')
+         write (i) decomp%x1dist, decomp%y1dist, decomp%y2dist, decomp%z2dist, &
+            decomp%xsz, decomp%ysz, decomp%zsz
+         close (i, status='delete')
+      end if
+
+      ! MPI_ALLTOALLV buffer information
+
+      do i = 0, dims(1) - 1
+         decomp%x1cnts(i) = decomp%x1dist(i)*decomp%xsz(2)*decomp%xsz(3)
+         decomp%y1cnts(i) = decomp%ysz(1)*decomp%y1dist(i)*decomp%ysz(3)
+         if (i == 0) then
+            decomp%x1disp(i) = 0  ! displacement is 0-based index
+            decomp%y1disp(i) = 0
+         else
+            decomp%x1disp(i) = decomp%x1disp(i - 1) + decomp%x1cnts(i - 1)
+            decomp%y1disp(i) = decomp%y1disp(i - 1) + decomp%y1cnts(i - 1)
+         end if
+      end do
+
+      do i = 0, dims(2) - 1
+         decomp%y2cnts(i) = decomp%ysz(1)*decomp%y2dist(i)*decomp%ysz(3)
+         decomp%z2cnts(i) = decomp%zsz(1)*decomp%zsz(2)*decomp%z2dist(i)
+         if (i == 0) then
+            decomp%y2disp(i) = 0  ! displacement is 0-based index
+            decomp%z2disp(i) = 0
+         else
+            decomp%y2disp(i) = decomp%y2disp(i - 1) + decomp%y2cnts(i - 1)
+            decomp%z2disp(i) = decomp%z2disp(i - 1) + decomp%z2cnts(i - 1)
+         end if
+      end do
+
+      ! MPI_ALLTOALL buffer information
+
+      ! For evenly distributed data, following is an easier implementation.
+      ! But it should be covered by the more general formulation below.
+      !decomp%x1count = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)/dims(1)
+      !decomp%y1count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(1)
+      !decomp%y2count = decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3)/dims(2)
+      !decomp%z2count = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)/dims(2)
+
+      ! For unevenly distributed data, pad smaller messages. Note the
+      ! last blocks along pencils always get assigned more mesh points
+      ! for X <=> Y transposes
+      decomp%x1count = decomp%x1dist(dims(1) - 1)* &
+                       decomp%y1dist(dims(1) - 1)*decomp%xsz(3)
+      decomp%y1count = decomp%x1count
+      ! for Y <=> Z transposes
+      decomp%y2count = decomp%y2dist(dims(2) - 1)* &
+                       decomp%z2dist(dims(2) - 1)*decomp%zsz(1)
+      decomp%z2count = decomp%y2count
+
+      return
+   end subroutine prepare_buffer
 
 #ifdef SHM
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  Generate shared-memory information 
+   !  Generate shared-memory information
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_info_init_shm(decomp)
+   subroutine decomp_info_init_shm(decomp)
 
-    implicit none
+      implicit none
 
-    TYPE(DECOMP_INFO), intent(INOUT) :: decomp
+      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
 
-    ! a copy of old displacement array (will be overwritten by shm code)
-    allocate(decomp%x1disp_o(0:dims(1)-1),decomp%y1disp_o(0:dims(1)-1), &
-         decomp%y2disp_o(0:dims(2)-1),decomp%z2disp_o(0:dims(2)-1))
-    decomp%x1disp_o = decomp%x1disp
-    decomp%y1disp_o = decomp%y1disp
-    decomp%y2disp_o = decomp%y2disp
-    decomp%z2disp_o = decomp%z2disp
+      ! a copy of old displacement array (will be overwritten by shm code)
+      allocate (decomp%x1disp_o(0:dims(1) - 1), decomp%y1disp_o(0:dims(1) - 1), &
+                decomp%y2disp_o(0:dims(2) - 1), decomp%z2disp_o(0:dims(2) - 1))
+      decomp%x1disp_o = decomp%x1disp
+      decomp%y1disp_o = decomp%y1disp
+      decomp%y2disp_o = decomp%y2disp
+      decomp%z2disp_o = decomp%z2disp
 
-    call prepare_shared_buffer(decomp%ROW_INFO,DECOMP_2D_COMM_ROW,decomp)
-    call prepare_shared_buffer(decomp%COL_INFO,DECOMP_2D_COMM_COL,decomp)
-
-    return
-  end subroutine decomp_info_init_shm
+      call prepare_shared_buffer(decomp%ROW_INFO, DECOMP_2D_COMM_ROW, decomp)
+      call prepare_shared_buffer(decomp%COL_INFO, DECOMP_2D_COMM_COL, decomp)
 
+      return
+   end subroutine decomp_info_init_shm
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! For shared-memory implementation, prepare send/recv shared buffer
+   ! For shared-memory implementation, prepare send/recv shared buffer
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine prepare_shared_buffer(C,MPI_COMM,decomp)
-
-    implicit none
-
-    TYPE(SMP_INFO) :: C
-    INTEGER :: MPI_COMM
-    TYPE(DECOMP_INFO) :: decomp
-
-    INTEGER, ALLOCATABLE :: KTBL(:,:),NARY(:,:),KTBLALL(:,:)
-    INTEGER MYSMP, MYCORE, COLOR
-
-    integer :: ierror
-
-    C%MPI_COMM = MPI_COMM
-    CALL MPI_COMM_SIZE(MPI_COMM,C%NCPU,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-    CALL MPI_COMM_RANK(MPI_COMM,C%NODE_ME,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_RANK")
-    C%SMP_COMM  = MPI_COMM_NULL
-    C%CORE_COMM = MPI_COMM_NULL
-    C%SMP_ME= 0
-    C%NCORE = 0
-    C%CORE_ME = 0
-    C%MAXCORE = 0
-    C%NSMP  = 0
-    C%N_SND = 0
-    C%N_RCV = 0
-    C%SND_P = 0
-    C%RCV_P = 0
-    C%SND_P_c = 0
-    C%RCV_P_c = 0
-
-    ! get smp-node map for this communicator and set up smp communicators
-    CALL GET_SMP_MAP(C%MPI_COMM, C%NSMP, MYSMP, &
-         C%NCORE, MYCORE, C%MAXCORE)
-    C%SMP_ME = MYSMP + 1
-    C%CORE_ME = MYCORE + 1
-    ! - set up inter/intra smp-node communicators
-    COLOR = MYCORE
-    IF (COLOR.GT.0) COLOR = MPI_UNDEFINED
-    CALL MPI_Comm_split(C%MPI_COMM, COLOR, MYSMP, C%SMP_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-    CALL MPI_Comm_split(C%MPI_COMM, MYSMP, MYCORE, C%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-    ! - allocate work space
-    ALLOCATE(KTBL(C%MAXCORE,C%NSMP),NARY(C%NCPU,C%NCORE))
-    ALLOCATE(KTBLALL(C%MAXCORE,C%NSMP))
-    ! - set up smp-node/core to node_me lookup table
-    KTBL = 0
-    KTBL(C%CORE_ME,C%SMP_ME) = C%NODE_ME + 1
-    CALL MPI_ALLREDUCE(KTBL,KTBLALL,C%NSMP*C%MAXCORE,MPI_INTEGER, &
-         MPI_SUM,MPI_COMM,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-    KTBL=KTBLALL
-    !  IF (SUM(KTBL) /= C%NCPU*(C%NCPU+1)/2) &
-    !       CALL MPI_ABORT(...
-
-    ! compute offsets in shared SNDBUF and RCVBUF
-    CALL MAPSET_SMPSHM(C, KTBL, NARY, decomp)
-
-    DEALLOCATE(KTBL,NARY)
-
-    return
-  end subroutine prepare_shared_buffer
+   subroutine prepare_shared_buffer(C, MPI_COMM, decomp)
+
+      implicit none
+
+      TYPE(SMP_INFO) :: C
+      INTEGER :: MPI_COMM
+      TYPE(DECOMP_INFO) :: decomp
+
+      INTEGER, ALLOCATABLE :: KTBL(:, :), NARY(:, :), KTBLALL(:, :)
+      INTEGER MYSMP, MYCORE, COLOR
+
+      integer :: ierror
+
+      C%MPI_COMM = MPI_COMM
+      CALL MPI_COMM_SIZE(MPI_COMM, C%NCPU, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+      CALL MPI_COMM_RANK(MPI_COMM, C%NODE_ME, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_RANK")
+      C%SMP_COMM = MPI_COMM_NULL
+      C%CORE_COMM = MPI_COMM_NULL
+      C%SMP_ME = 0
+      C%NCORE = 0
+      C%CORE_ME = 0
+      C%MAXCORE = 0
+      C%NSMP = 0
+      C%N_SND = 0
+      C%N_RCV = 0
+      C%SND_P = 0
+      C%RCV_P = 0
+      C%SND_P_c = 0
+      C%RCV_P_c = 0
+
+      ! get smp-node map for this communicator and set up smp communicators
+      CALL GET_SMP_MAP(C%MPI_COMM, C%NSMP, MYSMP, &
+                       C%NCORE, MYCORE, C%MAXCORE)
+      C%SMP_ME = MYSMP + 1
+      C%CORE_ME = MYCORE + 1
+      ! - set up inter/intra smp-node communicators
+      COLOR = MYCORE
+      IF (COLOR > 0) COLOR = MPI_UNDEFINED
+      CALL MPI_Comm_split(C%MPI_COMM, COLOR, MYSMP, C%SMP_COMM, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+      CALL MPI_Comm_split(C%MPI_COMM, MYSMP, MYCORE, C%CORE_COMM, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+      ! - allocate work space
+      ALLOCATE (KTBL(C%MAXCORE, C%NSMP), NARY(C%NCPU, C%NCORE))
+      ALLOCATE (KTBLALL(C%MAXCORE, C%NSMP))
+      ! - set up smp-node/core to node_me lookup table
+      KTBL = 0
+      KTBL(C%CORE_ME, C%SMP_ME) = C%NODE_ME + 1
+      CALL MPI_ALLREDUCE(KTBL, KTBLALL, C%NSMP*C%MAXCORE, MPI_INTEGER, &
+                         MPI_SUM, MPI_COMM, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+      KTBL = KTBLALL
+      !  IF (SUM(KTBL) /= C%NCPU*(C%NCPU+1)/2) &
+      !       CALL MPI_ABORT(...
+
+      ! compute offsets in shared SNDBUF and RCVBUF
+      CALL MAPSET_SMPSHM(C, KTBL, NARY, decomp)
+
+      DEALLOCATE (KTBL, NARY)
+
+      return
+   end subroutine prepare_shared_buffer
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Use Ian Bush's FreeIPC to generate shared-memory information
-  !  - system independent solution
-  !  - replacing David Tanqueray's implementation in alloc_shm.c
-  !    (old C code renamed to get_smp_map2)
+   ! Use Ian Bush's FreeIPC to generate shared-memory information
+   !  - system independent solution
+   !  - replacing David Tanqueray's implementation in alloc_shm.c
+   !    (old C code renamed to get_smp_map2)
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine get_smp_map(comm, nnodes, my_node, ncores, my_core, maxcor)
-
-    use FIPC_module
+   subroutine get_smp_map(comm, nnodes, my_node, ncores, my_core, maxcor)
 
-    implicit none
+      use FIPC_module
 
-    integer, intent(IN) :: comm
-    integer, intent(OUT) :: nnodes, my_node, ncores, my_core, maxcor
+      implicit none
 
-    integer :: intra_comm, extra_comm
-    integer :: ierror
+      integer, intent(IN) :: comm
+      integer, intent(OUT) :: nnodes, my_node, ncores, my_core, maxcor
 
-    call FIPC_init(comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_init")
+      integer :: intra_comm, extra_comm
+      integer :: ierror
 
-    ! intra_comm: communicator for processes on this shared memory node
-    ! extra_comm: communicator for all rank 0 on each shared memory node
-    call FIPC_ctxt_intra_comm(FIPC_ctxt_world, intra_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_intra_comm")
-    call FIPC_ctxt_extra_comm(FIPC_ctxt_world, extra_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_extra_comm")
+      call FIPC_init(comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_init")
 
-    call MPI_COMM_SIZE(intra_comm,  ncores, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-    call MPI_COMM_RANK(intra_comm, my_core, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+      ! intra_comm: communicator for processes on this shared memory node
+      ! extra_comm: communicator for all rank 0 on each shared memory node
+      call FIPC_ctxt_intra_comm(FIPC_ctxt_world, intra_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_intra_comm")
+      call FIPC_ctxt_extra_comm(FIPC_ctxt_world, extra_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_extra_comm")
 
-    ! only rank 0 on each shared memory node member of extra_comm
-    ! for others extra_comm = MPI_COMM_NULL
-    if (extra_comm /= MPI_COMM_NULL) then
-       call MPI_COMM_SIZE(extra_comm,  nnodes, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-       call MPI_COMM_RANK(extra_comm, my_node, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-    end if
+      call MPI_COMM_SIZE(intra_comm, ncores, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+      call MPI_COMM_RANK(intra_comm, my_core, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
 
-    ! other ranks share the same information as their leaders
-    call MPI_BCAST( nnodes, 1, MPI_INTEGER, 0, intra_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
-    call MPI_BCAST(my_node, 1, MPI_INTEGER, 0, intra_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+      ! only rank 0 on each shared memory node member of extra_comm
+      ! for others extra_comm = MPI_COMM_NULL
+      if (extra_comm /= MPI_COMM_NULL) then
+         call MPI_COMM_SIZE(extra_comm, nnodes, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+         call MPI_COMM_RANK(extra_comm, my_node, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+      end if
 
-    ! maxcor
-    call MPI_ALLREDUCE(ncores, maxcor, 1, MPI_INTEGER, MPI_MAX, &
-         decomp_2d_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+      ! other ranks share the same information as their leaders
+      call MPI_BCAST(nnodes, 1, MPI_INTEGER, 0, intra_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+      call MPI_BCAST(my_node, 1, MPI_INTEGER, 0, intra_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
 
-    call FIPC_finalize(ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_finalize")
+      ! maxcor
+      call MPI_ALLREDUCE(ncores, maxcor, 1, MPI_INTEGER, MPI_MAX, &
+                         decomp_2d_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
 
-    return
+      call FIPC_finalize(ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_finalize")
 
-  end subroutine get_smp_map
+      return
 
+   end subroutine get_smp_map
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Set up smp-node based shared memory maps
+   ! Set up smp-node based shared memory maps
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  SUBROUTINE MAPSET_SMPSHM(C, KTBL, NARY, decomp)
-
-    IMPLICIT NONE
-
-    TYPE (SMP_INFO) C
-    INTEGER KTBL(C%MAXCORE,C%NSMP)
-    INTEGER NARY(C%NCPU,C%NCORE)
-    TYPE (DECOMP_INFO) :: decomp
-
-    INTEGER i, j, k, l, N, PTR, BSIZ, ierror, status, seed
-    character*16 s
-
-    BSIZ = C%N_SND
-
-    ! a - SNDBUF
-    IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
-       ALLOCATE(decomp%x1cnts_s(C%NSMP),decomp%x1disp_s(C%NSMP+1), &
-            stat=status)
-       CALL MPI_Allgather(decomp%x1cnts, C%NCPU, MPI_INTEGER, &
-            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-       PTR = 0
-       DO i=1,C%NSMP
-          decomp%x1disp_s(i) = PTR
-          N = 0
-          DO j=1,C%MAXCORE
-             k = KTBL(j,i)
-             IF (k > 0) then
-                DO l=1,C%NCORE
-                   IF (l == C%CORE_ME) decomp%x1disp_o(k-1) = PTR
-                   N = N + NARY(k,l)
-                   PTR = PTR + NARY(k,l)
-                END DO
-             END IF
-          END DO
-          decomp%x1cnts_s(i) = N
-       END DO
-       decomp%x1disp_s(C%NSMP+1) = PTR
-       IF (PTR > BSIZ) BSIZ = PTR
-
-    ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
-       ALLOCATE(decomp%y2cnts_s(C%NSMP),decomp%y2disp_s(C%NSMP+1), &
-            stat=status)
-       CALL MPI_Allgather(decomp%y2cnts, C%NCPU, MPI_INTEGER, &
-            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-       PTR = 0
-       DO i=1,C%NSMP
-          decomp%y2disp_s(i) = PTR
-          N = 0
-          DO j=1,C%MAXCORE
-             k = KTBL(j,i)
-             IF (k > 0) then
-                DO l=1,C%NCORE
-                   IF (l == C%CORE_ME) decomp%y2disp_o(k-1) = PTR
-                   N = N + NARY(k,l)
-                   PTR = PTR + NARY(k,l)
-                END DO
-             END IF
-          END DO
-          decomp%y2cnts_s(i) = N
-       END DO
-       decomp%y2disp_s(C%NSMP+1) = PTR
-       IF (PTR > BSIZ) BSIZ = PTR
-    END IF
-
-    ! b - RCVBUF
-
-    IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
-       ALLOCATE(decomp%y1cnts_s(C%NSMP),decomp%y1disp_s(C%NSMP+1), &
-            stat=status)
-       CALL MPI_Allgather(decomp%y1cnts, C%NCPU, MPI_INTEGER, &
-            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-       PTR = 0
-       DO i=1,C%NSMP
-          decomp%y1disp_s(i) = PTR
-          N=0
-          DO j=1,C%NCORE
-             DO l=1,C%MAXCORE
-                k = KTBL(l,i)
-                IF (k > 0) then
-                   IF (j == C%CORE_ME) decomp%y1disp_o(k-1) = PTR
-                   N = N + NARY(k,j)
-                   PTR = PTR + NARY(k,j)
-                END IF
-             END DO
-          END DO
-          decomp%y1cnts_s(i) = N
-       END DO
-       decomp%y1disp_s(C%NSMP+1) = PTR
-       IF (PTR > BSIZ) BSIZ = PTR
-
-    ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
-       ALLOCATE(decomp%z2cnts_s(C%NSMP),decomp%z2disp_s(C%NSMP+1), &
-            stat=status)
-       CALL MPI_Allgather(decomp%z2cnts, C%NCPU, MPI_INTEGER, &
-            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-       PTR = 0
-       DO i=1,C%NSMP
-          decomp%z2disp_s(i) = PTR
-          N=0
-          DO j=1,C%NCORE
-             DO l=1,C%MAXCORE
-                k = KTBL(l,i)
-                IF (k > 0) then
-                   IF (j == C%CORE_ME) decomp%z2disp_o(k-1) = PTR
-                   N = N + NARY(k,j)
-                   PTR = PTR + NARY(k,j)
-                END IF
-             END DO
-          END DO
-          decomp%z2cnts_s(i) = N
-       END DO
-       decomp%z2disp_s(C%NSMP+1) = PTR
-       IF (PTR > BSIZ) BSIZ = PTR
-
-    END IF
-
-    ! check buffer size and (re)-allocate buffer space if necessary
-    IF (BSIZ > C%N_SND) then
-       IF (C%SND_P /= 0) CALL DEALLOC_SHM(C%SND_P, C%CORE_COMM)
-       ! make sure each rank has unique keys to get shared memory
-       !IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
-       !   seed = nrank+nproc*0+1 ! has to be non-zero
-       !ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
-       !   seed = nrank+nproc*1+1
-       !END IF
-       status = 1
-       !CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status, &
-       !     seed)
-       CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status)
-       C%N_SND = BSIZ
-
-       IF (C%RCV_P /= 0) CALL DEALLOC_SHM(C%RCV_P, C%CORE_COMM)
-       status = 1
-       CALL ALLOC_SHM(C%RCV_P, BSIZ, real_type, C%CORE_COMM, status)
-       C%N_RCV = BSIZ
-
-       IF (C%SND_P_c /= 0) CALL DEALLOC_SHM(C%SND_P_c, C%CORE_COMM)
-       status = 1
-       CALL ALLOC_SHM(C%SND_P_c, BSIZ, complex_type, C%CORE_COMM, status)
-       C%N_SND = BSIZ
-
-       IF (C%RCV_P_c /= 0) CALL DEALLOC_SHM(C%RCV_P_c, C%CORE_COMM)
-       status = 1
-       CALL ALLOC_SHM(C%RCV_P_c, BSIZ, complex_type, C%CORE_COMM, status)
-       C%N_RCV = BSIZ
-
-
-    END IF
-
-    RETURN
-  END SUBROUTINE MAPSET_SMPSHM
+   SUBROUTINE MAPSET_SMPSHM(C, KTBL, NARY, decomp)
+
+      IMPLICIT NONE
+
+      TYPE(SMP_INFO) C
+      INTEGER KTBL(C%MAXCORE, C%NSMP)
+      INTEGER NARY(C%NCPU, C%NCORE)
+      TYPE(DECOMP_INFO) :: decomp
+
+      INTEGER i, j, k, l, N, PTR, BSIZ, ierror, status, seed
+      character*16 s
+
+      BSIZ = C%N_SND
+
+      ! a - SNDBUF
+      IF (C%MPI_COMM == DECOMP_2D_COMM_COL) THEN
+         ALLOCATE (decomp%x1cnts_s(C%NSMP), decomp%x1disp_s(C%NSMP + 1), &
+                   stat=status)
+         CALL MPI_Allgather(decomp%x1cnts, C%NCPU, MPI_INTEGER, &
+                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+         PTR = 0
+         DO i = 1, C%NSMP
+            decomp%x1disp_s(i) = PTR
+            N = 0
+            DO j = 1, C%MAXCORE
+               k = KTBL(j, i)
+               IF (k > 0) then
+                  DO l = 1, C%NCORE
+                     IF (l == C%CORE_ME) decomp%x1disp_o(k - 1) = PTR
+                     N = N + NARY(k, l)
+                     PTR = PTR + NARY(k, l)
+                  END DO
+               END IF
+            END DO
+            decomp%x1cnts_s(i) = N
+         END DO
+         decomp%x1disp_s(C%NSMP + 1) = PTR
+         IF (PTR > BSIZ) BSIZ = PTR
+
+      ELSE IF (C%MPI_COMM == DECOMP_2D_COMM_ROW) THEN
+         ALLOCATE (decomp%y2cnts_s(C%NSMP), decomp%y2disp_s(C%NSMP + 1), &
+                   stat=status)
+         CALL MPI_Allgather(decomp%y2cnts, C%NCPU, MPI_INTEGER, &
+                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+         PTR = 0
+         DO i = 1, C%NSMP
+            decomp%y2disp_s(i) = PTR
+            N = 0
+            DO j = 1, C%MAXCORE
+               k = KTBL(j, i)
+               IF (k > 0) then
+                  DO l = 1, C%NCORE
+                     IF (l == C%CORE_ME) decomp%y2disp_o(k - 1) = PTR
+                     N = N + NARY(k, l)
+                     PTR = PTR + NARY(k, l)
+                  END DO
+               END IF
+            END DO
+            decomp%y2cnts_s(i) = N
+         END DO
+         decomp%y2disp_s(C%NSMP + 1) = PTR
+         IF (PTR > BSIZ) BSIZ = PTR
+      END IF
+
+      ! b - RCVBUF
+
+      IF (C%MPI_COMM == DECOMP_2D_COMM_COL) THEN
+         ALLOCATE (decomp%y1cnts_s(C%NSMP), decomp%y1disp_s(C%NSMP + 1), &
+                   stat=status)
+         CALL MPI_Allgather(decomp%y1cnts, C%NCPU, MPI_INTEGER, &
+                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+         PTR = 0
+         DO i = 1, C%NSMP
+            decomp%y1disp_s(i) = PTR
+            N = 0
+            DO j = 1, C%NCORE
+               DO l = 1, C%MAXCORE
+                  k = KTBL(l, i)
+                  IF (k > 0) then
+                     IF (j == C%CORE_ME) decomp%y1disp_o(k - 1) = PTR
+                     N = N + NARY(k, j)
+                     PTR = PTR + NARY(k, j)
+                  END IF
+               END DO
+            END DO
+            decomp%y1cnts_s(i) = N
+         END DO
+         decomp%y1disp_s(C%NSMP + 1) = PTR
+         IF (PTR > BSIZ) BSIZ = PTR
+
+      ELSE IF (C%MPI_COMM == DECOMP_2D_COMM_ROW) THEN
+         ALLOCATE (decomp%z2cnts_s(C%NSMP), decomp%z2disp_s(C%NSMP + 1), &
+                   stat=status)
+         CALL MPI_Allgather(decomp%z2cnts, C%NCPU, MPI_INTEGER, &
+                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
+         PTR = 0
+         DO i = 1, C%NSMP
+            decomp%z2disp_s(i) = PTR
+            N = 0
+            DO j = 1, C%NCORE
+               DO l = 1, C%MAXCORE
+                  k = KTBL(l, i)
+                  IF (k > 0) then
+                     IF (j == C%CORE_ME) decomp%z2disp_o(k - 1) = PTR
+                     N = N + NARY(k, j)
+                     PTR = PTR + NARY(k, j)
+                  END IF
+               END DO
+            END DO
+            decomp%z2cnts_s(i) = N
+         END DO
+         decomp%z2disp_s(C%NSMP + 1) = PTR
+         IF (PTR > BSIZ) BSIZ = PTR
+
+      END IF
+
+      ! check buffer size and (re)-allocate buffer space if necessary
+      IF (BSIZ > C%N_SND) then
+         IF (C%SND_P /= 0) CALL DEALLOC_SHM(C%SND_P, C%CORE_COMM)
+         ! make sure each rank has unique keys to get shared memory
+         !IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
+         !   seed = nrank+nproc*0+1 ! has to be non-zero
+         !ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
+         !   seed = nrank+nproc*1+1
+         !END IF
+         status = 1
+         !CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status, &
+         !     seed)
+         CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status)
+         C%N_SND = BSIZ
+
+         IF (C%RCV_P /= 0) CALL DEALLOC_SHM(C%RCV_P, C%CORE_COMM)
+         status = 1
+         CALL ALLOC_SHM(C%RCV_P, BSIZ, real_type, C%CORE_COMM, status)
+         C%N_RCV = BSIZ
+
+         IF (C%SND_P_c /= 0) CALL DEALLOC_SHM(C%SND_P_c, C%CORE_COMM)
+         status = 1
+         CALL ALLOC_SHM(C%SND_P_c, BSIZ, complex_type, C%CORE_COMM, status)
+         C%N_SND = BSIZ
+
+         IF (C%RCV_P_c /= 0) CALL DEALLOC_SHM(C%RCV_P_c, C%CORE_COMM)
+         status = 1
+         CALL ALLOC_SHM(C%RCV_P_c, BSIZ, complex_type, C%CORE_COMM, status)
+         C%N_RCV = BSIZ
+
+      END IF
+
+      RETURN
+   END SUBROUTINE MAPSET_SMPSHM
 
 #endif
 
-
 #ifdef OCC
-  ! For non-blocking communication code, progress the comminication stack
-  subroutine transpose_test(handle)
+   ! For non-blocking communication code, progress the comminication stack
+   subroutine transpose_test(handle)
 
-    implicit none
+      implicit none
 
-    integer :: handle, ierror
+      integer :: handle, ierror
 
-    call NBC_TEST(handle,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "NBC_TEST")
+      call NBC_TEST(handle, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "NBC_TEST")
 
-    return
-  end subroutine transpose_test
+      return
+   end subroutine transpose_test
 #endif
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Transposition routines 
+   ! Transposition routines
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "transpose_x_to_y.f90"
 #include "transpose_y_to_z.f90"
@@ -1641,175 +1633,174 @@ end subroutine transpose_test
 #include "transpose_y_to_x.f90"
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Halo cell support
+   ! Halo cell support
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "halo.f90"
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Error handling
+   ! Error handling
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_2d_abort_basic(errorcode, msg)
+   subroutine decomp_2d_abort_basic(errorcode, msg)
 
-    use iso_fortran_env, only : error_unit
+      use iso_fortran_env, only: error_unit
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: errorcode
-    character(len=*), intent(IN) :: msg
+      integer, intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
 
-    integer :: ierror
+      integer :: ierror
 
-    if (nrank==0) then
-       write(*,*) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-       write(*,*) 'ERROR MESSAGE: ' // msg
-       write(error_unit,*) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-       write(error_unit,*) 'ERROR MESSAGE: ' // msg
-    end if
-    call MPI_ABORT(decomp_2d_comm,errorcode,ierror)
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+         write (*, *) 'ERROR MESSAGE: '//msg
+         write (error_unit, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+         write (error_unit, *) 'ERROR MESSAGE: '//msg
+      end if
+      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
 
-  end subroutine decomp_2d_abort_basic
+   end subroutine decomp_2d_abort_basic
 
-  subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
+   subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
 
-    use iso_fortran_env, only : error_unit
+      use iso_fortran_env, only: error_unit
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: errorcode, line
-    character(len=*), intent(IN) :: msg, file
+      integer, intent(IN) :: errorcode, line
+      character(len=*), intent(IN) :: msg, file
 
-    integer :: ierror
+      integer :: ierror
 
-    if (nrank==0) then
-       write(*,*) '2DECOMP&FFT ERROR'
-       write(*,*) '  errorcode:     ', errorcode
-       write(*,*) '  error in file  ' // file
-       write(*,*) '           line  ', line
-       write(*,*) '  error message: ' // msg
-       write(error_unit,*) '2DECOMP&FFT ERROR'
-       write(error_unit,*) '  errorcode:     ', errorcode
-       write(error_unit,*) '  error in file  ' // file
-       write(error_unit,*) '           line  ', line
-       write(error_unit,*) '  error message: ' // msg
-    end if
-    call MPI_ABORT(decomp_2d_comm,errorcode,ierror)
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT ERROR'
+         write (*, *) '  errorcode:     ', errorcode
+         write (*, *) '  error in file  '//file
+         write (*, *) '           line  ', line
+         write (*, *) '  error message: '//msg
+         write (error_unit, *) '2DECOMP&FFT ERROR'
+         write (error_unit, *) '  errorcode:     ', errorcode
+         write (error_unit, *) '  error in file  '//file
+         write (error_unit, *) '           line  ', line
+         write (error_unit, *) '  error message: '//msg
+      end if
+      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
 
-  end subroutine decomp_2d_abort_file_line
+   end subroutine decomp_2d_abort_file_line
 
 #if defined(_GPU) && defined(_NCCL)
-  !
-  ! This is based on the file "nccl.h" in nvhpc 22.1
-  !
-  function _ncclresult_to_integer(errorcode)
-
-    implicit none
-
-    type(ncclresult), intent(IN) :: errorcode
-    integer :: _ncclresult_to_integer
-
-    if (errorcode == ncclSuccess) then
-        _ncclresult_to_integer = 0
-    elseif (errorcode == ncclUnhandledCudaError) then
-        _ncclresult_to_integer = 1
-    elseif (errorcode == ncclSystemError) then
-        _ncclresult_to_integer = 2
-    elseif (errorcode == ncclInternalError) then
-        _ncclresult_to_integer = 3
-    elseif (errorcode == ncclInvalidArgument) then
-        _ncclresult_to_integer = 4
-    elseif (errorcode == ncclInvalidUsage) then
-        _ncclresult_to_integer = 5
-    elseif (errorcode == ncclNumResults) then
-        _ncclresult_to_integer = 6
-    else
-      _ncclresult_to_integer = -1
-      call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
-                             "NCCL error handling needs some update")
-    end if
-
-  end function _ncclresult_to_integer
-
-  !
-  ! Small wrapper for basic NCCL errors
-  !
-  subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
-
-    implicit none
-
-    type(ncclresult), intent(IN) :: errorcode
-    character(len=*), intent(IN) :: msg
-
-    call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
-                         msg // " " // ncclGetErrorString(errorcode))
-
-  end subroutine decomp_2d_abort_nccl_basic
-
-  !
-  ! Small wrapper for NCCL errors
-  !
-  subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
-
-    implicit none
-
-    type(ncclresult), intent(IN) :: errorcode
-    integer, intent(in) :: line
-    character(len=*), intent(IN) :: msg, file
-
-    call decomp_2d_abort(file, &
-                         line, &
-                         _ncclresult_to_integer(errorcode), &
-                         msg // " " // ncclGetErrorString(errorcode))
-
-  end subroutine decomp_2d_abort_nccl_file_line
+   !
+   ! This is based on the file "nccl.h" in nvhpc 22.1
+   !
+   function _ncclresult_to_integer(errorcode)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      integer :: _ncclresult_to_integer
+
+      if (errorcode == ncclSuccess) then
+         _ncclresult_to_integer = 0
+      elseif (errorcode == ncclUnhandledCudaError) then
+         _ncclresult_to_integer = 1
+      elseif (errorcode == ncclSystemError) then
+         _ncclresult_to_integer = 2
+      elseif (errorcode == ncclInternalError) then
+         _ncclresult_to_integer = 3
+      elseif (errorcode == ncclInvalidArgument) then
+         _ncclresult_to_integer = 4
+      elseif (errorcode == ncclInvalidUsage) then
+         _ncclresult_to_integer = 5
+      elseif (errorcode == ncclNumResults) then
+         _ncclresult_to_integer = 6
+      else
+         _ncclresult_to_integer = -1
+         call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
+                                "NCCL error handling needs some update")
+      end if
+
+   end function _ncclresult_to_integer
+
+   !
+   ! Small wrapper for basic NCCL errors
+   !
+   subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
+
+      call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
+                           msg//" "//ncclGetErrorString(errorcode))
+
+   end subroutine decomp_2d_abort_nccl_basic
+
+   !
+   ! Small wrapper for NCCL errors
+   !
+   subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      integer, intent(in) :: line
+      character(len=*), intent(IN) :: msg, file
+
+      call decomp_2d_abort(file, &
+                           line, &
+                           _ncclresult_to_integer(errorcode), &
+                           msg//" "//ncclGetErrorString(errorcode))
+
+   end subroutine decomp_2d_abort_nccl_file_line
 #endif
 
-  subroutine decomp_2d_warning_basic(errorcode, msg)
+   subroutine decomp_2d_warning_basic(errorcode, msg)
 
-    use iso_fortran_env, only : error_unit
+      use iso_fortran_env, only: error_unit
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: errorcode
-    character(len=*), intent(IN) :: msg
+      integer, intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
 
-    if (nrank==0) then
-       write(*,*) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-       write(*,*) 'ERROR MESSAGE: ' // msg
-       write(error_unit,*) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-       write(error_unit,*) 'ERROR MESSAGE: ' // msg
-    end if
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+         write (*, *) 'ERROR MESSAGE: '//msg
+         write (error_unit, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+         write (error_unit, *) 'ERROR MESSAGE: '//msg
+      end if
 
-  end subroutine decomp_2d_warning_basic
+   end subroutine decomp_2d_warning_basic
 
-  subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
+   subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
 
-    use iso_fortran_env, only : error_unit
+      use iso_fortran_env, only: error_unit
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: errorcode, line
-    character(len=*), intent(IN) :: msg, file
+      integer, intent(IN) :: errorcode, line
+      character(len=*), intent(IN) :: msg, file
 
-    if (nrank==0) then
-       write(*,*) '2DECOMP&FFT WARNING'
-       write(*,*) '  errorcode:     ', errorcode
-       write(*,*) '  error in file  ' // file
-       write(*,*) '           line  ', line
-       write(*,*) '  error message: ' // msg
-       write(error_unit,*) '2DECOMP&FFT WARNING'
-       write(error_unit,*) '  errorcode:     ', errorcode
-       write(error_unit,*) '  error in file  ' // file
-       write(error_unit,*) '           line  ', line
-       write(error_unit,*) '  error message: ' // msg
-    end if
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT WARNING'
+         write (*, *) '  errorcode:     ', errorcode
+         write (*, *) '  error in file  '//file
+         write (*, *) '           line  ', line
+         write (*, *) '  error message: '//msg
+         write (error_unit, *) '2DECOMP&FFT WARNING'
+         write (error_unit, *) '  errorcode:     ', errorcode
+         write (error_unit, *) '  error in file  '//file
+         write (error_unit, *) '           line  ', line
+         write (error_unit, *) '  error message: '//msg
+      end if
 
-  end subroutine decomp_2d_warning_file_line
+   end subroutine decomp_2d_warning_file_line
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Utility routines to help allocate 3D arrays
+   ! Utility routines to help allocate 3D arrays
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "alloc.f90"
-  
+
 end module decomp_2d
 
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index df0c4668..edec7259 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -1,7 +1,7 @@
 !!!=======================================================================
 !!! This is part of the 2DECOMP&FFT library
-!!! 
-!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!!!
+!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 !!! decomposition. It also implements a highly scalable distributed
 !!! three-dimensional Fast Fourier Transform (FFT).
 !!!
@@ -19,273 +19,272 @@
   !     all internal data structures initialised properly
   !     library ready to use
   !======================================================================
-  subroutine decomp_2d_init_ref(nx,ny,nz,p_row,p_col,periodic_bc,comm)
+  subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
 
-    use mpi
-    use iso_fortran_env, only : output_unit
+     use mpi
+     use iso_fortran_env, only: output_unit
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: nx,ny,nz
-    integer, intent(INOUT) :: p_row,p_col
-    logical, dimension(3), intent(IN), optional :: periodic_bc
-    integer, intent(in), optional :: comm
+     integer, intent(IN) :: nx, ny, nz
+     integer, intent(INOUT) :: p_row, p_col
+     logical, dimension(3), intent(IN), optional :: periodic_bc
+     integer, intent(in), optional :: comm
 
-    integer :: errorcode, ierror, row, col, iounit
-    logical, dimension(2) :: periodic
+     integer :: errorcode, ierror, row, col, iounit
+     logical, dimension(2) :: periodic
 #if defined(_GPU) && defined(_NCCL)
-    integer :: cuda_stat
-    type(ncclResult) :: nccl_stat
+     integer :: cuda_stat
+     type(ncclResult) :: nccl_stat
 #endif
-    character(len=7) fname ! Sufficient for up to O(1M) ranks
+     character(len=7) fname ! Sufficient for up to O(1M) ranks
 
 #ifdef PROFILER
-    ! Prepare the profiler if it was not already prepared
-    if (decomp_profiler.eq.decomp_profiler_none) call decomp_profiler_prep()
-    ! Start the profiler
-    call decomp_profiler_init()
-    ! Start the timer for decomp_2d_init
-    if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_init")
+     ! Prepare the profiler if it was not already prepared
+     if (decomp_profiler == decomp_profiler_none) call decomp_profiler_prep()
+     ! Start the profiler
+     call decomp_profiler_init()
+     ! Start the timer for decomp_2d_init
+     if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_init")
 #endif
 
-    ! Use the provided MPI communicator if present
-    if (present(comm)) then
-       decomp_2d_comm = comm
-    else
-       decomp_2d_comm = MPI_COMM_WORLD
-    endif
-
-    ! Safety check
-    if (MPI_SUCCESS /= 0) call decomp_2d_abort(__FILE__, &
-                                               __LINE__, &
-                                               MPI_SUCCESS, &
-                                               "MPI error check is broken")
-
-    ! If the external code has not set nrank and nproc
-    if (nrank == -1) then
-       call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                             __LINE__, &
-                                             ierror, &
-                                             "MPI_COMM_RANK")
-    endif
-    if (nproc == -1) then
-       call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                             __LINE__, &
-                                             ierror, &
-                                             "MPI_COMM_SIZE")
-    endif
+     ! Use the provided MPI communicator if present
+     if (present(comm)) then
+        decomp_2d_comm = comm
+     else
+        decomp_2d_comm = MPI_COMM_WORLD
+     end if
+
+     ! Safety check
+     if (MPI_SUCCESS /= 0) call decomp_2d_abort(__FILE__, &
+                                                __LINE__, &
+                                                MPI_SUCCESS, &
+                                                "MPI error check is broken")
+
+     ! If the external code has not set nrank and nproc
+     if (nrank == -1) then
+        call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, &
+                                              __LINE__, &
+                                              ierror, &
+                                              "MPI_COMM_RANK")
+     end if
+     if (nproc == -1) then
+        call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, &
+                                              __LINE__, &
+                                              ierror, &
+                                              "MPI_COMM_SIZE")
+     end if
 #ifdef DEBUG
-    ! Check if a modification of the debug level is needed
-    call decomp_2d_debug()
+     ! Check if a modification of the debug level is needed
+     call decomp_2d_debug()
 #endif
 
-    nx_global = nx
-    ny_global = ny
-    nz_global = nz
-
-    if (present(periodic_bc)) then
-       periodic_x = periodic_bc(1)
-       periodic_y = periodic_bc(2)
-       periodic_z = periodic_bc(3)
-    else
-       periodic_x = .false.
-       periodic_y = .false.
-       periodic_z = .false.
-    end if
-
-    if (p_row==0 .and. p_col==0) then
-       ! determine the best 2D processor grid
-       call best_2d_grid(nproc, row, col)
-       p_row = row
-       p_col = col
-    else
-       if (nproc /= p_row*p_col) then
-          errorcode = 1
-          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-               'Invalid 2D processor grid - nproc /= p_row*p_col')
-       else
-          row = p_row
-          col = p_col
-       end if
-    end if
-
-    ! Create 2D Catersian topology
-    ! Note that in order to support periodic B.C. in the halo-cell code,
-    ! need to create multiple topology objects: DECOMP_2D_COMM_CART_?,
-    ! corresponding to three pencil orientations. They contain almost
-    ! identical topological information but allow different combinations
-    ! of periodic conditions.
-    dims(1) = row
-    dims(2) = col
-    periodic(1) = periodic_y
-    periodic(2) = periodic_z
-    call MPI_CART_CREATE(decomp_2d_comm,2,dims,periodic, &
-         .false., &  ! do not reorder rank
-         DECOMP_2D_COMM_CART_X, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
-    periodic(1) = periodic_x
-    periodic(2) = periodic_z
-    call MPI_CART_CREATE(decomp_2d_comm,2,dims,periodic, &
-         .false., DECOMP_2D_COMM_CART_Y, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
-    periodic(1) = periodic_x
-    periodic(2) = periodic_y
-    call MPI_CART_CREATE(decomp_2d_comm,2,dims,periodic, &
-         .false., DECOMP_2D_COMM_CART_Z, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
-
-    call MPI_CART_COORDS(DECOMP_2D_COMM_CART_X,nrank,2,coord,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_COORDS")
-
-    ! derive communicators defining sub-groups for ALLTOALL(V)
-    call MPI_CART_SUB(DECOMP_2D_COMM_CART_X,(/.true.,.false./), &
-         DECOMP_2D_COMM_COL,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
-    call MPI_CART_SUB(DECOMP_2D_COMM_CART_X,(/.false.,.true./), &
-         DECOMP_2D_COMM_ROW,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
-
-    ! gather information for halo-cell support code
-    call init_neighbour
-
-    ! actually generate all 2D decomposition information
-    call decomp_info_init(nx,ny,nz,decomp_main)
-
-    ! make a copy of the decomposition information associated with the
-    ! default global size in these global variables so applications can
-    ! use them to create data structures 
-    xstart = decomp_main%xst
-    ystart = decomp_main%yst
-    zstart = decomp_main%zst
-    xend   = decomp_main%xen
-    yend   = decomp_main%yen
-    zend   = decomp_main%zen
-    xsize  = decomp_main%xsz
-    ysize  = decomp_main%ysz
-    zsize  = decomp_main%zsz
-
-    ! determine the number of bytes per float number
-    ! do not use 'mytype' which is compiler dependent
-    ! also possible to use inquire(iolength=...) 
-    call MPI_TYPE_SIZE(real_type,mytype_bytes,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+     nx_global = nx
+     ny_global = ny
+     nz_global = nz
+
+     if (present(periodic_bc)) then
+        periodic_x = periodic_bc(1)
+        periodic_y = periodic_bc(2)
+        periodic_z = periodic_bc(3)
+     else
+        periodic_x = .false.
+        periodic_y = .false.
+        periodic_z = .false.
+     end if
+
+     if (p_row == 0 .and. p_col == 0) then
+        ! determine the best 2D processor grid
+        call best_2d_grid(nproc, row, col)
+        p_row = row
+        p_col = col
+     else
+        if (nproc /= p_row*p_col) then
+           errorcode = 1
+           call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                                'Invalid 2D processor grid - nproc /= p_row*p_col')
+        else
+           row = p_row
+           col = p_col
+        end if
+     end if
+
+     ! Create 2D Catersian topology
+     ! Note that in order to support periodic B.C. in the halo-cell code,
+     ! need to create multiple topology objects: DECOMP_2D_COMM_CART_?,
+     ! corresponding to three pencil orientations. They contain almost
+     ! identical topological information but allow different combinations
+     ! of periodic conditions.
+     dims(1) = row
+     dims(2) = col
+     periodic(1) = periodic_y
+     periodic(2) = periodic_z
+     call MPI_CART_CREATE(decomp_2d_comm, 2, dims, periodic, &
+                          .false., &  ! do not reorder rank
+                          DECOMP_2D_COMM_CART_X, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
+     periodic(1) = periodic_x
+     periodic(2) = periodic_z
+     call MPI_CART_CREATE(decomp_2d_comm, 2, dims, periodic, &
+                          .false., DECOMP_2D_COMM_CART_Y, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
+     periodic(1) = periodic_x
+     periodic(2) = periodic_y
+     call MPI_CART_CREATE(decomp_2d_comm, 2, dims, periodic, &
+                          .false., DECOMP_2D_COMM_CART_Z, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_CREATE")
+
+     call MPI_CART_COORDS(DECOMP_2D_COMM_CART_X, nrank, 2, coord, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_COORDS")
+
+     ! derive communicators defining sub-groups for ALLTOALL(V)
+     call MPI_CART_SUB(DECOMP_2D_COMM_CART_X, (/.true., .false./), &
+                       DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
+     call MPI_CART_SUB(DECOMP_2D_COMM_CART_X, (/.false., .true./), &
+                       DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SUB")
+
+     ! gather information for halo-cell support code
+     call init_neighbour
+
+     ! actually generate all 2D decomposition information
+     call decomp_info_init(nx, ny, nz, decomp_main)
+
+     ! make a copy of the decomposition information associated with the
+     ! default global size in these global variables so applications can
+     ! use them to create data structures
+     xstart = decomp_main%xst
+     ystart = decomp_main%yst
+     zstart = decomp_main%zst
+     xend = decomp_main%xen
+     yend = decomp_main%yen
+     zend = decomp_main%zen
+     xsize = decomp_main%xsz
+     ysize = decomp_main%ysz
+     zsize = decomp_main%zsz
+
+     ! determine the number of bytes per float number
+     ! do not use 'mytype' which is compiler dependent
+     ! also possible to use inquire(iolength=...)
+     call MPI_TYPE_SIZE(real_type, mytype_bytes, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
 
 #ifdef EVEN
-    if (nrank==0) write(*,*) 'Padded ALLTOALL optimisation on'
-#endif 
+     if (nrank == 0) write (*, *) 'Padded ALLTOALL optimisation on'
+#endif
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    call MPI_COMM_RANK(DECOMP_2D_COMM_COL,col_rank,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-    call MPI_COMM_RANK(DECOMP_2D_COMM_ROW,row_rank,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-    call MPI_COMM_SIZE(DECOMP_2D_COMM_COL,col_comm_size,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-    call MPI_COMM_SIZE(DECOMP_2D_COMM_ROW,row_comm_size,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-
-    allocate(local_to_global_col(col_comm_size), local_to_global_row(row_comm_size))
-    
-    local_to_global_col(:) = 0
-    local_to_global_row(:) = 0
-    local_to_global_col(col_rank+1) = nrank
-    local_to_global_row(row_rank+1) = nrank
-    
-    call mpi_allreduce(MPI_IN_PLACE,local_to_global_col,col_comm_size,MPI_INTEGER,MPI_SUM,DECOMP_2D_COMM_COL,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-    call mpi_allreduce(MPI_IN_PLACE,local_to_global_row,row_comm_size,MPI_INTEGER,MPI_SUM,DECOMP_2D_COMM_ROW,ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-
-    if (nrank .eq. 0) then
-       nccl_stat = ncclGetUniqueId(nccl_uid_2decomp)
-    end if
-    call MPI_Bcast(nccl_uid_2decomp, int(sizeof(ncclUniqueId)), MPI_BYTE, 0, decomp_2d_comm, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
-
-    nccl_stat = ncclCommInitRank(nccl_comm_2decomp, nproc, nccl_uid_2decomp, nrank)
-    cuda_stat = cudaStreamCreate(cuda_stream_2decomp)
+     call MPI_COMM_RANK(DECOMP_2D_COMM_COL, col_rank, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+     call MPI_COMM_RANK(DECOMP_2D_COMM_ROW, row_rank, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+     call MPI_COMM_SIZE(DECOMP_2D_COMM_COL, col_comm_size, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+     call MPI_COMM_SIZE(DECOMP_2D_COMM_ROW, row_comm_size, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+
+     allocate (local_to_global_col(col_comm_size), local_to_global_row(row_comm_size))
+
+     local_to_global_col(:) = 0
+     local_to_global_row(:) = 0
+     local_to_global_col(col_rank + 1) = nrank
+     local_to_global_row(row_rank + 1) = nrank
+
+     call mpi_allreduce(MPI_IN_PLACE, local_to_global_col, col_comm_size, MPI_INTEGER, MPI_SUM, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+     call mpi_allreduce(MPI_IN_PLACE, local_to_global_row, row_comm_size, MPI_INTEGER, MPI_SUM, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+
+     if (nrank == 0) then
+        nccl_stat = ncclGetUniqueId(nccl_uid_2decomp)
+     end if
+     call MPI_Bcast(nccl_uid_2decomp, int(sizeof(ncclUniqueId)), MPI_BYTE, 0, decomp_2d_comm, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+
+     nccl_stat = ncclCommInitRank(nccl_comm_2decomp, nproc, nccl_uid_2decomp, nrank)
+     cuda_stat = cudaStreamCreate(cuda_stream_2decomp)
 #endif
 #endif
 
-    !
-    ! Select the IO unit for decomp_2d setup
-    !
-    if (decomp_log == D2D_LOG_TOFILE_FULL) then
-       write(fname, "(I0)") nrank ! Adapt to magnitude of nrank
-       open(newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
-    elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
-       open(newunit=iounit, file="decomp_2d_setup.log", iostat=ierror)
-    else
-       iounit = output_unit
-       ierror = 0
-    endif
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
-    !
-    ! Print the decomp_2d setup
-    !
-    call d2d_listing(iounit)
-    !
-    ! Close the IO unit if it was not stdout
-    !
-    if (iounit /= output_unit) then
-       close(iounit, iostat=ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-    endif
+     !
+     ! Select the IO unit for decomp_2d setup
+     !
+     if (decomp_log == D2D_LOG_TOFILE_FULL) then
+        write (fname, "(I0)") nrank ! Adapt to magnitude of nrank
+        open (newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
+     elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
+        open (newunit=iounit, file="decomp_2d_setup.log", iostat=ierror)
+     else
+        iounit = output_unit
+        ierror = 0
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
+     !
+     ! Print the decomp_2d setup
+     !
+     call d2d_listing(iounit)
+     !
+     ! Close the IO unit if it was not stdout
+     !
+     if (iounit /= output_unit) then
+        close (iounit, iostat=ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+     end if
 
 #ifdef PROFILER
-    ! Stop the timer for decomp_2d_init
-    if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_init")
+     ! Stop the timer for decomp_2d_init
+     if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_init")
 #endif
 
-    return
+     return
   end subroutine decomp_2d_init_ref
 
   !======================================================================
   ! Routine to be called by applications to clean things up
   !======================================================================
   subroutine decomp_2d_finalize_ref
-    
-    implicit none
+
+     implicit none
 #if defined(_GPU) && defined(_NCCL)
-    type(ncclResult) :: nccl_stat
+     type(ncclResult) :: nccl_stat
 #endif
-    
- 
+
 #ifdef PROFILER
-    if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_fin")
+     if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_fin")
 #endif
 
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_ROW)
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_COL)
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_X)
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Y)
-    call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Z)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_ROW)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_COL)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_X)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Y)
+     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Z)
 
-    call decomp_info_finalize(decomp_main)
+     call decomp_info_finalize(decomp_main)
 
-    decomp_buf_size = 0
-    deallocate(work1_r, work2_r, work1_c, work2_c)
+     decomp_buf_size = 0
+     deallocate (work1_r, work2_r, work1_c, work2_c)
 #if defined(_GPU)
-    deallocate(work1_r_d, work2_r_d, work1_c_d, work2_c_d)
+     deallocate (work1_r_d, work2_r_d, work1_c_d, work2_c_d)
 
 #if defined(_NCCL)
-    nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
+     nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
 #endif
 #endif
 
-    nrank = -1
-    nproc = -1
+     nrank = -1
+     nproc = -1
 
 #ifdef PROFILER
-    if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_fin")
-    ! Finalize the profiler
-    call decomp_profiler_fin()
+     if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_fin")
+     ! Finalize the profiler
+     call decomp_profiler_fin()
 #endif
 
-    return
+     return
   end subroutine decomp_2d_finalize_ref
 
 #ifdef DEBUG
@@ -296,33 +295,33 @@ end subroutine decomp_2d_finalize_ref
   !
   subroutine decomp_2d_debug
 
-    implicit none
+     implicit none
 
-    integer :: ierror
-    character(len=4) :: val
-    character(len=*), parameter :: varname = "DECOMP_2D_DEBUG"
+     integer :: ierror
+     character(len=4) :: val
+     character(len=*), parameter :: varname = "DECOMP_2D_DEBUG"
 
-    ! Read the variable
-    call get_environment_variable(varname, value=val, status=ierror)
+     ! Read the variable
+     call get_environment_variable(varname, value=val, status=ierror)
 
-    ! Return if no variable, or no support for env. variable
-    if (ierror >= 1) return
+     ! Return if no variable, or no support for env. variable
+     if (ierror >= 1) return
 
-    ! Minor error, print warning and return
-    if (ierror /= 0) then
-       call decomp_2d_warning(__FILE__, &
-                              __LINE__, &
-                              ierror, &
-                              "Error when reading DECOMP_2D_DEBUG : "//val)
-       return
-    endif
+     ! Minor error, print warning and return
+     if (ierror /= 0) then
+        call decomp_2d_warning(__FILE__, &
+                               __LINE__, &
+                               ierror, &
+                               "Error when reading DECOMP_2D_DEBUG : "//val)
+        return
+     end if
 
-    ! Conversion to integer if possible
-    read(val, '(i4)', iostat=ierror) decomp_debug
-    if (ierror /= 0) call decomp_2d_warning(__FILE__, &
-                                            __LINE__, &
-                                            ierror, &
-                                            "Error when reading DECOMP_2D_DEBUG : "//val)
+     ! Conversion to integer if possible
+     read (val, '(i4)', iostat=ierror) decomp_debug
+     if (ierror /= 0) call decomp_2d_warning(__FILE__, &
+                                             __LINE__, &
+                                             ierror, &
+                                             "Error when reading DECOMP_2D_DEBUG : "//val)
 
   end subroutine decomp_2d_debug
 #endif
@@ -332,34 +331,34 @@ end subroutine decomp_2d_debug
   !---------------------------------------------------------------------
   subroutine best_2d_grid(iproc, best_p_row, best_p_col)
 
-    implicit none
+     implicit none
+
+     integer, intent(IN) :: iproc
+     integer, intent(OUT) :: best_p_row, best_p_col
 
-    integer, intent(IN) :: iproc
-    integer, intent(OUT) :: best_p_row, best_p_col
+     integer, allocatable, dimension(:) :: factors
+     integer :: nfact, i, col, i_best
 
-    integer, allocatable, dimension(:) :: factors
-    integer :: nfact, i, col, i_best
+     if (nrank == 0) write (*, *) 'In auto-tuning mode......'
 
-    if (nrank==0) write(*,*) 'In auto-tuning mode......'
+     i = int(sqrt(real(iproc))) + 10  ! enough space to save all factors
+     allocate (factors(i))
+     call findfactor(iproc, factors, nfact)
+     if (nrank == 0) write (*, *) 'factors: ', (factors(i), i=1, nfact)
 
-    i = int(sqrt(real(iproc))) + 10  ! enough space to save all factors 
-    allocate(factors(i))
-    call findfactor(iproc, factors, nfact)
-    if (nrank==0) write(*,*) 'factors: ', (factors(i), i=1,nfact)
+     i_best = nfact/2 + 1
+     col = factors(i_best)
 
-    i_best=nfact/2+1
-    col=factors(i_best)
+     best_p_col = col
+     best_p_row = iproc/col
+     if (nrank == 0) print *, 'p_row x p_col', best_p_row, best_p_col
+     if ((best_p_col == 1) .and. (nrank == 0)) then
+        print *, 'WARNING: current 2D DECOMP set-up might not work'
+     end if
 
-    best_p_col = col
-    best_p_row=iproc/col
-    if (nrank==0) print *,'p_row x p_col', best_p_row, best_p_col
-    if ((best_p_col==1).and.(nrank==0)) then
-       print *,'WARNING: current 2D DECOMP set-up might not work'
-    endif
-    
-    deallocate(factors)
+     deallocate (factors)
 
-    return
+     return
   end subroutine best_2d_grid
 
   !---------------------------------------------------------------------
@@ -369,36 +368,36 @@ end subroutine best_2d_grid
   !---------------------------------------------------------------------
   subroutine init_neighbour
 
-    integer :: ierror
-
-    ! For X-pencil
-    neighbour(1,1) = MPI_PROC_NULL               ! east
-    neighbour(1,2) = MPI_PROC_NULL               ! west
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 0, 1, &
-         neighbour(1,4), neighbour(1,3), ierror) ! north & south
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 1, 1, &
-         neighbour(1,6), neighbour(1,5), ierror) ! top & bottom
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-
-    ! For Y-pencil
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 0, 1, &
-         neighbour(2,2), neighbour(2,1), ierror) ! east & west
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-    neighbour(2,3) = MPI_PROC_NULL               ! north
-    neighbour(2,4) = MPI_PROC_NULL               ! south
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 1, 1, &
-         neighbour(2,6), neighbour(2,5), ierror) ! top & bottom
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-
-    ! For Z-pencil
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 0, 1, &
-         neighbour(3,2), neighbour(3,1), ierror) ! east & west
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-    call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 1, 1, &
-         neighbour(3,4), neighbour(3,3), ierror) ! north & south
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
-    neighbour(3,5) = MPI_PROC_NULL               ! top
-    neighbour(3,6) = MPI_PROC_NULL               ! bottom
-    return
+     integer :: ierror
+
+     ! For X-pencil
+     neighbour(1, 1) = MPI_PROC_NULL               ! east
+     neighbour(1, 2) = MPI_PROC_NULL               ! west
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 0, 1, &
+                         neighbour(1, 4), neighbour(1, 3), ierror) ! north & south
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_X, 1, 1, &
+                         neighbour(1, 6), neighbour(1, 5), ierror) ! top & bottom
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+
+     ! For Y-pencil
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 0, 1, &
+                         neighbour(2, 2), neighbour(2, 1), ierror) ! east & west
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+     neighbour(2, 3) = MPI_PROC_NULL               ! north
+     neighbour(2, 4) = MPI_PROC_NULL               ! south
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Y, 1, 1, &
+                         neighbour(2, 6), neighbour(2, 5), ierror) ! top & bottom
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+
+     ! For Z-pencil
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 0, 1, &
+                         neighbour(3, 2), neighbour(3, 1), ierror) ! east & west
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+     call MPI_CART_SHIFT(DECOMP_2D_COMM_CART_Z, 1, 1, &
+                         neighbour(3, 4), neighbour(3, 3), ierror) ! north & south
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_SHIFT")
+     neighbour(3, 5) = MPI_PROC_NULL               ! top
+     neighbour(3, 6) = MPI_PROC_NULL               ! bottom
+     return
   end subroutine init_neighbour
diff --git a/src/factor.f90 b/src/factor.f90
index 73d96e15..5f7e530a 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -1,7 +1,7 @@
 !!!=======================================================================
 !!! This is part of the 2DECOMP&FFT library
-!!! 
-!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!!!
+!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 !!! decomposition. It also implements a highly scalable distributed
 !!! three-dimensional Fast Fourier Transform (FFT).
 !!!
@@ -13,80 +13,80 @@
 !!! A few utility routines to find factors of integer numbers
 module factor
 
-  implicit none
+   implicit none
+
+   private
 
-  private
+   public :: findfactor
+   public :: primefactors
 
-  public :: findfactor
-  public :: primefactors
-  
 contains
-  
-  subroutine findfactor(num, factors, nfact)
-    
-    implicit none
-
-    integer, intent(IN) :: num
-    integer, intent(OUT), dimension(*) :: factors
-    integer, intent(OUT) :: nfact
-    integer :: i, m
-
-    ! find the factors <= sqrt(num)
-    m = int(sqrt(real(num)))
-    nfact = 1
-    do i=1,m
-       if (num/i*i == num) then
-          factors(nfact) = i
-          nfact = nfact + 1
-       end if
-    end do
-    nfact = nfact - 1
-
-    ! derive those > sqrt(num)
-    if (factors(nfact)**2/=num) then
-       do i=nfact+1, 2*nfact
-          factors(i) = num / factors(2*nfact-i+1)
-       end do
-       nfact = nfact * 2
-    else
-       do i=nfact+1, 2*nfact-1
-          factors(i) = num / factors(2*nfact-i)
-       end do
-       nfact = nfact * 2 - 1
-    endif
-       
-    return
-
-  end subroutine findfactor
-
-  subroutine primefactors(num, factors, nfact)
-
-    implicit none
-  
-    integer, intent(IN) :: num
-    integer, intent(OUT), dimension(*) :: factors
-    integer, intent(INOUT) :: nfact
-
-    integer :: i, n
-    
-    i = 2  
-    nfact = 1
-    n = num 
-    do
-       if (mod(n,i) == 0) then
-          factors(nfact) = i
-          nfact = nfact + 1
-          n = n / i
-       else
-          i = i + 1
-       end if
-       if (n == 1) then
-          nfact = nfact - 1
-          exit
-       end if
-    end do
-    
-    return
-
-  end subroutine primefactors
+
+   subroutine findfactor(num, factors, nfact)
+
+      implicit none
+
+      integer, intent(IN) :: num
+      integer, intent(OUT), dimension(*) :: factors
+      integer, intent(OUT) :: nfact
+      integer :: i, m
+
+      ! find the factors <= sqrt(num)
+      m = int(sqrt(real(num)))
+      nfact = 1
+      do i = 1, m
+         if (num/i*i == num) then
+            factors(nfact) = i
+            nfact = nfact + 1
+         end if
+      end do
+      nfact = nfact - 1
+
+      ! derive those > sqrt(num)
+      if (factors(nfact)**2 /= num) then
+         do i = nfact + 1, 2*nfact
+            factors(i) = num/factors(2*nfact - i + 1)
+         end do
+         nfact = nfact*2
+      else
+         do i = nfact + 1, 2*nfact - 1
+            factors(i) = num/factors(2*nfact - i)
+         end do
+         nfact = nfact*2 - 1
+      end if
+
+      return
+
+   end subroutine findfactor
+
+   subroutine primefactors(num, factors, nfact)
+
+      implicit none
+
+      integer, intent(IN) :: num
+      integer, intent(OUT), dimension(*) :: factors
+      integer, intent(INOUT) :: nfact
+
+      integer :: i, n
+
+      i = 2
+      nfact = 1
+      n = num
+      do
+         if (mod(n, i) == 0) then
+            factors(nfact) = i
+            nfact = nfact + 1
+            n = n/i
+         else
+            i = i + 1
+         end if
+         if (n == 1) then
+            nfact = nfact - 1
+            exit
+         end if
+      end do
+
+      return
+
+   end subroutine primefactors
 end module
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 8c17df27..e0cd9cac 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -16,12 +16,12 @@
 
 ! Physical space data can be stored in either X-pencil or Z-pencil
 integer, parameter, public :: PHYSICAL_IN_X = 1
-integer, parameter, public :: PHYSICAL_IN_Z = 3 
+integer, parameter, public :: PHYSICAL_IN_Z = 3
 
 integer, save :: format                 ! input X-pencil or Z-pencil
 
 ! The libary can only be initialised once
-logical, save :: initialised = .false. 
+logical, save :: initialised = .false.
 
 ! Global size of the FFT
 integer, save :: nx_fft, ny_fft, nz_fft
@@ -31,85 +31,84 @@
 integer, save, dimension(2) :: dims
 
 ! Decomposition objects
-TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
+TYPE(DECOMP_INFO), pointer, save :: ph => null()  ! physical space
 TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
 
 ! Workspace to store the intermediate Y-pencil data
 ! *** TODO: investigate how to use only one workspace array
-complex(mytype), allocatable, dimension(:,:,:) :: wk2_c2c, wk2_r2c
-complex(mytype), allocatable, dimension(:,:,:) :: wk13
+complex(mytype), allocatable, dimension(:, :, :) :: wk2_c2c, wk2_r2c
+complex(mytype), allocatable, dimension(:, :, :) :: wk13
 
 public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
-decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
+          decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+          decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
 
 ! Declare generic interfaces to handle different inputs
 
 interface decomp_2d_fft_init
-module procedure fft_init_noarg
-module procedure fft_init_arg
-module procedure fft_init_general
+   module procedure fft_init_noarg
+   module procedure fft_init_arg
+   module procedure fft_init_general
 end interface
 
 interface decomp_2d_fft_3d
-module procedure fft_3d_c2c
-module procedure fft_3d_r2c
-module procedure fft_3d_c2r
+   module procedure fft_3d_c2c
+   module procedure fft_3d_r2c
+   module procedure fft_3d_c2r
 end interface
 
 contains
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Initialise the FFT module
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_init_noarg
 
-implicit none
+   implicit none
 
-call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
+   call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
 
-return
+   return
 end subroutine fft_init_noarg
 
 subroutine fft_init_arg(pencil)     ! allow to handle Z-pencil input
 
-implicit none
+   implicit none
 
-integer, intent(IN) :: pencil
+   integer, intent(IN) :: pencil
 
-call fft_init_general(pencil, nx_global, ny_global, nz_global)
+   call fft_init_general(pencil, nx_global, ny_global, nz_global)
 
-return
+   return
 end subroutine fft_init_arg
 
 ! Initialise the FFT library to perform arbitrary size transforms
 subroutine fft_init_general(pencil, nx, ny, nz)
 
-implicit none
+   implicit none
 
-integer, intent(IN) :: pencil
-integer, intent(IN) :: nx, ny, nz
+   integer, intent(IN) :: pencil
+   integer, intent(IN) :: nx, ny, nz
 
-integer :: status, errorcode
+   integer :: status, errorcode
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
 #endif
 
-if (initialised) then
-errorcode = 4
-call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-'FFT library should only be initialised once')
-end if
+   if (initialised) then
+      errorcode = 4
+      call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                           'FFT library should only be initialised once')
+   end if
 
-format = pencil
-nx_fft = nx
-ny_fft = ny
-nz_fft = nz
+   format = pencil
+   nx_fft = nx
+   ny_fft = ny
+   nz_fft = nz
 
 ! determine the processor grid in use
-dims = get_decomp_dims()
+   dims = get_decomp_dims()
 
 ! for c2r/r2c interface:
 ! if in physical space, a real array is of size: nx*ny*nz
@@ -117,98 +116,96 @@ subroutine fft_init_general(pencil, nx, ny, nz)
 !         (nx/2+1)*ny*nz, if PHYSICAL_IN_X
 !      or nx*ny*(nz/2+1), if PHYSICAL_IN_Z
 
-if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
-ph=>decomp_main
-else
-if (.not.associated(ph)) allocate(ph)
-call decomp_info_init(nx, ny, nz, ph)
-endif
-if (format==PHYSICAL_IN_X) then
-call decomp_info_init(nx/2+1, ny, nz, sp)
-else if (format==PHYSICAL_IN_Z) then
-call decomp_info_init(nx, ny, nz/2+1, sp)
-end if
-
-allocate(wk2_c2c(ph%ysz(1),ph%ysz(2),ph%ysz(3)), STAT=status)
-allocate(wk2_r2c(sp%ysz(1),sp%ysz(2),sp%ysz(3)), STAT=status)
-if (format==PHYSICAL_IN_X) then
-allocate(wk13(sp%xsz(1),sp%xsz(2),sp%xsz(3)), STAT=status)
-else if (format==PHYSICAL_IN_Z) then
-allocate(wk13(sp%zsz(1),sp%zsz(2),sp%zsz(3)), STAT=status)
-end if
-if (status /= 0) then
-errorcode = 3
-call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-'Out of memory when initialising FFT')
-end if
-
-call init_fft_engine
-
-initialised = .true.
+   if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
+      ph => decomp_main
+   else
+      if (.not. associated(ph)) allocate (ph)
+      call decomp_info_init(nx, ny, nz, ph)
+   end if
+   if (format == PHYSICAL_IN_X) then
+      call decomp_info_init(nx/2 + 1, ny, nz, sp)
+   else if (format == PHYSICAL_IN_Z) then
+      call decomp_info_init(nx, ny, nz/2 + 1, sp)
+   end if
+
+   allocate (wk2_c2c(ph%ysz(1), ph%ysz(2), ph%ysz(3)), STAT=status)
+   allocate (wk2_r2c(sp%ysz(1), sp%ysz(2), sp%ysz(3)), STAT=status)
+   if (format == PHYSICAL_IN_X) then
+      allocate (wk13(sp%xsz(1), sp%xsz(2), sp%xsz(3)), STAT=status)
+   else if (format == PHYSICAL_IN_Z) then
+      allocate (wk13(sp%zsz(1), sp%zsz(2), sp%zsz(3)), STAT=status)
+   end if
+   if (status /= 0) then
+      errorcode = 3
+      call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                           'Out of memory when initialising FFT')
+   end if
+
+   call init_fft_engine
+
+   initialised = .true.
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
 #endif
 
-return
+   return
 end subroutine fft_init_general
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Final clean up
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine decomp_2d_fft_finalize
 
-implicit none
+   implicit none
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
-call decomp_info_finalize(ph)
-deallocate(ph)
-endif
-nullify(ph)
-call decomp_info_finalize(sp)
+   if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
+      call decomp_info_finalize(ph)
+      deallocate (ph)
+   end if
+   nullify (ph)
+   call decomp_info_finalize(sp)
 
-if (allocated(wk2_c2c)) deallocate(wk2_c2c)
-if (allocated(wk2_r2c)) deallocate(wk2_r2c)
-if (allocated(wk13)) deallocate(wk13)
+   if (allocated(wk2_c2c)) deallocate (wk2_c2c)
+   if (allocated(wk2_r2c)) deallocate (wk2_r2c)
+   if (allocated(wk13)) deallocate (wk13)
 
-call finalize_fft_engine
+   call finalize_fft_engine
 
-initialised = .false.
+   initialised = .false.
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
 #endif
 
-return
+   return
 end subroutine decomp_2d_fft_finalize
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-! Return the size, starting/ending index of the distributed array 
+! Return the size, starting/ending index of the distributed array
 !  whose global size is (nx/2+1)*ny*nz, for defining data structures
 !  in r2c and c2r interfaces
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine decomp_2d_fft_get_size(istart, iend, isize)
 
-implicit none
-integer, dimension(3), intent(OUT) :: istart, iend, isize
+   implicit none
+   integer, dimension(3), intent(OUT) :: istart, iend, isize
 
-if (format==PHYSICAL_IN_X) then
-istart = sp%zst
-iend   = sp%zen
-isize  = sp%zsz
-else if (format==PHYSICAL_IN_Z) then
-istart = sp%xst
-iend   = sp%xen
-isize  = sp%xsz
-end if
+   if (format == PHYSICAL_IN_X) then
+      istart = sp%zst
+      iend = sp%zen
+      isize = sp%zsz
+   else if (format == PHYSICAL_IN_Z) then
+      istart = sp%xst
+      iend = sp%xen
+      isize = sp%xsz
+   end if
 
-return
+   return
 end subroutine decomp_2d_fft_get_size
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -218,15 +215,15 @@ end subroutine decomp_2d_fft_get_size
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 function decomp_2d_fft_get_ph()
 
-implicit none
+   implicit none
 
-type(decomp_info), pointer :: decomp_2d_fft_get_ph
+   type(decomp_info), pointer :: decomp_2d_fft_get_ph
 
-if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
-                                              __LINE__, &
-                                             -1, &
-                                             'FFT library must be initialised first')
-decomp_2d_fft_get_ph => ph
+   if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  -1, &
+                                                  'FFT library must be initialised first')
+   decomp_2d_fft_get_ph => ph
 
 end function decomp_2d_fft_get_ph
 
@@ -237,14 +234,14 @@ end function decomp_2d_fft_get_ph
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 function decomp_2d_fft_get_sp()
 
-implicit none
+   implicit none
 
-type(decomp_info), pointer :: decomp_2d_fft_get_sp
+   type(decomp_info), pointer :: decomp_2d_fft_get_sp
 
-if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
-                                              __LINE__, &
-                                             -1, &
-                                             'FFT library must be initialised first')
-decomp_2d_fft_get_sp => sp
+   if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  -1, &
+                                                  'FFT library must be initialised first')
+   decomp_2d_fft_get_sp => sp
 
 end function decomp_2d_fft_get_sp
diff --git a/src/fft_common_3d.f90 b/src/fft_common_3d.f90
index 3c0abc1b..eb809b37 100644
--- a/src/fft_common_3d.f90
+++ b/src/fft_common_3d.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,290 +10,288 @@
 !=======================================================================
 
 ! This file contains 3D c2c/r2c/c2r transform subroutines which are
-! identical for several FFT engines 
+! identical for several FFT engines
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_3d_c2c(in, out, isign)
 
-implicit none
+   implicit none
 
-complex(mytype), dimension(:,:,:), intent(INOUT) :: in
-complex(mytype), dimension(:,:,:), intent(OUT) :: out
-integer, intent(IN) :: isign
+   complex(mytype), dimension(:, :, :), intent(INOUT) :: in
+   complex(mytype), dimension(:, :, :), intent(OUT) :: out
+   integer, intent(IN) :: isign
 
-integer :: i, j, k
+   integer :: i, j, k
 
 #ifndef OVERWRITE
-complex(mytype), allocatable, dimension(:,:,:) :: wk1
+   complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
+   if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+       format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
 
 ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-call c2c_1m_x(in,isign,ph)
+      call c2c_1m_x(in, isign, ph)
 #else
-allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-do concurrent (k=1:ph%xsz(3), j=1:ph%xsz(2), i=1:ph%xsz(1))
-wk1(i,j,k) = in(i,j,k)
-end do
-call c2c_1m_x(wk1,isign,ph)
+      allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+      do concurrent(k=1:ph%xsz(3), j=1:ph%xsz(2), i=1:ph%xsz(1))
+         wk1(i, j, k) = in(i, j, k)
+      end do
+      call c2c_1m_x(wk1, isign, ph)
 #endif
 
 ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-if (dims(1)>1) then
+      if (dims(1) > 1) then
 #ifdef OVERWRITE
-call transpose_x_to_y(in,wk2_c2c,ph)
+         call transpose_x_to_y(in, wk2_c2c, ph)
 #else
-call transpose_x_to_y(wk1,wk2_c2c,ph)
+         call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-call c2c_1m_y(wk2_c2c,isign,ph)
-else
+         call c2c_1m_y(wk2_c2c, isign, ph)
+      else
 #ifdef OVERWRITE
-call c2c_1m_y(in,isign,ph)
+         call c2c_1m_y(in, isign, ph)
 #else
-call c2c_1m_y(wk1,isign,ph)
+         call c2c_1m_y(wk1, isign, ph)
 #endif
-end if
+      end if
 
 ! ===== Swap Y --> Z; 1D FFTs in Z =====
-if (dims(1)>1) then
-call transpose_y_to_z(wk2_c2c,out,ph)
-else
+      if (dims(1) > 1) then
+         call transpose_y_to_z(wk2_c2c, out, ph)
+      else
 #ifdef OVERWRITE
-call transpose_y_to_z(in,out,ph)
+         call transpose_y_to_z(in, out, ph)
 #else
-call transpose_y_to_z(wk1,out,ph)
+         call transpose_y_to_z(wk1, out, ph)
 #endif
-end if
-call c2c_1m_z(out,isign,ph)
+      end if
+      call c2c_1m_z(out, isign, ph)
 
-else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-.OR. & 
-format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
+   else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+            .OR. &
+            format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
 
 ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-call c2c_1m_z(in,isign,ph)
+      call c2c_1m_z(in, isign, ph)
 #else
-allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-do concurrent (k=1:ph%zsz(3), j=1:ph%zsz(2), i=1:ph%zsz(1))
-wk1(i,j,k) = in(i,j,k)
-end do
-call c2c_1m_z(wk1,isign,ph)
+      allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+      do concurrent(k=1:ph%zsz(3), j=1:ph%zsz(2), i=1:ph%zsz(1))
+         wk1(i, j, k) = in(i, j, k)
+      end do
+      call c2c_1m_z(wk1, isign, ph)
 #endif
 
 ! ===== Swap Z --> Y; 1D FFTs in Y =====
-if (dims(1)>1) then
+      if (dims(1) > 1) then
 #ifdef OVERWRITE
-call transpose_z_to_y(in,wk2_c2c,ph)
+         call transpose_z_to_y(in, wk2_c2c, ph)
 #else
-call transpose_z_to_y(wk1,wk2_c2c,ph)
+         call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-call c2c_1m_y(wk2_c2c,isign,ph)
-else  ! out==wk2_c2c if 1D decomposition
+         call c2c_1m_y(wk2_c2c, isign, ph)
+      else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-call transpose_z_to_y(in,out,ph)
+         call transpose_z_to_y(in, out, ph)
 #else
-call transpose_z_to_y(wk1,out,ph)
+         call transpose_z_to_y(wk1, out, ph)
 #endif
-call c2c_1m_y(out,isign,ph)
-end if
+         call c2c_1m_y(out, isign, ph)
+      end if
 
 ! ===== Swap Y --> X; 1D FFTs in X =====
-if (dims(1)>1) then
-call transpose_y_to_x(wk2_c2c,out,ph)
-end if
-call c2c_1m_x(out,isign,ph)
+      if (dims(1) > 1) then
+         call transpose_y_to_x(wk2_c2c, out, ph)
+      end if
+      call c2c_1m_x(out, isign, ph)
 
-end if
+   end if
 
 #ifndef OVERWRITE
 ! Free memory
-if (allocated(wk1)) deallocate(wk1)
+   if (allocated(wk1)) deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-return
+   return
 end subroutine fft_3d_c2c
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_3d_r2c(in_r, out_c)
 
-implicit none
+   implicit none
 
-real(mytype), dimension(:,:,:), intent(IN) :: in_r
-complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
+   real(mytype), dimension(:, :, :), intent(IN) :: in_r
+   complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-if (format==PHYSICAL_IN_X) then
+   if (format == PHYSICAL_IN_X) then
 
 ! ===== 1D FFTs in X =====
-call r2c_1m_x(in_r,wk13)
+      call r2c_1m_x(in_r, wk13)
 
 ! ===== Swap X --> Y; 1D FFTs in Y =====
-if (dims(1)>1) then
-call transpose_x_to_y(wk13,wk2_r2c,sp)
-call c2c_1m_y(wk2_r2c,-1,sp)
-else
-call c2c_1m_y(wk13,-1,sp)
-end if
+      if (dims(1) > 1) then
+         call transpose_x_to_y(wk13, wk2_r2c, sp)
+         call c2c_1m_y(wk2_r2c, -1, sp)
+      else
+         call c2c_1m_y(wk13, -1, sp)
+      end if
 
 ! ===== Swap Y --> Z; 1D FFTs in Z =====
-if (dims(1)>1) then
-call transpose_y_to_z(wk2_r2c,out_c,sp)
-else
-call transpose_y_to_z(wk13,out_c,sp)
-end if
-call c2c_1m_z(out_c,-1,sp)
+      if (dims(1) > 1) then
+         call transpose_y_to_z(wk2_r2c, out_c, sp)
+      else
+         call transpose_y_to_z(wk13, out_c, sp)
+      end if
+      call c2c_1m_z(out_c, -1, sp)
 
-else if (format==PHYSICAL_IN_Z) then
+   else if (format == PHYSICAL_IN_Z) then
 
 ! ===== 1D FFTs in Z =====
-call r2c_1m_z(in_r,wk13)
+      call r2c_1m_z(in_r, wk13)
 
 ! ===== Swap Z --> Y; 1D FFTs in Y =====
-if (dims(1)>1) then
-call transpose_z_to_y(wk13,wk2_r2c,sp)
-call c2c_1m_y(wk2_r2c,-1,sp)
-else  ! out_c==wk2_r2c if 1D decomposition
-call transpose_z_to_y(wk13,out_c,sp)
-call c2c_1m_y(out_c,-1,sp)
-end if
+      if (dims(1) > 1) then
+         call transpose_z_to_y(wk13, wk2_r2c, sp)
+         call c2c_1m_y(wk2_r2c, -1, sp)
+      else  ! out_c==wk2_r2c if 1D decomposition
+         call transpose_z_to_y(wk13, out_c, sp)
+         call c2c_1m_y(out_c, -1, sp)
+      end if
 
 ! ===== Swap Y --> X; 1D FFTs in X =====
-if (dims(1)>1) then
-call transpose_y_to_x(wk2_r2c,out_c,sp)
-end if
-call c2c_1m_x(out_c,-1,sp)
+      if (dims(1) > 1) then
+         call transpose_y_to_x(wk2_r2c, out_c, sp)
+      end if
+      call c2c_1m_x(out_c, -1, sp)
 
-end if
+   end if
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-return
+   return
 end subroutine fft_3d_r2c
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 subroutine fft_3d_c2r(in_c, out_r)
 
-implicit none
+   implicit none
 
-complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
-real(mytype), dimension(:,:,:), intent(OUT) :: out_r
+   complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
+   real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
-integer :: i, j, k
+   integer :: i, j, k
 
 #ifndef OVERWRITE
-complex(mytype), allocatable, dimension(:,:,:) :: wk1
+   complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+   if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-if (format==PHYSICAL_IN_X) then
+   if (format == PHYSICAL_IN_X) then
 
 ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-call c2c_1m_z(in_c,1,sp)       
+      call c2c_1m_z(in_c, 1, sp)
 #else
-allocate(wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-do concurrent (k=1:sp%zsz(3), j=1:sp%zsz(2), i=1:sp%zsz(1))
-wk1(i,j,k) = in_c(i,j,k)
-end do
-call c2c_1m_z(wk1,1,sp)
+      allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+      do concurrent(k=1:sp%zsz(3), j=1:sp%zsz(2), i=1:sp%zsz(1))
+         wk1(i, j, k) = in_c(i, j, k)
+      end do
+      call c2c_1m_z(wk1, 1, sp)
 #endif
 
 ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-call transpose_z_to_y(in_c,wk2_r2c,sp)
+      call transpose_z_to_y(in_c, wk2_r2c, sp)
 #else
-call transpose_z_to_y(wk1,wk2_r2c,sp)
+      call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-call c2c_1m_y(wk2_r2c,1,sp)
+      call c2c_1m_y(wk2_r2c, 1, sp)
 
 ! ===== Swap Y --> X; 1D FFTs in X =====
-if (dims(1)>1) then
-call transpose_y_to_x(wk2_r2c,wk13,sp)
-call c2r_1m_x(wk13,out_r)
-else
-call c2r_1m_x(wk2_r2c,out_r)
-end if
+      if (dims(1) > 1) then
+         call transpose_y_to_x(wk2_r2c, wk13, sp)
+         call c2r_1m_x(wk13, out_r)
+      else
+         call c2r_1m_x(wk2_r2c, out_r)
+      end if
 
-else if (format==PHYSICAL_IN_Z) then
+   else if (format == PHYSICAL_IN_Z) then
 
 ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-call c2c_1m_x(in_c,1,sp)
+      call c2c_1m_x(in_c, 1, sp)
 #else
-allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-do concurrent (k=1:sp%xsz(3), j=1:sp%xsz(2), i=1:sp%xsz(1))
-wk1(i,j,k) = in_c(i,j,k)
-end do
-call c2c_1m_x(wk1,1,sp)
+      allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+      do concurrent(k=1:sp%xsz(3), j=1:sp%xsz(2), i=1:sp%xsz(1))
+         wk1(i, j, k) = in_c(i, j, k)
+      end do
+      call c2c_1m_x(wk1, 1, sp)
 #endif
 
 ! ===== Swap X --> Y; 1D FFTs in Y =====
-if (dims(1)>1) then
+      if (dims(1) > 1) then
 #ifdef OVERWRITE
-call transpose_x_to_y(in_c,wk2_r2c,sp)
+         call transpose_x_to_y(in_c, wk2_r2c, sp)
 #else
-call transpose_x_to_y(wk1,wk2_r2c,sp)
+         call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-call c2c_1m_y(wk2_r2c,1,sp)
-else  ! in_c==wk2_r2c if 1D decomposition
+         call c2c_1m_y(wk2_r2c, 1, sp)
+      else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-call c2c_1m_y(in_c,1,sp)
+         call c2c_1m_y(in_c, 1, sp)
 #else
-call c2c_1m_y(wk1,1,sp)
+         call c2c_1m_y(wk1, 1, sp)
 #endif
-end if
+      end if
 
 ! ===== Swap Y --> Z; 1D FFTs in Z =====
-if (dims(1)>1) then
-call transpose_y_to_z(wk2_r2c,wk13,sp)
-else
+      if (dims(1) > 1) then
+         call transpose_y_to_z(wk2_r2c, wk13, sp)
+      else
 #ifdef OVERWRITE
-call transpose_y_to_z(in_c,wk13,sp)
+         call transpose_y_to_z(in_c, wk13, sp)
 #else
-call transpose_y_to_z(wk1,wk13,sp)
+         call transpose_y_to_z(wk1, wk13, sp)
 #endif
-end if
-call c2r_1m_z(wk13,out_r)
+      end if
+      call c2r_1m_z(wk13, out_r)
 
-end if
+   end if
 
 #ifndef OVERWRITE
 ! Free memory
-if (allocated(wk1)) deallocate(wk1)
+   if (allocated(wk1)) deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+   if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-return
+   return
 end subroutine fft_3d_c2r
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 0622f958..5f7575df 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,1032 +13,1022 @@
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use iso_c_binding
-  use cudafor
-  use cufft
+   use decomp_2d  ! 2D decomposition module
+   use iso_c_binding
+   use cudafor
+   use cufft
 
-  implicit none
+   implicit none
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
-  ! engine-specific global variables
-  ! integer, save :: plan_type = FFTW_MEASURE
+   ! engine-specific global variables
+   ! integer, save :: plan_type = FFTW_MEASURE
 
-  ! FFTW plans
-  ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
-  ! For c2c transforms: 
-  !     use plan(-1,j) for  forward transform; 
-  !     use plan( 1,j) for backward transform;
-  ! For r2c/c2r transforms:
-  !     use plan(0,j) for r2c transforms;
-  !     use plan(2,j) for c2r transforms;
-  integer*4, save :: plan(-1:2,3)
-  complex*8, device, allocatable, dimension(:) :: cufft_workspace
+   ! FFTW plans
+   ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
+   ! For c2c transforms:
+   !     use plan(-1,j) for  forward transform;
+   !     use plan( 1,j) for backward transform;
+   ! For r2c/c2r transforms:
+   !     use plan(0,j) for r2c transforms;
+   !     use plan(2,j) for c2r transforms;
+   integer*4, save :: plan(-1:2, 3)
+   complex*8, device, allocatable, dimension(:) :: cufft_workspace
 
-  ! common code used for all engines, including global variables, 
-  ! generic interface definitions and several subroutines
+   ! common code used for all engines, including global variables,
+   ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2C case
-  subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
-    
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-   
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp%xsz(1),     &
-                                      decomp%xsz(1),1,decomp%xsz(1),     &
-                                      decomp%xsz(1),1,decomp%xsz(1),     &
-                                      cufft_type,decomp%xsz(2)*decomp%xsz(3),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2c_1m_x_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in Y direction: C2C case
-  subroutine c2c_1m_y_plan(plan1, decomp, cufft_type,worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: cufft_type
-
-    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-    ! done one Z-plane at a time. So plan for 2D data sets here.
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-   
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp%ysz(2),     &
-                                      decomp%ysz(2),decomp%ysz(1),1,     &
-                                      decomp%ysz(2),decomp%ysz(1),1,     &
-                                      cufft_type,decomp%ysz(1),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2c_1m_y_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in Z direction: C2C case
-  subroutine c2c_1m_z_plan(plan1, decomp, cufft_type,worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp%zsz(3),                           &
-                                      decomp%zsz(3),decomp%zsz(1)*decomp%zsz(2),1,     &
-                                      decomp%zsz(3),decomp%zsz(1)*decomp%zsz(2),1,     &
-                                      cufft_type,decomp%zsz(1)*decomp%zsz(2),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2c_1m_z_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
-  subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp_ph%xsz(1),     &
-                                      decomp_ph%xsz(1),1,decomp_ph%xsz(1),     &
-                                      decomp_sp%xsz(1),1,decomp_sp%xsz(1),     &
-                                      cufft_type,decomp_ph%xsz(2)*decomp_ph%xsz(3),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine r2c_1m_x_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
-  subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp_ph%xsz(1),     &
-                                      decomp_sp%xsz(1),1,decomp_sp%xsz(1),     &
-                                      decomp_ph%xsz(1),1,decomp_ph%xsz(1),     &
-                                      cufft_type,decomp_ph%xsz(2)*decomp_ph%xsz(3),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2r_1m_x_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
-  subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp_ph%zsz(3),                           &
-                                      decomp_ph%zsz(3),decomp_ph%zsz(1)*decomp_ph%zsz(2),1,     &
-                                      decomp_sp%zsz(3),decomp_sp%zsz(1)*decomp_sp%zsz(2),1,     &
-                                      cufft_type,decomp_ph%zsz(1)*decomp_ph%zsz(2),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine r2c_1m_z_plan
-
-  ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
-  subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
-
-    implicit none
-
-    integer*4, intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    integer, intent(IN) :: cufft_type
-
-    integer :: istat
-    integer(int_ptr_kind()), intent(out) :: worksize
-    integer, pointer :: null_fptr
-    call c_f_pointer( c_null_ptr, null_fptr )
-
-    istat = cufftCreate(plan1)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
-    istat = cufftSetAutoAllocation(plan1,0)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
-    istat = cufftMakePlanMany(plan1,1,decomp_ph%zsz(3),                           &
-                                      decomp_sp%zsz(3),decomp_sp%zsz(1)*decomp_sp%zsz(2),1,     &
-                                      decomp_ph%zsz(3),decomp_ph%zsz(1)*decomp_ph%zsz(2),1,     &
-                                      cufft_type,decomp_ph%zsz(1)*decomp_ph%zsz(2),worksize)
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
-
-  end subroutine c2r_1m_z_plan
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2C case
+   subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp%xsz(1), &
+                                decomp%xsz(1), 1, decomp%xsz(1), &
+                                decomp%xsz(1), 1, decomp%xsz(1), &
+                                cufft_type, decomp%xsz(2)*decomp%xsz(3), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2c_1m_x_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in Y direction: C2C case
+   subroutine c2c_1m_y_plan(plan1, decomp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: cufft_type
+
+      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+      ! done one Z-plane at a time. So plan for 2D data sets here.
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp%ysz(2), &
+                                decomp%ysz(2), decomp%ysz(1), 1, &
+                                decomp%ysz(2), decomp%ysz(1), 1, &
+                                cufft_type, decomp%ysz(1), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2c_1m_y_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in Z direction: C2C case
+   subroutine c2c_1m_z_plan(plan1, decomp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp%zsz(3), &
+                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
+                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
+                                cufft_type, decomp%zsz(1)*decomp%zsz(2), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2c_1m_z_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
+   subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
+                                decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine r2c_1m_x_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
+   subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
+                                decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2r_1m_x_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: R2C case
+   subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
+                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
+                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
+                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine r2c_1m_z_plan
+
+   ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2R case
+   subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
+
+      implicit none
+
+      integer*4, intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      integer, intent(IN) :: cufft_type
+
+      integer :: istat
+      integer(int_ptr_kind()), intent(out) :: worksize
+      integer, pointer :: null_fptr
+      call c_f_pointer(c_null_ptr, null_fptr)
+
+      istat = cufftCreate(plan1)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftCreate")
+      istat = cufftSetAutoAllocation(plan1, 0)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
+      istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
+                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
+                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
+                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
+
+   end subroutine c2r_1m_z_plan
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
+   subroutine init_fft_engine
+
+      implicit none
 
-    implicit none
+      !integer*4 :: cufft_ws, ws
+      integer(int_ptr_kind()) :: cufft_ws, ws
+      integer   :: i, j, istat
 
-    !integer*4 :: cufft_ws, ws
-    integer(int_ptr_kind()) :: cufft_ws, ws
-    integer   :: i, j, istat
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the New cuFFT engine *****'
+         write (*, *) ' '
+      end if
 
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the New cuFFT engine *****'
-       write(*,*) ' '
-    end if
-    
-     cufft_ws = 0 
+      cufft_ws = 0
 #ifdef DOUBLE_PREC
-    if (format == PHYSICAL_IN_X) then
-       ! For C2C transforms
-       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       ! For R2C/C2R tranforms
-       call r2c_1m_x_plan(plan(0,1), ph, sp, CUFFT_D2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(0,3), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(2,3), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2r_1m_x_plan(plan(2,1), sp, ph, CUFFT_Z2D,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-    else if (format == PHYSICAL_IN_Z) then
-
-       ! For C2C transforms
-       write(*,*) 'Create the plans'
-       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-       ! For R2C/C2R tranforms
-       call r2c_1m_z_plan(plan(0,3), ph, sp, CUFFT_D2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(0,1), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(2,1), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_Z2Z,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2r_1m_z_plan(plan(2,3), sp, ph, CUFFT_Z2D,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-    end if
+      if (format == PHYSICAL_IN_X) then
+         ! For C2C transforms
+         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         ! For R2C/C2R tranforms
+         call r2c_1m_x_plan(plan(0, 1), ph, sp, CUFFT_D2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(0, 3), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(2, 3), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2r_1m_x_plan(plan(2, 1), sp, ph, CUFFT_Z2D, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! For C2C transforms
+         write (*, *) 'Create the plans'
+         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_z_plan(plan(0, 3), ph, sp, CUFFT_D2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(0, 1), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(2, 1), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_Z2Z, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2r_1m_z_plan(plan(2, 3), sp, ph, CUFFT_Z2D, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+      end if
 #else
-    if (format == PHYSICAL_IN_X) then
-       ! For C2C transforms
-       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       ! For R2C/C2R tranforms
-       call r2c_1m_x_plan(plan(0,1), ph, sp, CUFFT_R2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(0,3), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan(2,3), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2r_1m_x_plan(plan(2,1), sp, ph, CUFFT_C2R,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-    else if (format == PHYSICAL_IN_Z) then
-
-       ! For C2C transforms
-       write(*,*) 'Create the plans on rank ', nproc
-       call c2c_1m_z_plan(plan(-1,3), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(-1,2), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(-1,1), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan( 1,1), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan( 1,2), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_z_plan(plan( 1,3), ph, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-       ! For R2C/C2R tranforms
-       call r2c_1m_z_plan(plan(0,3), ph, sp, CUFFT_R2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(0,2), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(0,1), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_x_plan(plan(2,1), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2c_1m_y_plan(plan(2,2), sp, CUFFT_C2C,ws)
-       cufft_ws = max (cufft_ws,ws)
-       call c2r_1m_z_plan(plan(2,3), sp, ph, CUFFT_C2R,ws)
-       cufft_ws = max (cufft_ws,ws)
-
-    end if
+      if (format == PHYSICAL_IN_X) then
+         ! For C2C transforms
+         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         ! For R2C/C2R tranforms
+         call r2c_1m_x_plan(plan(0, 1), ph, sp, CUFFT_R2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(0, 3), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(2, 3), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2r_1m_x_plan(plan(2, 1), sp, ph, CUFFT_C2R, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! For C2C transforms
+         write (*, *) 'Create the plans on rank ', nproc
+         call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(-1, 1), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(1, 1), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(1, 2), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_z_plan(plan(1, 3), ph, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_z_plan(plan(0, 3), ph, sp, CUFFT_R2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(0, 2), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(0, 1), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_x_plan(plan(2, 1), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2c_1m_y_plan(plan(2, 2), sp, CUFFT_C2C, ws)
+         cufft_ws = max(cufft_ws, ws)
+         call c2r_1m_z_plan(plan(2, 3), sp, ph, CUFFT_C2R, ws)
+         cufft_ws = max(cufft_ws, ws)
+
+      end if
 #endif
-    allocate(cufft_workspace(cufft_ws))
-    do j=1,3
-       do i=-1,2
-         istat = cufftSetWorkArea(plan(i,j),cufft_workspace)
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetWorkArea")
-      enddo
-    enddo
-
-  end subroutine init_fft_engine
+      allocate (cufft_workspace(cufft_ws))
+      do j = 1, 3
+         do i = -1, 2
+            istat = cufftSetWorkArea(plan(i, j), cufft_workspace)
+            if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetWorkArea")
+         end do
+      end do
 
+   end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
-
-    implicit none
+   subroutine finalize_fft_engine
 
-    integer :: i,j, istat
+      implicit none
 
-    do j=1,3
-       do i=-1,2
-          istat = cufftDestroy(plan(i,j))
-          if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftDestroy")
-       end do
-    end do
+      integer :: i, j, istat
 
-  end subroutine finalize_fft_engine
+      do j = 1, 3
+         do i = -1, 2
+            istat = cufftDestroy(plan(i, j))
+            if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftDestroy")
+         end do
+      end do
 
+   end subroutine finalize_fft_engine
 
-  ! Following routines calculate multiple one-dimensional FFTs to form 
-  ! the basis of three-dimensional FFTs.
+   ! Following routines calculate multiple one-dimensional FFTs to form
+   ! the basis of three-dimensional FFTs.
 
-  ! c2c transform, multiple 1D FFTs in x direction
-  subroutine c2c_1m_x(inout, isign, plan1)
+   ! c2c transform, multiple 1D FFTs in x direction
+   subroutine c2c_1m_x(inout, isign, plan1)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    integer*4, intent(IN) :: plan1
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      integer*4, intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), allocatable :: output
-    integer :: istat
+      complex(mytype), dimension(:, :, :), allocatable :: output
+      integer :: istat
 
-    allocate(output,mold=inout)
+      allocate (output, mold=inout)
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(inout,output)
-    istat = cufftExecZ2Z(plan1, inout, output,isign)
-    !$acc end host_data
+      !$acc host_data use_device(inout,output)
+      istat = cufftExecZ2Z(plan1, inout, output, isign)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(inout,output)
-    istat = cufftExecC2C(plan1, inout, output,isign)
-    !$acc end host_data
+      !$acc host_data use_device(inout,output)
+      istat = cufftExecC2C(plan1, inout, output, isign)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-    !$acc kernels
-    inout = output
-    !$acc end kernels 
-    deallocate(output)
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
+      !$acc kernels
+      inout = output
+      !$acc end kernels
+      deallocate (output)
 
-  end subroutine c2c_1m_x
+   end subroutine c2c_1m_x
 
+   ! c2c transform, multiple 1D FFTs in y direction
+   subroutine c2c_1m_y(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in y direction
-  subroutine c2c_1m_y(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      integer*4, intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    integer*4, intent(IN) :: plan1
+      complex(mytype), dimension(:, :, :), allocatable :: output
+      integer :: s3, k, istat
 
-    complex(mytype), dimension(:,:,:), allocatable :: output
-    integer :: s3, k, istat
-    
-    allocate(output,mold=inout)
-    ! transform on one Z-plane at a time
-    s3 = size(inout,3)
-    do k=1,s3
+      allocate (output, mold=inout)
+      ! transform on one Z-plane at a time
+      s3 = size(inout, 3)
+      do k = 1, s3
 #ifdef DOUBLE_PREC
-       !$acc host_data use_device(inout,output)
-       istat = cufftExecZ2Z(plan1, inout(:,:,k), output(:,:,k),isign)
-       !$acc end host_data
+         !$acc host_data use_device(inout,output)
+         istat = cufftExecZ2Z(plan1, inout(:, :, k), output(:, :, k), isign)
+         !$acc end host_data
 #else
-       !$acc host_data use_device(inout,output)
-       istat = cufftExecC2C(plan1, inout(:,:,k), output(:,:,k),isign)
-       !$acc end host_data
+         !$acc host_data use_device(inout,output)
+         istat = cufftExecC2C(plan1, inout(:, :, k), output(:, :, k), isign)
+         !$acc end host_data
 #endif
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-    enddo
-    !$acc kernels
-    inout = output
-    !$acc end kernels 
-    deallocate(output)
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
+      end do
+      !$acc kernels
+      inout = output
+      !$acc end kernels
+      deallocate (output)
 
-  end subroutine c2c_1m_y
+   end subroutine c2c_1m_y
 
-  ! c2c transform, multiple 1D FFTs in z direction
-  subroutine c2c_1m_z(inout, isign, plan1)
+   ! c2c transform, multiple 1D FFTs in z direction
+   subroutine c2c_1m_z(inout, isign, plan1)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    integer*4, intent(IN) :: plan1
-    
-    complex(mytype), dimension(:,:,:), allocatable :: output
-    integer :: istat
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      integer*4, intent(IN) :: plan1
 
-    allocate(output,mold=inout)
+      complex(mytype), dimension(:, :, :), allocatable :: output
+      integer :: istat
 
+      allocate (output, mold=inout)
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(inout,output)
-    istat = cufftExecZ2Z(plan1, inout, output,isign)
-    !$acc end host_data
+      !$acc host_data use_device(inout,output)
+      istat = cufftExecZ2Z(plan1, inout, output, isign)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(inout,output)
-    istat = cufftExecC2C(plan1, inout, output,isign)
-    !$acc end host_data
+      !$acc host_data use_device(inout,output)
+      istat = cufftExecC2C(plan1, inout, output, isign)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-    !$acc kernels
-    inout = output
-    !$acc end kernels
-    deallocate(output)
-
-  end subroutine c2c_1m_z
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
+      !$acc kernels
+      inout = output
+      !$acc end kernels
+      deallocate (output)
 
-  ! r2c transform, multiple 1D FFTs in x direction
-  subroutine r2c_1m_x(input, output)
+   end subroutine c2c_1m_z
 
-    implicit none
+   ! r2c transform, multiple 1D FFTs in x direction
+   subroutine r2c_1m_x(input, output)
 
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
-    integer :: istat
+      implicit none
 
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+      integer :: istat
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(input,output)
-    istat =  cufftExecD2Z(plan(0,1), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecD2Z(plan(0, 1), input, output)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(input,output)
-    istat =  cufftExecR2C(plan(0,1), input, output)
-    !$acc end host_data
-#endif    
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
+      !$acc host_data use_device(input,output)
+      istat = cufftExecR2C(plan(0, 1), input, output)
+      !$acc end host_data
+#endif
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
 
-  end subroutine r2c_1m_x
+   end subroutine r2c_1m_x
 
-  ! r2c transform, multiple 1D FFTs in z direction
-  subroutine r2c_1m_z(input, output)
+   ! r2c transform, multiple 1D FFTs in z direction
+   subroutine r2c_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN)     ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(IN)     ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
-    integer :: istat
+      integer :: istat
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(input,output)
-    istat = cufftExecD2Z(plan(0,3), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecD2Z(plan(0, 3), input, output)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(input,output)
-    istat = cufftExecR2C(plan(0,3), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecR2C(plan(0, 3), input, output)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecR2C/D2Z")
 
-  end subroutine r2c_1m_z
+   end subroutine r2c_1m_z
 
-  ! c2r transform, multiple 1D FFTs in x direction
-  subroutine c2r_1m_x(input, output)
+   ! c2r transform, multiple 1D FFTs in x direction
+   subroutine c2r_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT)    :: output
+      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT)    :: output
 
-    integer :: istat
+      integer :: istat
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(input,output)
-    istat = cufftExecZ2D(plan(2,1), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecZ2D(plan(2, 1), input, output)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(input,output)
-    istat = cufftExecC2R(plan(2,1), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecC2R(plan(2, 1), input, output)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
 
-  end subroutine c2r_1m_x
+   end subroutine c2r_1m_x
 
-  ! c2r transform, multiple 1D FFTs in z direction
-  subroutine c2r_1m_z(input, output)
+   ! c2r transform, multiple 1D FFTs in z direction
+   subroutine c2r_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(IN) :: input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
-    integer :: istat
+      integer :: istat
 
 #ifdef DOUBLE_PREC
-    !$acc host_data use_device(input,output)
-    istat = cufftExecZ2D(plan(2,3), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecZ2D(plan(2, 3), input, output)
+      !$acc end host_data
 #else
-    !$acc host_data use_device(input,output)
-    istat = cufftExecC2R(plan(2,3), input, output)
-    !$acc end host_data
+      !$acc host_data use_device(input,output)
+      istat = cufftExecC2R(plan(2, 3), input, output)
+      !$acc end host_data
 #endif
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
-
-  end subroutine c2r_1m_z
-
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2R/Z2D")
 
+   end subroutine c2r_1m_z
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D FFT - complex to complex
+   ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2c(in, out, isign)
+   subroutine fft_3d_c2c(in, out, isign)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out
-    integer, intent(IN) :: isign
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out
+      integer, intent(IN) :: isign
 
 #ifndef OVERWRITE
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
+      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
 
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in,isign,plan(isign,1))
+         call c2c_1m_x(in, isign, plan(isign, 1))
 #else
-       allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-       wk1 = in
-       call c2c_1m_x(wk1,isign,plan(isign,1))
+         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+         wk1 = in
+         call c2c_1m_x(wk1, isign, plan(isign, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-       if (dims(1)>1) then
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in,wk2_c2c,ph)
+            call transpose_x_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_x_to_y(wk1,wk2_c2c,ph)
+            call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else
 #ifdef OVERWRITE
-          call c2c_1m_y(in,isign,plan(isign,2))
+            call c2c_1m_y(in, isign, plan(isign, 2))
 #else
-          call c2c_1m_y(wk1,isign,plan(isign,2))
+            call c2c_1m_y(wk1, isign, plan(isign, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_c2c,out,ph)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_c2c, out, ph)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in,out,ph)
+            call transpose_y_to_z(in, out, ph)
 #else
-          call transpose_y_to_z(wk1,out,ph)
+            call transpose_y_to_z(wk1, out, ph)
 #endif
-       end if
-       call c2c_1m_z(out,isign,plan(isign,3))
+         end if
+         call c2c_1m_z(out, isign, plan(isign, 3))
 
-    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-         .OR. & 
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
+      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+               .OR. &
+               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in,isign,plan(isign,3))
+         call c2c_1m_z(in, isign, plan(isign, 3))
 #else
-       allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-       wk1 = in
-       call c2c_1m_z(wk1,isign,plan(isign,3))
+         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+         wk1 = in
+         call c2c_1m_z(wk1, isign, plan(isign, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,wk2_c2c,ph)
+            call transpose_z_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_z_to_y(wk1,wk2_c2c,ph)
+            call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else  ! out==wk2_c2c if 1D decomposition
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,out,ph)
+            call transpose_z_to_y(in, out, ph)
 #else
-          call transpose_z_to_y(wk1,out,ph)
+            call transpose_z_to_y(wk1, out, ph)
 #endif
-          call c2c_1m_y(out,isign,plan(isign,2))
-       end if
+            call c2c_1m_y(out, isign, plan(isign, 2))
+         end if
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_c2c,out,ph)
-       end if
-       call c2c_1m_x(out,isign,plan(isign,1))
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_c2c, out, ph)
+         end if
+         call c2c_1m_x(out, isign, plan(isign, 1))
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    deallocate (wk1)
+      deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-  end subroutine fft_3d_c2c
-
+   end subroutine fft_3d_c2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D forward FFT - real to complex
+   ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_r2c(in_r, out_c)
-    !use nvtx
-    implicit none
+   subroutine fft_3d_r2c(in_r, out_c)
+      !use nvtx
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: in_r
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
-    integer :: i, j ,k
-    integer, dimension(3) :: dim3d
+      real(mytype), dimension(:, :, :), intent(IN) :: in_r
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
+      integer :: i, j, k
+      integer, dimension(3) :: dim3d
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in X =====
-       call r2c_1m_x(in_r,wk13)
+         ! ===== 1D FFTs in X =====
+         call r2c_1m_x(in_r, wk13)
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_x_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else
-          call c2c_1m_y(wk13,-1,plan(0,2))
-       end if
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_x_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else
+            call c2c_1m_y(wk13, -1, plan(0, 2))
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,out_c,sp)
-       else
-          call transpose_y_to_z(wk13,out_c,sp)
-       end if
-       call c2c_1m_z(out_c,-1,plan(0,3))
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, out_c, sp)
+         else
+            call transpose_y_to_z(wk13, out_c, sp)
+         end if
+         call c2c_1m_z(out_c, -1, plan(0, 3))
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in Z =====
-       !call nvtxStartRange("Z r2c_1m_z")
+         ! ===== 1D FFTs in Z =====
+         !call nvtxStartRange("Z r2c_1m_z")
 #ifdef DEBUG
-       dim3d = shape(in_r)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_r(i,j,k))
+         dim3d = shape(in_r)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_r(i, j, k))
+               end do
             end do
          end do
-       end do
 #endif
-       call r2c_1m_z(in_r,wk13)
-
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          !call nvtxStartRange("Z1 transpose_z_to_y")
-          call transpose_z_to_y(wk13,wk2_r2c,sp)
-          !call nvtxEndRange
-          !call nvtxStartRange("Z1 c2c_1m_y")
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-          !call nvtxEndRange
+         call r2c_1m_z(in_r, wk13)
+
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            !call nvtxStartRange("Z1 transpose_z_to_y")
+            call transpose_z_to_y(wk13, wk2_r2c, sp)
+            !call nvtxEndRange
+            !call nvtxStartRange("Z1 c2c_1m_y")
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+            !call nvtxEndRange
 #ifdef DEBUG
-          write(*,*) 'c2c_1m_y'
-          dim3d = shape(wk2_r2c)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i,j,k)),&
-                                aimag(wk2_r2c(i,j,k))
+            write (*, *) 'c2c_1m_y'
+            dim3d = shape(wk2_r2c)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
+                        aimag(wk2_r2c(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
-       else  ! out_c==wk2_r2c if 1D decomposition
-          !call nvtxStartRange("Z transpose_z_to_y")
-          call transpose_z_to_y(wk13,out_c,sp)
-          !call nvtxEndRange
-          !call nvtxStartRange("Z c2c_1m_y")
-          call c2c_1m_y(out_c,-1,plan(0,2))
-          !call nvtxEndRange
+         else  ! out_c==wk2_r2c if 1D decomposition
+            !call nvtxStartRange("Z transpose_z_to_y")
+            call transpose_z_to_y(wk13, out_c, sp)
+            !call nvtxEndRange
+            !call nvtxStartRange("Z c2c_1m_y")
+            call c2c_1m_y(out_c, -1, plan(0, 2))
+            !call nvtxEndRange
 #ifdef DEBUG
-          write(*,*) 'c2c_1m_y2'
-          dim3d = shape(out_c)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i,j,k)),&
-                                aimag(out_c(i,j,k))
+            write (*, *) 'c2c_1m_y2'
+            dim3d = shape(out_c)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
+                        aimag(out_c(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
-       end if
-
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          !call nvtxStartRange("Z1 transpose_y_to_x")
-          call transpose_y_to_x(wk2_r2c,out_c,sp)
-          !call nvtxEndRange
-       end if
-       !call nvtxStartRange("c2c_1m_x")
-       call c2c_1m_x(out_c,-1,plan(0,1))
-       !call nvtxEndRange
+         end if
+
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            !call nvtxStartRange("Z1 transpose_y_to_x")
+            call transpose_y_to_x(wk2_r2c, out_c, sp)
+            !call nvtxEndRange
+         end if
+         !call nvtxStartRange("c2c_1m_x")
+         call c2c_1m_x(out_c, -1, plan(0, 1))
+         !call nvtxEndRange
 #ifdef DEBUG
-       write(*,*) 'c2c_1m_x'
-       dim3d = shape(out_c)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i,j,k)),&
-                             aimag(out_c(i,j,k))
+         write (*, *) 'c2c_1m_x'
+         dim3d = shape(out_c)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
+                     aimag(out_c(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
-    end if
+      end if
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-  end subroutine fft_3d_r2c
-
+   end subroutine fft_3d_r2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D inverse FFT - complex to real
+   ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2r(in_c, out_r)
+   subroutine fft_3d_c2r(in_c, out_r)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
-    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
-    integer :: i,j,k
-    integer, dimension(3) :: dim3d
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
+      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
+      integer :: i, j, k
+      integer, dimension(3) :: dim3d
 #ifndef OVERWRITE
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in_c,1,plan(2,3))       
+         call c2c_1m_z(in_c, 1, plan(2, 3))
 #else
-       allocate (wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       wk1 = in_c
-       call c2c_1m_z(wk1,1,plan(2,3))
+         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         wk1 = in_c
+         call c2c_1m_z(wk1, 1, plan(2, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-       call transpose_z_to_y(in_c,wk2_r2c,sp)
+         call transpose_z_to_y(in_c, wk2_r2c, sp)
 #else
-       call transpose_z_to_y(wk1,wk2_r2c,sp)
+         call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-       call c2c_1m_y(wk2_r2c,1,plan(2,2))
+         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,wk13,sp)
-          call c2r_1m_x(wk13,out_r)
-       else
-          call c2r_1m_x(wk2_r2c,out_r)
-       end if
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, wk13, sp)
+            call c2r_1m_x(wk13, out_r)
+         else
+            call c2r_1m_x(wk2_r2c, out_r)
+         end if
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 #ifdef DEBUG
-       write(*,*) 'Back Init c2c_1m_x line 788'
-       dim3d = shape(in_c)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i,j,k)),&
-                             aimag(in_c(i,j,k))
+         write (*, *) 'Back Init c2c_1m_x line 788'
+         dim3d = shape(in_c)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
+                     aimag(in_c(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in_c,1,plan(2,1))
+         call c2c_1m_x(in_c, 1, plan(2, 1))
 #ifdef DEBUG
-       write(*,*) 'Back c2c_1m_x overwrite line 804'
-       dim3d = shape(in_c)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i,j,k)),&
-                             aimag(in_c(i,j,k))
+         write (*, *) 'Back c2c_1m_x overwrite line 804'
+         dim3d = shape(in_c)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
+                     aimag(in_c(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
 #else
-       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       wk1 = in_c
-       call c2c_1m_x(wk1,1,plan(2,1))
+         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         wk1 = in_c
+         call c2c_1m_x(wk1, 1, plan(2, 1))
 #ifdef DEBUG
-       write(*,*) 'Back2 c2c_1m_x line 821'
-       dim3d = shape(wk1)
-       do k = 1, dim3d(3),dim3d(1)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i,j,k)),&
-                             aimag(wk1(i,j,k))
+         write (*, *) 'Back2 c2c_1m_x line 821'
+         dim3d = shape(wk1)
+         do k = 1, dim3d(3), dim3d(1)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
+                     aimag(wk1(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in_c,wk2_r2c,sp)
+            call transpose_x_to_y(in_c, wk2_r2c, sp)
 #else
-          call transpose_x_to_y(wk1,wk2_r2c,sp)
+            call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-          call c2c_1m_y(wk2_r2c,1,plan(2,2))
+            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
 #ifdef DEBUG
-          write(*,*) 'Back c2c_1m_y line 844'
-          dim3d = shape(wk2_r2c)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i,j,k)),&
-                                aimag(wk2_r2c(i,j,k))
+            write (*, *) 'Back c2c_1m_y line 844'
+            dim3d = shape(wk2_r2c)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
+                        aimag(wk2_r2c(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
-       else  ! in_c==wk2_r2c if 1D decomposition
+         else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-          call c2c_1m_y(in_c,1,plan(2,2))
+            call c2c_1m_y(in_c, 1, plan(2, 2))
 #ifdef DEBUG
-          write(*,*) 'Back2 c2c_1m_y line 860'
-          dim3d = shape(in_c)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i,j,k)),&
-                                aimag(in_c(i,j,k))
+            write (*, *) 'Back2 c2c_1m_y line 860'
+            dim3d = shape(in_c)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
+                        aimag(in_c(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
 #else
-          call c2c_1m_y(wk1,1,plan(2,2))
+            call c2c_1m_y(wk1, 1, plan(2, 2))
 #ifdef DEBUG
-          write(*,*) 'Back3 c2c_1m_y line 875'
-          dim3d = shape(wk1)
-          do k = 1, dim3d(3),dim3d(3)/8
-            do j = 1, dim3d(2),dim3d(2)/8
-               do i = 1, dim3d(1),dim3d(1)/8
-                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i,j,k)),&
-                                aimag(wk1(i,j,k))
+            write (*, *) 'Back3 c2c_1m_y line 875'
+            dim3d = shape(wk1)
+            do k = 1, dim3d(3), dim3d(3)/8
+               do j = 1, dim3d(2), dim3d(2)/8
+                  do i = 1, dim3d(1), dim3d(1)/8
+                     print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
+                        aimag(wk1(i, j, k))
+                  end do
                end do
             end do
-          end do
-          write(*,*)
-          write(*,*)
+            write (*, *)
+            write (*, *)
 #endif
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,wk13,sp)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, wk13, sp)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in_c,wk13,sp)
+            call transpose_y_to_z(in_c, wk13, sp)
 #else
-          call transpose_y_to_z(wk1,wk13,sp)
+            call transpose_y_to_z(wk1, wk13, sp)
 #endif
-       end if
+         end if
 #ifdef DEBUG
-       write(*,*) 'Back2 after tr_y2z'
-       dim3d = shape(wk13)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-              print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk13(i,j,k)),&
-                               aimag(wk13(i,j,k))
+         write (*, *) 'Back2 after tr_y2z'
+         dim3d = shape(wk13)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk13(i, j, k)), &
+                     aimag(wk13(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
-       call c2r_1m_z(wk13,out_r)
+         call c2r_1m_z(wk13, out_r)
 #ifdef DEBUG
-       write(*,*) 'Back2 c2r_1m_z out_r line 902'
-       dim3d = shape(out_r)
-       do k = 1, dim3d(3),dim3d(3)/8
-         do j = 1, dim3d(2),dim3d(2)/8
-            do i = 1, dim3d(1),dim3d(1)/8
-                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_r(i,j,k))
+         write (*, *) 'Back2 c2r_1m_z out_r line 902'
+         dim3d = shape(out_r)
+         do k = 1, dim3d(3), dim3d(3)/8
+            do j = 1, dim3d(2), dim3d(2)/8
+               do i = 1, dim3d(1), dim3d(1)/8
+                  print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_r(i, j, k))
+               end do
             end do
          end do
-       end do
-       write(*,*)
-       write(*,*)
+         write (*, *)
+         write (*, *)
 #endif
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    deallocate (wk1)
+      deallocate (wk1)
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-  end subroutine fft_3d_c2r
-
+   end subroutine fft_3d_c2r
 
 end module decomp_2d_fft
 
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index 91cc0670..fb7646ba 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,733 +13,720 @@
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use iso_c_binding
+   use decomp_2d  ! 2D decomposition module
+   use iso_c_binding
 
-  implicit none
+   implicit none
 
-  include "fftw3.f"
+   include "fftw3.f"
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
-  ! engine-specific global variables
-  integer, save :: plan_type = FFTW_MEASURE
+   ! engine-specific global variables
+   integer, save :: plan_type = FFTW_MEASURE
 
-  ! FFTW plans
-  ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
-  ! For c2c transforms: 
-  !     use plan(-1,j) for  forward transform; 
-  !     use plan( 1,j) for backward transform;
-  ! For r2c/c2r transforms:
-  !     use plan(0,j) for r2c transforms;
-  !     use plan(2,j) for c2r transforms;
-  type(C_PTR), save :: plan(-1:2,3)
+   ! FFTW plans
+   ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
+   ! For c2c transforms:
+   !     use plan(-1,j) for  forward transform;
+   !     use plan( 1,j) for backward transform;
+   ! For r2c/c2r transforms:
+   !     use plan(0,j) for r2c transforms;
+   !     use plan(2,j) for c2r transforms;
+   type(C_PTR), save :: plan(-1:2, 3)
 
-  ! This is defined in fftw3.f03 but not in fftw3.f
-  interface
-    subroutine fftw_cleanup() bind(C, name='fftw_cleanup')
-      import
-    end subroutine fftw_cleanup
-  end interface
+   ! This is defined in fftw3.f03 but not in fftw3.f
+   interface
+      subroutine fftw_cleanup() bind(C, name='fftw_cleanup')
+         import
+      end subroutine fftw_cleanup
+   end interface
 
-  ! common code used for all engines, including global variables, 
-  ! generic interface definitions and several subroutines
+   ! common code used for all engines, including global variables,
+   ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
-  subroutine c2c_1m_x_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
+   subroutine c2c_1m_x_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
-    complex(mytype), allocatable, dimension(:,:,:) :: a1
+      complex(mytype), allocatable, dimension(:, :, :) :: a1
 
-    allocate(a1(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)))
+      allocate (a1(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)))
 
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-         decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
-         isign, plan_type)
+      call dfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
+                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                               decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
+                               isign, plan_type)
 #else
-    call sfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-         decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
-         isign, plan_type)
+      call sfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
+                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                               decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
+                               isign, plan_type)
 #endif
 
-    deallocate(a1)
+      deallocate (a1)
 
-    return
-  end subroutine c2c_1m_x_plan
+      return
+   end subroutine c2c_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
-  subroutine c2c_1m_y_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
+   subroutine c2c_1m_y_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
-    complex(mytype), allocatable, dimension(:,:) :: a1
+      complex(mytype), allocatable, dimension(:, :) :: a1
 
-    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-    ! done one Z-plane at a time. So plan for 2D data sets here.
+      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+      ! done one Z-plane at a time. So plan for 2D data sets here.
 
-    allocate(a1(decomp%ysz(1),decomp%ysz(2)))
+      allocate (a1(decomp%ysz(1), decomp%ysz(2)))
 
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
-         a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
-         decomp%ysz(1), 1, isign, plan_type)
+      call dfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
+                               a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
+                               decomp%ysz(1), 1, isign, plan_type)
 #else
-    call sfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
-         a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
-         decomp%ysz(1), 1, isign, plan_type)
+      call sfftw_plan_many_dft(plan1, 1, decomp%ysz(2), decomp%ysz(1), &
+                               a1, decomp%ysz(2), decomp%ysz(1), 1, a1, decomp%ysz(2), &
+                               decomp%ysz(1), 1, isign, plan_type)
 #endif
 
-    deallocate(a1)
-
-    return
-  end subroutine c2c_1m_y_plan
+      deallocate (a1)
 
+      return
+   end subroutine c2c_1m_y_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
-  subroutine c2c_1m_z_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
+   subroutine c2c_1m_z_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
-    complex(mytype), allocatable, dimension(:,:,:) :: a1
+      complex(mytype), allocatable, dimension(:, :, :) :: a1
 
-    allocate(a1(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)))
+      allocate (a1(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)))
 
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+      call dfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #else
-    call sfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+      call sfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
+                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #endif
 
-    deallocate(a1)
+      deallocate (a1)
 
-    return
-  end subroutine c2c_1m_z_plan
+      return
+   end subroutine c2c_1m_z_plan
 
+   ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
+   subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
 
-  ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
-  subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
+      implicit none
 
-    implicit none
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      real(mytype), allocatable, dimension(:, :, :) :: a1
+      complex(mytype), allocatable, dimension(:, :, :) :: a2
 
-    real(mytype), allocatable, dimension(:,:,:) :: a1
-    complex(mytype), allocatable, dimension(:,:,:) :: a2
-
-    allocate(a1(decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)))
-    allocate(a2(decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)))
+      allocate (a1(decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)))
+      allocate (a2(decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)))
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-         plan_type)
+      call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
+                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                   decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                   plan_type)
 #else
-    call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-         plan_type)
+      call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
+                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                   decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                   plan_type)
 #endif
-    deallocate(a1,a2)    
-
-    return
-  end subroutine r2c_1m_x_plan
+      deallocate (a1, a2)
 
+      return
+   end subroutine r2c_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
-  subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
+   ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
+   subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-    complex(mytype), allocatable, dimension(:,:,:) :: a1
-    real(mytype), allocatable, dimension(:,:,:) :: a2
+      complex(mytype), allocatable, dimension(:, :, :) :: a1
+      real(mytype), allocatable, dimension(:, :, :) :: a2
 
-    allocate(a1(decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)))
-    allocate(a2(decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)))
+      allocate (a1(decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)))
+      allocate (a2(decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)))
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-         plan_type)
+      call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
+                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                   decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                   plan_type)
 #else
-    call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-         plan_type)
+      call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
+                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                   decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                   plan_type)
 #endif
-    deallocate(a1,a2)
+      deallocate (a1, a2)
 
-    return
-  end subroutine c2r_1m_x_plan
+      return
+   end subroutine c2r_1m_x_plan
 
+   ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
+   subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
 
-  ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
-  subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
+      implicit none
 
-    implicit none
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      real(mytype), allocatable, dimension(:, :, :) :: a1
+      complex(mytype), allocatable, dimension(:, :, :) :: a2
 
-    real(mytype), allocatable, dimension(:,:,:) :: a1
-    complex(mytype), allocatable, dimension(:,:,:) :: a2
-
-    allocate(a1(decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)))
-    allocate(a2(decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)))
+      allocate (a1(decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)))
+      allocate (a2(decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)))
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+      call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #else
-    call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+      call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #endif
-    deallocate(a1,a2)
-
-    return
-  end subroutine r2c_1m_z_plan
+      deallocate (a1, a2)
 
+      return
+   end subroutine r2c_1m_z_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
-  subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
+   ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
+   subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
 
-    implicit none
+      implicit none
 
-    type(C_PTR), intent(OUT) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      type(C_PTR), intent(OUT) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-    complex(mytype), allocatable, dimension(:,:,:) :: a1
-    real(mytype), allocatable, dimension(:,:,:) :: a2
+      complex(mytype), allocatable, dimension(:, :, :) :: a1
+      real(mytype), allocatable, dimension(:, :, :) :: a2
 
-    allocate(a1(decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)))
-    allocate(a2(decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)))
+      allocate (a1(decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)))
+      allocate (a2(decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)))
 
 #ifdef DOUBLE_PREC
-    call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+      call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #else
-    call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+      call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #endif
-    deallocate(a1,a2)     
-
-    return
-  end subroutine c2r_1m_z_plan
+      deallocate (a1, a2)
 
+      return
+   end subroutine c2r_1m_z_plan
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
-
-    implicit none
-
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the FFTW (version 3.x) engine *****'
-       write(*,*) ' '
-    end if
-
-    if (format == PHYSICAL_IN_X) then
-
-       ! For C2C transforms
-       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
-       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
-       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
-
-       ! For R2C/C2R tranforms
-       call r2c_1m_x_plan(plan(0,1), ph, sp)
-       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(0,3), sp, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(2,3), sp, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
-       call c2r_1m_x_plan(plan(2,1), sp, ph)
-
-    else if (format == PHYSICAL_IN_Z) then
-
-       ! For C2C transforms
-       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
-       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD ) 
-       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
-       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
-
-       ! For R2C/C2R tranforms
-       call r2c_1m_z_plan(plan(0,3), ph, sp)
-       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan(0,1), sp, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan(2,1), sp, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
-       call c2r_1m_z_plan(plan(2,3), sp, ph)
-
-    end if
-
-    return
-  end subroutine init_fft_engine
+   subroutine init_fft_engine
+
+      implicit none
+
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the FFTW (version 3.x) engine *****'
+         write (*, *) ' '
+      end if
+
+      if (format == PHYSICAL_IN_X) then
+
+         ! For C2C transforms
+         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
+         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
+         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_x_plan(plan(0, 1), ph, sp)
+         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(0, 3), sp, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(2, 3), sp, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
+         call c2r_1m_x_plan(plan(2, 1), sp, ph)
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! For C2C transforms
+         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
+         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
+         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_z_plan(plan(0, 3), ph, sp)
+         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(0, 1), sp, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(2, 1), sp, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
+         call c2r_1m_z_plan(plan(2, 3), sp, ph)
+
+      end if
+
+      return
+   end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
+   subroutine finalize_fft_engine
 
-    implicit none
+      implicit none
 
-    integer :: i,j
+      integer :: i, j
 
-    do j=1,3
-       do i=-1,2
+      do j = 1, 3
+         do i = -1, 2
 #ifdef DOUBLE_PREC
-          call dfftw_destroy_plan(plan(i,j))
+            call dfftw_destroy_plan(plan(i, j))
 #else
-          call sfftw_destroy_plan(plan(i,j))
+            call sfftw_destroy_plan(plan(i, j))
 #endif
-       end do
-    end do
+         end do
+      end do
 
-    call fftw_cleanup()
+      call fftw_cleanup()
 
-    return
-  end subroutine finalize_fft_engine
+      return
+   end subroutine finalize_fft_engine
 
+   ! Following routines calculate multiple one-dimensional FFTs to form
+   ! the basis of three-dimensional FFTs.
 
-  ! Following routines calculate multiple one-dimensional FFTs to form 
-  ! the basis of three-dimensional FFTs.
+   ! c2c transform, multiple 1D FFTs in x direction
+   subroutine c2c_1m_x(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in x direction
-  subroutine c2c_1m_x(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR), intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR), intent(IN) :: plan1
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft(plan1, inout, inout)
+      call dfftw_execute_dft(plan1, inout, inout)
 #else
-    call sfftw_execute_dft(plan1, inout, inout)
+      call sfftw_execute_dft(plan1, inout, inout)
 #endif
 
-    return
-  end subroutine c2c_1m_x
+      return
+   end subroutine c2c_1m_x
 
+   ! c2c transform, multiple 1D FFTs in y direction
+   subroutine c2c_1m_y(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in y direction
-  subroutine c2c_1m_y(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR), intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR), intent(IN) :: plan1
+      integer :: k, s3
 
-    integer :: k, s3
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
-    ! transform on one Z-plane at a time
-    s3 = size(inout,3)
-    do k=1,s3
+      ! transform on one Z-plane at a time
+      s3 = size(inout, 3)
+      do k = 1, s3
 #ifdef DOUBLE_PREC
-       call dfftw_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
+         call dfftw_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
 #else
-       call sfftw_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
+         call sfftw_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
 #endif
-    end do
+      end do
+
+      return
+   end subroutine c2c_1m_y
 
-    return
-  end subroutine c2c_1m_y
+   ! c2c transform, multiple 1D FFTs in z direction
+   subroutine c2c_1m_z(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in z direction
-  subroutine c2c_1m_z(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR), intent(IN) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR), intent(IN) :: plan1
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft(plan1, inout, inout)
+      call dfftw_execute_dft(plan1, inout, inout)
 #else
-    call sfftw_execute_dft(plan1, inout, inout)
+      call sfftw_execute_dft(plan1, inout, inout)
 #endif
 
-    return
-  end subroutine c2c_1m_z
+      return
+   end subroutine c2c_1m_z
 
-  ! r2c transform, multiple 1D FFTs in x direction
-  subroutine r2c_1m_x(input, output)
+   ! r2c transform, multiple 1D FFTs in x direction
+   subroutine r2c_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft_r2c(plan(0,1), input, output)
+      call dfftw_execute_dft_r2c(plan(0, 1), input, output)
 #else
-    call sfftw_execute_dft_r2c(plan(0,1), input, output)
-#endif    
+      call sfftw_execute_dft_r2c(plan(0, 1), input, output)
+#endif
 
-    return
+      return
 
-  end subroutine r2c_1m_x
+   end subroutine r2c_1m_x
 
-  ! r2c transform, multiple 1D FFTs in z direction
-  subroutine r2c_1m_z(input, output)
+   ! r2c transform, multiple 1D FFTs in z direction
+   subroutine r2c_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft_r2c(plan(0,3), input, output)
+      call dfftw_execute_dft_r2c(plan(0, 3), input, output)
 #else
-    call sfftw_execute_dft_r2c(plan(0,3), input, output)
+      call sfftw_execute_dft_r2c(plan(0, 3), input, output)
 #endif
 
-    return
+      return
 
-  end subroutine r2c_1m_z
+   end subroutine r2c_1m_z
 
-  ! c2r transform, multiple 1D FFTs in x direction
-  subroutine c2r_1m_x(input, output)
+   ! c2r transform, multiple 1D FFTs in x direction
+   subroutine c2r_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft_c2r(plan(2,1), input, output)
+      call dfftw_execute_dft_c2r(plan(2, 1), input, output)
 #else
-    call sfftw_execute_dft_c2r(plan(2,1), input, output)
+      call sfftw_execute_dft_c2r(plan(2, 1), input, output)
 #endif
 
-    return
+      return
 
-  end subroutine c2r_1m_x
+   end subroutine c2r_1m_x
 
-  ! c2r transform, multiple 1D FFTs in z direction
-  subroutine c2r_1m_z(input, output)
+   ! c2r transform, multiple 1D FFTs in z direction
+   subroutine c2r_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(IN) :: input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call dfftw_execute_dft_c2r(plan(2,3), input, output)
+      call dfftw_execute_dft_c2r(plan(2, 3), input, output)
 #else
-    call sfftw_execute_dft_c2r(plan(2,3), input, output)
-#endif    
-
-    return
-
-  end subroutine c2r_1m_z
+      call sfftw_execute_dft_c2r(plan(2, 3), input, output)
+#endif
 
+      return
 
+   end subroutine c2r_1m_z
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D FFT - complex to complex
+   ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2c(in, out, isign)
+   subroutine fft_3d_c2c(in, out, isign)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out
-    integer, intent(IN) :: isign
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out
+      integer, intent(IN) :: isign
 
 #ifndef OVERWRITE
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
-    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
+      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
 
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in,isign,plan(isign,1))
+         call c2c_1m_x(in, isign, plan(isign, 1))
 #else
-       allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-       wk1 = in
-       call c2c_1m_x(wk1,isign,plan(isign,1))
+         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+         wk1 = in
+         call c2c_1m_x(wk1, isign, plan(isign, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-       if (dims(1)>1) then
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in,wk2_c2c,ph)
+            call transpose_x_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_x_to_y(wk1,wk2_c2c,ph)
+            call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else
 #ifdef OVERWRITE
-          call c2c_1m_y(in,isign,plan(isign,2))
+            call c2c_1m_y(in, isign, plan(isign, 2))
 #else
-          call c2c_1m_y(wk1,isign,plan(isign,2))
+            call c2c_1m_y(wk1, isign, plan(isign, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_c2c,out,ph)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_c2c, out, ph)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in,out,ph)
+            call transpose_y_to_z(in, out, ph)
 #else
-          call transpose_y_to_z(wk1,out,ph)
+            call transpose_y_to_z(wk1, out, ph)
 #endif
-       end if
-       call c2c_1m_z(out,isign,plan(isign,3))
+         end if
+         call c2c_1m_z(out, isign, plan(isign, 3))
 
-    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-         .OR. & 
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
+      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+               .OR. &
+               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in,isign,plan(isign,3))
+         call c2c_1m_z(in, isign, plan(isign, 3))
 #else
-       allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-       wk1 = in
-       call c2c_1m_z(wk1,isign,plan(isign,3))
+         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+         wk1 = in
+         call c2c_1m_z(wk1, isign, plan(isign, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,wk2_c2c,ph)
+            call transpose_z_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_z_to_y(wk1,wk2_c2c,ph)
+            call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else  ! out==wk2_c2c if 1D decomposition
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,out,ph)
+            call transpose_z_to_y(in, out, ph)
 #else
-          call transpose_z_to_y(wk1,out,ph)
+            call transpose_z_to_y(wk1, out, ph)
 #endif
-          call c2c_1m_y(out,isign,plan(isign,2))
-       end if
+            call c2c_1m_y(out, isign, plan(isign, 2))
+         end if
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_c2c,out,ph)
-       end if
-       call c2c_1m_x(out,isign,plan(isign,1))
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_c2c, out, ph)
+         end if
+         call c2c_1m_x(out, isign, plan(isign, 1))
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    deallocate (wk1)
+      deallocate (wk1)
 #endif
 
-    return
-  end subroutine fft_3d_c2c
-
+      return
+   end subroutine fft_3d_c2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D forward FFT - real to complex
+   ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_r2c(in_r, out_c)
-
-    implicit none
+   subroutine fft_3d_r2c(in_r, out_c)
 
-    real(mytype), dimension(:,:,:), intent(IN) :: in_r
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
+      implicit none
 
-    if (format==PHYSICAL_IN_X) then
+      real(mytype), dimension(:, :, :), intent(IN) :: in_r
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
-       ! ===== 1D FFTs in X =====
-       call r2c_1m_x(in_r,wk13)
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_x_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else
-          call c2c_1m_y(wk13,-1,plan(0,2))
-       end if
+         ! ===== 1D FFTs in X =====
+         call r2c_1m_x(in_r, wk13)
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,out_c,sp)
-       else
-          call transpose_y_to_z(wk13,out_c,sp)
-       end if
-       call c2c_1m_z(out_c,-1,plan(0,3))
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_x_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else
+            call c2c_1m_y(wk13, -1, plan(0, 2))
+         end if
 
-    else if (format==PHYSICAL_IN_Z) then
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, out_c, sp)
+         else
+            call transpose_y_to_z(wk13, out_c, sp)
+         end if
+         call c2c_1m_z(out_c, -1, plan(0, 3))
 
-       ! ===== 1D FFTs in Z =====
-       call r2c_1m_z(in_r,wk13)
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_z_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else  ! out_c==wk2_r2c if 1D decomposition
-          call transpose_z_to_y(wk13,out_c,sp)
-          call c2c_1m_y(out_c,-1,plan(0,2))
-       end if
+         ! ===== 1D FFTs in Z =====
+         call r2c_1m_z(in_r, wk13)
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,out_c,sp)
-       end if
-       call c2c_1m_x(out_c,-1,plan(0,1))
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_z_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else  ! out_c==wk2_r2c if 1D decomposition
+            call transpose_z_to_y(wk13, out_c, sp)
+            call c2c_1m_y(out_c, -1, plan(0, 2))
+         end if
 
-    end if
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, out_c, sp)
+         end if
+         call c2c_1m_x(out_c, -1, plan(0, 1))
 
-    return
-  end subroutine fft_3d_r2c
+      end if
 
+      return
+   end subroutine fft_3d_r2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D inverse FFT - complex to real
+   ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2r(in_c, out_r)
+   subroutine fft_3d_c2r(in_c, out_r)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
-    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
+      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
 #ifndef OVERWRITE
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in_c,1,plan(2,3))       
+         call c2c_1m_z(in_c, 1, plan(2, 3))
 #else
-       allocate (wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       wk1 = in_c
-       call c2c_1m_z(wk1,1,plan(2,3))
+         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         wk1 = in_c
+         call c2c_1m_z(wk1, 1, plan(2, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-       call transpose_z_to_y(in_c,wk2_r2c,sp)
+         call transpose_z_to_y(in_c, wk2_r2c, sp)
 #else
-       call transpose_z_to_y(wk1,wk2_r2c,sp)
+         call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-       call c2c_1m_y(wk2_r2c,1,plan(2,2))
+         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,wk13,sp)
-          call c2r_1m_x(wk13,out_r)
-       else
-          call c2r_1m_x(wk2_r2c,out_r)
-       end if
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, wk13, sp)
+            call c2r_1m_x(wk13, out_r)
+         else
+            call c2r_1m_x(wk2_r2c, out_r)
+         end if
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in_c,1,plan(2,1))
+         call c2c_1m_x(in_c, 1, plan(2, 1))
 #else
-       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       wk1 = in_c
-       call c2c_1m_x(wk1,1,plan(2,1))
+         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         wk1 = in_c
+         call c2c_1m_x(wk1, 1, plan(2, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in_c,wk2_r2c,sp)
+            call transpose_x_to_y(in_c, wk2_r2c, sp)
 #else
-          call transpose_x_to_y(wk1,wk2_r2c,sp)
+            call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-          call c2c_1m_y(wk2_r2c,1,plan(2,2))
-       else  ! in_c==wk2_r2c if 1D decomposition
+            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+         else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-          call c2c_1m_y(in_c,1,plan(2,2))
+            call c2c_1m_y(in_c, 1, plan(2, 2))
 #else
-          call c2c_1m_y(wk1,1,plan(2,2))
+            call c2c_1m_y(wk1, 1, plan(2, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,wk13,sp)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, wk13, sp)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in_c,wk13,sp)
+            call transpose_y_to_z(in_c, wk13, sp)
 #else
-          call transpose_y_to_z(wk1,wk13,sp)
+            call transpose_y_to_z(wk1, wk13, sp)
 #endif
-       end if
-       call c2r_1m_z(wk13,out_r)
+         end if
+         call c2r_1m_z(wk13, out_r)
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    deallocate (wk1)
+      deallocate (wk1)
 #endif
 
-    return
-  end subroutine fft_3d_c2r
-
+      return
+   end subroutine fft_3d_c2r
 
 end module decomp_2d_fft
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 5765ab66..e8c97081 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,1083 +9,1070 @@
 !
 !=======================================================================
 
-! This is the FFTW implementation of the FFT library using 
+! This is the FFTW implementation of the FFT library using
 ! the Fortran 2003 interface introduced in FFTW 3.3-beta1
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use, intrinsic :: iso_c_binding
-  
-  implicit none
-
-  include "fftw3.f03"
-  
-  private        ! Make everything private unless declared public
-
-  ! engine-specific global variables
-  integer, save :: plan_type = FFTW_MEASURE
-
-  ! FFTW plans
-  ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
-  ! For c2c transforms: 
-  !     use plan(-1,j) for  forward transform; 
-  !     use plan( 1,j) for backward transform;
-  ! For r2c/c2r transforms:
-  !     use plan(0,j) for r2c transforms;
-  !     use plan(2,j) for c2r transforms;
-  type(C_PTR), save :: plan(-1:2,3)
-
-  integer, parameter, public :: DECOMP_2D_FFT_FORWARD = -1
-  integer, parameter, public :: DECOMP_2D_FFT_BACKWARD = 1
-  
-  ! Physical space data can be stored in either X-pencil or Z-pencil
-  integer, parameter, public :: PHYSICAL_IN_X = 1
-  integer, parameter, public :: PHYSICAL_IN_Z = 3 
-
-  integer, save :: format                 ! input X-pencil or Z-pencil
-  
-  ! The libary can only be initialised once
-  logical, save :: initialised = .false. 
-
-  ! Global size of the FFT
-  integer, save :: nx_fft, ny_fft, nz_fft
-
-  ! 2D processor grid
-  ! FIXME this is already available in the module decomp_2d
-  integer, save, dimension(2) :: dims
-
-  ! Decomposition objects
-  TYPE(DECOMP_INFO), pointer, save :: ph=>null()  ! physical space
-  TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
-
-  ! Workspace to store the intermediate Y-pencil data
-  ! *** TODO: investigate how to use only one workspace array
-  complex(mytype), pointer :: wk2_c2c(:,:,:), wk2_r2c(:,:,:), wk13(:,:,:)
-  type(C_PTR) :: wk2_c2c_p, wk2_r2c_p, wk13_p
-
-  public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-       decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
-       decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
-  
-  ! Declare generic interfaces to handle different inputs
-  
-  interface decomp_2d_fft_init
-     module procedure fft_init_noarg
-     module procedure fft_init_arg
-     module procedure fft_init_general
-  end interface
-  
-  interface decomp_2d_fft_3d
-     module procedure fft_3d_c2c
-     module procedure fft_3d_r2c
-     module procedure fft_3d_c2r
-  end interface
-
-  
+   use decomp_2d  ! 2D decomposition module
+   use, intrinsic :: iso_c_binding
+
+   implicit none
+
+   include "fftw3.f03"
+
+   private        ! Make everything private unless declared public
+
+   ! engine-specific global variables
+   integer, save :: plan_type = FFTW_MEASURE
+
+   ! FFTW plans
+   ! j=1,2,3 corresponds to the 1D FFTs in X,Y,Z direction, respectively
+   ! For c2c transforms:
+   !     use plan(-1,j) for  forward transform;
+   !     use plan( 1,j) for backward transform;
+   ! For r2c/c2r transforms:
+   !     use plan(0,j) for r2c transforms;
+   !     use plan(2,j) for c2r transforms;
+   type(C_PTR), save :: plan(-1:2, 3)
+
+   integer, parameter, public :: DECOMP_2D_FFT_FORWARD = -1
+   integer, parameter, public :: DECOMP_2D_FFT_BACKWARD = 1
+
+   ! Physical space data can be stored in either X-pencil or Z-pencil
+   integer, parameter, public :: PHYSICAL_IN_X = 1
+   integer, parameter, public :: PHYSICAL_IN_Z = 3
+
+   integer, save :: format                 ! input X-pencil or Z-pencil
+
+   ! The libary can only be initialised once
+   logical, save :: initialised = .false.
+
+   ! Global size of the FFT
+   integer, save :: nx_fft, ny_fft, nz_fft
+
+   ! 2D processor grid
+   ! FIXME this is already available in the module decomp_2d
+   integer, save, dimension(2) :: dims
+
+   ! Decomposition objects
+   TYPE(DECOMP_INFO), pointer, save :: ph => null()  ! physical space
+   TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
+
+   ! Workspace to store the intermediate Y-pencil data
+   ! *** TODO: investigate how to use only one workspace array
+   complex(mytype), pointer :: wk2_c2c(:, :, :), wk2_r2c(:, :, :), wk13(:, :, :)
+   type(C_PTR) :: wk2_c2c_p, wk2_r2c_p, wk13_p
+
+   public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
+             decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+             decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
+
+   ! Declare generic interfaces to handle different inputs
+
+   interface decomp_2d_fft_init
+      module procedure fft_init_noarg
+      module procedure fft_init_arg
+      module procedure fft_init_general
+   end interface
+
+   interface decomp_2d_fft_3d
+      module procedure fft_3d_c2c
+      module procedure fft_3d_r2c
+      module procedure fft_3d_c2r
+   end interface
+
 contains
-  
-  
+
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Initialise the FFT module
+   ! Initialise the FFT module
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_init_noarg
-    
-    implicit none
-    
-    call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
-    
-    return
-  end subroutine fft_init_noarg
+   subroutine fft_init_noarg
+
+      implicit none
+
+      call fft_init_arg(PHYSICAL_IN_X)  ! default input is X-pencil data
 
-  subroutine fft_init_arg(pencil)     ! allow to handle Z-pencil input
+      return
+   end subroutine fft_init_noarg
 
-    implicit none
+   subroutine fft_init_arg(pencil)     ! allow to handle Z-pencil input
 
-    integer, intent(IN) :: pencil
+      implicit none
 
-    call fft_init_general(pencil, nx_global, ny_global, nz_global)
+      integer, intent(IN) :: pencil
 
-    return
-  end subroutine fft_init_arg
+      call fft_init_general(pencil, nx_global, ny_global, nz_global)
 
-  ! Initialise the FFT library to perform arbitrary size transforms
-  subroutine fft_init_general(pencil, nx, ny, nz)
+      return
+   end subroutine fft_init_arg
 
-    implicit none
+   ! Initialise the FFT library to perform arbitrary size transforms
+   subroutine fft_init_general(pencil, nx, ny, nz)
 
-    integer, intent(IN) :: pencil
-    integer, intent(IN) :: nx, ny, nz
+      implicit none
 
-    integer :: errorcode
-    integer(C_SIZE_T) :: sz
+      integer, intent(IN) :: pencil
+      integer, intent(IN) :: nx, ny, nz
+
+      integer :: errorcode
+      integer(C_SIZE_T) :: sz
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
 #endif
 
-    if (initialised) then
-       errorcode = 4
-       call decomp_2d_abort(errorcode, &
-            'FFT library should only be initialised once')
-    end if
-    
-    format = pencil
-    nx_fft = nx
-    ny_fft = ny
-    nz_fft = nz
-
-    ! determine the processor grid in use
-    dims = get_decomp_dims()
-
-    ! for c2r/r2c interface:
-    ! if in physical space, a real array is of size: nx*ny*nz
-    ! in spectral space, the complex array is of size:
-    !         (nx/2+1)*ny*nz, if PHYSICAL_IN_X
-    !      or nx*ny*(nz/2+1), if PHYSICAL_IN_Z
-
-    if (nx_fft==nx_global.and.ny_fft==ny_global.and.nz_fft==nz_global) then
-       ph => decomp_main
-    else
-       if (.not.associated(ph)) allocate(ph)
-       call decomp_info_init(nx, ny, nz, ph)
-    endif
-    if (format==PHYSICAL_IN_X) then
-       call decomp_info_init(nx/2+1, ny, nz, sp)
-    else if (format==PHYSICAL_IN_Z) then
-       call decomp_info_init(nx, ny, nz/2+1, sp)
-    end if
-
-    sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)
-    wk2_c2c_p = fftw_alloc_complex(sz)
-    call c_f_pointer(wk2_c2c_p,wk2_c2c,[ph%ysz(1),ph%ysz(2),ph%ysz(3)])
-
-    sz = sp%ysz(1)*sp%ysz(2)*sp%ysz(3)
-    wk2_r2c_p = fftw_alloc_complex(sz)
-    call c_f_pointer(wk2_r2c_p,wk2_r2c,[sp%ysz(1),sp%ysz(2),sp%ysz(3)])
-
-
-    if (format==PHYSICAL_IN_X) then
-       sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
-       wk13_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk13_p,wk13,[sp%xsz(1),sp%xsz(2),sp%xsz(3)])
-    else if (format==PHYSICAL_IN_Z) then
-       sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
-       wk13_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk13_p,wk13,[sp%zsz(1),sp%zsz(2),sp%zsz(3)])
-    end if
-
-    call init_fft_engine
-    
-    initialised = .true.
+      if (initialised) then
+         errorcode = 4
+         call decomp_2d_abort(errorcode, &
+                              'FFT library should only be initialised once')
+      end if
+
+      format = pencil
+      nx_fft = nx
+      ny_fft = ny
+      nz_fft = nz
+
+      ! determine the processor grid in use
+      dims = get_decomp_dims()
+
+      ! for c2r/r2c interface:
+      ! if in physical space, a real array is of size: nx*ny*nz
+      ! in spectral space, the complex array is of size:
+      !         (nx/2+1)*ny*nz, if PHYSICAL_IN_X
+      !      or nx*ny*(nz/2+1), if PHYSICAL_IN_Z
+
+      if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
+         ph => decomp_main
+      else
+         if (.not. associated(ph)) allocate (ph)
+         call decomp_info_init(nx, ny, nz, ph)
+      end if
+      if (format == PHYSICAL_IN_X) then
+         call decomp_info_init(nx/2 + 1, ny, nz, sp)
+      else if (format == PHYSICAL_IN_Z) then
+         call decomp_info_init(nx, ny, nz/2 + 1, sp)
+      end if
+
+      sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)
+      wk2_c2c_p = fftw_alloc_complex(sz)
+      call c_f_pointer(wk2_c2c_p, wk2_c2c, [ph%ysz(1), ph%ysz(2), ph%ysz(3)])
+
+      sz = sp%ysz(1)*sp%ysz(2)*sp%ysz(3)
+      wk2_r2c_p = fftw_alloc_complex(sz)
+      call c_f_pointer(wk2_r2c_p, wk2_r2c, [sp%ysz(1), sp%ysz(2), sp%ysz(3)])
+
+      if (format == PHYSICAL_IN_X) then
+         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+         wk13_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk13_p, wk13, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
+      else if (format == PHYSICAL_IN_Z) then
+         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+         wk13_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk13_p, wk13, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
+      end if
+
+      call init_fft_engine
+
+      initialised = .true.
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_init")
 #endif
 
-    return
-  end subroutine fft_init_general
+      return
+   end subroutine fft_init_general
 
-  
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Final clean up
+   ! Final clean up
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_2d_fft_finalize
-    
-    implicit none
+   subroutine decomp_2d_fft_finalize
+
+      implicit none
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_fin")
 #endif
 
-    if (nx_fft/=nx_global.or.ny_fft/=ny_global.or.nz_fft/=nz_global) then
-      call decomp_info_finalize(ph)
-      deallocate(ph)
-    endif
-    nullify(ph)
-    call decomp_info_finalize(sp)
+      if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
+         call decomp_info_finalize(ph)
+         deallocate (ph)
+      end if
+      nullify (ph)
+      call decomp_info_finalize(sp)
 
-    call fftw_free(wk2_c2c_p)
-    call fftw_free(wk2_r2c_p)
-    call fftw_free(wk13_p)
+      call fftw_free(wk2_c2c_p)
+      call fftw_free(wk2_r2c_p)
+      call fftw_free(wk13_p)
 
-    call finalize_fft_engine
+      call finalize_fft_engine
 
-    initialised = .false.
+      initialised = .false.
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_fin")
 #endif
 
-    return
-  end subroutine decomp_2d_fft_finalize
-
+      return
+   end subroutine decomp_2d_fft_finalize
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Return the size, starting/ending index of the distributed array 
-  !  whose global size is (nx/2+1)*ny*nz, for defining data structures
-  !  in r2c and c2r interfaces
+   ! Return the size, starting/ending index of the distributed array
+   !  whose global size is (nx/2+1)*ny*nz, for defining data structures
+   !  in r2c and c2r interfaces
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine decomp_2d_fft_get_size(istart, iend, isize)
-    
-    implicit none
-    integer, dimension(3), intent(OUT) :: istart, iend, isize
-    
-    if (format==PHYSICAL_IN_X) then
-       istart = sp%zst
-       iend   = sp%zen
-       isize  = sp%zsz
-    else if (format==PHYSICAL_IN_Z) then
-       istart = sp%xst
-       iend   = sp%xen
-       isize  = sp%xsz
-    end if
-    
-    return
-  end subroutine decomp_2d_fft_get_size
+   subroutine decomp_2d_fft_get_size(istart, iend, isize)
+
+      implicit none
+      integer, dimension(3), intent(OUT) :: istart, iend, isize
+
+      if (format == PHYSICAL_IN_X) then
+         istart = sp%zst
+         iend = sp%zen
+         isize = sp%zsz
+      else if (format == PHYSICAL_IN_Z) then
+         istart = sp%xst
+         iend = sp%xen
+         isize = sp%xsz
+      end if
+
+      return
+   end subroutine decomp_2d_fft_get_size
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Return a pointer to the decomp_info object ph
-  !
-  ! The caller should not apply decomp_info_finalize on the pointer
+   ! Return a pointer to the decomp_info object ph
+   !
+   ! The caller should not apply decomp_info_finalize on the pointer
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  function decomp_2d_fft_get_ph()
+   function decomp_2d_fft_get_ph()
 
-    implicit none
+      implicit none
 
-    type(decomp_info), pointer :: decomp_2d_fft_get_ph
+      type(decomp_info), pointer :: decomp_2d_fft_get_ph
 
-    if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  -1, &
-                                                  'FFT library must be initialised first')
-    decomp_2d_fft_get_ph => ph
+      if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                     __LINE__, &
+                                                     -1, &
+                                                     'FFT library must be initialised first')
+      decomp_2d_fft_get_ph => ph
 
-  end function decomp_2d_fft_get_ph
+   end function decomp_2d_fft_get_ph
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Return a pointer to the decomp_info object sp
-  !
-  ! The caller should not apply decomp_info_finalize on the pointer
+   ! Return a pointer to the decomp_info object sp
+   !
+   ! The caller should not apply decomp_info_finalize on the pointer
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  function decomp_2d_fft_get_sp()
+   function decomp_2d_fft_get_sp()
 
-    implicit none
+      implicit none
 
-    type(decomp_info), pointer :: decomp_2d_fft_get_sp
+      type(decomp_info), pointer :: decomp_2d_fft_get_sp
 
-    if (.not.associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  -1, &
-                                                  'FFT library must be initialised first')
-    decomp_2d_fft_get_sp => sp
+      if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
+                                                     __LINE__, &
+                                                     -1, &
+                                                     'FFT library must be initialised first')
+      decomp_2d_fft_get_sp => sp
 
-  end function decomp_2d_fft_get_sp
+   end function decomp_2d_fft_get_sp
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
-  subroutine c2c_1m_x_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
+   subroutine c2c_1m_x_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
 #ifdef DOUBLE_PREC
-    complex(C_DOUBLE_COMPLEX), pointer :: a1(:,:,:)
-    complex(C_DOUBLE_COMPLEX), pointer :: a1o(:,:,:)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1(:, :, :)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1o(:, :, :)
 #else
-    complex(C_FLOAT_COMPLEX), pointer :: a1(:,:,:)
-    complex(C_FLOAT_COMPLEX), pointer :: a1o(:,:,:)
+      complex(C_FLOAT_COMPLEX), pointer :: a1(:, :, :)
+      complex(C_FLOAT_COMPLEX), pointer :: a1o(:, :, :)
 #endif
-    type(C_PTR) :: a1_p
-    integer(C_SIZE_T) :: sz
+      type(C_PTR) :: a1_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1,[decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)])
-    call c_f_pointer(a1_p,a1o,[decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)])
+      sz = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
+      call c_f_pointer(a1_p, a1o, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 = fftw_plan_many_dft(1, decomp%xsz(1), &
-         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-         decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
-         isign, plan_type)
+      plan1 = fftw_plan_many_dft(1, decomp%xsz(1), &
+                                 decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                                 decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
+                                 isign, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft(1, decomp%xsz(1), &
-         decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
-         decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
-         isign, plan_type)
+      plan1 = fftwf_plan_many_dft(1, decomp%xsz(1), &
+                                  decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                                  decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
+                                  isign, plan_type)
 #endif
 
-    call fftw_free(a1_p)
+      call fftw_free(a1_p)
 
-    return
-  end subroutine c2c_1m_x_plan
+      return
+   end subroutine c2c_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
-  subroutine c2c_1m_y_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Y direction
+   subroutine c2c_1m_y_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
 #ifdef DOUBLE_PREC
-    complex(C_DOUBLE_COMPLEX), pointer :: a1(:,:)
-    complex(C_DOUBLE_COMPLEX), pointer :: a1o(:,:)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1(:, :)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1o(:, :)
 #else
-    complex(C_FLOAT_COMPLEX), pointer :: a1(:,:)
-    complex(C_FLOAT_COMPLEX), pointer :: a1o(:,:)
+      complex(C_FLOAT_COMPLEX), pointer :: a1(:, :)
+      complex(C_FLOAT_COMPLEX), pointer :: a1o(:, :)
 #endif
-    type(C_PTR) :: a1_p
-    integer(C_SIZE_T) :: sz
+      type(C_PTR) :: a1_p
+      integer(C_SIZE_T) :: sz
 
-    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-    ! done one Z-plane at a time. So plan for 2D data sets here.
-    sz = decomp%ysz(1)*decomp%ysz(2)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1,[decomp%ysz(1),decomp%ysz(2)])
-    call c_f_pointer(a1_p,a1o,[decomp%ysz(1),decomp%ysz(2)])
+      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+      ! done one Z-plane at a time. So plan for 2D data sets here.
+      sz = decomp%ysz(1)*decomp%ysz(2)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, [decomp%ysz(1), decomp%ysz(2)])
+      call c_f_pointer(a1_p, a1o, [decomp%ysz(1), decomp%ysz(2)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
-         a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
-         decomp%ysz(1), 1, isign, plan_type)
+      plan1 = fftw_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
+                                 a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
+                                 decomp%ysz(1), 1, isign, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
-         a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
-         decomp%ysz(1), 1, isign, plan_type)
+      plan1 = fftwf_plan_many_dft(1, decomp%ysz(2), decomp%ysz(1), &
+                                  a1, decomp%ysz(2), decomp%ysz(1), 1, a1o, decomp%ysz(2), &
+                                  decomp%ysz(1), 1, isign, plan_type)
 #endif
 
-    call fftw_free(a1_p)
+      call fftw_free(a1_p)
 
-    return
-  end subroutine c2c_1m_y_plan
+      return
+   end subroutine c2c_1m_y_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
-  subroutine c2c_1m_z_plan(plan1, decomp, isign)
+   ! Return a FFTW3 plan for multiple 1D c2c FFTs in Z direction
+   subroutine c2c_1m_z_plan(plan1, decomp, isign)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: isign
 
 #ifdef DOUBLE_PREC
-    complex(C_DOUBLE_COMPLEX), pointer :: a1(:,:,:)
-    complex(C_DOUBLE_COMPLEX), pointer :: a1o(:,:,:)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1(:, :, :)
+      complex(C_DOUBLE_COMPLEX), pointer :: a1o(:, :, :)
 #else
-    complex(C_FLOAT_COMPLEX), pointer :: a1(:,:,:)
-    complex(C_FLOAT_COMPLEX), pointer :: a1o(:,:,:)
+      complex(C_FLOAT_COMPLEX), pointer :: a1(:, :, :)
+      complex(C_FLOAT_COMPLEX), pointer :: a1o(:, :, :)
 #endif
-    type(C_PTR) :: a1_p
-    integer(C_SIZE_T) :: sz
+      type(C_PTR) :: a1_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1,[decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)])
-    call c_f_pointer(a1_p,a1o,[decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)])
+      sz = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
+      call c_f_pointer(a1_p, a1o, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft(1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+      plan1 = fftw_plan_many_dft(1, decomp%zsz(3), &
+                                 decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+                                 decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+                                 decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft(1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-         decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+      plan1 = fftwf_plan_many_dft(1, decomp%zsz(3), &
+                                  decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
+                                  decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+                                  decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
 #endif
 
-    call fftw_free(a1_p)
+      call fftw_free(a1_p)
 
-    return
-  end subroutine c2c_1m_z_plan
+      return
+   end subroutine c2c_1m_z_plan
 
-  ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
-  subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
+   ! Return a FFTW3 plan for multiple 1D r2c FFTs in X direction
+   subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-    real(mytype), pointer :: a1(:,:,:)
-    complex(mytype), pointer :: a2(:,:,:)
-    type(C_PTR) :: a1_p, a2_p
-    integer(C_SIZE_T) :: sz
+      real(mytype), pointer :: a1(:, :, :)
+      complex(mytype), pointer :: a2(:, :, :)
+      type(C_PTR) :: a1_p, a2_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
-    a1_p = fftw_alloc_real(sz)
-    call c_f_pointer(a1_p,a1, &
-         [decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)])
-    sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
-    a2_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a2_p,a2, &
-         [decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)])
+      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+      a1_p = fftw_alloc_real(sz)
+      call c_f_pointer(a1_p, a1, &
+                       [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
+      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+      a2_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a2_p, a2, &
+                       [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-         plan_type)
+      plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
+                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                     decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                     plan_type)
 #else
-    plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
-         decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-         plan_type)
+      plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
+                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                      decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
+                                      plan_type)
 #endif
 
-    call fftw_free(a1_p)
-    call fftw_free(a2_p)
+      call fftw_free(a1_p)
+      call fftw_free(a2_p)
 
-    return
-  end subroutine r2c_1m_x_plan
+      return
+   end subroutine r2c_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
-  subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
+   ! Return a FFTW3 plan for multiple 1D c2r FFTs in X direction
+   subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-    complex(mytype), pointer :: a1(:,:,:)
-    real(mytype), pointer :: a2(:,:,:)
-    type(C_PTR) :: a1_p, a2_p
-    integer(C_SIZE_T) :: sz
+      complex(mytype), pointer :: a1(:, :, :)
+      real(mytype), pointer :: a2(:, :, :)
+      type(C_PTR) :: a1_p, a2_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1, &
-         [decomp_sp%xsz(1),decomp_sp%xsz(2),decomp_sp%xsz(3)])
-    sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
-    a2_p = fftw_alloc_real(sz)
-    call c_f_pointer(a2_p,a2, &
-         [decomp_ph%xsz(1),decomp_ph%xsz(2),decomp_ph%xsz(3)])
+      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, &
+                       [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
+      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+      a2_p = fftw_alloc_real(sz)
+      call c_f_pointer(a2_p, a2, &
+                       [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-         plan_type)
+      plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
+                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                     decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                     plan_type)
 #else
-    plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
-         decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-         plan_type)
+      plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
+                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                      decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
+                                      plan_type)
 #endif
 
-    call fftw_free(a1_p)
-    call fftw_free(a2_p)
+      call fftw_free(a1_p)
+      call fftw_free(a2_p)
 
-    return
-  end subroutine c2r_1m_x_plan
+      return
+   end subroutine c2r_1m_x_plan
 
-  ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
-  subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
+   ! Return a FFTW3 plan for multiple 1D r2c FFTs in Z direction
+   subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
 
-    real(mytype), pointer :: a1(:,:,:)
-    complex(mytype), pointer :: a2(:,:,:)
-    type(C_PTR) :: a1_p, a2_p
-    integer(C_SIZE_T) :: sz
+      real(mytype), pointer :: a1(:, :, :)
+      complex(mytype), pointer :: a2(:, :, :)
+      type(C_PTR) :: a1_p, a2_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
-    a1_p = fftw_alloc_real(sz)
-    call c_f_pointer(a1_p,a1, &
-         [decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)])
-    sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
-    a2_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a2_p,a2, &
-         [decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)])
+      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+      a1_p = fftw_alloc_real(sz)
+      call c_f_pointer(a1_p, a1, &
+                       [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
+      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+      a2_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a2_p, a2, &
+                       [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+      plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+      plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
 #endif
 
-    call fftw_free(a1_p)
-    call fftw_free(a2_p)
+      call fftw_free(a1_p)
+      call fftw_free(a2_p)
 
-    return
-  end subroutine r2c_1m_z_plan
+      return
+   end subroutine r2c_1m_z_plan
 
-  ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
-  subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
+   ! Return a FFTW3 plan for multiple 1D c2r FFTs in Z direction
+   subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
 
-    implicit none
+      implicit none
 
-    type(C_PTR) :: plan1
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
+      type(C_PTR) :: plan1
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_sp
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph
 
-    complex(mytype), pointer :: a1(:,:,:)
-    real(mytype), pointer :: a2(:,:,:)
-    type(C_PTR) :: a1_p, a2_p
-    integer(C_SIZE_T) :: sz
+      complex(mytype), pointer :: a1(:, :, :)
+      real(mytype), pointer :: a2(:, :, :)
+      type(C_PTR) :: a1_p, a2_p
+      integer(C_SIZE_T) :: sz
 
-    sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
-    a1_p = fftw_alloc_complex(sz)
-    call c_f_pointer(a1_p,a1, &
-         [decomp_sp%zsz(1),decomp_sp%zsz(2),decomp_sp%zsz(3)])
-    sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
-    a2_p = fftw_alloc_real(sz)
-    call c_f_pointer(a2_p,a2, &
-         [decomp_ph%zsz(1),decomp_ph%zsz(2),decomp_ph%zsz(3)])
+      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+      a1_p = fftw_alloc_complex(sz)
+      call c_f_pointer(a1_p, a1, &
+                       [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
+      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+      a2_p = fftw_alloc_real(sz)
+      call c_f_pointer(a2_p, a2, &
+                       [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
 
 #ifdef DOUBLE_PREC
-    plan1 =  fftw_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+      plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #else
-    plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-         decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+      plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
 #endif
 
-    call fftw_free(a1_p)
-    call fftw_free(a2_p)
-
-    return
-  end subroutine c2r_1m_z_plan
+      call fftw_free(a1_p)
+      call fftw_free(a2_p)
 
+      return
+   end subroutine c2r_1m_z_plan
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
-
-    implicit none
-
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the FFTW (F2003 interface) engine *****'
-       write(*,*) ' '
-    end if
-
-    if (format == PHYSICAL_IN_X) then
-
-       ! For C2C transforms
-       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
-       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
-       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
-       
-       ! For R2C/C2R tranforms
-       call r2c_1m_x_plan(plan(0,1), ph, sp)
-       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(0,3), sp, FFTW_FORWARD )
-       call c2c_1m_z_plan(plan(2,3), sp, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
-       call c2r_1m_x_plan(plan(2,1), sp, ph)
-
-    else if (format == PHYSICAL_IN_Z) then
-
-       ! For C2C transforms
-       call c2c_1m_z_plan(plan(-1,3), ph, FFTW_FORWARD )
-       call c2c_1m_y_plan(plan(-1,2), ph, FFTW_FORWARD ) 
-       call c2c_1m_x_plan(plan(-1,1), ph, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan( 1,1), ph, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan( 1,2), ph, FFTW_BACKWARD)
-       call c2c_1m_z_plan(plan( 1,3), ph, FFTW_BACKWARD)
-       
-       ! For R2C/C2R tranforms
-       call r2c_1m_z_plan(plan(0,3), ph, sp)
-       call c2c_1m_y_plan(plan(0,2), sp, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan(0,1), sp, FFTW_FORWARD )
-       call c2c_1m_x_plan(plan(2,1), sp, FFTW_BACKWARD)
-       call c2c_1m_y_plan(plan(2,2), sp, FFTW_BACKWARD)
-       call c2r_1m_z_plan(plan(2,3), sp, ph)
-       
-    end if
-
-    return
-  end subroutine init_fft_engine
+   subroutine init_fft_engine
+
+      implicit none
+
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the FFTW (F2003 interface) engine *****'
+         write (*, *) ' '
+      end if
+
+      if (format == PHYSICAL_IN_X) then
+
+         ! For C2C transforms
+         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
+         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
+         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_x_plan(plan(0, 1), ph, sp)
+         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(0, 3), sp, FFTW_FORWARD)
+         call c2c_1m_z_plan(plan(2, 3), sp, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
+         call c2r_1m_x_plan(plan(2, 1), sp, ph)
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! For C2C transforms
+         call c2c_1m_z_plan(plan(-1, 3), ph, FFTW_FORWARD)
+         call c2c_1m_y_plan(plan(-1, 2), ph, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(-1, 1), ph, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(1, 1), ph, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(1, 2), ph, FFTW_BACKWARD)
+         call c2c_1m_z_plan(plan(1, 3), ph, FFTW_BACKWARD)
+
+         ! For R2C/C2R tranforms
+         call r2c_1m_z_plan(plan(0, 3), ph, sp)
+         call c2c_1m_y_plan(plan(0, 2), sp, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(0, 1), sp, FFTW_FORWARD)
+         call c2c_1m_x_plan(plan(2, 1), sp, FFTW_BACKWARD)
+         call c2c_1m_y_plan(plan(2, 2), sp, FFTW_BACKWARD)
+         call c2r_1m_z_plan(plan(2, 3), sp, ph)
+
+      end if
+
+      return
+   end subroutine init_fft_engine
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
+   subroutine finalize_fft_engine
+
+      implicit none
 
-    implicit none
+      integer :: i, j
 
-    integer :: i,j
-    
-    do j=1,3
-       do i=-1,2
+      do j = 1, 3
+         do i = -1, 2
 #ifdef DOUBLE_PREC
-          call fftw_destroy_plan(plan(i,j))
+            call fftw_destroy_plan(plan(i, j))
 #else
-          call fftwf_destroy_plan(plan(i,j))
+            call fftwf_destroy_plan(plan(i, j))
 #endif
-       end do
-    end do
+         end do
+      end do
 
-    call fftw_cleanup()
+      call fftw_cleanup()
 
-    return
-  end subroutine finalize_fft_engine
+      return
+   end subroutine finalize_fft_engine
 
+   ! Following routines calculate multiple one-dimensional FFTs to form
+   ! the basis of three-dimensional FFTs.
 
-  ! Following routines calculate multiple one-dimensional FFTs to form 
-  ! the basis of three-dimensional FFTs.
+   ! c2c transform, multiple 1D FFTs in x direction
+   subroutine c2c_1m_x(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in x direction
-  subroutine c2c_1m_x(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR) :: plan1
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft(plan1, inout, inout)
+      call fftw_execute_dft(plan1, inout, inout)
 #else
-    call fftwf_execute_dft(plan1, inout, inout)
+      call fftwf_execute_dft(plan1, inout, inout)
 #endif
 
-    return
-  end subroutine c2c_1m_x
+      return
+   end subroutine c2c_1m_x
 
+   ! c2c transform, multiple 1D FFTs in y direction
+   subroutine c2c_1m_y(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in y direction
-  subroutine c2c_1m_y(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR) :: plan1
+      integer :: k, s3
 
-    integer :: k, s3
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
-    s3 = size(inout,3)
+      s3 = size(inout, 3)
 
-    do k=1,s3  ! transform on one Z-plane at a time
+      do k = 1, s3  ! transform on one Z-plane at a time
 #ifdef DOUBLE_PREC
-       call fftw_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
+         call fftw_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
 #else
-       call fftwf_execute_dft(plan1, inout(:,:,k), inout(:,:,k))
+         call fftwf_execute_dft(plan1, inout(:, :, k), inout(:, :, k))
 #endif
-    end do
+      end do
+
+      return
+   end subroutine c2c_1m_y
 
-    return
-  end subroutine c2c_1m_y
+   ! c2c transform, multiple 1D FFTs in z direction
+   subroutine c2c_1m_z(inout, isign, plan1)
 
-  ! c2c transform, multiple 1D FFTs in z direction
-  subroutine c2c_1m_z(inout, isign, plan1)
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      type(C_PTR) :: plan1
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    type(C_PTR) :: plan1
+      integer :: foo
 
-    integer :: foo
+      foo = isign ! Silence unused dummy argument
 
-    foo = isign ! Silence unused dummy argument
-    
 #ifdef DOUBLE_PREC
-       call fftw_execute_dft(plan1, inout, inout)
+      call fftw_execute_dft(plan1, inout, inout)
 #else
-       call fftwf_execute_dft(plan1, inout, inout)
+      call fftwf_execute_dft(plan1, inout, inout)
 #endif
 
-    return
-  end subroutine c2c_1m_z
+      return
+   end subroutine c2c_1m_z
 
-  ! r2c transform, multiple 1D FFTs in x direction
-  subroutine r2c_1m_x(input, output)
+   ! r2c transform, multiple 1D FFTs in x direction
+   subroutine r2c_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(INOUT)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(INOUT)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft_r2c(plan(0,1), input, output)
+      call fftw_execute_dft_r2c(plan(0, 1), input, output)
 #else
-    call fftwf_execute_dft_r2c(plan(0,1), input, output)
-#endif    
+      call fftwf_execute_dft_r2c(plan(0, 1), input, output)
+#endif
 
-    return
+      return
 
-  end subroutine r2c_1m_x
+   end subroutine r2c_1m_x
 
-  ! r2c transform, multiple 1D FFTs in z direction
-  subroutine r2c_1m_z(input, output)
+   ! r2c transform, multiple 1D FFTs in z direction
+   subroutine r2c_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(INOUT)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
+      real(mytype), dimension(:, :, :), intent(INOUT)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft_r2c(plan(0,3), input, output)
+      call fftw_execute_dft_r2c(plan(0, 3), input, output)
 #else
-    call fftwf_execute_dft_r2c(plan(0,3), input, output)
+      call fftwf_execute_dft_r2c(plan(0, 3), input, output)
 #endif
 
-    return
+      return
 
-  end subroutine r2c_1m_z
+   end subroutine r2c_1m_z
 
-  ! c2r transform, multiple 1D FFTs in x direction
-  subroutine c2r_1m_x(input, output)
+   ! c2r transform, multiple 1D FFTs in x direction
+   subroutine c2r_1m_x(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(INOUT)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft_c2r(plan(2,1), input, output)
+      call fftw_execute_dft_c2r(plan(2, 1), input, output)
 #else
-    call fftwf_execute_dft_c2r(plan(2,1), input, output)
+      call fftwf_execute_dft_c2r(plan(2, 1), input, output)
 #endif
 
-    return
+      return
 
-  end subroutine c2r_1m_x
+   end subroutine c2r_1m_x
 
-  ! c2r transform, multiple 1D FFTs in z direction
-  subroutine c2r_1m_z(input, output)
+   ! c2r transform, multiple 1D FFTs in z direction
+   subroutine c2r_1m_z(input, output)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
-    call fftw_execute_dft_c2r(plan(2,3), input, output)
+      call fftw_execute_dft_c2r(plan(2, 3), input, output)
 #else
-    call fftwf_execute_dft_c2r(plan(2,3), input, output)
-#endif    
-
-    return
-
-  end subroutine c2r_1m_z
+      call fftwf_execute_dft_c2r(plan(2, 3), input, output)
+#endif
 
+      return
 
+   end subroutine c2r_1m_z
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D FFT - complex to complex
+   ! 3D FFT - complex to complex
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2c(in, out, isign)
-    
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out
-    integer, intent(IN) :: isign
+   subroutine fft_3d_c2c(in, out, isign)
+
+      implicit none
+
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out
+      integer, intent(IN) :: isign
 
 #ifndef OVERWRITE
-    complex(mytype), pointer :: wk1(:,:,:)
-    integer(C_SIZE_T) :: sz
-    type(C_PTR) :: wk1_p
+      complex(mytype), pointer :: wk1(:, :, :)
+      integer(C_SIZE_T) :: sz
+      type(C_PTR) :: wk1_p
 
-    wk1_p = c_null_ptr ! Initialise to NULL pointer
+      wk1_p = c_null_ptr ! Initialise to NULL pointer
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
-       
-       ! ===== 1D FFTs in X =====
+      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
+
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in,isign,plan(isign,1))
+         call c2c_1m_x(in, isign, plan(isign, 1))
 #else
-       sz = ph%xsz(1)*ph%xsz(2)*ph%xsz(3)
-       wk1_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk1_p, wk1, [ph%xsz(1),ph%xsz(2),ph%xsz(3)])
-       wk1 = in
-       call c2c_1m_x(wk1,isign,plan(isign,1))
+         sz = ph%xsz(1)*ph%xsz(2)*ph%xsz(3)
+         wk1_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk1_p, wk1, [ph%xsz(1), ph%xsz(2), ph%xsz(3)])
+         wk1 = in
+         call c2c_1m_x(wk1, isign, plan(isign, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
 
-       if (dims(1)>1) then
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in,wk2_c2c,ph)
+            call transpose_x_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_x_to_y(wk1,wk2_c2c,ph)
+            call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else
 #ifdef OVERWRITE
-          call c2c_1m_y(in,isign,plan(isign,2))
+            call c2c_1m_y(in, isign, plan(isign, 2))
 #else
-          call c2c_1m_y(wk1,isign,plan(isign,2))
+            call c2c_1m_y(wk1, isign, plan(isign, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_c2c,out,ph)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_c2c, out, ph)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in,out,ph)
+            call transpose_y_to_z(in, out, ph)
 #else
-          call transpose_y_to_z(wk1,out,ph)
+            call transpose_y_to_z(wk1, out, ph)
 #endif
-       end if
-       call c2c_1m_z(out,isign,plan(isign,3))
+         end if
+         call c2c_1m_z(out, isign, plan(isign, 3))
 
-    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-         .OR. & 
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
+      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+               .OR. &
+               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in,isign,plan(isign,3))
+         call c2c_1m_z(in, isign, plan(isign, 3))
 #else
-       sz = ph%zsz(1)*ph%zsz(2)*ph%zsz(3)
-       wk1_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk1_p, wk1, [ph%zsz(1),ph%zsz(2),ph%zsz(3)])
-       wk1 = in
-       call c2c_1m_z(wk1,isign,plan(isign,3))
+         sz = ph%zsz(1)*ph%zsz(2)*ph%zsz(3)
+         wk1_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk1_p, wk1, [ph%zsz(1), ph%zsz(2), ph%zsz(3)])
+         wk1 = in
+         call c2c_1m_z(wk1, isign, plan(isign, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,wk2_c2c,ph)
+            call transpose_z_to_y(in, wk2_c2c, ph)
 #else
-          call transpose_z_to_y(wk1,wk2_c2c,ph)
+            call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-          call c2c_1m_y(wk2_c2c,isign,plan(isign,2))
-       else  ! out==wk2_c2c if 1D decomposition
+            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+         else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
-          call transpose_z_to_y(in,out,ph)
+            call transpose_z_to_y(in, out, ph)
 #else
-          call transpose_z_to_y(wk1,out,ph)
+            call transpose_z_to_y(wk1, out, ph)
 #endif
-          call c2c_1m_y(out,isign,plan(isign,2))
-       end if
+            call c2c_1m_y(out, isign, plan(isign, 2))
+         end if
+
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_c2c, out, ph)
+         end if
+         call c2c_1m_x(out, isign, plan(isign, 1))
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_c2c,out,ph)
-       end if
-       call c2c_1m_x(out,isign,plan(isign,1))
-       
-    end if
+      end if
 
 #ifndef OVERWRITE
-    call fftw_free(wk1_p)
+      call fftw_free(wk1_p)
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-    return
-  end subroutine fft_3d_c2c
+      return
+   end subroutine fft_3d_c2c
 
-  
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D forward FFT - real to complex
+   ! 3D forward FFT - real to complex
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_r2c(in_r, out_c)
-    
-    implicit none
-    
-    real(mytype), dimension(:,:,:), intent(INOUT) :: in_r
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
+   subroutine fft_3d_r2c(in_r, out_c)
+
+      implicit none
+
+      real(mytype), dimension(:, :, :), intent(INOUT) :: in_r
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-    if (format==PHYSICAL_IN_X) then
-
-       ! ===== 1D FFTs in X =====
-       call r2c_1m_x(in_r,wk13)
-
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_x_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else
-          call c2c_1m_y(wk13,-1,plan(0,2))
-       end if
-
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,out_c,sp)
-       else
-          call transpose_y_to_z(wk13,out_c,sp)
-       end if
-       call c2c_1m_z(out_c,-1,plan(0,3))
-                
-    else if (format==PHYSICAL_IN_Z) then
-
-       ! ===== 1D FFTs in Z =====
-       call r2c_1m_z(in_r,wk13)
-
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
-          call transpose_z_to_y(wk13,wk2_r2c,sp)
-          call c2c_1m_y(wk2_r2c,-1,plan(0,2))
-       else  ! out_c==wk2_r2c if 1D decomposition
-          call transpose_z_to_y(wk13,out_c,sp)
-          call c2c_1m_y(out_c,-1,plan(0,2))
-       end if
-
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,out_c,sp)
-       end if
-       call c2c_1m_x(out_c,-1,plan(0,1))
-
-    end if
+      if (format == PHYSICAL_IN_X) then
+
+         ! ===== 1D FFTs in X =====
+         call r2c_1m_x(in_r, wk13)
+
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_x_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else
+            call c2c_1m_y(wk13, -1, plan(0, 2))
+         end if
+
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, out_c, sp)
+         else
+            call transpose_y_to_z(wk13, out_c, sp)
+         end if
+         call c2c_1m_z(out_c, -1, plan(0, 3))
+
+      else if (format == PHYSICAL_IN_Z) then
+
+         ! ===== 1D FFTs in Z =====
+         call r2c_1m_z(in_r, wk13)
+
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
+            call transpose_z_to_y(wk13, wk2_r2c, sp)
+            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+         else  ! out_c==wk2_r2c if 1D decomposition
+            call transpose_z_to_y(wk13, out_c, sp)
+            call c2c_1m_y(out_c, -1, plan(0, 2))
+         end if
+
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, out_c, sp)
+         end if
+         call c2c_1m_x(out_c, -1, plan(0, 1))
+
+      end if
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-    return
-  end subroutine fft_3d_r2c
-  
-  
+      return
+   end subroutine fft_3d_r2c
+
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D inverse FFT - complex to real
+   ! 3D inverse FFT - complex to real
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2r(in_c, out_r)
-    
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: in_c
-    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
+   subroutine fft_3d_c2r(in_c, out_r)
+
+      implicit none
+
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
+      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
 #ifndef OVERWRITE
-    complex(mytype), pointer :: wk1(:,:,:)
-    integer(C_SIZE_T) :: sz
-    type(C_PTR) :: wk1_p
+      complex(mytype), pointer :: wk1(:, :, :)
+      integer(C_SIZE_T) :: sz
+      type(C_PTR) :: wk1_p
 
-    wk1_p = c_null_ptr ! Initialise to NULL pointer
+      wk1_p = c_null_ptr ! Initialise to NULL pointer
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in Z =====
+         ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-       call c2c_1m_z(in_c,1,plan(2,3))       
+         call c2c_1m_z(in_c, 1, plan(2, 3))
 #else
-       sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
-       wk1_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk1_p, wk1, [sp%zsz(1),sp%zsz(2),sp%zsz(3)])
-       wk1 = in_c
-       call c2c_1m_z(wk1,1,plan(2,3))
+         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+         wk1_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk1_p, wk1, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
+         wk1 = in_c
+         call c2c_1m_z(wk1, 1, plan(2, 3))
 #endif
 
-       ! ===== Swap Z --> Y; 1D FFTs in Y =====
+         ! ===== Swap Z --> Y; 1D FFTs in Y =====
 #ifdef OVERWRITE
-       call transpose_z_to_y(in_c,wk2_r2c,sp)
+         call transpose_z_to_y(in_c, wk2_r2c, sp)
 #else
-       call transpose_z_to_y(wk1,wk2_r2c,sp)
+         call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-       call c2c_1m_y(wk2_r2c,1,plan(2,2))
+         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
 
-       ! ===== Swap Y --> X; 1D FFTs in X =====
-       if (dims(1)>1) then
-          call transpose_y_to_x(wk2_r2c,wk13,sp)
-          call c2r_1m_x(wk13,out_r)
-       else
-          call c2r_1m_x(wk2_r2c,out_r)
-       end if
+         ! ===== Swap Y --> X; 1D FFTs in X =====
+         if (dims(1) > 1) then
+            call transpose_y_to_x(wk2_r2c, wk13, sp)
+            call c2r_1m_x(wk13, out_r)
+         else
+            call c2r_1m_x(wk2_r2c, out_r)
+         end if
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in X =====
+         ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-       call c2c_1m_x(in_c,1,plan(2,1))
+         call c2c_1m_x(in_c, 1, plan(2, 1))
 #else
-       sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
-       wk1_p = fftw_alloc_complex(sz)
-       call c_f_pointer(wk1_p, wk1, [sp%xsz(1),sp%xsz(2),sp%xsz(3)])
-       wk1 = in_c
-       call c2c_1m_x(wk1,1,plan(2,1))
+         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+         wk1_p = fftw_alloc_complex(sz)
+         call c_f_pointer(wk1_p, wk1, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
+         wk1 = in_c
+         call c2c_1m_x(wk1, 1, plan(2, 1))
 #endif
 
-       ! ===== Swap X --> Y; 1D FFTs in Y =====
-       if (dims(1)>1) then
+         ! ===== Swap X --> Y; 1D FFTs in Y =====
+         if (dims(1) > 1) then
 #ifdef OVERWRITE
-          call transpose_x_to_y(in_c,wk2_r2c,sp)
+            call transpose_x_to_y(in_c, wk2_r2c, sp)
 #else
-          call transpose_x_to_y(wk1,wk2_r2c,sp)
+            call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-          call c2c_1m_y(wk2_r2c,1,plan(2,2))
-       else  ! in_c==wk2_r2c if 1D decomposition
+            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+         else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-          call c2c_1m_y(in_c,1,plan(2,2))
+            call c2c_1m_y(in_c, 1, plan(2, 2))
 #else
-          call c2c_1m_y(wk1,1,plan(2,2))
+            call c2c_1m_y(wk1, 1, plan(2, 2))
 #endif
-       end if
+         end if
 
-       ! ===== Swap Y --> Z; 1D FFTs in Z =====
-       if (dims(1)>1) then
-          call transpose_y_to_z(wk2_r2c,wk13,sp)
-       else
+         ! ===== Swap Y --> Z; 1D FFTs in Z =====
+         if (dims(1) > 1) then
+            call transpose_y_to_z(wk2_r2c, wk13, sp)
+         else
 #ifdef OVERWRITE
-          call transpose_y_to_z(in_c,wk13,sp)
+            call transpose_y_to_z(in_c, wk13, sp)
 #else
-          call transpose_y_to_z(wk1,wk13,sp)
+            call transpose_y_to_z(wk1, wk13, sp)
 #endif
-       end if
-       call c2r_1m_z(wk13,out_r)
+         end if
+         call c2r_1m_z(wk13, out_r)
 
-    end if
+      end if
 
 #ifndef OVERWRITE
-    call fftw_free(wk1_p)
+      call fftw_free(wk1_p)
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-    return
-  end subroutine fft_3d_c2r
+      return
+   end subroutine fft_3d_c2r
 
-  
 end module decomp_2d_fft
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index d66425bd..8bab0ae3 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,320 +13,316 @@
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use glassman
+   use decomp_2d  ! 2D decomposition module
+   use glassman
 
-  implicit none
+   implicit none
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
-  ! engine-specific global variables
-  complex(mytype), allocatable, dimension(:) :: buf, scratch
+   ! engine-specific global variables
+   complex(mytype), allocatable, dimension(:) :: buf, scratch
 
-  ! common code used for all engines, including global variables, 
-  ! generic interface definitions and several subroutines
+   ! common code used for all engines, including global variables,
+   ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
-
-    implicit none
+   subroutine init_fft_engine
 
-    integer :: cbuf_size
+      implicit none
 
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the generic FFT engine *****'
-       write(*,*) ' '
-    end if
+      integer :: cbuf_size
 
-    cbuf_size = max(ph%xsz(1), ph%ysz(2))
-    cbuf_size = max(cbuf_size, ph%zsz(3))
-    allocate(buf(cbuf_size))
-    allocate(scratch(cbuf_size))
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the generic FFT engine *****'
+         write (*, *) ' '
+      end if
 
-    return
-  end subroutine init_fft_engine
+      cbuf_size = max(ph%xsz(1), ph%ysz(2))
+      cbuf_size = max(cbuf_size, ph%zsz(3))
+      allocate (buf(cbuf_size))
+      allocate (scratch(cbuf_size))
 
+      return
+   end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
-
-    implicit none
+   subroutine finalize_fft_engine
 
-    if (allocated(buf)) deallocate(buf)
-    if (allocated(scratch)) deallocate(scratch)
+      implicit none
 
-    return
-  end subroutine finalize_fft_engine
+      if (allocated(buf)) deallocate (buf)
+      if (allocated(scratch)) deallocate (scratch)
 
+      return
+   end subroutine finalize_fft_engine
 
-  ! Following routines calculate multiple one-dimensional FFTs to form 
-  ! the basis of three-dimensional FFTs.
+   ! Following routines calculate multiple one-dimensional FFTs to form
+   ! the basis of three-dimensional FFTs.
 
-  ! c2c transform, multiple 1D FFTs in x direction
-  subroutine c2c_1m_x(inout, isign, decomp)
+   ! c2c transform, multiple 1D FFTs in x direction
+   subroutine c2c_1m_x(inout, isign, decomp)
 
-    !$acc routine(spcfft) seq
+      !$acc routine(spcfft) seq
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer :: i,j,k
+      integer :: i, j, k
 
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do k=1,decomp%xsz(3)
-       do j=1,decomp%xsz(2)
-          do i=1,decomp%xsz(1)
-             buf(i) = inout(i,j,k)
-          end do
-          call spcfft(buf,decomp%xsz(1),isign,scratch)
-          do i=1,decomp%xsz(1)
-             inout(i,j,k) = buf(i)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do k = 1, decomp%xsz(3)
+         do j = 1, decomp%xsz(2)
+            do i = 1, decomp%xsz(1)
+               buf(i) = inout(i, j, k)
+            end do
+            call spcfft(buf, decomp%xsz(1), isign, scratch)
+            do i = 1, decomp%xsz(1)
+               inout(i, j, k) = buf(i)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
 
-    return
+      return
 
-  end subroutine c2c_1m_x
+   end subroutine c2c_1m_x
 
-  ! c2c transform, multiple 1D FFTs in y direction
-  subroutine c2c_1m_y(inout, isign, decomp)
+   ! c2c transform, multiple 1D FFTs in y direction
+   subroutine c2c_1m_y(inout, isign, decomp)
 
-    !$acc routine(spcfft) seq
+      !$acc routine(spcfft) seq
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer :: i,j,k
+      integer :: i, j, k
 
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do k=1,decomp%ysz(3)
-       do i=1,decomp%ysz(1)
-          do j=1,decomp%ysz(2)
-             buf(j) = inout(i,j,k)
-          end do
-          call spcfft(buf,decomp%ysz(2),isign,scratch)
-          do j=1,decomp%ysz(2)
-             inout(i,j,k) = buf(j)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do k = 1, decomp%ysz(3)
+         do i = 1, decomp%ysz(1)
+            do j = 1, decomp%ysz(2)
+               buf(j) = inout(i, j, k)
+            end do
+            call spcfft(buf, decomp%ysz(2), isign, scratch)
+            do j = 1, decomp%ysz(2)
+               inout(i, j, k) = buf(j)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
 
-    return
+      return
 
-  end subroutine c2c_1m_y
+   end subroutine c2c_1m_y
 
-  ! c2c transform, multiple 1D FFTs in z direction
-  subroutine c2c_1m_z(inout, isign, decomp)
+   ! c2c transform, multiple 1D FFTs in z direction
+   subroutine c2c_1m_z(inout, isign, decomp)
 
-    !$acc routine(spcfft) seq
+      !$acc routine(spcfft) seq
+
+      implicit none
 
-    implicit none
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+      integer, intent(IN) :: isign
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    integer, intent(IN) :: isign
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer :: i, j, k
 
-    integer :: i,j,k
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do j = 1, decomp%zsz(2)
+         do i = 1, decomp%zsz(1)
+            do k = 1, decomp%zsz(3)
+               buf(k) = inout(i, j, k)
+            end do
+            call spcfft(buf, decomp%zsz(3), isign, scratch)
+            do k = 1, decomp%zsz(3)
+               inout(i, j, k) = buf(k)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
 
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do j=1,decomp%zsz(2)
-       do i=1,decomp%zsz(1)
-          do k=1,decomp%zsz(3)
-             buf(k) = inout(i,j,k)
-          end do
-          call spcfft(buf,decomp%zsz(3),isign,scratch)
-          do k=1,decomp%zsz(3)
-             inout(i,j,k) = buf(k)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
+      return
 
-    return
+   end subroutine c2c_1m_z
 
-  end subroutine c2c_1m_z
-
-  ! r2c transform, multiple 1D FFTs in x direction
-  subroutine r2c_1m_x(input, output)
-
-    !$acc routine(spcfft) seq
-
-    implicit none
-
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
-
-    integer :: i,j,k, s1,s2,s3, d1
-
-    s1 = size(input,1)
-    s2 = size(input,2)
-    s3 = size(input,3)
-    d1 = size(output,1)
-
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do k=1,s3
-       do j=1,s2
-          ! Glassman's FFT is c2c only, 
-          ! needing some pre- and post-processing for r2c
-          ! pack real input in complex storage
-          do i=1,s1
-             buf(i) = cmplx(input(i,j,k),0._mytype, kind=mytype)
-          end do
-          call spcfft(buf,s1,-1,scratch)
-          ! note d1 ~ s1/2+1
-          ! simply drop the redundant part of the complex output
-          do i=1,d1
-             output(i,j,k) = buf(i)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
-
-    return
-
-  end subroutine r2c_1m_x
-
-  ! r2c transform, multiple 1D FFTs in z direction
-  subroutine r2c_1m_z(input, output)
-
-    !$acc routine(spcfft) seq
-
-    implicit none
-
-    real(mytype), dimension(:,:,:), intent(IN)  ::  input
-    complex(mytype), dimension(:,:,:), intent(OUT) :: output
-
-    integer :: i,j,k, s1,s2,s3, d3
-
-    s1 = size(input,1)
-    s2 = size(input,2)
-    s3 = size(input,3)
-    d3 = size(output,3)
-
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do j=1,s2
-       do i=1,s1
-          ! Glassman's FFT is c2c only, 
-          ! needing some pre- and post-processing for r2c
-          ! pack real input in complex storage
-          do k=1,s3
-             buf(k) = cmplx(input(i,j,k),0._mytype, kind=mytype)
-          end do
-          call spcfft(buf,s3,-1,scratch)
-          ! note d3 ~ s3/2+1
-          ! simply drop the redundant part of the complex output
-          do k=1,d3
-             output(i,j,k) = buf(k)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
-
-    return
-
-  end subroutine r2c_1m_z
-
-  ! c2r transform, multiple 1D FFTs in x direction
-  subroutine c2r_1m_x(input, output)
-
-    !$acc routine(spcfft) seq
-
-    implicit none
-
-    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
-
-    integer :: i,j,k, d1,d2,d3
-
-    d1 = size(output,1)
-    d2 = size(output,2)
-    d3 = size(output,3)
-
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do k=1,d3
-       do j=1,d2
-          ! Glassman's FFT is c2c only, 
-          ! needing some pre- and post-processing for c2r
-          do i=1,d1/2+1
-             buf(i) = input(i,j,k)
-          end do
-          ! expanding to a full-size complex array
-          ! For odd N, the storage is:
-          !  1, 2, ...... N/2+1   integer division rounded down
-          !     N, ...... N/2+2   => a(i) is conjugate of a(N+2-i)
-          ! For even N, the storage is:
-          !  1, 2, ...... N/2  , N/2+1
-          !     N, ...... N/2+2  again a(i) conjugate of a(N+2-i)
-          do i=d1/2+2,d1
-             buf(i) =  conjg(buf(d1+2-i))
-          end do
-          call spcfft(buf,d1,1,scratch)
-          do i=1,d1
-             ! simply drop imaginary part
-             output(i,j,k) = real(buf(i), kind=mytype)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
-
-    return
-
-  end subroutine c2r_1m_x
-
-  ! c2r transform, multiple 1D FFTs in z direction
-  subroutine c2r_1m_z(input, output)
-
-    !$acc routine(spcfft) seq
-
-    implicit none
-
-    complex(mytype), dimension(:,:,:), intent(IN)  ::  input
-    real(mytype), dimension(:,:,:), intent(OUT) :: output
-
-    integer :: i,j,k, d1,d2,d3
-
-    d1 = size(output,1)
-    d2 = size(output,2)
-    d3 = size(output,3)
-
-    !$acc parallel loop gang vector collapse(2) private(buf, scratch)
-    do j=1,d2
-       do i=1,d1
-          do k=1,d3/2+1
-             buf(k) = input(i,j,k)
-          end do
-          do k=d3/2+2,d3
-             buf(k) =  conjg(buf(d3+2-k))
-          end do
-          call spcfft(buf,d3,1,scratch)
-          do k=1,d3
-             output(i,j,k) = real(buf(k), kind=mytype)
-          end do
-       end do
-    end do
-    !$acc end parallel loop
-
-    return
-
-  end subroutine c2r_1m_z
+   ! r2c transform, multiple 1D FFTs in x direction
+   subroutine r2c_1m_x(input, output)
 
+      !$acc routine(spcfft) seq
+
+      implicit none
+
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+
+      integer :: i, j, k, s1, s2, s3, d1
+
+      s1 = size(input, 1)
+      s2 = size(input, 2)
+      s3 = size(input, 3)
+      d1 = size(output, 1)
+
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do k = 1, s3
+         do j = 1, s2
+            ! Glassman's FFT is c2c only,
+            ! needing some pre- and post-processing for r2c
+            ! pack real input in complex storage
+            do i = 1, s1
+               buf(i) = cmplx(input(i, j, k), 0._mytype, kind=mytype)
+            end do
+            call spcfft(buf, s1, -1, scratch)
+            ! note d1 ~ s1/2+1
+            ! simply drop the redundant part of the complex output
+            do i = 1, d1
+               output(i, j, k) = buf(i)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
+
+      return
+
+   end subroutine r2c_1m_x
+
+   ! r2c transform, multiple 1D FFTs in z direction
+   subroutine r2c_1m_z(input, output)
+
+      !$acc routine(spcfft) seq
+
+      implicit none
+
+      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(OUT) :: output
+
+      integer :: i, j, k, s1, s2, s3, d3
+
+      s1 = size(input, 1)
+      s2 = size(input, 2)
+      s3 = size(input, 3)
+      d3 = size(output, 3)
+
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do j = 1, s2
+         do i = 1, s1
+            ! Glassman's FFT is c2c only,
+            ! needing some pre- and post-processing for r2c
+            ! pack real input in complex storage
+            do k = 1, s3
+               buf(k) = cmplx(input(i, j, k), 0._mytype, kind=mytype)
+            end do
+            call spcfft(buf, s3, -1, scratch)
+            ! note d3 ~ s3/2+1
+            ! simply drop the redundant part of the complex output
+            do k = 1, d3
+               output(i, j, k) = buf(k)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
+
+      return
+
+   end subroutine r2c_1m_z
+
+   ! c2r transform, multiple 1D FFTs in x direction
+   subroutine c2r_1m_x(input, output)
+
+      !$acc routine(spcfft) seq
+
+      implicit none
+
+      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
+
+      integer :: i, j, k, d1, d2, d3
+
+      d1 = size(output, 1)
+      d2 = size(output, 2)
+      d3 = size(output, 3)
+
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do k = 1, d3
+         do j = 1, d2
+            ! Glassman's FFT is c2c only,
+            ! needing some pre- and post-processing for c2r
+            do i = 1, d1/2 + 1
+               buf(i) = input(i, j, k)
+            end do
+            ! expanding to a full-size complex array
+            ! For odd N, the storage is:
+            !  1, 2, ...... N/2+1   integer division rounded down
+            !     N, ...... N/2+2   => a(i) is conjugate of a(N+2-i)
+            ! For even N, the storage is:
+            !  1, 2, ...... N/2  , N/2+1
+            !     N, ...... N/2+2  again a(i) conjugate of a(N+2-i)
+            do i = d1/2 + 2, d1
+               buf(i) = conjg(buf(d1 + 2 - i))
+            end do
+            call spcfft(buf, d1, 1, scratch)
+            do i = 1, d1
+               ! simply drop imaginary part
+               output(i, j, k) = real(buf(i), kind=mytype)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
+
+      return
+
+   end subroutine c2r_1m_x
+
+   ! c2r transform, multiple 1D FFTs in z direction
+   subroutine c2r_1m_z(input, output)
+
+      !$acc routine(spcfft) seq
+
+      implicit none
+
+      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(OUT) :: output
+
+      integer :: i, j, k, d1, d2, d3
+
+      d1 = size(output, 1)
+      d2 = size(output, 2)
+      d3 = size(output, 3)
+
+      !$acc parallel loop gang vector collapse(2) private(buf, scratch)
+      do j = 1, d2
+         do i = 1, d1
+            do k = 1, d3/2 + 1
+               buf(k) = input(i, j, k)
+            end do
+            do k = d3/2 + 2, d3
+               buf(k) = conjg(buf(d3 + 2 - k))
+            end do
+            call spcfft(buf, d3, 1, scratch)
+            do k = 1, d3
+               output(i, j, k) = real(buf(k), kind=mytype)
+            end do
+         end do
+      end do
+      !$acc end parallel loop
+
+      return
+
+   end subroutine c2r_1m_z
 
 #include "fft_common_3d.f90"
 
-
 end module decomp_2d_fft
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index b50120aa..cebed05c 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,666 +13,660 @@
 
 module decomp_2d_fft
 
-  use decomp_2d  ! 2D decomposition module
-  use MKL_DFTI   ! MKL FFT module
+   use decomp_2d  ! 2D decomposition module
+   use MKL_DFTI   ! MKL FFT module
 
-  implicit none
+   implicit none
 
-  private        ! Make everything private unless declared public
+   private        ! Make everything private unless declared public
 
-  ! engine-specific global variables
+   ! engine-specific global variables
 
-  ! Descriptors for MKL FFT, one for each set of 1D FFTs
-  !  for c2c transforms
-  type(DFTI_DESCRIPTOR), pointer :: c2c_x, c2c_y, c2c_z     
-  !  for r2c/c2r transforms, PHYSICAL_IN_X
-  type(DFTI_DESCRIPTOR), pointer :: r2c_x, c2c_y2, c2c_z2, c2r_x
-  !  for r2c/c2r transforms, PHYSICAL_IN_Z
-  type(DFTI_DESCRIPTOR), pointer :: r2c_z, c2c_x2, c2r_z
+   ! Descriptors for MKL FFT, one for each set of 1D FFTs
+   !  for c2c transforms
+   type(DFTI_DESCRIPTOR), pointer :: c2c_x, c2c_y, c2c_z
+   !  for r2c/c2r transforms, PHYSICAL_IN_X
+   type(DFTI_DESCRIPTOR), pointer :: r2c_x, c2c_y2, c2c_z2, c2r_x
+   !  for r2c/c2r transforms, PHYSICAL_IN_Z
+   type(DFTI_DESCRIPTOR), pointer :: r2c_z, c2c_x2, c2r_z
 
-  ! common code used for all engines, including global variables, 
-  ! generic interface definitions and several subroutines
+   ! common code used for all engines, including global variables,
+   ! generic interface definitions and several subroutines
 #include "fft_common.f90"
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time initialisations for the FFT engine
+   !  This routine performs one-time initialisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine init_fft_engine
+   subroutine init_fft_engine
 
-    implicit none
+      implicit none
 
-    if (nrank==0) then
-       write(*,*) ' '
-       write(*,*) '***** Using the MKL engine *****'
-       write(*,*) ' '
-    end if
+      if (nrank == 0) then
+         write (*, *) ' '
+         write (*, *) '***** Using the MKL engine *****'
+         write (*, *) ' '
+      end if
 #ifdef OVERWRITE
-    call decomp_2d_warning(0, "MKL FFT engine does not support the OVERWRITE flag.")
+      call decomp_2d_warning(0, "MKL FFT engine does not support the OVERWRITE flag.")
 #endif
 
-    ! For C2C transforms
-    call c2c_1m_x_plan(c2c_x, ph)
-    call c2c_1m_y_plan(c2c_y, ph)
-    call c2c_1m_z_plan(c2c_z, ph)
-
-    ! For R2C/C2R tranfroms with physical space in X-pencil
-    if (format == PHYSICAL_IN_X) then
-       call r2c_1m_x_plan(r2c_x, ph, sp, -1)
-       call c2c_1m_y_plan(c2c_y2, sp)
-       call c2c_1m_z_plan(c2c_z2, sp)
-       call r2c_1m_x_plan(c2r_x, ph, sp,  1)
+      ! For C2C transforms
+      call c2c_1m_x_plan(c2c_x, ph)
+      call c2c_1m_y_plan(c2c_y, ph)
+      call c2c_1m_z_plan(c2c_z, ph)
 
-       ! For R2C/C2R tranfroms with physical space in Z-pencil
-    else if (format == PHYSICAL_IN_Z) then
-       call r2c_1m_z_plan(r2c_z, ph, sp, -1)
-       call c2c_1m_y_plan(c2c_y2, sp)
-       call c2c_1m_x_plan(c2c_x2, sp)
-       call r2c_1m_z_plan(c2r_z, ph, sp,  1)
-    end if
+      ! For R2C/C2R tranfroms with physical space in X-pencil
+      if (format == PHYSICAL_IN_X) then
+         call r2c_1m_x_plan(r2c_x, ph, sp, -1)
+         call c2c_1m_y_plan(c2c_y2, sp)
+         call c2c_1m_z_plan(c2c_z2, sp)
+         call r2c_1m_x_plan(c2r_x, ph, sp, 1)
 
-    return
-  end subroutine init_fft_engine
+         ! For R2C/C2R tranfroms with physical space in Z-pencil
+      else if (format == PHYSICAL_IN_Z) then
+         call r2c_1m_z_plan(r2c_z, ph, sp, -1)
+         call c2c_1m_y_plan(c2c_y2, sp)
+         call c2c_1m_x_plan(c2c_x2, sp)
+         call r2c_1m_z_plan(c2r_z, ph, sp, 1)
+      end if
 
+      return
+   end subroutine init_fft_engine
 
-  ! Return an MKL plan for multiple 1D c2c FFTs in X direction
-  subroutine c2c_1m_x_plan(desc, decomp)
+   ! Return an MKL plan for multiple 1D c2c FFTs in X direction
+   subroutine c2c_1m_x_plan(desc, decomp)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer :: status
+      integer :: status
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_COMPLEX, 1, decomp%xsz(1))
-#else       
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_COMPLEX, 1, decomp%xsz(1))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_COMPLEX, 1, decomp%xsz(1))
+#else
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_COMPLEX, 1, decomp%xsz(1))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-         decomp%xsz(2)*decomp%xsz(3))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, decomp%xsz(1))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, decomp%xsz(1))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+                            decomp%xsz(2)*decomp%xsz(3))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, decomp%xsz(1))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, decomp%xsz(1))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
 
-    return
-  end subroutine c2c_1m_x_plan
+      return
+   end subroutine c2c_1m_x_plan
 
-  ! Return an MKL plan for multiple 1D c2c FFTs in Y direction
-  subroutine c2c_1m_y_plan(desc, decomp)
+   ! Return an MKL plan for multiple 1D c2c FFTs in Y direction
+   subroutine c2c_1m_y_plan(desc, decomp)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer :: status, strides(2)
+      integer :: status, strides(2)
 
-    ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
-    ! done one Z-plane at a time. So plan for 2D data sets here.
+      ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
+      ! done one Z-plane at a time. So plan for 2D data sets here.
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_COMPLEX, 1, decomp%ysz(2))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_COMPLEX, 1, decomp%ysz(2))
 #else
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_COMPLEX, 1, decomp%ysz(2))
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_COMPLEX, 1, decomp%ysz(2))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, decomp%ysz(1))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    strides(1) = 0
-    strides(2) = decomp%ysz(1)
-    status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-    return
-  end subroutine c2c_1m_y_plan
-
-  ! Return an MKL plan for multiple 1D c2c FFTs in Z direction
-  subroutine c2c_1m_z_plan(desc, decomp)
-
-    implicit none
-
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-
-    integer :: status, strides(2)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, decomp%ysz(1))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      strides(1) = 0
+      strides(2) = decomp%ysz(1)
+      status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+      return
+   end subroutine c2c_1m_y_plan
+
+   ! Return an MKL plan for multiple 1D c2c FFTs in Z direction
+   subroutine c2c_1m_z_plan(desc, decomp)
+
+      implicit none
+
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+
+      integer :: status, strides(2)
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_COMPLEX, 1, decomp%zsz(3))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_COMPLEX, 1, decomp%zsz(3))
 #else
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_COMPLEX, 1, decomp%zsz(3))
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_COMPLEX, 1, decomp%zsz(3))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-         decomp%zsz(1)*decomp%zsz(2))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    strides(1) = 0
-    strides(2) = decomp%zsz(1)*decomp%zsz(2)
-    status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-    return
-  end subroutine c2c_1m_z_plan
-
-  ! Return an MKL plan for multiple 1D r2c FFTs in X direction
-  subroutine r2c_1m_x_plan(desc, decomp_ph, decomp_sp, direction)
-
-    implicit none
-
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
-    integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
-
-    integer :: status
-
-    ! c2r and r2c plans are almost the same, just swap input/output
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+                            decomp%zsz(1)*decomp%zsz(2))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      strides(1) = 0
+      strides(2) = decomp%zsz(1)*decomp%zsz(2)
+      status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+      return
+   end subroutine c2c_1m_z_plan
+
+   ! Return an MKL plan for multiple 1D r2c FFTs in X direction
+   subroutine r2c_1m_x_plan(desc, decomp_ph, decomp_sp, direction)
+
+      implicit none
+
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
+      integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
+
+      integer :: status
+
+      ! c2r and r2c plans are almost the same, just swap input/output
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_REAL, 1, decomp_ph%xsz(1))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_REAL, 1, decomp_ph%xsz(1))
 #else
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_REAL, 1, decomp_ph%xsz(1))
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_REAL, 1, decomp_ph%xsz(1))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-         decomp_ph%xsz(2)*decomp_ph%xsz(3))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
-         DFTI_COMPLEX_COMPLEX)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    if (direction == -1) then  ! r2c
-       status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
-            decomp_ph%xsz(1))
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
-            decomp_sp%xsz(1))
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    else if (direction == 1) then  ! c2r
-       status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
-            decomp_sp%xsz(1))
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
-            decomp_ph%xsz(1))
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    end if
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-    return
-  end subroutine r2c_1m_x_plan
-
-  ! Return an MKL plan for multiple 1D r2c FFTs in Z direction
-  subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
-
-    implicit none
-
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
-    integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
-
-    integer :: status, strides(2)
-
-    ! c2r and r2c plans are almost the same, just swap input/output
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+                            decomp_ph%xsz(2)*decomp_ph%xsz(3))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
+                            DFTI_COMPLEX_COMPLEX)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      if (direction == -1) then  ! r2c
+         status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
+                               decomp_ph%xsz(1))
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+         status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
+                               decomp_sp%xsz(1))
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      else if (direction == 1) then  ! c2r
+         status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, &
+                               decomp_sp%xsz(1))
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+         status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, &
+                               decomp_ph%xsz(1))
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      end if
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+      return
+   end subroutine r2c_1m_x_plan
+
+   ! Return an MKL plan for multiple 1D r2c FFTs in Z direction
+   subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
+
+      implicit none
+
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      TYPE(DECOMP_INFO), intent(IN) :: decomp_ph, decomp_sp
+      integer, intent(IN) :: direction ! (-1=r2c; 1=c2r)
+
+      integer :: status, strides(2)
+
+      ! c2r and r2c plans are almost the same, just swap input/output
 
 #ifdef DOUBLE_PREC
-    status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
-         DFTI_REAL, 1, decomp_ph%zsz(3))
+      status = DftiCreateDescriptor(desc, DFTI_DOUBLE, &
+                                    DFTI_REAL, 1, decomp_ph%zsz(3))
 #else
-    status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
-         DFTI_REAL, 1, decomp_ph%zsz(3))
+      status = DftiCreateDescriptor(desc, DFTI_SINGLE, &
+                                    DFTI_REAL, 1, decomp_ph%zsz(3))
 #endif
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
-    status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-         decomp_ph%zsz(1)*decomp_ph%zsz(2))
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
-         DFTI_COMPLEX_COMPLEX)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    strides(1) = 0
-    strides(2) = decomp_ph%zsz(1)*decomp_ph%zsz(2)
-    if (direction == -1) then
-       status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-    else if (direction == 1) then
-       status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-    end if
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    strides(2) = decomp_sp%zsz(1)*decomp_sp%zsz(2)
-    if (direction == -1) then
-       status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
-    else if (direction == 1) then
-       status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
-    end if
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-    status = DftiCommitDescriptor(desc)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
-
-    return
-  end subroutine r2c_1m_z_plan
-
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+      status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
+                            decomp_ph%zsz(1)*decomp_ph%zsz(2))
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
+                            DFTI_COMPLEX_COMPLEX)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      strides(1) = 0
+      strides(2) = decomp_ph%zsz(1)*decomp_ph%zsz(2)
+      if (direction == -1) then
+         status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+      else if (direction == 1) then
+         status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+      end if
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      strides(2) = decomp_sp%zsz(1)*decomp_sp%zsz(2)
+      if (direction == -1) then
+         status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
+      else if (direction == 1) then
+         status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
+      end if
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+      status = DftiCommitDescriptor(desc)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCommitDescriptor")
+
+      return
+   end subroutine r2c_1m_z_plan
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !  This routine performs one-time finalisations for the FFT engine
+   !  This routine performs one-time finalisations for the FFT engine
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine finalize_fft_engine
-
-    implicit none
-
-    integer :: status
-
-    status = DftiFreeDescriptor(c2c_x)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    status = DftiFreeDescriptor(c2c_y)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    status = DftiFreeDescriptor(c2c_z)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    if (format==PHYSICAL_IN_X) then
-       status = DftiFreeDescriptor(r2c_x)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-       status = DftiFreeDescriptor(c2c_z2)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-       status = DftiFreeDescriptor(c2r_x)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    else if (format==PHYSICAL_IN_Z) then
-       status = DftiFreeDescriptor(r2c_z)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-       status = DftiFreeDescriptor(c2c_x2)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-       status = DftiFreeDescriptor(c2r_z)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-    end if
-    status = DftiFreeDescriptor(c2c_y2)
-    if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
-
-    return
-  end subroutine finalize_fft_engine
-
+   subroutine finalize_fft_engine
+
+      implicit none
+
+      integer :: status
+
+      status = DftiFreeDescriptor(c2c_x)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      status = DftiFreeDescriptor(c2c_y)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      status = DftiFreeDescriptor(c2c_z)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      if (format == PHYSICAL_IN_X) then
+         status = DftiFreeDescriptor(r2c_x)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+         status = DftiFreeDescriptor(c2c_z2)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+         status = DftiFreeDescriptor(c2r_x)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      else if (format == PHYSICAL_IN_Z) then
+         status = DftiFreeDescriptor(r2c_z)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+         status = DftiFreeDescriptor(c2c_x2)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+         status = DftiFreeDescriptor(c2r_z)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+      end if
+      status = DftiFreeDescriptor(c2c_y2)
+      if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiFreeDescriptor")
+
+      return
+   end subroutine finalize_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D FFT - complex to complex
+   ! 3D FFT - complex to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2c(in, out, isign)
+   subroutine fft_3d_c2c(in, out, isign)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: in
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out
-    integer, intent(IN) :: isign   
+      complex(mytype), dimension(:, :, :), intent(IN) :: in
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out
+      integer, intent(IN) :: isign
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1,wk2,wk2b,wk3
-    integer :: k, status
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
+      integer :: k, status
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
-    if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_FORWARD .OR.  &
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_BACKWARD) then
-
-       ! ===== 1D FFTs in X =====
-       allocate (wk1(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-       !       if (isign==DECOMP_2D_FFT_FORWARD) then
-       !          status = DftiComputeForward(c2c_x, in(:,1,1), wk1(:,1,1))
-       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-       !          status = DftiComputeBackward(c2c_x, in(:,1,1), wk1(:,1,1))
-       !       end if
-       status = wrapper_c2c(c2c_x, in, wk1, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-       ! ===== Swap X --> Y =====
-       allocate (wk2(ph%ysz(1),ph%ysz(2),ph%ysz(3)))
-       call transpose_x_to_y(wk1,wk2,ph)
-
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(ph%ysz(1),ph%ysz(2),ph%ysz(3))) 
-       do k=1,ph%xsz(3) ! one Z-plane at a time
-          !          if (isign==DECOMP_2D_FFT_FORWARD) then
-          !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-          !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
-          !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-          !          end if
-          status = wrapper_c2c(c2c_y, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
-
-       ! ===== Swap Y --> Z =====
-       allocate (wk3(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-       call transpose_y_to_z(wk2b,wk3,ph)
-
-       ! ===== 1D FFTs in Z =====
-       !       if (isign==DECOMP_2D_FFT_FORWARD) then
-       !          status = DftiComputeForward(c2c_z, wk3(:,1,1), out(:,1,1))
-       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-       !          status = DftiComputeBackward(c2c_z, wk3(:,1,1), out(:,1,1))
-       !       end if
-       status = wrapper_c2c(c2c_z, wk3, out, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-    else if (format==PHYSICAL_IN_X .AND. isign==DECOMP_2D_FFT_BACKWARD &
-         .OR. & 
-         format==PHYSICAL_IN_Z .AND. isign==DECOMP_2D_FFT_FORWARD) then
-
-       ! ===== 1D FFTs in Z =====
-       allocate (wk1(ph%zsz(1),ph%zsz(2),ph%zsz(3)))
-       !       if (isign==DECOMP_2D_FFT_FORWARD) then
-       !          status = DftiComputeForward(c2c_z, in(:,1,1), wk1(:,1,1))
-       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-       !          status = DftiComputeBackward(c2c_z, in(:,1,1), wk1(:,1,1))
-       !       end if
-       status = wrapper_c2c(c2c_z, in, wk1, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-       ! ===== Swap Z --> Y =====
-       allocate (wk2(ph%ysz(1),ph%ysz(2),ph%ysz(3)))
-       call transpose_z_to_y(wk1,wk2,ph)
-
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(ph%ysz(1),ph%ysz(2),ph%ysz(3)))
-       do k=1,ph%xsz(3) ! one Z-plane at a time
-          !          if (isign==DECOMP_2D_FFT_FORWARD) then
-          !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-          !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
-          !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
-          !          end if
-          status = wrapper_c2c(c2c_y, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
-
-       ! ===== Swap Y --> X =====
-       allocate (wk3(ph%xsz(1),ph%xsz(2),ph%xsz(3)))
-       call transpose_y_to_x(wk2b,wk3,ph)
-
-       ! ===== 1D FFTs in X =====
-       !       if (isign==DECOMP_2D_FFT_FORWARD) then
-       !          status = DftiComputeForward(c2c_x, wk3(:,1,1), out(:,1,1))
-       !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
-       !          status = DftiComputeBackward(c2c_x, wk3(:,1,1), out(:,1,1))
-       !       end if
-       status = wrapper_c2c(c2c_x, wk3, out, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-
-    end if
+      if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
+          format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
+
+         ! ===== 1D FFTs in X =====
+         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+         !       if (isign==DECOMP_2D_FFT_FORWARD) then
+         !          status = DftiComputeForward(c2c_x, in(:,1,1), wk1(:,1,1))
+         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+         !          status = DftiComputeBackward(c2c_x, in(:,1,1), wk1(:,1,1))
+         !       end if
+         status = wrapper_c2c(c2c_x, in, wk1, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+         ! ===== Swap X --> Y =====
+         allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+         call transpose_x_to_y(wk1, wk2, ph)
+
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+         do k = 1, ph%xsz(3) ! one Z-plane at a time
+            !          if (isign==DECOMP_2D_FFT_FORWARD) then
+            !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+            !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
+            !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+            !          end if
+            status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+
+         ! ===== Swap Y --> Z =====
+         allocate (wk3(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+         call transpose_y_to_z(wk2b, wk3, ph)
+
+         ! ===== 1D FFTs in Z =====
+         !       if (isign==DECOMP_2D_FFT_FORWARD) then
+         !          status = DftiComputeForward(c2c_z, wk3(:,1,1), out(:,1,1))
+         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+         !          status = DftiComputeBackward(c2c_z, wk3(:,1,1), out(:,1,1))
+         !       end if
+         status = wrapper_c2c(c2c_z, wk3, out, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+      else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
+               .OR. &
+               format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
+
+         ! ===== 1D FFTs in Z =====
+         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+         !       if (isign==DECOMP_2D_FFT_FORWARD) then
+         !          status = DftiComputeForward(c2c_z, in(:,1,1), wk1(:,1,1))
+         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+         !          status = DftiComputeBackward(c2c_z, in(:,1,1), wk1(:,1,1))
+         !       end if
+         status = wrapper_c2c(c2c_z, in, wk1, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+         ! ===== Swap Z --> Y =====
+         allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+         call transpose_z_to_y(wk1, wk2, ph)
+
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+         do k = 1, ph%xsz(3) ! one Z-plane at a time
+            !          if (isign==DECOMP_2D_FFT_FORWARD) then
+            !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+            !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
+            !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
+            !          end if
+            status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+
+         ! ===== Swap Y --> X =====
+         allocate (wk3(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+         call transpose_y_to_x(wk2b, wk3, ph)
+
+         ! ===== 1D FFTs in X =====
+         !       if (isign==DECOMP_2D_FFT_FORWARD) then
+         !          status = DftiComputeForward(c2c_x, wk3(:,1,1), out(:,1,1))
+         !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
+         !          status = DftiComputeBackward(c2c_x, wk3(:,1,1), out(:,1,1))
+         !       end if
+         status = wrapper_c2c(c2c_x, wk3, out, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+
+      end if
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
 
-    return
-  end subroutine fft_3d_c2c
-
+      return
+   end subroutine fft_3d_c2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D forward FFT - real to complex
+   ! 3D forward FFT - real to complex
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_r2c(in_r, out_c)
+   subroutine fft_3d_r2c(in_r, out_c)
 
-    implicit none
+      implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: in_r
-    complex(mytype), dimension(:,:,:), intent(OUT) :: out_c
+      real(mytype), dimension(:, :, :), intent(IN) :: in_r
+      complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1,wk2,wk2b,wk3
-    integer :: k, status, isign
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
+      integer :: k, status, isign
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
-    isign = DECOMP_2D_FFT_FORWARD
+      isign = DECOMP_2D_FFT_FORWARD
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in X =====
-       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       !       status = DftiComputeForward(r2c_x, in_r(:,1,1), wk1(:,1,1))       
-       status = wrapper_r2c(r2c_x, in_r, wk1)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
+         ! ===== 1D FFTs in X =====
+         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         !       status = DftiComputeForward(r2c_x, in_r(:,1,1), wk1(:,1,1))
+         status = wrapper_r2c(r2c_x, in_r, wk1)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
-       ! ===== Swap X --> Y =====
-       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       call transpose_x_to_y(wk1,wk2,sp)
+         ! ===== Swap X --> Y =====
+         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         call transpose_x_to_y(wk1, wk2, sp)
 
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       do k=1,sp%ysz(3)
-          !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         do k = 1, sp%ysz(3)
+            !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
 
-       ! ===== Swap Y --> Z =====
-       allocate (wk3(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       call transpose_y_to_z(wk2b,wk3,sp)
+         ! ===== Swap Y --> Z =====
+         allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         call transpose_y_to_z(wk2b, wk3, sp)
 
-       ! ===== 1D FFTs in Z =====
-       !       status = DftiComputeForward(c2c_z2, wk3(:,1,1), out_c(:,1,1))
-       status = wrapper_c2c(c2c_z2, wk3, out_c, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         ! ===== 1D FFTs in Z =====
+         !       status = DftiComputeForward(c2c_z2, wk3(:,1,1), out_c(:,1,1))
+         status = wrapper_c2c(c2c_z2, wk3, out_c, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in Z =====
-       allocate(wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       !       status = DftiComputeForward(r2c_z, in_r(:,1,1), wk1(:,1,1))
-       status = wrapper_r2c(r2c_z, in_r, wk1)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
+         ! ===== 1D FFTs in Z =====
+         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         !       status = DftiComputeForward(r2c_z, in_r(:,1,1), wk1(:,1,1))
+         status = wrapper_r2c(r2c_z, in_r, wk1)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
-       ! ===== Swap Z --> Y =====
-       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       call transpose_z_to_y(wk1,wk2,sp)
+         ! ===== Swap Z --> Y =====
+         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         call transpose_z_to_y(wk1, wk2, sp)
 
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       do k=1,sp%ysz(3)
-          !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         do k = 1, sp%ysz(3)
+            !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
 
-       ! ===== Swap Y --> X =====
-       allocate (wk3(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       call transpose_y_to_x(wk2b,wk3,sp)
+         ! ===== Swap Y --> X =====
+         allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         call transpose_y_to_x(wk2b, wk3, sp)
 
-       ! ===== 1D FFTs in X =====
-       !       status = DftiComputeForward(c2c_x2, wk3(:,1,1), out_c(:,1,1))
-       status = wrapper_c2c(c2c_x2, wk3, out_c, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         ! ===== 1D FFTs in X =====
+         !       status = DftiComputeForward(c2c_x2, wk3(:,1,1), out_c(:,1,1))
+         status = wrapper_c2c(c2c_x2, wk3, out_c, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-    end if
+      end if
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
 
-    return
-  end subroutine fft_3d_r2c
-
+      return
+   end subroutine fft_3d_r2c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! 3D inverse FFT - complex to real
+   ! 3D inverse FFT - complex to real
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine fft_3d_c2r(in_c, out_r)
+   subroutine fft_3d_c2r(in_c, out_r)
 
-    implicit none
+      implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: in_c
-    real(mytype), dimension(:,:,:), intent(OUT) :: out_r
+      complex(mytype), dimension(:, :, :), intent(IN) :: in_c
+      real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk1,wk2,wk2b,wk3
-    integer :: k, status, isign
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
+      integer :: k, status, isign
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
-    isign = DECOMP_2D_FFT_BACKWARD
+      isign = DECOMP_2D_FFT_BACKWARD
 
-    if (format==PHYSICAL_IN_X) then
+      if (format == PHYSICAL_IN_X) then
 
-       ! ===== 1D FFTs in Z =====
-       allocate (wk1(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       !       status = DftiComputeBackward(c2c_z2, in_c(:,1,1), wk1(:,1,1))
-       status = wrapper_c2c(c2c_z2, in_c, wk1, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         ! ===== 1D FFTs in Z =====
+         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         !       status = DftiComputeBackward(c2c_z2, in_c(:,1,1), wk1(:,1,1))
+         status = wrapper_c2c(c2c_z2, in_c, wk1, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-       ! ===== Swap Z --> Y =====
-       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       call transpose_z_to_y(wk1,wk2,sp)
+         ! ===== Swap Z --> Y =====
+         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         call transpose_z_to_y(wk1, wk2, sp)
 
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       do k=1,sp%ysz(3)
-          !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         do k = 1, sp%ysz(3)
+            !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
 
-       ! ===== Swap Y --> X =====
-       allocate (wk3(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       call transpose_y_to_x(wk2b,wk3,sp)
+         ! ===== Swap Y --> X =====
+         allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         call transpose_y_to_x(wk2b, wk3, sp)
 
-       ! ===== 1D FFTs in X =====
-       !       status = DftiComputeBackward(c2r_x, wk3(:,1,1), out_r(:,1,1))
-       status =  wrapper_c2r(c2r_x, wk3, out_r)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
+         ! ===== 1D FFTs in X =====
+         !       status = DftiComputeBackward(c2r_x, wk3(:,1,1), out_r(:,1,1))
+         status = wrapper_c2r(c2r_x, wk3, out_r)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
 
-    else if (format==PHYSICAL_IN_Z) then
+      else if (format == PHYSICAL_IN_Z) then
 
-       ! ===== 1D FFTs in X =====
-       allocate(wk1(sp%xsz(1),sp%xsz(2),sp%xsz(3)))
-       !       status = DftiComputeBackward(c2c_x2, in_c(:,1,1), wk1(:,1,1))
-       status = wrapper_c2c(c2c_x2, in_c, wk1, isign)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         ! ===== 1D FFTs in X =====
+         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         !       status = DftiComputeBackward(c2c_x2, in_c(:,1,1), wk1(:,1,1))
+         status = wrapper_c2c(c2c_x2, in_c, wk1, isign)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
-       ! ===== Swap X --> Y =====
-       allocate (wk2(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       call transpose_x_to_y(wk1,wk2,sp)
+         ! ===== Swap X --> Y =====
+         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         call transpose_x_to_y(wk1, wk2, sp)
 
-       ! ===== 1D FFTs in Y =====
-       allocate (wk2b(sp%ysz(1),sp%ysz(2),sp%ysz(3)))
-       do k=1,sp%ysz(3)
-          !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-          status = wrapper_c2c(c2c_y2, wk2(1,1,k), wk2b(1,1,k), isign)
-          if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
-       end do
+         ! ===== 1D FFTs in Y =====
+         allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+         do k = 1, sp%ysz(3)
+            !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
+            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
 
-       ! ===== Swap Y --> Z =====
-       allocate (wk3(sp%zsz(1),sp%zsz(2),sp%zsz(3)))
-       call transpose_y_to_z(wk2b,wk3,sp)
+         ! ===== Swap Y --> Z =====
+         allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         call transpose_y_to_z(wk2b, wk3, sp)
 
-       ! ===== 1D FFTs in Z =====
-       !       status = DftiComputeBackward(c2r_z, wk3(:,1,1), out_r(:,1,1))
-       status =  wrapper_c2r(c2r_z, wk3, out_r)
-       if (status/=0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
+         ! ===== 1D FFTs in Z =====
+         !       status = DftiComputeBackward(c2r_z, wk3(:,1,1), out_r(:,1,1))
+         status = wrapper_c2r(c2r_z, wk3, out_r)
+         if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
 
-    end if
+      end if
 
 #ifdef PROFILER
-if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
+      if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif
 
-    return
-  end subroutine fft_3d_c2r
-
+      return
+   end subroutine fft_3d_c2r
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Wrapper functions so that one can pass 3D arrays to DftiCompute
-  !  -- MKL accepts only 1D arrays as input/output for its multi-
-  !     dimensional FFTs.
-  !  -- Using EQUIVALENCE as suggested by MKL documents is impossible 
-  !     for allocated arrays, not to mention bad coding style
-  !  -- All code commented out above may well work but not safe. There 
-  !     is no guarantee that compiler wouldn't make copies of 1D arrays
-  !     (which would contain only one slice of the original 3D data) 
-  !     rather than referring to the same memory address, i.e. 3D array
-  !     A and 1D array A(:,1,1) may refer to different memory location.
-  !  -- Using the following wrappers is safe and standard conforming.
+   ! Wrapper functions so that one can pass 3D arrays to DftiCompute
+   !  -- MKL accepts only 1D arrays as input/output for its multi-
+   !     dimensional FFTs.
+   !  -- Using EQUIVALENCE as suggested by MKL documents is impossible
+   !     for allocated arrays, not to mention bad coding style
+   !  -- All code commented out above may well work but not safe. There
+   !     is no guarantee that compiler wouldn't make copies of 1D arrays
+   !     (which would contain only one slice of the original 3D data)
+   !     rather than referring to the same memory address, i.e. 3D array
+   !     A and 1D array A(:,1,1) may refer to different memory location.
+   !  -- Using the following wrappers is safe and standard conforming.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  integer function wrapper_c2c(desc, in, out, isign)
+   integer function wrapper_c2c(desc, in, out, isign)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    complex(mytype), dimension(*) :: in, out
-    integer :: isign, status
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      complex(mytype), dimension(*) :: in, out
+      integer :: isign, status
 
-    if (isign == DECOMP_2D_FFT_FORWARD) then
-       status = DftiComputeForward(desc, in, out)
-    else if (isign == DECOMP_2D_FFT_BACKWARD) then
-       status = DftiComputeBackward(desc, in, out)
-    end if
+      if (isign == DECOMP_2D_FFT_FORWARD) then
+         status = DftiComputeForward(desc, in, out)
+      else if (isign == DECOMP_2D_FFT_BACKWARD) then
+         status = DftiComputeBackward(desc, in, out)
+      end if
 
-    wrapper_c2c = status
+      wrapper_c2c = status
 
-    return
-  end function wrapper_c2c
+      return
+   end function wrapper_c2c
 
-  integer function wrapper_r2c(desc, in, out)
+   integer function wrapper_r2c(desc, in, out)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    real(mytype), dimension(*) :: in 
-    complex(mytype), dimension(*) :: out
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      real(mytype), dimension(*) :: in
+      complex(mytype), dimension(*) :: out
 
-    wrapper_r2c = DftiComputeForward(desc, in, out)
+      wrapper_r2c = DftiComputeForward(desc, in, out)
 
-    return
-  end function wrapper_r2c
+      return
+   end function wrapper_r2c
 
-  integer function wrapper_c2r(desc, in, out)
+   integer function wrapper_c2r(desc, in, out)
 
-    implicit none
+      implicit none
 
-    type(DFTI_DESCRIPTOR), pointer :: desc
-    complex(mytype), dimension(*) :: in 
-    real(mytype), dimension(*) :: out
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      complex(mytype), dimension(*) :: in
+      real(mytype), dimension(*) :: out
 
-    wrapper_c2r = DftiComputeBackward(desc, in, out)
+      wrapper_c2r = DftiComputeBackward(desc, in, out)
 
-    return
-  end function wrapper_c2r
+      return
+   end function wrapper_c2r
 
 end module decomp_2d_fft
diff --git a/src/glassman.f90 b/src/glassman.f90
index 89fcf41e..d0572101 100644
--- a/src/glassman.f90
+++ b/src/glassman.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,184 +9,181 @@
 !
 !=======================================================================
 
-! This module contains a few 'generic' FFT routines, making the 
+! This module contains a few 'generic' FFT routines, making the
 ! 2DECOMP&FFT library not dependent on any external libraries
 
 module glassman
 
-  use decomp_2d, only : mytype
+   use decomp_2d, only: mytype
 
-  implicit none
+   implicit none
 
 contains
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Following is a FFT implementation based on algorithm proposed by
-  ! Glassman, a general FFT algorithm supporting arbitrary input length.
-  !
-  ! W. E. Ferguson, Jr., "A simple derivation of Glassman general-n fast
-  ! Fourier transform," Comput. and Math. with Appls., vol. 8, no. 6, pp.
-  ! 401-411, 1982.
-  !
-  ! Original implemtation online at http://www.jjj.de/fft/fftpage.html
-  !
-  ! Updated  
-  !  -  to handle double-precision as well
-  !  -  unnecessary scaling code removed
+   ! Following is a FFT implementation based on algorithm proposed by
+   ! Glassman, a general FFT algorithm supporting arbitrary input length.
+   !
+   ! W. E. Ferguson, Jr., "A simple derivation of Glassman general-n fast
+   ! Fourier transform," Comput. and Math. with Appls., vol. 8, no. 6, pp.
+   ! 401-411, 1982.
+   !
+   ! Original implemtation online at http://www.jjj.de/fft/fftpage.html
+   !
+   ! Updated
+   !  -  to handle double-precision as well
+   !  -  unnecessary scaling code removed
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-  PURE SUBROUTINE SPCFFT(U,N,ISIGN,WORK)
-
-    !$acc routine seq
-
-    IMPLICIT NONE
-
-    LOGICAL :: INU
-    INTEGER :: A,B,C,I
-    INTEGER, INTENT(IN) :: N, ISIGN
-    COMPLEX(mytype), INTENT(INOUT) :: U(*),WORK(*)
-
-    A = 1
-    B = N
-    C = 1
-    INU = .TRUE.
-
-    DO WHILE ( B .GT. 1 )
-       A = C * A
-       C = 2
-       DO WHILE ( MOD(B,C) .NE. 0 )
-          C = C + 1
-       END DO
-       B = B / C
-       IF ( INU ) THEN
-          CALL SPCPFT (A,B,C,U,WORK,ISIGN)
-       ELSE
-          CALL SPCPFT (A,B,C,WORK,U,ISIGN)
-       END IF
-       INU = ( .NOT. INU )
-    END DO
-
-    IF ( .NOT. INU ) THEN
-       DO I = 1, N
-          U(I) = WORK(I)
-       END DO
-    END IF
-
-    RETURN
-  END SUBROUTINE SPCFFT
-
-
-  PURE SUBROUTINE SPCPFT( A, B, C, UIN, UOUT, ISIGN )
-
-    !$acc routine seq
-
-    IMPLICIT NONE
-
-    INTEGER, INTENT(IN), VALUE :: A, B, C, ISIGN
-    INTEGER :: IA,IB,IC,JCR,JC
-
-    DOUBLE PRECISION :: ANGLE
-
-    COMPLEX(mytype), INTENT(IN) :: UIN(B,C,A)
-    COMPLEX(mytype), INTENT(OUT) :: UOUT(B,A,C)
-    COMPLEX(mytype) :: DELTA,OMEGA,SUM
-
-    ANGLE = 6.28318530717958_mytype / REAL( A * C, kind=mytype )
-    OMEGA = CMPLX( 1.0, 0.0, kind=mytype )
-
-    IF( ISIGN .EQ. 1 ) THEN
-       DELTA = CMPLX( COS(ANGLE), SIN(ANGLE), kind=mytype )
-    ELSE
-       DELTA = CMPLX( COS(ANGLE), -SIN(ANGLE), kind=mytype )
-    END IF
-
-    DO IC = 1, C
-       DO IA = 1, A
-          DO IB = 1, B
-             SUM = UIN( IB, C, IA )
-             DO JCR = 2, C
-                JC = C + 1 - JCR
-                SUM = UIN( IB, JC, IA ) + OMEGA * SUM
-             END DO
-             UOUT( IB, IA, IC ) = SUM
-          END DO
-          OMEGA = DELTA * OMEGA
-       END DO
-    END DO
-
-    RETURN
-  END SUBROUTINE SPCPFT
-
+   PURE SUBROUTINE SPCFFT(U, N, ISIGN, WORK)
+
+      !$acc routine seq
+
+      IMPLICIT NONE
+
+      LOGICAL :: INU
+      INTEGER :: A, B, C, I
+      INTEGER, INTENT(IN) :: N, ISIGN
+      COMPLEX(mytype), INTENT(INOUT) :: U(*), WORK(*)
+
+      A = 1
+      B = N
+      C = 1
+      INU = .TRUE.
+
+      DO WHILE (B > 1)
+         A = C*A
+         C = 2
+         DO WHILE (MOD(B, C) /= 0)
+            C = C + 1
+         END DO
+         B = B/C
+         IF (INU) THEN
+            CALL SPCPFT(A, B, C, U, WORK, ISIGN)
+         ELSE
+            CALL SPCPFT(A, B, C, WORK, U, ISIGN)
+         END IF
+         INU = (.NOT. INU)
+      END DO
+
+      IF (.NOT. INU) THEN
+         DO I = 1, N
+            U(I) = WORK(I)
+         END DO
+      END IF
+
+      RETURN
+   END SUBROUTINE SPCFFT
+
+   PURE SUBROUTINE SPCPFT(A, B, C, UIN, UOUT, ISIGN)
+
+      !$acc routine seq
+
+      IMPLICIT NONE
+
+      INTEGER, INTENT(IN), VALUE :: A, B, C, ISIGN
+      INTEGER :: IA, IB, IC, JCR, JC
+
+      DOUBLE PRECISION :: ANGLE
+
+      COMPLEX(mytype), INTENT(IN) :: UIN(B, C, A)
+      COMPLEX(mytype), INTENT(OUT) :: UOUT(B, A, C)
+      COMPLEX(mytype) :: DELTA, OMEGA, SUM
+
+      ANGLE = 6.28318530717958_mytype/REAL(A*C, kind=mytype)
+      OMEGA = CMPLX(1.0, 0.0, kind=mytype)
+
+      IF (ISIGN == 1) THEN
+         DELTA = CMPLX(COS(ANGLE), SIN(ANGLE), kind=mytype)
+      ELSE
+         DELTA = CMPLX(COS(ANGLE), -SIN(ANGLE), kind=mytype)
+      END IF
+
+      DO IC = 1, C
+         DO IA = 1, A
+            DO IB = 1, B
+               SUM = UIN(IB, C, IA)
+               DO JCR = 2, C
+                  JC = C + 1 - JCR
+                  SUM = UIN(IB, JC, IA) + OMEGA*SUM
+               END DO
+               UOUT(IB, IA, IC) = SUM
+            END DO
+            OMEGA = DELTA*OMEGA
+         END DO
+      END DO
+
+      RETURN
+   END SUBROUTINE SPCPFT
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! A 3D real-to-complex routine implemented using the 1D FFT above
-  !   Input:   nx*ny*nz real numbers
-  !   Output:  (nx/2+1)*ny*nz complex numbers
-  ! Just like big FFT libraries (such as FFTW) do
+   ! A 3D real-to-complex routine implemented using the 1D FFT above
+   !   Input:   nx*ny*nz real numbers
+   !   Output:  (nx/2+1)*ny*nz complex numbers
+   ! Just like big FFT libraries (such as FFTW) do
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine glassman_3d_r2c(in_r,nx,ny,nz,out_c)
-
-    implicit none
-
-    integer, intent(IN) :: nx,ny,nz
-    real(mytype), dimension(nx,ny,nz) :: in_r
-    complex(mytype), dimension(nx/2+1,ny,nz) :: out_c
-
-    complex(mytype), allocatable, dimension(:) :: buf, scratch
-    integer :: maxsize, i,j,k
-
-    maxsize = max(nx, max(ny,nz))
-    allocate(buf(maxsize))
-    allocate(scratch(maxsize))
-
-    ! ===== 1D FFTs in X =====
-    do k=1,nz
-       do j=1,ny
-          ! Glassman's 1D FFT is c2c only, 
-          ! needing some pre- and post-processing for r2c
-          ! pack real input in complex storage
-          do i=1,nx
-             buf(i) = cmplx(in_r(i,j,k),0._mytype, kind=mytype)
-          end do
-          call spcfft(buf,nx,-1,scratch)
-          ! simply drop the redundant part of the complex output
-          do i=1,nx/2+1
-             out_c(i,j,k) = buf(i)
-          end do
-       end do
-    end do
-
-    ! ===== 1D FFTs in Y =====
-    do k=1,nz
-       do i=1,nx/2+1
-          do j=1,ny
-             buf(j) = out_c(i,j,k)
-          end do
-          call spcfft(buf,ny,-1,scratch)
-          do j=1,ny
-             out_c(i,j,k) = buf(j)
-          end do
-       end do
-    end do
-
-    ! ===== 1D FFTs in Z =====
-    do j=1,ny
-       do i=1,nx/2+1
-          do k=1,nz
-             buf(k) = out_c(i,j,k)
-          end do
-          call spcfft(buf,nz,-1,scratch)
-          do k=1,nz
-             out_c(i,j,k) = buf(k)
-          end do
-       end do
-    end do
-
-    deallocate(buf,scratch)
-
-    return
-  end subroutine glassman_3d_r2c
-
+   subroutine glassman_3d_r2c(in_r, nx, ny, nz, out_c)
+
+      implicit none
+
+      integer, intent(IN) :: nx, ny, nz
+      real(mytype), dimension(nx, ny, nz) :: in_r
+      complex(mytype), dimension(nx/2 + 1, ny, nz) :: out_c
+
+      complex(mytype), allocatable, dimension(:) :: buf, scratch
+      integer :: maxsize, i, j, k
+
+      maxsize = max(nx, max(ny, nz))
+      allocate (buf(maxsize))
+      allocate (scratch(maxsize))
+
+      ! ===== 1D FFTs in X =====
+      do k = 1, nz
+         do j = 1, ny
+            ! Glassman's 1D FFT is c2c only,
+            ! needing some pre- and post-processing for r2c
+            ! pack real input in complex storage
+            do i = 1, nx
+               buf(i) = cmplx(in_r(i, j, k), 0._mytype, kind=mytype)
+            end do
+            call spcfft(buf, nx, -1, scratch)
+            ! simply drop the redundant part of the complex output
+            do i = 1, nx/2 + 1
+               out_c(i, j, k) = buf(i)
+            end do
+         end do
+      end do
+
+      ! ===== 1D FFTs in Y =====
+      do k = 1, nz
+         do i = 1, nx/2 + 1
+            do j = 1, ny
+               buf(j) = out_c(i, j, k)
+            end do
+            call spcfft(buf, ny, -1, scratch)
+            do j = 1, ny
+               out_c(i, j, k) = buf(j)
+            end do
+         end do
+      end do
+
+      ! ===== 1D FFTs in Z =====
+      do j = 1, ny
+         do i = 1, nx/2 + 1
+            do k = 1, nz
+               buf(k) = out_c(i, j, k)
+            end do
+            call spcfft(buf, nz, -1, scratch)
+            do k = 1, nz
+               out_c(i, j, k) = buf(k)
+            end do
+         end do
+      end do
+
+      deallocate (buf, scratch)
+
+      return
+   end subroutine glassman_3d_r2c
 
 end module glassman
 
diff --git a/src/halo.f90 b/src/halo.f90
index ef324bf5..d0082f5c 100644
--- a/src/halo.f90
+++ b/src/halo.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -14,98 +14,98 @@
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   subroutine update_halo_real_short(in, out, level, opt_global, opt_pencil)
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: level      ! levels of halo cells required
-    real(mytype), dimension(:,:,:), intent(IN) :: in
-    real(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
-    logical, optional :: opt_global
-    integer, intent(in), optional :: opt_pencil
+     integer, intent(IN) :: level      ! levels of halo cells required
+     real(mytype), dimension(:, :, :), intent(IN) :: in
+     real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+     logical, optional :: opt_global
+     integer, intent(in), optional :: opt_pencil
 
-    call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
+     call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
 
   end subroutine update_halo_real_short
 
   subroutine update_halo_real(in, out, level, decomp, opt_global, opt_pencil)
 
-    implicit none
+     implicit none
+
+     integer, intent(IN) :: level      ! levels of halo cells required
+     real(mytype), dimension(:, :, :), intent(IN) :: in
+     real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+     TYPE(DECOMP_INFO), intent(in) :: decomp
+     logical, optional :: opt_global
+     integer, intent(in), optional :: opt_pencil
 
-    integer, intent(IN) :: level      ! levels of halo cells required
-    real(mytype), dimension(:,:,:), intent(IN) :: in    
-    real(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
-    TYPE(DECOMP_INFO), intent(in) :: decomp
-    logical, optional :: opt_global
-    integer, intent(in), optional :: opt_pencil
+     logical :: global
 
-    logical :: global
+     ! starting/ending index of array with halo cells
+     integer :: xs, ys, zs, xe, ye, ze
 
-    ! starting/ending index of array with halo cells
-    integer :: xs, ys, zs, xe, ye, ze
+     integer :: i, j, k, s1, s2, s3, ierror
+     integer :: data_type
 
-    integer :: i, j, k, s1, s2, s3, ierror
-    integer :: data_type
+     integer :: icount, ilength, ijump
+     integer :: halo12, halo21, halo31, halo32
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
 
-    integer :: icount, ilength, ijump 
-    integer :: halo12, halo21, halo31, halo32                 
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
+     integer :: ipencil
+     logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
 
-    integer :: ipencil
-    logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
-    
-    data_type = real_type
+     data_type = real_type
 
 #include "halo_common.f90"
 
-    return
+     return
   end subroutine update_halo_real
 
   subroutine update_halo_complex_short(in, out, level, opt_global, opt_pencil)
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: level      ! levels of halo cells required
-    complex(mytype), dimension(:,:,:), intent(IN) :: in
-    complex(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
-    logical, optional :: opt_global
-    integer, intent(in), optional :: opt_pencil
+     integer, intent(IN) :: level      ! levels of halo cells required
+     complex(mytype), dimension(:, :, :), intent(IN) :: in
+     complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+     logical, optional :: opt_global
+     integer, intent(in), optional :: opt_pencil
 
-    call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
+     call update_halo(in, out, level, decomp_main, opt_global, opt_pencil)
 
   end subroutine update_halo_complex_short
 
   subroutine update_halo_complex(in, out, level, decomp, opt_global, opt_pencil)
 
-    implicit none
+     implicit none
+
+     integer, intent(IN) :: level      ! levels of halo cells required
+     complex(mytype), dimension(:, :, :), intent(IN) :: in
+     complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+     TYPE(DECOMP_INFO), intent(in) :: decomp
+     logical, optional :: opt_global
+     integer, intent(in), optional :: opt_pencil
 
-    integer, intent(IN) :: level      ! levels of halo cells required
-    complex(mytype), dimension(:,:,:), intent(IN) :: in    
-    complex(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
-    TYPE(DECOMP_INFO), intent(in) :: decomp
-    logical, optional :: opt_global
-    integer, intent(in), optional :: opt_pencil
+     logical :: global
 
-    logical :: global
+     ! starting/ending index of array with halo cells
+     integer :: xs, ys, zs, xe, ye, ze
 
-    ! starting/ending index of array with halo cells
-    integer :: xs, ys, zs, xe, ye, ze
+     integer :: i, j, k, s1, s2, s3, ierror
+     integer :: data_type
 
-    integer :: i, j, k, s1, s2, s3, ierror
-    integer :: data_type
+     integer :: icount, ilength, ijump
+     integer :: halo12, halo21, halo31, halo32
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
 
-    integer :: icount, ilength, ijump 
-    integer :: halo12, halo21, halo31, halo32                 
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_e, tag_w, tag_n, tag_s, tag_t, tag_b
+     integer :: ipencil
+     logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
 
-    integer :: ipencil
-    logical, save :: first_call_x = .true., first_call_y = .true., first_call_z = .true.
-    
-    data_type = complex_type
+     data_type = complex_type
 
 #include "halo_common.f90"
 
-    return
+     return
   end subroutine update_halo_complex
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index 11859eae..13fa5d43 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,7 +9,7 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'update_halo_...' in halo.f90
 
     if (present(opt_global)) then
@@ -18,9 +18,9 @@
        global = .false.
     end if
 
-    s1 = size(in,1)
-    s2 = size(in,2)
-    s3 = size(in,3)
+    s1 = size(in, 1)
+    s2 = size(in, 2)
+    s3 = size(in, 3)
 
     if (present(opt_pencil)) then
        ipencil = opt_pencil
@@ -31,21 +31,21 @@
           if (first_call_x) then
              first_call_x = .false.
              call decomp_2d_warning(__FILE__, __LINE__, &
-                  0, "Deprecated interface - calling halo in X without explicit pencil")
+                                    0, "Deprecated interface - calling halo in X without explicit pencil")
           end if
        else if (s2 == decomp%ysz(2)) then
           ipencil = 2
           if (first_call_y) then
              first_call_y = .false.
              call decomp_2d_warning(__FILE__, __LINE__, &
-                  0, "Deprecated interface - calling halo in Y without explicit pencil")
+                                    0, "Deprecated interface - calling halo in Y without explicit pencil")
           end if
        else if (s3 == decomp%zsz(3)) then
           ipencil = 3
           if (first_call_z) then
              first_call_z = .false.
              call decomp_2d_warning(__FILE__, __LINE__, &
-                  0, "Deprecated interface - calling halo in Z without explicit pencil")
+                                    0, "Deprecated interface - calling halo in Z without explicit pencil")
           end if
        else
           ipencil = 0
@@ -56,23 +56,23 @@
     ! Calculate the starting index and ending index of output
     if (ipencil == 1) then  ! X-pencil input
        if (global) then
-          xs = decomp%xst(1) 
+          xs = decomp%xst(1)
           xe = decomp%xen(1)
           ys = decomp%xst(2) - level
           ye = decomp%xen(2) + level
           zs = decomp%xst(3) - level
           ze = decomp%xen(3) + level
        else
-          xs = 1 
+          xs = 1
           xe = s1
           ys = 1 - level
-          ye = s2 + level 
+          ye = s2 + level
           zs = 1 - level
           ze = s3 + level
        end if
     else if (ipencil == 2) then  ! Y-pencil input
        if (global) then
-          xs = decomp%yst(1) - level 
+          xs = decomp%yst(1) - level
           xe = decomp%yen(1) + level
           ys = decomp%yst(2)
           ye = decomp%yen(2)
@@ -88,7 +88,7 @@
        end if
     else if (ipencil == 3) then  ! Z-pencil input
        if (global) then
-          xs = decomp%zst(1) - level 
+          xs = decomp%zst(1) - level
           xe = decomp%zen(1) + level
           ys = decomp%zst(2) - level
           ye = decomp%zen(2) + level
@@ -109,49 +109,49 @@
        xs = 1; xe = 1
        ys = 1; ye = 1
        zs = 1; ze = 1
-       
+
        call decomp_2d_abort(__FILE__, __LINE__, 10, &
-            'Invalid data passed to update_halo')
+                            'Invalid data passed to update_halo')
     end if
-    allocate(out(xs:xe, ys:ye, zs:ze))
+    allocate (out(xs:xe, ys:ye, zs:ze))
 !    out = -1.0_mytype ! fill the halo for debugging
 
     ! copy input data to output
     if (global) then
        ! using global coordinate
-       ! note the input array passed in always has index starting from 1 
+       ! note the input array passed in always has index starting from 1
        ! need to work out the corresponding global index
        if (ipencil == 1) then
-          do k=decomp%xst(3),decomp%xen(3)
-             do j=decomp%xst(2),decomp%xen(2)
-                do i=1,s1  ! x all local
-                   out(i,j,k) = in(i,j-decomp%xst(2)+1,k-decomp%xst(3)+1)
+          do k = decomp%xst(3), decomp%xen(3)
+             do j = decomp%xst(2), decomp%xen(2)
+                do i = 1, s1  ! x all local
+                   out(i, j, k) = in(i, j - decomp%xst(2) + 1, k - decomp%xst(3) + 1)
                 end do
              end do
           end do
        else if (ipencil == 2) then
-          do k=decomp%yst(3),decomp%yen(3)
-             do j=1,s2  ! y all local
-                do i=decomp%yst(1),decomp%yen(1)
-                   out(i,j,k) = in(i-decomp%yst(1)+1,j,k-decomp%yst(3)+1)
+          do k = decomp%yst(3), decomp%yen(3)
+             do j = 1, s2  ! y all local
+                do i = decomp%yst(1), decomp%yen(1)
+                   out(i, j, k) = in(i - decomp%yst(1) + 1, j, k - decomp%yst(3) + 1)
                 end do
              end do
           end do
        else if (ipencil == 3) then
-          do k=1,s3  ! z all local
-             do j=decomp%zst(2),decomp%zen(2)
-                do i=decomp%zst(1),decomp%zen(1)
-                   out(i,j,k) = in(i-decomp%zst(1)+1,j-decomp%zst(2)+1,k)
+          do k = 1, s3  ! z all local
+             do j = decomp%zst(2), decomp%zen(2)
+                do i = decomp%zst(1), decomp%zen(1)
+                   out(i, j, k) = in(i - decomp%zst(1) + 1, j - decomp%zst(2) + 1, k)
                 end do
              end do
           end do
        end if
     else
        ! not using global coordinate
-       do k=1,s3
-          do j=1,s2
-             do i=1,s1
-                out(i,j,k) = in(i,j,k)
+       do k = 1, s3
+          do j = 1, s2
+             do i = 1, s1
+                out(i, j, k) = in(i, j, k)
              end do
           end do
        end do
@@ -159,19 +159,19 @@
 
     ! If needed, define MPI derived data type to pack halo data,
     ! then call MPI send/receive to exchange halo data
-    
+
     !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     ! X-pencil
     if (ipencil == 1) then
 
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'X-pencil input'
-          write(*,*) '=============='
-          write(*,*) 'Data on a y-z plane is shown'
-          write(*,*) 'Before halo exchange'
-          do j=ye,ys,-1
-             write(*,'(10F4.0)') (out(1,j,k),k=zs,ze)
+       if (nrank == 4) then
+          write (*, *) 'X-pencil input'
+          write (*, *) '=============='
+          write (*, *) 'Data on a y-z plane is shown'
+          write (*, *) 'Before halo exchange'
+          do j = ye, ys, -1
+             write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
           end do
        end if
 #endif
@@ -179,50 +179,50 @@
        ! *** east/west ***
        ! all data in local memory already, no halo exchange
 
-       ! *** north/south *** 
+       ! *** north/south ***
        tag_s = coord(1)
-       if (coord(1)==dims(1)-1 .AND. periodic_y) then
+       if (coord(1) == dims(1) - 1 .AND. periodic_y) then
           tag_n = 0
        else
           tag_n = coord(1) + 1
        end if
        icount = s3 + 2*level
-       ilength = level * s1
-       ijump = s1*(s2+2*level)
+       ilength = level*s1
+       ijump = s1*(s2 + 2*level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-            data_type, halo12, ierror)
+                            data_type, halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from south
-       call MPI_IRECV(out(xs,ys,zs), 1, halo12, &
-            neighbour(1,4), tag_s, DECOMP_2D_COMM_CART_X, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), 1, halo12, &
+                      neighbour(1, 4), tag_s, DECOMP_2D_COMM_CART_X, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from north
-       call MPI_IRECV(out(xs,ye-level+1,zs), 1, halo12, &
-            neighbour(1,3), tag_n, DECOMP_2D_COMM_CART_X, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xs, ye - level + 1, zs), 1, halo12, &
+                      neighbour(1, 3), tag_n, DECOMP_2D_COMM_CART_X, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to south
-       call MPI_ISSEND(out(xs,ys+level,zs), 1, halo12, &
-            neighbour(1,4), tag_s, DECOMP_2D_COMM_CART_X, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs, ys + level, zs), 1, halo12, &
+                       neighbour(1, 4), tag_s, DECOMP_2D_COMM_CART_X, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to north
-       call MPI_ISSEND(out(xs,ye-level-level+1,zs), 1, halo12, &
-            neighbour(1,3), tag_n, DECOMP_2D_COMM_CART_X, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xs, ye - level - level + 1, zs), 1, halo12, &
+                       neighbour(1, 3), tag_n, DECOMP_2D_COMM_CART_X, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
        call MPI_TYPE_FREE(halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'After exchange in Y'
-          do j=ye,ys,-1
-             write(*,'(10F4.0)') (out(1,j,k),k=zs,ze)
+       if (nrank == 4) then
+          write (*, *) 'After exchange in Y'
+          do j = ye, ys, -1
+             write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
           end do
        end if
 #endif
@@ -232,103 +232,103 @@
        ! all contiguous in memory, which can be sent/received using
        ! MPI directly
        tag_b = coord(2)
-       if (coord(2)==dims(2)-1 .AND. periodic_z) then
+       if (coord(2) == dims(2) - 1 .AND. periodic_z) then
           tag_t = 0
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s1 * (s2+2*level)) * level
+       icount = (s1*(s2 + 2*level))*level
        ! receive from bottom
-       call MPI_IRECV(out(xs,ys,zs), icount, data_type, &
-            neighbour(1,6), tag_b, DECOMP_2D_COMM_CART_X, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
+                      neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from top
-       call MPI_IRECV(out(xs,ys,ze-level+1), icount, data_type, &
-            neighbour(1,5), tag_t, DECOMP_2D_COMM_CART_X, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xs, ys, ze - level + 1), icount, data_type, &
+                      neighbour(1, 5), tag_t, DECOMP_2D_COMM_CART_X, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to bottom
-       call MPI_ISSEND(out(xs,ys,zs+level), icount, data_type, &
-            neighbour(1,6), tag_b, DECOMP_2D_COMM_CART_X, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs, ys, zs + level), icount, data_type, &
+                       neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to top
-       call MPI_ISSEND(out(xs,ys,ze-level-level+1), icount, data_type, &
-            neighbour(1,5), tag_t, DECOMP_2D_COMM_CART_X, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xs, ys, ze - level - level + 1), icount, data_type, &
+                       neighbour(1, 5), tag_t, DECOMP_2D_COMM_CART_X, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
-#ifdef HALO_DEBUG       
-       if (nrank==4) then
-          write(*,*) 'After exchange in Z'
-          do j=ye,ys,-1
-             write(*,'(10F4.0)') (out(1,j,k),k=zs,ze)
+#ifdef HALO_DEBUG
+       if (nrank == 4) then
+          write (*, *) 'After exchange in Z'
+          do j = ye, ys, -1
+             write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
           end do
        end if
-#endif       
+#endif
 
-    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   
-    ! Y-pencil   
+    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+       ! Y-pencil
     else if (ipencil == 2) then
 
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'Y-pencil input'
-          write(*,*) '=============='
-          write(*,*) 'Data on a x-z plane is shown'
-          write(*,*) 'Before halo exchange'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,1,k),k=zs,ze)
+       if (nrank == 4) then
+          write (*, *) 'Y-pencil input'
+          write (*, *) '=============='
+          write (*, *) 'Data on a x-z plane is shown'
+          write (*, *) 'Before halo exchange'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
           end do
        end if
 #endif
 
        ! *** east/west ***
        tag_w = coord(1)
-       if (coord(1)==dims(1)-1 .AND. periodic_x) then
+       if (coord(1) == dims(1) - 1 .AND. periodic_x) then
           tag_e = 0
        else
           tag_e = coord(1) + 1
        end if
-       icount = s2*(s3+2*level)
+       icount = s2*(s3 + 2*level)
        ilength = level
-       ijump = s1+2*level
+       ijump = s1 + 2*level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-            data_type, halo21, ierror)
+                            data_type, halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from west
-       call MPI_IRECV(out(xs,ys,zs), 1, halo21, &
-            neighbour(2,2), tag_w, DECOMP_2D_COMM_CART_Y, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), 1, halo21, &
+                      neighbour(2, 2), tag_w, DECOMP_2D_COMM_CART_Y, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from east
-       call MPI_IRECV(out(xe-level+1,ys,zs), 1, halo21, &
-            neighbour(2,1), tag_e, DECOMP_2D_COMM_CART_Y, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xe - level + 1, ys, zs), 1, halo21, &
+                      neighbour(2, 1), tag_e, DECOMP_2D_COMM_CART_Y, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to west
-       call MPI_ISSEND(out(xs+level,ys,zs), 1, halo21, &
-            neighbour(2,2), tag_w, DECOMP_2D_COMM_CART_Y, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs + level, ys, zs), 1, halo21, &
+                       neighbour(2, 2), tag_w, DECOMP_2D_COMM_CART_Y, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to east
-       call MPI_ISSEND(out(xe-level-level+1,ys,zs), 1, halo21, &
-            neighbour(2,1), tag_e, DECOMP_2D_COMM_CART_Y, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xe - level - level + 1, ys, zs), 1, halo21, &
+                       neighbour(2, 1), tag_e, DECOMP_2D_COMM_CART_Y, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
        call MPI_TYPE_FREE(halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-#ifdef HALO_DEBUG       
-       if (nrank==4) then
-          write(*,*) 'After exchange in X'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,1,k),k=zs,ze)
+#ifdef HALO_DEBUG
+       if (nrank == 4) then
+          write (*, *) 'After exchange in X'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
           end do
        end if
 #endif
@@ -341,150 +341,150 @@
        ! all contiguous in memory, which can be sent/received using
        ! MPI directly
        tag_b = coord(2)
-       if (coord(2)==dims(2)-1 .AND. periodic_z) then
+       if (coord(2) == dims(2) - 1 .AND. periodic_z) then
           tag_t = 0
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s2 * (s1+2*level)) * level
+       icount = (s2*(s1 + 2*level))*level
        ! receive from bottom
-       call MPI_IRECV(out(xs,ys,zs), icount, data_type, &
-            neighbour(2,6), tag_b, DECOMP_2D_COMM_CART_Y, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
+                      neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from top
-       call MPI_IRECV(out(xs,ys,ze-level+1), icount, data_type, &
-            neighbour(2,5), tag_t, DECOMP_2D_COMM_CART_Y, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xs, ys, ze - level + 1), icount, data_type, &
+                      neighbour(2, 5), tag_t, DECOMP_2D_COMM_CART_Y, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to bottom
-       call MPI_ISSEND(out(xs,ys,zs+level), icount, data_type, &
-            neighbour(2,6), tag_b, DECOMP_2D_COMM_CART_Y, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs, ys, zs + level), icount, data_type, &
+                       neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to top
-       call MPI_ISSEND(out(xs,ys,ze-level-level+1), icount, data_type, &
-            neighbour(2,5), tag_t, DECOMP_2D_COMM_CART_Y, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xs, ys, ze - level - level + 1), icount, data_type, &
+                       neighbour(2, 5), tag_t, DECOMP_2D_COMM_CART_Y, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'After exchange in Z'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,1,k),k=zs,ze)
+       if (nrank == 4) then
+          write (*, *) 'After exchange in Z'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
           end do
        end if
 #endif
 
-    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   
-    ! Z-pencil
-    else if (ipencil == 3) then   
+    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+       ! Z-pencil
+    else if (ipencil == 3) then
 
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'Z-pencil input'
-          write(*,*) '=============='
-          write(*,*) 'Data on a x-y plane is shown'
-          write(*,*) 'Before halo exchange'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,j,1),j=ys,ye)
+       if (nrank == 4) then
+          write (*, *) 'Z-pencil input'
+          write (*, *) '=============='
+          write (*, *) 'Data on a x-y plane is shown'
+          write (*, *) 'Before halo exchange'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
           end do
        end if
 #endif
        ! *** east/west ***
        tag_w = coord(1)
-       if (coord(1)==dims(1)-1 .AND. periodic_x) then
+       if (coord(1) == dims(1) - 1 .AND. periodic_x) then
           tag_e = 0
        else
           tag_e = coord(1) + 1
        end if
-       icount = (s2+2*level)*s3
+       icount = (s2 + 2*level)*s3
        ilength = level
-       ijump = s1+2*level
+       ijump = s1 + 2*level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-            data_type, halo31, ierror)
+                            data_type, halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from west
-       call MPI_IRECV(out(xs,ys,zs), 1, halo31, &
-            neighbour(3,2), tag_w, DECOMP_2D_COMM_CART_Z, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), 1, halo31, &
+                      neighbour(3, 2), tag_w, DECOMP_2D_COMM_CART_Z, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from east
-       call MPI_IRECV(out(xe-level+1,ys,zs), 1, halo31, &
-            neighbour(3,1), tag_e, DECOMP_2D_COMM_CART_Z, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xe - level + 1, ys, zs), 1, halo31, &
+                      neighbour(3, 1), tag_e, DECOMP_2D_COMM_CART_Z, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to west
-       call MPI_ISSEND(out(xs+level,ys,zs), 1, halo31, &
-            neighbour(3,2), tag_w, DECOMP_2D_COMM_CART_Z, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs + level, ys, zs), 1, halo31, &
+                       neighbour(3, 2), tag_w, DECOMP_2D_COMM_CART_Z, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to east
-       call MPI_ISSEND(out(xe-level-level+1,ys,zs), 1, halo31, &
-            neighbour(3,1), tag_e, DECOMP_2D_COMM_CART_Z, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xe - level - level + 1, ys, zs), 1, halo31, &
+                       neighbour(3, 1), tag_e, DECOMP_2D_COMM_CART_Z, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
-       call MPI_TYPE_FREE(halo31, ierror)       
+       call MPI_TYPE_FREE(halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'After exchange in X'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,j,1),j=ys,ye)
+       if (nrank == 4) then
+          write (*, *) 'After exchange in X'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
           end do
        end if
 #endif
 
-       ! *** north/south *** 
+       ! *** north/south ***
        tag_s = coord(2)
-       if (coord(2)==dims(2)-1 .AND. periodic_y) then
+       if (coord(2) == dims(2) - 1 .AND. periodic_y) then
           tag_n = 0
        else
           tag_n = coord(2) + 1
        end if
        icount = s3
-       ilength = level * (s1+2*level)
-       ijump = (s1+2*level) * (s2+2*level)
+       ilength = level*(s1 + 2*level)
+       ijump = (s1 + 2*level)*(s2 + 2*level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-            data_type, halo32, ierror)
+                            data_type, halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
        call MPI_TYPE_COMMIT(halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
        ! receive from south
-       call MPI_IRECV(out(xs,ys,zs), 1, halo32, &
-            neighbour(3,4), tag_s, DECOMP_2D_COMM_CART_Z, &
-            requests(1), ierror)
+       call MPI_IRECV(out(xs, ys, zs), 1, halo32, &
+                      neighbour(3, 4), tag_s, DECOMP_2D_COMM_CART_Z, &
+                      requests(1), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! receive from north
-       call MPI_IRECV(out(xs,ye-level+1,zs), 1, halo32, &
-            neighbour(3,3), tag_n, DECOMP_2D_COMM_CART_Z, &
-            requests(2), ierror)
+       call MPI_IRECV(out(xs, ye - level + 1, zs), 1, halo32, &
+                      neighbour(3, 3), tag_n, DECOMP_2D_COMM_CART_Z, &
+                      requests(2), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
        ! send to south
-       call MPI_ISSEND(out(xs,ys+level,zs), 1, halo32, &
-            neighbour(3,4), tag_s, DECOMP_2D_COMM_CART_Z, &
-            requests(3), ierror)
+       call MPI_ISSEND(out(xs, ys + level, zs), 1, halo32, &
+                       neighbour(3, 4), tag_s, DECOMP_2D_COMM_CART_Z, &
+                       requests(3), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        ! send to north
-       call MPI_ISSEND(out(xs,ye-level-level+1,zs), 1, halo32, &
-            neighbour(3,3), tag_n, DECOMP_2D_COMM_CART_Z, &
-            requests(4), ierror)
+       call MPI_ISSEND(out(xs, ye - level - level + 1, zs), 1, halo32, &
+                       neighbour(3, 3), tag_n, DECOMP_2D_COMM_CART_Z, &
+                       requests(4), ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
        call MPI_TYPE_FREE(halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank==4) then
-          write(*,*) 'After exchange in Y'
-          do i=xe,xs,-1
-             write(*,'(10F4.0)') (out(i,j,1),j=ys,ye)
+       if (nrank == 4) then
+          write (*, *) 'After exchange in Y'
+          do i = xe, xs, -1
+             write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
           end do
        end if
 #endif
diff --git a/src/io.f90 b/src/io.f90
index 42705f1f..b1d49641 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -15,2044 +15,2025 @@
 
 module decomp_2d_io
 
-  use decomp_2d
-  use MPI
+   use decomp_2d
+   use MPI
 
 #ifdef ADIOS2
-  use adios2
+   use adios2
 #endif
 
-  implicit none
+   implicit none
 
-  integer, parameter, public :: decomp_2d_write_mode = 1, decomp_2d_read_mode = 2, &
-       decomp_2d_append_mode = 3
-  integer, parameter :: MAX_IOH = 10 ! How many live IO things should we handle?
-  character(len=*), parameter :: io_sep = "::"
-  integer, save :: nreg_io = 0
+   integer, parameter, public :: decomp_2d_write_mode = 1, decomp_2d_read_mode = 2, &
+                                 decomp_2d_append_mode = 3
+   integer, parameter :: MAX_IOH = 10 ! How many live IO things should we handle?
+   character(len=*), parameter :: io_sep = "::"
+   integer, save :: nreg_io = 0
 #ifndef ADIOS2
-  integer, dimension(MAX_IOH), save :: fh_registry
-  logical, dimension(MAX_IOH), target, save :: fh_live
-  character(len=1024), dimension(MAX_IOH), target, save :: fh_names
-  integer(kind=MPI_OFFSET_KIND), dimension(MAX_IOH), save :: fh_disp
+   integer, dimension(MAX_IOH), save :: fh_registry
+   logical, dimension(MAX_IOH), target, save :: fh_live
+   character(len=1024), dimension(MAX_IOH), target, save :: fh_names
+   integer(kind=MPI_OFFSET_KIND), dimension(MAX_IOH), save :: fh_disp
 #else
-  type(adios2_adios) :: adios
-  character(len=1024), dimension(MAX_IOH), target, save :: engine_names
-  logical, dimension(MAX_IOH), target, save :: engine_live
-  type(adios2_engine), dimension(MAX_IOH), save :: engine_registry
-#endif
-  
-  private        ! Make everything private unless declared public
-
-  public :: decomp_2d_write_one, decomp_2d_read_one, &
-       decomp_2d_write_var, decomp_2d_read_var, &
-       decomp_2d_write_scalar, decomp_2d_read_scalar, &
-       decomp_2d_write_plane, decomp_2d_write_every, &
-       decomp_2d_write_subdomain, &
-       decomp_2d_write_outflow, decomp_2d_read_inflow, &
-       decomp_2d_io_init, decomp_2d_io_finalise, & ! XXX: initialise/finalise 2decomp&fft IO module
-       decomp_2d_init_io, & ! XXX: initialise an io process - awful naming
-       decomp_2d_register_variable, &
-       decomp_2d_open_io, decomp_2d_close_io, &
-       decomp_2d_start_io, decomp_2d_end_io, &
-       gen_iodir_name
-
-  ! Generic interface to handle multiple data types
-
-  interface decomp_2d_write_one
-     module procedure write_one_real
-     module procedure write_one_complex
-     module procedure mpiio_write_real_coarse
-     module procedure mpiio_write_real_probe
-  end interface decomp_2d_write_one
-
-  interface decomp_2d_read_one
-     module procedure read_one_real
-     module procedure read_one_complex
-  end interface decomp_2d_read_one
-
-  interface decomp_2d_write_var
-     module procedure write_var_real
-     module procedure write_var_complex
-  end interface decomp_2d_write_var
-
-  interface decomp_2d_read_var
-     module procedure read_var_real
-     module procedure read_var_complex
-  end interface decomp_2d_read_var
-
-  interface decomp_2d_write_scalar
-     module procedure write_scalar_real
-     module procedure write_scalar_complex
-     module procedure write_scalar_integer
-     module procedure write_scalar_logical
-  end interface decomp_2d_write_scalar
-
-  interface decomp_2d_read_scalar
-     module procedure read_scalar_real
-     module procedure read_scalar_complex
-     module procedure read_scalar_integer
-     module procedure read_scalar_logical
-  end interface decomp_2d_read_scalar
-
-  interface decomp_2d_write_plane
-     module procedure write_plane_3d_real
-     module procedure write_plane_3d_complex
-     !     module procedure write_plane_2d
-  end interface decomp_2d_write_plane
-
-  interface decomp_2d_write_every
-     module procedure write_every_real
-     module procedure write_every_complex
-  end interface decomp_2d_write_every
-
-  interface decomp_2d_write_subdomain
-     module procedure write_subdomain
-  end interface decomp_2d_write_subdomain
-
-  interface decomp_2d_write_outflow
-     module procedure write_outflow
-  end interface decomp_2d_write_outflow
-
-  interface decomp_2d_read_inflow
-     module procedure read_inflow
-  end interface decomp_2d_read_inflow
+   type(adios2_adios) :: adios
+   character(len=1024), dimension(MAX_IOH), target, save :: engine_names
+   logical, dimension(MAX_IOH), target, save :: engine_live
+   type(adios2_engine), dimension(MAX_IOH), save :: engine_registry
+#endif
+
+   private        ! Make everything private unless declared public
+
+   public :: decomp_2d_write_one, decomp_2d_read_one, &
+             decomp_2d_write_var, decomp_2d_read_var, &
+             decomp_2d_write_scalar, decomp_2d_read_scalar, &
+             decomp_2d_write_plane, decomp_2d_write_every, &
+             decomp_2d_write_subdomain, &
+             decomp_2d_write_outflow, decomp_2d_read_inflow, &
+             decomp_2d_io_init, decomp_2d_io_finalise, & ! XXX: initialise/finalise 2decomp&fft IO module
+             decomp_2d_init_io, & ! XXX: initialise an io process - awful naming
+             decomp_2d_register_variable, &
+             decomp_2d_open_io, decomp_2d_close_io, &
+             decomp_2d_start_io, decomp_2d_end_io, &
+             gen_iodir_name
+
+   ! Generic interface to handle multiple data types
+
+   interface decomp_2d_write_one
+      module procedure write_one_real
+      module procedure write_one_complex
+      module procedure mpiio_write_real_coarse
+      module procedure mpiio_write_real_probe
+   end interface decomp_2d_write_one
+
+   interface decomp_2d_read_one
+      module procedure read_one_real
+      module procedure read_one_complex
+   end interface decomp_2d_read_one
+
+   interface decomp_2d_write_var
+      module procedure write_var_real
+      module procedure write_var_complex
+   end interface decomp_2d_write_var
+
+   interface decomp_2d_read_var
+      module procedure read_var_real
+      module procedure read_var_complex
+   end interface decomp_2d_read_var
+
+   interface decomp_2d_write_scalar
+      module procedure write_scalar_real
+      module procedure write_scalar_complex
+      module procedure write_scalar_integer
+      module procedure write_scalar_logical
+   end interface decomp_2d_write_scalar
+
+   interface decomp_2d_read_scalar
+      module procedure read_scalar_real
+      module procedure read_scalar_complex
+      module procedure read_scalar_integer
+      module procedure read_scalar_logical
+   end interface decomp_2d_read_scalar
+
+   interface decomp_2d_write_plane
+      module procedure write_plane_3d_real
+      module procedure write_plane_3d_complex
+      !     module procedure write_plane_2d
+   end interface decomp_2d_write_plane
+
+   interface decomp_2d_write_every
+      module procedure write_every_real
+      module procedure write_every_complex
+   end interface decomp_2d_write_every
+
+   interface decomp_2d_write_subdomain
+      module procedure write_subdomain
+   end interface decomp_2d_write_subdomain
+
+   interface decomp_2d_write_outflow
+      module procedure write_outflow
+   end interface decomp_2d_write_outflow
+
+   interface decomp_2d_read_inflow
+      module procedure read_inflow
+   end interface decomp_2d_read_inflow
 
 contains
 
-  subroutine decomp_2d_io_init()
+   subroutine decomp_2d_io_init()
 
 #ifdef ADIOS2
-  integer :: ierror
-  logical :: adios2_debug_mode
-  character(len=80) :: config_file="adios2_config.xml"
+      integer :: ierror
+      logical :: adios2_debug_mode
+      character(len=80) :: config_file = "adios2_config.xml"
 #endif
 
 #ifdef ADIOS2
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_start("io_init")
+      if (decomp_profiler_io) call decomp_profiler_start("io_init")
 #endif
 
   !! TODO: make this a runtime-option
-  adios2_debug_mode = .true.
+      adios2_debug_mode = .true.
 
-  call adios2_init(adios, trim(config_file), decomp_2d_comm, adios2_debug_mode, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-     "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
+      call adios2_init(adios, trim(config_file), decomp_2d_comm, adios2_debug_mode, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                                            "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
 
-  engine_live(:) = .false.
+      engine_live(:) = .false.
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_end("io_init")
+      if (decomp_profiler_io) call decomp_profiler_end("io_init")
 #endif
 #endif
-  
-  end subroutine decomp_2d_io_init
-  subroutine decomp_2d_io_finalise()
+
+   end subroutine decomp_2d_io_init
+   subroutine decomp_2d_io_finalise()
 
 #ifdef ADIOS2
-    use adios2
+      use adios2
 #endif
 
-    implicit none
+      implicit none
 
 #ifdef ADIOS2
-    integer :: ierror
+      integer :: ierror
 #endif
-    
+
 #ifdef ADIOS2
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_fin")
+      if (decomp_profiler_io) call decomp_profiler_start("io_fin")
 #endif
-    call adios2_finalize(adios, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-       "adios2_finalize")
+      call adios2_finalize(adios, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                                            "adios2_finalize")
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_fin")
+      if (decomp_profiler_io) call decomp_profiler_end("io_fin")
 #endif
 #endif
 
-  end subroutine decomp_2d_io_finalise
-  
+   end subroutine decomp_2d_io_finalise
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Using MPI-IO library to write a single 3D array to a file
+   ! Using MPI-IO library to write a single 3D array to a file
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_one_real(ipencil,var,filename,opt_decomp)
+   subroutine write_one_real(ipencil, var, filename, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    character(len=*), intent(IN) :: filename
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: ipencil
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      character(len=*), intent(IN) :: filename
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, fh, data_type
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_start("io_write_one_real")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_one_real")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_one.inc"
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_end("io_write_one_real")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_one_real")
 #endif
 
-    return
-  end subroutine write_one_real
+      return
+   end subroutine write_one_real
 
-  subroutine write_one_complex(ipencil,var,filename,opt_decomp)
+   subroutine write_one_complex(ipencil, var, filename, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil
-    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    character(len=*), intent(IN) :: filename
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: ipencil
+      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      character(len=*), intent(IN) :: filename
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, fh, data_type
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_start("io_write_one_cplx")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_one_cplx")
 #endif
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_write_one.inc"
 
 #ifdef PROFILER
-  if (decomp_profiler_io) call decomp_profiler_end("io_write_one_cplx")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_one_cplx")
 #endif
 
-    return
-  end subroutine write_one_complex
-
+      return
+   end subroutine write_one_complex
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Using MPI-IO library to read from a file a single 3D array
+   ! Using MPI-IO library to read from a file a single 3D array
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine read_one_real(ipencil,var,dirname,varname,io_name,opt_decomp,reduce_prec)
+   subroutine read_one_real(ipencil, var, dirname, varname, io_name, opt_decomp, reduce_prec)
+
+      implicit none
 
-    implicit none
+      integer, intent(IN) :: ipencil
+      real(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      character(len=*), intent(IN) :: varname, dirname, io_name
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      logical, intent(in), optional :: reduce_prec
 
-    integer, intent(IN) :: ipencil
-    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    character(len=*), intent(IN) :: varname, dirname, io_name
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
-    logical, intent(in), optional :: reduce_prec
+      logical :: read_reduce_prec
 
-    logical :: read_reduce_prec
-    
-    integer :: idx
+      integer :: idx
 #ifndef ADIOS2
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    real(mytype_single), allocatable, dimension(:,:,:) :: varsingle
-    integer :: data_type
-    logical :: dir_exists
-    integer :: disp_bytes
-    integer :: ierror, newtype
-    character(len=:), allocatable :: full_io_name
-    logical :: opened_new
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      real(mytype_single), allocatable, dimension(:, :, :) :: varsingle
+      integer :: data_type
+      logical :: dir_exists
+      integer :: disp_bytes
+      integer :: ierror, newtype
+      character(len=:), allocatable :: full_io_name
+      logical :: opened_new
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_read_one_real")
+      if (decomp_profiler_io) call decomp_profiler_start("io_read_one_real")
 #endif
-    
-    read_reduce_prec = .true.
-    
-    idx = get_io_idx(io_name, dirname)
+
+      read_reduce_prec = .true.
+
+      idx = get_io_idx(io_name, dirname)
 #ifndef ADIOS2
-    opened_new = .false.
-    if (idx .lt. 1) then
-       ! Check file exists
-       full_io_name = trim(dirname)//"/"//trim(varname)
-       if (nrank==0) then
-          inquire(file=full_io_name, exist=dir_exists)
-          if (.not.dir_exists) then
-             print *, "ERROR: cannot read from", full_io_name, " directory doesn't exist!"
-             stop
-          end if
-       end if
-       
-       call decomp_2d_open_io(io_name, full_io_name, decomp_2d_read_mode)
-       idx = get_io_idx(io_name, full_io_name)
-       opened_new = .true.
-    else
-       full_io_name = "" ! Ensure string is not unset
-    end if
-    
-    if (present(reduce_prec)) then
-       if (.not. reduce_prec) then
-          read_reduce_prec = .false.
-       end if
-    end if
-    if (read_reduce_prec) then
-       data_type = real_type_single
-    else
-       data_type = real_type
-    end if
-    call MPI_TYPE_SIZE(data_type,disp_bytes,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    
-    if (present(opt_decomp)) then
-       decomp = opt_decomp
-    else
-       call get_decomp_info(decomp)
-    end if
-
-    ! determine subarray parameters
-    sizes(1) = decomp%xsz(1)
-    sizes(2) = decomp%ysz(2)
-    sizes(3) = decomp%zsz(3)
-
-    if (ipencil == 1) then
-       subsizes(1) = decomp%xsz(1)
-       subsizes(2) = decomp%xsz(2)
-       subsizes(3) = decomp%xsz(3)
-       starts(1) = decomp%xst(1)-1  ! 0-based index
-       starts(2) = decomp%xst(2)-1
-       starts(3) = decomp%xst(3)-1
-    else if (ipencil == 2) then
-       subsizes(1) = decomp%ysz(1)
-       subsizes(2) = decomp%ysz(2)
-       subsizes(3) = decomp%ysz(3)
-       starts(1) = decomp%yst(1)-1
-       starts(2) = decomp%yst(2)-1
-       starts(3) = decomp%yst(3)-1
-    else if (ipencil == 3) then
-       subsizes(1) = decomp%zsz(1)
-       subsizes(2) = decomp%zsz(2)
-       subsizes(3) = decomp%zsz(3)
-       starts(1) = decomp%zst(1)-1
-       starts(2) = decomp%zst(2)-1
-       starts(3) = decomp%zst(3)-1
-    else
-       call decomp_2d_abort(-1, "IO/read_one_real : Wrong value for ipencil")
-    endif
-
-    associate(fh => fh_registry(idx), &
-         disp => fh_disp(idx))
-      call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-           MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-      call MPI_TYPE_COMMIT(newtype,ierror)
-      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-      call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-           newtype,'native',MPI_INFO_NULL,ierror)
-      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      opened_new = .false.
+      if (idx < 1) then
+         ! Check file exists
+         full_io_name = trim(dirname)//"/"//trim(varname)
+         if (nrank == 0) then
+            inquire (file=full_io_name, exist=dir_exists)
+            if (.not. dir_exists) then
+               print *, "ERROR: cannot read from", full_io_name, " directory doesn't exist!"
+               stop
+            end if
+         end if
+
+         call decomp_2d_open_io(io_name, full_io_name, decomp_2d_read_mode)
+         idx = get_io_idx(io_name, full_io_name)
+         opened_new = .true.
+      else
+         full_io_name = "" ! Ensure string is not unset
+      end if
+
+      if (present(reduce_prec)) then
+         if (.not. reduce_prec) then
+            read_reduce_prec = .false.
+         end if
+      end if
       if (read_reduce_prec) then
-         allocate (varsingle(xstV(1):xenV(1),xstV(2):xenV(2),xstV(3):xenV(3)))
-         call MPI_FILE_READ_ALL(fh, varsingle, &
-              subsizes(1)*subsizes(2)*subsizes(3), &
-              data_type, MPI_STATUS_IGNORE, ierror)
-         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-         var = real(varsingle,mytype)
-         deallocate(varsingle)
+         data_type = real_type_single
+      else
+         data_type = real_type
+      end if
+      call MPI_TYPE_SIZE(data_type, disp_bytes, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+
+      if (present(opt_decomp)) then
+         decomp = opt_decomp
       else
-         call MPI_FILE_READ_ALL(fh, var, &
-              subsizes(1)*subsizes(2)*subsizes(3), &
-              data_type, MPI_STATUS_IGNORE, ierror)
-         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-      endif
-      call MPI_TYPE_FREE(newtype,ierror)
-      if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-
-      disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                  * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                  * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                  * int(disp_bytes, kind=MPI_OFFSET_KIND)
-    end associate
-
-    if (opened_new) then
-       call decomp_2d_close_io(io_name, full_io_name)
-       deallocate(full_io_name)
-    end if
+         call get_decomp_info(decomp)
+      end if
+
+      ! determine subarray parameters
+      sizes(1) = decomp%xsz(1)
+      sizes(2) = decomp%ysz(2)
+      sizes(3) = decomp%zsz(3)
+
+      if (ipencil == 1) then
+         subsizes(1) = decomp%xsz(1)
+         subsizes(2) = decomp%xsz(2)
+         subsizes(3) = decomp%xsz(3)
+         starts(1) = decomp%xst(1) - 1  ! 0-based index
+         starts(2) = decomp%xst(2) - 1
+         starts(3) = decomp%xst(3) - 1
+      else if (ipencil == 2) then
+         subsizes(1) = decomp%ysz(1)
+         subsizes(2) = decomp%ysz(2)
+         subsizes(3) = decomp%ysz(3)
+         starts(1) = decomp%yst(1) - 1
+         starts(2) = decomp%yst(2) - 1
+         starts(3) = decomp%yst(3) - 1
+      else if (ipencil == 3) then
+         subsizes(1) = decomp%zsz(1)
+         subsizes(2) = decomp%zsz(2)
+         subsizes(3) = decomp%zsz(3)
+         starts(1) = decomp%zst(1) - 1
+         starts(2) = decomp%zst(2) - 1
+         starts(3) = decomp%zst(3) - 1
+      else
+         call decomp_2d_abort(-1, "IO/read_one_real : Wrong value for ipencil")
+      end if
+
+      associate (fh => fh_registry(idx), &
+                 disp => fh_disp(idx))
+         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+         call MPI_TYPE_COMMIT(newtype, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+         call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                                newtype, 'native', MPI_INFO_NULL, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+         if (read_reduce_prec) then
+            allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
+            call MPI_FILE_READ_ALL(fh, varsingle, &
+                                   subsizes(1)*subsizes(2)*subsizes(3), &
+                                   data_type, MPI_STATUS_IGNORE, ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+            var = real(varsingle, mytype)
+            deallocate (varsingle)
+         else
+            call MPI_FILE_READ_ALL(fh, var, &
+                                   subsizes(1)*subsizes(2)*subsizes(3), &
+                                   data_type, MPI_STATUS_IGNORE, ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+         end if
+         call MPI_TYPE_FREE(newtype, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+
+         disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                *int(disp_bytes, kind=MPI_OFFSET_KIND)
+      end associate
+
+      if (opened_new) then
+         call decomp_2d_close_io(io_name, full_io_name)
+         deallocate (full_io_name)
+      end if
 #else
-    call adios2_read_one_real(var, dirname, varname, io_name)
+      call adios2_read_one_real(var, dirname, varname, io_name)
 
-    associate(pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
-    end associate
+      associate (pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
+      end associate
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_read_one_real")
+      if (decomp_profiler_io) call decomp_profiler_end("io_read_one_real")
 #endif
 
-  end subroutine read_one_real
-
+   end subroutine read_one_real
 
-  subroutine read_one_complex(ipencil,var,filename,opt_decomp)
+   subroutine read_one_complex(ipencil, var, filename, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil
-    complex(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    character(len=*), intent(IN) :: filename
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: ipencil
+      complex(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      character(len=*), intent(IN) :: filename
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer(kind=MPI_OFFSET_KIND) :: disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer(kind=MPI_OFFSET_KIND) :: disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, fh, data_type
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_read_one_cplx")
+      if (decomp_profiler_io) call decomp_profiler_start("io_read_one_cplx")
 #endif
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_read_one.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_read_one_cplx")
+      if (decomp_profiler_io) call decomp_profiler_end("io_read_one_cplx")
 #endif
 
-  end subroutine read_one_complex
+   end subroutine read_one_complex
 
 #ifdef ADIOS2
-  subroutine adios2_read_one_real(var,engine_name,varname,io_name)
+   subroutine adios2_read_one_real(var, engine_name, varname, io_name)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: engine_name
-    character(len=*), intent(in) :: io_name
-    character*(*), intent(in) :: varname
-    real(mytype), contiguous, dimension(:,:,:), intent(out) :: var
+      character(len=*), intent(in) :: engine_name
+      character(len=*), intent(in) :: io_name
+      character*(*), intent(in) :: varname
+      real(mytype), contiguous, dimension(:, :, :), intent(out) :: var
 
-    integer :: ierror
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
-    integer :: idx
+      integer :: ierror
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
+      integer :: idx
 
-    integer(kind=8) :: nsteps
-    integer(kind=8) :: curstep
+      integer(kind=8) :: nsteps
+      integer(kind=8) :: curstep
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("adios2_read_one_real")
-#endif
-
-    call adios2_at_io(io_handle, adios, io_name, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-    call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-    if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-       "ERROR: trying to read variable without registering first! "//trim(varname))
-
-    call adios2_variable_steps(nsteps, var_handle, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_variable_steps")
-    print *, "AVAILABLE steps for ", nsteps
-    
-    print *, "IO_NAME: ", io_name
-    print *, "ENGINE_NAME: ", engine_name
-    print *, "VAR_NAME: ", varname
-    idx = get_io_idx(io_name, engine_name)
-    print *, idx
-    call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_deferred, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
-
-    print *, "MAX: ", maxval(var)
-    
-    call adios2_current_step(curstep, engine_registry(idx), ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_current_step")
-    print *, "Current step: ", curstep
+      if (decomp_profiler_io) call decomp_profiler_start("adios2_read_one_real")
+#endif
+
+      call adios2_at_io(io_handle, adios, io_name, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+      call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+      if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                                                       "ERROR: trying to read variable without registering first! "//trim(varname))
+
+      call adios2_variable_steps(nsteps, var_handle, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_variable_steps")
+      print *, "AVAILABLE steps for ", nsteps
+
+      print *, "IO_NAME: ", io_name
+      print *, "ENGINE_NAME: ", engine_name
+      print *, "VAR_NAME: ", varname
+      idx = get_io_idx(io_name, engine_name)
+      print *, idx
+      call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_deferred, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
+
+      print *, "MAX: ", maxval(var)
+
+      call adios2_current_step(curstep, engine_registry(idx), ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_current_step")
+      print *, "Current step: ", curstep
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("adios2_read_one_real")
+      if (decomp_profiler_io) call decomp_profiler_end("adios2_read_one_real")
 #endif
-    
-  end subroutine adios2_read_one_real
+
+   end subroutine adios2_read_one_real
 #endif
-  
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write a 3D array as part of a big MPI-IO file, starting from 
-  !  displacement 'disp'; 'disp' will be updated after the writing
-  !  operation to prepare the writing of next chunk of data.
+   ! Write a 3D array as part of a big MPI-IO file, starting from
+   !  displacement 'disp'; 'disp' will be updated after the writing
+   !  operation to prepare the writing of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_var_real(fh,disp,ipencil,var,opt_decomp)
+   subroutine write_var_real(fh, disp, ipencil, var, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: ipencil
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: ipencil
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, data_type
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_var.inc"
 
-    return
-  end subroutine write_var_real
+      return
+   end subroutine write_var_real
 
-  subroutine write_var_complex(fh,disp,ipencil,var,opt_decomp)
+   subroutine write_var_complex(fh, disp, ipencil, var, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: ipencil
-    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: ipencil
+      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, data_type
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_write_var.inc"
 
-    return
-  end subroutine write_var_complex
-
+      return
+   end subroutine write_var_complex
 
-  subroutine write_outflow(dirname,varname,ntimesteps,var,io_name,opt_decomp)
+   subroutine write_outflow(dirname, varname, ntimesteps, var, io_name, opt_decomp)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: dirname, varname, io_name
-    integer, intent(IN) :: ntimesteps 
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      character(len=*), intent(in) :: dirname, varname, io_name
+      integer, intent(IN) :: ntimesteps
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, data_type
-    integer :: idx
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, data_type
+      integer :: idx
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
 #else
-    integer :: newtype
+      integer :: newtype
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_outflow")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_outflow")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_outflow.f90"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_outflow")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_outflow")
 #endif
 
-  end subroutine write_outflow
-
+   end subroutine write_outflow
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Read a 3D array as part of a big MPI-IO file, starting from 
-  !  displacement 'disp'; 'disp' will be updated after the reading
-  !  operation to prepare the reading of next chunk of data.
+   ! Read a 3D array as part of a big MPI-IO file, starting from
+   !  displacement 'disp'; 'disp' will be updated after the reading
+   !  operation to prepare the reading of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine read_var_real(fh,disp,ipencil,var,opt_decomp)
+   subroutine read_var_real(fh, disp, ipencil, var, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: ipencil
-    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: ipencil
+      real(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, data_type
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_read_var.inc"
 
-    return
-  end subroutine read_var_real
-
+      return
+   end subroutine read_var_real
 
-  subroutine read_var_complex(fh,disp,ipencil,var,opt_decomp)
+   subroutine read_var_complex(fh, disp, ipencil, var, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: ipencil
-    complex(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: ipencil
+      complex(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, newtype, data_type
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, newtype, data_type
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_read_var.inc"
 
-    return
-  end subroutine read_var_complex
+      return
+   end subroutine read_var_complex
 
+   subroutine read_inflow(dirname, varname, ntimesteps, var, io_name, opt_decomp)
 
-  subroutine read_inflow(dirname,varname,ntimesteps,var,io_name,opt_decomp)
+      implicit none
 
-    implicit none
+      character(len=*), intent(in) :: dirname, varname, io_name
+      integer, intent(IN) :: ntimesteps
+      real(mytype), contiguous, dimension(:, :, :), intent(INOUT) :: var
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    character(len=*), intent(in) :: dirname, varname, io_name
-    integer, intent(IN) :: ntimesteps
-    real(mytype), contiguous, dimension(:,:,:), intent(INOUT) :: var
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
-
-    TYPE(DECOMP_INFO) :: decomp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: ierror, data_type
-    integer :: idx
+      TYPE(DECOMP_INFO) :: decomp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: ierror, data_type
+      integer :: idx
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
 #else
-    integer :: newtype
+      integer :: newtype
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_read_inflow")
+      if (decomp_profiler_io) call decomp_profiler_start("io_read_inflow")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_read_inflow.f90"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_read_inflow")
+      if (decomp_profiler_io) call decomp_profiler_end("io_read_inflow")
 #endif
 
-  end subroutine read_inflow
+   end subroutine read_inflow
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write scalar variables as part of a big MPI-IO file, starting from 
-  !  displacement 'disp'; 'disp' will be updated after the reading
-  !  operation to prepare the reading of next chunk of data.
+   ! Write scalar variables as part of a big MPI-IO file, starting from
+   !  displacement 'disp'; 'disp' will be updated after the reading
+   !  operation to prepare the reading of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_scalar_real(fh,disp,n,var)
+   subroutine write_scalar_real(fh, disp, n, var)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh             ! file handle
-    integer(KIND=MPI_OFFSET_KIND), &
+      integer, intent(IN) :: fh             ! file handle
+      integer(KIND=MPI_OFFSET_KIND), &
          intent(INOUT) :: disp            ! displacement
-    integer, intent(IN) :: n              ! number of scalars
-    real(mytype), dimension(n), &
+      integer, intent(IN) :: n              ! number of scalars
+      real(mytype), dimension(n), &
          intent(IN) :: var                ! array of scalars
 
-    integer :: m, ierror
-
-    call MPI_FILE_SET_VIEW(fh,disp,real_type, &
-         real_type,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    if (nrank==0) then
-       m = n ! only one rank needs to write
-    else
-       m = 0
-    end if
-    call MPI_FILE_WRITE_ALL(fh, var, m, real_type, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(mytype_bytes, kind=MPI_OFFSET_KIND)
-
-    return
-  end subroutine write_scalar_real
-
-
-  subroutine write_scalar_complex(fh,disp,n,var)
-
-    implicit none
-
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    complex(mytype), dimension(n), intent(IN) :: var
-
-    integer :: m, ierror
-
-    call MPI_FILE_SET_VIEW(fh,disp,complex_type, &
-         complex_type,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    if (nrank==0) then
-       m = n
-    else
-       m = 0
-    end if
-    call MPI_FILE_WRITE_ALL(fh, var, m, complex_type, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-                * 2_MPI_OFFSET_KIND
-
-    return
-  end subroutine write_scalar_complex
-
-
-  subroutine write_scalar_integer(fh,disp,n,var)
-
-    implicit none
-
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    integer, dimension(n), intent(IN) :: var
-
-    integer :: m, ierror
-
-    call MPI_FILE_SET_VIEW(fh,disp,MPI_INTEGER, &
-         MPI_INTEGER,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    if (nrank==0) then
-       m = n
-    else
-       m = 0
-    end if
-    call MPI_FILE_WRITE_ALL(fh, var, m, MPI_INTEGER, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    call MPI_TYPE_SIZE(MPI_INTEGER,m,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(m, kind=MPI_OFFSET_KIND)
-
-    return
-  end subroutine write_scalar_integer
-
-
-  subroutine write_scalar_logical(fh,disp,n,var)
-
-    implicit none
-
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    logical, dimension(n), intent(IN) :: var
-
-    integer :: m, ierror
-
-    call MPI_FILE_SET_VIEW(fh,disp,MPI_LOGICAL, &
-         MPI_LOGICAL,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    if (nrank==0) then
-       m = n
-    else
-       m = 0
-    end if
-    call MPI_FILE_WRITE_ALL(fh, var, m, MPI_LOGICAL, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    call MPI_TYPE_SIZE(MPI_LOGICAL,m,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(m, kind=MPI_OFFSET_KIND)
-
-    return
-  end subroutine write_scalar_logical
+      integer :: m, ierror
+
+      call MPI_FILE_SET_VIEW(fh, disp, real_type, &
+                             real_type, 'native', MPI_INFO_NULL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      if (nrank == 0) then
+         m = n ! only one rank needs to write
+      else
+         m = 0
+      end if
+      call MPI_FILE_WRITE_ALL(fh, var, m, real_type, &
+                              MPI_STATUS_IGNORE, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+
+      return
+   end subroutine write_scalar_real
+
+   subroutine write_scalar_complex(fh, disp, n, var)
+
+      implicit none
+
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      complex(mytype), dimension(n), intent(IN) :: var
+
+      integer :: m, ierror
+
+      call MPI_FILE_SET_VIEW(fh, disp, complex_type, &
+                             complex_type, 'native', MPI_INFO_NULL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      if (nrank == 0) then
+         m = n
+      else
+         m = 0
+      end if
+      call MPI_FILE_WRITE_ALL(fh, var, m, complex_type, &
+                              MPI_STATUS_IGNORE, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+             *2_MPI_OFFSET_KIND
+
+      return
+   end subroutine write_scalar_complex
+
+   subroutine write_scalar_integer(fh, disp, n, var)
+
+      implicit none
+
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      integer, dimension(n), intent(IN) :: var
+
+      integer :: m, ierror
+
+      call MPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, &
+                             MPI_INTEGER, 'native', MPI_INFO_NULL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      if (nrank == 0) then
+         m = n
+      else
+         m = 0
+      end if
+      call MPI_FILE_WRITE_ALL(fh, var, m, MPI_INTEGER, &
+                              MPI_STATUS_IGNORE, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(m, kind=MPI_OFFSET_KIND)
+
+      return
+   end subroutine write_scalar_integer
+
+   subroutine write_scalar_logical(fh, disp, n, var)
 
+      implicit none
+
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      logical, dimension(n), intent(IN) :: var
+
+      integer :: m, ierror
+
+      call MPI_FILE_SET_VIEW(fh, disp, MPI_LOGICAL, &
+                             MPI_LOGICAL, 'native', MPI_INFO_NULL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      if (nrank == 0) then
+         m = n
+      else
+         m = 0
+      end if
+      call MPI_FILE_WRITE_ALL(fh, var, m, MPI_LOGICAL, &
+                              MPI_STATUS_IGNORE, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(m, kind=MPI_OFFSET_KIND)
+
+      return
+   end subroutine write_scalar_logical
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Read scalar variables as part of a big MPI-IO file, starting from 
-  !  displacement 'disp'; 'disp' will be updated after the reading
-  !  operation to prepare the reading of next chunk of data.
+   ! Read scalar variables as part of a big MPI-IO file, starting from
+   !  displacement 'disp'; 'disp' will be updated after the reading
+   !  operation to prepare the reading of next chunk of data.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine read_scalar_real(fh,disp,n,var)
+   subroutine read_scalar_real(fh, disp, n, var)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: fh             ! file handle
-    integer(KIND=MPI_OFFSET_KIND), &
+      integer, intent(IN) :: fh             ! file handle
+      integer(KIND=MPI_OFFSET_KIND), &
          intent(INOUT) :: disp            ! displacement
-    integer, intent(IN) :: n              ! number of scalars
-    real(mytype), dimension(n), &
+      integer, intent(IN) :: n              ! number of scalars
+      real(mytype), dimension(n), &
          intent(INOUT) :: var             ! array of scalars
 
-    integer :: ierror
+      integer :: ierror
 
-    call MPI_FILE_SET_VIEW(fh,disp,real_type, &
-         real_type,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_READ_ALL(fh, var, n, real_type, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+      call MPI_FILE_SET_VIEW(fh, disp, real_type, &
+                             real_type, 'native', MPI_INFO_NULL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_READ_ALL(fh, var, n, real_type, &
+                             MPI_STATUS_IGNORE, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 
-    return
-  end subroutine read_scalar_real
+      return
+   end subroutine read_scalar_real
 
+   subroutine read_scalar_complex(fh, disp, n, var)
 
-  subroutine read_scalar_complex(fh,disp,n,var)
+      implicit none
 
-    implicit none
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      complex(mytype), dimension(n), intent(INOUT) :: var
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    complex(mytype), dimension(n), intent(INOUT) :: var
+      integer :: ierror
 
-    integer :: ierror
+      call MPI_FILE_SET_VIEW(fh, disp, complex_type, &
+                             complex_type, 'native', MPI_INFO_NULL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_READ_ALL(fh, var, n, complex_type, &
+                             MPI_STATUS_IGNORE, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+             *2_MPI_OFFSET_KIND
 
-    call MPI_FILE_SET_VIEW(fh,disp,complex_type, &
-         complex_type,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_READ_ALL(fh, var, n, complex_type, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-                * 2_MPI_OFFSET_KIND
+      return
+   end subroutine read_scalar_complex
 
-    return
-  end subroutine read_scalar_complex
+   subroutine read_scalar_integer(fh, disp, n, var)
 
+      implicit none
 
-  subroutine read_scalar_integer(fh,disp,n,var)
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      integer, dimension(n), intent(INOUT) :: var
 
-    implicit none
+      integer :: m, ierror
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    integer, dimension(n), intent(INOUT) :: var
+      call MPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, &
+                             MPI_INTEGER, 'native', MPI_INFO_NULL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_READ_ALL(fh, var, n, MPI_INTEGER, &
+                             MPI_STATUS_IGNORE, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+      call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(m, kind=MPI_OFFSET_KIND)
 
-    integer :: m, ierror
+      return
+   end subroutine read_scalar_integer
 
-    call MPI_FILE_SET_VIEW(fh,disp,MPI_INTEGER, &
-         MPI_INTEGER,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_READ_ALL(fh, var, n, MPI_INTEGER, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    call MPI_TYPE_SIZE(MPI_INTEGER,m,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(m, kind=MPI_OFFSET_KIND)
+   subroutine read_scalar_logical(fh, disp, n, var)
 
-    return
-  end subroutine read_scalar_integer
+      implicit none
 
+      integer, intent(IN) :: fh
+      integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
+      integer, intent(IN) :: n
+      logical, dimension(n), intent(INOUT) :: var
 
-  subroutine read_scalar_logical(fh,disp,n,var)
+      integer :: m, ierror
 
-    implicit none
+      call MPI_FILE_SET_VIEW(fh, disp, MPI_LOGICAL, &
+                             MPI_LOGICAL, 'native', MPI_INFO_NULL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_READ_ALL(fh, var, n, MPI_LOGICAL, &
+                             MPI_STATUS_IGNORE, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+      call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+      disp = disp + int(n, kind=MPI_OFFSET_KIND) &
+             *int(m, kind=MPI_OFFSET_KIND)
 
-    integer, intent(IN) :: fh
-    integer(KIND=MPI_OFFSET_KIND), intent(INOUT) :: disp
-    integer, intent(IN) :: n
-    logical, dimension(n), intent(INOUT) :: var
+      return
+   end subroutine read_scalar_logical
 
-    integer :: m, ierror
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   ! Write a 2D slice of the 3D data to a file
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   subroutine plane_extents(sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes)
 
-    call MPI_FILE_SET_VIEW(fh,disp,MPI_LOGICAL, &
-         MPI_LOGICAL,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_READ_ALL(fh, var, n, MPI_LOGICAL, &
-         MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-    call MPI_TYPE_SIZE(MPI_LOGICAL,m,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-                * int(m, kind=MPI_OFFSET_KIND)
+      integer, intent(in) :: iplane
+      type(decomp_info), intent(in), optional :: opt_decomp
+      integer, intent(in), optional :: opt_nplanes
 
-    return
-  end subroutine read_scalar_logical
+      integer, dimension(3), intent(out) :: sizes, subsizes, starts
 
+      integer :: nplanes
+      type(decomp_info) :: decomp
 
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write a 2D slice of the 3D data to a file
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine plane_extents (sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes)
-
-    integer, intent(in) :: iplane
-    type(decomp_info), intent(in), optional :: opt_decomp
-    integer, intent(in), optional :: opt_nplanes
-    
-    integer, dimension(3), intent(out) :: sizes, subsizes, starts
-
-    integer :: nplanes
-    type(decomp_info) :: decomp
-
-    if (present(opt_decomp)) then
-       decomp = opt_decomp
-    else
-       call get_decomp_info(decomp)
-    end if
-
-    if (present(opt_nplanes)) then
-       nplanes = opt_nplanes
-    else
-       nplanes = 1
-    end if
-    
-    if (iplane == 1) then
-       sizes(1) = nplanes
-       sizes(2) = decomp%ysz(2)
-       sizes(3) = decomp%zsz(3)
-       subsizes(1) = nplanes
-       subsizes(2) = decomp%xsz(2)
-       subsizes(3) = decomp%xsz(3)
-       starts(1) = 0
-       starts(2) = decomp%xst(2)-1
-       starts(3) = decomp%xst(3)-1
-    else if (iplane == 2) then
-       sizes(1) = decomp%xsz(1)
-       sizes(2) = nplanes
-       sizes(3) = decomp%zsz(3)
-       subsizes(1) = decomp%ysz(1)
-       subsizes(2) = nplanes
-       subsizes(3) = decomp%ysz(3)
-       starts(1) = decomp%yst(1)-1
-       starts(2) = 0
-       starts(3) = decomp%yst(3)-1
-    else if (iplane == 3) then
-       sizes(1) = decomp%xsz(1)
-       sizes(2) = decomp%ysz(2)
-       sizes(3) = nplanes
-       subsizes(1) = decomp%zsz(1)
-       subsizes(2) = decomp%zsz(2)
-       subsizes(3) = nplanes
-       starts(1) = decomp%zst(1)-1
-       starts(2) = decomp%zst(2)-1
-       starts(3) = 0
-    else
-       print *, "Can't work with plane ", iplane
-       stop
-    endif
-    
-  end subroutine plane_extents
-
-  subroutine write_plane_3d_real(ipencil,var,iplane,n,dirname,varname,io_name, &
-       opt_decomp)
-
-    implicit none
-
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
-    integer, intent(IN) :: n ! which plane to write (global coordinate)
-    character(len=*), intent(IN) :: dirname,varname,io_name
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
-
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    real(mytype), allocatable, dimension(:,:,:) :: wk2d
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: i,j,k, ierror, data_type
-
-    logical :: opened_new
-    integer :: idx
+      if (present(opt_decomp)) then
+         decomp = opt_decomp
+      else
+         call get_decomp_info(decomp)
+      end if
+
+      if (present(opt_nplanes)) then
+         nplanes = opt_nplanes
+      else
+         nplanes = 1
+      end if
+
+      if (iplane == 1) then
+         sizes(1) = nplanes
+         sizes(2) = decomp%ysz(2)
+         sizes(3) = decomp%zsz(3)
+         subsizes(1) = nplanes
+         subsizes(2) = decomp%xsz(2)
+         subsizes(3) = decomp%xsz(3)
+         starts(1) = 0
+         starts(2) = decomp%xst(2) - 1
+         starts(3) = decomp%xst(3) - 1
+      else if (iplane == 2) then
+         sizes(1) = decomp%xsz(1)
+         sizes(2) = nplanes
+         sizes(3) = decomp%zsz(3)
+         subsizes(1) = decomp%ysz(1)
+         subsizes(2) = nplanes
+         subsizes(3) = decomp%ysz(3)
+         starts(1) = decomp%yst(1) - 1
+         starts(2) = 0
+         starts(3) = decomp%yst(3) - 1
+      else if (iplane == 3) then
+         sizes(1) = decomp%xsz(1)
+         sizes(2) = decomp%ysz(2)
+         sizes(3) = nplanes
+         subsizes(1) = decomp%zsz(1)
+         subsizes(2) = decomp%zsz(2)
+         subsizes(3) = nplanes
+         starts(1) = decomp%zst(1) - 1
+         starts(2) = decomp%zst(2) - 1
+         starts(3) = 0
+      else
+         print *, "Can't work with plane ", iplane
+         stop
+      end if
+
+   end subroutine plane_extents
+
+   subroutine write_plane_3d_real(ipencil, var, iplane, n, dirname, varname, io_name, &
+                                  opt_decomp)
+
+      implicit none
+
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
+      integer, intent(IN) :: n ! which plane to write (global coordinate)
+      character(len=*), intent(IN) :: dirname, varname, io_name
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      real(mytype), allocatable, dimension(:, :, :) :: wk2d
+      TYPE(DECOMP_INFO) :: decomp
+      integer :: i, j, k, ierror, data_type
+
+      logical :: opened_new
+      integer :: idx
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
 #else
-    integer, dimension(3) :: sizes, subsizes, starts
-    logical :: dir_exists
-    character(len=:), allocatable :: full_io_name
-    integer :: newtype
+      integer, dimension(3) :: sizes, subsizes, starts
+      logical :: dir_exists
+      character(len=:), allocatable :: full_io_name
+      integer :: newtype
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_real")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_real")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_plane.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_real")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_real")
 #endif
 
-  end subroutine write_plane_3d_real
-
+   end subroutine write_plane_3d_real
 
-  subroutine write_plane_3d_complex(ipencil,var,iplane,n, &
-       dirname,varname,io_name,opt_decomp)
+   subroutine write_plane_3d_complex(ipencil, var, iplane, n, &
+                                     dirname, varname, io_name, opt_decomp)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
-    integer, intent(IN) :: n ! which plane to write (global coordinate)
-    character(len=*), intent(IN) :: dirname,varname,io_name
-    TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: iplane !(x-plane=1; y-plane=2; z-plane=3)
+      integer, intent(IN) :: n ! which plane to write (global coordinate)
+      character(len=*), intent(IN) :: dirname, varname, io_name
+      TYPE(DECOMP_INFO), intent(IN), optional :: opt_decomp
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    complex(mytype), allocatable, dimension(:,:,:) :: wk2d
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: i,j,k, ierror, data_type
-    logical :: opened_new
-    integer :: idx
+      complex(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      complex(mytype), allocatable, dimension(:, :, :) :: wk2d
+      TYPE(DECOMP_INFO) :: decomp
+      integer :: i, j, k, ierror, data_type
+      logical :: opened_new
+      integer :: idx
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
 #else
-    integer, dimension(3) :: sizes, subsizes, starts
-    logical :: dir_exists
-    character(len=:), allocatable :: full_io_name
-    integer :: newtype
+      integer, dimension(3) :: sizes, subsizes, starts
+      logical :: dir_exists
+      character(len=:), allocatable :: full_io_name
+      integer :: newtype
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_cplx")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_plane_3d_cplx")
 #endif
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_write_plane.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_cplx")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_plane_3d_cplx")
 #endif
 
-  end subroutine write_plane_3d_complex
-
+   end subroutine write_plane_3d_complex
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write a 2D array to a file
+   ! Write a 2D array to a file
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  !************** TO DO ***************
-  !* Consider handling distributed 2D data set
-  !  subroutine write_plane_2d(ipencil,var,filename)
-  !    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-  !    real(mytype), dimension(:,:), intent(IN) :: var ! 2D array
-  !    character(len=*), intent(IN) :: filename
-  !
-  !    if (ipencil==1) then
-  !       ! var should be defined as var(xsize(2)
-  !
-  !    else if (ipencil==2) then
-  !
-  !    else if (ipencil==3) then
-  !
-  !    end if
-  !
-  !    return
-  !  end subroutine write_plane_2d
-
+   !************** TO DO ***************
+   !* Consider handling distributed 2D data set
+   !  subroutine write_plane_2d(ipencil,var,filename)
+   !    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+   !    real(mytype), dimension(:,:), intent(IN) :: var ! 2D array
+   !    character(len=*), intent(IN) :: filename
+   !
+   !    if (ipencil==1) then
+   !       ! var should be defined as var(xsize(2)
+   !
+   !    else if (ipencil==2) then
+   !
+   !    else if (ipencil==3) then
+   !
+   !    end if
+   !
+   !    return
+   !  end subroutine write_plane_2d
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write 3D array data for every specified mesh point
+   ! Write 3D array data for every specified mesh point
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_every_real(ipencil,var,iskip,jskip,kskip, &
-       filename, from1)
+   subroutine write_every_real(ipencil, var, iskip, jskip, kskip, &
+                               filename, from1)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: iskip,jskip,kskip 
-    character(len=*), intent(IN) :: filename
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: iskip, jskip, kskip
+      character(len=*), intent(IN) :: filename
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
 
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: i,j,k, ierror, newtype, fh, key,color,newcomm, data_type
-    integer, dimension(3) :: xsz,ysz,zsz,xst,yst,zst,xen,yen,zen,skip
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: i, j, k, ierror, newtype, fh, key, color, newcomm, data_type
+      integer, dimension(3) :: xsz, ysz, zsz, xst, yst, zst, xen, yen, zen, skip
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_every_real")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_every_real")
 #endif
 
-    data_type = real_type
+      data_type = real_type
 
 #include "io_write_every.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_every_real")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_every_real")
 #endif
 
-  end subroutine write_every_real
-
+   end subroutine write_every_real
 
-  subroutine write_every_complex(ipencil,var,iskip,jskip,kskip, &
-       filename, from1)
+   subroutine write_every_complex(ipencil, var, iskip, jskip, kskip, &
+                                  filename, from1)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    complex(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: iskip,jskip,kskip 
-    character(len=*), intent(IN) :: filename
-    logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
-    ! .false. - save n,2n,3n...
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      complex(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: iskip, jskip, kskip
+      character(len=*), intent(IN) :: filename
+      logical, intent(IN) :: from1  ! .true.  - save 1,n+1,2n+1...
+      ! .false. - save n,2n,3n...
 
-    complex(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: i,j,k, ierror, newtype, fh, key,color,newcomm, data_type
-    integer, dimension(3) :: xsz,ysz,zsz,xst,yst,zst,xen,yen,zen,skip
+      complex(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: i, j, k, ierror, newtype, fh, key, color, newcomm, data_type
+      integer, dimension(3) :: xsz, ysz, zsz, xst, yst, zst, xen, yen, zen, skip
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_every_cplx")
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_every_cplx")
 #endif
 
-    data_type = complex_type
+      data_type = complex_type
 
 #include "io_write_every.inc"
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_every_cplx")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_every_cplx")
 #endif
 
-  end subroutine write_every_complex
+   end subroutine write_every_complex
 
-  subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
+   subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
 
-    implicit none
-    
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-    type(decomp_info), intent(in), optional :: opt_decomp
+      implicit none
 
-    integer, dimension(3) :: sizes, subsizes, starts
-    type(decomp_info) :: decomp
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
+      type(decomp_info), intent(in), optional :: opt_decomp
 
-    if ((icoarse.lt.0).or.(icoarse.gt.2)) then
-       call decomp_2d_abort(__FILE__, __LINE__, icoarse, "Error invalid value of icoarse")
-    endif
-    if ((ipencil.lt.1).or.(ipencil.gt.3)) then
-       call decomp_2d_abort(__FILE__, __LINE__, ipencil, "Error invalid value of ipencil ")
-    endif
+      integer, dimension(3) :: sizes, subsizes, starts
+      type(decomp_info) :: decomp
 
-    if (icoarse==0) then
+      if ((icoarse < 0) .or. (icoarse > 2)) then
+         call decomp_2d_abort(__FILE__, __LINE__, icoarse, "Error invalid value of icoarse")
+      end if
+      if ((ipencil < 1) .or. (ipencil > 3)) then
+         call decomp_2d_abort(__FILE__, __LINE__, ipencil, "Error invalid value of ipencil ")
+      end if
+
+      if (icoarse == 0) then
        !! Use full fields
-       
-       if (present(opt_decomp)) then
-          decomp = opt_decomp
-       else
-          call get_decomp_info(decomp)
-       endif
-
-       sizes(1) = decomp%xsz(1)
-       sizes(2) = decomp%ysz(2)
-       sizes(3) = decomp%zsz(3)
-
-       if (ipencil == 1) then
-          subsizes(1:3) = decomp%xsz(1:3)
-          starts(1:3) = decomp%xst(1:3) - 1
-       elseif (ipencil == 2) then
-          subsizes(1:3) = decomp%ysz(1:3)
-          starts(1:3) = decomp%yst(1:3) - 1
-       elseif (ipencil == 3) then
-          subsizes(1:3) = decomp%zsz(1:3)
-          starts(1:3) = decomp%zst(1:3) - 1
-       else
-          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
-       endif
-    elseif (icoarse==1) then
-       sizes(1) = xszS(1)
-       sizes(2) = yszS(2)
-       sizes(3) = zszS(3)
-
-       if (ipencil == 1) then
-          subsizes(1) = xszS(1)
-          subsizes(2) = xszS(2)
-          subsizes(3) = xszS(3)
-          starts(1) = xstS(1)-1  ! 0-based index
-          starts(2) = xstS(2)-1
-          starts(3) = xstS(3)-1
-       else if (ipencil == 2) then
-          subsizes(1) = yszS(1)
-          subsizes(2) = yszS(2)
-          subsizes(3) = yszS(3)
-          starts(1) = ystS(1)-1
-          starts(2) = ystS(2)-1
-          starts(3) = ystS(3)-1
-       else if (ipencil == 3) then
-          subsizes(1) = zszS(1)
-          subsizes(2) = zszS(2)
-          subsizes(3) = zszS(3)
-          starts(1) = zstS(1)-1
-          starts(2) = zstS(2)-1
-          starts(3) = zstS(3)-1
-       else
-          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
-       endif
-    elseif (icoarse==2) then
-       sizes(1) = xszV(1)
-       sizes(2) = yszV(2)
-       sizes(3) = zszV(3)
-
-       if (ipencil == 1) then
-          subsizes(1) = xszV(1)
-          subsizes(2) = xszV(2)
-          subsizes(3) = xszV(3)
-          starts(1) = xstV(1)-1  ! 0-based index
-          starts(2) = xstV(2)-1
-          starts(3) = xstV(3)-1
-       else if (ipencil == 2) then
-          subsizes(1) = yszV(1)
-          subsizes(2) = yszV(2)
-          subsizes(3) = yszV(3)
-          starts(1) = ystV(1)-1
-          starts(2) = ystV(2)-1
-          starts(3) = ystV(3)-1
-       else if (ipencil == 3) then
-          subsizes(1) = zszV(1)
-          subsizes(2) = zszV(2)
-          subsizes(3) = zszV(3)
-          starts(1) = zstV(1)-1
-          starts(2) = zstV(2)-1
-          starts(3) = zstV(3)-1
-       else
-          call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
-       endif
-    endif
-    
-  end subroutine coarse_extents
-
-  subroutine mpiio_write_real_coarse(ipencil,var,dirname,varname,icoarse,io_name,opt_decomp,reduce_prec,opt_deferred_writes)
-
-    ! USE param
-    ! USE variables
-
-    implicit none
-
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    character(len=*), intent(in) :: dirname, varname, io_name
-    type(decomp_info), intent(in), optional :: opt_decomp
-    logical, intent(in), optional :: reduce_prec
-    logical, intent(in), optional :: opt_deferred_writes
-
-    logical :: write_reduce_prec
-    logical :: deferred_writes
-    
-    integer :: ierror
-    integer :: idx
-    logical :: opened_new
+
+         if (present(opt_decomp)) then
+            decomp = opt_decomp
+         else
+            call get_decomp_info(decomp)
+         end if
+
+         sizes(1) = decomp%xsz(1)
+         sizes(2) = decomp%ysz(2)
+         sizes(3) = decomp%zsz(3)
+
+         if (ipencil == 1) then
+            subsizes(1:3) = decomp%xsz(1:3)
+            starts(1:3) = decomp%xst(1:3) - 1
+         elseif (ipencil == 2) then
+            subsizes(1:3) = decomp%ysz(1:3)
+            starts(1:3) = decomp%yst(1:3) - 1
+         elseif (ipencil == 3) then
+            subsizes(1:3) = decomp%zsz(1:3)
+            starts(1:3) = decomp%zst(1:3) - 1
+         else
+            call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
+         end if
+      elseif (icoarse == 1) then
+         sizes(1) = xszS(1)
+         sizes(2) = yszS(2)
+         sizes(3) = zszS(3)
+
+         if (ipencil == 1) then
+            subsizes(1) = xszS(1)
+            subsizes(2) = xszS(2)
+            subsizes(3) = xszS(3)
+            starts(1) = xstS(1) - 1  ! 0-based index
+            starts(2) = xstS(2) - 1
+            starts(3) = xstS(3) - 1
+         else if (ipencil == 2) then
+            subsizes(1) = yszS(1)
+            subsizes(2) = yszS(2)
+            subsizes(3) = yszS(3)
+            starts(1) = ystS(1) - 1
+            starts(2) = ystS(2) - 1
+            starts(3) = ystS(3) - 1
+         else if (ipencil == 3) then
+            subsizes(1) = zszS(1)
+            subsizes(2) = zszS(2)
+            subsizes(3) = zszS(3)
+            starts(1) = zstS(1) - 1
+            starts(2) = zstS(2) - 1
+            starts(3) = zstS(3) - 1
+         else
+            call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
+         end if
+      elseif (icoarse == 2) then
+         sizes(1) = xszV(1)
+         sizes(2) = yszV(2)
+         sizes(3) = zszV(3)
+
+         if (ipencil == 1) then
+            subsizes(1) = xszV(1)
+            subsizes(2) = xszV(2)
+            subsizes(3) = xszV(3)
+            starts(1) = xstV(1) - 1  ! 0-based index
+            starts(2) = xstV(2) - 1
+            starts(3) = xstV(3) - 1
+         else if (ipencil == 2) then
+            subsizes(1) = yszV(1)
+            subsizes(2) = yszV(2)
+            subsizes(3) = yszV(3)
+            starts(1) = ystV(1) - 1
+            starts(2) = ystV(2) - 1
+            starts(3) = ystV(3) - 1
+         else if (ipencil == 3) then
+            subsizes(1) = zszV(1)
+            subsizes(2) = zszV(2)
+            subsizes(3) = zszV(3)
+            starts(1) = zstV(1) - 1
+            starts(2) = zstV(2) - 1
+            starts(3) = zstV(3) - 1
+         else
+            call decomp_2d_abort(-1, "IO/coarse_extents : Wrong value for ipencil")
+         end if
+      end if
+
+   end subroutine coarse_extents
+
+  subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_name, opt_decomp, reduce_prec, opt_deferred_writes)
+
+      ! USE param
+      ! USE variables
+
+      implicit none
+
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      character(len=*), intent(in) :: dirname, varname, io_name
+      type(decomp_info), intent(in), optional :: opt_decomp
+      logical, intent(in), optional :: reduce_prec
+      logical, intent(in), optional :: opt_deferred_writes
+
+      logical :: write_reduce_prec
+      logical :: deferred_writes
+
+      integer :: ierror
+      integer :: idx
+      logical :: opened_new
 #ifdef ADIOS2
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
-    integer :: write_mode
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
+      integer :: write_mode
 #else
-    real(mytype_single), allocatable, dimension(:,:,:) :: varsingle
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: newtype
-    logical :: dir_exists
-    integer :: disp_bytes
-    character(len=:), allocatable :: full_io_name
+      real(mytype_single), allocatable, dimension(:, :, :) :: varsingle
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: newtype
+      logical :: dir_exists
+      integer :: disp_bytes
+      character(len=:), allocatable :: full_io_name
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_coarse")
+      if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_coarse")
 #endif
 
     !! Set defaults
-    write_reduce_prec = .true.
-    if (present(opt_deferred_writes)) then
-       deferred_writes = opt_deferred_writes
-    else
-       deferred_writes = .true.
-    end if
-    
-    opened_new = .false.
-    idx = get_io_idx(io_name, dirname)
+      write_reduce_prec = .true.
+      if (present(opt_deferred_writes)) then
+         deferred_writes = opt_deferred_writes
+      else
+         deferred_writes = .true.
+      end if
+
+      opened_new = .false.
+      idx = get_io_idx(io_name, dirname)
 #ifndef ADIOS2
-    if (present(reduce_prec)) then
-       if (.not. reduce_prec) then
-          write_reduce_prec = .false.
-       end if
-    end if
-    if (write_reduce_prec) then
-       call MPI_TYPE_SIZE(real_type_single,disp_bytes,ierror)
-    else
-       call MPI_TYPE_SIZE(real_type,disp_bytes,ierror)
-    end if
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
-    
+      if (present(reduce_prec)) then
+         if (.not. reduce_prec) then
+            write_reduce_prec = .false.
+         end if
+      end if
+      if (write_reduce_prec) then
+         call MPI_TYPE_SIZE(real_type_single, disp_bytes, ierror)
+      else
+         call MPI_TYPE_SIZE(real_type, disp_bytes, ierror)
+      end if
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
+
     !! Use original MPIIO writers
-    if (present(opt_decomp)) then
-       call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
-    else
-       call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
-    end if
-    if (write_reduce_prec) then
-       allocate (varsingle(xstV(1):xenV(1),xstV(2):xenV(2),xstV(3):xenV(3)))
-       varsingle=real(var, mytype_single)
-    end if
-
-    if (write_reduce_prec) then
-       call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-            MPI_ORDER_FORTRAN, real_type_single, newtype, ierror)
-    else
-       call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-            MPI_ORDER_FORTRAN, real_type, newtype, ierror)
-    end if
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-    call MPI_TYPE_COMMIT(newtype,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-
-    if (idx .lt. 1) then
-       ! Create folder if needed
-       if (nrank==0) then
-          inquire(file=dirname, exist=dir_exists)
-          if (.not.dir_exists) then
-             call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
-          end if
-       end if
-       full_io_name = trim(dirname)//"/"//trim(varname)
-       call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
-       idx = get_io_idx(io_name, full_io_name)
-       opened_new = .true.
-    else
-       full_io_name = "" ! Ensure string is set
-    end if
-
-    if (write_reduce_prec) then
-       call MPI_FILE_SET_VIEW(fh_registry(idx),fh_disp(idx),real_type_single, &
-            newtype,'native',MPI_INFO_NULL,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-       call MPI_FILE_WRITE_ALL(fh_registry(idx), varsingle, &
-            subsizes(1)*subsizes(2)*subsizes(3), &
-            real_type_single, MPI_STATUS_IGNORE, ierror)
-    else
-       call MPI_FILE_SET_VIEW(fh_registry(idx),fh_disp(idx),real_type, &
-            newtype,'native',MPI_INFO_NULL,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-       call MPI_FILE_WRITE_ALL(fh_registry(idx), var, &
-            subsizes(1)*subsizes(2)*subsizes(3), &
-            real_type, MPI_STATUS_IGNORE, ierror)
-    end if
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    
-    fh_disp(idx) = fh_disp(idx) + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                * int(disp_bytes, kind=MPI_OFFSET_KIND)
-    
-    if (opened_new) then
-       call decomp_2d_close_io(io_name, full_io_name)
-       deallocate(full_io_name)
-    end if
-
-    call MPI_TYPE_FREE(newtype,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-    if (write_reduce_prec) then
-       deallocate(varsingle)
-    end if
+      if (present(opt_decomp)) then
+         call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
+      else
+         call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
+      end if
+      if (write_reduce_prec) then
+         allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
+         varsingle = real(var, mytype_single)
+      end if
+
+      if (write_reduce_prec) then
+         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                       MPI_ORDER_FORTRAN, real_type_single, newtype, ierror)
+      else
+         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                       MPI_ORDER_FORTRAN, real_type, newtype, ierror)
+      end if
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+      call MPI_TYPE_COMMIT(newtype, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+
+      if (idx < 1) then
+         ! Create folder if needed
+         if (nrank == 0) then
+            inquire (file=dirname, exist=dir_exists)
+            if (.not. dir_exists) then
+               call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
+            end if
+         end if
+         full_io_name = trim(dirname)//"/"//trim(varname)
+         call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
+         idx = get_io_idx(io_name, full_io_name)
+         opened_new = .true.
+      else
+         full_io_name = "" ! Ensure string is set
+      end if
+
+      if (write_reduce_prec) then
+         call MPI_FILE_SET_VIEW(fh_registry(idx), fh_disp(idx), real_type_single, &
+                                newtype, 'native', MPI_INFO_NULL, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+         call MPI_FILE_WRITE_ALL(fh_registry(idx), varsingle, &
+                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 real_type_single, MPI_STATUS_IGNORE, ierror)
+      else
+         call MPI_FILE_SET_VIEW(fh_registry(idx), fh_disp(idx), real_type, &
+                                newtype, 'native', MPI_INFO_NULL, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+         call MPI_FILE_WRITE_ALL(fh_registry(idx), var, &
+                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 real_type, MPI_STATUS_IGNORE, ierror)
+      end if
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+
+      fh_disp(idx) = fh_disp(idx) + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                     *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                     *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                     *int(disp_bytes, kind=MPI_OFFSET_KIND)
+
+      if (opened_new) then
+         call decomp_2d_close_io(io_name, full_io_name)
+         deallocate (full_io_name)
+      end if
+
+      call MPI_TYPE_FREE(newtype, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+      if (write_reduce_prec) then
+         deallocate (varsingle)
+      end if
 #else
-    if (.not. engine_live(idx)) then
-       call decomp_2d_abort(__FILE__, __LINE__, -1, "ERROR: Engine is not live!")
-    end if
-    
-    call adios2_at_io(io_handle, adios, io_name, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-    call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-    if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-       "ERROR: trying to write variable before registering! "//trim(varname))
-
-    if (idx .lt. 1) call decomp_2d_abort(__FILE__, __LINE__, idx, &
-       "You haven't opened "//trim(io_name)//":"//trim(dirname))
-
-    if (deferred_writes) then
-       write_mode = adios2_mode_deferred
-    else
-       write_mode = adios2_mode_sync
-    end if
-
-    if (engine_registry(idx)%valid) then
-       call adios2_put(engine_registry(idx), var_handle, var, write_mode, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
-    else
-       call decomp_2d_abort(__FILE__, __LINE__, -1, &
-          "ERROR: decomp2d thinks engine is live, but adios2 engine object is not valid")
-    end if
-
-    associate(crs => icoarse, pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
-    end associate
+      if (.not. engine_live(idx)) then
+         call decomp_2d_abort(__FILE__, __LINE__, -1, "ERROR: Engine is not live!")
+      end if
+
+      call adios2_at_io(io_handle, adios, io_name, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+      call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+      if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                                                       "ERROR: trying to write variable before registering! "//trim(varname))
+
+      if (idx < 1) call decomp_2d_abort(__FILE__, __LINE__, idx, &
+                                        "You haven't opened "//trim(io_name)//":"//trim(dirname))
+
+      if (deferred_writes) then
+         write_mode = adios2_mode_deferred
+      else
+         write_mode = adios2_mode_sync
+      end if
+
+      if (engine_registry(idx)%valid) then
+         call adios2_put(engine_registry(idx), var_handle, var, write_mode, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
+      else
+         call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                              "ERROR: decomp2d thinks engine is live, but adios2 engine object is not valid")
+      end if
+
+      associate (crs => icoarse, pncl => ipencil, opdcmp => opt_decomp, rdprec => reduce_prec) ! Silence unused arguments
+      end associate
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_coarse")
+      if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_coarse")
 #endif
 
-  end subroutine mpiio_write_real_coarse
+   end subroutine mpiio_write_real_coarse
 
-  subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplane, type, opt_decomp, opt_nplanes)
+   subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplane, type, opt_decomp, opt_nplanes)
 
-    use, intrinsic :: iso_fortran_env, only : real32, real64
+      use, intrinsic :: iso_fortran_env, only: real32, real64
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
-    character(len=*), intent(in) :: io_name
-    integer, intent(in) :: type
-    integer, intent(in) :: iplane
-    type(decomp_info), intent(in), optional :: opt_decomp
-    integer, intent(in), optional :: opt_nplanes
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      integer, intent(IN) :: icoarse !(nstat=1; nvisu=2)
+      character(len=*), intent(in) :: io_name
+      integer, intent(in) :: type
+      integer, intent(in) :: iplane
+      type(decomp_info), intent(in), optional :: opt_decomp
+      integer, intent(in), optional :: opt_nplanes
 
-    integer :: nplanes
-    character*(*), intent(in) :: varname
+      integer :: nplanes
+      character*(*), intent(in) :: varname
 #ifdef ADIOS2
-    integer, dimension(3) :: sizes, subsizes, starts
-    type(adios2_io) :: io_handle
-    type(adios2_variable) :: var_handle
-    integer, parameter :: ndims = 3
-    logical, parameter :: adios2_constant_dims = .true.
-    integer :: data_type
-    integer :: ierror
-
-    if (iplane .eq. 0) then
-       if (present(opt_decomp)) then
-          call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
-       else
-          call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
-       endif
-    else
-       if (present(opt_nplanes)) then
-          nplanes = opt_nplanes
-       else
-          nplanes = 1
-       end if
-       if (present(opt_decomp)) then
-          call plane_extents(sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes=nplanes)
-       else
-          call plane_extents(sizes, subsizes, starts, iplane, opt_nplanes=nplanes)
-       endif
-    end if
-    
-    ! Check if variable already exists, if not create it
-    call adios2_at_io(io_handle, adios, io_name, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-    if (io_handle%valid) then
-       call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-       if (.not.var_handle % valid) then
+      integer, dimension(3) :: sizes, subsizes, starts
+      type(adios2_io) :: io_handle
+      type(adios2_variable) :: var_handle
+      integer, parameter :: ndims = 3
+      logical, parameter :: adios2_constant_dims = .true.
+      integer :: data_type
+      integer :: ierror
+
+      if (iplane == 0) then
+         if (present(opt_decomp)) then
+            call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
+         else
+            call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
+         end if
+      else
+         if (present(opt_nplanes)) then
+            nplanes = opt_nplanes
+         else
+            nplanes = 1
+         end if
+         if (present(opt_decomp)) then
+            call plane_extents(sizes, subsizes, starts, iplane, opt_decomp, opt_nplanes=nplanes)
+         else
+            call plane_extents(sizes, subsizes, starts, iplane, opt_nplanes=nplanes)
+         end if
+      end if
+
+      ! Check if variable already exists, if not create it
+      call adios2_at_io(io_handle, adios, io_name, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+      if (io_handle%valid) then
+         call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+         if (.not. var_handle%valid) then
           !! New variable
-          if (nrank .eq. 0) then
-             print *, "Registering variable for IO: ", varname
-          endif
+            if (nrank == 0) then
+               print *, "Registering variable for IO: ", varname
+            end if
 
-          ! Need to set the ADIOS2 data type
-          if (type.eq.kind(0._real64)) then
+            ! Need to set the ADIOS2 data type
+            if (type == kind(0._real64)) then
              !! Double
-             data_type = adios2_type_dp
-          else if (type.eq.kind(0._real32)) then
+               data_type = adios2_type_dp
+            else if (type == kind(0._real32)) then
              !! Single
-             data_type = adios2_type_real
-          else
-             call decomp_2d_abort(__FILE__, __LINE__, -1, "Trying to write unknown data type!")
-          endif
-
-          call adios2_define_variable(var_handle, io_handle, varname, data_type, &
-               ndims, int(sizes, kind=8), int(starts, kind=8), int(subsizes, kind=8), &
-               adios2_constant_dims, ierror)
-          if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-             "adios2_define_variable, ERROR registering variable "//trim(varname))
-       endif
-    else
-       call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to register variable with invalid IO!")
-    end if
+               data_type = adios2_type_real
+            else
+               call decomp_2d_abort(__FILE__, __LINE__, -1, "Trying to write unknown data type!")
+            end if
+
+            call adios2_define_variable(var_handle, io_handle, varname, data_type, &
+                                        ndims, int(sizes, kind=8), int(starts, kind=8), int(subsizes, kind=8), &
+                                        adios2_constant_dims, ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                                                  "adios2_define_variable, ERROR registering variable "//trim(varname))
+         end if
+      else
+         call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to register variable with invalid IO!")
+      end if
 #else
-    nplanes = 1 ! Silence unused variable
-    associate(crs => icoarse, nm =>io_name, pncl => ipencil, pln => iplane, &
-         opdcmp => opt_decomp, opnpl => opt_nplanes, tp => type, &
-         vnm => varname) ! Silence unused dummy argument
-    end associate
+      nplanes = 1 ! Silence unused variable
+      associate (crs => icoarse, nm => io_name, pncl => ipencil, pln => iplane, &
+                 opdcmp => opt_decomp, opnpl => opt_nplanes, tp => type, &
+                 vnm => varname) ! Silence unused dummy argument
+      end associate
 #endif
-    
-  end subroutine decomp_2d_register_variable
-  
-  subroutine mpiio_write_real_probe(ipencil,var,filename,nlength)
 
-    ! USE param
-    ! USE variables
+   end subroutine decomp_2d_register_variable
+
+   subroutine mpiio_write_real_probe(ipencil, var, filename, nlength)
+
+      ! USE param
+      ! USE variables
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    integer, intent(in) :: nlength
-    real(mytype), contiguous, dimension(:,:,:,:), intent(IN) :: var
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      integer, intent(in) :: nlength
+      real(mytype), contiguous, dimension(:, :, :, :), intent(IN) :: var
 
-    character(len=*) :: filename
+      character(len=*) :: filename
 
-    integer (kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(4) :: sizes, subsizes, starts
-    integer :: ierror, newtype, fh
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(4) :: sizes, subsizes, starts
+      integer :: ierror, newtype, fh
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_probe")
-#endif
-
-    sizes(1) = xszP(1)
-    sizes(2) = yszP(2)
-    sizes(3) = zszP(3)
-    sizes(4) = nlength
-    if (ipencil == 1) then
-       subsizes(1) = xszP(1)
-       subsizes(2) = xszP(2)
-       subsizes(3) = xszP(3)
-       subsizes(4) = nlength
-       starts(1) = xstP(1)-1  ! 0-based index
-       starts(2) = xstP(2)-1
-       starts(3) = xstP(3)-1
-       starts(4) = 0
-    else if (ipencil == 2) then
-       subsizes(1) = yszP(1)
-       subsizes(2) = yszP(2)
-       subsizes(3) = yszP(3)
-       starts(1) = ystP(1)-1
-       starts(2) = ystP(2)-1
-       starts(3) = ystP(3)-1
-    else if (ipencil == 3) then
-       subsizes(1) = zszP(1)
-       subsizes(2) = zszP(2)
-       subsizes(3) = zszP(3)
-       starts(1) = zstP(1)-1
-       starts(2) = zstP(2)-1
-       starts(3) = zstP(3)-1
-    endif
-    !   print *,nrank,starts(1),starts(2),starts(3),starts(4)
-    call MPI_TYPE_CREATE_SUBARRAY(4, sizes, subsizes, starts,  &
-         MPI_ORDER_FORTRAN, real_type, newtype, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-    call MPI_TYPE_COMMIT(newtype,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-    call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-         MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-         fh, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
-    filesize = 0_MPI_OFFSET_KIND
-    call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
-    disp = 0_MPI_OFFSET_KIND
-    call MPI_FILE_SET_VIEW(fh,disp,real_type, &
-         newtype,'native',MPI_INFO_NULL,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-    call MPI_FILE_WRITE_ALL(fh, var, &
-         subsizes(1)*subsizes(2)*subsizes(3)*subsizes(4), &
-         real_type, MPI_STATUS_IGNORE, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-    call MPI_FILE_CLOSE(fh,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-    call MPI_TYPE_FREE(newtype,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+      if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_probe")
+#endif
+
+      sizes(1) = xszP(1)
+      sizes(2) = yszP(2)
+      sizes(3) = zszP(3)
+      sizes(4) = nlength
+      if (ipencil == 1) then
+         subsizes(1) = xszP(1)
+         subsizes(2) = xszP(2)
+         subsizes(3) = xszP(3)
+         subsizes(4) = nlength
+         starts(1) = xstP(1) - 1  ! 0-based index
+         starts(2) = xstP(2) - 1
+         starts(3) = xstP(3) - 1
+         starts(4) = 0
+      else if (ipencil == 2) then
+         subsizes(1) = yszP(1)
+         subsizes(2) = yszP(2)
+         subsizes(3) = yszP(3)
+         starts(1) = ystP(1) - 1
+         starts(2) = ystP(2) - 1
+         starts(3) = ystP(3) - 1
+      else if (ipencil == 3) then
+         subsizes(1) = zszP(1)
+         subsizes(2) = zszP(2)
+         subsizes(3) = zszP(3)
+         starts(1) = zstP(1) - 1
+         starts(2) = zstP(2) - 1
+         starts(3) = zstP(3) - 1
+      end if
+      !   print *,nrank,starts(1),starts(2),starts(3),starts(4)
+      call MPI_TYPE_CREATE_SUBARRAY(4, sizes, subsizes, starts, &
+                                    MPI_ORDER_FORTRAN, real_type, newtype, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+      call MPI_TYPE_COMMIT(newtype, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+      call MPI_FILE_OPEN(decomp_2d_comm, filename, &
+                         MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                         fh, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+      filesize = 0_MPI_OFFSET_KIND
+      call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+      disp = 0_MPI_OFFSET_KIND
+      call MPI_FILE_SET_VIEW(fh, disp, real_type, &
+                             newtype, 'native', MPI_INFO_NULL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+      call MPI_FILE_WRITE_ALL(fh, var, &
+                              subsizes(1)*subsizes(2)*subsizes(3)*subsizes(4), &
+                              real_type, MPI_STATUS_IGNORE, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+      call MPI_FILE_CLOSE(fh, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+      call MPI_TYPE_FREE(newtype, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_probe")
+      if (decomp_profiler_io) call decomp_profiler_end("mpiio_write_real_probe")
 #endif
 
-  end subroutine mpiio_write_real_probe
+   end subroutine mpiio_write_real_probe
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  ! Write a 3D data set covering a smaller sub-domain only
+   ! Write a 3D data set covering a smaller sub-domain only
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-  subroutine write_subdomain(ipencil,var,is,ie,js,je,ks,ke,filename)
+   subroutine write_subdomain(ipencil, var, is, ie, js, je, ks, ke, filename)
 
-    implicit none
+      implicit none
 
-    integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
-    real(mytype), contiguous, dimension(:,:,:), intent(IN) :: var
-    integer, intent(IN) :: is, ie, js, je, ks, ke
-    character(len=*), intent(IN) :: filename
+      integer, intent(IN) :: ipencil !(x-pencil=1; y-pencil=2; z-pencil=3)
+      real(mytype), contiguous, dimension(:, :, :), intent(IN) :: var
+      integer, intent(IN) :: is, ie, js, je, ks, ke
+      character(len=*), intent(IN) :: filename
 
-    real(mytype), allocatable, dimension(:,:,:) :: wk, wk2
-    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
-    integer, dimension(3) :: sizes, subsizes, starts
-    integer :: color, key, errorcode, newcomm, ierror
-    integer :: newtype, fh, data_type, i, j, k
-    integer :: i1, i2, j1, j2, k1, k2
+      real(mytype), allocatable, dimension(:, :, :) :: wk, wk2
+      integer(kind=MPI_OFFSET_KIND) :: filesize, disp
+      integer, dimension(3) :: sizes, subsizes, starts
+      integer :: color, key, errorcode, newcomm, ierror
+      integer :: newtype, fh, data_type, i, j, k
+      integer :: i1, i2, j1, j2, k1, k2
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_write_subdomain")
-#endif
-
-    data_type = real_type
-
-    ! validate the input paramters
-    if (is<1 .OR. ie>nx_global .OR. js<1 .OR. je>ny_global .OR. &
-         ks<1 .OR. ke>nz_global) then
-       errorcode = 10
-       call decomp_2d_abort(errorcode, &
-            'Invalid subdomain specified in I/O')
-    end if
-
-    ! create a communicator for all those MPI ranks containing the subdomain
-    color = 1
-    key = 1
-    if (ipencil==1) then
-       if (xstart(1)>ie .OR. xend(1)<is .OR. xstart(2)>je .OR. xend(2)<js &
-            .OR. xstart(3)>ke .OR. xend(3)<ks) then
-          color = 2
-       end if
-    else if (ipencil==2) then
-       if (ystart(1)>ie .OR. yend(1)<is .OR. ystart(2)>je .OR. yend(2)<js &
-            .OR. ystart(3)>ke .OR. yend(3)<ks) then
-          color = 2
-       end if
-    else if (ipencil==3) then
-       if (zstart(1)>ie .OR. zend(1)<is .OR. zstart(2)>je .OR. zend(2)<js &
-            .OR. zstart(3)>ke .OR. zend(3)<ks) then
-          color = 2
-       end if
-    end if
-    call MPI_COMM_SPLIT(decomp_2d_comm,color,key,newcomm,ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-
-    if (color==1) then ! only ranks in this group do IO collectively
-
-       ! generate MPI-IO subarray information
-
-       ! global size of the sub-domain to write
-       sizes(1) = ie - is + 1
-       sizes(2) = je - js + 1
-       sizes(3) = ke - ks + 1
-
-       ! 'subsizes' & 'starts' as required by MPI_TYPE_CREATE_SUBARRAY
-       ! note the special code whe subdomain only occupy part of the pencil
-       if (ipencil==1) then
-
-          subsizes(1) = xsize(1)
-          starts(1) = xstart(1) - is
-          if (xend(1)>ie .AND. xstart(1)<is) then
-             subsizes(1) = ie - is + 1
-             starts(1) = 0
-          else if (xstart(1)<is) then
-             subsizes(1) = xend(1) - is + 1
-             starts(1) = 0
-          else if (xend(1)>ie) then
-             subsizes(1) = ie - xstart(1) + 1
-          end if
-          subsizes(2) = xsize(2)
-          starts(2) = xstart(2) - js
-          if (xend(2)>je .AND. xstart(2)<js) then
-             subsizes(2) = je - js + 1
-             starts(2) = 0
-          else if (xstart(2)<js) then
-             subsizes(2) = xend(2) - js + 1
-             starts(2) = 0
-          else if (xend(2)>je) then
-             subsizes(2) = je - xstart(2) + 1
-          end if
-          subsizes(3) = xsize(3)
-          starts(3) = xstart(3) - ks
-          if (xend(3)>ke .AND. xstart(3)<ks) then
-             subsizes(3) = ke - ks + 1
-             starts(3) = 0
-          else if (xstart(3)<ks) then
-             subsizes(3) = xend(3) - ks + 1
-             starts(3) = 0
-          else if (xend(3)>ke) then
-             subsizes(3) = ke - xstart(3) + 1
-          end if
-
-       else if (ipencil==2) then
-
-          ! TODO 
-
-       else if (ipencil==3) then
-
-          ! TODO
-
-       end if
-
-
-       ! copy data from orginal to a temp array
-       ! pay attention to blocks only partially cover the sub-domain
-       if (ipencil==1) then
-
-          if (xend(1)>ie .AND. xstart(1)<is) then
-             i1 = is
-             i2 = ie
-          else if (xend(1)>ie) then
-             i1 = xstart(1)
-             i2 = ie
-          else if (xstart(1)<is) then
-             i1 = is
-             i2 = xend(1)
-          else
-             i1 = xstart(1)
-             i2 = xend(1)
-          end if
-
-          if (xend(2)>je .AND. xstart(2)<js) then
-             j1 = js
-             j2 = je
-          else if (xend(2)>je) then
-             j1 = xstart(2)
-             j2 = je
-          else if (xstart(2)<js) then
-             j1 = js
-             j2 = xend(2)
-          else
-             j1 = xstart(2)
-             j2 = xend(2)
-          end if
-
-          if (xend(3)>ke .AND. xstart(3)<ks) then
-             k1 = ks
-             k2 = ke
-          else if (xend(3)>ke) then
-             k1 = xstart(3)
-             k2 = ke
-          else if (xstart(3)<ks) then
-             k1 = ks
-             k2 = xend(3)
-          else
-             k1 = xstart(3)
-             k2 = xend(3)
-          end if
-
-          allocate(wk(i1:i2, j1:j2, k1:k2))
-          allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-          wk2 = var
-          do k=k1,k2
-             do j=j1,j2
-                do i=i1,i2
-                   wk(i,j,k) = wk2(i,j,k)
-                end do
-             end do
-          end do
-
-       else if (ipencil==2) then
-
-          ! TODO
-
-       else if (ipencil==3) then
-
-          ! TODO
-
-       end if
-
-       deallocate(wk2)
-
-       ! MPI-IO
-       call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-            MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-       call MPI_TYPE_COMMIT(newtype,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-       call MPI_FILE_OPEN(newcomm, filename, &
-            MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-            fh, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
-       filesize = 0_MPI_OFFSET_KIND
-       call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
-       disp = 0_MPI_OFFSET_KIND
-       call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-            newtype,'native',MPI_INFO_NULL,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-       call MPI_FILE_WRITE_ALL(fh, wk, &
-            subsizes(1)*subsizes(2)*subsizes(3), &
-            data_type, MPI_STATUS_IGNORE, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-       call MPI_FILE_CLOSE(fh,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-       call MPI_TYPE_FREE(newtype,ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
-
-       deallocate(wk)
-
-    end if
-
-    call decomp_mpi_comm_free(newcomm)
+      if (decomp_profiler_io) call decomp_profiler_start("io_write_subdomain")
+#endif
+
+      data_type = real_type
+
+      ! validate the input paramters
+      if (is < 1 .OR. ie > nx_global .OR. js < 1 .OR. je > ny_global .OR. &
+          ks < 1 .OR. ke > nz_global) then
+         errorcode = 10
+         call decomp_2d_abort(errorcode, &
+                              'Invalid subdomain specified in I/O')
+      end if
+
+      ! create a communicator for all those MPI ranks containing the subdomain
+      color = 1
+      key = 1
+      if (ipencil == 1) then
+         if (xstart(1) > ie .OR. xend(1) < is .OR. xstart(2) > je .OR. xend(2) < js &
+             .OR. xstart(3) > ke .OR. xend(3) < ks) then
+            color = 2
+         end if
+      else if (ipencil == 2) then
+         if (ystart(1) > ie .OR. yend(1) < is .OR. ystart(2) > je .OR. yend(2) < js &
+             .OR. ystart(3) > ke .OR. yend(3) < ks) then
+            color = 2
+         end if
+      else if (ipencil == 3) then
+         if (zstart(1) > ie .OR. zend(1) < is .OR. zstart(2) > je .OR. zend(2) < js &
+             .OR. zstart(3) > ke .OR. zend(3) < ks) then
+            color = 2
+         end if
+      end if
+      call MPI_COMM_SPLIT(decomp_2d_comm, color, key, newcomm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+
+      if (color == 1) then ! only ranks in this group do IO collectively
+
+         ! generate MPI-IO subarray information
+
+         ! global size of the sub-domain to write
+         sizes(1) = ie - is + 1
+         sizes(2) = je - js + 1
+         sizes(3) = ke - ks + 1
+
+         ! 'subsizes' & 'starts' as required by MPI_TYPE_CREATE_SUBARRAY
+         ! note the special code whe subdomain only occupy part of the pencil
+         if (ipencil == 1) then
+
+            subsizes(1) = xsize(1)
+            starts(1) = xstart(1) - is
+            if (xend(1) > ie .AND. xstart(1) < is) then
+               subsizes(1) = ie - is + 1
+               starts(1) = 0
+            else if (xstart(1) < is) then
+               subsizes(1) = xend(1) - is + 1
+               starts(1) = 0
+            else if (xend(1) > ie) then
+               subsizes(1) = ie - xstart(1) + 1
+            end if
+            subsizes(2) = xsize(2)
+            starts(2) = xstart(2) - js
+            if (xend(2) > je .AND. xstart(2) < js) then
+               subsizes(2) = je - js + 1
+               starts(2) = 0
+            else if (xstart(2) < js) then
+               subsizes(2) = xend(2) - js + 1
+               starts(2) = 0
+            else if (xend(2) > je) then
+               subsizes(2) = je - xstart(2) + 1
+            end if
+            subsizes(3) = xsize(3)
+            starts(3) = xstart(3) - ks
+            if (xend(3) > ke .AND. xstart(3) < ks) then
+               subsizes(3) = ke - ks + 1
+               starts(3) = 0
+            else if (xstart(3) < ks) then
+               subsizes(3) = xend(3) - ks + 1
+               starts(3) = 0
+            else if (xend(3) > ke) then
+               subsizes(3) = ke - xstart(3) + 1
+            end if
+
+         else if (ipencil == 2) then
+
+            ! TODO
+
+         else if (ipencil == 3) then
+
+            ! TODO
+
+         end if
+
+         ! copy data from orginal to a temp array
+         ! pay attention to blocks only partially cover the sub-domain
+         if (ipencil == 1) then
+
+            if (xend(1) > ie .AND. xstart(1) < is) then
+               i1 = is
+               i2 = ie
+            else if (xend(1) > ie) then
+               i1 = xstart(1)
+               i2 = ie
+            else if (xstart(1) < is) then
+               i1 = is
+               i2 = xend(1)
+            else
+               i1 = xstart(1)
+               i2 = xend(1)
+            end if
+
+            if (xend(2) > je .AND. xstart(2) < js) then
+               j1 = js
+               j2 = je
+            else if (xend(2) > je) then
+               j1 = xstart(2)
+               j2 = je
+            else if (xstart(2) < js) then
+               j1 = js
+               j2 = xend(2)
+            else
+               j1 = xstart(2)
+               j2 = xend(2)
+            end if
+
+            if (xend(3) > ke .AND. xstart(3) < ks) then
+               k1 = ks
+               k2 = ke
+            else if (xend(3) > ke) then
+               k1 = xstart(3)
+               k2 = ke
+            else if (xstart(3) < ks) then
+               k1 = ks
+               k2 = xend(3)
+            else
+               k1 = xstart(3)
+               k2 = xend(3)
+            end if
+
+            allocate (wk(i1:i2, j1:j2, k1:k2))
+            allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+            wk2 = var
+            do k = k1, k2
+               do j = j1, j2
+                  do i = i1, i2
+                     wk(i, j, k) = wk2(i, j, k)
+                  end do
+               end do
+            end do
+
+         else if (ipencil == 2) then
+
+            ! TODO
+
+         else if (ipencil == 3) then
+
+            ! TODO
+
+         end if
+
+         deallocate (wk2)
+
+         ! MPI-IO
+         call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+         call MPI_TYPE_COMMIT(newtype, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+         call MPI_FILE_OPEN(newcomm, filename, &
+                            MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                            fh, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+         filesize = 0_MPI_OFFSET_KIND
+         call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+         disp = 0_MPI_OFFSET_KIND
+         call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                                newtype, 'native', MPI_INFO_NULL, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+         call MPI_FILE_WRITE_ALL(fh, wk, &
+                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 data_type, MPI_STATUS_IGNORE, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+         call MPI_FILE_CLOSE(fh, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+         call MPI_TYPE_FREE(newtype, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+
+         deallocate (wk)
+
+      end if
+
+      call decomp_mpi_comm_free(newcomm)
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_write_subdomain")
+      if (decomp_profiler_io) call decomp_profiler_end("io_write_subdomain")
 #endif
 
-  end subroutine write_subdomain
+   end subroutine write_subdomain
 
-  subroutine decomp_2d_init_io(io_name)
+   subroutine decomp_2d_init_io(io_name)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: io_name
+      character(len=*), intent(in) :: io_name
 #ifdef ADIOS2
-    integer :: ierror
-    type(adios2_io) :: io
+      integer :: ierror
+      type(adios2_io) :: io
 #endif
-    
-    if (nrank .eq. 0) then
-       print *, "Initialising IO for ", io_name
-    end if
+
+      if (nrank == 0) then
+         print *, "Initialising IO for ", io_name
+      end if
 
 #ifdef ADIOS2
-    if (adios%valid) then
-       call adios2_declare_io(io, adios, io_name, ierror)
-       if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_declare_io "//trim(io_name))
-    else
-       call decomp_2d_abort(__FILE__, __LINE__, -1, "couldn't declare IO - adios object not valid")
-    end if
-#endif
-    
-  end subroutine decomp_2d_init_io
-  
-  subroutine decomp_2d_open_io(io_name, io_dir, mode)
-
-    implicit none
-
-    character(len=*), intent(in) :: io_name, io_dir
-    integer, intent(in) :: mode
-
-    logical, dimension(:), pointer :: live_ptrh
-    character(len=1024), dimension(:), pointer :: names_ptr
-    character(len=(len(io_name)+len(io_sep)+len(io_dir))) :: full_name
-    
-    integer :: idx, ierror
-    integer :: access_mode
+      if (adios%valid) then
+         call adios2_declare_io(io, adios, io_name, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_declare_io "//trim(io_name))
+      else
+         call decomp_2d_abort(__FILE__, __LINE__, -1, "couldn't declare IO - adios object not valid")
+      end if
+#endif
+
+   end subroutine decomp_2d_init_io
+
+   subroutine decomp_2d_open_io(io_name, io_dir, mode)
+
+      implicit none
+
+      character(len=*), intent(in) :: io_name, io_dir
+      integer, intent(in) :: mode
+
+      logical, dimension(:), pointer :: live_ptrh
+      character(len=1024), dimension(:), pointer :: names_ptr
+      character(len=(len(io_name) + len(io_sep) + len(io_dir))) :: full_name
+
+      integer :: idx, ierror
+      integer :: access_mode
 #ifndef ADIOS2
-    integer(MPI_OFFSET_KIND) :: filesize
+      integer(MPI_OFFSET_KIND) :: filesize
 #else
-    type(adios2_io) :: io
+      type(adios2_io) :: io
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_start("io_open_close")
+      if (decomp_profiler_io) call decomp_profiler_start("io_open_close")
 #endif
 
 #ifndef ADIOS2
-    live_ptrh => fh_live
-    names_ptr => fh_names
+      live_ptrh => fh_live
+      names_ptr => fh_names
 #else
-    live_ptrh => engine_live
-    names_ptr => engine_names
+      live_ptrh => engine_live
+      names_ptr => engine_names
 #endif
-    
-    idx = get_io_idx(io_name, io_dir)
-    if (idx .lt. 1) then
+
+      idx = get_io_idx(io_name, io_dir)
+      if (idx < 1) then
        !! New io destination
-       if (nreg_io .lt. MAX_IOH) then
-          nreg_io = nreg_io + 1
-          do idx = 1, MAX_IOH
-             if (.not. live_ptrh(idx)) then
-                live_ptrh(idx) = .true.
-                exit
-             end if
-          end do
-          
-          full_name = io_name//io_sep//io_dir
-          names_ptr(idx) = full_name
-
-          if (mode .eq. decomp_2d_write_mode) then
+         if (nreg_io < MAX_IOH) then
+            nreg_io = nreg_io + 1
+            do idx = 1, MAX_IOH
+               if (.not. live_ptrh(idx)) then
+                  live_ptrh(idx) = .true.
+                  exit
+               end if
+            end do
+
+            full_name = io_name//io_sep//io_dir
+            names_ptr(idx) = full_name
+
+            if (mode == decomp_2d_write_mode) then
              !! Setup writers
 #ifndef ADIOS2
-             filesize = 0_MPI_OFFSET_KIND
-             fh_disp(idx) = 0_MPI_OFFSET_KIND
-             access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
+               filesize = 0_MPI_OFFSET_KIND
+               fh_disp(idx) = 0_MPI_OFFSET_KIND
+               access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
 #else
-             access_mode = adios2_mode_write
+               access_mode = adios2_mode_write
 #endif
-          else if (mode .eq. decomp_2d_read_mode) then
+            else if (mode == decomp_2d_read_mode) then
              !! Setup readers
 #ifndef ADIOS2
-             fh_disp(idx) = 0_MPI_OFFSET_KIND
-             access_mode = MPI_MODE_RDONLY
+               fh_disp(idx) = 0_MPI_OFFSET_KIND
+               access_mode = MPI_MODE_RDONLY
 #else
-             access_mode = adios2_mode_read
+               access_mode = adios2_mode_read
 #endif
-          else if (mode .eq. decomp_2d_append_mode) then
+            else if (mode == decomp_2d_append_mode) then
 #ifndef ADIOS2
-             filesize = 0_MPI_OFFSET_KIND
-             fh_disp(idx) = 0_MPI_OFFSET_KIND
-             access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
+               filesize = 0_MPI_OFFSET_KIND
+               fh_disp(idx) = 0_MPI_OFFSET_KIND
+               access_mode = MPI_MODE_CREATE + MPI_MODE_WRONLY
 #else
-             access_mode = adios2_mode_append
+               access_mode = adios2_mode_append
 #endif
-          else
-             print *, "ERROR: Unknown mode!"
-             stop
-          endif
+            else
+               print *, "ERROR: Unknown mode!"
+               stop
+            end if
 
           !! Open IO
 #ifndef ADIOS2
-          call MPI_FILE_OPEN(decomp_2d_comm, io_dir, &
-               access_mode, MPI_INFO_NULL, &
-               fh_registry(idx), ierror)
-          if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
-          if (mode .eq. decomp_2d_write_mode) then
+            call MPI_FILE_OPEN(decomp_2d_comm, io_dir, &
+                               access_mode, MPI_INFO_NULL, &
+                               fh_registry(idx), ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+            if (mode == decomp_2d_write_mode) then
              !! Guarantee overwriting
-             call MPI_FILE_SET_SIZE(fh_registry(idx), filesize, ierror)
-             if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
-          end if
+               call MPI_FILE_SET_SIZE(fh_registry(idx), filesize, ierror)
+               if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+            end if
 #else
-          call adios2_at_io(io, adios, io_name, ierror)
-          if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-          if (io%valid) then
-             call adios2_open(engine_registry(idx), io, trim(gen_iodir_name(io_dir, io_name)), access_mode, ierror)
-             if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "ERROR opening engine!")
-          else
-             call decomp_2d_abort(__FILE__, __LINE__, -1, "Couldn't find IO handle")
-          end if
-#endif
-       end if
-    end if
+            call adios2_at_io(io, adios, io_name, ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+            if (io%valid) then
+               call adios2_open(engine_registry(idx), io, trim(gen_iodir_name(io_dir, io_name)), access_mode, ierror)
+               if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "ERROR opening engine!")
+            else
+               call decomp_2d_abort(__FILE__, __LINE__, -1, "Couldn't find IO handle")
+            end if
+#endif
+         end if
+      end if
+
+   end subroutine decomp_2d_open_io
 
-  end subroutine decomp_2d_open_io
+   subroutine decomp_2d_close_io(io_name, io_dir)
 
-  subroutine decomp_2d_close_io(io_name, io_dir)
+      implicit none
 
-    implicit none
+      character(len=*), intent(in) :: io_name, io_dir
 
-    character(len=*), intent(in) :: io_name, io_dir
-    
-    character(len=1024), dimension(:), pointer :: names_ptr
-    logical, dimension(:), pointer :: live_ptrh
-    integer :: idx, ierror
+      character(len=1024), dimension(:), pointer :: names_ptr
+      logical, dimension(:), pointer :: live_ptrh
+      integer :: idx, ierror
 
-    idx = get_io_idx(io_name, io_dir)
+      idx = get_io_idx(io_name, io_dir)
 #ifndef ADIOS2
-    names_ptr => fh_names
-    live_ptrh => fh_live
-    call MPI_FILE_CLOSE(fh_registry(idx), ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+      names_ptr => fh_names
+      live_ptrh => fh_live
+      call MPI_FILE_CLOSE(fh_registry(idx), ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
 #else
-    names_ptr => engine_names
-    live_ptrh => engine_live
-    call adios2_close(engine_registry(idx), ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_close")
+      names_ptr => engine_names
+      live_ptrh => engine_live
+      call adios2_close(engine_registry(idx), ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_close")
 #endif
-    names_ptr(idx) = ""
-    live_ptrh(idx) = .false.
-    nreg_io = nreg_io - 1
+      names_ptr(idx) = ""
+      live_ptrh(idx) = .false.
+      nreg_io = nreg_io - 1
 
 #ifdef PROFILER
-    if (decomp_profiler_io) call decomp_profiler_end("io_open_close")
+      if (decomp_profiler_io) call decomp_profiler_end("io_open_close")
 #endif
 
-  end subroutine decomp_2d_close_io
+   end subroutine decomp_2d_close_io
 
-  subroutine decomp_2d_start_io(io_name, io_dir)
+   subroutine decomp_2d_start_io(io_name, io_dir)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: io_name, io_dir
+      character(len=*), intent(in) :: io_name, io_dir
 #ifdef ADIOS2
-    integer :: idx, ierror
-
-    idx = get_io_idx(io_name, io_dir)
-    associate(engine => engine_registry(idx))
-      if (engine%valid) then
-         call adios2_begin_step(engine, ierror)
-         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_begin_step")
-      else
-         call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to begin step with invalid engine")
-      end if
-    end associate
+      integer :: idx, ierror
+
+      idx = get_io_idx(io_name, io_dir)
+      associate (engine => engine_registry(idx))
+         if (engine%valid) then
+            call adios2_begin_step(engine, ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_begin_step")
+         else
+            call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to begin step with invalid engine")
+         end if
+      end associate
 #else
-    associate(nm => io_name, dr => io_dir) ! Silence unused dummy argument
-    end associate
+      associate (nm => io_name, dr => io_dir) ! Silence unused dummy argument
+      end associate
 #endif
-    
-  end subroutine decomp_2d_start_io
 
-  subroutine decomp_2d_end_io(io_name, io_dir)
+   end subroutine decomp_2d_start_io
 
-    implicit none
+   subroutine decomp_2d_end_io(io_name, io_dir)
 
-    character(len=*), intent(in) :: io_name, io_dir
-#ifdef ADIOS2
-    integer :: idx, ierror
+      implicit none
 
-    idx = get_io_idx(io_name, io_dir)
-    associate(engine => engine_registry(idx))
-      if (engine%valid) then
-         call adios2_end_step(engine, ierror)
-         if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_end_step")
-      else
-         call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to end step with invalid engine")
-      end if
-    end associate
+      character(len=*), intent(in) :: io_name, io_dir
+#ifdef ADIOS2
+      integer :: idx, ierror
+
+      idx = get_io_idx(io_name, io_dir)
+      associate (engine => engine_registry(idx))
+         if (engine%valid) then
+            call adios2_end_step(engine, ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_end_step")
+         else
+            call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to end step with invalid engine")
+         end if
+      end associate
 #else
-    associate(nm => io_name, dr => io_dir) ! Silence unused dummy argument
-    end associate
+      associate (nm => io_name, dr => io_dir) ! Silence unused dummy argument
+      end associate
 #endif
 
-  end subroutine decomp_2d_end_io
-  
-  integer function get_io_idx(io_name, engine_name)
+   end subroutine decomp_2d_end_io
+
+   integer function get_io_idx(io_name, engine_name)
 
-    implicit none
+      implicit none
 
-    character(len=*), intent(in) :: io_name
-    character(len=*), intent(in) :: engine_name
+      character(len=*), intent(in) :: io_name
+      character(len=*), intent(in) :: engine_name
 
-    character(len=(len(io_name)+len(io_sep)+len(engine_name))) :: full_name
-    integer :: idx
-    logical :: found
+      character(len=(len(io_name) + len(io_sep) + len(engine_name))) :: full_name
+      integer :: idx
+      logical :: found
 
-    character(len=1024), dimension(:), pointer :: names_ptr
+      character(len=1024), dimension(:), pointer :: names_ptr
 
 #ifndef ADIOS2
-    names_ptr => fh_names
+      names_ptr => fh_names
 #else
-    names_ptr => engine_names
+      names_ptr => engine_names
 #endif
 
-    full_name = io_name//io_sep//engine_name
-    
-    found = .false.
-    do idx = 1, MAX_IOH
-       if (names_ptr(idx) .eq. full_name) then
-          found = .true.
-          exit
-       end if
-    end do
+      full_name = io_name//io_sep//engine_name
+
+      found = .false.
+      do idx = 1, MAX_IOH
+         if (names_ptr(idx) == full_name) then
+            found = .true.
+            exit
+         end if
+      end do
+
+      if (.not. found) then
+         idx = -1
+      end if
 
-    if (.not. found) then
-       idx = -1
-    end if
+      get_io_idx = idx
 
-    get_io_idx = idx
-    
-  end function get_io_idx
+   end function get_io_idx
 
-  function gen_iodir_name(io_dir, io_name)
+   function gen_iodir_name(io_dir, io_name)
 
-    character(len=*), intent(in) :: io_dir, io_name
-    character(len=(len(io_dir) + 5)) :: gen_iodir_name
+      character(len=*), intent(in) :: io_dir, io_name
+      character(len=(len(io_dir) + 5)) :: gen_iodir_name
 #ifdef ADIOS2
-    integer :: ierror
-    type(adios2_io) :: io
-    character(len=5) :: ext
+      integer :: ierror
+      type(adios2_io) :: io
+      character(len=5) :: ext
 #endif
 
 #ifndef ADIOS2
-    associate(nm => io_name) ! Silence unused dummy argument
-    end associate
-    write(gen_iodir_name, "(A)") io_dir
+      associate (nm => io_name) ! Silence unused dummy argument
+      end associate
+      write (gen_iodir_name, "(A)") io_dir
 #else
-    call adios2_at_io(io, adios, io_name, ierror)
-    if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
-    if (io%engine_type .eq. "BP4") then
-       ext = ".bp4"
-    else if (io%engine_type .eq. "HDF5") then
-       ext = ".hdf5"
-    else if (io%engine_type .eq. "SST") then
-       ext = ""
-    else
-       print *, "ERROR: Unkown engine type! ", io%engine_type
-       print *, "-  IO: ", io_name
-       print *, "- DIR:", io_dir
-       stop
-    endif
-    write(gen_iodir_name, "(A,A)") io_dir, trim(ext)
-#endif
-    
-  end function gen_iodir_name
+      call adios2_at_io(io, adios, io_name, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+      if (io%engine_type == "BP4") then
+         ext = ".bp4"
+      else if (io%engine_type == "HDF5") then
+         ext = ".hdf5"
+      else if (io%engine_type == "SST") then
+         ext = ""
+      else
+         print *, "ERROR: Unkown engine type! ", io%engine_type
+         print *, "-  IO: ", io_name
+         print *, "- DIR:", io_dir
+         stop
+      end if
+      write (gen_iodir_name, "(A,A)") io_dir, trim(ext)
+#endif
+
+   end function gen_iodir_name
 
 end module decomp_2d_io
diff --git a/src/io_read_inflow.f90 b/src/io_read_inflow.f90
index 3e5a9fb1..9484ddcb 100644
--- a/src/io_read_inflow.f90
+++ b/src/io_read_inflow.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,12 +9,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'read_var_...' in io.f90
 
-! Using MPI-IO to read a distributed 3D variable from a file. File 
-! operations (open/close) need to be done in calling application. This 
-! allows multiple variables to be read from a single file. Together 
+! Using MPI-IO to read a distributed 3D variable from a file. File
+! operations (open/close) need to be done in calling application. This
+! allows multiple variables to be read from a single file. Together
 ! with the corresponding write operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -25,62 +25,62 @@
 end if
 
 ! Create file type and set file view
-sizes(1) = ntimesteps 
+sizes(1) = ntimesteps
 sizes(2) = decomp%ysz(2)
 sizes(3) = decomp%zsz(3)
 subsizes(1) = ntimesteps
 subsizes(2) = decomp%xsz(2)
 subsizes(3) = decomp%xsz(3)
 starts(1) = 0  ! 0-based index
-starts(2) = decomp%xst(2)-1
-starts(3) = decomp%xst(3)-1
+starts(2) = decomp%xst(2) - 1
+starts(3) = decomp%xst(3) - 1
 
 idx = get_io_idx(io_name, dirname)
 
 #ifndef ADIOS2
 !! Use default MPIIO
-associate(fh=>fh_registry(idx), &
-  disp => fh_disp(idx))
-  call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-  call MPI_TYPE_COMMIT(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-  call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-       newtype,'native',MPI_INFO_NULL,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-  call MPI_FILE_READ_ALL(fh, var, &
-       subsizes(1)*subsizes(2)*subsizes(3), &
-       data_type, MPI_STATUS_IGNORE, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-  call MPI_TYPE_FREE(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+associate (fh => fh_registry(idx), &
+           disp => fh_disp(idx))
+   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+   call MPI_TYPE_COMMIT(newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                          newtype, 'native', MPI_INFO_NULL, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+   call MPI_FILE_READ_ALL(fh, var, &
+                          subsizes(1)*subsizes(2)*subsizes(3), &
+                          data_type, MPI_STATUS_IGNORE, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+   call MPI_TYPE_FREE(newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-  ! update displacement for the next read operation
-  disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-              * int(sizes(2), kind=MPI_OFFSET_KIND) &
-              * int(sizes(3), kind=MPI_OFFSET_KIND) &
-              * int(mytype_bytes, kind=MPI_OFFSET_KIND)
-  if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                             * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                             * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+   ! update displacement for the next read operation
+   disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+          *int(sizes(2), kind=MPI_OFFSET_KIND) &
+          *int(sizes(3), kind=MPI_OFFSET_KIND) &
+          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+   if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
-associate(vnm => varname) ! Silence unused argument
+associate (vnm => varname) ! Silence unused argument
 end associate
 
 #else
 !! Use ADIOS2
 call adios2_at_io(io_handle, adios, io_name, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-   "trying to write variable before registering! "//trim(varname))
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                                                 "trying to write variable before registering! "//trim(varname))
 
 !! Note - need to use sync mode as we are using a view into the array - unsure how this works with deferred writes
 ! call adios2_set_step_selection(var_handle, int(0, kind=8), int(1, kind=8), ierror)
 call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_sync, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
 #endif
diff --git a/src/io_read_one.inc b/src/io_read_one.inc
index 211cbc2c..1167e872 100644
--- a/src/io_read_one.inc
+++ b/src/io_read_one.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,7 +10,7 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'mpiio_read_one_...' in io.f90
 
 ! Using MPI-IO to write a distributed 3D array into a file
@@ -30,43 +30,43 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1)-1  ! 0-based index
-   starts(2) = decomp%xst(2)-1
-   starts(3) = decomp%xst(3)-1
+   starts(1) = decomp%xst(1) - 1  ! 0-based index
+   starts(2) = decomp%xst(2) - 1
+   starts(3) = decomp%xst(3) - 1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1)-1
-   starts(2) = decomp%yst(2)-1
-   starts(3) = decomp%yst(3)-1
+   starts(1) = decomp%yst(1) - 1
+   starts(2) = decomp%yst(2) - 1
+   starts(3) = decomp%yst(3) - 1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1)-1
-   starts(2) = decomp%zst(2)-1
-   starts(3) = decomp%zst(3)-1
-endif
+   starts(1) = decomp%zst(1) - 1
+   starts(2) = decomp%zst(2) - 1
+   starts(3) = decomp%zst(3) - 1
+end if
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
 call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-     MPI_MODE_RDONLY, MPI_INFO_NULL, &
-     fh, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+                   MPI_MODE_RDONLY, MPI_INFO_NULL, &
+                   fh, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
 disp = 0_MPI_OFFSET_KIND
-call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-     newtype,'native',MPI_INFO_NULL,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                       newtype, 'native', MPI_INFO_NULL, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_READ_ALL(fh, var, &
-     subsizes(1)*subsizes(2)*subsizes(3), &
-     data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-call MPI_FILE_CLOSE(fh,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-call MPI_TYPE_FREE(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                       subsizes(1)*subsizes(2)*subsizes(3), &
+                       data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+call MPI_FILE_CLOSE(fh, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+call MPI_TYPE_FREE(newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
diff --git a/src/io_read_var.inc b/src/io_read_var.inc
index 489247cb..b3fa33f9 100644
--- a/src/io_read_var.inc
+++ b/src/io_read_var.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,12 +10,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'read_var_...' in io.f90
 
-! Using MPI-IO to read a distributed 3D variable from a file. File 
-! operations (open/close) need to be done in calling application. This 
-! allows multiple variables to be read from a single file. Together 
+! Using MPI-IO to read a distributed 3D variable from a file. File
+! operations (open/close) need to be done in calling application. This
+! allows multiple variables to be read from a single file. Together
 ! with the corresponding write operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -33,46 +33,46 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1)-1  ! 0-based index
-   starts(2) = decomp%xst(2)-1
-   starts(3) = decomp%xst(3)-1
+   starts(1) = decomp%xst(1) - 1  ! 0-based index
+   starts(2) = decomp%xst(2) - 1
+   starts(3) = decomp%xst(3) - 1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1)-1
-   starts(2) = decomp%yst(2)-1
-   starts(3) = decomp%yst(3)-1
+   starts(1) = decomp%yst(1) - 1
+   starts(2) = decomp%yst(2) - 1
+   starts(3) = decomp%yst(3) - 1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1)-1
-   starts(2) = decomp%zst(2)-1
-   starts(3) = decomp%zst(3)-1
-endif
+   starts(1) = decomp%zst(1) - 1
+   starts(2) = decomp%zst(2) - 1
+   starts(3) = decomp%zst(3) - 1
+end if
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-     newtype,'native',MPI_INFO_NULL,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                       newtype, 'native', MPI_INFO_NULL, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_READ_ALL(fh, var, &
-     subsizes(1)*subsizes(2)*subsizes(3), &
-     data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
-call MPI_TYPE_FREE(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                       subsizes(1)*subsizes(2)*subsizes(3), &
+                       data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
+call MPI_TYPE_FREE(newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 ! update displacement for the next read operation
 disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-            * int(sizes(2), kind=MPI_OFFSET_KIND) &
-            * int(sizes(3), kind=MPI_OFFSET_KIND) &
-            * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+       *int(sizes(2), kind=MPI_OFFSET_KIND) &
+       *int(sizes(3), kind=MPI_OFFSET_KIND) &
+       *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                           * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                           * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                           * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                      *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                      *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                      *int(mytype_bytes, kind=MPI_OFFSET_KIND)
diff --git a/src/io_write_every.inc b/src/io_write_every.inc
index cbf98b7a..3289264f 100644
--- a/src/io_write_every.inc
+++ b/src/io_write_every.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,12 +10,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'write_every_...' in io.f90
 
 ! To write every few points of a 3D array to a file
 
-! work out the distribution parameters, which may be different from 
+! work out the distribution parameters, which may be different from
 ! the default distribution used by the decomposition library
 !  For exmample if nx=17 and p_row=4
 !    distribution is: 4 4 4 5
@@ -38,199 +38,199 @@
 !             4th block (13-17) contains then 12th & 15th point
 !    giving a 1 2 2 1 distribution
 
-skip(1)=iskip
-skip(2)=jskip
-skip(3)=kskip
+skip(1) = iskip
+skip(2) = jskip
+skip(3) = kskip
 
-do i=1,3
+do i = 1, 3
    if (from1) then
-      xst(i) = (xstart(i)+skip(i)-1)/skip(i)
-      if (mod(xstart(i)+skip(i)-1,skip(i))/=0) xst(i)=xst(i)+1
-      xen(i) = (xend(i)+skip(i)-1)/skip(i)
+      xst(i) = (xstart(i) + skip(i) - 1)/skip(i)
+      if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xst(i) = xst(i) + 1
+      xen(i) = (xend(i) + skip(i) - 1)/skip(i)
    else
       xst(i) = xstart(i)/skip(i)
-      if (mod(xstart(i),skip(i))/=0) xst(i)=xst(i)+1
+      if (mod(xstart(i), skip(i)) /= 0) xst(i) = xst(i) + 1
       xen(i) = xend(i)/skip(i)
    end if
-   xsz(i) = xen(i)-xst(i)+1
+   xsz(i) = xen(i) - xst(i) + 1
 end do
 
-do i=1,3
+do i = 1, 3
    if (from1) then
-      yst(i) = (ystart(i)+skip(i)-1)/skip(i)
-      if (mod(ystart(i)+skip(i)-1,skip(i))/=0) yst(i)=yst(i)+1
-      yen(i) = (yend(i)+skip(i)-1)/skip(i)
+      yst(i) = (ystart(i) + skip(i) - 1)/skip(i)
+      if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) yst(i) = yst(i) + 1
+      yen(i) = (yend(i) + skip(i) - 1)/skip(i)
    else
       yst(i) = ystart(i)/skip(i)
-      if (mod(ystart(i),skip(i))/=0) yst(i)=yst(i)+1
+      if (mod(ystart(i), skip(i)) /= 0) yst(i) = yst(i) + 1
       yen(i) = yend(i)/skip(i)
    end if
-   ysz(i) = yen(i)-yst(i)+1
+   ysz(i) = yen(i) - yst(i) + 1
 end do
 
-do i=1,3
+do i = 1, 3
    if (from1) then
-      zst(i) = (zstart(i)+skip(i)-1)/skip(i)
-      if (mod(zstart(i)+skip(i)-1,skip(i))/=0) zst(i)=zst(i)+1
-      zen(i) = (zend(i)+skip(i)-1)/skip(i)
+      zst(i) = (zstart(i) + skip(i) - 1)/skip(i)
+      if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zst(i) = zst(i) + 1
+      zen(i) = (zend(i) + skip(i) - 1)/skip(i)
    else
       zst(i) = zstart(i)/skip(i)
-      if (mod(zstart(i),skip(i))/=0) zst(i)=zst(i)+1
+      if (mod(zstart(i), skip(i)) /= 0) zst(i) = zst(i) + 1
       zen(i) = zend(i)/skip(i)
    end if
-   zsz(i) = zen(i)-zst(i)+1
+   zsz(i) = zen(i) - zst(i) + 1
 end do
 
-! if 'skip' value is large it is possible that some ranks do not 
-! contain any points to be written. Subarray constructor requires 
+! if 'skip' value is large it is possible that some ranks do not
+! contain any points to be written. Subarray constructor requires
 ! nonzero size so it is not possible to use decomp_2d_comm for IO.
 ! Create a sub communicator for this...
 color = 1
 key = 0  ! rank order doesn't matter
-if (ipencil==1) then
-   if (xsz(1)==0 .or. xsz(2)==0 .or. xsz(3)==0) then
+if (ipencil == 1) then
+   if (xsz(1) == 0 .or. xsz(2) == 0 .or. xsz(3) == 0) then
       color = 2
    end if
-else if (ipencil==2) then
-   if (ysz(1)==0 .or. ysz(2)==0 .or. ysz(3)==0) then
+else if (ipencil == 2) then
+   if (ysz(1) == 0 .or. ysz(2) == 0 .or. ysz(3) == 0) then
       color = 2
    end if
-else if (ipencil==3) then
-   if (zsz(1)==0 .or. zsz(2)==0 .or. zsz(3)==0) then
+else if (ipencil == 3) then
+   if (zsz(1) == 0 .or. zsz(2) == 0 .or. zsz(3) == 0) then
       color = 2
    end if
 end if
-call MPI_COMM_SPLIT(decomp_2d_comm,color,key,newcomm,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
+call MPI_COMM_SPLIT(decomp_2d_comm, color, key, newcomm, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
 
-if (color==1) then ! only ranks in this group do IO collectively
+if (color == 1) then ! only ranks in this group do IO collectively
 
    ! generate subarray information
    sizes(1) = xsz(1)
    sizes(2) = ysz(2)
    sizes(3) = zsz(3)
-   if (ipencil==1) then
+   if (ipencil == 1) then
       subsizes(1) = xsz(1)
       subsizes(2) = xsz(2)
       subsizes(3) = xsz(3)
-      starts(1) = xst(1)-1
-      starts(2) = xst(2)-1
-      starts(3) = xst(3)-1
-   else if (ipencil==2) then
+      starts(1) = xst(1) - 1
+      starts(2) = xst(2) - 1
+      starts(3) = xst(3) - 1
+   else if (ipencil == 2) then
       subsizes(1) = ysz(1)
       subsizes(2) = ysz(2)
       subsizes(3) = ysz(3)
-      starts(1) = yst(1)-1
-      starts(2) = yst(2)-1
-      starts(3) = yst(3)-1
-   else if (ipencil==3) then
+      starts(1) = yst(1) - 1
+      starts(2) = yst(2) - 1
+      starts(3) = yst(3) - 1
+   else if (ipencil == 3) then
       subsizes(1) = zsz(1)
       subsizes(2) = zsz(2)
       subsizes(3) = zsz(3)
-      starts(1) = zst(1)-1
-      starts(2) = zst(2)-1
-      starts(3) = zst(3)-1
+      starts(1) = zst(1) - 1
+      starts(2) = zst(2) - 1
+      starts(3) = zst(3) - 1
    end if
 
    ! copy data from original array
-   ! needs a copy of original array in global coordinate 
-   if (ipencil==1) then
-      allocate(wk(xst(1):xen(1),xst(2):xen(2),xst(3):xen(3)))
-      allocate(wk2(xstart(1):xend(1),xstart(2):xend(2),xstart(3):xend(3)))
-      wk2=var
+   ! needs a copy of original array in global coordinate
+   if (ipencil == 1) then
+      allocate (wk(xst(1):xen(1), xst(2):xen(2), xst(3):xen(3)))
+      allocate (wk2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      wk2 = var
       if (from1) then
-         do k=xst(3),xen(3)
-            do j=xst(2),xen(2)
-               do i=xst(1),xen(1)
-                  wk(i,j,k) = wk2((i-1)*iskip+1,(j-1)*jskip+1,(k-1)*kskip+1)
+         do k = xst(3), xen(3)
+            do j = xst(2), xen(2)
+               do i = xst(1), xen(1)
+                  wk(i, j, k) = wk2((i - 1)*iskip + 1, (j - 1)*jskip + 1, (k - 1)*kskip + 1)
                end do
             end do
          end do
       else
-         do k=xst(3),xen(3)
-            do j=xst(2),xen(2)
-               do i=xst(1),xen(1)
-                  wk(i,j,k) = wk2(i*iskip,j*jskip,k*kskip)
+         do k = xst(3), xen(3)
+            do j = xst(2), xen(2)
+               do i = xst(1), xen(1)
+                  wk(i, j, k) = wk2(i*iskip, j*jskip, k*kskip)
                end do
             end do
          end do
       end if
-   else if (ipencil==2) then
-      allocate(wk(yst(1):yen(1),yst(2):yen(2),yst(3):yen(3)))
-      allocate(wk2(ystart(1):yend(1),ystart(2):yend(2),ystart(3):yend(3)))
-      wk2=var
+   else if (ipencil == 2) then
+      allocate (wk(yst(1):yen(1), yst(2):yen(2), yst(3):yen(3)))
+      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
+      wk2 = var
       if (from1) then
-         do k=yst(3),yen(3)
-            do j=yst(2),yen(2)
-               do i=yst(1),yen(1)
-                  wk(i,j,k) = wk2((i-1)*iskip+1,(j-1)*jskip+1,(k-1)*kskip+1)
+         do k = yst(3), yen(3)
+            do j = yst(2), yen(2)
+               do i = yst(1), yen(1)
+                  wk(i, j, k) = wk2((i - 1)*iskip + 1, (j - 1)*jskip + 1, (k - 1)*kskip + 1)
                end do
             end do
          end do
       else
-         do k=yst(3),yen(3)
-            do j=yst(2),yen(2)
-               do i=yst(1),yen(1)
-                  wk(i,j,k) = wk2(i*iskip,j*jskip,k*kskip)
+         do k = yst(3), yen(3)
+            do j = yst(2), yen(2)
+               do i = yst(1), yen(1)
+                  wk(i, j, k) = wk2(i*iskip, j*jskip, k*kskip)
                end do
             end do
          end do
       end if
-   else if (ipencil==3) then
-      allocate(wk(zst(1):zen(1),zst(2):zen(2),zst(3):zen(3)))
-      allocate(wk2(zstart(1):zend(1),zstart(2):zend(2),zstart(3):zend(3)))
-      wk2=var
+   else if (ipencil == 3) then
+      allocate (wk(zst(1):zen(1), zst(2):zen(2), zst(3):zen(3)))
+      allocate (wk2(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+      wk2 = var
       if (from1) then
-         do k=zst(3),zen(3)
-            do j=zst(2),zen(2)
-               do i=zst(1),zen(1)
-                  wk(i,j,k) = wk2((i-1)*iskip+1,(j-1)*jskip+1,(k-1)*kskip+1)
+         do k = zst(3), zen(3)
+            do j = zst(2), zen(2)
+               do i = zst(1), zen(1)
+                  wk(i, j, k) = wk2((i - 1)*iskip + 1, (j - 1)*jskip + 1, (k - 1)*kskip + 1)
                end do
             end do
          end do
       else
-         do k=zst(3),zen(3)
-            do j=zst(2),zen(2)
-               do i=zst(1),zen(1)
-                  wk(i,j,k) = wk2(i*iskip,j*jskip,k*kskip)
+         do k = zst(3), zen(3)
+            do j = zst(2), zen(2)
+               do i = zst(1), zen(1)
+                  wk(i, j, k) = wk2(i*iskip, j*jskip, k*kskip)
                end do
             end do
          end do
       end if
    end if
-   deallocate(wk2)
+   deallocate (wk2)
 
    ! MPI-IO
-   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-        MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-   call MPI_TYPE_COMMIT(newtype,ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+   call MPI_TYPE_COMMIT(newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
    call MPI_FILE_OPEN(newcomm, filename, &
-        MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-        fh, ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+                      MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                      fh, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
    filesize = 0_MPI_OFFSET_KIND
-   call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+   call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
    disp = 0_MPI_OFFSET_KIND
-   call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-        newtype,'native',MPI_INFO_NULL,ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                          newtype, 'native', MPI_INFO_NULL, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
    call MPI_FILE_WRITE_ALL(fh, wk, &
-        subsizes(1)*subsizes(2)*subsizes(3), &
-        data_type, MPI_STATUS_IGNORE, ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-   call MPI_FILE_CLOSE(fh,ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-   call MPI_TYPE_FREE(newtype,ierror)
-   if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                           subsizes(1)*subsizes(2)*subsizes(3), &
+                           data_type, MPI_STATUS_IGNORE, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+   call MPI_FILE_CLOSE(fh, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+   call MPI_TYPE_FREE(newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-   deallocate(wk)
+   deallocate (wk)
 
 end if ! color==1
 
 call decomp_mpi_comm_free(newcomm)
 
 call MPI_BARRIER(decomp_2d_comm, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
diff --git a/src/io_write_one.inc b/src/io_write_one.inc
index 8492be5a..93a89317 100644
--- a/src/io_write_one.inc
+++ b/src/io_write_one.inc
@@ -1,17 +1,17 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG) 
+! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
 ! Copyright (C) 2021               the University of Edinburgh (UoE)
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'mpiio_write_one_...' in io.f90
 
 ! Using MPI-IO to write a distributed 3D array into a file
@@ -31,46 +31,46 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1)-1  ! 0-based index
-   starts(2) = decomp%xst(2)-1
-   starts(3) = decomp%xst(3)-1
+   starts(1) = decomp%xst(1) - 1  ! 0-based index
+   starts(2) = decomp%xst(2) - 1
+   starts(3) = decomp%xst(3) - 1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1)-1
-   starts(2) = decomp%yst(2)-1
-   starts(3) = decomp%yst(3)-1
+   starts(1) = decomp%yst(1) - 1
+   starts(2) = decomp%yst(2) - 1
+   starts(3) = decomp%yst(3) - 1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1)-1
-   starts(2) = decomp%zst(2)-1
-   starts(3) = decomp%zst(3)-1
-endif
+   starts(1) = decomp%zst(1) - 1
+   starts(2) = decomp%zst(2) - 1
+   starts(3) = decomp%zst(3) - 1
+end if
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
 call MPI_FILE_OPEN(decomp_2d_comm, filename, &
-     MPI_MODE_CREATE+MPI_MODE_WRONLY, MPI_INFO_NULL, &
-     fh, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
+                   MPI_MODE_CREATE + MPI_MODE_WRONLY, MPI_INFO_NULL, &
+                   fh, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
 filesize = 0_MPI_OFFSET_KIND
-call MPI_FILE_SET_SIZE(fh,filesize,ierror)  ! guarantee overwriting
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
+call MPI_FILE_SET_SIZE(fh, filesize, ierror)  ! guarantee overwriting
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
 disp = 0_MPI_OFFSET_KIND
-call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-     newtype,'native',MPI_INFO_NULL,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                       newtype, 'native', MPI_INFO_NULL, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_WRITE_ALL(fh, var, &
-     subsizes(1)*subsizes(2)*subsizes(3), &
-     data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-call MPI_FILE_CLOSE(fh,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
-call MPI_TYPE_FREE(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                        subsizes(1)*subsizes(2)*subsizes(3), &
+                        data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+call MPI_FILE_CLOSE(fh, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_CLOSE")
+call MPI_TYPE_FREE(newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
diff --git a/src/io_write_outflow.f90 b/src/io_write_outflow.f90
index ccbd0314..fe126584 100644
--- a/src/io_write_outflow.f90
+++ b/src/io_write_outflow.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -9,12 +9,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'write_var_...' in io.f90
 
-! Using MPI-IO to write a distributed 3D variable to a file. File 
+! Using MPI-IO to write a distributed 3D variable to a file. File
 ! operations (open/close) need to be done in calling application. This
-! allows multiple variables to be written to a single file. Together 
+! allows multiple variables to be written to a single file. Together
 ! with the corresponding read operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -25,62 +25,62 @@
 end if
 
 ! Create file type and set file view
-sizes(1) = ntimesteps 
+sizes(1) = ntimesteps
 sizes(2) = decomp%ysz(2)
 sizes(3) = decomp%zsz(3)
-subsizes(1) = ntimesteps 
+subsizes(1) = ntimesteps
 subsizes(2) = decomp%xsz(2)
 subsizes(3) = decomp%xsz(3)
 starts(1) = 0  ! 0-based index
-starts(2) = decomp%xst(2)-1
-starts(3) = decomp%xst(3)-1
+starts(2) = decomp%xst(2) - 1
+starts(3) = decomp%xst(3) - 1
 
 idx = get_io_idx(io_name, dirname)
 
 #ifndef ADIOS2
 !! Use default MPIIO
-associate(fh=>fh_registry(idx), &
-     disp=>fh_disp(idx))
-  call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-  call MPI_TYPE_COMMIT(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-  call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-       newtype,'native',MPI_INFO_NULL,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-  call MPI_FILE_WRITE_ALL(fh, var, &
-       subsizes(1)*subsizes(2)*subsizes(3), &
-       data_type, MPI_STATUS_IGNORE, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-  call MPI_TYPE_FREE(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+associate (fh => fh_registry(idx), &
+           disp => fh_disp(idx))
+   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+   call MPI_TYPE_COMMIT(newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                          newtype, 'native', MPI_INFO_NULL, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+   call MPI_FILE_WRITE_ALL(fh, var, &
+                           subsizes(1)*subsizes(2)*subsizes(3), &
+                           data_type, MPI_STATUS_IGNORE, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+   call MPI_TYPE_FREE(newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
-  ! update displacement for the next write operation
-  disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-              * int(sizes(2), kind=MPI_OFFSET_KIND) &
-              * int(sizes(3), kind=MPI_OFFSET_KIND) &
-              * int(mytype_bytes, kind=MPI_OFFSET_KIND)
-  if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                             * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                             * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+   ! update displacement for the next write operation
+   disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+          *int(sizes(2), kind=MPI_OFFSET_KIND) &
+          *int(sizes(3), kind=MPI_OFFSET_KIND) &
+          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+   if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
+                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
-associate(vnm => varname) ! Silence unused dummy argument
+associate (vnm => varname) ! Silence unused dummy argument
 end associate
 
 #else
 !! Use ADIOS2
 call adios2_at_io(io_handle, adios, io_name, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not.var_handle % valid) then
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+if (.not. var_handle%valid) then
    call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to write variable before registering!"//trim(varname))
-endif
+end if
 
 !! Note - need to use sync mode as we are using a view into the array - unsure how this works with deferred writes
 call adios2_put(engine_registry(idx), var_handle, var, adios2_mode_sync, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
 #endif
diff --git a/src/io_write_plane.inc b/src/io_write_plane.inc
index 088fe097..10f23074 100644
--- a/src/io_write_plane.inc
+++ b/src/io_write_plane.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,7 +10,7 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'mpiio_write_plane_3d_...' in io.f90
 
 ! It is much easier to implement if all mpi ranks participate I/O.
@@ -24,81 +24,81 @@ end if
 
 opened_new = .false.
 
-if (iplane==1) then
-   allocate(wk(decomp%xsz(1),decomp%xsz(2),decomp%xsz(3)))
-   if (ipencil==1) then
+if (iplane == 1) then
+   allocate (wk(decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)))
+   if (ipencil == 1) then
       wk = var
-   else if (ipencil==2) then
-      call transpose_y_to_x(var,wk,decomp)
-   else if (ipencil==3) then
-      allocate(wk2(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)))
-      call transpose_z_to_y(var,wk2,decomp)
-      call transpose_y_to_x(wk2,wk,decomp)
-      deallocate(wk2)
+   else if (ipencil == 2) then
+      call transpose_y_to_x(var, wk, decomp)
+   else if (ipencil == 3) then
+      allocate (wk2(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)))
+      call transpose_z_to_y(var, wk2, decomp)
+      call transpose_y_to_x(wk2, wk, decomp)
+      deallocate (wk2)
    end if
-   allocate(wk2d(1,decomp%xsz(2),decomp%xsz(3)))
-   if (n.ge.1) then
-      do k=1,decomp%xsz(3)
-         do j=1,decomp%xsz(2)
-            wk2d(1,j,k)=wk(n,j,k)
+   allocate (wk2d(1, decomp%xsz(2), decomp%xsz(3)))
+   if (n >= 1) then
+      do k = 1, decomp%xsz(3)
+         do j = 1, decomp%xsz(2)
+            wk2d(1, j, k) = wk(n, j, k)
          end do
       end do
    else
-      do k=1,decomp%xsz(3)
-         do j=1,decomp%xsz(2)
-            wk2d(1,j,k)=sum(wk(:,j,k))/real(decomp%xsz(1),kind=mytype)
+      do k = 1, decomp%xsz(3)
+         do j = 1, decomp%xsz(2)
+            wk2d(1, j, k) = sum(wk(:, j, k))/real(decomp%xsz(1), kind=mytype)
          end do
       end do
-   endif
-else if (iplane==2) then
-   allocate(wk(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)))
-   if (ipencil==1) then
-      call transpose_x_to_y(var,wk,decomp)
-   else if (ipencil==2) then
+   end if
+else if (iplane == 2) then
+   allocate (wk(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)))
+   if (ipencil == 1) then
+      call transpose_x_to_y(var, wk, decomp)
+   else if (ipencil == 2) then
       wk = var
-   else if (ipencil==3) then
-      call transpose_z_to_y(var,wk,decomp)
+   else if (ipencil == 3) then
+      call transpose_z_to_y(var, wk, decomp)
    end if
-   allocate(wk2d(decomp%ysz(1),1,decomp%ysz(3)))
-   if (n.ge.1) then
-      do k=1,decomp%ysz(3)
-         do i=1,decomp%ysz(1)
-            wk2d(i,1,k)=wk(i,n,k)
+   allocate (wk2d(decomp%ysz(1), 1, decomp%ysz(3)))
+   if (n >= 1) then
+      do k = 1, decomp%ysz(3)
+         do i = 1, decomp%ysz(1)
+            wk2d(i, 1, k) = wk(i, n, k)
          end do
       end do
    else
-      do k=1,decomp%ysz(3)
-         do i=1,decomp%ysz(1)
-            wk2d(i,1,k)=sum(wk(i,:,k))/real(decomp%ysz(2),kind=mytype)
+      do k = 1, decomp%ysz(3)
+         do i = 1, decomp%ysz(1)
+            wk2d(i, 1, k) = sum(wk(i, :, k))/real(decomp%ysz(2), kind=mytype)
          end do
       end do
-   endif
-else if (iplane==3) then
-   allocate(wk(decomp%zsz(1),decomp%zsz(2),decomp%zsz(3)))
-   if (ipencil==1) then
-      allocate(wk2(decomp%ysz(1),decomp%ysz(2),decomp%ysz(3)))
-      call transpose_x_to_y(var,wk2,decomp)
-      call transpose_y_to_z(wk2,wk,decomp)
-      deallocate(wk2)
-   else if (ipencil==2) then
-      call transpose_y_to_z(var,wk,decomp)
-   else if (ipencil==3) then
+   end if
+else if (iplane == 3) then
+   allocate (wk(decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)))
+   if (ipencil == 1) then
+      allocate (wk2(decomp%ysz(1), decomp%ysz(2), decomp%ysz(3)))
+      call transpose_x_to_y(var, wk2, decomp)
+      call transpose_y_to_z(wk2, wk, decomp)
+      deallocate (wk2)
+   else if (ipencil == 2) then
+      call transpose_y_to_z(var, wk, decomp)
+   else if (ipencil == 3) then
       wk = var
    end if
-   allocate(wk2d(decomp%zsz(1),decomp%zsz(2),1))
-   if (n.ge.1) then
-      do j=1,decomp%zsz(2)
-         do i=1,decomp%zsz(1) 
-            wk2d(i,j,1)=wk(i,j,n)
+   allocate (wk2d(decomp%zsz(1), decomp%zsz(2), 1))
+   if (n >= 1) then
+      do j = 1, decomp%zsz(2)
+         do i = 1, decomp%zsz(1)
+            wk2d(i, j, 1) = wk(i, j, n)
          end do
       end do
    else
-      do j=1,decomp%zsz(2)
-         do i=1,decomp%zsz(1)
-            wk2d(i,j,1)=sum(wk(i,j,:))/real(decomp%zsz(3),kind=mytype)
+      do j = 1, decomp%zsz(2)
+         do i = 1, decomp%zsz(1)
+            wk2d(i, j, 1) = sum(wk(i, j, :))/real(decomp%zsz(3), kind=mytype)
          end do
       end do
-   endif
+   end if
 end if
 
 idx = get_io_idx(io_name, dirname)
@@ -106,15 +106,15 @@ idx = get_io_idx(io_name, dirname)
 #ifndef ADIOS2
 !! Use default MPIIO writers
 
-if (idx .lt. 1) then
+if (idx < 1) then
    ! Create folder if needed
-   if (nrank==0) then
-      inquire(file=dirname, exist=dir_exists)
-      if (.not.dir_exists) then
+   if (nrank == 0) then
+      inquire (file=dirname, exist=dir_exists)
+      if (.not. dir_exists) then
          call execute_command_line("mkdir "//dirname//" 2> /dev/null", wait=.true.)
       end if
    end if
-   allocate(character(len(trim(dirname)) + 1 + len(trim(varname))) :: full_io_name)
+   allocate (character(len(trim(dirname)) + 1 + len(trim(varname))) :: full_io_name)
    full_io_name = dirname//"/"//varname
    call decomp_2d_open_io(io_name, full_io_name, decomp_2d_write_mode)
    idx = get_io_idx(io_name, full_io_name)
@@ -122,40 +122,40 @@ if (idx .lt. 1) then
 end if
 
 call plane_extents(sizes, subsizes, starts, iplane, decomp)
-associate(fh=>fh_registry(idx), &
-     disp=>fh_disp(idx))
-  call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-       MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-  call MPI_TYPE_COMMIT(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-  call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-       newtype,'native',MPI_INFO_NULL,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
-  call MPI_FILE_WRITE_ALL(fh, wk2d, &
-       subsizes(1)*subsizes(2)*subsizes(3), &
-       data_type, MPI_STATUS_IGNORE, ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-  call MPI_TYPE_FREE(newtype,ierror)
-  if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+associate (fh => fh_registry(idx), &
+           disp => fh_disp(idx))
+   call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                                 MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+   call MPI_TYPE_COMMIT(newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+   call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                          newtype, 'native', MPI_INFO_NULL, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+   call MPI_FILE_WRITE_ALL(fh, wk2d, &
+                           subsizes(1)*subsizes(2)*subsizes(3), &
+                           data_type, MPI_STATUS_IGNORE, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+   call MPI_TYPE_FREE(newtype, ierror)
+   if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 end associate
-    
+
 if (opened_new) then
    call decomp_2d_close_io(io_name, full_io_name)
-   deallocate(full_io_name)
+   deallocate (full_io_name)
 end if
 #else
 !! Write using ADIOS2
 call adios2_at_io(io_handle, adios, io_name, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not.var_handle % valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-   "trying to write variable before registering! "//trim(varname))
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
+if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
+                                                 "trying to write variable before registering! "//trim(varname))
 
 !! Note - need to use sync mode as the array for the output plane gets reused.
 call adios2_put(engine_registry(idx), var_handle, wk2d, adios2_mode_sync, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_put")
 #endif
 
-deallocate(wk,wk2d)
+deallocate (wk, wk2d)
diff --git a/src/io_write_var.inc b/src/io_write_var.inc
index 23c8b617..f5b34b82 100644
--- a/src/io_write_var.inc
+++ b/src/io_write_var.inc
@@ -1,8 +1,8 @@
-!                          -*- mode: f90 -*-                            
+!                          -*- mode: f90 -*-
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,12 +10,12 @@
 !
 !=======================================================================
 
-! This file contain common code to be included by subroutines 
+! This file contain common code to be included by subroutines
 ! 'write_var_...' in io.f90
 
-! Using MPI-IO to write a distributed 3D variable to a file. File 
+! Using MPI-IO to write a distributed 3D variable to a file. File
 ! operations (open/close) need to be done in calling application. This
-! allows multiple variables to be written to a single file. Together 
+! allows multiple variables to be written to a single file. Together
 ! with the corresponding read operation, this is the perfect solution
 ! for applications to perform restart/checkpointing.
 
@@ -33,46 +33,46 @@ if (ipencil == 1) then
    subsizes(1) = decomp%xsz(1)
    subsizes(2) = decomp%xsz(2)
    subsizes(3) = decomp%xsz(3)
-   starts(1) = decomp%xst(1)-1  ! 0-based index
-   starts(2) = decomp%xst(2)-1
-   starts(3) = decomp%xst(3)-1
+   starts(1) = decomp%xst(1) - 1  ! 0-based index
+   starts(2) = decomp%xst(2) - 1
+   starts(3) = decomp%xst(3) - 1
 else if (ipencil == 2) then
    subsizes(1) = decomp%ysz(1)
    subsizes(2) = decomp%ysz(2)
    subsizes(3) = decomp%ysz(3)
-   starts(1) = decomp%yst(1)-1
-   starts(2) = decomp%yst(2)-1
-   starts(3) = decomp%yst(3)-1
+   starts(1) = decomp%yst(1) - 1
+   starts(2) = decomp%yst(2) - 1
+   starts(3) = decomp%yst(3) - 1
 else if (ipencil == 3) then
    subsizes(1) = decomp%zsz(1)
    subsizes(2) = decomp%zsz(2)
    subsizes(3) = decomp%zsz(3)
-   starts(1) = decomp%zst(1)-1
-   starts(2) = decomp%zst(2)-1
-   starts(3) = decomp%zst(3)-1
-endif
+   starts(1) = decomp%zst(1) - 1
+   starts(2) = decomp%zst(2) - 1
+   starts(3) = decomp%zst(3) - 1
+end if
 
-call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts,  &
-     MPI_ORDER_FORTRAN, data_type, newtype, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
-call MPI_TYPE_COMMIT(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
-call MPI_FILE_SET_VIEW(fh,disp,data_type, &
-     newtype,'native',MPI_INFO_NULL,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
+call MPI_TYPE_CREATE_SUBARRAY(3, sizes, subsizes, starts, &
+                              MPI_ORDER_FORTRAN, data_type, newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_CREATE_SUBARRAY")
+call MPI_TYPE_COMMIT(newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
+call MPI_FILE_SET_VIEW(fh, disp, data_type, &
+                       newtype, 'native', MPI_INFO_NULL, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
 call MPI_FILE_WRITE_ALL(fh, var, &
-     subsizes(1)*subsizes(2)*subsizes(3), &
-     data_type, MPI_STATUS_IGNORE, ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
-call MPI_TYPE_FREE(newtype,ierror)
-if (ierror.ne.0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
+                        subsizes(1)*subsizes(2)*subsizes(3), &
+                        data_type, MPI_STATUS_IGNORE, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
+call MPI_TYPE_FREE(newtype, ierror)
+if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 ! update displacement for the next write operation
 disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-            * int(sizes(2), kind=MPI_OFFSET_KIND) &
-            * int(sizes(3), kind=MPI_OFFSET_KIND) &
-            * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+       *int(sizes(2), kind=MPI_OFFSET_KIND) &
+       *int(sizes(3), kind=MPI_OFFSET_KIND) &
+       *int(mytype_bytes, kind=MPI_OFFSET_KIND)
 if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                           * int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                           * int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                           * int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                      *int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                      *int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                      *int(mytype_bytes, kind=MPI_OFFSET_KIND)
diff --git a/src/log.f90 b/src/log.f90
index 866cd97e..53e3165a 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -10,296 +10,296 @@
 !
 !=======================================================================
 
-submodule (decomp_2d) d2d_log
+submodule(decomp_2d) d2d_log
 
-  implicit none
+   implicit none
 
-  contains
+contains
 
-  !
-  ! Print some information about decomp_2d
-  !
-  module subroutine d2d_listing(given_io_unit)
+   !
+   ! Print some information about decomp_2d
+   !
+   module subroutine d2d_listing(given_io_unit)
 
-    use iso_fortran_env, only : output_unit, compiler_version, compiler_options
+      use iso_fortran_env, only: output_unit, compiler_version, compiler_options
 
-    implicit none
+      implicit none
 
-    ! Argument
-    integer, intent(in), optional :: given_io_unit
+      ! Argument
+      integer, intent(in), optional :: given_io_unit
 
-    ! Local variable
-    integer :: io_unit
-    integer :: version, subversion, ierror
+      ! Local variable
+      integer :: io_unit
+      integer :: version, subversion, ierror
 #ifdef DEBUG
-    character(len=64) :: fname
+      character(len=64) :: fname
 #endif
 
-    ! Output log if needed
-    if (decomp_log == D2D_LOG_QUIET) return
-    if (decomp_log == D2D_LOG_STDOUT .and. nrank /= 0) return
-    if (decomp_log == D2D_LOG_TOFILE .and. nrank /= 0) return
+      ! Output log if needed
+      if (decomp_log == D2D_LOG_QUIET) return
+      if (decomp_log == D2D_LOG_STDOUT .and. nrank /= 0) return
+      if (decomp_log == D2D_LOG_TOFILE .and. nrank /= 0) return
 
-    ! If no IO unit provided, use stdout
-    if (present(given_io_unit)) then
-       io_unit = given_io_unit
-    else
-       io_unit = output_unit
-    endif
+      ! If no IO unit provided, use stdout
+      if (present(given_io_unit)) then
+         io_unit = given_io_unit
+      else
+         io_unit = output_unit
+      end if
 
-    ! Header
-    write (io_unit, *) '==========================================================='
-    write (io_unit, *) '=================== Decomp2D - log ========================'
-    write (io_unit, *) '==========================================================='
+      ! Header
+      write (io_unit, *) '==========================================================='
+      write (io_unit, *) '=================== Decomp2D - log ========================'
+      write (io_unit, *) '==========================================================='
 
-    ! Git hash if available
+      ! Git hash if available
 #if defined(VERSION)
-    write (io_unit, *) 'Git version        : ', VERSION
+      write (io_unit, *) 'Git version        : ', VERSION
 #else
-    write (io_unit, *) 'Git version        : unknown'
+      write (io_unit, *) 'Git version        : unknown'
 #endif
 
-    ! Basic info
+      ! Basic info
 #ifdef DEBUG
-    if (decomp_debug >= D2D_DEBUG_LEVEL_INFO) &
-       write (io_unit, *) 'I am mpi rank ', nrank
+      if (decomp_debug >= D2D_DEBUG_LEVEL_INFO) &
+         write (io_unit, *) 'I am mpi rank ', nrank
 #endif
-    write (io_unit, *) 'Total ranks ', nproc
-    write (io_unit, *) 'Global data size : ', nx_global, ny_global, nz_global
-    write (io_unit, *) 'p_row, p_col : ', dims(1), dims(2)
-    write (io_unit, *) 'Periodicity : ', periodic_x, periodic_y, periodic_z
-    write (io_unit, *) 'Number of bytes / float number : ', mytype_bytes
-    write (io_unit, *) '==========================================================='
-
-    ! Show detected flags, compiler options, version of the MPI library
-    write (io_unit, *) 'Compile flags detected :'
+      write (io_unit, *) 'Total ranks ', nproc
+      write (io_unit, *) 'Global data size : ', nx_global, ny_global, nz_global
+      write (io_unit, *) 'p_row, p_col : ', dims(1), dims(2)
+      write (io_unit, *) 'Periodicity : ', periodic_x, periodic_y, periodic_z
+      write (io_unit, *) 'Number of bytes / float number : ', mytype_bytes
+      write (io_unit, *) '==========================================================='
+
+      ! Show detected flags, compiler options, version of the MPI library
+      write (io_unit, *) 'Compile flags detected :'
 #ifdef DOUBLE_PREC
 #ifdef SAVE_SINGLE
-    write (io_unit, *) 'Numerical precision: Double, saving in single'
+      write (io_unit, *) 'Numerical precision: Double, saving in single'
 #else
-    write (io_unit, *) 'Numerical precision: Double'
+      write (io_unit, *) 'Numerical precision: Double'
 #endif
 #else
-    write (io_unit, *) 'Numerical precision: Single'
+      write (io_unit, *) 'Numerical precision: Single'
 #endif
-    write (io_unit, *) 'Compiled with ', compiler_version()
-    write (io_unit, *) 'Compiler options : ', compiler_options()
-    call MPI_Get_version(version, subversion, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_Get_version")
-    write (io_unit, '(" Version of the MPI library : ",I0,".",I0)') version, subversion
+      write (io_unit, *) 'Compiled with ', compiler_version()
+      write (io_unit, *) 'Compiler options : ', compiler_options()
+      call MPI_Get_version(version, subversion, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_Get_version")
+      write (io_unit, '(" Version of the MPI library : ",I0,".",I0)') version, subversion
 #ifdef DEBUG
-    write (io_unit, *) 'Compile flag DEBUG detected'
-    write (io_unit, *) '   debug level : ', decomp_debug
+      write (io_unit, *) 'Compile flag DEBUG detected'
+      write (io_unit, *) '   debug level : ', decomp_debug
 #endif
 #ifdef PROFILER
-    write (io_unit, *) 'Compile flag PROFILER detected'
+      write (io_unit, *) 'Compile flag PROFILER detected'
 #endif
 #ifdef SHM
-    write (io_unit, *) 'Compile flag SHM detected'
+      write (io_unit, *) 'Compile flag SHM detected'
 #endif
 #ifdef EVEN
-    write (io_unit, *) 'Compile flag EVEN detected'
+      write (io_unit, *) 'Compile flag EVEN detected'
 #endif
 #ifdef OCC
-    write (io_unit, *) 'Compile flag OCC detected'
+      write (io_unit, *) 'Compile flag OCC detected'
 #endif
 #ifdef OVERWRITE
-    write (io_unit, *) 'Compile flag OVERWRITE detected'
+      write (io_unit, *) 'Compile flag OVERWRITE detected'
 #endif
 #ifdef HALO_DEBUG
-    write (io_unit, *) 'Compile flag HALO_DEBUG detected'
+      write (io_unit, *) 'Compile flag HALO_DEBUG detected'
 #endif
 #ifdef SHM_DEBUG
-    write (io_unit, *) 'Compile flag SHM_DEBUG detected'
+      write (io_unit, *) 'Compile flag SHM_DEBUG detected'
 #endif
 #ifdef _GPU
-    write (io_unit, *) 'Compile flag _GPU detected'
+      write (io_unit, *) 'Compile flag _GPU detected'
 #endif
 #ifdef _NCCL
-    write (io_unit, *) 'Compile flag _NCCL detected'
+      write (io_unit, *) 'Compile flag _NCCL detected'
 #endif
-    write (io_unit, *) '==========================================================='
-    write (io_unit, *) 'Profiler id : ', decomp_profiler
+      write (io_unit, *) '==========================================================='
+      write (io_unit, *) 'Profiler id : ', decomp_profiler
 #ifdef PROFILER
-    call decomp_profiler_log(io_unit)
-    write(io_unit, *) "   Profiling transpose : ", decomp_profiler_transpose
-    write(io_unit, *) "   Profiling IO : ", decomp_profiler_io
-    write(io_unit, *) "   Profiling FFT : ", decomp_profiler_fft
-    write(io_unit, *) "   Profiling decomp_2d : ", decomp_profiler_d2d
+      call decomp_profiler_log(io_unit)
+      write (io_unit, *) "   Profiling transpose : ", decomp_profiler_transpose
+      write (io_unit, *) "   Profiling IO : ", decomp_profiler_io
+      write (io_unit, *) "   Profiling FFT : ", decomp_profiler_fft
+      write (io_unit, *) "   Profiling decomp_2d : ", decomp_profiler_d2d
 #endif
-    write (io_unit, *) '==========================================================='
-    ! Info about each decomp_info object
-    call decomp_info_print(decomp_main, io_unit, "decomp_main")
-    call decomp_info_print(phG, io_unit, "phG")
-    call decomp_info_print(ph1, io_unit, "ph1")
-    call decomp_info_print(ph2, io_unit, "ph2")
-    call decomp_info_print(ph3, io_unit, "ph3")
-    call decomp_info_print(ph4, io_unit, "ph4")
+      write (io_unit, *) '==========================================================='
+      ! Info about each decomp_info object
+      call decomp_info_print(decomp_main, io_unit, "decomp_main")
+      call decomp_info_print(phG, io_unit, "phG")
+      call decomp_info_print(ph1, io_unit, "ph1")
+      call decomp_info_print(ph2, io_unit, "ph2")
+      call decomp_info_print(ph3, io_unit, "ph3")
+      call decomp_info_print(ph4, io_unit, "ph4")
 #ifdef SHM_DEBUG
-    write (io_unit, *) '==========================================================='
-    call print_smp(io_unit)
+      write (io_unit, *) '==========================================================='
+      call print_smp(io_unit)
 #endif
-    write (io_unit, *) '==========================================================='
-    write (io_unit, *) '==========================================================='
+      write (io_unit, *) '==========================================================='
+      write (io_unit, *) '==========================================================='
 #ifdef DEBUG
-    !
-    ! In DEBUG mode, rank 0 will also print environment variables
-    !
-    ! At high debug level, all ranks will print env. variables
-    !
-    ! The system call, if writing to a file, is not blocking if supported
-    !
-    if (nrank == 0 .or. decomp_debug >= D2D_DEBUG_LEVEL_INFO) then
-       write (io_unit, *) '============== Environment variables ======================'
-       write (io_unit, *) '==========================================================='
-       write (io_unit, *) '==========================================================='
-       if (io_unit == output_unit ) then
-          call execute_command_line("env", wait = .true.)
-       else
-          inquire(unit = io_unit, name = fname, iostat = ierror)
-          if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                                __LINE__, &
-                                                ierror, &
-                                                "No name for the log file")
-          call execute_command_line("env >> "//trim(fname), wait = .false.)
-       endif
-    endif
+      !
+      ! In DEBUG mode, rank 0 will also print environment variables
+      !
+      ! At high debug level, all ranks will print env. variables
+      !
+      ! The system call, if writing to a file, is not blocking if supported
+      !
+      if (nrank == 0 .or. decomp_debug >= D2D_DEBUG_LEVEL_INFO) then
+         write (io_unit, *) '============== Environment variables ======================'
+         write (io_unit, *) '==========================================================='
+         write (io_unit, *) '==========================================================='
+         if (io_unit == output_unit) then
+            call execute_command_line("env", wait=.true.)
+         else
+            inquire (unit=io_unit, name=fname, iostat=ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, &
+                                                  __LINE__, &
+                                                  ierror, &
+                                                  "No name for the log file")
+            call execute_command_line("env >> "//trim(fname), wait=.false.)
+         end if
+      end if
 #endif
 
-  end subroutine d2d_listing
+   end subroutine d2d_listing
 
-  !
-  ! Print some information about given decomp_info object
-  !
-  module subroutine decomp_info_print(d2d, io_unit, d2dname)
+   !
+   ! Print some information about given decomp_info object
+   !
+   module subroutine decomp_info_print(d2d, io_unit, d2dname)
 
-    implicit none
+      implicit none
 
-    ! Arguments
-    type(decomp_info), intent(in) :: d2d
-    integer, intent(in) :: io_unit
-    character(len=*), intent(in) :: d2dname
+      ! Arguments
+      type(decomp_info), intent(in) :: d2d
+      integer, intent(in) :: io_unit
+      character(len=*), intent(in) :: d2dname
 
-    ! Nothing to print if not initialized
-    if (.not.allocated(d2d%x1dist)) then
-      write (io_unit, *) 'Uninitialized decomp_info ', d2dname
-      return
-    endif
+      ! Nothing to print if not initialized
+      if (.not. allocated(d2d%x1dist)) then
+         write (io_unit, *) 'Uninitialized decomp_info ', d2dname
+         return
+      end if
 
-    !
-    ! If DEBUG mode, print everything
-    ! Otherwise, print only global size
-    !
-    write (io_unit, *) 'Decomp_info : ', d2dname
-    write (io_unit, *) '   Global size : ', d2d%xsz(1), d2d%ysz(2), d2d%zsz(3)
+      !
+      ! If DEBUG mode, print everything
+      ! Otherwise, print only global size
+      !
+      write (io_unit, *) 'Decomp_info : ', d2dname
+      write (io_unit, *) '   Global size : ', d2d%xsz(1), d2d%ysz(2), d2d%zsz(3)
 #ifdef DEBUG
-    write (io_unit, *) '   xsz, xst, xen : ', d2d%xsz, d2d%xst, d2d%xen
-    write (io_unit, *) '   ysz, yst, yen : ', d2d%ysz, d2d%yst, d2d%yen
-    write (io_unit, *) '   zsz, zst, zen : ', d2d%zsz, d2d%zst, d2d%zen
-    write (io_unit, *) '   x1dist : ', d2d%x1dist
-    write (io_unit, *) '   y1dist : ', d2d%y1dist
-    write (io_unit, *) '   y2dist : ', d2d%y2dist
-    write (io_unit, *) '   z2dist : ', d2d%z2dist
-    write (io_unit, *) '   x1cnts : ', d2d%x1cnts
-    write (io_unit, *) '   y1cnts : ', d2d%y1cnts
-    write (io_unit, *) '   y2cnts : ', d2d%y2cnts
-    write (io_unit, *) '   z2cnts : ', d2d%z2cnts
-    write (io_unit, *) '   x1disp : ', d2d%x1disp
-    write (io_unit, *) '   y1disp : ', d2d%y1disp
-    write (io_unit, *) '   y2disp : ', d2d%y2disp
-    write (io_unit, *) '   z2disp : ', d2d%z2disp
-    write (io_unit, *) '   x1count : ', d2d%x1count
-    write (io_unit, *) '   y1count : ', d2d%y1count
-    write (io_unit, *) '   y2count : ', d2d%y2count
-    write (io_unit, *) '   z2count : ', d2d%z2count
-    write (io_unit, *) '   even : ', d2d%even
+      write (io_unit, *) '   xsz, xst, xen : ', d2d%xsz, d2d%xst, d2d%xen
+      write (io_unit, *) '   ysz, yst, yen : ', d2d%ysz, d2d%yst, d2d%yen
+      write (io_unit, *) '   zsz, zst, zen : ', d2d%zsz, d2d%zst, d2d%zen
+      write (io_unit, *) '   x1dist : ', d2d%x1dist
+      write (io_unit, *) '   y1dist : ', d2d%y1dist
+      write (io_unit, *) '   y2dist : ', d2d%y2dist
+      write (io_unit, *) '   z2dist : ', d2d%z2dist
+      write (io_unit, *) '   x1cnts : ', d2d%x1cnts
+      write (io_unit, *) '   y1cnts : ', d2d%y1cnts
+      write (io_unit, *) '   y2cnts : ', d2d%y2cnts
+      write (io_unit, *) '   z2cnts : ', d2d%z2cnts
+      write (io_unit, *) '   x1disp : ', d2d%x1disp
+      write (io_unit, *) '   y1disp : ', d2d%y1disp
+      write (io_unit, *) '   y2disp : ', d2d%y2disp
+      write (io_unit, *) '   z2disp : ', d2d%z2disp
+      write (io_unit, *) '   x1count : ', d2d%x1count
+      write (io_unit, *) '   y1count : ', d2d%y1count
+      write (io_unit, *) '   y2count : ', d2d%y2count
+      write (io_unit, *) '   z2count : ', d2d%z2count
+      write (io_unit, *) '   even : ', d2d%even
 #ifdef SHM
-    write (io_unit, *) '   listing of the SHM part is not yet implemented'
+      write (io_unit, *) '   listing of the SHM part is not yet implemented'
 #endif
 #endif
 
-  end subroutine decomp_info_print
+   end subroutine decomp_info_print
 
 #ifdef SHM_DEBUG
 
-  ! For debugging, print the shared-memory structure
-  module subroutine print_smp(io_unit)
-
-    implicit none
-    
-    ! Argument
-    integer, intent(in) :: io_unit
-
-    ! print out shared-memory information
-    write(io_unit,*)'I am mpi rank ', nrank, 'Total ranks ', nproc
-    write(io_unit,*)' '
-    write(io_unit,*)'Global data size:'
-    write(io_unit,*)'nx*ny*nz', nx,ny,nz
-    write(io_unit,*)' '
-    write(io_unit,*)'2D processor grid:'
-    write(io_unit,*)'p_row*p_col:', dims(1), dims(2)
-    write(io_unit,*)' '
-    write(io_unit,*)'Portion of global data held locally:'
-    write(io_unit,*)'xsize:',xsize
-    write(io_unit,*)'ysize:',ysize
-    write(io_unit,*)'zsize:',zsize
-    write(io_unit,*)' '
-    write(io_unit,*)'How pensils are to be divided and sent in alltoallv:'
-    write(io_unit,*)'x1dist:',decomp_main%x1dist
-    write(io_unit,*)'y1dist:',decomp_main%y1dist
-    write(io_unit,*)'y2dist:',decomp_main%y2dist
-    write(io_unit,*)'z2dist:',decomp_main%z2dist
-    write(io_unit,*)' '
-    write(io_unit,*)'######Shared buffer set up after this point######'
-    write(io_unit,*)' '
-    write(io_unit,*) 'col communicator detais:'
-    call print_smp_info(decomp_main%COL_INFO, io_unit)
-    write(io_unit,*)' '
-    write(io_unit,*) 'row communicator detais:'
-    call print_smp_info(decomp_main%ROW_INFO; io_unit)
-    write(io_unit,*)' '
-    write(io_unit,*)'Buffer count and displacement of per-core buffers'
-    write(io_unit,*)'x1cnts:',decomp_main%x1cnts
-    write(io_unit,*)'y1cnts:',decomp_main%y1cnts
-    write(io_unit,*)'y2cnts:',decomp_main%y2cnts
-    write(io_unit,*)'z2cnts:',decomp_main%z2cnts
-    write(io_unit,*)'x1disp:',decomp_main%x1disp
-    write(io_unit,*)'y1disp:',decomp_main%y1disp
-    write(io_unit,*)'y2disp:',decomp_main%y2disp
-    write(io_unit,*)'z2disp:',decomp_main%z2disp
-    write(io_unit,*)' '
-    write(io_unit,*)'Buffer count and displacement of shared buffers'
-    write(io_unit,*)'x1cnts:',decomp_main%x1cnts_s
-    write(io_unit,*)'y1cnts:',decomp_main%y1cnts_s
-    write(io_unit,*)'y2cnts:',decomp_main%y2cnts_s
-    write(io_unit,*)'z2cnts:',decomp_main%z2cnts_s
-    write(io_unit,*)'x1disp:',decomp_main%x1disp_s
-    write(io_unit,*)'y1disp:',decomp_main%y1disp_s
-    write(io_unit,*)'y2disp:',decomp_main%y2disp_s
-    write(io_unit,*)'z2disp:',decomp_main%z2disp_s
-
-  end subroutine print_smp
-
-  ! For debugging, print the shared-memory structure
-  module subroutine print_smp_info(s, io_unit)
-
-    implicit none
-
-    ! Argument
-    TYPE(SMP_INFO), intent(in) :: s
-    integer, intent(in) :: io_unit
-
-    write(io_unit,*) 'size of current communicator:', s%NCPU
-    write(io_unit,*) 'rank in current communicator:', s%NODE_ME
-    write(io_unit,*) 'number of SMP-nodes in this communicator:', s%NSMP
-    write(io_unit,*) 'SMP-node id (1 ~ NSMP):', s%SMP_ME
-    write(io_unit,*) 'NCORE - number of cores on this SMP-node', s%NCORE
-    write(io_unit,*) 'core id (1 ~ NCORE):', s%CORE_ME
-    write(io_unit,*) 'maximum no. cores on any SMP-node:', s%MAXCORE
-    write(io_unit,*) 'size of SMP shared memory SND buffer:', s%N_SND
-    write(io_unit,*) 'size of SMP shared memory RCV buffer:', s%N_RCV
-
-  end subroutine print_smp_info
+   ! For debugging, print the shared-memory structure
+   module subroutine print_smp(io_unit)
+
+      implicit none
+
+      ! Argument
+      integer, intent(in) :: io_unit
+
+      ! print out shared-memory information
+      write (io_unit, *) 'I am mpi rank ', nrank, 'Total ranks ', nproc
+      write (io_unit, *) ' '
+      write (io_unit, *) 'Global data size:'
+      write (io_unit, *) 'nx*ny*nz', nx, ny, nz
+      write (io_unit, *) ' '
+      write (io_unit, *) '2D processor grid:'
+      write (io_unit, *) 'p_row*p_col:', dims(1), dims(2)
+      write (io_unit, *) ' '
+      write (io_unit, *) 'Portion of global data held locally:'
+      write (io_unit, *) 'xsize:', xsize
+      write (io_unit, *) 'ysize:', ysize
+      write (io_unit, *) 'zsize:', zsize
+      write (io_unit, *) ' '
+      write (io_unit, *) 'How pensils are to be divided and sent in alltoallv:'
+      write (io_unit, *) 'x1dist:', decomp_main%x1dist
+      write (io_unit, *) 'y1dist:', decomp_main%y1dist
+      write (io_unit, *) 'y2dist:', decomp_main%y2dist
+      write (io_unit, *) 'z2dist:', decomp_main%z2dist
+      write (io_unit, *) ' '
+      write (io_unit, *) '######Shared buffer set up after this point######'
+      write (io_unit, *) ' '
+      write (io_unit, *) 'col communicator detais:'
+      call print_smp_info(decomp_main%COL_INFO, io_unit)
+      write (io_unit, *) ' '
+      write (io_unit, *) 'row communicator detais:'
+      call print_smp_info(decomp_main%ROW_INFO; io_unit)
+      write (io_unit, *) ' '
+      write (io_unit, *) 'Buffer count and displacement of per-core buffers'
+      write (io_unit, *) 'x1cnts:', decomp_main%x1cnts
+      write (io_unit, *) 'y1cnts:', decomp_main%y1cnts
+      write (io_unit, *) 'y2cnts:', decomp_main%y2cnts
+      write (io_unit, *) 'z2cnts:', decomp_main%z2cnts
+      write (io_unit, *) 'x1disp:', decomp_main%x1disp
+      write (io_unit, *) 'y1disp:', decomp_main%y1disp
+      write (io_unit, *) 'y2disp:', decomp_main%y2disp
+      write (io_unit, *) 'z2disp:', decomp_main%z2disp
+      write (io_unit, *) ' '
+      write (io_unit, *) 'Buffer count and displacement of shared buffers'
+      write (io_unit, *) 'x1cnts:', decomp_main%x1cnts_s
+      write (io_unit, *) 'y1cnts:', decomp_main%y1cnts_s
+      write (io_unit, *) 'y2cnts:', decomp_main%y2cnts_s
+      write (io_unit, *) 'z2cnts:', decomp_main%z2cnts_s
+      write (io_unit, *) 'x1disp:', decomp_main%x1disp_s
+      write (io_unit, *) 'y1disp:', decomp_main%y1disp_s
+      write (io_unit, *) 'y2disp:', decomp_main%y2disp_s
+      write (io_unit, *) 'z2disp:', decomp_main%z2disp_s
+
+   end subroutine print_smp
+
+   ! For debugging, print the shared-memory structure
+   module subroutine print_smp_info(s, io_unit)
+
+      implicit none
+
+      ! Argument
+      TYPE(SMP_INFO), intent(in) :: s
+      integer, intent(in) :: io_unit
+
+      write (io_unit, *) 'size of current communicator:', s%NCPU
+      write (io_unit, *) 'rank in current communicator:', s%NODE_ME
+      write (io_unit, *) 'number of SMP-nodes in this communicator:', s%NSMP
+      write (io_unit, *) 'SMP-node id (1 ~ NSMP):', s%SMP_ME
+      write (io_unit, *) 'NCORE - number of cores on this SMP-node', s%NCORE
+      write (io_unit, *) 'core id (1 ~ NCORE):', s%CORE_ME
+      write (io_unit, *) 'maximum no. cores on any SMP-node:', s%MAXCORE
+      write (io_unit, *) 'size of SMP shared memory SND buffer:', s%N_SND
+      write (io_unit, *) 'size of SMP shared memory RCV buffer:', s%N_RCV
+
+   end subroutine print_smp_info
 #endif
 
 end submodule d2d_log
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index c5d83fa4..bd783d7e 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,480 +13,474 @@
 
   subroutine transpose_x_to_y_real_short(src, dst)
 
-    implicit none
+     implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_x_to_y(src, dst, decomp_main)
+     call transpose_x_to_y(src, dst, decomp_main)
 
   end subroutine transpose_x_to_y_real_short
 
   subroutine transpose_x_to_y_real(src, dst, decomp)
 
-    implicit none
-    
-    Real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
 
+     Real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU) && defined(_NCCL)
-    integer :: col_rank_id, cuda_stat
-    type(ncclResult) :: nccl_stat
+     integer :: col_rank_id, cuda_stat
+     type(ncclResult) :: nccl_stat
 #endif
 
 #ifdef SHM
-    real(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     real(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-    ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%COL_INFO%SND_P
-    call mem_split_xy_real(src, s1, s2, s3, work1, dims(1), &
-         decomp%x1dist, decomp)
+     work1_p = decomp%COL_INFO%SND_P
+     call mem_split_xy_real(src, s1, s2, s3, work1, dims(1), &
+                            decomp%x1dist, decomp)
 #else
-  
+
 #if defined(_GPU)
-    call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
+                            decomp%x1dist, decomp)
 #else
-    call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
+                            decomp%x1dist, decomp)
 #endif
 
 #endif
 
-    ! define receive buffer
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%COL_INFO%RCV_P
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%COL_INFO%RCV_P
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
-    ! transpose using MPI_ALLTOALL(V)
+
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-    if (decomp%COL_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
-            real_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
-            real_type, decomp%COL_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%COL_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
+                           real_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
+                           real_type, decomp%COL_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    call MPI_ALLTOALL(work1_r, decomp%x1count, &
-         real_type, work2_r, decomp%y1count, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_r, decomp%x1count, &
+                       real_type, work2_r, decomp%y1count, &
+                       real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    nccl_stat = ncclGroupStart()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-    do col_rank_id = 0, (col_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d( decomp%x1disp(col_rank_id)+1 ), decomp%x1cnts(col_rank_id), &
-          ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+     nccl_stat = ncclGroupStart()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+     do col_rank_id = 0, (col_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
+                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d( decomp%y1disp(col_rank_id)+1 ), decomp%y1cnts(col_rank_id), &
-          ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+        nccl_stat = ncclRecv(work2_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
+                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-    end do
-    nccl_stat = ncclGroupEnd()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     end do
+     nccl_stat = ncclGroupEnd()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-    call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
-         real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
+                        real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-    call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
-         real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
+                        real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_xy_real(work2, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_xy_real(work2, d1, d2, d3, dst, dims(1), &
+                            decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+                            decomp%y1dist, decomp)
 #else
-    call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
+                            decomp%y1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
 #endif
 
-    return
+     return
   end subroutine transpose_x_to_y_real
 
-
   subroutine transpose_x_to_y_complex_short(src, dst)
 
-    implicit none
+     implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_x_to_y(src, dst, decomp_main)
+     call transpose_x_to_y(src, dst, decomp_main)
 
   end subroutine transpose_x_to_y_complex_short
 
   subroutine transpose_x_to_y_complex(src, dst, decomp)
 
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #ifdef SHM
-    complex(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     complex(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
-    
-    ! rearrange source array as send buffer
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
+
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%COL_INFO%SND_P_c
-    call mem_split_xy_complex(src, s1, s2, s3, work1, dims(1), &
-         decomp%x1dist, decomp)
+     work1_p = decomp%COL_INFO%SND_P_c
+     call mem_split_xy_complex(src, s1, s2, s3, work1, dims(1), &
+                               decomp%x1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+                               decomp%x1dist, decomp)
 #else
-    call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
+                               decomp%x1dist, decomp)
 #endif
 
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%COL_INFO%RCV_P_c
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%COL_INFO%RCV_P_c
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
-    ! transpose using MPI_ALLTOALL(V)
+
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-    if (decomp%COL_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
-            complex_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
-            complex_type, decomp%COL_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%COL_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
+                           complex_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
+                           complex_type, decomp%COL_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    call MPI_ALLTOALL(work1_c, decomp%x1count, &
-         complex_type, work2_c, decomp%y1count, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_c, decomp%x1count, &
+                       complex_type, work2_c, decomp%y1count, &
+                       complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-    call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
-         complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-    call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
-         complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_xy_complex(work2, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_xy_complex(work2, d1, d2, d3, dst, dims(1), &
+                               decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+                               decomp%y1dist, decomp)
 #else
-    call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
+                               decomp%y1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
 #endif
 
-    return
+     return
   end subroutine transpose_x_to_y_complex
 
+  subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_xy_real(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    real(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
-    
-    integer :: i,j,k, m,i1,i2,pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%x1disp_o(m) + 1
+        pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%x1count + 1
+        pos = m*decomp%x1count + 1
 #else
-       pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=1,n2
-             do i=i1,i2
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_xy_real
 
+  subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_xy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    complex(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%x1disp_o(m) + 1
+        pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%x1count + 1
+        pos = m*decomp%x1count + 1
 #else
-       pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=1,n2
-             do i=i1,i2
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_xy_complex
 
+  subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_xy_real(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(*), intent(IN) :: in
-    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
-        
-    integer :: i,j,k, m,i1,i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y1disp_o(m) + 1
+        pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y1count + 1
+        pos = m*decomp%y1count + 1
 #else
-       pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_xy_real
 
+  subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_xy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(*), intent(IN) :: in
-    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y1disp_o(m) + 1
+        pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y1count + 1
+        pos = m*decomp%y1count + 1
 #else
-       pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_xy_complex
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index 1e7c62c2..a742ead6 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,479 +13,474 @@
 
   subroutine transpose_y_to_x_real_short(src, dst)
 
-    implicit none
+     implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_y_to_x(src, dst, decomp_main)
+     call transpose_y_to_x(src, dst, decomp_main)
 
   end subroutine transpose_y_to_x_real_short
 
   subroutine transpose_y_to_x_real(src, dst, decomp)
 
-    implicit none
-    
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU) && defined(_NCCL)
-    type(ncclResult) :: nccl_stat
-    integer :: col_rank_id, cuda_stat
+     type(ncclResult) :: nccl_stat
+     integer :: col_rank_id, cuda_stat
 #endif
 
 #ifdef SHM
-    real(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     real(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-    ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%COL_INFO%SND_P
-    call mem_split_yx_real(src, s1, s2, s3, work1, dims(1), &
-         decomp%y1dist, decomp)
+     work1_p = decomp%COL_INFO%SND_P
+     call mem_split_yx_real(src, s1, s2, s3, work1, dims(1), &
+                            decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
+                            decomp%y1dist, decomp)
 #else
-    call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
+                            decomp%y1dist, decomp)
 #endif
 
 #endif
 
-    ! define receive buffer
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%COL_INFO%RCV_P
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%COL_INFO%RCV_P
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
-    ! transpose using MPI_ALLTOALL(V)
+
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-    if (decomp%COL_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
-            real_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
-            real_type, decomp%COL_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%COL_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
+                           real_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
+                           real_type, decomp%COL_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    call MPI_ALLTOALL(work1_r, decomp%y1count, &
-         real_type, work2_r, decomp%x1count, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_r, decomp%y1count, &
+                       real_type, work2_r, decomp%x1count, &
+                       real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    nccl_stat = ncclGroupStart()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-    do col_rank_id = 0, (col_comm_size - 1)
-     nccl_stat = ncclSend(work1_r_d( decomp%y1disp(col_rank_id)+1 ), decomp%y1cnts(col_rank_id), &
-       ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-     nccl_stat = ncclRecv(work2_r_d( decomp%x1disp(col_rank_id)+1 ), decomp%x1cnts(col_rank_id), &
-       ncclDouble, local_to_global_col(col_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-    end do
-    nccl_stat = ncclGroupEnd()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     nccl_stat = ncclGroupStart()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+     do col_rank_id = 0, (col_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
+                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
+        nccl_stat = ncclRecv(work2_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
+                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
+     end do
+     nccl_stat = ncclGroupEnd()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-    call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
-         real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
+                        real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-    call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
-         real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
-         real_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
+                        real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_yx_real(work2, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_yx_real(work2, d1, d2, d3, dst, dims(1), &
+                            decomp%x1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+                            decomp%x1dist, decomp)
 #else
-    call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
+                            decomp%x1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
 #endif
 
-    return
+     return
   end subroutine transpose_y_to_x_real
 
-
   subroutine transpose_y_to_x_complex_short(src, dst)
 
-    implicit none
+     implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_y_to_x(src, dst, decomp_main)
+     call transpose_y_to_x(src, dst, decomp_main)
 
   end subroutine transpose_y_to_x_complex_short
 
   subroutine transpose_y_to_x_complex(src, dst, decomp)
 
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #ifdef SHM
-    complex(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     complex(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
-    
-    ! rearrange source array as send buffer
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
+
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%COL_INFO%SND_P_c
-    call mem_split_yx_complex(src, s1, s2, s3, work1, dims(1), &
-         decomp%y1dist, decomp)
+     work1_p = decomp%COL_INFO%SND_P_c
+     call mem_split_yx_complex(src, s1, s2, s3, work1, dims(1), &
+                               decomp%y1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+                               decomp%y1dist, decomp)
 #else
-    call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
-         decomp%y1dist, decomp)
+     call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
+                               decomp%y1dist, decomp)
 #endif
 
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%COL_INFO%RCV_P_c
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%COL_INFO%RCV_P_c
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
-    ! transpose using MPI_ALLTOALL(V)
+
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef SHM
-    if (decomp%COL_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
-            complex_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
-            complex_type, decomp%COL_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%COL_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
+                           complex_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
+                           complex_type, decomp%COL_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    call MPI_ALLTOALL(work1_c, decomp%y1count, &
-         complex_type, work2_c, decomp%x1count, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_c, decomp%y1count, &
+                       complex_type, work2_c, decomp%x1count, &
+                       complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-    call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
-         complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-    call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
-         complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
-         complex_type, DECOMP_2D_COMM_COL, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_yx_complex(work2, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_yx_complex(work2, d1, d2, d3, dst, dims(1), &
+                               decomp%x1dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+                               decomp%x1dist, decomp)
 #else
-    call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
-         decomp%x1dist, decomp)
+     call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
+                               decomp%x1dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
 #endif
 
-    return
+     return
   end subroutine transpose_y_to_x_complex
 
+  subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_yx_real(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    real(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y1disp_o(m) + 1
+        pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y1count + 1
+        pos = m*decomp%y1count + 1
 #else
-       pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_yx_real
 
+  subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_yx_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    complex(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y1disp_o(m) + 1
+        pos = decomp%y1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y1count + 1
+        pos = m*decomp%y1count + 1
 #else
-       pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_yx_complex
 
+  subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_yx_real(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(*), intent(IN) :: in
-    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%x1disp_o(m) + 1
+        pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%x1count + 1
+        pos = m*decomp%x1count + 1
 #else
-       pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=1,n2
-             do i=i1,i2
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_yx_real
 
+  subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_yx_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(*), intent(IN) :: in
-    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%x1disp_o(m) + 1
+        pos = decomp%x1disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%x1count + 1
+        pos = m*decomp%x1count + 1
 #else
-       pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=1,n2
-             do i=i1,i2
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_yx_complex
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 70f427cf..62bc9071 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,488 +13,484 @@
 
   subroutine transpose_y_to_z_real_short(src, dst)
 
-    implicit none
+     implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_y_to_z(src, dst, decomp_main)
+     call transpose_y_to_z(src, dst, decomp_main)
 
   end subroutine transpose_y_to_z_real_short
 
   subroutine transpose_y_to_z_real(src, dst, decomp)
 
-    implicit none
-    
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    type(ncclResult) :: nccl_stat
-    integer :: row_rank_id, cuda_stat
+     type(ncclResult) :: nccl_stat
+     integer :: row_rank_id, cuda_stat
 #endif
-    integer :: istat
+     integer :: istat
 #endif
 
 #ifdef SHM
-    real(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     real(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-    ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%ROW_INFO%SND_P
-    call mem_split_yz_real(src, s1, s2, s3, work1, dims(2), &
-         decomp%y2dist, decomp)
+     work1_p = decomp%ROW_INFO%SND_P
+     call mem_split_yz_real(src, s1, s2, s3, work1, dims(2), &
+                            decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
+                            decomp%y2dist, decomp)
 #else
-    call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
+                            decomp%y2dist, decomp)
 #endif
 
 #endif
 
-    ! define receive buffer
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%ROW_INFO%RCV_P
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%ROW_INFO%RCV_P
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
+
 #ifdef SHM
-    if (decomp%ROW_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
-            real_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
-            real_type, decomp%ROW_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%ROW_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
+                           real_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
+                           real_type, decomp%ROW_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    if (decomp%even) then
-       call MPI_ALLTOALL(work1_r, decomp%y2count, &
-            real_type, dst, decomp%z2count, &
-            real_type, DECOMP_2D_COMM_ROW, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    else
-       call MPI_ALLTOALL(work1_r, decomp%y2count, &
-            real_type, work2_r, decomp%z2count, &
-            real_type, DECOMP_2D_COMM_ROW, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
-    end if
+     if (decomp%even) then
+        call MPI_ALLTOALL(work1_r, decomp%y2count, &
+                          real_type, dst, decomp%z2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     else
+        call MPI_ALLTOALL(work1_r, decomp%y2count, &
+                          real_type, work2_r, decomp%z2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     end if
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    nccl_stat = ncclGroupStart()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-    do row_rank_id = 0, (row_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d( decomp%y2disp(row_rank_id)+1 ), decomp%y2cnts(row_rank_id), &
-          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+     nccl_stat = ncclGroupStart()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+     do row_rank_id = 0, (row_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
+                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d( decomp%z2disp(row_rank_id)+1 ), decomp%z2cnts(row_rank_id), &
-          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+        nccl_stat = ncclRecv(work2_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
+                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-    end do
-    nccl_stat = ncclGroupEnd()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     end do
+     nccl_stat = ncclGroupEnd()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-    call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
-         real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
-         real_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
+                        real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-    call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
-         real_type, dst, decomp%z2cnts, decomp%z2disp, &
-         real_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
+                        real_type, dst, decomp%z2cnts, decomp%z2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_yz_real(work2, d1, d2, d3, dst, dims(2), &
-         decomp%z2dist, decomp)
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_yz_real(work2, d1, d2, d3, dst, dims(2), &
+                            decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-    if (.not. decomp%even) then
-       call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
-         decomp%z2dist, decomp)
-    end if
+     if (.not. decomp%even) then
+        call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
+                               decomp%z2dist, decomp)
+     end if
 #else
-    ! note the receive buffer is already in natural (i,j,k) order
-    ! so no merge operation needed
+     ! note the receive buffer is already in natural (i,j,k) order
+     ! so no merge operation needed
 
 #if defined(_GPU)
-    istat = cudaMemcpy( dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
+     istat = cudaMemcpy(dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
 #endif
 
 #endif
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
 #endif
 
-    return
+     return
   end subroutine transpose_y_to_z_real
 
-
   subroutine transpose_y_to_z_complex_short(src, dst)
 
-    implicit none
+     implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_y_to_z(src, dst, decomp_main)
+     call transpose_y_to_z(src, dst, decomp_main)
 
   end subroutine transpose_y_to_z_complex_short
 
   subroutine transpose_y_to_z_complex(src, dst, decomp)
 
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-    integer :: istat
+     integer :: istat
 #endif
 
 #ifdef SHM
-    complex(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     complex(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
-    
-    ! rearrange source array as send buffer
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
+
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%ROW_INFO%SND_P_c
-    call mem_split_yz_complex(src, s1, s2, s3, work1, dims(2), &
-         decomp%y2dist, decomp)
+     work1_p = decomp%ROW_INFO%SND_P_c
+     call mem_split_yz_complex(src, s1, s2, s3, work1, dims(2), &
+                               decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
+                               decomp%y2dist, decomp)
 #else
-    call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
+                               decomp%y2dist, decomp)
 #endif
 
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%ROW_INFO%RCV_P_c
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%ROW_INFO%RCV_P_c
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
+
 #ifdef SHM
-    if (decomp%ROW_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
-            complex_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
-            complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%ROW_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
+                           complex_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
+                           complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    if (decomp%even) then
-       call MPI_ALLTOALL(work1_c, decomp%y2count, &
-            complex_type, dst, decomp%z2count, &
-            complex_type, DECOMP_2D_COMM_ROW, ierror)
-    else
-       call MPI_ALLTOALL(work1_c, decomp%y2count, &
-            complex_type, work2_c, decomp%z2count, &
-            complex_type, DECOMP_2D_COMM_ROW, ierror)
-    end if
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     if (decomp%even) then
+        call MPI_ALLTOALL(work1_c, decomp%y2count, &
+                          complex_type, dst, decomp%z2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     else
+        call MPI_ALLTOALL(work1_c, decomp%y2count, &
+                          complex_type, work2_c, decomp%z2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-    call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
-         complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
-         complex_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-    call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
-         complex_type, dst, decomp%z2cnts, decomp%z2disp, &
-         complex_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, dst, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_yz_complex(work2, d1, d2, d3, dst, dims(2), &
-         decomp%z2dist, decomp)
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_yz_complex(work2, d1, d2, d3, dst, dims(2), &
+                               decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-    if (.not. decomp%even) then
-       call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
-         decomp%z2dist, decomp)
-    end if
+     if (.not. decomp%even) then
+        call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
+                                  decomp%z2dist, decomp)
+     end if
 #else
-    ! note the receive buffer is already in natural (i,j,k) order
-    ! so no merge operation needed
+     ! note the receive buffer is already in natural (i,j,k) order
+     ! so no merge operation needed
 
 #if defined(_GPU)
-    istat = cudaMemcpy( dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
+     istat = cudaMemcpy(dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
 #endif
 
 #endif
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
 #endif
 
-    return
+     return
   end subroutine transpose_y_to_z_complex
 
-  subroutine mem_split_yz_real(in,n1,n2,n3,out,iproc,dist,decomp)
+  subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    real(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y2disp_o(m) + 1
+        pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y2count + 1
+        pos = m*decomp%y2count + 1
 #else
-       pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_yz_real
 
+  subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_yz_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    complex(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: out
-    integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y2disp_o(m) + 1
+        pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y2count + 1
+        pos = m*decomp%y2count + 1
 #else
-       pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_split_yz_complex
 
+  subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
+
+     implicit none
+
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-  subroutine mem_merge_yz_real(in,n1,n2,n3,out,iproc,dist,decomp)
-
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(*), intent(IN) :: in
-    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
-    integer :: i,j,k, m,i1,i2, pos
-
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%z2disp_o(m) + 1
+        pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%z2count + 1
+        pos = m*decomp%z2count + 1
 #else
-       pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-       do k=i1,i2
-          do j=1,n2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-    end do
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
 
-    return
+     return
   end subroutine mem_merge_yz_real
 
+  subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+
+     implicit none
+
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+
+     integer :: i, j, k, m, i1, i2, pos
 
-  subroutine mem_merge_yz_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-    
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(*), intent(IN) :: in
-    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
-    integer :: i,j,k, m,i1,i2, pos
-
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%z2disp_o(m) + 1
+        pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%z2count + 1
+        pos = m*decomp%z2count + 1
 #else
-       pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-       do k=i1,i2
-          do j=1,n2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-    end do
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
 
-    return
+     return
   end subroutine mem_merge_yz_complex
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index 33030a7d..a2f86844 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -1,7 +1,7 @@
 !=======================================================================
 ! This is part of the 2DECOMP&FFT library
-! 
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil) 
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
 ! decomposition. It also implements a highly scalable distributed
 ! three-dimensional Fast Fourier Transform (FFT).
 !
@@ -13,490 +13,485 @@
 
   subroutine transpose_z_to_y_real_short(src, dst)
 
-    implicit none
+     implicit none
 
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_z_to_y(src, dst, decomp_main)
+     call transpose_z_to_y(src, dst, decomp_main)
 
   end subroutine transpose_z_to_y_real_short
 
   subroutine transpose_z_to_y_real(src, dst, decomp)
 
-    implicit none
-    
-    real(mytype), dimension(:,:,:), intent(IN) :: src
-    real(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    type(ncclResult) :: nccl_stat
-    integer :: row_rank_id, cuda_stat
+     type(ncclResult) :: nccl_stat
+     integer :: row_rank_id, cuda_stat
 #endif
-    integer :: istat
+     integer :: istat
 #endif
 
 #ifdef SHM
-    real(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     real(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-    ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%ROW_INFO%SND_P
-    call mem_split_zy_real(src, s1, s2, s3, work1, dims(2), &
-         decomp%z2dist, decomp)
+     work1_p = decomp%ROW_INFO%SND_P
+     call mem_split_zy_real(src, s1, s2, s3, work1, dims(2), &
+                            decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-    if (.not. decomp%even) then
-       call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
-            decomp%z2dist, decomp)
-    end if
+     if (.not. decomp%even) then
+        call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
+                               decomp%z2dist, decomp)
+     end if
 #else
-    ! note the src array is suitable to be a send buffer
-    ! so no split operation needed
+     ! note the src array is suitable to be a send buffer
+     ! so no split operation needed
 
 #if defined(_GPU)
-    istat = cudaMemcpy( work1_r_d, src, s1*s2*s3 )
+     istat = cudaMemcpy(work1_r_d, src, s1*s2*s3)
 #endif
 
 #endif
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%ROW_INFO%RCV_P
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%ROW_INFO%RCV_P
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
+
 #ifdef SHM
-    if (decomp%ROW_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
-            real_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
-            real_type, decomp%ROW_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%ROW_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
+                           real_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
+                           real_type, decomp%ROW_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    if (decomp%even) then
-       call MPI_ALLTOALL(src, decomp%z2count, &
-            real_type, work2_r, decomp%y2count, &
-            real_type, DECOMP_2D_COMM_ROW, ierror)
-    else
-       call MPI_ALLTOALL(work1_r, decomp%z2count, &
-            real_type, work2_r, decomp%y2count, &
-            real_type, DECOMP_2D_COMM_ROW, ierror)
-    end if
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     if (decomp%even) then
+        call MPI_ALLTOALL(src, decomp%z2count, &
+                          real_type, work2_r, decomp%y2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+     else
+        call MPI_ALLTOALL(work1_r, decomp%z2count, &
+                          real_type, work2_r, decomp%y2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-    nccl_stat = ncclGroupStart()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-    do row_rank_id = 0, (row_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d( decomp%z2disp(row_rank_id)+1 ), decomp%z2cnts(row_rank_id), &
-          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+     nccl_stat = ncclGroupStart()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+     do row_rank_id = 0, (row_comm_size - 1)
+        nccl_stat = ncclSend(work1_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
+                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d( decomp%y2disp(row_rank_id)+1 ), decomp%y2cnts(row_rank_id), &
-          ncclDouble, local_to_global_row(row_rank_id+1), nccl_comm_2decomp, cuda_stream_2decomp)
+        nccl_stat = ncclRecv(work2_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
+                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-    end do
-    nccl_stat = ncclGroupEnd()
-    if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-    cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-    if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     end do
+     nccl_stat = ncclGroupEnd()
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
+     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 #else
-    call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
-         real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
-         real_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
+                        real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-    call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-         real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
-         real_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+                        real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_zy_real(work2, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_zy_real(work2, d1, d2, d3, dst, dims(2), &
+                            decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
+                            decomp%y2dist, decomp)
 #else
-    call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
+                            decomp%y2dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
 #endif
 
-    return
+     return
   end subroutine transpose_z_to_y_real
 
-
   subroutine transpose_z_to_y_complex_short(src, dst)
 
-    implicit none
+     implicit none
 
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-    call transpose_z_to_y(src, dst, decomp_main)
+     call transpose_z_to_y(src, dst, decomp_main)
 
   end subroutine transpose_z_to_y_complex_short
 
   subroutine transpose_z_to_y_complex(src, dst, decomp)
 
-    implicit none
-    
-    complex(mytype), dimension(:,:,:), intent(IN) :: src
-    complex(mytype), dimension(:,:,:), intent(OUT) :: dst
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-    integer :: istat
+     integer :: istat
 #endif
 
 #ifdef SHM
-    complex(mytype) :: work1(*), work2(*)
-    POINTER  (work1_p, work1), (work2_p, work2)  ! Cray pointers
+     complex(mytype) :: work1(*), work2(*)
+     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
 #endif
-    
-    integer :: s1,s2,s3,d1,d2,d3
-    integer :: ierror
+
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
 #endif
 
-    s1 = SIZE(src,1)
-    s2 = SIZE(src,2)
-    s3 = SIZE(src,3)
-    d1 = SIZE(dst,1)
-    d2 = SIZE(dst,2)
-    d3 = SIZE(dst,3)
-    
-    ! rearrange source array as send buffer
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
+
+     ! rearrange source array as send buffer
 #ifdef SHM
-    work1_p = decomp%ROW_INFO%SND_P_c
-    call mem_split_zy_complex(src, s1, s2, s3, work1, dims(2), &
-         decomp%z2dist, decomp)
+     work1_p = decomp%ROW_INFO%SND_P_c
+     call mem_split_zy_complex(src, s1, s2, s3, work1, dims(2), &
+                               decomp%z2dist, decomp)
 #else
 #ifdef EVEN
-    if (.not. decomp%even) then
-       call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
-            decomp%z2dist, decomp)
-    end if
+     if (.not. decomp%even) then
+        call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
+                                  decomp%z2dist, decomp)
+     end if
 #else
-    ! note the src array is suitable to be a send buffer
-    ! so no split operation needed
+     ! note the src array is suitable to be a send buffer
+     ! so no split operation needed
 
 #if defined(_GPU)
-    istat = cudaMemcpy( work1_c_d, src, s1*s2*s3 )
+     istat = cudaMemcpy(work1_c_d, src, s1*s2*s3)
 #endif
 
 #endif
 #endif
-    
-    ! define receive buffer
+
+     ! define receive buffer
 #ifdef SHM
-    work2_p = decomp%ROW_INFO%RCV_P_c
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     work2_p = decomp%ROW_INFO%RCV_P_c
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
 #endif
-    
+
 #ifdef SHM
-    if (decomp%ROW_INFO%CORE_ME==1) THEN
-       call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
-            complex_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
-            complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
-       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-    end if
+     if (decomp%ROW_INFO%CORE_ME == 1) THEN
+        call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
+                           complex_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
+                           complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     end if
 #else
 #ifdef EVEN
-    if (decomp%even) then
-       call MPI_ALLTOALL(src, decomp%z2count, &
-            complex_type, work2_c, decomp%y2count, &
-            complex_type, DECOMP_2D_COMM_ROW, ierror)
-    else
-       call MPI_ALLTOALL(work1_c, decomp%z2count, &
-            complex_type, work2_c, decomp%y2count, &
-            complex_type, DECOMP_2D_COMM_ROW, ierror)
-    end if
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     if (decomp%even) then
+        call MPI_ALLTOALL(src, decomp%z2count, &
+                          complex_type, work2_c, decomp%y2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     else
+        call MPI_ALLTOALL(work1_c, decomp%z2count, &
+                          complex_type, work2_c, decomp%y2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
-    call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
-         complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
-         complex_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #else
-    call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-         complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
-         complex_type, DECOMP_2D_COMM_ROW, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #endif
 
 #endif
 #endif
 
-    ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef SHM
-    call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-    call mem_merge_zy_complex(work2, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
+     call mem_merge_zy_complex(work2, d1, d2, d3, dst, dims(2), &
+                               decomp%y2dist, decomp)
 #else
 
 #if defined(_GPU)
-    call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
+                               decomp%y2dist, decomp)
 #else
-    call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
-         decomp%y2dist, decomp)
+     call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
+                               decomp%y2dist, decomp)
 #endif
 
 #endif
 
 #ifdef PROFILER
-    if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
 #endif
 
-    return
+     return
   end subroutine transpose_z_to_y_complex
 
-
   ! pack/unpack ALLTOALL(V) buffers
 
-  subroutine mem_split_zy_real(in,n1,n2,n3,out,iproc,dist,decomp)
+  subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-    implicit none
+     implicit none
 
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    real(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
-    integer :: i,j,k, m,i1,i2,pos
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%z2disp_o(m) + 1
+        pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%z2count + 1
+        pos = m*decomp%z2count + 1
 #else
-       pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-       do k=i1,i2
-          do j=1,n2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-    end do
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
 
-    return
+     return
   end subroutine mem_split_zy_real
 
+  subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_split_zy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(n1,n2,n3), intent(IN) :: in
-    complex(mytype), dimension(*), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
-    integer :: i,j,k, m,i1,i2,pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then 
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%z2disp_o(m) + 1
+        pos = decomp%z2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%z2count + 1
+        pos = m*decomp%z2count + 1
 #else
-       pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 #endif
 
-       do k=i1,i2
-          do j=1,n2
-             do i=1,n1
-                out(pos) = in(i,j,k)
-                pos = pos + 1
-             end do
-          end do
-       end do
-    end do
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
 
-    return
+     return
   end subroutine mem_split_zy_complex
 
+  subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-  subroutine mem_merge_zy_real(in,n1,n2,n3,out,iproc,dist,decomp)
+     implicit none
 
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    real(mytype), dimension(*), intent(IN) :: in
-    real(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y2disp_o(m) + 1
+        pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y2count + 1
+        pos = m*decomp%y2count + 1
 #else
-       pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_zy_real
 
+  subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+
+     implicit none
+
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-  subroutine mem_merge_zy_complex(in,n1,n2,n3,out,iproc,dist,decomp)
-    
-    implicit none
-    
-    integer, intent(IN) :: n1,n2,n3
-    complex(mytype), dimension(*), intent(IN) :: in
-    complex(mytype), dimension(n1,n2,n3), intent(OUT) :: out
-    integer, intent(IN) :: iproc
-    integer, dimension(0:iproc-1), intent(IN) :: dist
-    TYPE(DECOMP_INFO), intent(IN) :: decomp
-    
 #if defined(_GPU)
-    attributes(device) :: in
-    integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-    integer :: i,j,k, m,i1,i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-    do m=0,iproc-1
-       if (m==0) then
-          i1 = 1
-          i2 = dist(0)
-       else
-          i1 = i2+1
-          i2 = i1+dist(m)-1
-       end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef SHM
-       pos = decomp%y2disp_o(m) + 1
+        pos = decomp%y2disp_o(m) + 1
 #else
 #ifdef EVEN
-       pos = m * decomp%y2count + 1
+        pos = m*decomp%y2count + 1
 #else
-       pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 #endif
 
 #if defined(_GPU)
-       istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-       do k=1,n3
-          do j=i1,i2
-             do i=1,n1
-                out(i,j,k) = in(pos)
-                pos = pos + 1
-             end do
-          end do
-       end do
-#endif
-    end do
-
-    return
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+#endif
+     end do
+
+     return
   end subroutine mem_merge_zy_complex

From d93b241d1d7f536baa500ff50e0d525e143e1cc8 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Tue, 13 Dec 2022 17:56:27 +0100
Subject: [PATCH 030/436] Describe preprocessor variables in the README

---
 README.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/README.md b/README.md
index a4ccea56..66a95cc5 100644
--- a/README.md
+++ b/README.md
@@ -89,6 +89,56 @@ The default value used is `D2D_LOG_TOFILE` for the default build and `D2D_LOG_TO
 
 Before calling `decomp_2d_init`, the external code can modify the variable `decomp_debug` to change the debug level. The user can also modify this variable using the environment variable `DECOMP_2D_DEBUG`. Please note that the environment variable is processed only for debug builds. The expected value for the variable `decomp_debug` is some integer between 0 and 6, bounds included.
 
+### List of preprocessor variables
+
+#### DEBUG
+
+This variable is automatically added in debug and dev builds. Extra information is printed when it is present.
+
+#### DOUBLE_PREC
+
+When this variable is not present, the library uses single precision. When it is present, the library uses double precision
+
+#### SAVE_SINGLE
+
+This variable is valid for double precision builds only. When it is present, snapshots are written in single precision.
+
+#### PROFILER
+
+This variable is automatically added when selecting the profiler. It activates the profiling sectionsof the code.
+
+#### EVEN
+
+This preprocessor variable is not valid for GPU builds. It leads to padded alltoall operations.
+
+#### OVERWRITE
+
+This variable is not supported by the MKL FFT backend. It leads to overwrite the input array when computing FFT (complex-to-complex and complex-to-real).
+
+#### HALO_DEBUG
+
+This variable is used to debug the halo operations.
+
+#### _GPU
+
+This variable is automatically added in GPU builds.
+
+#### _NCCL
+
+This variable is valid only for GPU builds. The NVIDIA Collective Communication Library (NCCL) implements multi-GPU and multi-node communication primitives optimized for NVIDIA GPUs and Networking.
+
+#### OCC
+
+This variable is not supported
+
+#### SHM
+
+This variable is not supported.
+
+#### SHM_DEBUG
+
+This variable is not supported
+
 ## Optional dependencies
 
 ### FFTW

From fc10d27128f765248ca3da3b13c5d7f276b4dc9a Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Dec 2022 07:02:54 +0100
Subject: [PATCH 031/436] Add support for OVERWRITE in the MKL FFT backend

---
 src/fft_mkl.f90 | 151 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 149 insertions(+), 2 deletions(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index cebed05c..ba0f580f 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -94,7 +94,11 @@ subroutine c2c_1m_x_plan(desc, decomp)
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
                             decomp%xsz(2)*decomp%xsz(3))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
+#ifdef OVERWRITE
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_INPLACE)
+#else
       status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+#endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, decomp%xsz(1))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
@@ -127,7 +131,11 @@ subroutine c2c_1m_y_plan(desc, decomp)
                                     DFTI_COMPLEX, 1, decomp%ysz(2))
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+#ifdef OVERWRITE
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_INPLACE)
+#else
       status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+#endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, decomp%ysz(1))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
@@ -165,7 +173,11 @@ subroutine c2c_1m_z_plan(desc, decomp)
                                     DFTI_COMPLEX, 1, decomp%zsz(3))
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
+#ifdef OVERWRITE
+      status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_INPLACE)
+#else
       status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
+#endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
                             decomp%zsz(1)*decomp%zsz(2))
@@ -348,20 +360,34 @@ subroutine fft_3d_c2c(in, out, isign)
           format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
 
          ! ===== 1D FFTs in X =====
-         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
          !       if (isign==DECOMP_2D_FFT_FORWARD) then
          !          status = DftiComputeForward(c2c_x, in(:,1,1), wk1(:,1,1))
          !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
          !          status = DftiComputeBackward(c2c_x, in(:,1,1), wk1(:,1,1))
          !       end if
+#ifdef OVERWRITE
+         status = wrapper_c2c_inplace(c2c_x, in, isign)
+#else
+         allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
          status = wrapper_c2c(c2c_x, in, wk1, isign)
+#endif
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
          ! ===== Swap X --> Y =====
          allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+#ifdef OVERWRITE
+         call transpose_x_to_y(in, wk2, ph)
+#else
          call transpose_x_to_y(wk1, wk2, ph)
+#endif
 
          ! ===== 1D FFTs in Y =====
+#ifdef OVERWRITE
+         do k = 1, ph%xsz(3)
+            status = wrapper_c2c_inplace(c2c_y, wk2(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+#else
          allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
          do k = 1, ph%xsz(3) ! one Z-plane at a time
             !          if (isign==DECOMP_2D_FFT_FORWARD) then
@@ -372,10 +398,15 @@ subroutine fft_3d_c2c(in, out, isign)
             status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
+#endif
 
          ! ===== Swap Y --> Z =====
+#ifdef OVERWRITE
+         call transpose_y_to_z(wk2, out, ph)
+#else
          allocate (wk3(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
          call transpose_y_to_z(wk2b, wk3, ph)
+#endif
 
          ! ===== 1D FFTs in Z =====
          !       if (isign==DECOMP_2D_FFT_FORWARD) then
@@ -383,7 +414,11 @@ subroutine fft_3d_c2c(in, out, isign)
          !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
          !          status = DftiComputeBackward(c2c_z, wk3(:,1,1), out(:,1,1))
          !       end if
+#ifdef OVERWRITE
+         status = wrapper_c2c_inplace(c2c_z, out, isign)
+#else
          status = wrapper_c2c(c2c_z, wk3, out, isign)
+#endif
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
       else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
@@ -391,20 +426,34 @@ subroutine fft_3d_c2c(in, out, isign)
                format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_FORWARD) then
 
          ! ===== 1D FFTs in Z =====
-         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
          !       if (isign==DECOMP_2D_FFT_FORWARD) then
          !          status = DftiComputeForward(c2c_z, in(:,1,1), wk1(:,1,1))
          !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
          !          status = DftiComputeBackward(c2c_z, in(:,1,1), wk1(:,1,1))
          !       end if
+#ifdef OVERWRITE
+         status = wrapper_c2c_inplace(c2c_z, in, isign)
+#else
+         allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
          status = wrapper_c2c(c2c_z, in, wk1, isign)
+#endif
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
          ! ===== Swap Z --> Y =====
          allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
+#ifdef OVERWRITE
+         call transpose_z_to_y(in, wk2, ph)
+#else
          call transpose_z_to_y(wk1, wk2, ph)
+#endif
 
          ! ===== 1D FFTs in Y =====
+#ifdef OVERWRITE
+         do k = 1, ph%xsz(3)
+            status = wrapper_c2c_inplace(c2c_y, wk2(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+#else
          allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
          do k = 1, ph%xsz(3) ! one Z-plane at a time
             !          if (isign==DECOMP_2D_FFT_FORWARD) then
@@ -415,10 +464,15 @@ subroutine fft_3d_c2c(in, out, isign)
             status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
+#endif
 
          ! ===== Swap Y --> X =====
+#ifdef OVERWRITE
+         call transpose_y_to_x(wk2, out, ph)
+#else
          allocate (wk3(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
          call transpose_y_to_x(wk2b, wk3, ph)
+#endif
 
          ! ===== 1D FFTs in X =====
          !       if (isign==DECOMP_2D_FFT_FORWARD) then
@@ -426,7 +480,11 @@ subroutine fft_3d_c2c(in, out, isign)
          !       else if (isign==DECOMP_2D_FFT_BACKWARD) then
          !          status = DftiComputeBackward(c2c_x, wk3(:,1,1), out(:,1,1))
          !       end if
+#ifdef OVERWRITE
+         status = wrapper_c2c_inplace(c2c_x, out, isign)
+#else
          status = wrapper_c2c(c2c_x, wk3, out, isign)
+#endif
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
       end if
@@ -470,20 +528,35 @@ subroutine fft_3d_r2c(in_r, out_c)
          call transpose_x_to_y(wk1, wk2, sp)
 
          ! ===== 1D FFTs in Y =====
+#ifdef OVERWRITE
+         do k = 1, sp%ysz(3)
+            status = wrapper_c2c_inplace(c2c_y2, wk2(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+#else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
             status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
+#endif
 
          ! ===== Swap Y --> Z =====
+#ifdef OVERWRITE
+         call transpose_y_to_z(wk2, out_c, sp)
+#else
          allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
          call transpose_y_to_z(wk2b, wk3, sp)
+#endif
 
          ! ===== 1D FFTs in Z =====
          !       status = DftiComputeForward(c2c_z2, wk3(:,1,1), out_c(:,1,1))
+#ifdef OVERWRITE
+         status = wrapper_c2c_inplace(c2c_z2, out_c, isign)
+#else
          status = wrapper_c2c(c2c_z2, wk3, out_c, isign)
+#endif
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
       else if (format == PHYSICAL_IN_Z) then
@@ -499,20 +572,35 @@ subroutine fft_3d_r2c(in_r, out_c)
          call transpose_z_to_y(wk1, wk2, sp)
 
          ! ===== 1D FFTs in Y =====
+#ifdef OVERWRITE
+         do k = 1, sp%ysz(3)
+            status = wrapper_c2c_inplace(c2c_y2, wk2(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+#else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
             status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
+#endif
 
          ! ===== Swap Y --> X =====
+#ifdef OVERWRITE
+         call transpose_y_to_x(wk2, out_c, sp)
+#else
          allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
          call transpose_y_to_x(wk2b, wk3, sp)
+#endif
 
          ! ===== 1D FFTs in X =====
          !       status = DftiComputeForward(c2c_x2, wk3(:,1,1), out_c(:,1,1))
+#ifdef OVERWRITE
+         status = wrapper_c2c_inplace(c2c_x2, out_c, isign)
+#else
          status = wrapper_c2c(c2c_x2, wk3, out_c, isign)
+#endif
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
       end if
@@ -546,26 +634,45 @@ subroutine fft_3d_c2r(in_c, out_r)
       if (format == PHYSICAL_IN_X) then
 
          ! ===== 1D FFTs in Z =====
+#ifdef OVERWRITE
+         status = wrapper_c2c_inplace(c2c_z2, in_c, isign)
+#else
          allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
          !       status = DftiComputeBackward(c2c_z2, in_c(:,1,1), wk1(:,1,1))
          status = wrapper_c2c(c2c_z2, in_c, wk1, isign)
+#endif
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
          ! ===== Swap Z --> Y =====
          allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+#ifdef OVERWRITE
+         call transpose_z_to_y(in_c, wk2, sp)
+#else
          call transpose_z_to_y(wk1, wk2, sp)
+#endif
 
          ! ===== 1D FFTs in Y =====
+#ifdef OVERWRITE
+         do k = 1, sp%ysz(3)
+            status = wrapper_c2c_inplace(c2c_y2, wk2(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+#else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
             status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
+#endif
 
          ! ===== Swap Y --> X =====
          allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+#ifdef OVERWRITE
+         call transpose_y_to_x(wk2, wk3, sp)
+#else
          call transpose_y_to_x(wk2b, wk3, sp)
+#endif
 
          ! ===== 1D FFTs in X =====
          !       status = DftiComputeBackward(c2r_x, wk3(:,1,1), out_r(:,1,1))
@@ -575,26 +682,45 @@ subroutine fft_3d_c2r(in_c, out_r)
       else if (format == PHYSICAL_IN_Z) then
 
          ! ===== 1D FFTs in X =====
+#ifdef OVERWRITE
+         status = wrapper_c2c_inplace(c2c_x2, in_c, isign)
+#else
          allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
          !       status = DftiComputeBackward(c2c_x2, in_c(:,1,1), wk1(:,1,1))
          status = wrapper_c2c(c2c_x2, in_c, wk1, isign)
+#endif
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
          ! ===== Swap X --> Y =====
          allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
+#ifdef OVERWRITE
+         call transpose_x_to_y(in_c, wk2, sp)
+#else
          call transpose_x_to_y(wk1, wk2, sp)
+#endif
 
          ! ===== 1D FFTs in Y =====
+#ifdef OVERWRITE
+         do k = 1, sp%ysz(3)
+            status = wrapper_c2c_inplace(c2c_y2, wk2(1, 1, k), isign)
+            if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
+         end do
+#else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
             status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
+#endif
 
          ! ===== Swap Y --> Z =====
          allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+#ifdef OVERWRITE
+         call transpose_y_to_z(wk2, wk3, sp)
+#else
          call transpose_y_to_z(wk2b, wk3, sp)
+#endif
 
          ! ===== 1D FFTs in Z =====
          !       status = DftiComputeBackward(c2r_z, wk3(:,1,1), out_r(:,1,1))
@@ -643,6 +769,27 @@ integer function wrapper_c2c(desc, in, out, isign)
       return
    end function wrapper_c2c
 
+#ifdef OVERWRITE
+   integer function wrapper_c2c_inplace(desc, inout, isign)
+
+      implicit none
+
+      type(DFTI_DESCRIPTOR), pointer :: desc
+      complex(mytype), dimension(*) :: inout
+      integer :: isign, status
+
+      if (isign == DECOMP_2D_FFT_FORWARD) then
+         status = DftiComputeForward(desc, inout)
+      else if (isign == DECOMP_2D_FFT_BACKWARD) then
+         status = DftiComputeBackward(desc, inout)
+      end if
+
+      wrapper_c2c = status
+
+      return
+   end function wrapper_c2c_inplace
+#endif
+
    integer function wrapper_r2c(desc, in, out)
 
       implicit none

From 4a4e69e5fb6352fd3d5ac2836677f4a435f5c933 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Dec 2022 07:08:33 +0100
Subject: [PATCH 032/436] MKL : array wk2 is not needed for c2c transform

---
 src/fft_mkl.f90 | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index ba0f580f..65e74087 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -349,7 +349,7 @@ subroutine fft_3d_c2c(in, out, isign)
       complex(mytype), dimension(:, :, :), intent(OUT) :: out
       integer, intent(IN) :: isign
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2b, wk3
       integer :: k, status
 
 #ifdef PROFILER
@@ -374,17 +374,16 @@ subroutine fft_3d_c2c(in, out, isign)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
          ! ===== Swap X --> Y =====
-         allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
 #ifdef OVERWRITE
-         call transpose_x_to_y(in, wk2, ph)
+         call transpose_x_to_y(in, wk2_c2c, ph)
 #else
-         call transpose_x_to_y(wk1, wk2, ph)
+         call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, ph%xsz(3)
-            status = wrapper_c2c_inplace(c2c_y, wk2(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y, wk2_c2c(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
@@ -395,14 +394,14 @@ subroutine fft_3d_c2c(in, out, isign)
             !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
             !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
             !          end if
-            status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y, wk2_c2c(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif
 
          ! ===== Swap Y --> Z =====
 #ifdef OVERWRITE
-         call transpose_y_to_z(wk2, out, ph)
+         call transpose_y_to_z(wk2_c2c, out, ph)
 #else
          allocate (wk3(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
          call transpose_y_to_z(wk2b, wk3, ph)
@@ -440,17 +439,16 @@ subroutine fft_3d_c2c(in, out, isign)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
          ! ===== Swap Z --> Y =====
-         allocate (wk2(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
 #ifdef OVERWRITE
-         call transpose_z_to_y(in, wk2, ph)
+         call transpose_z_to_y(in, wk2_c2c, ph)
 #else
-         call transpose_z_to_y(wk1, wk2, ph)
+         call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, ph%xsz(3)
-            status = wrapper_c2c_inplace(c2c_y, wk2(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y, wk2_c2c(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
@@ -461,14 +459,14 @@ subroutine fft_3d_c2c(in, out, isign)
             !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
             !             status = DftiComputeBackward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
             !          end if
-            status = wrapper_c2c(c2c_y, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y, wk2_c2c(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif
 
          ! ===== Swap Y --> X =====
 #ifdef OVERWRITE
-         call transpose_y_to_x(wk2, out, ph)
+         call transpose_y_to_x(wk2_c2c, out, ph)
 #else
          allocate (wk3(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
          call transpose_y_to_x(wk2b, wk3, ph)

From 02a2804f3b109e1944419176e0b97c4410350e22 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Dec 2022 07:11:13 +0100
Subject: [PATCH 033/436] MKL : array wk2 is not needed for r2c transform

---
 src/fft_mkl.f90 | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 65e74087..b5eaf1db 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -504,7 +504,7 @@ subroutine fft_3d_r2c(in_r, out_c)
       real(mytype), dimension(:, :, :), intent(IN) :: in_r
       complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2b, wk3
       integer :: k, status, isign
 
 #ifdef PROFILER
@@ -522,27 +522,26 @@ subroutine fft_3d_r2c(in_r, out_c)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
          ! ===== Swap X --> Y =====
-         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         call transpose_x_to_y(wk1, wk2, sp)
+         call transpose_x_to_y(wk1, wk2_r2c, sp)
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, sp%ysz(3)
-            status = wrapper_c2c_inplace(c2c_y2, wk2(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y2, wk2_r2c(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif
 
          ! ===== Swap Y --> Z =====
 #ifdef OVERWRITE
-         call transpose_y_to_z(wk2, out_c, sp)
+         call transpose_y_to_z(wk2_r2c, out_c, sp)
 #else
          allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
          call transpose_y_to_z(wk2b, wk3, sp)
@@ -566,27 +565,26 @@ subroutine fft_3d_r2c(in_r, out_c)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
          ! ===== Swap Z --> Y =====
-         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
-         call transpose_z_to_y(wk1, wk2, sp)
+         call transpose_z_to_y(wk1, wk2_r2c, sp)
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, sp%ysz(3)
-            status = wrapper_c2c_inplace(c2c_y2, wk2(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y2, wk2_r2c(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif
 
          ! ===== Swap Y --> X =====
 #ifdef OVERWRITE
-         call transpose_y_to_x(wk2, out_c, sp)
+         call transpose_y_to_x(wk2_r2c, out_c, sp)
 #else
          allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
          call transpose_y_to_x(wk2b, wk3, sp)

From 294ea4bf05cb30b01bc28d33333f6bcf776a0eae Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Dec 2022 07:12:58 +0100
Subject: [PATCH 034/436] MKL : array wk2 is not needed for c2r transform

---
 src/fft_mkl.f90 | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index b5eaf1db..19545a7c 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -618,7 +618,7 @@ subroutine fft_3d_c2r(in_c, out_r)
       complex(mytype), dimension(:, :, :), intent(IN) :: in_c
       real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2, wk2b, wk3
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2b, wk3
       integer :: k, status, isign
 
 #ifdef PROFILER
@@ -640,24 +640,23 @@ subroutine fft_3d_c2r(in_c, out_r)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
          ! ===== Swap Z --> Y =====
-         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
 #ifdef OVERWRITE
-         call transpose_z_to_y(in_c, wk2, sp)
+         call transpose_z_to_y(in_c, wk2_r2c, sp)
 #else
-         call transpose_z_to_y(wk1, wk2, sp)
+         call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, sp%ysz(3)
-            status = wrapper_c2c_inplace(c2c_y2, wk2(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y2, wk2_r2c(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif
@@ -665,7 +664,7 @@ subroutine fft_3d_c2r(in_c, out_r)
          ! ===== Swap Y --> X =====
          allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
 #ifdef OVERWRITE
-         call transpose_y_to_x(wk2, wk3, sp)
+         call transpose_y_to_x(wk2_r2c, wk3, sp)
 #else
          call transpose_y_to_x(wk2b, wk3, sp)
 #endif
@@ -688,24 +687,23 @@ subroutine fft_3d_c2r(in_c, out_r)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
 
          ! ===== Swap X --> Y =====
-         allocate (wk2(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
 #ifdef OVERWRITE
-         call transpose_x_to_y(in_c, wk2, sp)
+         call transpose_x_to_y(in_c, wk2_r2c, sp)
 #else
-         call transpose_x_to_y(wk1, wk2, sp)
+         call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, sp%ysz(3)
-            status = wrapper_c2c_inplace(c2c_y2, wk2(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y2, wk2_r2c(1, 1, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif
@@ -713,7 +711,7 @@ subroutine fft_3d_c2r(in_c, out_r)
          ! ===== Swap Y --> Z =====
          allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
 #ifdef OVERWRITE
-         call transpose_y_to_z(wk2, wk3, sp)
+         call transpose_y_to_z(wk2_r2c, wk3, sp)
 #else
          call transpose_y_to_z(wk2b, wk3, sp)
 #endif

From 00be54989d945d72b8cfcef0ffb992178488e715 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Dec 2022 07:15:46 +0100
Subject: [PATCH 035/436] MKL : array wk1 is not needed for r2c transform

---
 src/fft_mkl.f90 | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 19545a7c..2b2fe09b 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -504,7 +504,7 @@ subroutine fft_3d_r2c(in_r, out_c)
       real(mytype), dimension(:, :, :), intent(IN) :: in_r
       complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2b, wk3
+      complex(mytype), allocatable, dimension(:, :, :) :: wk2b, wk3
       integer :: k, status, isign
 
 #ifdef PROFILER
@@ -516,13 +516,12 @@ subroutine fft_3d_r2c(in_r, out_c)
       if (format == PHYSICAL_IN_X) then
 
          ! ===== 1D FFTs in X =====
-         allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
          !       status = DftiComputeForward(r2c_x, in_r(:,1,1), wk1(:,1,1))
-         status = wrapper_r2c(r2c_x, in_r, wk1)
+         status = wrapper_r2c(r2c_x, in_r, wk13)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
          ! ===== Swap X --> Y =====
-         call transpose_x_to_y(wk1, wk2_r2c, sp)
+         call transpose_x_to_y(wk13, wk2_r2c, sp)
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
@@ -559,13 +558,12 @@ subroutine fft_3d_r2c(in_r, out_c)
       else if (format == PHYSICAL_IN_Z) then
 
          ! ===== 1D FFTs in Z =====
-         allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
          !       status = DftiComputeForward(r2c_z, in_r(:,1,1), wk1(:,1,1))
-         status = wrapper_r2c(r2c_z, in_r, wk1)
+         status = wrapper_r2c(r2c_z, in_r, wk13)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_r2c")
 
          ! ===== Swap Z --> Y =====
-         call transpose_z_to_y(wk1, wk2_r2c, sp)
+         call transpose_z_to_y(wk13, wk2_r2c, sp)
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE

From 2e0384c85c88b2f45bcfbd4d3ce65e1f7bad43c6 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Dec 2022 07:17:20 +0100
Subject: [PATCH 036/436] MKL : array wk3 is not needed for c2r transform

---
 src/fft_mkl.f90 | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 2b2fe09b..d5e72525 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -616,7 +616,7 @@ subroutine fft_3d_c2r(in_c, out_r)
       complex(mytype), dimension(:, :, :), intent(IN) :: in_c
       real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
-      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2b, wk3
+      complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2b
       integer :: k, status, isign
 
 #ifdef PROFILER
@@ -660,16 +660,15 @@ subroutine fft_3d_c2r(in_c, out_r)
 #endif
 
          ! ===== Swap Y --> X =====
-         allocate (wk3(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
 #ifdef OVERWRITE
-         call transpose_y_to_x(wk2_r2c, wk3, sp)
+         call transpose_y_to_x(wk2_r2c, wk13, sp)
 #else
-         call transpose_y_to_x(wk2b, wk3, sp)
+         call transpose_y_to_x(wk2b, wk13, sp)
 #endif
 
          ! ===== 1D FFTs in X =====
          !       status = DftiComputeBackward(c2r_x, wk3(:,1,1), out_r(:,1,1))
-         status = wrapper_c2r(c2r_x, wk3, out_r)
+         status = wrapper_c2r(c2r_x, wk13, out_r)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
 
       else if (format == PHYSICAL_IN_Z) then
@@ -707,16 +706,15 @@ subroutine fft_3d_c2r(in_c, out_r)
 #endif
 
          ! ===== Swap Y --> Z =====
-         allocate (wk3(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
 #ifdef OVERWRITE
-         call transpose_y_to_z(wk2_r2c, wk3, sp)
+         call transpose_y_to_z(wk2_r2c, wk13, sp)
 #else
-         call transpose_y_to_z(wk2b, wk3, sp)
+         call transpose_y_to_z(wk2b, wk13, sp)
 #endif
 
          ! ===== 1D FFTs in Z =====
          !       status = DftiComputeBackward(c2r_z, wk3(:,1,1), out_r(:,1,1))
-         status = wrapper_c2r(c2r_z, wk3, out_r)
+         status = wrapper_c2r(c2r_z, wk13, out_r)
          if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2r")
 
       end if

From 3867ee1538069c8792c38f3f187b5216952f79df Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Dec 2022 07:21:14 +0100
Subject: [PATCH 037/436] MKL : fix memory leaks

---
 src/fft_mkl.f90 | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index d5e72525..b6391ee2 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -349,7 +349,9 @@ subroutine fft_3d_c2c(in, out, isign)
       complex(mytype), dimension(:, :, :), intent(OUT) :: out
       integer, intent(IN) :: isign
 
+#ifndef OVERWRITE
       complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2b, wk3
+#endif
       integer :: k, status
 
 #ifdef PROFILER
@@ -487,6 +489,12 @@ subroutine fft_3d_c2c(in, out, isign)
 
       end if
 
+      ! Free memory
+#ifndef OVERWRITE
+      deallocate(wk1, wk2b, wk3)
+#endif
+
+
 #ifdef PROFILER
       if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
@@ -504,7 +512,9 @@ subroutine fft_3d_r2c(in_r, out_c)
       real(mytype), dimension(:, :, :), intent(IN) :: in_r
       complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
+#ifndef OVERWRITE
       complex(mytype), allocatable, dimension(:, :, :) :: wk2b, wk3
+#endif
       integer :: k, status, isign
 
 #ifdef PROFILER
@@ -599,6 +609,11 @@ subroutine fft_3d_r2c(in_r, out_c)
 
       end if
 
+      ! Free memory
+#ifndef OVERWRITE
+      deallocate(wk2b, wk3)
+#endif
+
 #ifdef PROFILER
       if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
 #endif
@@ -616,7 +631,9 @@ subroutine fft_3d_c2r(in_c, out_r)
       complex(mytype), dimension(:, :, :), intent(IN) :: in_c
       real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
+#ifndef OVERWRITE
       complex(mytype), allocatable, dimension(:, :, :) :: wk1, wk2b
+#endif
       integer :: k, status, isign
 
 #ifdef PROFILER
@@ -719,6 +736,11 @@ subroutine fft_3d_c2r(in_c, out_r)
 
       end if
 
+      ! Free memory
+#ifndef OVERWRITE
+      deallocate(wk1, wk2b)
+#endif
+
 #ifdef PROFILER
       if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
 #endif

From ccedcdea6b4544f27ad504c0b4608d92042926c2 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Dec 2022 09:30:14 +0100
Subject: [PATCH 038/436] MKL : fix wrapper_c2c_inplace

---
 src/fft_mkl.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index b6391ee2..7fa6edd6 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -796,7 +796,7 @@ integer function wrapper_c2c_inplace(desc, inout, isign)
          status = DftiComputeBackward(desc, inout)
       end if
 
-      wrapper_c2c = status
+      wrapper_c2c_inplace = status
 
       return
    end function wrapper_c2c_inplace

From 347765cdd3c133eae66abd3131a5474d7622b811 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Dec 2022 09:32:42 +0100
Subject: [PATCH 039/436] MKL : remove warning in case of overwrite

---
 src/fft_mkl.f90 | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 7fa6edd6..04fa5a58 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -46,9 +46,6 @@ subroutine init_fft_engine
          write (*, *) '***** Using the MKL engine *****'
          write (*, *) ' '
       end if
-#ifdef OVERWRITE
-      call decomp_2d_warning(0, "MKL FFT engine does not support the OVERWRITE flag.")
-#endif
 
       ! For C2C transforms
       call c2c_1m_x_plan(c2c_x, ph)

From badba948939aa41d930ecb4627a4f38dab61b956 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Wed, 14 Dec 2022 09:55:14 +0100
Subject: [PATCH 040/436] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 66a95cc5..25b1bdd3 100644
--- a/README.md
+++ b/README.md
@@ -113,7 +113,7 @@ This preprocessor variable is not valid for GPU builds. It leads to padded allto
 
 #### OVERWRITE
 
-This variable is not supported by the MKL FFT backend. It leads to overwrite the input array when computing FFT (complex-to-complex and complex-to-real).
+This variable leads to overwrite the input array when computing FFT (complex-to-complex and complex-to-real).
 
 #### HALO_DEBUG
 

From 7c724e6bc321bfe5e1c5c600882e89c9685b76bc Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Wed, 14 Dec 2022 16:26:54 +0100
Subject: [PATCH 041/436] Typo in README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 25b1bdd3..976f9f6e 100644
--- a/README.md
+++ b/README.md
@@ -66,7 +66,7 @@ By default the compute architecture for the GPU is 80 (i.e. Ampere), to change i
 Profiling can be activated in the Makefile. Set the variable `PROFILER` to one of the supported profilers (only `caliper` currently). If using `caliper`, provide the installation path in the variable `CALIPER_PATH`. When the profiling is active, one can tune it before calling `decomp_2d_init` using the subroutine `decomp_profiler_prep`. The input argument for this subroutine is a logical array of size 4. Each input allow activation / deactivation of the profiling as follows :
 
 1. Profile transpose operations (default : true)
-2. Profile IO operations (default : true))
+2. Profile IO operations (default : true)
 3. Profile FFT operations (default : true)
 4. Profile decomp_2d init / fin subroutines (default : true)
 

From 5d31559c48a2c70e1519ce799bd0ece5a6489624 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 15 Dec 2022 09:54:28 +0100
Subject: [PATCH 042/436] Expose the selected FFT backend in a public variable

---
 README.md             | 12 ++++++++++++
 src/fft_common.f90    |  7 +++++++
 src/fft_cufft.f90     |  1 +
 src/fft_fftw3.f90     |  1 +
 src/fft_fftw3_f03.f90 |  8 ++++++++
 src/fft_generic.f90   |  1 +
 src/fft_mkl.f90       |  1 +
 7 files changed, 31 insertions(+)

diff --git a/README.md b/README.md
index 976f9f6e..f10f3caf 100644
--- a/README.md
+++ b/README.md
@@ -70,6 +70,18 @@ Profiling can be activated in the Makefile. Set the variable `PROFILER` to one o
 3. Profile FFT operations (default : true)
 4. Profile decomp_2d init / fin subroutines (default : true)
 
+## FFT backends
+
+The library provides a built-in FFT engine and supports various FFT backends : FFTW, Intel oneMKL, Nvidia cuFFT. The FFT engine selected during compilation is available through the variable `D2D_FFT_BACKEND` defined in the module `decomp_2d_fft`. The expected value is defined by the integer constants
+```
+integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0   ! Built-in engine
+integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1     ! FFTW
+integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2 ! FFTW (Fortran 2003)
+integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3       ! Intel oneMKL
+integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4     ! Nvidia cuFFT
+```
+The external code can use the named variables to check the FFT backend used in a given build.
+
 ## Miscellaneous
 
 ### Print the log to a file or to stdout
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index e0cd9cac..1feb5166 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -18,6 +18,13 @@
 integer, parameter, public :: PHYSICAL_IN_X = 1
 integer, parameter, public :: PHYSICAL_IN_Z = 3
 
+! Supported FFT backends
+integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
+integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
+integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
+integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
+integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
+
 integer, save :: format                 ! input X-pencil or Z-pencil
 
 ! The libary can only be initialised once
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 5f7575df..08b9355a 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -39,6 +39,7 @@ module decomp_2d_fft
    ! common code used for all engines, including global variables,
    ! generic interface definitions and several subroutines
 #include "fft_common.f90"
+   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_CUFFT
 
    ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2C case
    subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index fb7646ba..71ebcf8a 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -45,6 +45,7 @@ end subroutine fftw_cleanup
    ! common code used for all engines, including global variables,
    ! generic interface definitions and several subroutines
 #include "fft_common.f90"
+   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_FFTW3
 
    ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
    subroutine c2c_1m_x_plan(plan1, decomp, isign)
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index e8c97081..28422968 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -43,6 +43,14 @@ module decomp_2d_fft
    integer, parameter, public :: PHYSICAL_IN_X = 1
    integer, parameter, public :: PHYSICAL_IN_Z = 3
 
+   ! Supported FFT backends
+   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
+   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
+   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
+   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_FFTW3_F03
+
    integer, save :: format                 ! input X-pencil or Z-pencil
 
    ! The libary can only be initialised once
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index 8bab0ae3..90483a1c 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -26,6 +26,7 @@ module decomp_2d_fft
    ! common code used for all engines, including global variables,
    ! generic interface definitions and several subroutines
 #include "fft_common.f90"
+   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_GENERIC
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    !  This routine performs one-time initialisations for the FFT engine
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index cebed05c..4521fb8e 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -33,6 +33,7 @@ module decomp_2d_fft
    ! common code used for all engines, including global variables,
    ! generic interface definitions and several subroutines
 #include "fft_common.f90"
+   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_MKL
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    !  This routine performs one-time initialisations for the FFT engine

From 605a036d19e4b8e52c529695bfa42918cd7d7b45 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 15 Dec 2022 14:24:10 +0100
Subject: [PATCH 043/436] Fix 5d31559c48a2c70e1519ce799bd0ece5a6489624

---
 src/fft_common.f90  | 7 -------
 src/fft_cufft.f90   | 9 ++++++++-
 src/fft_fftw3.f90   | 9 ++++++++-
 src/fft_generic.f90 | 9 ++++++++-
 src/fft_mkl.f90     | 9 ++++++++-
 5 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 1feb5166..e0cd9cac 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -18,13 +18,6 @@
 integer, parameter, public :: PHYSICAL_IN_X = 1
 integer, parameter, public :: PHYSICAL_IN_Z = 3
 
-! Supported FFT backends
-integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
-integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
-integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
-integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
-integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
-
 integer, save :: format                 ! input X-pencil or Z-pencil
 
 ! The libary can only be initialised once
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 08b9355a..a44152a6 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -36,10 +36,17 @@ module decomp_2d_fft
    integer*4, save :: plan(-1:2, 3)
    complex*8, device, allocatable, dimension(:) :: cufft_workspace
 
+   ! Supported FFT backends
+   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
+   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
+   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
+   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_CUFFT
+
    ! common code used for all engines, including global variables,
    ! generic interface definitions and several subroutines
 #include "fft_common.f90"
-   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_CUFFT
 
    ! Return a cuFFT plan for multiple 1D FFTs in X direction: C2C case
    subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index 71ebcf8a..6fcf8d0e 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -42,10 +42,17 @@ subroutine fftw_cleanup() bind(C, name='fftw_cleanup')
       end subroutine fftw_cleanup
    end interface
 
+   ! Supported FFT backends
+   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
+   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
+   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
+   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_FFTW3
+
    ! common code used for all engines, including global variables,
    ! generic interface definitions and several subroutines
 #include "fft_common.f90"
-   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_FFTW3
 
    ! Return a FFTW3 plan for multiple 1D c2c FFTs in X direction
    subroutine c2c_1m_x_plan(plan1, decomp, isign)
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index 90483a1c..d6edcb83 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -23,10 +23,17 @@ module decomp_2d_fft
    ! engine-specific global variables
    complex(mytype), allocatable, dimension(:) :: buf, scratch
 
+   ! Supported FFT backends
+   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
+   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
+   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
+   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_GENERIC
+
    ! common code used for all engines, including global variables,
    ! generic interface definitions and several subroutines
 #include "fft_common.f90"
-   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_GENERIC
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    !  This routine performs one-time initialisations for the FFT engine
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 4521fb8e..255b9a4a 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -30,10 +30,17 @@ module decomp_2d_fft
    !  for r2c/c2r transforms, PHYSICAL_IN_Z
    type(DFTI_DESCRIPTOR), pointer :: r2c_z, c2c_x2, c2r_z
 
+   ! Supported FFT backends
+   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
+   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
+   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
+   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_MKL
+
    ! common code used for all engines, including global variables,
    ! generic interface definitions and several subroutines
 #include "fft_common.f90"
-   integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_MKL
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    !  This routine performs one-time initialisations for the FFT engine

From c5b8af4f19ea8fb39a485e9e6a06ff3c6fb500eb Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 15 Dec 2022 15:24:08 +0100
Subject: [PATCH 044/436] Update the FFT backends for the case decomp_log ==
 D2D_LOG_STDOUT

---
 src/fft_cufft.f90     | 4 +++-
 src/fft_fftw3.f90     | 4 +++-
 src/fft_fftw3_f03.f90 | 4 +++-
 src/fft_generic.f90   | 4 +++-
 src/fft_mkl.f90       | 4 +++-
 5 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index a44152a6..387787ee 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -247,7 +247,9 @@ subroutine init_fft_engine
       integer(int_ptr_kind()) :: cufft_ws, ws
       integer   :: i, j, istat
 
-      if (nrank == 0) then
+      ! FIXME decomp_log == D2D_LOG_TOFILE
+      !                     D2D_LOG_TOFILE_FULL
+      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
          write (*, *) ' '
          write (*, *) '***** Using the New cuFFT engine *****'
          write (*, *) ' '
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index 6fcf8d0e..15871ac5 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -273,7 +273,9 @@ subroutine init_fft_engine
 
       implicit none
 
-      if (nrank == 0) then
+      ! FIXME decomp_log == D2D_LOG_TOFILE
+      !                     D2D_LOG_TOFILE_FULL
+      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
          write (*, *) ' '
          write (*, *) '***** Using the FFTW (version 3.x) engine *****'
          write (*, *) ' '
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 28422968..9e2571a2 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -579,7 +579,9 @@ subroutine init_fft_engine
 
       implicit none
 
-      if (nrank == 0) then
+      ! FIXME decomp_log == D2D_LOG_TOFILE
+      !                     D2D_LOG_TOFILE_FULL
+      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
          write (*, *) ' '
          write (*, *) '***** Using the FFTW (F2003 interface) engine *****'
          write (*, *) ' '
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index d6edcb83..7e56b22f 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -44,7 +44,9 @@ subroutine init_fft_engine
 
       integer :: cbuf_size
 
-      if (nrank == 0) then
+      ! FIXME decomp_log == D2D_LOG_TOFILE
+      !                     D2D_LOG_TOFILE_FULL
+      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
          write (*, *) ' '
          write (*, *) '***** Using the generic FFT engine *****'
          write (*, *) ' '
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 255b9a4a..eeaff021 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -49,7 +49,9 @@ subroutine init_fft_engine
 
       implicit none
 
-      if (nrank == 0) then
+      ! FIXME decomp_log == D2D_LOG_TOFILE
+      !                     D2D_LOG_TOFILE_FULL
+      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
          write (*, *) ' '
          write (*, *) '***** Using the MKL engine *****'
          write (*, *) ' '

From b790e894918bc35bb1a31691549adee5d6ce6772 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 15 Dec 2022 16:09:19 +0100
Subject: [PATCH 045/436] log : Close the IO unit before writing the
 environment variables

---
 src/decomp_2d.f90          |  4 +++
 src/decomp_2d_init_fin.f90 | 21 ++----------
 src/log.f90                | 70 +++++++++++++++++++++++++++++++++++++-
 3 files changed, 75 insertions(+), 20 deletions(-)

diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index e4142fd9..be25c1fc 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -359,6 +359,10 @@ module decomp_2d
 
    interface
 
+      module function d2d_listing_get_unit()
+         integer :: d2d_listing_get_unit
+      end function d2d_listing_get_unit
+
       module subroutine d2d_listing(given_io_unit)
          integer, intent(in), optional :: given_io_unit
       end subroutine d2d_listing
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index edec7259..58560c9c 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -37,7 +37,6 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
      integer :: cuda_stat
      type(ncclResult) :: nccl_stat
 #endif
-     character(len=7) fname ! Sufficient for up to O(1M) ranks
 
 #ifdef PROFILER
      ! Prepare the profiler if it was not already prepared
@@ -211,29 +210,13 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
 #endif
 
      !
-     ! Select the IO unit for decomp_2d setup
+     ! Get the IO unit for decomp_2d setup
      !
-     if (decomp_log == D2D_LOG_TOFILE_FULL) then
-        write (fname, "(I0)") nrank ! Adapt to magnitude of nrank
-        open (newunit=iounit, file='decomp_2d_setup_'//trim(fname)//'.log', iostat=ierror)
-     elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
-        open (newunit=iounit, file="decomp_2d_setup.log", iostat=ierror)
-     else
-        iounit = output_unit
-        ierror = 0
-     end if
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
+     iounit = d2d_listing_get_unit()
      !
      ! Print the decomp_2d setup
      !
      call d2d_listing(iounit)
-     !
-     ! Close the IO unit if it was not stdout
-     !
-     if (iounit /= output_unit) then
-        close (iounit, iostat=ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-     end if
 
 #ifdef PROFILER
      ! Stop the timer for decomp_2d_init
diff --git a/src/log.f90 b/src/log.f90
index 53e3165a..139647e1 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -16,6 +16,62 @@
 
 contains
 
+   !
+   ! Get the IO unit for the log
+   !
+   module function d2d_listing_get_unit()
+
+      use iso_fortran_env, only: output_unit
+
+      implicit none
+
+      ! Output
+      integer :: d2d_listing_get_unit
+
+      ! Local variables
+      logical :: found
+      integer :: ierror
+      character(len=7) fname ! Sufficient for up to O(1M) ranks
+
+      if (decomp_log == D2D_LOG_TOFILE_FULL) then
+         write (fname, "(I0)") nrank ! Adapt to magnitude of nrank
+         inquire(file='decomp_2d_setup_'//trim(fname)//'.log', &
+                 exist=found)
+         if (found) then
+            open (newunit=d2d_listing_get_unit, &
+                  file='decomp_2d_setup_'//trim(fname)//'.log', &
+                  status="old", &
+                  position="append", &
+                  iostat=ierror)
+         else
+            open (newunit=d2d_listing_get_unit, &
+                  file='decomp_2d_setup_'//trim(fname)//'.log', &
+                  status="new", &
+                  iostat=ierror)
+         endif
+      elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
+         inquire (file="decomp_2d_setup.log", &
+                  exist=found)
+         if (found) then
+            open (newunit=d2d_listing_get_unit, &
+                  file="decomp_2d_setup.log", &
+                  status="old", &
+                  position="append", &
+                  iostat=ierror)
+         else
+            open (newunit=d2d_listing_get_unit, &                                                    
+                  file="decomp_2d_setup.log", &
+                  status="new", &
+                  iostat=ierror)
+         endif
+      else
+         d2d_listing_get_unit = output_unit
+         ierror = 0
+      endif
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
+
+   end function d2d_listing_get_unit
+
    !
    ! Print some information about decomp_2d
    !
@@ -141,6 +197,7 @@ module subroutine d2d_listing(given_io_unit)
 #endif
       write (io_unit, *) '==========================================================='
       write (io_unit, *) '==========================================================='
+
 #ifdef DEBUG
       !
       ! In DEBUG mode, rank 0 will also print environment variables
@@ -161,9 +218,20 @@ module subroutine d2d_listing(given_io_unit)
                                                   __LINE__, &
                                                   ierror, &
                                                   "No name for the log file")
-            call execute_command_line("env >> "//trim(fname), wait=.false.)
+            ! Close the IO unit to print the environment variables
+            close (io_unit, iostat=ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+            call execute_command_line("env >> "//trim(fname), wait=.true.)
          end if
       end if
+#else
+      !
+      ! Close the IO unit if it was not stdout
+      !
+      if (io_unit /= output_unit) then
+         close (io_unit, iostat=ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+      endif
 #endif
 
    end subroutine d2d_listing

From 144841a6b557a10ca13b98c4dbe0a4db79e08885 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 15 Dec 2022 16:10:06 +0100
Subject: [PATCH 046/436] Use decomp_log inside the FFT module

---
 src/decomp_2d.f90     |  3 ++-
 src/fft_cufft.f90     | 21 +++++++++++++--------
 src/fft_fftw3.f90     | 19 +++++++++++++------
 src/fft_fftw3_f03.f90 | 19 +++++++++++++------
 src/fft_generic.f90   | 23 +++++++++++++++--------
 src/fft_mkl.f90       | 20 ++++++++++++++------
 6 files changed, 70 insertions(+), 35 deletions(-)

diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index be25c1fc..f77ffee1 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -259,7 +259,8 @@ module decomp_2d
              alloc_x, alloc_y, alloc_z, &
              update_halo, decomp_2d_abort, &
              decomp_2d_warning, get_decomp_info, &
-             decomp_mpi_comm_free, get_decomp_dims
+             decomp_mpi_comm_free, get_decomp_dims, &
+             d2d_listing_get_unit
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! These are routines to perform global data transpositions
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 387787ee..d54293f1 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -245,14 +245,19 @@ subroutine init_fft_engine
 
       !integer*4 :: cufft_ws, ws
       integer(int_ptr_kind()) :: cufft_ws, ws
-      integer   :: i, j, istat
-
-      ! FIXME decomp_log == D2D_LOG_TOFILE
-      !                     D2D_LOG_TOFILE_FULL
-      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the New cuFFT engine *****'
-         write (*, *) ' '
+      integer   :: i, j, istat, iounit, ierror
+
+      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE_FULL)) then
+         iounit = d2d_listing_get_unit()
+         write (iounit, *) ' '
+         write (iounit, *) '***** Using the New cuFFT engine *****'
+         write (iounit, *) ' '
+         if (iounit /= output_unit) then
+            close(iounit, iostat=ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+         endif
       end if
 
       cufft_ws = 0
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index 15871ac5..17ae70b7 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -273,12 +273,19 @@ subroutine init_fft_engine
 
       implicit none
 
-      ! FIXME decomp_log == D2D_LOG_TOFILE
-      !                     D2D_LOG_TOFILE_FULL
-      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the FFTW (version 3.x) engine *****'
-         write (*, *) ' '
+      integer :: iounit, ierror
+
+      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE_FULL)) then
+         iounit = d2d_listing_get_unit()
+         write (iounit, *) ' '
+         write (iounit, *) '***** Using the FFTW (version 3.x) engine *****'
+         write (iounit, *) ' '
+         if (iounit /= output_unit) then
+            close(iounit, iostat=ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+         endif
       end if
 
       if (format == PHYSICAL_IN_X) then
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 9e2571a2..2782211b 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -579,12 +579,19 @@ subroutine init_fft_engine
 
       implicit none
 
-      ! FIXME decomp_log == D2D_LOG_TOFILE
-      !                     D2D_LOG_TOFILE_FULL
-      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the FFTW (F2003 interface) engine *****'
-         write (*, *) ' '
+      integer :: iounit, ierror
+
+      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE_FULL)) then
+         iounit = d2d_listing_get_unit()
+         write (iounit, *) ' '
+         write (iounit, *) '***** Using the FFTW (F2003 interface) engine *****'
+         write (iounit, *) ' '
+         if (iounit /= output_unit) then
+            close(iounit, iostat=ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+         endif
       end if
 
       if (format == PHYSICAL_IN_X) then
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index 7e56b22f..f3639067 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -40,16 +40,23 @@ module decomp_2d_fft
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
-      implicit none
+      use iso_fortran_env, only : output_unit
 
-      integer :: cbuf_size
+      implicit none
 
-      ! FIXME decomp_log == D2D_LOG_TOFILE
-      !                     D2D_LOG_TOFILE_FULL
-      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the generic FFT engine *****'
-         write (*, *) ' '
+      integer :: cbuf_size, iounit, ierror
+
+      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE_FULL)) then
+         iounit = d2d_listing_get_unit()
+         write (iounit, *) ' '
+         write (iounit, *) '***** Using the generic FFT engine *****'
+         write (iounit, *) ' '
+         if (iounit /= output_unit) then
+            close(iounit, iostat=ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+         endif
       end if
 
       cbuf_size = max(ph%xsz(1), ph%ysz(2))
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index eeaff021..f9688217 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -49,13 +49,21 @@ subroutine init_fft_engine
 
       implicit none
 
-      ! FIXME decomp_log == D2D_LOG_TOFILE
-      !                     D2D_LOG_TOFILE_FULL
-      if (decomp_log == D2D_LOG_STDOUT .and. nrank == 0) then
-         write (*, *) ' '
-         write (*, *) '***** Using the MKL engine *****'
-         write (*, *) ' '
+      integer :: iounit, ierror
+
+      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE_FULL)) then
+         iounit = d2d_listing_get_unit()
+         write (iounit, *) ' '
+         write (iounit, *) '***** Using the MKL engine *****'
+         write (iounit, *) ' '
+         if (iounit /= output_unit) then
+            close(iounit, iostat=ierror)
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+         endif
       end if
+
 #ifdef OVERWRITE
       call decomp_2d_warning(0, "MKL FFT engine does not support the OVERWRITE flag.")
 #endif

From 70f5979dfc4a1e07ea5f6d01630ba2b4bd4d4666 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 15 Dec 2022 16:25:21 +0100
Subject: [PATCH 047/436] Move all the constants / parameters in a dedicated
 file / module

---
 Makefile                    |  2 +-
 README.md                   |  3 +-
 src/decomp_2d.f90           | 18 +--------
 src/decomp_2d_constants.f90 | 74 +++++++++++++++++++++++++++++++++++++
 src/fft_cufft.f90           |  6 ---
 src/fft_fftw3.f90           |  6 ---
 src/fft_fftw3_f03.f90       |  6 ---
 src/fft_generic.f90         |  7 +---
 src/fft_mkl.f90             |  6 ---
 9 files changed, 79 insertions(+), 49 deletions(-)
 create mode 100644 src/decomp_2d_constants.f90

diff --git a/Makefile b/Makefile
index 29817754..f74736ea 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,7 @@ CMPINC = Makefile.compilers
 include $(CMPINC)
 
 ### List of files for the main code
-SRCDECOMP = factor.f90 decomp_2d.f90 log.f90 io.f90
+SRCDECOMP = decomp_2d_constants.f90 factor.f90 decomp_2d.f90 log.f90 io.f90
 
 #######FFT settings##########
 ifeq ($(FFT),fftw3)
diff --git a/README.md b/README.md
index f10f3caf..aa4777ef 100644
--- a/README.md
+++ b/README.md
@@ -80,6 +80,7 @@ integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2 ! FFTW (Fortran 2003
 integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3       ! Intel oneMKL
 integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4     ! Nvidia cuFFT
 ```
+exported by the module `decomp_2d_constants`.
 The external code can use the named variables to check the FFT backend used in a given build.
 
 ## Miscellaneous
@@ -93,7 +94,7 @@ integer, parameter, public :: D2D_LOG_STDOUT = 1      ! Root rank logs output to
 integer, parameter, public :: D2D_LOG_TOFILE = 2      ! Root rank logs output to the file "decomp_2d_setup.log"
 integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3 ! All ranks log output to a dedicated file
 ```
-exported by the `decomp_2d` module.
+exported by the `decomp_2d_constants` module.
 Although their values are shown here, users should not rely on these and are recommended to prefer to use the named variables `D2D_LOG_QUIET`, etc. instead.
 The default value used is `D2D_LOG_TOFILE` for the default build and `D2D_LOG_TOFILE_FULL` for a debug build.
 
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index f77ffee1..b442b820 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -17,6 +17,7 @@ module decomp_2d
    use MPI
    use, intrinsic :: iso_fortran_env, only: real32, real64
    use factor
+   use decomp_2d_constants
 #if defined(_GPU)
    use cudafor
 #if defined(_NCCL)
@@ -84,10 +85,6 @@ module decomp_2d
    !
    ! The default value is 2 (3 for debug builds)
    !
-   integer, parameter, public :: D2D_LOG_QUIET = 0
-   integer, parameter, public :: D2D_LOG_STDOUT = 1
-   integer, parameter, public :: D2D_LOG_TOFILE = 2
-   integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
 #ifdef DEBUG
    integer, public, save :: decomp_log = D2D_LOG_TOFILE_FULL
 #else
@@ -101,15 +98,6 @@ module decomp_2d
    !
    ! Debug checks are performed only when the preprocessor variable DEBUG is defined
    !
-   enum, bind(c)
-      enumerator :: D2D_DEBUG_LEVEL_OFF = 0
-      enumerator :: D2D_DEBUG_LEVEL_CRITICAL = 1
-      enumerator :: D2D_DEBUG_LEVEL_ERROR = 2
-      enumerator :: D2D_DEBUG_LEVEL_WARN = 3
-      enumerator :: D2D_DEBUG_LEVEL_INFO = 4
-      enumerator :: D2D_DEBUG_LEVEL_DEBUG = 5
-      enumerator :: D2D_DEBUG_LEVEL_TRACE = 6
-   end enum
 #ifdef DEBUG
    integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_INFO
 #else
@@ -235,10 +223,6 @@ module decomp_2d
    !    0 => no profiling, default
    !    1 => Caliper (https://github.com/LLNL/Caliper)
    !
-   enum, bind(c)
-      enumerator :: decomp_profiler_none = 0
-      enumerator :: decomp_profiler_caliper = 1
-   end enum
    integer(kind(decomp_profiler_none)), save, public :: decomp_profiler = decomp_profiler_none
    ! Default : profile everything
    logical, save, public :: decomp_profiler_transpose = .true.
diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
new file mode 100644
index 00000000..8d2c54ab
--- /dev/null
+++ b/src/decomp_2d_constants.f90
@@ -0,0 +1,74 @@
+!=======================================================================
+! This is part of the 2DECOMP&FFT library
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! decomposition. It also implements a highly scalable distributed
+! three-dimensional Fast Fourier Transform (FFT).
+!
+! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
+! Copyright (C) 2021               the University of Edinburgh (UoE)
+!
+!=======================================================================
+
+! Constants for the 2decomp&fft library
+
+module decomp_2d_constants
+
+   implicit none
+
+   !
+   ! Output for the log can be changed by the external code before calling decomp_2d_init
+   !
+   !    0 => No log output
+   !    1 => Master rank log output to stdout
+   !    2 => Master rank log output to the file "decomp_2d_setup.log"
+   !    3 => All ranks log output to a dedicated file
+   !
+   ! The default value is 2 (3 for debug builds)
+   !
+   integer, parameter, public :: D2D_LOG_QUIET = 0
+   integer, parameter, public :: D2D_LOG_STDOUT = 1
+   integer, parameter, public :: D2D_LOG_TOFILE = 2
+   integer, parameter, public :: D2D_LOG_TOFILE_FULL = 3
+
+   !
+   ! Debug level can be changed by the external code before calling decomp_2d_init
+   !
+   ! The environment variable "DECOMP_2D_DEBUG" can be used to change the debug level
+   !
+   ! Debug checks are performed only when the preprocessor variable DEBUG is defined
+   !
+   enum, bind(c)
+      enumerator :: D2D_DEBUG_LEVEL_OFF = 0
+      enumerator :: D2D_DEBUG_LEVEL_CRITICAL = 1
+      enumerator :: D2D_DEBUG_LEVEL_ERROR = 2
+      enumerator :: D2D_DEBUG_LEVEL_WARN = 3
+      enumerator :: D2D_DEBUG_LEVEL_INFO = 4
+      enumerator :: D2D_DEBUG_LEVEL_DEBUG = 5
+      enumerator :: D2D_DEBUG_LEVEL_TRACE = 6
+   end enum
+
+   !
+   ! Profiler section
+   !
+   ! Integer to select the profiling tool
+   !    0 => no profiling, default
+   !    1 => Caliper (https://github.com/LLNL/Caliper)
+   !
+   enum, bind(c)
+      enumerator :: decomp_profiler_none = 0
+      enumerator :: decomp_profiler_caliper = 1
+   end enum
+
+   !
+   ! Supported FFT backends
+   !
+   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
+   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
+   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
+   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
+
+
+end module decomp_2d_constants
+
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index d54293f1..b0c68e7a 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -36,12 +36,6 @@ module decomp_2d_fft
    integer*4, save :: plan(-1:2, 3)
    complex*8, device, allocatable, dimension(:) :: cufft_workspace
 
-   ! Supported FFT backends
-   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
-   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
-   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
    integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_CUFFT
 
    ! common code used for all engines, including global variables,
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index 17ae70b7..4d90d2b5 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -42,12 +42,6 @@ subroutine fftw_cleanup() bind(C, name='fftw_cleanup')
       end subroutine fftw_cleanup
    end interface
 
-   ! Supported FFT backends
-   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
-   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
-   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
    integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_FFTW3
 
    ! common code used for all engines, including global variables,
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 2782211b..62e66a4e 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -43,12 +43,6 @@ module decomp_2d_fft
    integer, parameter, public :: PHYSICAL_IN_X = 1
    integer, parameter, public :: PHYSICAL_IN_Z = 3
 
-   ! Supported FFT backends
-   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
-   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
-   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
    integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_FFTW3_F03
 
    integer, save :: format                 ! input X-pencil or Z-pencil
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index f3639067..30e58f6d 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -13,6 +13,7 @@
 
 module decomp_2d_fft
 
+   use decomp_2d_constants
    use decomp_2d  ! 2D decomposition module
    use glassman
 
@@ -23,12 +24,6 @@ module decomp_2d_fft
    ! engine-specific global variables
    complex(mytype), allocatable, dimension(:) :: buf, scratch
 
-   ! Supported FFT backends
-   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
-   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
-   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
    integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_GENERIC
 
    ! common code used for all engines, including global variables,
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index f9688217..69e10188 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -30,12 +30,6 @@ module decomp_2d_fft
    !  for r2c/c2r transforms, PHYSICAL_IN_Z
    type(DFTI_DESCRIPTOR), pointer :: r2c_z, c2c_x2, c2r_z
 
-   ! Supported FFT backends
-   integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1
-   integer, parameter, public :: D2D_FFT_BACKEND_FFTW3_F03 = 2
-   integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
-   integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
    integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_MKL
 
    ! common code used for all engines, including global variables,

From 8bee08070329eb8121fbb6071770dcc7561437fa Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 15 Dec 2022 16:28:14 +0100
Subject: [PATCH 048/436] fprettify the new file

---
 src/decomp_2d_constants.f90 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 8d2c54ab..41496e66 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -69,6 +69,5 @@ module decomp_2d_constants
    integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
    integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
 
-
 end module decomp_2d_constants
 

From 723a24fe3502020033a3f759e2c1101ee5d2e36d Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 15 Dec 2022 16:40:00 +0100
Subject: [PATCH 049/436] Fix commit 70f5979dfc4a1e07ea5f6d01630ba2b4bd4d4666

---
 src/fft_cufft.f90     |  7 +++++--
 src/fft_fftw3.f90     |  7 +++++--
 src/fft_fftw3_f03.f90 |  7 +++++--
 src/fft_generic.f90   |  6 +++---
 src/fft_mkl.f90       |  7 +++++--
 src/log.f90           | 14 +++++++-------
 6 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index b0c68e7a..3c60be8f 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -13,6 +13,7 @@
 
 module decomp_2d_fft
 
+   use decomp_2d_constants
    use decomp_2d  ! 2D decomposition module
    use iso_c_binding
    use cudafor
@@ -235,6 +236,8 @@ end subroutine c2r_1m_z_plan
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
+      use iso_fortran_env, only: output_unit
+
       implicit none
 
       !integer*4 :: cufft_ws, ws
@@ -249,9 +252,9 @@ subroutine init_fft_engine
          write (iounit, *) '***** Using the New cuFFT engine *****'
          write (iounit, *) ' '
          if (iounit /= output_unit) then
-            close(iounit, iostat=ierror)
+            close (iounit, iostat=ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         endif
+         end if
       end if
 
       cufft_ws = 0
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index 4d90d2b5..d8b6d73c 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -13,6 +13,7 @@
 
 module decomp_2d_fft
 
+   use decomp_2d_constants
    use decomp_2d  ! 2D decomposition module
    use iso_c_binding
 
@@ -265,6 +266,8 @@ end subroutine c2r_1m_z_plan
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
+      use iso_fortran_env, only: output_unit
+
       implicit none
 
       integer :: iounit, ierror
@@ -277,9 +280,9 @@ subroutine init_fft_engine
          write (iounit, *) '***** Using the FFTW (version 3.x) engine *****'
          write (iounit, *) ' '
          if (iounit /= output_unit) then
-            close(iounit, iostat=ierror)
+            close (iounit, iostat=ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         endif
+         end if
       end if
 
       if (format == PHYSICAL_IN_X) then
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 62e66a4e..155beef7 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -14,6 +14,7 @@
 
 module decomp_2d_fft
 
+   use decomp_2d_constants
    use decomp_2d  ! 2D decomposition module
    use, intrinsic :: iso_c_binding
 
@@ -571,6 +572,8 @@ end subroutine c2r_1m_z_plan
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
+      use iso_fortran_env, only: output_unit
+
       implicit none
 
       integer :: iounit, ierror
@@ -583,9 +586,9 @@ subroutine init_fft_engine
          write (iounit, *) '***** Using the FFTW (F2003 interface) engine *****'
          write (iounit, *) ' '
          if (iounit /= output_unit) then
-            close(iounit, iostat=ierror)
+            close (iounit, iostat=ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         endif
+         end if
       end if
 
       if (format == PHYSICAL_IN_X) then
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index 30e58f6d..13d1001d 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -35,7 +35,7 @@ module decomp_2d_fft
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
-      use iso_fortran_env, only : output_unit
+      use iso_fortran_env, only: output_unit
 
       implicit none
 
@@ -49,9 +49,9 @@ subroutine init_fft_engine
          write (iounit, *) '***** Using the generic FFT engine *****'
          write (iounit, *) ' '
          if (iounit /= output_unit) then
-            close(iounit, iostat=ierror)
+            close (iounit, iostat=ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         endif
+         end if
       end if
 
       cbuf_size = max(ph%xsz(1), ph%ysz(2))
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 69e10188..6eca40f1 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -13,6 +13,7 @@
 
 module decomp_2d_fft
 
+   use decomp_2d_constants
    use decomp_2d  ! 2D decomposition module
    use MKL_DFTI   ! MKL FFT module
 
@@ -41,6 +42,8 @@ module decomp_2d_fft
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
+      use iso_fortran_env, only: output_unit
+
       implicit none
 
       integer :: iounit, ierror
@@ -53,9 +56,9 @@ subroutine init_fft_engine
          write (iounit, *) '***** Using the MKL engine *****'
          write (iounit, *) ' '
          if (iounit /= output_unit) then
-            close(iounit, iostat=ierror)
+            close (iounit, iostat=ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         endif
+         end if
       end if
 
 #ifdef OVERWRITE
diff --git a/src/log.f90 b/src/log.f90
index 139647e1..77cd2570 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -35,8 +35,8 @@ module function d2d_listing_get_unit()
 
       if (decomp_log == D2D_LOG_TOFILE_FULL) then
          write (fname, "(I0)") nrank ! Adapt to magnitude of nrank
-         inquire(file='decomp_2d_setup_'//trim(fname)//'.log', &
-                 exist=found)
+         inquire (file='decomp_2d_setup_'//trim(fname)//'.log', &
+                  exist=found)
          if (found) then
             open (newunit=d2d_listing_get_unit, &
                   file='decomp_2d_setup_'//trim(fname)//'.log', &
@@ -48,7 +48,7 @@ module function d2d_listing_get_unit()
                   file='decomp_2d_setup_'//trim(fname)//'.log', &
                   status="new", &
                   iostat=ierror)
-         endif
+         end if
       elseif (nrank == 0 .and. decomp_log == D2D_LOG_TOFILE) then
          inquire (file="decomp_2d_setup.log", &
                   exist=found)
@@ -59,15 +59,15 @@ module function d2d_listing_get_unit()
                   position="append", &
                   iostat=ierror)
          else
-            open (newunit=d2d_listing_get_unit, &                                                    
+            open (newunit=d2d_listing_get_unit, &
                   file="decomp_2d_setup.log", &
                   status="new", &
                   iostat=ierror)
-         endif
+         end if
       else
          d2d_listing_get_unit = output_unit
          ierror = 0
-      endif
+      end if
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not open log file")
 
    end function d2d_listing_get_unit
@@ -231,7 +231,7 @@ module subroutine d2d_listing(given_io_unit)
       if (io_unit /= output_unit) then
          close (io_unit, iostat=ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-      endif
+      end if
 #endif
 
    end subroutine d2d_listing

From 4df10d32ef3d8df375a8f5caac22147409deb4eb Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 15 Dec 2022 16:55:02 +0100
Subject: [PATCH 050/436] fprettify the code

---
 src/fft_mkl.f90 | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index bd8d3c99..7bfc146c 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -502,10 +502,9 @@ subroutine fft_3d_c2c(in, out, isign)
 
       ! Free memory
 #ifndef OVERWRITE
-      deallocate(wk1, wk2b, wk3)
+      deallocate (wk1, wk2b, wk3)
 #endif
 
-
 #ifdef PROFILER
       if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
 #endif
@@ -622,7 +621,7 @@ subroutine fft_3d_r2c(in_r, out_c)
 
       ! Free memory
 #ifndef OVERWRITE
-      deallocate(wk2b, wk3)
+      deallocate (wk2b, wk3)
 #endif
 
 #ifdef PROFILER
@@ -749,7 +748,7 @@ subroutine fft_3d_c2r(in_c, out_r)
 
       ! Free memory
 #ifndef OVERWRITE
-      deallocate(wk1, wk2b)
+      deallocate (wk1, wk2b)
 #endif
 
 #ifdef PROFILER

From fd1d81292c16d6e6faf78dec136a6c5d73acf6e6 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Thu, 15 Dec 2022 16:58:13 +0100
Subject: [PATCH 051/436] Fix CI for the Nvidia compiler

---
 src/log.f90 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/log.f90 b/src/log.f90
index 77cd2570..04342b7d 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -12,6 +12,8 @@
 
 submodule(decomp_2d) d2d_log
 
+   use decomp_2d_constants
+
    implicit none
 
 contains

From a099cb14faebeab063f105a18a85c6c046c8d4f0 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 16 Dec 2022 13:54:00 +0000
Subject: [PATCH 052/436] Add -DOVERWRITE option to Makefile

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f74736ea..7d0311f0 100644
--- a/Makefile
+++ b/Makefile
@@ -5,10 +5,11 @@
 #   -DDOUBLE_PREC - use double-precision
 #   -DSAVE_SINGLE - Save 3D data in single-precision
 #   -DDEBG        - debugging
+#   -DOVERWIR     - Enable overwriting optimisation in MKL FFT
 # generate a Git version string
 GIT_VERSION := $(shell git describe --tag --long --always)
 
-DEFS = -DDOUBLE_PREC -DVERSION=\"$(GIT_VERSION)\"
+DEFS = -DDOUBLE_PREC -DVERSION=\"$(GIT_VERSION)\" -DOVERWRITE
 
 LCL = local# local,lad,sdu,archer
 CMP = gcc# intel,gcc,nagfor,cray,nvhpc

From 8eaec4471c3747e7c5667c4b24fbc6ddeeaa5bbf Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 16 Dec 2022 13:54:26 +0000
Subject: [PATCH 053/436] Don't actually use OVERWRITE by default...

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 7d0311f0..9e579f8f 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@
 # generate a Git version string
 GIT_VERSION := $(shell git describe --tag --long --always)
 
-DEFS = -DDOUBLE_PREC -DVERSION=\"$(GIT_VERSION)\" -DOVERWRITE
+DEFS = -DDOUBLE_PREC -DVERSION=\"$(GIT_VERSION)\"
 
 LCL = local# local,lad,sdu,archer
 CMP = gcc# intel,gcc,nagfor,cray,nvhpc

From 1fb76917fb9aff179ee6fdf8ee5a0ee354201280 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 16 Dec 2022 14:11:03 +0000
Subject: [PATCH 054/436] Remove unused variables from OVERWRITE case (#124)

---
 src/fft_common_3d.f90 | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/fft_common_3d.f90 b/src/fft_common_3d.f90
index eb809b37..fdee1b43 100644
--- a/src/fft_common_3d.f90
+++ b/src/fft_common_3d.f90
@@ -23,9 +23,8 @@ subroutine fft_3d_c2c(in, out, isign)
    complex(mytype), dimension(:, :, :), intent(OUT) :: out
    integer, intent(IN) :: isign
 
-   integer :: i, j, k
-
 #ifndef OVERWRITE
+   integer :: i, j, k
    complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 
@@ -202,9 +201,8 @@ subroutine fft_3d_c2r(in_c, out_r)
    complex(mytype), dimension(:, :, :), intent(INOUT) :: in_c
    real(mytype), dimension(:, :, :), intent(OUT) :: out_r
 
-   integer :: i, j, k
-
 #ifndef OVERWRITE
+   integer :: i, j, k
    complex(mytype), allocatable, dimension(:, :, :) :: wk1
 #endif
 

From 26394e25a2230d80341efb1a3f63e5729884919d Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 16 Dec 2022 18:09:59 +0000
Subject: [PATCH 055/436] Replace character*(*) -> character(len=*) (#125)

The old style character*(*) is deprecated by newer compilers
---
 src/io.f90 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/io.f90 b/src/io.f90
index b1d49641..b6d137c7 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -423,7 +423,7 @@ subroutine adios2_read_one_real(var, engine_name, varname, io_name)
 
       character(len=*), intent(in) :: engine_name
       character(len=*), intent(in) :: io_name
-      character*(*), intent(in) :: varname
+      character(len=*), intent(in) :: varname
       real(mytype), contiguous, dimension(:, :, :), intent(out) :: var
 
       integer :: ierror
@@ -1387,7 +1387,7 @@ subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplan
       integer, intent(in), optional :: opt_nplanes
 
       integer :: nplanes
-      character*(*), intent(in) :: varname
+      character(len=*), intent(in) :: varname
 #ifdef ADIOS2
       integer, dimension(3) :: sizes, subsizes, starts
       type(adios2_io) :: io_handle

From 8c52a54656c259779ef03c3bc8cd115f8f196325 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 16 Dec 2022 21:35:24 +0000
Subject: [PATCH 056/436] Don't track example executables

---
 examples/fft_physical_x/.gitignore | 2 ++
 examples/fft_physical_z/.gitignore | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/examples/fft_physical_x/.gitignore b/examples/fft_physical_x/.gitignore
index d32650f6..91b1e5ef 100644
--- a/examples/fft_physical_x/.gitignore
+++ b/examples/fft_physical_x/.gitignore
@@ -1,2 +1,4 @@
 fft_physical_x
 fft_grid_x
+fft_c2c_x
+fft_r2c_x
diff --git a/examples/fft_physical_z/.gitignore b/examples/fft_physical_z/.gitignore
index 26337569..229609ed 100644
--- a/examples/fft_physical_z/.gitignore
+++ b/examples/fft_physical_z/.gitignore
@@ -1 +1,3 @@
 fft_physical_z
+fft_c2c_z
+fft_r2c_z

From a26b50fb471e5bb3590faafd18739c601b240178 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 17 Dec 2022 10:01:45 +0100
Subject: [PATCH 057/436] Add a subroutine to close the IO unit of the log

---
 src/decomp_2d.f90     |  6 +++++-
 src/fft_cufft.f90     |  9 ++-------
 src/fft_fftw3.f90     |  9 ++-------
 src/fft_fftw3_f03.f90 |  9 ++-------
 src/fft_generic.f90   | 10 ++--------
 src/fft_mkl.f90       |  9 ++-------
 src/log.f90           | 34 +++++++++++++++++++++++++++-------
 7 files changed, 42 insertions(+), 44 deletions(-)

diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index b442b820..79fec912 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -244,7 +244,7 @@ module decomp_2d
              update_halo, decomp_2d_abort, &
              decomp_2d_warning, get_decomp_info, &
              decomp_mpi_comm_free, get_decomp_dims, &
-             d2d_listing_get_unit
+             d2d_listing_get_unit, d2d_listing_close_unit
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! These are routines to perform global data transpositions
@@ -348,6 +348,10 @@ module function d2d_listing_get_unit()
          integer :: d2d_listing_get_unit
       end function d2d_listing_get_unit
 
+      module subroutine d2d_listing_close_unit(io_unit)
+         integer, intent(in) :: io_unit
+      end subroutine d2d_listing_close_unit
+
       module subroutine d2d_listing(given_io_unit)
          integer, intent(in), optional :: given_io_unit
       end subroutine d2d_listing
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 3c60be8f..28894fce 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -236,13 +236,11 @@ end subroutine c2r_1m_z_plan
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
-      use iso_fortran_env, only: output_unit
-
       implicit none
 
       !integer*4 :: cufft_ws, ws
       integer(int_ptr_kind()) :: cufft_ws, ws
-      integer   :: i, j, istat, iounit, ierror
+      integer :: i, j, istat, iounit
 
       if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
           (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
@@ -251,10 +249,7 @@ subroutine init_fft_engine
          write (iounit, *) ' '
          write (iounit, *) '***** Using the New cuFFT engine *****'
          write (iounit, *) ' '
-         if (iounit /= output_unit) then
-            close (iounit, iostat=ierror)
-            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         end if
+         call d2d_listing_close_unit(iounit)
       end if
 
       cufft_ws = 0
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index d8b6d73c..2530ad1c 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -266,11 +266,9 @@ end subroutine c2r_1m_z_plan
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
-      use iso_fortran_env, only: output_unit
-
       implicit none
 
-      integer :: iounit, ierror
+      integer :: iounit
 
       if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
           (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
@@ -279,10 +277,7 @@ subroutine init_fft_engine
          write (iounit, *) ' '
          write (iounit, *) '***** Using the FFTW (version 3.x) engine *****'
          write (iounit, *) ' '
-         if (iounit /= output_unit) then
-            close (iounit, iostat=ierror)
-            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         end if
+         call d2d_listing_close_unit(iounit)
       end if
 
       if (format == PHYSICAL_IN_X) then
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 155beef7..d4084d67 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -572,11 +572,9 @@ end subroutine c2r_1m_z_plan
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
-      use iso_fortran_env, only: output_unit
-
       implicit none
 
-      integer :: iounit, ierror
+      integer :: iounit
 
       if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
           (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
@@ -585,10 +583,7 @@ subroutine init_fft_engine
          write (iounit, *) ' '
          write (iounit, *) '***** Using the FFTW (F2003 interface) engine *****'
          write (iounit, *) ' '
-         if (iounit /= output_unit) then
-            close (iounit, iostat=ierror)
-            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         end if
+         call d2d_listing_close_unit(iounit)
       end if
 
       if (format == PHYSICAL_IN_X) then
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index 13d1001d..d4786710 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -35,11 +35,9 @@ module decomp_2d_fft
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
-      use iso_fortran_env, only: output_unit
-
       implicit none
 
-      integer :: cbuf_size, iounit, ierror
+      integer :: cbuf_size, iounit
 
       if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
           (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
@@ -48,10 +46,7 @@ subroutine init_fft_engine
          write (iounit, *) ' '
          write (iounit, *) '***** Using the generic FFT engine *****'
          write (iounit, *) ' '
-         if (iounit /= output_unit) then
-            close (iounit, iostat=ierror)
-            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         end if
+         call d2d_listing_close_unit(iounit)
       end if
 
       cbuf_size = max(ph%xsz(1), ph%ysz(2))
@@ -59,7 +54,6 @@ subroutine init_fft_engine
       allocate (buf(cbuf_size))
       allocate (scratch(cbuf_size))
 
-      return
    end subroutine init_fft_engine
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 7bfc146c..537fde46 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -42,11 +42,9 @@ module decomp_2d_fft
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine init_fft_engine
 
-      use iso_fortran_env, only: output_unit
-
       implicit none
 
-      integer :: iounit, ierror
+      integer :: iounit
 
       if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
           (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
@@ -55,10 +53,7 @@ subroutine init_fft_engine
          write (iounit, *) ' '
          write (iounit, *) '***** Using the MKL engine *****'
          write (iounit, *) ' '
-         if (iounit /= output_unit) then
-            close (iounit, iostat=ierror)
-            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-         end if
+         call d2d_listing_close_unit(iounit)
       end if
 
       ! For C2C transforms
diff --git a/src/log.f90 b/src/log.f90
index 04342b7d..73e493cd 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -74,6 +74,31 @@ module function d2d_listing_get_unit()
 
    end function d2d_listing_get_unit
 
+   !
+   ! Close the IO unit for the log if needed
+   !
+   module subroutine d2d_listing_close_unit(io_unit)
+
+      use iso_fortran_env, only: output_unit
+
+      implicit none
+
+      ! Input
+      integer, intent(in) :: io_unit
+
+      ! Local variables
+      integer :: ierror
+
+      !
+      ! Close the IO unit if it was not stdout
+      !
+      if (io_unit /= output_unit) then
+         close (io_unit, iostat=ierror)
+         if (ierror /= 0) call decomp_2d_abort(ierror, "Could not close log file")
+      end if
+
+   end subroutine d2d_listing_close_unit
+
    !
    ! Print some information about decomp_2d
    !
@@ -227,13 +252,8 @@ module subroutine d2d_listing(given_io_unit)
          end if
       end if
 #else
-      !
-      ! Close the IO unit if it was not stdout
-      !
-      if (io_unit /= output_unit) then
-         close (io_unit, iostat=ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
-      end if
+      ! Close the IO unit if needed
+      call d2d_listing_close_unit(io_unit)
 #endif
 
    end subroutine d2d_listing

From 451befb7c2e4e949106d3313b1214cd894337206 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 17 Dec 2022 10:13:03 +0100
Subject: [PATCH 058/436] Update examples : use the built-in memory allocator
 for C2C and R2C / C2R

---
 examples/fft_physical_x/fft_c2c_x.f90 |  8 +++++---
 examples/fft_physical_x/fft_r2c_x.f90 | 16 +++++++++-------
 examples/fft_physical_z/fft_c2c_z.f90 |  9 +++++++--
 examples/fft_physical_z/fft_r2c_z.f90 | 16 +++++++++-------
 4 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 0e46d01d..95a78623 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -19,6 +19,7 @@ program fft_c2c_x
 
    integer, parameter :: ntest = 10  ! repeat test this times
 
+   type(decomp_info), pointer :: ph => null()
    complex(mytype), allocatable, dimension(:, :, :) :: in, out
 
    real(mytype) :: dr, di, error, err_all, n1, flops
@@ -40,11 +41,11 @@ program fft_c2c_x
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
    call decomp_2d_fft_init(PHYSICAL_IN_X) ! force the default x pencil
-
+   ph => decomp_2d_fft_get_ph()
    !  input is X-pencil data
    ! output is Z-pencil data
-   allocate (in(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (out(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   call alloc_x(in, ph, .true.)
+   call alloc_z(out, ph, .true.)
    ! initilise input
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
@@ -120,6 +121,7 @@ program fft_c2c_x
    end if
 
    deallocate (in, out)
+   nullify (ph)
    call decomp_2d_fft_finalize
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 0603ab37..9324202a 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -19,11 +19,10 @@ program fft_r2c_x
 
    integer, parameter :: ntest = 10  ! repeat test this times
 
+   type(decomp_info), pointer :: ph => null(), sp => null()
    complex(mytype), allocatable, dimension(:, :, :) :: out
    real(mytype), allocatable, dimension(:, :, :) :: in_r
 
-   integer, dimension(3) :: fft_start, fft_end, fft_size
-
    real(mytype) :: dr, error, err_all
    integer :: ierror, i, j, k, m
    real(mytype) :: t1, t2, t3, t4
@@ -43,11 +42,12 @@ program fft_r2c_x
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    call decomp_2d_fft_init(PHYSICAL_IN_X)
 
-   allocate (in_r(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   call decomp_2d_fft_get_size(fft_start, fft_end, fft_size)
-   allocate (out(fft_start(1):fft_end(1), &
-                 fft_start(2):fft_end(2), &
-                 fft_start(3):fft_end(3)))
+   ph => decomp_2d_fft_get_ph()
+   sp => decomp_2d_fft_get_sp()
+   !  input is X-pencil data
+   ! output is Z-pencil data
+   call alloc_x(in_r, ph, .true.)
+   call alloc_z(out, sp, .true.)
 
    ! initilise input
    do k = xstart(3), xend(3)
@@ -111,6 +111,8 @@ program fft_r2c_x
    end if
 
    deallocate (in_r, out)
+   nullify (ph)
+   nullify (sp)
    call decomp_2d_fft_finalize
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index e3581bf6..68823bf3 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -20,6 +20,7 @@ program fft_c2c_z
 
    integer, parameter :: ntest = 10  ! repeat test this times
 
+   type(decomp_info), pointer :: ph => null()
    complex(mytype), allocatable, dimension(:, :, :) :: in, out
 
    real(mytype) :: dr, di, error, err_all, n1, flops
@@ -40,10 +41,13 @@ program fft_c2c_z
    ! Test the c2c interface
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
+
+   ph => decomp_2d_fft_get_ph()
    !  input is Z-pencil data
    ! output is X-pencil data
-   allocate (in(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   allocate (out(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   call alloc_z(in, ph, .true.)
+   call alloc_x(out, ph, .true.)
+
    ! initilise input
    do k = zstart(3), zend(3)
       do j = zstart(2), zend(2)
@@ -119,6 +123,7 @@ program fft_c2c_z
    end if
 
    deallocate (in, out)
+   nullify(ph)
    call decomp_2d_fft_finalize
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 321f2421..f9fdc13e 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -20,11 +20,10 @@ program fft_r2c_z
 
    integer, parameter :: ntest = 10  ! repeat test this times
 
+   type(decomp_info), pointer :: ph => null(), sp => null()
    complex(mytype), allocatable, dimension(:, :, :) :: out
    real(mytype), allocatable, dimension(:, :, :) :: in_r
 
-   integer, dimension(3) :: fft_start, fft_end, fft_size
-
    real(mytype) :: dr, error, err_all
    integer :: ierror, i, j, k, m
    real(mytype) :: t1, t2, t3, t4
@@ -44,11 +43,12 @@ program fft_r2c_z
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
 
-   allocate (in_r(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   call decomp_2d_fft_get_size(fft_start, fft_end, fft_size)
-   allocate (out(fft_start(1):fft_end(1), &
-                 fft_start(2):fft_end(2), &
-                 fft_start(3):fft_end(3)))
+   ph => decomp_2d_fft_get_ph()
+   sp => decomp_2d_fft_get_sp()
+   !  input is Z-pencil data
+   ! output is X-pencil data
+   call alloc_z(in_r, ph, .true.)
+   call alloc_x(out, sp, .true.)
 
    ! initilise input
    do k = zstart(3), zend(3)
@@ -114,6 +114,8 @@ program fft_r2c_z
    end if
 
    deallocate (in_r, out)
+   nullify(ph)
+   nullify(sp)
    call decomp_2d_fft_finalize
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)

From b2c3f1b4c1a69ccbfcc19bc8210075b51bf7b4e6 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 17 Dec 2022 10:23:56 +0100
Subject: [PATCH 059/436] Update examples : use the built-in allocator for
 io_test

---
 examples/io_test/io_bench.f90      |  4 +--
 examples/io_test/io_plane_test.f90 |  8 ++---
 examples/io_test/io_read.f90       |  8 ++---
 examples/io_test/io_test.f90       | 16 ++++-----
 examples/io_test/io_var_test.f90   | 52 +++++++++++++-----------------
 5 files changed, 41 insertions(+), 47 deletions(-)

diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index 4e3200ca..b1cabb76 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -19,7 +19,7 @@ program io_bench
    call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+   call alloc_x(u1, .true.)
    call random_number(u1)
 
    t1 = MPI_WTIME()
@@ -28,9 +28,9 @@ program io_bench
 
    if (nrank == 0) write (*, *) 'I/O time: ', t2 - t1
 
+   deallocate (u1)
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
-   deallocate (u1)
 
 end program io_bench
 
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 7bebfa03..e883d5c1 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -33,9 +33,9 @@ program io_plane_test
       end do
    end do
 
-   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   call alloc_x(u1, .true.)
+   call alloc_y(u2, .true.)
+   call alloc_z(u3, .true.)
 
    ! original X-pensil based data
    do k = xstart(3), xend(3)
@@ -121,8 +121,8 @@ program io_plane_test
 
    end if
 
+   deallocate (u1, u2, u3)
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
-   deallocate (u1, u2, u3)
 
 end program io_plane_test
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 694f1f21..4803cf3c 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -45,9 +45,9 @@ program io_read
       end do
    end do
 
-   allocate (u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   call alloc_x(u1b, .true.)
+   call alloc_y(u2b, .true.)
+   call alloc_z(u3b, .true.)
 
    ! read back to different arrays
    call decomp_2d_read_one(1, u1b, '.', 'u1.dat', 'test', reduce_prec=.false.)
@@ -79,8 +79,8 @@ program io_read
       end do
    end do
 
+   deallocate (u1b, u2b, u3b)
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
-   deallocate (u1b, u2b, u3b)
 
 end program io_read
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index eecdcfeb..7e0b1fbf 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -44,13 +44,13 @@ program io_test
       end do
    end do
 
-   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   call alloc_x(u1, .true.)
+   call alloc_y(u2, .true.)
+   call alloc_z(u3, .true.)
 
-   allocate (u1b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
+   call alloc_x(u1b, .true.)
+   call alloc_y(u2b, .true.)
+   call alloc_z(u3b, .true.)
 
    ! original x-pencil based data
    do k = xstart(3), xend(3)
@@ -125,9 +125,9 @@ program io_test
       end do
    end do
 
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
    deallocate (u1, u2, u3)
    deallocate (u1b, u2b, u3b)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
 end program io_test
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 7f91e01d..a35eeeca 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -90,32 +90,26 @@ program io_var_test
    end do
 
    ! allocate memory
-   allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   allocate (u1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (u2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (u3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-   allocate (cu1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (cu2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (cu3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-   allocate (cu1_b(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-   allocate (cu2_b(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-   allocate (cu3_b(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-
-   allocate (u1l(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
-                 large%xst(3):large%xen(3)))
-   allocate (u2l(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
-                 large%yst(3):large%yen(3)))
-   allocate (u3l(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
-                 large%zst(3):large%zen(3)))
-   allocate (u1l_b(large%xst(1):large%xen(1), large%xst(2):large%xen(2), &
-                   large%xst(3):large%xen(3)))
-   allocate (u2l_b(large%yst(1):large%yen(1), large%yst(2):large%yen(2), &
-                   large%yst(3):large%yen(3)))
-   allocate (u3l_b(large%zst(1):large%zen(1), large%zst(2):large%zen(2), &
-                   large%zst(3):large%zen(3)))
+   call alloc_x(u1, .true.)
+   call alloc_y(u2, .true.)
+   call alloc_z(u3, .true.)
+   call alloc_x(u1_b, .true.)
+   call alloc_y(u2_b, .true.)
+   call alloc_z(u3_b, .true.)
+
+   call alloc_x(cu1, .true.)
+   call alloc_y(cu2, .true.)
+   call alloc_z(cu3, .true.)
+   call alloc_x(cu1_b, .true.)
+   call alloc_y(cu2_b, .true.)
+   call alloc_z(cu3_b, .true.)
+
+   call alloc_x(u1l, large, .true.)
+   call alloc_y(u2l, large, .true.)
+   call alloc_z(u3l, large, .true.)
+   call alloc_x(u1l_b, large, .true.)
+   call alloc_y(u2l_b, large, .true.)
+   call alloc_z(u3l_b, large, .true.)
 
    ! distribute the data
    do k = xstart(3), xend(3)
@@ -271,15 +265,15 @@ program io_var_test
    if (nrank == 0) write (*, *) 'passed self test'
 
    ! clean up
-   call decomp_info_finalize(large)
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
    deallocate (u1, u2, u3)
    deallocate (u1l, u2l, u3l)
    deallocate (cu1, cu2, cu3)
    deallocate (u1_b, u2_b, u3_b)
    deallocate (u1l_b, u2l_b, u3l_b)
    deallocate (cu1_b, cu2_b, cu3_b)
+   call decomp_info_finalize(large)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
    deallocate (data1_large)
 
 end program io_var_test

From dfa78a0561d051383d38b73987ea2957a9c9d1f8 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 17 Dec 2022 10:37:17 +0100
Subject: [PATCH 060/436] Update examples : use the built-in allocator for
 halo_test

---
 examples/halo_test/halo_test.f90 | 110 ++++++++++++++++---------------
 1 file changed, 58 insertions(+), 52 deletions(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 2222d3c4..8c64b7a6 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -73,17 +73,19 @@ program halo_test
 
    subroutine initialise()
 
-      ! initialise u,v,w with random numbers in X-pencil
+      implicit none
+
 #ifdef HALO_GLOBAL
-      allocate (u1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      allocate (v1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      allocate (w1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
+      logical, parameter :: global = .true.
 #else
-      allocate (u1(xsize(1), xsize(2), xsize(3)))
-      allocate (v1(xsize(1), xsize(2), xsize(3)))
-      allocate (w1(xsize(1), xsize(2), xsize(3)))
+      logical, parameter :: global = .false.
 #endif
 
+      ! initialise u,v,w with random numbers in X-pencil
+      call alloc_x(u1, global)
+      call alloc_x(v1, global)
+      call alloc_x(w1, global)
+
       call random_seed(size=n)
       allocate (seed(n))
       seed = nrank + 1
@@ -101,17 +103,23 @@ end subroutine initialise
    !=====================================================================
    subroutine test_div_transpose()
 
+      implicit none
+
+#ifdef HALO_GLOBAL
+      logical, parameter :: global = .true.
+#else
+      logical, parameter :: global = .false.
+#endif
       integer :: i1, in ! I loop start/end
       integer :: j1, jn ! J loop start/end
       integer :: k1, kn ! K loop start/end
 
       ! du/dx calculated on X-pencil
+      call alloc_x(div1, global)
 #ifdef HALO_GLOBAL
-      allocate (div1(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
       k1 = xstart(3); kn = xend(3)
       j1 = xstart(2); jn = xend(2)
 #else
-      allocate (div1(xsize(1), xsize(2), xsize(3)))
       k1 = 1; kn = xsize(3)
       j1 = 1; jn = xsize(2)
 #endif
@@ -127,14 +135,12 @@ subroutine test_div_transpose()
       end do
 
       ! dv/dy calculated on Y-pencil
+      call alloc_y(v2, global)
+      call alloc_y(wk2, global)
 #ifdef HALO_GLOBAL
-      allocate (v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
       k1 = ystart(3); kn = yend(3)
       i1 = ystart(1); in = yend(1)
 #else
-      allocate (v2(ysize(1), ysize(2), ysize(3)))
-      allocate (wk2(ysize(1), ysize(2), ysize(3)))
       k1 = 1; kn = ysize(3)
       i1 = 1; in = ysize(1)
 #endif
@@ -152,16 +158,13 @@ subroutine test_div_transpose()
       end do
 
       ! dw/dz calculated on Z-pencil
+      call alloc_y(w2, global)
+      call alloc_z(w3, global)
+      call alloc_z(wk3, global)
 #ifdef HALO_GLOBAL
-      allocate (w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-      allocate (wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
       j1 = zstart(2); jn = zend(2)
       i1 = zstart(1); in = zend(1)
 #else
-      allocate (w2(ysize(1), ysize(2), ysize(3)))
-      allocate (w3(zsize(1), zsize(2), zsize(3)))
-      allocate (wk3(zsize(1), zsize(2), zsize(3)))
       j1 = 1; jn = zsize(2)
       i1 = 1; in = zsize(1)
 #endif
@@ -199,6 +202,13 @@ end subroutine test_div_transpose
    !=====================================================================
    subroutine test_div_haloX()
 
+      implicit none
+
+#ifdef HALO_GLOBAL
+      logical, parameter :: global = .true.
+#else
+      logical, parameter :: global = .false.
+#endif
       integer :: i1, in ! I loop start/end
       integer :: j1, jn ! J loop start/end
       integer :: k1, kn ! K loop start/end
@@ -208,15 +218,14 @@ subroutine test_div_haloX()
       ny_expected = xsize(2) + 2
       nz_expected = xsize(3) + 2
 
+      call alloc_x(div2, global)
 #ifdef HALO_GLOBAL
-      allocate (div2(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
       call update_halo(v1, vh, 1, opt_global=.true., opt_pencil=1)
       call update_halo(w1, wh, 1, opt_global=.true., opt_pencil=1)
 
       k1 = xstart(3); kn = xend(3)
       j1 = xstart(2); jn = xend(2)
 #else
-      allocate (div2(xsize(1), xsize(2), xsize(3)))
       call update_halo(v1, vh, 1, opt_pencil=1)
       call update_halo(w1, wh, 1, opt_pencil=1)
 
@@ -251,6 +260,13 @@ end subroutine test_div_haloX
    !=====================================================================
    subroutine test_div_haloY()
 
+      implicit none
+
+#ifdef HALO_GLOBAL
+      logical, parameter :: global = .true.
+#else
+      logical, parameter :: global = .false.
+#endif
       integer :: i1, in ! I loop start/end
       integer :: j1, jn ! J loop start/end
       integer :: k1, kn ! K loop start/end
@@ -260,19 +276,12 @@ subroutine test_div_haloY()
       ny_expected = ny
       nz_expected = ysize(3) + 2
 
-#ifdef HALO_GLOBAL
-      allocate (u2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (v2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (w2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (div3(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-#else
-      allocate (u2(ysize(1), ysize(2), ysize(3)))
-      allocate (v2(ysize(1), ysize(2), ysize(3)))
-      allocate (w2(ysize(1), ysize(2), ysize(3)))
-      allocate (div3(xsize(1), xsize(2), xsize(3)))
-      allocate (wk2(ysize(1), ysize(2), ysize(3)))
-#endif
+      call alloc_y(u2, global)
+      call alloc_y(v2, global)
+      call alloc_y(w2, global)
+      call alloc_x(div3, global)
+      call alloc_y(wk2, global)
+
       call transpose_x_to_y(u1, u2)
       call transpose_x_to_y(v1, v2)
       call transpose_x_to_y(w1, w2)
@@ -318,33 +327,30 @@ end subroutine test_div_haloY
    !=====================================================================
    subroutine test_div_haloZ()
 
+      implicit none
+
+#ifdef HALO_GLOBAL
+      logical, parameter :: global = .true.
+#else
+      logical, parameter :: global = .false.
+#endif
+
       ! Expected sizes
       nx_expected = zsize(1) + 2
       ny_expected = zsize(2) + 2
       nz_expected = nz
 
-#ifdef HALO_GLOBAL
-      allocate (u3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-      allocate (v3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-      allocate (w3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-#else
-      allocate (u3(zsize(1), zsize(2), zsize(3)))
-      allocate (v3(zsize(1), zsize(2), zsize(3)))
-      allocate (w3(zsize(1), zsize(2), zsize(3)))
-#endif
+      call alloc_z(u3, global)
+      call alloc_z(v3, global)
+      call alloc_z(w3, global)
+
       call transpose_y_to_z(u2, u3)
       call transpose_y_to_z(v2, v3)
       call transpose_y_to_z(w2, w3)
 
-#ifdef HALO_GLOBAL
-      allocate (div4(xstart(1):xend(1), xstart(2):xend(2), xstart(3):xend(3)))
-      allocate (wk2(ystart(1):yend(1), ystart(2):yend(2), ystart(3):yend(3)))
-      allocate (wk3(zstart(1):zend(1), zstart(2):zend(2), zstart(3):zend(3)))
-#else
-      allocate (div4(xsize(1), xsize(2), xsize(3)))
-      allocate (wk2(ysize(1), ysize(2), ysize(3)))
-      allocate (wk3(zsize(1), zsize(2), zsize(3)))
-#endif
+      call alloc_x(div4, global)
+      call alloc_y(wk2, global)
+      call alloc_z(wk3, global)
 
       ! du/dx
 #ifdef HALO_GLOBAL

From f3e4bfc80ac03bb7c83ad9b677d42e76430328e2 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 17 Dec 2022 10:44:28 +0100
Subject: [PATCH 061/436] Minor update in the log

---
 src/log.f90 | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/log.f90 b/src/log.f90
index 73e493cd..9f6933ed 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -246,8 +246,7 @@ module subroutine d2d_listing(given_io_unit)
                                                   ierror, &
                                                   "No name for the log file")
             ! Close the IO unit to print the environment variables
-            close (io_unit, iostat=ierror)
-            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "Could not close log file")
+            call d2d_listing_close_unit(io_unit)
             call execute_command_line("env >> "//trim(fname), wait=.true.)
          end if
       end if

From 9c1d36b9eca7becabb6fd225fac0b52ca4fbdc4b Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 19 Dec 2022 10:30:01 +0100
Subject: [PATCH 062/436] Fuse working arrays wk2_c2c and wk2_r2c

---
 src/fft_common.f90  | 10 ++++++----
 src/fft_generic.f90 |  1 +
 src/fft_mkl.f90     |  1 +
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index e0cd9cac..8d2e2c34 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -35,8 +35,10 @@
 TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
 
 ! Workspace to store the intermediate Y-pencil data
-! *** TODO: investigate how to use only one workspace array
-complex(mytype), allocatable, dimension(:, :, :) :: wk2_c2c, wk2_r2c
+complex(mytype), allocatable, target, dimension(:, :, :) :: wk2_c2c
+complex(mytype), pointer, dimension(:,:,:) :: wk2_r2c
+! Workspace for r2c and c2r transforms
+! FIXME could be removed using in-place r2c and c2r ?
 complex(mytype), allocatable, dimension(:, :, :) :: wk13
 
 public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
@@ -129,7 +131,7 @@ subroutine fft_init_general(pencil, nx, ny, nz)
    end if
 
    allocate (wk2_c2c(ph%ysz(1), ph%ysz(2), ph%ysz(3)), STAT=status)
-   allocate (wk2_r2c(sp%ysz(1), sp%ysz(2), sp%ysz(3)), STAT=status)
+   call c_f_pointer(c_loc(wk2_c2c), wk2_r2c, sp%ysz)
    if (format == PHYSICAL_IN_X) then
       allocate (wk13(sp%xsz(1), sp%xsz(2), sp%xsz(3)), STAT=status)
    else if (format == PHYSICAL_IN_Z) then
@@ -171,7 +173,7 @@ subroutine decomp_2d_fft_finalize
    call decomp_info_finalize(sp)
 
    if (allocated(wk2_c2c)) deallocate (wk2_c2c)
-   if (allocated(wk2_r2c)) deallocate (wk2_r2c)
+   if (associated(wk2_r2c)) nullify (wk2_r2c)
    if (allocated(wk13)) deallocate (wk13)
 
    call finalize_fft_engine
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index d4786710..dfd1d25c 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -13,6 +13,7 @@
 
 module decomp_2d_fft
 
+   use iso_c_binding, only : c_f_pointer, c_loc
    use decomp_2d_constants
    use decomp_2d  ! 2D decomposition module
    use glassman
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 537fde46..e66799ef 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -13,6 +13,7 @@
 
 module decomp_2d_fft
 
+   use iso_c_binding, only : c_f_pointer, c_loc
    use decomp_2d_constants
    use decomp_2d  ! 2D decomposition module
    use MKL_DFTI   ! MKL FFT module

From d72bb462a3ddba8c9afde2b8503da9957bd84de0 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 19 Dec 2022 10:33:28 +0100
Subject: [PATCH 063/436] Initialize wk2_r2c to null

---
 src/fft_common.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 8d2e2c34..abcc5e10 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -36,7 +36,7 @@
 
 ! Workspace to store the intermediate Y-pencil data
 complex(mytype), allocatable, target, dimension(:, :, :) :: wk2_c2c
-complex(mytype), pointer, dimension(:,:,:) :: wk2_r2c
+complex(mytype), pointer, dimension(:,:,:) :: wk2_r2c => null()
 ! Workspace for r2c and c2r transforms
 ! FIXME could be removed using in-place r2c and c2r ?
 complex(mytype), allocatable, dimension(:, :, :) :: wk13

From 62693d3e4bc6f824954b1d61c7d4eac4672088bb Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 19 Dec 2022 13:21:17 +0100
Subject: [PATCH 064/436] Update comments

---
 src/fft_common.f90  | 25 ++++++++++++++++++++++++-
 src/fft_generic.f90 |  2 +-
 src/fft_mkl.f90     |  2 +-
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index abcc5e10..48f6dd4a 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -36,7 +36,7 @@
 
 ! Workspace to store the intermediate Y-pencil data
 complex(mytype), allocatable, target, dimension(:, :, :) :: wk2_c2c
-complex(mytype), pointer, dimension(:,:,:) :: wk2_r2c => null()
+complex(mytype), pointer, dimension(:, :, :) :: wk2_r2c => null()
 ! Workspace for r2c and c2r transforms
 ! FIXME could be removed using in-place r2c and c2r ?
 complex(mytype), allocatable, dimension(:, :, :) :: wk13
@@ -130,8 +130,31 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       call decomp_info_init(nx, ny, nz/2 + 1, sp)
    end if
 
+   !
+   ! Allocate the workspace for intermediate y-pencil data
+   ! The largest memory block needed is the one for c2c transforms
+   !
    allocate (wk2_c2c(ph%ysz(1), ph%ysz(2), ph%ysz(3)), STAT=status)
+   if (status /= 0) then
+      errorcode = 3
+      call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                           'Out of memory when initialising FFT')
+   end if
+   !
+   ! A smaller memory block is needed for r2c and c2r transforms
+   ! wk2_c2c and wk2_r2c start at the same memory location
+   !
+   !    Size of wk2_c2c : ph%ysz(1), ph%ysz(2), ph%ysz(3)
+   !    Size of wk2_r2c : sp%ysz(1), sp%ysz(2), sp%ysz(3)
+   !
    call c_f_pointer(c_loc(wk2_c2c), wk2_r2c, sp%ysz)
+   !
+   ! Allocate the workspace for r2c and c2r transforms
+   !
+   ! wk13 can not be easily fused with wk2_*2c due to statements such as
+   ! transpose_y_to_x(wk2_r2c, wk13, sp)
+   ! transpose_y_to_z(wk2_r2c, wk13, sp)
+   !
    if (format == PHYSICAL_IN_X) then
       allocate (wk13(sp%xsz(1), sp%xsz(2), sp%xsz(3)), STAT=status)
    else if (format == PHYSICAL_IN_Z) then
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index dfd1d25c..18606290 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -13,7 +13,7 @@
 
 module decomp_2d_fft
 
-   use iso_c_binding, only : c_f_pointer, c_loc
+   use iso_c_binding, only: c_f_pointer, c_loc
    use decomp_2d_constants
    use decomp_2d  ! 2D decomposition module
    use glassman
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index e66799ef..3b7500a7 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -13,7 +13,7 @@
 
 module decomp_2d_fft
 
-   use iso_c_binding, only : c_f_pointer, c_loc
+   use iso_c_binding, only: c_f_pointer, c_loc
    use decomp_2d_constants
    use decomp_2d  ! 2D decomposition module
    use MKL_DFTI   ! MKL FFT module

From e535e031aa2d20e127a64e1b58dc602659863f35 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 19 Dec 2022 14:35:27 +0100
Subject: [PATCH 065/436] Fix MKL backend

---
 src/fft_common.f90 |  2 +-
 src/fft_mkl.f90    | 16 ++++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 48f6dd4a..3e749d0c 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -36,7 +36,7 @@
 
 ! Workspace to store the intermediate Y-pencil data
 complex(mytype), allocatable, target, dimension(:, :, :) :: wk2_c2c
-complex(mytype), pointer, dimension(:, :, :) :: wk2_r2c => null()
+complex(mytype), contiguous, pointer, dimension(:, :, :) :: wk2_r2c => null()
 ! Workspace for r2c and c2r transforms
 ! FIXME could be removed using in-place r2c and c2r ?
 complex(mytype), allocatable, dimension(:, :, :) :: wk13
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 3b7500a7..019efbe4 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -542,14 +542,14 @@ subroutine fft_3d_r2c(in_r, out_c)
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, sp%ysz(3)
-            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(:, :, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2_r2c(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y2, wk2_r2c(:, :, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif
@@ -584,14 +584,14 @@ subroutine fft_3d_r2c(in_r, out_c)
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, sp%ysz(3)
-            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(:, :, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeForward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2_r2c(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y2, wk2_r2c(:, :, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif
@@ -670,14 +670,14 @@ subroutine fft_3d_c2r(in_c, out_r)
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, sp%ysz(3)
-            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(:, :, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2_r2c(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y2, wk2_r2c(:, :, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif
@@ -716,14 +716,14 @@ subroutine fft_3d_c2r(in_c, out_r)
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
          do k = 1, sp%ysz(3)
-            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(1, 1, k), isign)
+            status = wrapper_c2c_inplace(c2c_y2, wk2_r2c(:, :, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(sp%ysz(1), sp%ysz(2), sp%ysz(3)))
          do k = 1, sp%ysz(3)
             !          status = DftiComputeBackward(c2c_y2, wk2(:,1,k), wk2b(:,1,k))
-            status = wrapper_c2c(c2c_y2, wk2_r2c(1, 1, k), wk2b(1, 1, k), isign)
+            status = wrapper_c2c(c2c_y2, wk2_r2c(:, :, k), wk2b(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #endif

From 6c8de8066da5c564d4566216db1fe6f6915e4bbd Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 09:46:47 +0100
Subject: [PATCH 066/436] fftw3 : r2c and c2r transforms do not always preserve
 their input argument

---
 src/fft_fftw3.f90 | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index 2530ad1c..da5bdcc3 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -425,7 +425,7 @@ subroutine r2c_1m_x(input, output)
 
       implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(INOUT)  ::  input
       complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
@@ -443,7 +443,7 @@ subroutine r2c_1m_z(input, output)
 
       implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN)  ::  input
+      real(mytype), dimension(:, :, :), intent(INOUT)  ::  input
       complex(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
@@ -461,7 +461,7 @@ subroutine c2r_1m_x(input, output)
 
       implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN)  ::  input
+      complex(mytype), dimension(:, :, :), intent(INOUT)  ::  input
       real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
@@ -479,7 +479,7 @@ subroutine c2r_1m_z(input, output)
 
       implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: input
+      complex(mytype), dimension(:, :, :), intent(INOUT) :: input
       real(mytype), dimension(:, :, :), intent(OUT) :: output
 
 #ifdef DOUBLE_PREC
@@ -600,7 +600,7 @@ subroutine fft_3d_r2c(in_r, out_c)
 
       implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN) :: in_r
+      real(mytype), dimension(:, :, :), intent(INOUT) :: in_r
       complex(mytype), dimension(:, :, :), intent(OUT) :: out_c
 
       if (format == PHYSICAL_IN_X) then

From dca71b8b1b91a0bee884f19fd5a47a2061fd8134 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 09:52:32 +0100
Subject: [PATCH 067/436] Add some error check for fft/init

---
 src/decomp_2d_init_fin.f90 | 5 ++++-
 src/fft_common.f90         | 6 ++++++
 src/fft_fftw3_f03.f90      | 6 ++++++
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index 58560c9c..001e7acd 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -59,6 +59,9 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
                                                 __LINE__, &
                                                 MPI_SUCCESS, &
                                                 "MPI error check is broken")
+     if (nx <= 0) call decomp_2d_abort(__FILE__, __LINE__, nx, "Invalid value for nx")
+     if (ny <= 0) call decomp_2d_abort(__FILE__, __LINE__, ny, "Invalid value for ny")
+     if (nz <= 0) call decomp_2d_abort(__FILE__, __LINE__, nz, "Invalid value for nz")
 
      ! If the external code has not set nrank and nproc
      if (nrank == -1) then
@@ -94,7 +97,7 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
         periodic_z = .false.
      end if
 
-     if (p_row == 0 .and. p_col == 0) then
+     if (p_row <= 0 .or. p_col <= 0) then
         ! determine the best 2D processor grid
         call best_2d_grid(nproc, row, col)
         p_row = row
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 3e749d0c..7cd4b04d 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -98,11 +98,15 @@ subroutine fft_init_general(pencil, nx, ny, nz)
    if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
 #endif
 
+   ! Safety checks
    if (initialised) then
       errorcode = 4
       call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
                            'FFT library should only be initialised once')
    end if
+   if (nx <= 0) call decomp_2d_abort(__FILE__, __LINE__, nx, "Invalid value for nx")
+   if (ny <= 0) call decomp_2d_abort(__FILE__, __LINE__, ny, "Invalid value for ny")
+   if (nz <= 0) call decomp_2d_abort(__FILE__, __LINE__, nz, "Invalid value for nz")
 
    format = pencil
    nx_fft = nx
@@ -128,6 +132,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       call decomp_info_init(nx/2 + 1, ny, nz, sp)
    else if (format == PHYSICAL_IN_Z) then
       call decomp_info_init(nx, ny, nz/2 + 1, sp)
+   else
+      call decomp_2d_abort(__FILE__, __LINE__, format, "Invalid value for format")
    end if
 
    !
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index d4084d67..5c5d51e1 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -125,11 +125,15 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       if (decomp_profiler_fft) call decomp_profiler_start("fft_init")
 #endif
 
+      ! Safety checks
       if (initialised) then
          errorcode = 4
          call decomp_2d_abort(errorcode, &
                               'FFT library should only be initialised once')
       end if
+      if (nx <= 0) call decomp_2d_abort(__FILE__, __LINE__, nx, "Invalid value for nx")
+      if (ny <= 0) call decomp_2d_abort(__FILE__, __LINE__, ny, "Invalid value for ny")
+      if (nz <= 0) call decomp_2d_abort(__FILE__, __LINE__, nz, "Invalid value for nz")
 
       format = pencil
       nx_fft = nx
@@ -155,6 +159,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
          call decomp_info_init(nx/2 + 1, ny, nz, sp)
       else if (format == PHYSICAL_IN_Z) then
          call decomp_info_init(nx, ny, nz/2 + 1, sp)
+      else
+         call decomp_2d_abort(__FILE__, __LINE__, format, "Invalid value for format")
       end if
 
       sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)

From 47c53f2db2071b619cb589f5bbbb81a1d475c3d6 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 09:57:14 +0100
Subject: [PATCH 068/436] FFT : move constants to the dedicated module

---
 examples/fft_physical_x/fft_c2c_x.f90  |  1 +
 examples/fft_physical_x/fft_grid_x.f90 |  1 +
 examples/fft_physical_x/fft_r2c_x.f90  |  1 +
 examples/fft_physical_z/fft_c2c_z.f90  |  3 ++-
 examples/fft_physical_z/fft_r2c_z.f90  |  1 +
 src/decomp_2d_constants.f90            | 12 ++++++++++++
 src/fft_common.f90                     |  7 -------
 src/fft_fftw3_f03.f90                  |  7 -------
 8 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 95a78623..eb251940 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -2,6 +2,7 @@ program fft_c2c_x
 
    use decomp_2d
    use decomp_2d_fft
+   use decomp_2d_constants
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 7955a81e..991828b8 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -2,6 +2,7 @@ program fft_physical_x
 
    use decomp_2d
    use decomp_2d_fft
+   use decomp_2d_constants
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 9324202a..26e26a1f 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -2,6 +2,7 @@ program fft_r2c_x
 
    use decomp_2d
    use decomp_2d_fft
+   use decomp_2d_constants
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 68823bf3..cc7ebdba 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -2,6 +2,7 @@ program fft_c2c_z
 
    use decomp_2d
    use decomp_2d_fft
+   use decomp_2d_constants
    use MPI
 #if defined(_GPU)
    use cudafor
@@ -123,7 +124,7 @@ program fft_c2c_z
    end if
 
    deallocate (in, out)
-   nullify(ph)
+   nullify (ph)
    call decomp_2d_fft_finalize
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index f9fdc13e..216cd6fc 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -2,6 +2,7 @@ program fft_r2c_z
 
    use decomp_2d
    use decomp_2d_fft
+   use decomp_2d_constants
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 41496e66..d1bf438c 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -69,5 +69,17 @@ module decomp_2d_constants
    integer, parameter, public :: D2D_FFT_BACKEND_MKL = 3
    integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4
 
+   !
+   ! Complex-to-complex FFT can be forward or backward
+   !
+   integer, parameter, public :: DECOMP_2D_FFT_FORWARD = -1
+   integer, parameter, public :: DECOMP_2D_FFT_BACKWARD = 1
+
+   !
+   ! Input / output of the FFT are distributed along different pencils
+   !
+   integer, parameter, public :: PHYSICAL_IN_X = 1 ! Forward is input in X, output in Z
+   integer, parameter, public :: PHYSICAL_IN_Z = 3 ! Forward is input in Z, output in X
+
 end module decomp_2d_constants
 
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 7cd4b04d..65693f25 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -11,13 +11,6 @@
 
 ! This file contains common code shared by all FFT engines
 
-integer, parameter, public :: DECOMP_2D_FFT_FORWARD = -1
-integer, parameter, public :: DECOMP_2D_FFT_BACKWARD = 1
-
-! Physical space data can be stored in either X-pencil or Z-pencil
-integer, parameter, public :: PHYSICAL_IN_X = 1
-integer, parameter, public :: PHYSICAL_IN_Z = 3
-
 integer, save :: format                 ! input X-pencil or Z-pencil
 
 ! The libary can only be initialised once
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 5c5d51e1..649c56e8 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -37,13 +37,6 @@ module decomp_2d_fft
    !     use plan(2,j) for c2r transforms;
    type(C_PTR), save :: plan(-1:2, 3)
 
-   integer, parameter, public :: DECOMP_2D_FFT_FORWARD = -1
-   integer, parameter, public :: DECOMP_2D_FFT_BACKWARD = 1
-
-   ! Physical space data can be stored in either X-pencil or Z-pencil
-   integer, parameter, public :: PHYSICAL_IN_X = 1
-   integer, parameter, public :: PHYSICAL_IN_Z = 3
-
    integer, parameter, public :: D2D_FFT_BACKEND = D2D_FFT_BACKEND_FFTW3_F03
 
    integer, save :: format                 ! input X-pencil or Z-pencil

From 01e9a4cc1f8a179f2e60c5583726fec12a194946 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 10:16:39 +0100
Subject: [PATCH 069/436] FFT : use a dedicated submodule to interact with the
 log

---
 Makefile              |  2 +-
 src/fft_common.f90    |  6 ++++
 src/fft_cufft.f90     | 14 ++------
 src/fft_fftw3.f90     | 12 +------
 src/fft_fftw3_f03.f90 | 18 ++++------
 src/fft_generic.f90   | 14 ++------
 src/fft_log.f90       | 79 +++++++++++++++++++++++++++++++++++++++++++
 src/fft_mkl.f90       | 12 +------
 8 files changed, 101 insertions(+), 56 deletions(-)
 create mode 100644 src/fft_log.f90

diff --git a/Makefile b/Makefile
index 9e579f8f..c3a9af46 100644
--- a/Makefile
+++ b/Makefile
@@ -99,7 +99,7 @@ SRCDIR = src
 DECOMPINC = mod
 FFLAGS += $(MODFLAG)$(DECOMPINC) -I$(DECOMPINC)
 
-SRCDECOMP := $(SRCDECOMP) fft_$(FFT).f90
+SRCDECOMP := $(SRCDECOMP) fft_$(FFT).f90 fft_log.f90
 SRCDECOMP_ = $(patsubst %.f90,$(SRCDIR)/%.f90,$(filter-out %/mkl_dfti.f90,$(SRCDECOMP)))
 SRCDECOMP_ += $(filter %/mkl_dfti.f90,$(SRCDECOMP))
 OBJDECOMP_MKL_ = $(patsubst $(MKLROOT)/include/%.f90,$(OBJDIR)/%.f90,$(filter %/mkl_dfti.f90,$(SRCDECOMP_)))
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 65693f25..ea1c51da 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -52,6 +52,12 @@
    module procedure fft_3d_c2r
 end interface
 
+interface
+   module subroutine decomp_2d_fft_log(backend)
+      character(len=*), intent(in) :: backend
+   end subroutine decomp_2d_fft_log
+end interface
+
 contains
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 28894fce..a3a1b057 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -240,17 +240,9 @@ subroutine init_fft_engine
 
       !integer*4 :: cufft_ws, ws
       integer(int_ptr_kind()) :: cufft_ws, ws
-      integer :: i, j, istat, iounit
-
-      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE_FULL)) then
-         iounit = d2d_listing_get_unit()
-         write (iounit, *) ' '
-         write (iounit, *) '***** Using the New cuFFT engine *****'
-         write (iounit, *) ' '
-         call d2d_listing_close_unit(iounit)
-      end if
+      integer :: i, j, istat
+
+      call decomp_2d_fft_log("cuFFT")
 
       cufft_ws = 0
 #ifdef DOUBLE_PREC
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index da5bdcc3..a6d400c4 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -268,17 +268,7 @@ subroutine init_fft_engine
 
       implicit none
 
-      integer :: iounit
-
-      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE_FULL)) then
-         iounit = d2d_listing_get_unit()
-         write (iounit, *) ' '
-         write (iounit, *) '***** Using the FFTW (version 3.x) engine *****'
-         write (iounit, *) ' '
-         call d2d_listing_close_unit(iounit)
-      end if
+      call decomp_2d_fft_log("FFTW (version 3.x)")
 
       if (format == PHYSICAL_IN_X) then
 
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 649c56e8..2d2ef2ec 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -78,6 +78,12 @@ module decomp_2d_fft
       module procedure fft_3d_c2r
    end interface
 
+   interface
+      module subroutine decomp_2d_fft_log(backend)
+         character(len=*), intent(in) :: backend
+      end subroutine decomp_2d_fft_log
+   end interface
+
 contains
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -573,17 +579,7 @@ subroutine init_fft_engine
 
       implicit none
 
-      integer :: iounit
-
-      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE_FULL)) then
-         iounit = d2d_listing_get_unit()
-         write (iounit, *) ' '
-         write (iounit, *) '***** Using the FFTW (F2003 interface) engine *****'
-         write (iounit, *) ' '
-         call d2d_listing_close_unit(iounit)
-      end if
+      call decomp_2d_fft_log("FFTW (F2003 interface)")
 
       if (format == PHYSICAL_IN_X) then
 
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index 18606290..047bc9aa 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -38,17 +38,9 @@ subroutine init_fft_engine
 
       implicit none
 
-      integer :: cbuf_size, iounit
-
-      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE_FULL)) then
-         iounit = d2d_listing_get_unit()
-         write (iounit, *) ' '
-         write (iounit, *) '***** Using the generic FFT engine *****'
-         write (iounit, *) ' '
-         call d2d_listing_close_unit(iounit)
-      end if
+      integer :: cbuf_size
+
+      call decomp_2d_fft_log("generic")
 
       cbuf_size = max(ph%xsz(1), ph%ysz(2))
       cbuf_size = max(cbuf_size, ph%zsz(3))
diff --git a/src/fft_log.f90 b/src/fft_log.f90
new file mode 100644
index 00000000..b293c6b8
--- /dev/null
+++ b/src/fft_log.f90
@@ -0,0 +1,79 @@
+!=======================================================================
+! This is part of the 2DECOMP&FFT library
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! decomposition. It also implements a highly scalable distributed
+! three-dimensional Fast Fourier Transform (FFT).
+!
+! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
+! Copyright (C) 2021               the University of Edinburgh (UoE)
+!
+!=======================================================================
+
+submodule(decomp_2d_fft) d2d_fft_log
+
+   use decomp_2d_constants
+   use decomp_2d
+
+   implicit none
+
+contains
+
+   !
+   ! Log subroutine for the decomp_2d_fft module
+   !
+   module subroutine decomp_2d_fft_log(backend)
+
+      implicit none
+
+      ! Argument
+      character(len=*), intent(in) :: backend
+
+      ! Local variable
+      integer :: io_unit
+
+      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
+          (decomp_log == D2D_LOG_TOFILE_FULL)) then
+         io_unit = d2d_listing_get_unit()
+         write (io_unit, *) ''
+         write (io_unit, *) '***** Using the '//trim(backend)//' FFT engine *****'
+         write (io_unit, *) ''
+         write (io_unit, *) 'Id of the backend : ', D2D_FFT_BACKEND
+         if (format == PHYSICAL_IN_X) then
+            write (io_unit, *) 'Format : Physical in x'
+         else
+            write (io_unit, *) 'Format : Physical in z'
+         end if
+         write (io_unit, *) ''
+         if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
+            write (io_unit, *) 'decomp_info object ph is a pointer to decomp_main'
+         end if
+         call decomp_info_print(ph, io_unit, "ph")
+         call decomp_info_print(sp, io_unit, "sp")
+         write (io_unit, *) ''
+#ifdef OVERWRITE
+         if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_GENERIC .or. &
+             D2D_FFT_BACKEND == D2D_FFT_BACKEND_CUFFT) then
+            call decomp_2d_warning("Selected FFT backend does not support overwrite")
+         else if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3 .or. &
+                  D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3_F03) then
+            write (io_unit, *) 'OVERWRITE support limited to c2C and c2r transforms'
+            write (io_unit, *) ''
+         else if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_MKL) then
+            write (io_unit, *) 'OVERWRITE support limited to c2c and c2r transforms'
+            write (io_unit, *) ''
+         endif
+#else
+         if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3 .or. &
+             D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3_F03) then
+            write (io_unit, *) 'Warning : c2c and c2r transforms do not preserve their complex input'
+            write (io_unit, *) ''
+         endif
+#endif
+         call d2d_listing_close_unit(io_unit)
+      end if
+
+   end subroutine decomp_2d_fft_log
+
+end submodule d2d_fft_log
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 019efbe4..29fe70ce 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -45,17 +45,7 @@ subroutine init_fft_engine
 
       implicit none
 
-      integer :: iounit
-
-      if ((decomp_log == D2D_LOG_STDOUT .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE .and. nrank == 0) .or. &
-          (decomp_log == D2D_LOG_TOFILE_FULL)) then
-         iounit = d2d_listing_get_unit()
-         write (iounit, *) ' '
-         write (iounit, *) '***** Using the MKL engine *****'
-         write (iounit, *) ' '
-         call d2d_listing_close_unit(iounit)
-      end if
+      call decomp_2d_fft_log("MKL")
 
       ! For C2C transforms
       call c2c_1m_x_plan(c2c_x, ph)

From 55ffcb0bcdf6de7529b684d563d08df507f29c1d Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 10:24:47 +0100
Subject: [PATCH 070/436] Fix 01e9a4cc1f8a179f2e60c5583726fec12a194946

---
 README.md       |  9 ++++++++-
 src/fft_log.f90 | 16 ++++------------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index aa4777ef..cca3f1bb 100644
--- a/README.md
+++ b/README.md
@@ -83,6 +83,13 @@ integer, parameter, public :: D2D_FFT_BACKEND_CUFFT = 4     ! Nvidia cuFFT
 exported by the module `decomp_2d_constants`.
 The external code can use the named variables to check the FFT backend used in a given build.
 
+### OVERWRITE flag
+
+- The generic backend does not support the OVERWRITE flag
+- The FFTW3 and FFTW3_F03 backends support the OVERWRITE flag for c2c and c2r transforms
+- The oneMKL backend supports the OVERWRITE flag for c2c and c2r transforms
+- The cuFFT backend supports the OVERWRITE flag for c2c anc c2r transforms
+
 ## Miscellaneous
 
 ### Print the log to a file or to stdout
@@ -126,7 +133,7 @@ This preprocessor variable is not valid for GPU builds. It leads to padded allto
 
 #### OVERWRITE
 
-This variable leads to overwrite the input array when computing FFT (complex-to-complex and complex-to-real).
+This variable leads to overwrite the input array when computing FFT. The support of this flag is variable, depending on the FFT backend selected, as described above.
 
 #### HALO_DEBUG
 
diff --git a/src/fft_log.f90 b/src/fft_log.f90
index b293c6b8..905833f9 100644
--- a/src/fft_log.f90
+++ b/src/fft_log.f90
@@ -53,22 +53,14 @@ module subroutine decomp_2d_fft_log(backend)
          call decomp_info_print(sp, io_unit, "sp")
          write (io_unit, *) ''
 #ifdef OVERWRITE
-         if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_GENERIC .or. &
-             D2D_FFT_BACKEND == D2D_FFT_BACKEND_CUFFT) then
+         if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_GENERIC) then
             call decomp_2d_warning("Selected FFT backend does not support overwrite")
          else if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3 .or. &
-                  D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3_F03) then
+                  D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3_F03 .or. &
+                  D2D_FFT_BACKEND == D2D_FFT_BACKEND_CUFFT .or. &
+                  D2D_FFT_BACKEND == D2D_FFT_BACKEND_MKL) then
             write (io_unit, *) 'OVERWRITE support limited to c2C and c2r transforms'
             write (io_unit, *) ''
-         else if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_MKL) then
-            write (io_unit, *) 'OVERWRITE support limited to c2c and c2r transforms'
-            write (io_unit, *) ''
-         endif
-#else
-         if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3 .or. &
-             D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3_F03) then
-            write (io_unit, *) 'Warning : c2c and c2r transforms do not preserve their complex input'
-            write (io_unit, *) ''
          endif
 #endif
          call d2d_listing_close_unit(io_unit)

From 3ba1c637f2457ae419a84b32022b84f62d93db7f Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 10:27:14 +0100
Subject: [PATCH 071/436] Remove the subroutine decomp_2d_get_size, the
 decomp_info object sp can be used instead

---
 src/fft_common.f90    | 25 +------------------------
 src/fft_fftw3_f03.f90 | 25 +------------------------
 2 files changed, 2 insertions(+), 48 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index ea1c51da..479c801a 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -35,7 +35,7 @@
 complex(mytype), allocatable, dimension(:, :, :) :: wk13
 
 public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-          decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+          decomp_2d_fft_finalize, &
           decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
 
 ! Declare generic interfaces to handle different inputs
@@ -215,29 +215,6 @@ subroutine decomp_2d_fft_finalize
    return
 end subroutine decomp_2d_fft_finalize
 
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-! Return the size, starting/ending index of the distributed array
-!  whose global size is (nx/2+1)*ny*nz, for defining data structures
-!  in r2c and c2r interfaces
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-subroutine decomp_2d_fft_get_size(istart, iend, isize)
-
-   implicit none
-   integer, dimension(3), intent(OUT) :: istart, iend, isize
-
-   if (format == PHYSICAL_IN_X) then
-      istart = sp%zst
-      iend = sp%zen
-      isize = sp%zsz
-   else if (format == PHYSICAL_IN_Z) then
-      istart = sp%xst
-      iend = sp%xen
-      isize = sp%xsz
-   end if
-
-   return
-end subroutine decomp_2d_fft_get_size
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Return a pointer to the decomp_info object ph
 !
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 2d2ef2ec..b4696864 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -61,7 +61,7 @@ module decomp_2d_fft
    type(C_PTR) :: wk2_c2c_p, wk2_r2c_p, wk13_p
 
    public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-             decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
+             decomp_2d_fft_finalize, &
              decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
 
    ! Declare generic interfaces to handle different inputs
@@ -224,29 +224,6 @@ subroutine decomp_2d_fft_finalize
       return
    end subroutine decomp_2d_fft_finalize
 
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Return the size, starting/ending index of the distributed array
-   !  whose global size is (nx/2+1)*ny*nz, for defining data structures
-   !  in r2c and c2r interfaces
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine decomp_2d_fft_get_size(istart, iend, isize)
-
-      implicit none
-      integer, dimension(3), intent(OUT) :: istart, iend, isize
-
-      if (format == PHYSICAL_IN_X) then
-         istart = sp%zst
-         iend = sp%zen
-         isize = sp%zsz
-      else if (format == PHYSICAL_IN_Z) then
-         istart = sp%xst
-         iend = sp%xen
-         isize = sp%xsz
-      end if
-
-      return
-   end subroutine decomp_2d_fft_get_size
-
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Return a pointer to the decomp_info object ph
    !

From 80142399d4a767e73a409211f447b31000690af3 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 10:28:43 +0100
Subject: [PATCH 072/436] wk2_r2c should not have the save attribute

---
 src/fft_common.f90 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 479c801a..abae5b9e 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -29,7 +29,7 @@
 
 ! Workspace to store the intermediate Y-pencil data
 complex(mytype), allocatable, target, dimension(:, :, :) :: wk2_c2c
-complex(mytype), contiguous, pointer, dimension(:, :, :) :: wk2_r2c => null()
+complex(mytype), contiguous, pointer, dimension(:, :, :) :: wk2_r2c
 ! Workspace for r2c and c2r transforms
 ! FIXME could be removed using in-place r2c and c2r ?
 complex(mytype), allocatable, dimension(:, :, :) :: wk13
@@ -152,6 +152,7 @@ subroutine fft_init_general(pencil, nx, ny, nz)
    !    Size of wk2_c2c : ph%ysz(1), ph%ysz(2), ph%ysz(3)
    !    Size of wk2_r2c : sp%ysz(1), sp%ysz(2), sp%ysz(3)
    !
+   nullify(wk2_r2c)
    call c_f_pointer(c_loc(wk2_c2c), wk2_r2c, sp%ysz)
    !
    ! Allocate the workspace for r2c and c2r transforms

From 5b31e93da9e38e98415961b2e294ca0da3baa660 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 10:39:45 +0100
Subject: [PATCH 073/436] fftw3_f03 : fuse working arrays wk2_c2c and wk2_r2c

---
 src/fft_fftw3_f03.f90 | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index b4696864..de7b18e5 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -57,8 +57,8 @@ module decomp_2d_fft
 
    ! Workspace to store the intermediate Y-pencil data
    ! *** TODO: investigate how to use only one workspace array
-   complex(mytype), pointer :: wk2_c2c(:, :, :), wk2_r2c(:, :, :), wk13(:, :, :)
-   type(C_PTR) :: wk2_c2c_p, wk2_r2c_p, wk13_p
+   complex(mytype), contiguous, pointer :: wk2_c2c(:, :, :), wk2_r2c(:, :, :), wk13(:, :, :)
+   type(C_PTR) :: wk2_c2c_p, wk13_p
 
    public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
              decomp_2d_fft_finalize, &
@@ -162,14 +162,28 @@ subroutine fft_init_general(pencil, nx, ny, nz)
          call decomp_2d_abort(__FILE__, __LINE__, format, "Invalid value for format")
       end if
 
+      !
+      ! Allocate the workspace fo intermediate y-pencil data
+      ! The largest memory block needed is the one for c2c transforms
+      !
       sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)
       wk2_c2c_p = fftw_alloc_complex(sz)
       call c_f_pointer(wk2_c2c_p, wk2_c2c, [ph%ysz(1), ph%ysz(2), ph%ysz(3)])
-
-      sz = sp%ysz(1)*sp%ysz(2)*sp%ysz(3)
-      wk2_r2c_p = fftw_alloc_complex(sz)
-      call c_f_pointer(wk2_r2c_p, wk2_r2c, [sp%ysz(1), sp%ysz(2), sp%ysz(3)])
-
+      !
+      ! A smaller memory block is needed for r2c and c2r transforms
+      ! wk2_c2c and wk2_r2c start at the same location
+      !
+      !    Size of wk2_c2c : ph%ysz(1), ph%ysz(2), ph%ysz(3)
+      !    Size of wk2_r2c : sp%ysz(1), sp%ysz(2), sp%ysz(3)
+      !
+      call c_f_pointer(wk2_c2c_p, wk2_r2c, [sp%ysz(1), sp%ysz(2), sp%ysz(3)])
+      !
+      ! Allocate the workspace for r2c and c2r transforms
+      !
+      ! wk13 can not be easily fused with wk2_*2c due to statements such as
+      ! transpose_y_to_x(wk2_r2c, wk13, sp)
+      ! transpose_y_to_z(wk2_r2c, wk13, sp)
+      !
       if (format == PHYSICAL_IN_X) then
          sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
          wk13_p = fftw_alloc_complex(sz)
@@ -210,8 +224,10 @@ subroutine decomp_2d_fft_finalize
       call decomp_info_finalize(sp)
 
       call fftw_free(wk2_c2c_p)
-      call fftw_free(wk2_r2c_p)
+      call nullify (wk2_c2c)
+      call nullify (wk2_r2c)
       call fftw_free(wk13_p)
+      call nullify (wk13)
 
       call finalize_fft_engine
 

From 7f2eaef5a5ea26a3928988b2b7bf32ae5be5ffda Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 10:41:41 +0100
Subject: [PATCH 074/436] Minor update

---
 src/fft_common.f90 | 1 -
 src/fft_log.f90    | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index abae5b9e..c12e6663 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -152,7 +152,6 @@ subroutine fft_init_general(pencil, nx, ny, nz)
    !    Size of wk2_c2c : ph%ysz(1), ph%ysz(2), ph%ysz(3)
    !    Size of wk2_r2c : sp%ysz(1), sp%ysz(2), sp%ysz(3)
    !
-   nullify(wk2_r2c)
    call c_f_pointer(c_loc(wk2_c2c), wk2_r2c, sp%ysz)
    !
    ! Allocate the workspace for r2c and c2r transforms
diff --git a/src/fft_log.f90 b/src/fft_log.f90
index 905833f9..ba6e3845 100644
--- a/src/fft_log.f90
+++ b/src/fft_log.f90
@@ -61,7 +61,7 @@ module subroutine decomp_2d_fft_log(backend)
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_MKL) then
             write (io_unit, *) 'OVERWRITE support limited to c2C and c2r transforms'
             write (io_unit, *) ''
-         endif
+         end if
 #endif
          call d2d_listing_close_unit(io_unit)
       end if

From a2a70652e7570ec429b7ff6e25f74f922b67143e Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 10:45:33 +0100
Subject: [PATCH 075/436] Fix 01e9a4cc1f8a179f2e60c5583726fec12a194946

---
 src/fft_log.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fft_log.f90 b/src/fft_log.f90
index ba6e3845..10e8b631 100644
--- a/src/fft_log.f90
+++ b/src/fft_log.f90
@@ -54,7 +54,7 @@ module subroutine decomp_2d_fft_log(backend)
          write (io_unit, *) ''
 #ifdef OVERWRITE
          if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_GENERIC) then
-            call decomp_2d_warning("Selected FFT backend does not support overwrite")
+            call decomp_2d_warning(D2D_FFT_BACKEND, "Selected FFT backend does not support overwrite")
          else if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3 .or. &
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3_F03 .or. &
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_CUFFT .or. &

From 758dd295036eec8f6729ce5cb7121e9499cfd332 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 10:49:16 +0100
Subject: [PATCH 076/436] Fix FFTW3_F03 backend

---
 src/fft_fftw3_f03.f90 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index de7b18e5..5095086a 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -224,10 +224,10 @@ subroutine decomp_2d_fft_finalize
       call decomp_info_finalize(sp)
 
       call fftw_free(wk2_c2c_p)
-      call nullify (wk2_c2c)
-      call nullify (wk2_r2c)
+      nullify (wk2_c2c)
+      nullify (wk2_r2c)
       call fftw_free(wk13_p)
-      call nullify (wk13)
+      nullify (wk13)
 
       call finalize_fft_engine
 

From a0c5964d7c94d8c54ad3e71c4ce824acfa91bdae Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 13:50:27 +0100
Subject: [PATCH 077/436] Revert "Remove the subroutine decomp_2d_get_size, the
 decomp_info object sp can be used instead"

This reverts commit 3ba1c637f2457ae419a84b32022b84f62d93db7f.
---
 src/fft_common.f90    | 25 ++++++++++++++++++++++++-
 src/fft_fftw3_f03.f90 | 25 ++++++++++++++++++++++++-
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index c12e6663..a9cf09a3 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -35,7 +35,7 @@
 complex(mytype), allocatable, dimension(:, :, :) :: wk13
 
 public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-          decomp_2d_fft_finalize, &
+          decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
           decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
 
 ! Declare generic interfaces to handle different inputs
@@ -215,6 +215,29 @@ subroutine decomp_2d_fft_finalize
    return
 end subroutine decomp_2d_fft_finalize
 
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+! Return the size, starting/ending index of the distributed array
+!  whose global size is (nx/2+1)*ny*nz, for defining data structures
+!  in r2c and c2r interfaces
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+subroutine decomp_2d_fft_get_size(istart, iend, isize)
+
+   implicit none
+   integer, dimension(3), intent(OUT) :: istart, iend, isize
+
+   if (format == PHYSICAL_IN_X) then
+      istart = sp%zst
+      iend = sp%zen
+      isize = sp%zsz
+   else if (format == PHYSICAL_IN_Z) then
+      istart = sp%xst
+      iend = sp%xen
+      isize = sp%xsz
+   end if
+
+   return
+end subroutine decomp_2d_fft_get_size
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! Return a pointer to the decomp_info object ph
 !
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 5095086a..f94f6854 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -61,7 +61,7 @@ module decomp_2d_fft
    type(C_PTR) :: wk2_c2c_p, wk13_p
 
    public :: decomp_2d_fft_init, decomp_2d_fft_3d, &
-             decomp_2d_fft_finalize, &
+             decomp_2d_fft_finalize, decomp_2d_fft_get_size, &
              decomp_2d_fft_get_ph, decomp_2d_fft_get_sp
 
    ! Declare generic interfaces to handle different inputs
@@ -240,6 +240,29 @@ subroutine decomp_2d_fft_finalize
       return
    end subroutine decomp_2d_fft_finalize
 
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   ! Return the size, starting/ending index of the distributed array
+   !  whose global size is (nx/2+1)*ny*nz, for defining data structures
+   !  in r2c and c2r interfaces
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   subroutine decomp_2d_fft_get_size(istart, iend, isize)
+
+      implicit none
+      integer, dimension(3), intent(OUT) :: istart, iend, isize
+
+      if (format == PHYSICAL_IN_X) then
+         istart = sp%zst
+         iend = sp%zen
+         isize = sp%zsz
+      else if (format == PHYSICAL_IN_Z) then
+         istart = sp%xst
+         iend = sp%xen
+         isize = sp%xsz
+      end if
+
+      return
+   end subroutine decomp_2d_fft_get_size
+
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Return a pointer to the decomp_info object ph
    !

From d1165ddaa0a99c9b0a9421b38eb9db4efc2a4b62 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 16:34:00 +0100
Subject: [PATCH 078/436] fftw3: Remove unused argument

---
 src/fft_fftw3.f90 | 71 +++++++++++++++++++----------------------------
 1 file changed, 28 insertions(+), 43 deletions(-)

diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index a6d400c4..5e9188c1 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -339,18 +339,13 @@ end subroutine finalize_fft_engine
    ! the basis of three-dimensional FFTs.
 
    ! c2c transform, multiple 1D FFTs in x direction
-   subroutine c2c_1m_x(inout, isign, plan1)
+   subroutine c2c_1m_x(inout, plan1)
 
       implicit none
 
       complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
       type(C_PTR), intent(IN) :: plan1
 
-      integer :: foo
-
-      foo = isign ! Silence unused dummy argument
-
 #ifdef DOUBLE_PREC
       call dfftw_execute_dft(plan1, inout, inout)
 #else
@@ -361,20 +356,15 @@ subroutine c2c_1m_x(inout, isign, plan1)
    end subroutine c2c_1m_x
 
    ! c2c transform, multiple 1D FFTs in y direction
-   subroutine c2c_1m_y(inout, isign, plan1)
+   subroutine c2c_1m_y(inout, plan1)
 
       implicit none
 
       complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
       type(C_PTR), intent(IN) :: plan1
 
       integer :: k, s3
 
-      integer :: foo
-
-      foo = isign ! Silence unused dummy argument
-
       ! transform on one Z-plane at a time
       s3 = size(inout, 3)
       do k = 1, s3
@@ -389,18 +379,13 @@ subroutine c2c_1m_y(inout, isign, plan1)
    end subroutine c2c_1m_y
 
    ! c2c transform, multiple 1D FFTs in z direction
-   subroutine c2c_1m_z(inout, isign, plan1)
+   subroutine c2c_1m_z(inout, plan1)
 
       implicit none
 
       complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
       type(C_PTR), intent(IN) :: plan1
 
-      integer :: foo
-
-      foo = isign ! Silence unused dummy argument
-
 #ifdef DOUBLE_PREC
       call dfftw_execute_dft(plan1, inout, inout)
 #else
@@ -502,11 +487,11 @@ subroutine fft_3d_c2c(in, out, isign)
 
          ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in, isign, plan(isign, 1))
+         call c2c_1m_x(in, plan(isign, 1))
 #else
          allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
          wk1 = in
-         call c2c_1m_x(wk1, isign, plan(isign, 1))
+         call c2c_1m_x(wk1, plan(isign, 1))
 #endif
 
          ! ===== Swap X --> Y; 1D FFTs in Y =====
@@ -517,12 +502,12 @@ subroutine fft_3d_c2c(in, out, isign)
 #else
             call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+            call c2c_1m_y(wk2_c2c, plan(isign, 2))
          else
 #ifdef OVERWRITE
-            call c2c_1m_y(in, isign, plan(isign, 2))
+            call c2c_1m_y(in, plan(isign, 2))
 #else
-            call c2c_1m_y(wk1, isign, plan(isign, 2))
+            call c2c_1m_y(wk1, plan(isign, 2))
 #endif
          end if
 
@@ -536,7 +521,7 @@ subroutine fft_3d_c2c(in, out, isign)
             call transpose_y_to_z(wk1, out, ph)
 #endif
          end if
-         call c2c_1m_z(out, isign, plan(isign, 3))
+         call c2c_1m_z(out, plan(isign, 3))
 
       else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
                .OR. &
@@ -544,11 +529,11 @@ subroutine fft_3d_c2c(in, out, isign)
 
          ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in, isign, plan(isign, 3))
+         call c2c_1m_z(in, plan(isign, 3))
 #else
          allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
          wk1 = in
-         call c2c_1m_z(wk1, isign, plan(isign, 3))
+         call c2c_1m_z(wk1, plan(isign, 3))
 #endif
 
          ! ===== Swap Z --> Y; 1D FFTs in Y =====
@@ -558,21 +543,21 @@ subroutine fft_3d_c2c(in, out, isign)
 #else
             call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+            call c2c_1m_y(wk2_c2c, plan(isign, 2))
          else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
             call transpose_z_to_y(in, out, ph)
 #else
             call transpose_z_to_y(wk1, out, ph)
 #endif
-            call c2c_1m_y(out, isign, plan(isign, 2))
+            call c2c_1m_y(out, plan(isign, 2))
          end if
 
          ! ===== Swap Y --> X; 1D FFTs in X =====
          if (dims(1) > 1) then
             call transpose_y_to_x(wk2_c2c, out, ph)
          end if
-         call c2c_1m_x(out, isign, plan(isign, 1))
+         call c2c_1m_x(out, plan(isign, 1))
 
       end if
 
@@ -601,9 +586,9 @@ subroutine fft_3d_r2c(in_r, out_c)
          ! ===== Swap X --> Y; 1D FFTs in Y =====
          if (dims(1) > 1) then
             call transpose_x_to_y(wk13, wk2_r2c, sp)
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+            call c2c_1m_y(wk2_r2c, plan(0, 2))
          else
-            call c2c_1m_y(wk13, -1, plan(0, 2))
+            call c2c_1m_y(wk13, plan(0, 2))
          end if
 
          ! ===== Swap Y --> Z; 1D FFTs in Z =====
@@ -612,7 +597,7 @@ subroutine fft_3d_r2c(in_r, out_c)
          else
             call transpose_y_to_z(wk13, out_c, sp)
          end if
-         call c2c_1m_z(out_c, -1, plan(0, 3))
+         call c2c_1m_z(out_c, plan(0, 3))
 
       else if (format == PHYSICAL_IN_Z) then
 
@@ -622,17 +607,17 @@ subroutine fft_3d_r2c(in_r, out_c)
          ! ===== Swap Z --> Y; 1D FFTs in Y =====
          if (dims(1) > 1) then
             call transpose_z_to_y(wk13, wk2_r2c, sp)
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+            call c2c_1m_y(wk2_r2c, plan(0, 2))
          else  ! out_c==wk2_r2c if 1D decomposition
             call transpose_z_to_y(wk13, out_c, sp)
-            call c2c_1m_y(out_c, -1, plan(0, 2))
+            call c2c_1m_y(out_c, plan(0, 2))
          end if
 
          ! ===== Swap Y --> X; 1D FFTs in X =====
          if (dims(1) > 1) then
             call transpose_y_to_x(wk2_r2c, out_c, sp)
          end if
-         call c2c_1m_x(out_c, -1, plan(0, 1))
+         call c2c_1m_x(out_c, plan(0, 1))
 
       end if
 
@@ -657,11 +642,11 @@ subroutine fft_3d_c2r(in_c, out_r)
 
          ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in_c, 1, plan(2, 3))
+         call c2c_1m_z(in_c, plan(2, 3))
 #else
          allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
          wk1 = in_c
-         call c2c_1m_z(wk1, 1, plan(2, 3))
+         call c2c_1m_z(wk1, plan(2, 3))
 #endif
 
          ! ===== Swap Z --> Y; 1D FFTs in Y =====
@@ -670,7 +655,7 @@ subroutine fft_3d_c2r(in_c, out_r)
 #else
          call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+         call c2c_1m_y(wk2_r2c, plan(2, 2))
 
          ! ===== Swap Y --> X; 1D FFTs in X =====
          if (dims(1) > 1) then
@@ -684,11 +669,11 @@ subroutine fft_3d_c2r(in_c, out_r)
 
          ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in_c, 1, plan(2, 1))
+         call c2c_1m_x(in_c, plan(2, 1))
 #else
          allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
          wk1 = in_c
-         call c2c_1m_x(wk1, 1, plan(2, 1))
+         call c2c_1m_x(wk1, plan(2, 1))
 #endif
 
          ! ===== Swap X --> Y; 1D FFTs in Y =====
@@ -698,12 +683,12 @@ subroutine fft_3d_c2r(in_c, out_r)
 #else
             call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+            call c2c_1m_y(wk2_r2c, plan(2, 2))
          else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-            call c2c_1m_y(in_c, 1, plan(2, 2))
+            call c2c_1m_y(in_c, plan(2, 2))
 #else
-            call c2c_1m_y(wk1, 1, plan(2, 2))
+            call c2c_1m_y(wk1, plan(2, 2))
 #endif
          end if
 

From 9aa941d716fd7b8c4f519204d08571db13c78910 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 26 Dec 2022 16:44:12 +0100
Subject: [PATCH 079/436] fftw3_f03: Remove unused argument

---
 src/fft_fftw3_f03.f90 | 73 ++++++++++++++++++-------------------------
 1 file changed, 30 insertions(+), 43 deletions(-)

diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index f94f6854..0ac3898e 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -666,18 +666,13 @@ end subroutine finalize_fft_engine
    ! the basis of three-dimensional FFTs.
 
    ! c2c transform, multiple 1D FFTs in x direction
-   subroutine c2c_1m_x(inout, isign, plan1)
+   subroutine c2c_1m_x(inout, plan1)
 
       implicit none
 
       complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
       type(C_PTR) :: plan1
 
-      integer :: foo
-
-      foo = isign ! Silence unused dummy argument
-
 #ifdef DOUBLE_PREC
       call fftw_execute_dft(plan1, inout, inout)
 #else
@@ -688,20 +683,15 @@ subroutine c2c_1m_x(inout, isign, plan1)
    end subroutine c2c_1m_x
 
    ! c2c transform, multiple 1D FFTs in y direction
-   subroutine c2c_1m_y(inout, isign, plan1)
+   subroutine c2c_1m_y(inout, plan1)
 
       implicit none
 
       complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
       type(C_PTR) :: plan1
 
       integer :: k, s3
 
-      integer :: foo
-
-      foo = isign ! Silence unused dummy argument
-
       s3 = size(inout, 3)
 
       do k = 1, s3  ! transform on one Z-plane at a time
@@ -716,18 +706,13 @@ subroutine c2c_1m_y(inout, isign, plan1)
    end subroutine c2c_1m_y
 
    ! c2c transform, multiple 1D FFTs in z direction
-   subroutine c2c_1m_z(inout, isign, plan1)
+   subroutine c2c_1m_z(inout, plan1)
 
       implicit none
 
       complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-      integer, intent(IN) :: isign
       type(C_PTR) :: plan1
 
-      integer :: foo
-
-      foo = isign ! Silence unused dummy argument
-
 #ifdef DOUBLE_PREC
       call fftw_execute_dft(plan1, inout, inout)
 #else
@@ -837,13 +822,13 @@ subroutine fft_3d_c2c(in, out, isign)
 
          ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in, isign, plan(isign, 1))
+         call c2c_1m_x(in, plan(isign, 1))
 #else
          sz = ph%xsz(1)*ph%xsz(2)*ph%xsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [ph%xsz(1), ph%xsz(2), ph%xsz(3)])
          wk1 = in
-         call c2c_1m_x(wk1, isign, plan(isign, 1))
+         call c2c_1m_x(wk1, plan(isign, 1))
 #endif
 
          ! ===== Swap X --> Y; 1D FFTs in Y =====
@@ -854,12 +839,12 @@ subroutine fft_3d_c2c(in, out, isign)
 #else
             call transpose_x_to_y(wk1, wk2_c2c, ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+            call c2c_1m_y(wk2_c2c, plan(isign, 2))
          else
 #ifdef OVERWRITE
-            call c2c_1m_y(in, isign, plan(isign, 2))
+            call c2c_1m_y(in, plan(isign, 2))
 #else
-            call c2c_1m_y(wk1, isign, plan(isign, 2))
+            call c2c_1m_y(wk1, plan(isign, 2))
 #endif
          end if
 
@@ -873,7 +858,7 @@ subroutine fft_3d_c2c(in, out, isign)
             call transpose_y_to_z(wk1, out, ph)
 #endif
          end if
-         call c2c_1m_z(out, isign, plan(isign, 3))
+         call c2c_1m_z(out, plan(isign, 3))
 
       else if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_BACKWARD &
                .OR. &
@@ -881,13 +866,13 @@ subroutine fft_3d_c2c(in, out, isign)
 
          ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in, isign, plan(isign, 3))
+         call c2c_1m_z(in, plan(isign, 3))
 #else
          sz = ph%zsz(1)*ph%zsz(2)*ph%zsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [ph%zsz(1), ph%zsz(2), ph%zsz(3)])
          wk1 = in
-         call c2c_1m_z(wk1, isign, plan(isign, 3))
+         call c2c_1m_z(wk1, plan(isign, 3))
 #endif
 
          ! ===== Swap Z --> Y; 1D FFTs in Y =====
@@ -897,26 +882,27 @@ subroutine fft_3d_c2c(in, out, isign)
 #else
             call transpose_z_to_y(wk1, wk2_c2c, ph)
 #endif
-            call c2c_1m_y(wk2_c2c, isign, plan(isign, 2))
+            call c2c_1m_y(wk2_c2c, plan(isign, 2))
          else  ! out==wk2_c2c if 1D decomposition
 #ifdef OVERWRITE
             call transpose_z_to_y(in, out, ph)
 #else
             call transpose_z_to_y(wk1, out, ph)
 #endif
-            call c2c_1m_y(out, isign, plan(isign, 2))
+            call c2c_1m_y(out, plan(isign, 2))
          end if
 
          ! ===== Swap Y --> X; 1D FFTs in X =====
          if (dims(1) > 1) then
             call transpose_y_to_x(wk2_c2c, out, ph)
          end if
-         call c2c_1m_x(out, isign, plan(isign, 1))
+         call c2c_1m_x(out, plan(isign, 1))
 
       end if
 
 #ifndef OVERWRITE
       call fftw_free(wk1_p)
+      nullify (wk1)
 #endif
 
 #ifdef PROFILER
@@ -948,9 +934,9 @@ subroutine fft_3d_r2c(in_r, out_c)
          ! ===== Swap X --> Y; 1D FFTs in Y =====
          if (dims(1) > 1) then
             call transpose_x_to_y(wk13, wk2_r2c, sp)
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+            call c2c_1m_y(wk2_r2c, plan(0, 2))
          else
-            call c2c_1m_y(wk13, -1, plan(0, 2))
+            call c2c_1m_y(wk13, plan(0, 2))
          end if
 
          ! ===== Swap Y --> Z; 1D FFTs in Z =====
@@ -959,7 +945,7 @@ subroutine fft_3d_r2c(in_r, out_c)
          else
             call transpose_y_to_z(wk13, out_c, sp)
          end if
-         call c2c_1m_z(out_c, -1, plan(0, 3))
+         call c2c_1m_z(out_c, plan(0, 3))
 
       else if (format == PHYSICAL_IN_Z) then
 
@@ -969,17 +955,17 @@ subroutine fft_3d_r2c(in_r, out_c)
          ! ===== Swap Z --> Y; 1D FFTs in Y =====
          if (dims(1) > 1) then
             call transpose_z_to_y(wk13, wk2_r2c, sp)
-            call c2c_1m_y(wk2_r2c, -1, plan(0, 2))
+            call c2c_1m_y(wk2_r2c, plan(0, 2))
          else  ! out_c==wk2_r2c if 1D decomposition
             call transpose_z_to_y(wk13, out_c, sp)
-            call c2c_1m_y(out_c, -1, plan(0, 2))
+            call c2c_1m_y(out_c, plan(0, 2))
          end if
 
          ! ===== Swap Y --> X; 1D FFTs in X =====
          if (dims(1) > 1) then
             call transpose_y_to_x(wk2_r2c, out_c, sp)
          end if
-         call c2c_1m_x(out_c, -1, plan(0, 1))
+         call c2c_1m_x(out_c, plan(0, 1))
 
       end if
 
@@ -1016,13 +1002,13 @@ subroutine fft_3d_c2r(in_c, out_r)
 
          ! ===== 1D FFTs in Z =====
 #ifdef OVERWRITE
-         call c2c_1m_z(in_c, 1, plan(2, 3))
+         call c2c_1m_z(in_c, plan(2, 3))
 #else
          sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
          wk1 = in_c
-         call c2c_1m_z(wk1, 1, plan(2, 3))
+         call c2c_1m_z(wk1, plan(2, 3))
 #endif
 
          ! ===== Swap Z --> Y; 1D FFTs in Y =====
@@ -1031,7 +1017,7 @@ subroutine fft_3d_c2r(in_c, out_r)
 #else
          call transpose_z_to_y(wk1, wk2_r2c, sp)
 #endif
-         call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+         call c2c_1m_y(wk2_r2c, plan(2, 2))
 
          ! ===== Swap Y --> X; 1D FFTs in X =====
          if (dims(1) > 1) then
@@ -1045,13 +1031,13 @@ subroutine fft_3d_c2r(in_c, out_r)
 
          ! ===== 1D FFTs in X =====
 #ifdef OVERWRITE
-         call c2c_1m_x(in_c, 1, plan(2, 1))
+         call c2c_1m_x(in_c, plan(2, 1))
 #else
          sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
          wk1 = in_c
-         call c2c_1m_x(wk1, 1, plan(2, 1))
+         call c2c_1m_x(wk1, plan(2, 1))
 #endif
 
          ! ===== Swap X --> Y; 1D FFTs in Y =====
@@ -1061,12 +1047,12 @@ subroutine fft_3d_c2r(in_c, out_r)
 #else
             call transpose_x_to_y(wk1, wk2_r2c, sp)
 #endif
-            call c2c_1m_y(wk2_r2c, 1, plan(2, 2))
+            call c2c_1m_y(wk2_r2c, plan(2, 2))
          else  ! in_c==wk2_r2c if 1D decomposition
 #ifdef OVERWRITE
-            call c2c_1m_y(in_c, 1, plan(2, 2))
+            call c2c_1m_y(in_c, plan(2, 2))
 #else
-            call c2c_1m_y(wk1, 1, plan(2, 2))
+            call c2c_1m_y(wk1, plan(2, 2))
 #endif
          end if
 
@@ -1086,6 +1072,7 @@ subroutine fft_3d_c2r(in_c, out_r)
 
 #ifndef OVERWRITE
       call fftw_free(wk1_p)
+      nullify (wk1)
 #endif
 
 #ifdef PROFILER

From cf72032409181a1e4e15bb153842c930830a2725 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Fri, 30 Dec 2022 10:13:23 +0100
Subject: [PATCH 080/436] Update fft (#132)

* Update README

* Update fft/log
---
 README.md       | 10 +++++-----
 src/fft_log.f90 |  8 +++++---
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index cca3f1bb..143beaf5 100644
--- a/README.md
+++ b/README.md
@@ -85,10 +85,10 @@ The external code can use the named variables to check the FFT backend used in a
 
 ### OVERWRITE flag
 
-- The generic backend does not support the OVERWRITE flag
-- The FFTW3 and FFTW3_F03 backends support the OVERWRITE flag for c2c and c2r transforms
-- The oneMKL backend supports the OVERWRITE flag for c2c and c2r transforms
-- The cuFFT backend supports the OVERWRITE flag for c2c anc c2r transforms
+- The generic backend supports the OVERWRITE flag but it can not perform in-place transforms
+- The FFTW3 and FFTW3_F03 backends support the OVERWRITE flag and can perform in-place complex 1D fft
+- The oneMKL backend supports the OVERWRITE flag and can perform in-place complex 1D fft
+- The cuFFT backend supports the OVERWRITE flag but it can not perform in-place transforms
 
 ## Miscellaneous
 
@@ -133,7 +133,7 @@ This preprocessor variable is not valid for GPU builds. It leads to padded allto
 
 #### OVERWRITE
 
-This variable leads to overwrite the input array when computing FFT. The support of this flag is variable, depending on the FFT backend selected, as described above.
+This variable leads to overwrite the input array when computing FFT. The support of this flag does not always correspond to in-place transforms, depending on the FFT backend selected, as described above.
 
 #### HALO_DEBUG
 
diff --git a/src/fft_log.f90 b/src/fft_log.f90
index 10e8b631..258a4c9d 100644
--- a/src/fft_log.f90
+++ b/src/fft_log.f90
@@ -53,13 +53,15 @@ module subroutine decomp_2d_fft_log(backend)
          call decomp_info_print(sp, io_unit, "sp")
          write (io_unit, *) ''
 #ifdef OVERWRITE
-         if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_GENERIC) then
-            call decomp_2d_warning(D2D_FFT_BACKEND, "Selected FFT backend does not support overwrite")
+         if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_GENERIC .or. &
+             D2D_FFT_BACKEND == D2D_FFT_BACKEND_CUFFT) then
+            write (io_unit, *) 'OVERWRITE is supported but transforms are not performed in-place'
+            write (io_unit, *) ''
          else if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3 .or. &
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3_F03 .or. &
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_CUFFT .or. &
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_MKL) then
-            write (io_unit, *) 'OVERWRITE support limited to c2C and c2r transforms'
+            write (io_unit, *) 'OVERWRITE is supported but in-place transforms is limited to complex transforms'
             write (io_unit, *) ''
          end if
 #endif

From a9a76c1a3bbcd577ed012ffdeecd2ca45860b705 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 3 Jan 2023 10:33:27 +0100
Subject: [PATCH 081/436] Update fft_log.f90

---
 src/fft_log.f90 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/fft_log.f90 b/src/fft_log.f90
index 258a4c9d..85d59352 100644
--- a/src/fft_log.f90
+++ b/src/fft_log.f90
@@ -59,7 +59,6 @@ module subroutine decomp_2d_fft_log(backend)
             write (io_unit, *) ''
          else if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3 .or. &
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3_F03 .or. &
-                  D2D_FFT_BACKEND == D2D_FFT_BACKEND_CUFFT .or. &
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_MKL) then
             write (io_unit, *) 'OVERWRITE is supported but in-place transforms is limited to complex transforms'
             write (io_unit, *) ''

From 5718945fdbabe5552d62ae3fb9727a6089ea47bf Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Wed, 11 Jan 2023 23:34:06 +0000
Subject: [PATCH 082/436] Update SRC for manual memory managment

---
 src/fft_cufft.f90        | 73 ++++++++++++++++++++++------------------
 src/transpose_x_to_y.f90 | 16 ++++++---
 src/transpose_y_to_x.f90 | 16 ++++++---
 src/transpose_y_to_z.f90 | 21 +++++++++---
 src/transpose_z_to_y.f90 | 18 +++++++---
 5 files changed, 95 insertions(+), 49 deletions(-)

diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index a3a1b057..b9465e82 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -368,6 +368,7 @@ subroutine init_fft_engine
 
       end if
 #endif
+      cufft_ws = cufft_ws / sizeof(1._mytype)
       allocate (cufft_workspace(cufft_ws))
       do j = 1, 3
          do i = -1, 2
@@ -408,24 +409,18 @@ subroutine c2c_1m_x(inout, isign, plan1)
       integer, intent(IN) :: isign
       integer*4, intent(IN) :: plan1
 
-      complex(mytype), dimension(:, :, :), allocatable :: output
       integer :: istat
 
-      allocate (output, mold=inout)
 #ifdef DOUBLE_PREC
-      !$acc host_data use_device(inout,output)
-      istat = cufftExecZ2Z(plan1, inout, output, isign)
+      !$acc host_data use_device(inout)
+      istat = cufftExecZ2Z(plan1, inout, inout,isign)
       !$acc end host_data
 #else
-      !$acc host_data use_device(inout,output)
-      istat = cufftExecC2C(plan1, inout, output, isign)
+      !$acc host_data use_device(inout)
+      istat = cufftExecC2C(plan1, inout, inout,isign)
       !$acc end host_data
 #endif
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-      !$acc kernels
-      inout = output
-      !$acc end kernels
-      deallocate (output)
 
    end subroutine c2c_1m_x
 
@@ -438,28 +433,22 @@ subroutine c2c_1m_y(inout, isign, plan1)
       integer, intent(IN) :: isign
       integer*4, intent(IN) :: plan1
 
-      complex(mytype), dimension(:, :, :), allocatable :: output
       integer :: s3, k, istat
 
-      allocate (output, mold=inout)
       ! transform on one Z-plane at a time
       s3 = size(inout, 3)
       do k = 1, s3
 #ifdef DOUBLE_PREC
-         !$acc host_data use_device(inout,output)
-         istat = cufftExecZ2Z(plan1, inout(:, :, k), output(:, :, k), isign)
-         !$acc end host_data
+        !$acc host_data use_device(inout)
+        istat = cufftExecZ2Z(plan1, inout(:,:,k), inout(:,:,k),isign)
+        !$acc end host_data
 #else
-         !$acc host_data use_device(inout,output)
-         istat = cufftExecC2C(plan1, inout(:, :, k), output(:, :, k), isign)
-         !$acc end host_data
+        !$acc host_data use_device(inout)
+        istat = cufftExecC2C(plan1, inout(:,:,k), inout(:,:,k),isign)
+        !$acc end host_data
 #endif
          if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
       end do
-      !$acc kernels
-      inout = output
-      !$acc end kernels
-      deallocate (output)
 
    end subroutine c2c_1m_y
 
@@ -472,25 +461,19 @@ subroutine c2c_1m_z(inout, isign, plan1)
       integer, intent(IN) :: isign
       integer*4, intent(IN) :: plan1
 
-      complex(mytype), dimension(:, :, :), allocatable :: output
       integer :: istat
 
-      allocate (output, mold=inout)
 
 #ifdef DOUBLE_PREC
-      !$acc host_data use_device(inout,output)
-      istat = cufftExecZ2Z(plan1, inout, output, isign)
+      !$acc host_data use_device(inout)
+      istat = cufftExecZ2Z(plan1, inout, inout,isign)
       !$acc end host_data
 #else
-      !$acc host_data use_device(inout,output)
-      istat = cufftExecC2C(plan1, inout, output, isign)
+      !$acc host_data use_device(inout)
+      istat = cufftExecC2C(plan1, inout, inout,isign)
       !$acc end host_data
 #endif
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
-      !$acc kernels
-      inout = output
-      !$acc end kernels
-      deallocate (output)
 
    end subroutine c2c_1m_z
 
@@ -604,6 +587,7 @@ subroutine fft_3d_c2c(in, out, isign)
       if (decomp_profiler_fft) call decomp_profiler_start("fft_c2c")
 #endif
 
+      !$acc data create(wk2_c2c) present(in,out)
       if (format == PHYSICAL_IN_X .AND. isign == DECOMP_2D_FFT_FORWARD .OR. &
           format == PHYSICAL_IN_Z .AND. isign == DECOMP_2D_FFT_BACKWARD) then
 
@@ -612,7 +596,11 @@ subroutine fft_3d_c2c(in, out, isign)
          call c2c_1m_x(in, isign, plan(isign, 1))
 #else
          allocate (wk1(ph%xsz(1), ph%xsz(2), ph%xsz(3)))
+         !$acc enter data create(wk1) async
+         !$acc wait
+         !$acc kernels default(present)
          wk1 = in
+         !$acc end kernels
          call c2c_1m_x(wk1, isign, plan(isign, 1))
 #endif
 
@@ -654,7 +642,11 @@ subroutine fft_3d_c2c(in, out, isign)
          call c2c_1m_z(in, isign, plan(isign, 3))
 #else
          allocate (wk1(ph%zsz(1), ph%zsz(2), ph%zsz(3)))
+         !$acc enter data create(wk1) async
+         !$acc wait
+         !$acc kernels default(present)
          wk1 = in
+         !$acc end kernels
          call c2c_1m_z(wk1, isign, plan(isign, 3))
 #endif
 
@@ -684,8 +676,11 @@ subroutine fft_3d_c2c(in, out, isign)
       end if
 
 #ifndef OVERWRITE
+      !$acc exit data delete(wk1) async
+      !$acc wait 
       deallocate (wk1)
 #endif
+      !$acc end data
 
 #ifdef PROFILER
       if (decomp_profiler_fft) call decomp_profiler_end("fft_c2c")
@@ -709,6 +704,7 @@ subroutine fft_3d_r2c(in_r, out_c)
       if (decomp_profiler_fft) call decomp_profiler_start("fft_r2c")
 #endif
 
+      !$acc data create(wk13,wk2_r2c) present(in_r,out_c)
       if (format == PHYSICAL_IN_X) then
 
          ! ===== 1D FFTs in X =====
@@ -815,6 +811,7 @@ subroutine fft_3d_r2c(in_r, out_c)
          write (*, *)
 #endif
       end if
+      !$acc end data
 
 #ifdef PROFILER
       if (decomp_profiler_fft) call decomp_profiler_end("fft_r2c")
@@ -841,6 +838,7 @@ subroutine fft_3d_c2r(in_c, out_r)
       if (decomp_profiler_fft) call decomp_profiler_start("fft_c2r")
 #endif
 
+      !$acc data create(wk2_r2c,wk13) present(in_c,out_r)
       if (format == PHYSICAL_IN_X) then
 
          ! ===== 1D FFTs in Z =====
@@ -848,7 +846,11 @@ subroutine fft_3d_c2r(in_c, out_r)
          call c2c_1m_z(in_c, 1, plan(2, 3))
 #else
          allocate (wk1(sp%zsz(1), sp%zsz(2), sp%zsz(3)))
+         !$acc enter data create(wk1) async
+         !$acc wait
+         !$acc kernels default(present)
          wk1 = in_c
+         !$acc end kernels
          call c2c_1m_z(wk1, 1, plan(2, 3))
 #endif
 
@@ -902,7 +904,11 @@ subroutine fft_3d_c2r(in_c, out_r)
 #endif
 #else
          allocate (wk1(sp%xsz(1), sp%xsz(2), sp%xsz(3)))
+         !$acc enter data create(wk1) async
+         !$acc wait
+         !$acc kernels default(present)
          wk1 = in_c
+         !$acc end kernels
          call c2c_1m_x(wk1, 1, plan(2, 1))
 #ifdef DEBUG
          write (*, *) 'Back2 c2c_1m_x line 821'
@@ -1020,8 +1026,11 @@ subroutine fft_3d_c2r(in_c, out_r)
       end if
 
 #ifndef OVERWRITE
+      !$acc exit data delete(wk1) async
+      !$acc wait
       deallocate (wk1)
 #endif
+      !$acc end data
 
 #ifdef PROFILER
       if (decomp_profiler_fft) call decomp_profiler_end("fft_c2r")
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index bd783d7e..02c58f55 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -306,7 +306,9 @@ subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
@@ -360,7 +362,9 @@ subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
@@ -414,7 +418,9 @@ subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
@@ -468,7 +474,9 @@ subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data 
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index a742ead6..33f0bc21 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -306,7 +306,9 @@ subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
@@ -360,7 +362,9 @@ subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
@@ -414,7 +418,9 @@ subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
@@ -468,7 +474,9 @@ subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 62bc9071..454b2f66 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -152,7 +152,11 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
      ! so no merge operation needed
 
 #if defined(_GPU)
-     istat = cudaMemcpy(dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
+     !If one of the array in cuda call is not device we need to add acc host_data
+     !$acc host_data use_device(dst)
+     istat = cudaMemcpy( dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
+     !$acc end host_data
+     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
@@ -284,7 +288,10 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
      ! so no merge operation needed
 
 #if defined(_GPU)
-     istat = cudaMemcpy(dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
+    !$acc host_data use_device(dst)
+    istat = cudaMemcpy( dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
+    !$acc end host_data
+    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
@@ -334,7 +341,9 @@ subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
@@ -388,8 +397,10 @@ subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+       !$acc host_data use_device(in)
+       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+       !$acc end host_data
+       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
            do j = i1, i2
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index a2f86844..f6f29a48 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -73,7 +73,10 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
      ! so no split operation needed
 
 #if defined(_GPU)
-     istat = cudaMemcpy(work1_r_d, src, s1*s2*s3)
+     !$acc host_data use_device(src)
+     istat = cudaMemcpy( work1_r_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice )
+     !$acc end host_data
+     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
@@ -220,7 +223,10 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
      ! so no split operation needed
 
 #if defined(_GPU)
-     istat = cudaMemcpy(work1_c_d, src, s1*s2*s3)
+     !$acc host_data use_device(src)
+     istat = cudaMemcpy( work1_c_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice )
+     !$acc end host_data
+     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
@@ -424,7 +430,9 @@ subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
@@ -479,7 +487,9 @@ subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3

From 1d3bc14601c3a19eab2d98a5f5df882de244b90a Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 12 Jan 2023 12:56:07 +0000
Subject: [PATCH 083/436] Update transpose and fft examples for no managed case

---
 Makefile.compilers                     | 10 ++++++++--
 examples/Makefile                      | 20 ++++++--------------
 examples/fft_physical_x/fft_c2c_x.f90  |  4 ++++
 examples/fft_physical_x/fft_grid_x.f90 | 23 +++++++++++++++++------
 examples/fft_physical_x/fft_r2c_x.f90  |  4 ++++
 examples/fft_physical_z/fft_c2c_z.f90  |  8 ++++++--
 examples/fft_physical_z/fft_r2c_z.f90  |  4 ++++
 examples/test2d/test2d.f90             | 14 ++++++++++++++
 8 files changed, 63 insertions(+), 24 deletions(-)

diff --git a/Makefile.compilers b/Makefile.compilers
index 7d0a2bda..b41599de 100644
--- a/Makefile.compilers
+++ b/Makefile.compilers
@@ -84,12 +84,18 @@ else ifeq ($(CMP),nvhpc)
      LFLAGS += -acc -lnvhpcwrapnvtx
   else ifeq ($(PARAMOD),gpu)
      CCXY=80
-     NCCL=no     
+     MANAGED=yes
+     ifeq ($(MANAGED),yes)
+       GPUOPT=-gpu=cc${CCXY},managed,lineinfo
+     else
+       GPUOPT=-gpu=cc${CCXY},lineinfo
+     endif
      FFLAGS += -D_GPU
+     NCCL=no     
      ifeq ($(NCCL),yes)
        FFLAGS += -D_NCCL
      endif
-     FFLAGS += -Mfree -Kieee -Minfo=accel,stdpar -stdpar=gpu -gpu=cc${CCXY},managed,lineinfo -acc -target=gpu -traceback -O3 -DUSE_CUDA -cuda 
+     FFLAGS += -Mfree -Kieee -Minfo=accel,stdpar ${GPUOPT} -acc -target=gpu -traceback -O3 -DUSE_CUDA -cuda 
      ifeq ($(NCCL),yes)
        FFLAGS += -cudalib=cufft,nccl
      else
diff --git a/examples/Makefile b/examples/Makefile
index 5934165d..91066efb 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,36 +1,28 @@
-.PHONY: test2d fft_physical_x fft_physical_z halo_test init_test io_test 
+.PHONY: init_test test2d fft_physical_x fft_physical_z 
 
 # Just build the examples
-examples: test2d fft_physical_x fft_physical_z halo_test init_test io_test
+examples: init_test test2d fft_physical_x fft_physical_z 
 	@echo "Built the examples"
 
+init_test:
+	$(MAKE) -C $@ all
 test2d:
 	$(MAKE) -C $@ $@
 fft_physical_x:
 	$(MAKE) -C $@ all
 fft_physical_z:
 	$(MAKE) -C $@ all
-halo_test:
-	$(MAKE) -C $@ $@
-init_test:
-	$(MAKE) -C $@ all
-io_test:
-	$(MAKE) -C $@ all
 
 check:
+	cd init_test; $(MAKE) $@
 	cd test2d; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
-	cd halo_test; $(MAKE) $@
-	cd init_test; $(MAKE) $@
-	cd io_test; $(MAKE) $@
 
 clean:
+	cd init_test; $(MAKE) $@
 	cd test2d; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
-	cd halo_test; $(MAKE) $@
-	cd init_test; $(MAKE) $@
-	cd io_test; $(MAKE) $@
 
 export
diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index eb251940..1078fa7c 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -61,6 +61,7 @@ program fft_c2c_x
 
    t2 = 0._mytype
    t4 = 0._mytype
+  !$acc data copyin(in,xstart,xend) copy(out)
    do m = 1, ntest
 
       ! forward FFT
@@ -92,6 +93,7 @@ program fft_c2c_x
 
    ! checking accuracy
    error = 0._mytype
+   !$acc parallel loop default(present) reduction(+:error)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -104,6 +106,7 @@ program fft_c2c_x
          end do
       end do
    end do
+   !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
 
@@ -120,6 +123,7 @@ program fft_c2c_x
       flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
       write (*, *) 'GFLOPS : ', flops/1000._mytype**3
    end if
+  !$acc end data
 
    deallocate (in, out)
    nullify (ph)
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 991828b8..9ff47d19 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -25,6 +25,8 @@ program fft_physical_x
 
    real(mytype) :: dr, di, error, err_all, n1, flops
    integer :: ierror, i, j, k, m
+   integer :: xst1, xst2, xst3
+   integer :: xen1, xen2, xen3
    real(mytype) :: t1, t2, t3, t4
 
    call MPI_INIT(ierror)
@@ -47,10 +49,15 @@ program fft_physical_x
    ! output is Z-pencil data
    call alloc_x(in, ph, .true.)
    call alloc_z(out, ph, .true.)
+   ! Convert pointers to loops start/end to scalar
+   ! This is define loop on GPUs
+   xst1 = ph%xst(1); xen1=ph%xen(1);
+   xst2 = ph%xst(2); xen2=ph%xen(2);
+   xst3 = ph%xst(3); xen3=ph%xen(3);
    ! initilise input
-   do k = ph%xst(3), ph%xen(3)
-      do j = ph%xst(2), ph%xen(2)
-         do i = ph%xst(1), ph%xen(1)
+   do k=xst3,xen3
+      do j=xst2,xen2
+         do i=xst1,xen1
             dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
                  /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
             di = dr
@@ -61,6 +68,7 @@ program fft_physical_x
 
    t2 = 0._mytype
    t4 = 0._mytype
+   !$acc data copyin(in) copy(out)
    do m = 1, ntest
 
       ! forward FFT
@@ -92,9 +100,10 @@ program fft_physical_x
 
    ! checking accuracy
    error = 0._mytype
-   do k = ph%xst(3), ph%xen(3)
-      do j = ph%xst(2), ph%xen(2)
-         do i = ph%xst(1), ph%xen(1)
+  !$acc parallel loop default(present) reduction(+:error)
+   do k=xst3,xen3
+      do j=xst2,xen2
+         do i=xst1,xen1
             dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
                  /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
             di = dr
@@ -104,6 +113,7 @@ program fft_physical_x
          end do
       end do
    end do
+   !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
 
@@ -120,6 +130,7 @@ program fft_physical_x
       flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
       write (*, *) 'GFLOPS : ', flops/1000._mytype**3
    end if
+  !$acc end data
 
    deallocate (in, out)
    nullify (ph)
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 26e26a1f..e0a6295c 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -62,6 +62,7 @@ program fft_r2c_x
 
    t2 = 0._mytype
    t4 = 0._mytype
+   !$acc data copyin(in_r,xstart,xend) copy(out)
    do m = 1, ntest
 
       ! 3D r2c FFT
@@ -93,6 +94,7 @@ program fft_r2c_x
 
    ! checking accuracy
    error = 0._mytype
+   !$acc parallel loop default(present) reduction(+:error)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -102,6 +104,7 @@ program fft_r2c_x
          end do
       end do
    end do
+   !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
 
@@ -110,6 +113,7 @@ program fft_r2c_x
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
    end if
+   !$acc end data
 
    deallocate (in_r, out)
    nullify (ph)
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index cc7ebdba..531c3189 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -38,9 +38,9 @@ program fft_c2c_z
    nz = nz_base*resize_domain
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Test the c2c interface
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    call decomp_2d_fft_init(PHYSICAL_IN_Z) ! non-default Z-pencil input
 
    ph => decomp_2d_fft_get_ph()
@@ -63,6 +63,7 @@ program fft_c2c_z
 
    t2 = 0._mytype
    t4 = 0._mytype
+   !$acc data copyin(in,zstart,zend) copy(out)
    do m = 1, ntest
 
       ! forward FFT
@@ -94,6 +95,7 @@ program fft_c2c_z
 
    ! checking accuracy
    error = 0._mytype
+   !$acc parallel loop default(present) reduction(+:error)
    do k = zstart(3), zend(3)
       do j = zstart(2), zend(2)
          do i = zstart(1), zend(1)
@@ -106,6 +108,7 @@ program fft_c2c_z
          end do
       end do
    end do
+  !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
 
@@ -122,6 +125,7 @@ program fft_c2c_z
       flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
       write (*, *) 'GFLOPS : ', flops/1000._mytype**3
    end if
+   !$acc end data
 
    deallocate (in, out)
    nullify (ph)
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 216cd6fc..4d0c2199 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -63,6 +63,7 @@ program fft_r2c_z
 
    t2 = 0._mytype
    t4 = 0._mytype
+   !$acc data copyin(in_r,zstart,zend) copy(out)
    do m = 1, ntest
 
       ! 3D r2c FFT
@@ -93,6 +94,7 @@ program fft_r2c_z
 
    ! checking accuracy
    error = 0._mytype
+   !$acc parallel loop default(present) reduction(+:error)
    do k = zstart(3), zend(3)
       do j = zstart(2), zend(2)
          do i = zstart(1), zend(1)
@@ -103,6 +105,7 @@ program fft_r2c_z
          end do
       end do
    end do
+   !$acc end loop
 !10 format('in_r final ', I2,1x,I2,1x,I2,1x,I2,1x,F12.6,1x,F12.6)
 
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
@@ -113,6 +116,7 @@ program fft_r2c_z
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
    end if
+  !$acc end data
 
    deallocate (in_r, out)
    nullify(ph)
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index cf8a0194..02a69258 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -45,7 +45,9 @@ program test2d
    call alloc_y(u2, opt_global=.true.)
    call alloc_z(u3, opt_global=.true.)
 
+   !$acc data copyin(data1,xstart,xend,ystart,yend,zstart,zend) copy(u1,u2,u3) 
    ! original x-pensil based data
+   !$acc parallel loop default(present) 
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -53,6 +55,7 @@ program test2d
          end do
       end do
    end do
+   !$acc end loop
 
 10 format(15I5)
 
@@ -73,6 +76,7 @@ program test2d
 
 #ifdef DEBUG
    if (nrank == 0) then
+      !$acc update self(u2)  
       write (*, *) ' '
       write (*, *) 'Y-pencil'
       write (*, 10) int(u2)
@@ -83,6 +87,7 @@ program test2d
    ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
 
    ! also check the transposition this way
+   !$acc parallel loop default(present) 
    do k = ystart(3), yend(3)
       do j = ystart(2), yend(2)
          do i = ystart(1), yend(1)
@@ -90,6 +95,7 @@ program test2d
          end do
       end do
    end do
+   !$acc end loop
    call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
    if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
    if (error_flag) call decomp_2d_abort(1, "error swaping x->y")
@@ -100,6 +106,7 @@ program test2d
 
 #ifdef DEBUG
    if (nrank == 0) then
+      !$acc update self(u3)  
       write (*, *) ' '
       write (*, *) 'Z-pencil'
       write (*, 10) int(u3)
@@ -109,6 +116,7 @@ program test2d
    ! call decomp_2d_write_one(3,u3,'u3.dat')
    ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
+   !$acc parallel loop default(present) 
    do k = zstart(3), zend(3)
       do j = zstart(2), zend(2)
          do i = zstart(1), zend(1)
@@ -116,6 +124,7 @@ program test2d
          end do
       end do
    end do
+   !$acc end loop
    call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
    if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
    if (error_flag) call decomp_2d_abort(2, "error swaping y->z")
@@ -125,6 +134,7 @@ program test2d
    call transpose_z_to_y(u3, u2)
    ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
+   !$acc parallel loop default(present) 
    do k = ystart(3), yend(3)
       do j = ystart(2), yend(2)
          do i = ystart(1), yend(1)
@@ -132,6 +142,7 @@ program test2d
          end do
       end do
    end do
+   !$acc end loop
    call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
    if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
    if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
@@ -141,6 +152,7 @@ program test2d
    call transpose_y_to_x(u2, u1)
    ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
+   !$acc parallel loop default(present) 
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -148,6 +160,7 @@ program test2d
          end do
       end do
    end do
+   !$acc end loop
    call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
    if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
    if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
@@ -160,6 +173,7 @@ program test2d
 
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
+   !$acc end data
    deallocate (u1, u2, u3)
 
 end program test2d

From 7086de4e2affd43b1132f045a8acf5f7a7744d11 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Fri, 13 Jan 2023 13:26:09 +0000
Subject: [PATCH 084/436] Updates examples for non managed case with exception
 of Halo

---
 examples/Makefile                  |  8 +++++--
 examples/io_test/io_plane_test.f90 | 35 +++++++++++++++++++++++++-----
 examples/io_test/io_test.f90       |  9 +++++++-
 examples/io_test/io_var_test.f90   | 15 +++++++++++++
 examples/test2d/test2d.f90         |  1 +
 5 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index 91066efb..1643583b 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,7 +1,7 @@
-.PHONY: init_test test2d fft_physical_x fft_physical_z 
+.PHONY: init_test test2d fft_physical_x fft_physical_z io_test 
 
 # Just build the examples
-examples: init_test test2d fft_physical_x fft_physical_z 
+examples: init_test test2d fft_physical_x fft_physical_z io_test
 	@echo "Built the examples"
 
 init_test:
@@ -12,17 +12,21 @@ fft_physical_x:
 	$(MAKE) -C $@ all
 fft_physical_z:
 	$(MAKE) -C $@ all
+io_test:
+	$(MAKE) -C $@ all
 
 check:
 	cd init_test; $(MAKE) $@
 	cd test2d; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
+	cd io_test; $(MAKE) $@
 
 clean:
 	cd init_test; $(MAKE) $@
 	cd test2d; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
+	cd io_test; $(MAKE) $@
 
 export
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index e883d5c1..ec539d31 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -1,9 +1,15 @@
+!! NB in case of GPU only the writing in the aligned pencil (i.e. X for a 1 array) is performed. 
+!! IO subrotines needs update for non managed GPU case
 program io_plane_test
 
    use mpi
 
    use decomp_2d
    use decomp_2d_io
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
 
    implicit none
 
@@ -37,7 +43,12 @@ program io_plane_test
    call alloc_y(u2, .true.)
    call alloc_z(u3, .true.)
 
+   ! For GPU we port the global data create the different pencil arrays 
+   ! Move back to host the arrays for writing on disk
+
+   !$acc data copyin(data1,xstart,xend) copy(u1,u2,u3) 
    ! original X-pensil based data
+   !$acc parallel loop default(present) 
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -45,22 +56,34 @@ program io_plane_test
          end do
       end do
    end do
+   !$acc end loop
+   write(*,*) 'End Loop'
+   call transpose_x_to_y(u1, u2)
+   call transpose_y_to_z(u2, u3)
+   !$acc update self(u1)  
+   !$acc update self(u2)  
+   !$acc update self(u3)  
+   !$acc end data
+   call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
+ 
    call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
+#if !defined(_GPU)
    call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
    call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
-
+#endif
    ! Y-pencil data
-   call transpose_x_to_y(u1, u2)
-   call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(2, u2, 2, ny/2, '.', 'y_pencil-y_plane.dat', 'test')
+#if !defined(_GPU)
+   call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(2, u2, 3, nz/2, '.', 'y_pencil-z_plane.dat', 'test')
+#endif
 
    ! Z-pencil data
-   call transpose_y_to_z(u2, u3)
+#if !defined(_GPU)
    call decomp_2d_write_plane(3, u3, 1, nx/2, '.', 'z_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(3, u3, 2, ny/2, '.', 'z_pencil-y_plane.dat', 'test')
+#endif
    call decomp_2d_write_plane(3, u3, 3, nz/2, '.', 'z_pencil-z_plane.dat', 'test')
-
    ! Attemp to read the files
    if (nrank == 0) then
       inquire (iolength=iol) data1(1, 1, 1)
@@ -84,6 +107,7 @@ program io_plane_test
       write (*, *) 'passed self test x-plane'
 
       ! Y-plane
+#if !defined(_GPU)
       allocate (work(nx, 1, nz))
       open (10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
             ACCESS='DIRECT', RECL=iol)
@@ -118,6 +142,7 @@ program io_plane_test
       deallocate (work)
 
       write (*, *) 'passed self test z-plane'
+#endif
 
    end if
 
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 7e0b1fbf..7ce2c150 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -53,6 +53,8 @@ program io_test
    call alloc_z(u3b, .true.)
 
    ! original x-pencil based data
+   !$acc data copyin(data1,xstart,xend) copy(u1,u2,u3) 
+   !$acc parallel loop default(present)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -60,10 +62,15 @@ program io_test
          end do
       end do
    end do
+   !$acc end loop
 
    ! transpose
    call transpose_x_to_y(u1, u2)
    call transpose_y_to_z(u2, u3)
+   !$acc update self(u1)  
+   !$acc update self(u2)  
+   !$acc update self(u3)
+   !$acc end data  
 
    ! write to disk
    call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
@@ -79,7 +86,7 @@ program io_test
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
-            if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) stop 1
+            if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 1
          end do
       end do
    end do
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index a35eeeca..57477df4 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -112,6 +112,8 @@ program io_var_test
    call alloc_z(u3l_b, large, .true.)
 
    ! distribute the data
+   !$acc data copyin(data1,cdata1,data1_large,xstart,xend,large%xst,large%xen) copy(u1,u2,u3,u1l,u2l,u3l,cu1,cu2,cu3) 
+   !$acc parallel loop default(present)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -120,6 +122,8 @@ program io_var_test
          end do
       end do
    end do
+   !$acc end loop
+   !$acc parallel loop default(present)
    do k = large%xst(3), large%xen(3)
       do j = large%xst(2), large%xen(2)
          do i = large%xst(1), large%xen(1)
@@ -127,6 +131,7 @@ program io_var_test
          end do
       end do
    end do
+   !$acc end loop
 
    ! transpose
    call transpose_x_to_y(u1, u2)
@@ -135,6 +140,16 @@ program io_var_test
    call transpose_y_to_z(u2l, u3l, large)
    call transpose_x_to_y(cu1, cu2)
    call transpose_y_to_z(cu2, cu3)
+   !$acc update self (u1)
+   !$acc update self (u2)
+   !$acc update self (u3)
+   !$acc update self (u1l)
+   !$acc update self (u2l)
+   !$acc update self (u3l)
+   !$acc update self (cu1)
+   !$acc update self (cu2)
+   !$acc update self (cu3)
+   !$acc end data
 
    ! open file for IO
    write (filename, '(A,I3.3)') 'io_var_data.', nproc
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 02a69258..a9a446ff 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -61,6 +61,7 @@ program test2d
 
 #ifdef DEBUG
    if (nrank == 0) then
+      !$acc update self(u1)  
       write (*, *) 'Numbers held on Rank 0'
       write (*, *) ' '
       write (*, *) 'X-pencil'

From 70d130ff1a217c30eb0b23310f6a8b9432c3c6c7 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 15 Jan 2023 09:37:07 +0100
Subject: [PATCH 085/436] Minor update

---
 README.md       | 2 +-
 src/fft_log.f90 | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 143beaf5..1de50561 100644
--- a/README.md
+++ b/README.md
@@ -88,7 +88,7 @@ The external code can use the named variables to check the FFT backend used in a
 - The generic backend supports the OVERWRITE flag but it can not perform in-place transforms
 - The FFTW3 and FFTW3_F03 backends support the OVERWRITE flag and can perform in-place complex 1D fft
 - The oneMKL backend supports the OVERWRITE flag and can perform in-place complex 1D fft
-- The cuFFT backend supports the OVERWRITE flag but it can not perform in-place transforms
+- The cuFFT backend supports the OVERWRITE flag and can perform in-place complex 1D fft
 
 ## Miscellaneous
 
diff --git a/src/fft_log.f90 b/src/fft_log.f90
index 85d59352..b0608486 100644
--- a/src/fft_log.f90
+++ b/src/fft_log.f90
@@ -53,11 +53,11 @@ module subroutine decomp_2d_fft_log(backend)
          call decomp_info_print(sp, io_unit, "sp")
          write (io_unit, *) ''
 #ifdef OVERWRITE
-         if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_GENERIC .or. &
-             D2D_FFT_BACKEND == D2D_FFT_BACKEND_CUFFT) then
+         if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_GENERIC) then
             write (io_unit, *) 'OVERWRITE is supported but transforms are not performed in-place'
             write (io_unit, *) ''
-         else if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3 .or. &
+         else if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_CUFFT .or. &
+                  D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3 .or. &
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_FFTW3_F03 .or. &
                   D2D_FFT_BACKEND == D2D_FFT_BACKEND_MKL) then
             write (io_unit, *) 'OVERWRITE is supported but in-place transforms is limited to complex transforms'

From f9d047fcc998f3737cb3ce61c3e04848b12751ad Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 15 Jan 2023 10:30:03 +0100
Subject: [PATCH 086/436] Fuse clean and realclean

---
 examples/io_test/Makefile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/examples/io_test/Makefile b/examples/io_test/Makefile
index 26a7ab61..3d4a004b 100644
--- a/examples/io_test/Makefile
+++ b/examples/io_test/Makefile
@@ -48,8 +48,6 @@ endif
 
 clean:
 	rm -f *.o io_test io_read io_var_test io_plane_test io_bench
-
-realclean: clean
 	rm -f *.dat io_var_data.* *.log
 
 %.o : %.f90

From b569a0a1526f602ba8e2b56c873a01735542969c Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 16 Jan 2023 06:04:39 +0000
Subject: [PATCH 087/436] Corrections for halo_test and io_test

---
 examples/halo_test/halo_test.f90 | 225 +++++++++++++++----------------
 examples/io_test/io_test.f90     |   2 +-
 2 files changed, 110 insertions(+), 117 deletions(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 8c64b7a6..9c99ded4 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -334,6 +334,9 @@ subroutine test_div_haloZ()
 #else
       logical, parameter :: global = .false.
 #endif
+      integer :: i1, in ! I loop start/end
+      integer :: j1, jn ! J loop start/end
+      integer :: k1, kn ! K loop start/end
 
       ! Expected sizes
       nx_expected = zsize(1) + 2
@@ -355,152 +358,142 @@ subroutine test_div_haloZ()
       ! du/dx
 #ifdef HALO_GLOBAL
       call update_halo(u3, uh, 1, opt_global=.true., opt_pencil=3)
+      i1 = zstart(1); in = zend(1)
+      j1 = zstart(2); jn = zend(2)
 #else
       call update_halo(u3, uh, 1, opt_pencil=3)
+      i1 = 1; in = zsize(1)
+      j1 = 1; jn = zsize(2)
 #endif
+      k1 = 2; kn = zsize(3) - 1
 
       call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
 
-#ifdef HALO_GLOBAL
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-#else
-            do j = 1, zsize(2)
-               do i = 1, zsize(1)
-#endif
-                  do k = 2, zsize(3) - 1
-                     wk3(i, j, k) = uh(i + 1, j, k) - uh(i - 1, j, k)
-                  end do
-               end do
+      do j = j1, jn
+         do i = i1, in
+            do k = k1, kn
+               wk3(i, j, k) = uh(i + 1, j, k) - uh(i - 1, j, k)
             end do
+         end do
+      end do
 
-            ! dv/dy
+      ! dv/dy
 #ifdef HALO_GLOBAL
-            call update_halo(v3, vh, 1, opt_global=.true., opt_pencil=3)
+      call update_halo(v3, vh, 1, opt_global=.true., opt_pencil=3)
 #else
-            call update_halo(v3, vh, 1, opt_pencil=3)
+      call update_halo(v3, vh, 1, opt_pencil=3)
 #endif
 
-            call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
+      call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
 
-#ifdef HALO_GLOBAL
-            do j = zstart(2), zend(2)
-               do i = zstart(1), zend(1)
-#else
-                  do j = 1, zsize(2)
-                     do i = 1, zsize(1)
-#endif
-                        do k = 2, zsize(3) - 1
-                           wk3(i, j, k) = wk3(i, j, k) + vh(i, j + 1, k) - vh(i, j - 1, k)
-                        end do
-                     end do
-                  end do
+      do j = j1, jn
+         do i = i1, in
+            do k = k1, kn
+               wk3(i, j, k) = wk3(i, j, k) + vh(i, j + 1, k) - vh(i, j - 1, k)
+            end do
+         end do
+      end do
 
-                  ! dw/dz
-#ifdef HALO_GLOBAL
-                  do j = zstart(2), zend(2)
-                     do i = zstart(1), zend(1)
-#else
-                        do j = 1, zsize(2)
-                           do i = 1, zsize(1)
-#endif
-                              do k = 2, zsize(3) - 1
-                                 wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
-                              end do
-                           end do
-                        end do
-
-                        call transpose_z_to_y(wk3, wk2)
-                        call transpose_y_to_x(wk2, div4)
-
-                        ! Compute error
-                        call check_err(div4, "Z")
-
-                        deallocate (uh, vh, wk2, wk3)
-                        end subroutine test_div_haloZ
-
-                        subroutine check_err(divh, pencil)
-
-                           real(mytype), dimension(:, :, :), intent(in) :: divh
-                           character(len=*), intent(in) :: pencil
-
-                           real(mytype), dimension(:, :, :), allocatable :: tmp
-
-                           real(mytype) :: divmag
-
-                           ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
-                           !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
-                           !      causes a SEGFAULT. Explicitly computing the difference in a temporary
-                           !      array seems to be OK.
-                           allocate (tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
-                           tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - div1(2:xlast, 2:ylast, 2:zlast)
-                           err = mag(tmp(2:xlast, 2:ylast, 2:zlast))
-                           deallocate (tmp)
-                           divmag = mag(div1(2:xlast, 2:ylast, 2:zlast))
-                           if (err < epsilon(divmag)*divmag) then
-                              passing = .true.
-                           else
-                              passing = .false.
-                           end if
-                           all_pass = all_pass .and. passing
-
-                           if (nrank == 0) then
-                              write (*, *) '-----------------------------------------------'
-                              write (*, *) 'Calculated via halo exchange (data in '//pencil//'-pencil)'
+      ! dw/dz
+      do j = j1, jn
+         do i = i1, in
+            do k = k1, kn
+               wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
+            end do
+         end do
+      end do
+
+      call transpose_z_to_y(wk3, wk2)
+      call transpose_y_to_x(wk2, div4)
+
+      ! Compute error
+      call check_err(div4, "Z")
+
+      deallocate (uh, vh, wk2, wk3)
+      end subroutine test_div_haloZ
+
+   subroutine check_err(divh, pencil)
+
+      real(mytype), dimension(:, :, :), intent(in) :: divh
+      character(len=*), intent(in) :: pencil
+
+      real(mytype), dimension(:, :, :), allocatable :: tmp
+
+      real(mytype) :: divmag
+
+      ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
+      !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
+      !      causes a SEGFAULT. Explicitly computing the difference in a temporary
+      !      array seems to be OK.
+      allocate (tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
+      tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - div1(2:xlast, 2:ylast, 2:zlast)
+      err = mag(tmp(2:xlast, 2:ylast, 2:zlast))
+      deallocate (tmp)
+      divmag = mag(div1(2:xlast, 2:ylast, 2:zlast))
+      if (err < epsilon(divmag)*divmag) then
+         passing = .true.
+      else
+         passing = .false.
+      end if
+      all_pass = all_pass .and. passing
+
+      if (nrank == 0) then
+         write (*, *) '-----------------------------------------------'
+         write (*, *) 'Calculated via halo exchange (data in '//pencil//'-pencil)'
 #ifdef DEBUG
-                              write (*, *) (divh(i, i, i), i=2, 13)
+         write (*, *) (divh(i, i, i), i=2, 13)
 #endif
-                              write (*, *) 'Error: ', err, '; Relative: ', err/divmag
-                              write (*, *) 'Pass: ', passing
-                           end if
+         write (*, *) 'Error: ', err, '; Relative: ', err/divmag
+         write (*, *) 'Pass: ', passing
+      end if
 
-                        end subroutine check_err
+   end subroutine check_err
 
-                        real(mytype) function mag(a)
+   real(mytype) function mag(a)
 
-                           real(mytype), dimension(:, :, :), intent(in) :: a
+      real(mytype), dimension(:, :, :), intent(in) :: a
 
-                           real(mytype) :: lmag, gmag
+      real(mytype) :: lmag, gmag
 
-                           lmag = sum(a(:, :, :)**2)
-                           call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-                           if (ierror /= 0) then
-                              call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-                                                   "halo_test::mag::MPI_Allreduce")
-                           end if
+      lmag = sum(a(:, :, :)**2)
+      call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+      if (ierror /= 0) then
+         call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                              "halo_test::mag::MPI_Allreduce")
+      end if
 
-                           mag = sqrt(gmag/(nx - 2)/(ny - 2)/(nz - 2))
+      mag = sqrt(gmag/(nx - 2)/(ny - 2)/(nz - 2))
 
-                        end function mag
+   end function mag
 
-                        subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
+   subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
 
-                           real(mytype), dimension(:, :, :), intent(in) :: arrh
-                           integer, intent(in) :: nx_expected, ny_expected, nz_expected
-                           character(len=*), intent(in) :: tag
+      real(mytype), dimension(:, :, :), intent(in) :: arrh
+      integer, intent(in) :: nx_expected, ny_expected, nz_expected
+      character(len=*), intent(in) :: tag
 
-                           integer :: nx, ny, nz
+      integer :: nx, ny, nz
 
-                           character(len=128) :: rank_lbl
+      character(len=128) :: rank_lbl
 
-                           nx = size(arrh, 1)
-                           ny = size(arrh, 2)
-                           nz = size(arrh, 3)
+      nx = size(arrh, 1)
+      ny = size(arrh, 2)
+      nz = size(arrh, 3)
 
-                           write (rank_lbl, "(A,I0,A)") "Rank", nrank, ":"
+      write (rank_lbl, "(A,I0,A)") "Rank", nrank, ":"
 
-                           if ((nx /= nx_expected) .or. &
-                               (ny /= ny_expected) .or. &
-                               (nz /= nz_expected)) then
-                              write (*, *) trim(rank_lbl), " ", tag, ":ERROR: halo size"
-                              write (*, *) trim(rank_lbl), " ", "+ Expected: ", nx_expected, " ", ny_expected, " ", nz_expected, " "
-                              write (*, *) trim(rank_lbl), " ", "+ Got:      ", nx, " ", ny, " ", nz, " "
+      if ((nx /= nx_expected) .or. &
+          (ny /= ny_expected) .or. &
+          (nz /= nz_expected)) then
+         write (*, *) trim(rank_lbl), " ", tag, ":ERROR: halo size"
+         write (*, *) trim(rank_lbl), " ", "+ Expected: ", nx_expected, " ", ny_expected, " ", nz_expected, " "
+         write (*, *) trim(rank_lbl), " ", "+ Got:      ", nx, " ", ny, " ", nz, " "
 
-                              all_pass = .false.
-                           else
-                              write (*, *) trim(rank_lbl), " ", tag, ":PASS"
-                           end if
+         all_pass = .false.
+      else
+         write (*, *) trim(rank_lbl), " ", tag, ":PASS"
+      end if
 
-                        end subroutine test_halo_size
+   end subroutine test_halo_size
 
-                        end program halo_test
+end program halo_test
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 7ce2c150..0bfceb94 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -86,7 +86,7 @@ program io_test
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
-            if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 1
+            if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) stop 1
          end do
       end do
    end do

From fd47d2d6cf48fc3f541c111e568b0c548ceb6765 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 16 Jan 2023 06:27:38 +0000
Subject: [PATCH 088/436] Update io_plane_test with GPU warning message

---
 examples/io_test/io_plane_test.f90 | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index ec539d31..dfe8620b 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -57,15 +57,20 @@ program io_plane_test
       end do
    end do
    !$acc end loop
-   write(*,*) 'End Loop'
    call transpose_x_to_y(u1, u2)
    call transpose_y_to_z(u2, u3)
    !$acc update self(u1)  
    !$acc update self(u2)  
    !$acc update self(u3)  
    !$acc end data
-   call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
  
+#if defined(_GPU)
+   write (*, *) 'WARNING for GPU IO: write of a plane is supported only in the aligned pencil'
+   write (*, *) '                    i.e. x_plane-x_pencil'
+   write (*, *) '                    i.e. y_plane-y_pencil'
+   write (*, *) '                    i.e. z_plane-z_pencil'
+#endif
+
    call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
 #if !defined(_GPU)
    call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')

From 44acc88e76596a00c84d603ce51efc7fb356c531 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Tue, 17 Jan 2023 08:02:56 +0100
Subject: [PATCH 089/436] Fix the code, not the example

---
 examples/io_test/io_plane_test.f90 | 121 +++++++++++++++--------------
 src/io_write_plane.inc             |  11 +++
 2 files changed, 73 insertions(+), 59 deletions(-)

diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index dfe8620b..1820a2f9 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -1,5 +1,9 @@
+!!
+!! FIXME The issue below is specific to GPU and should be discussed in a dedicated github issue
+!!
 !! NB in case of GPU only the writing in the aligned pencil (i.e. X for a 1 array) is performed. 
 !! IO subrotines needs update for non managed GPU case
+!!
 program io_plane_test
 
    use mpi
@@ -22,6 +26,7 @@ program io_plane_test
    real(mytype), allocatable, dimension(:, :, :) :: work
 
    integer :: i, j, k, m, ierror, iol
+   logical :: found
 
    call MPI_INIT(ierror)
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
@@ -64,90 +69,88 @@ program io_plane_test
    !$acc update self(u3)  
    !$acc end data
  
-#if defined(_GPU)
-   write (*, *) 'WARNING for GPU IO: write of a plane is supported only in the aligned pencil'
-   write (*, *) '                    i.e. x_plane-x_pencil'
-   write (*, *) '                    i.e. y_plane-y_pencil'
-   write (*, *) '                    i.e. z_plane-z_pencil'
-#endif
-
    call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
-#if !defined(_GPU)
    call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
    call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
-#endif
    ! Y-pencil data
    call decomp_2d_write_plane(2, u2, 2, ny/2, '.', 'y_pencil-y_plane.dat', 'test')
-#if !defined(_GPU)
    call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(2, u2, 3, nz/2, '.', 'y_pencil-z_plane.dat', 'test')
-#endif
 
    ! Z-pencil data
-#if !defined(_GPU)
    call decomp_2d_write_plane(3, u3, 1, nx/2, '.', 'z_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(3, u3, 2, ny/2, '.', 'z_pencil-y_plane.dat', 'test')
-#endif
    call decomp_2d_write_plane(3, u3, 3, nz/2, '.', 'z_pencil-z_plane.dat', 'test')
    ! Attemp to read the files
    if (nrank == 0) then
       inquire (iolength=iol) data1(1, 1, 1)
 
       ! X-plane
-      allocate (work(1, ny, nz))
-      open (10, FILE='x_pencil-x_plane.dat', FORM='unformatted', &
-            ACCESS='DIRECT', RECL=iol)
-      m = 1
-      do k = 1, nz
-         do j = 1, ny
-            read (10, rec=m) work(1, j, k)
-            m = m + 1
+      inquire (file='x_pencil-x_plane.dat', exist=found)
+      if (found) then
+         allocate (work(1, ny, nz))
+         open (10, FILE='x_pencil-x_plane.dat', FORM='unformatted', &
+               ACCESS='DIRECT', RECL=iol)
+         m = 1
+         do k = 1, nz
+            do j = 1, ny
+               read (10, rec=m) work(1, j, k)
+               m = m + 1
+            end do
          end do
-      end do
-!     write(*,*) ' '
-!     write(*,'(15I5)') int(work)
-      close (10)
-      deallocate (work)
+!        write(*,*) ' '
+!        write(*,'(15I5)') int(work)
+         close (10)
+         deallocate (work)
+         write (*, *) 'passed self test x-plane'
+      else
+         write (*, *) "Warning : x_pencil-x_plane.dat is missing"
+      endif
 
-      write (*, *) 'passed self test x-plane'
 
       ! Y-plane
-#if !defined(_GPU)
-      allocate (work(nx, 1, nz))
-      open (10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
-            ACCESS='DIRECT', RECL=iol)
-      m = 1
-      do k = 1, nz
-         do i = 1, nx
-            read (10, rec=m) work(i, 1, k)
-            m = m + 1
+      inquire (file='x_pencil-y_plane.dat', exist=found)
+      if (found) then
+         allocate (work(nx, 1, nz))
+         open (10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
+               ACCESS='DIRECT', RECL=iol)
+         m = 1
+         do k = 1, nz
+            do i = 1, nx
+               read (10, rec=m) work(i, 1, k)
+               m = m + 1
+            end do
          end do
-      end do
-!     write(*,*) ' '
-!     write(*,'(15I5)') int(work)
-      close (10)
-      deallocate (work)
-
-      write (*, *) 'passed self test y-plane'
+!        write(*,*) ' '
+!        write(*,'(15I5)') int(work)
+         close (10)
+         deallocate (work)
+         write (*, *) 'passed self test y-plane'
+      else
+         write (*, *) 'Warning : x_pencil-y_plane.dat is missing'
+      endif
 
       ! Z-plane
-      allocate (work(nx, ny, 1))
-      open (10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
-            ACCESS='DIRECT', RECL=iol)
-      m = 1
-      do j = 1, ny
-         do i = 1, nx
-            read (10, rec=m) work(i, j, 1)
-            m = m + 1
+      inquire (file='x_pencil-z_plane.dat', exist=found)
+      if (found) then
+         allocate (work(nx, ny, 1))
+         open (10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
+               ACCESS='DIRECT', RECL=iol)
+         m = 1
+         do j = 1, ny
+            do i = 1, nx
+               read (10, rec=m) work(i, j, 1)
+               m = m + 1
+            end do
          end do
-      end do
-!     write(*,*) ' '
-!     write(*,'(15I5)') int(work)
-      close (10)
-      deallocate (work)
-
-      write (*, *) 'passed self test z-plane'
-#endif
+!        write(*,*) ' '
+!        write(*,'(15I5)') int(work)
+         close (10)
+         deallocate (work)
+         write (*, *) 'passed self test z-plane'
+      else
+         write (*, *) 'Warning : x_pencil-z_plane.dat is missing'
+      endif
 
    end if
 
diff --git a/src/io_write_plane.inc b/src/io_write_plane.inc
index 10f23074..6965060e 100644
--- a/src/io_write_plane.inc
+++ b/src/io_write_plane.inc
@@ -22,6 +22,17 @@ else
    call get_decomp_info(decomp)
 end if
 
+! The current GPU build will not perform transpose operations when writing planes
+#if defined(_GPU)
+if (ipencil /= iplane) then
+   call decomp_2d_warning(__FILE__, &
+                          __LINE__, &
+                          ipencil, &
+                          "GPU build does not allow transpose operations when writing planes")
+   return
+endif
+#endif
+
 opened_new = .false.
 
 if (iplane == 1) then

From 3173d40421681d6f09bb11136c9e3915a9403839 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Tue, 17 Jan 2023 08:21:59 +0100
Subject: [PATCH 090/436] fprettify the code

---
 examples/fft_physical_x/fft_c2c_x.f90  |  4 ++--
 examples/fft_physical_x/fft_grid_x.f90 | 22 +++++++++++-----------
 examples/fft_physical_z/fft_c2c_z.f90  |  2 +-
 examples/fft_physical_z/fft_r2c_z.f90  |  6 +++---
 examples/halo_test/halo_test.f90       |  2 +-
 examples/io_test/io_plane_test.f90     | 23 +++++++++++------------
 examples/io_test/io_test.f90           |  8 ++++----
 examples/io_test/io_var_test.f90       |  2 +-
 examples/test2d/test2d.f90             | 18 +++++++++---------
 src/fft_cufft.f90                      | 25 ++++++++++++-------------
 src/transpose_x_to_y.f90               | 10 +++++-----
 src/transpose_y_to_x.f90               |  8 ++++----
 src/transpose_y_to_z.f90               | 20 ++++++++++----------
 src/transpose_z_to_y.f90               |  8 ++++----
 14 files changed, 78 insertions(+), 80 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 1078fa7c..f3226876 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -61,7 +61,7 @@ program fft_c2c_x
 
    t2 = 0._mytype
    t4 = 0._mytype
-  !$acc data copyin(in,xstart,xend) copy(out)
+   !$acc data copyin(in,xstart,xend) copy(out)
    do m = 1, ntest
 
       ! forward FFT
@@ -123,7 +123,7 @@ program fft_c2c_x
       flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
       write (*, *) 'GFLOPS : ', flops/1000._mytype**3
    end if
-  !$acc end data
+   !$acc end data
 
    deallocate (in, out)
    nullify (ph)
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 9ff47d19..5a17078a 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -51,13 +51,13 @@ program fft_physical_x
    call alloc_z(out, ph, .true.)
    ! Convert pointers to loops start/end to scalar
    ! This is define loop on GPUs
-   xst1 = ph%xst(1); xen1=ph%xen(1);
-   xst2 = ph%xst(2); xen2=ph%xen(2);
-   xst3 = ph%xst(3); xen3=ph%xen(3);
+   xst1 = ph%xst(1); xen1 = ph%xen(1); 
+   xst2 = ph%xst(2); xen2 = ph%xen(2); 
+   xst3 = ph%xst(3); xen3 = ph%xen(3); 
    ! initilise input
-   do k=xst3,xen3
-      do j=xst2,xen2
-         do i=xst1,xen1
+   do k = xst3, xen3
+      do j = xst2, xen2
+         do i = xst1, xen1
             dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
                  /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
             di = dr
@@ -100,10 +100,10 @@ program fft_physical_x
 
    ! checking accuracy
    error = 0._mytype
-  !$acc parallel loop default(present) reduction(+:error)
-   do k=xst3,xen3
-      do j=xst2,xen2
-         do i=xst1,xen1
+   !$acc parallel loop default(present) reduction(+:error)
+   do k = xst3, xen3
+      do j = xst2, xen2
+         do i = xst1, xen1
             dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
                  /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
             di = dr
@@ -130,7 +130,7 @@ program fft_physical_x
       flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
       write (*, *) 'GFLOPS : ', flops/1000._mytype**3
    end if
-  !$acc end data
+   !$acc end data
 
    deallocate (in, out)
    nullify (ph)
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 531c3189..e2d22ee8 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -108,7 +108,7 @@ program fft_c2c_z
          end do
       end do
    end do
-  !$acc end loop
+   !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
 
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 4d0c2199..bb9d5b84 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -116,11 +116,11 @@ program fft_r2c_z
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
    end if
-  !$acc end data
+   !$acc end data
 
    deallocate (in_r, out)
-   nullify(ph)
-   nullify(sp)
+   nullify (ph)
+   nullify (sp)
    call decomp_2d_fft_finalize
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 9c99ded4..29447a18 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -410,7 +410,7 @@ subroutine test_div_haloZ()
       call check_err(div4, "Z")
 
       deallocate (uh, vh, wk2, wk3)
-      end subroutine test_div_haloZ
+   end subroutine test_div_haloZ
 
    subroutine check_err(divh, pencil)
 
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 1820a2f9..7fc6ddbd 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -1,7 +1,7 @@
 !!
 !! FIXME The issue below is specific to GPU and should be discussed in a dedicated github issue
 !!
-!! NB in case of GPU only the writing in the aligned pencil (i.e. X for a 1 array) is performed. 
+!! NB in case of GPU only the writing in the aligned pencil (i.e. X for a 1 array) is performed.
 !! IO subrotines needs update for non managed GPU case
 !!
 program io_plane_test
@@ -48,12 +48,12 @@ program io_plane_test
    call alloc_y(u2, .true.)
    call alloc_z(u3, .true.)
 
-   ! For GPU we port the global data create the different pencil arrays 
+   ! For GPU we port the global data create the different pencil arrays
    ! Move back to host the arrays for writing on disk
 
-   !$acc data copyin(data1,xstart,xend) copy(u1,u2,u3) 
+   !$acc data copyin(data1,xstart,xend) copy(u1,u2,u3)
    ! original X-pensil based data
-   !$acc parallel loop default(present) 
+   !$acc parallel loop default(present)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -64,11 +64,11 @@ program io_plane_test
    !$acc end loop
    call transpose_x_to_y(u1, u2)
    call transpose_y_to_z(u2, u3)
-   !$acc update self(u1)  
-   !$acc update self(u2)  
-   !$acc update self(u3)  
+   !$acc update self(u1)
+   !$acc update self(u2)
+   !$acc update self(u3)
    !$acc end data
- 
+
    call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
    call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
@@ -105,8 +105,7 @@ program io_plane_test
          write (*, *) 'passed self test x-plane'
       else
          write (*, *) "Warning : x_pencil-x_plane.dat is missing"
-      endif
-
+      end if
 
       ! Y-plane
       inquire (file='x_pencil-y_plane.dat', exist=found)
@@ -128,7 +127,7 @@ program io_plane_test
          write (*, *) 'passed self test y-plane'
       else
          write (*, *) 'Warning : x_pencil-y_plane.dat is missing'
-      endif
+      end if
 
       ! Z-plane
       inquire (file='x_pencil-z_plane.dat', exist=found)
@@ -150,7 +149,7 @@ program io_plane_test
          write (*, *) 'passed self test z-plane'
       else
          write (*, *) 'Warning : x_pencil-z_plane.dat is missing'
-      endif
+      end if
 
    end if
 
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 0bfceb94..e3a71854 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -53,7 +53,7 @@ program io_test
    call alloc_z(u3b, .true.)
 
    ! original x-pencil based data
-   !$acc data copyin(data1,xstart,xend) copy(u1,u2,u3) 
+   !$acc data copyin(data1,xstart,xend) copy(u1,u2,u3)
    !$acc parallel loop default(present)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
@@ -67,10 +67,10 @@ program io_test
    ! transpose
    call transpose_x_to_y(u1, u2)
    call transpose_y_to_z(u2, u3)
-   !$acc update self(u1)  
-   !$acc update self(u2)  
+   !$acc update self(u1)
+   !$acc update self(u2)
    !$acc update self(u3)
-   !$acc end data  
+   !$acc end data
 
    ! write to disk
    call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 57477df4..5d6dfd7f 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -112,7 +112,7 @@ program io_var_test
    call alloc_z(u3l_b, large, .true.)
 
    ! distribute the data
-   !$acc data copyin(data1,cdata1,data1_large,xstart,xend,large%xst,large%xen) copy(u1,u2,u3,u1l,u2l,u3l,cu1,cu2,cu3) 
+   !$acc data copyin(data1,cdata1,data1_large,xstart,xend,large%xst,large%xen) copy(u1,u2,u3,u1l,u2l,u3l,cu1,cu2,cu3)
    !$acc parallel loop default(present)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index a9a446ff..cc2521b2 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -45,9 +45,9 @@ program test2d
    call alloc_y(u2, opt_global=.true.)
    call alloc_z(u3, opt_global=.true.)
 
-   !$acc data copyin(data1,xstart,xend,ystart,yend,zstart,zend) copy(u1,u2,u3) 
+   !$acc data copyin(data1,xstart,xend,ystart,yend,zstart,zend) copy(u1,u2,u3)
    ! original x-pensil based data
-   !$acc parallel loop default(present) 
+   !$acc parallel loop default(present)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -61,7 +61,7 @@ program test2d
 
 #ifdef DEBUG
    if (nrank == 0) then
-      !$acc update self(u1)  
+      !$acc update self(u1)
       write (*, *) 'Numbers held on Rank 0'
       write (*, *) ' '
       write (*, *) 'X-pencil'
@@ -77,7 +77,7 @@ program test2d
 
 #ifdef DEBUG
    if (nrank == 0) then
-      !$acc update self(u2)  
+      !$acc update self(u2)
       write (*, *) ' '
       write (*, *) 'Y-pencil'
       write (*, 10) int(u2)
@@ -88,7 +88,7 @@ program test2d
    ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
 
    ! also check the transposition this way
-   !$acc parallel loop default(present) 
+   !$acc parallel loop default(present)
    do k = ystart(3), yend(3)
       do j = ystart(2), yend(2)
          do i = ystart(1), yend(1)
@@ -107,7 +107,7 @@ program test2d
 
 #ifdef DEBUG
    if (nrank == 0) then
-      !$acc update self(u3)  
+      !$acc update self(u3)
       write (*, *) ' '
       write (*, *) 'Z-pencil'
       write (*, 10) int(u3)
@@ -117,7 +117,7 @@ program test2d
    ! call decomp_2d_write_one(3,u3,'u3.dat')
    ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
-   !$acc parallel loop default(present) 
+   !$acc parallel loop default(present)
    do k = zstart(3), zend(3)
       do j = zstart(2), zend(2)
          do i = zstart(1), zend(1)
@@ -135,7 +135,7 @@ program test2d
    call transpose_z_to_y(u3, u2)
    ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
-   !$acc parallel loop default(present) 
+   !$acc parallel loop default(present)
    do k = ystart(3), yend(3)
       do j = ystart(2), yend(2)
          do i = ystart(1), yend(1)
@@ -153,7 +153,7 @@ program test2d
    call transpose_y_to_x(u2, u1)
    ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
-   !$acc parallel loop default(present) 
+   !$acc parallel loop default(present)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index b9465e82..bbeabde5 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -368,7 +368,7 @@ subroutine init_fft_engine
 
       end if
 #endif
-      cufft_ws = cufft_ws / sizeof(1._mytype)
+      cufft_ws = cufft_ws/sizeof(1._mytype)
       allocate (cufft_workspace(cufft_ws))
       do j = 1, 3
          do i = -1, 2
@@ -413,11 +413,11 @@ subroutine c2c_1m_x(inout, isign, plan1)
 
 #ifdef DOUBLE_PREC
       !$acc host_data use_device(inout)
-      istat = cufftExecZ2Z(plan1, inout, inout,isign)
+      istat = cufftExecZ2Z(plan1, inout, inout, isign)
       !$acc end host_data
 #else
       !$acc host_data use_device(inout)
-      istat = cufftExecC2C(plan1, inout, inout,isign)
+      istat = cufftExecC2C(plan1, inout, inout, isign)
       !$acc end host_data
 #endif
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
@@ -439,13 +439,13 @@ subroutine c2c_1m_y(inout, isign, plan1)
       s3 = size(inout, 3)
       do k = 1, s3
 #ifdef DOUBLE_PREC
-        !$acc host_data use_device(inout)
-        istat = cufftExecZ2Z(plan1, inout(:,:,k), inout(:,:,k),isign)
-        !$acc end host_data
+         !$acc host_data use_device(inout)
+         istat = cufftExecZ2Z(plan1, inout(:, :, k), inout(:, :, k), isign)
+         !$acc end host_data
 #else
-        !$acc host_data use_device(inout)
-        istat = cufftExecC2C(plan1, inout(:,:,k), inout(:,:,k),isign)
-        !$acc end host_data
+         !$acc host_data use_device(inout)
+         istat = cufftExecC2C(plan1, inout(:, :, k), inout(:, :, k), isign)
+         !$acc end host_data
 #endif
          if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
       end do
@@ -463,14 +463,13 @@ subroutine c2c_1m_z(inout, isign, plan1)
 
       integer :: istat
 
-
 #ifdef DOUBLE_PREC
       !$acc host_data use_device(inout)
-      istat = cufftExecZ2Z(plan1, inout, inout,isign)
+      istat = cufftExecZ2Z(plan1, inout, inout, isign)
       !$acc end host_data
 #else
       !$acc host_data use_device(inout)
-      istat = cufftExecC2C(plan1, inout, inout,isign)
+      istat = cufftExecC2C(plan1, inout, inout, isign)
       !$acc end host_data
 #endif
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftExecC2C/Z2Z")
@@ -677,7 +676,7 @@ subroutine fft_3d_c2c(in, out, isign)
 
 #ifndef OVERWRITE
       !$acc exit data delete(wk1) async
-      !$acc wait 
+      !$acc wait
       deallocate (wk1)
 #endif
       !$acc end data
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index 02c58f55..d264c7be 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -307,7 +307,7 @@ subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -363,7 +363,7 @@ subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D( out(pos), i2-i1+1, in(i1,1,1), n1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -419,7 +419,7 @@ subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -475,8 +475,8 @@ subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-        !$acc end host_data 
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index 33f0bc21..5908b10e 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -307,7 +307,7 @@ subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -363,7 +363,7 @@ subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -419,7 +419,7 @@ subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -475,7 +475,7 @@ subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D( out(i1,1,1), n1, in(pos), i2-i1+1, i2-i1+1, n2*n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 454b2f66..30a760de 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -154,7 +154,7 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
 #if defined(_GPU)
      !If one of the array in cuda call is not device we need to add acc host_data
      !$acc host_data use_device(dst)
-     istat = cudaMemcpy( dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
+     istat = cudaMemcpy(dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -288,10 +288,10 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
      ! so no merge operation needed
 
 #if defined(_GPU)
-    !$acc host_data use_device(dst)
-    istat = cudaMemcpy( dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice )
-    !$acc end host_data
-    if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+     !$acc host_data use_device(dst)
+     istat = cudaMemcpy(dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
+     !$acc end host_data
+     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
@@ -342,7 +342,7 @@ subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -397,10 +397,10 @@ subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 #endif
 
 #if defined(_GPU)
-       !$acc host_data use_device(in)
-       istat = cudaMemcpy2D( out(pos), n1*(i2-i1+1), in(1,i1,1), n1*n2, n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
-       !$acc end host_data
-       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
         do k = 1, n3
            do j = i1, i2
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index f6f29a48..5950e0d1 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -74,7 +74,7 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
 
 #if defined(_GPU)
      !$acc host_data use_device(src)
-     istat = cudaMemcpy( work1_r_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice )
+     istat = cudaMemcpy(work1_r_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -224,7 +224,7 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
 
 #if defined(_GPU)
      !$acc host_data use_device(src)
-     istat = cudaMemcpy( work1_c_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice )
+     istat = cudaMemcpy(work1_c_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -431,7 +431,7 @@ subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -488,7 +488,7 @@ subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D( out(1,i1,1), n1*n2, in(pos), n1*(i2-i1+1), n1*(i2-i1+1), n3, cudaMemcpyDeviceToDevice )
+        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else

From f2dc2cda0b6b1a6e4f9e4f7b250c01eae4b98884 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 17 Jan 2023 10:17:24 +0000
Subject: [PATCH 091/436] Update IO write plane example

---
 examples/io_test/io_plane_test.f90 | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index dfe8620b..97d2da98 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -64,30 +64,16 @@ program io_plane_test
    !$acc update self(u3)  
    !$acc end data
  
-#if defined(_GPU)
-   write (*, *) 'WARNING for GPU IO: write of a plane is supported only in the aligned pencil'
-   write (*, *) '                    i.e. x_plane-x_pencil'
-   write (*, *) '                    i.e. y_plane-y_pencil'
-   write (*, *) '                    i.e. z_plane-z_pencil'
-#endif
-
    call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
-#if !defined(_GPU)
    call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
    call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
-#endif
    ! Y-pencil data
    call decomp_2d_write_plane(2, u2, 2, ny/2, '.', 'y_pencil-y_plane.dat', 'test')
-#if !defined(_GPU)
    call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(2, u2, 3, nz/2, '.', 'y_pencil-z_plane.dat', 'test')
-#endif
-
    ! Z-pencil data
-#if !defined(_GPU)
    call decomp_2d_write_plane(3, u3, 1, nx/2, '.', 'z_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(3, u3, 2, ny/2, '.', 'z_pencil-y_plane.dat', 'test')
-#endif
    call decomp_2d_write_plane(3, u3, 3, nz/2, '.', 'z_pencil-z_plane.dat', 'test')
    ! Attemp to read the files
    if (nrank == 0) then

From dd655de6c55e19d530fc3aa3abc6bfd9249cb121 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 17 Jan 2023 14:21:34 +0000
Subject: [PATCH 092/436] Bug correction for test2d with definition of local
 loop limits

---
 examples/test2d/test2d.f90 | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 37acfae5..5e5e64fb 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -40,6 +40,18 @@ program test2d
       end do
    end do
 
+   ! Fill the local index
+   xst1 = xstart(1); xen1 = xend(1)
+   xst2 = xstart(2); xen2 = xend(2)
+   xst3 = xstart(3); xen3 = xend(3)
+   yst1 = ystart(1); yen1 = yend(1)
+   yst2 = ystart(2); yen2 = yend(2)
+   yst3 = ystart(3); yen3 = yend(3)
+   zst1 = zstart(1); zen1 = zend(1)
+   zst2 = zstart(2); zen2 = zend(2)
+   zst3 = zstart(3); zen3 = zend(3)
+   
+
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Testing the swap routines
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -151,7 +163,7 @@ program test2d
    if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
    if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
 
-  !!!!!!!!!!!!!!!!!!!!!!!
+   !!!!!!!!!!!!!!!!!!!!!!!
    ! y-pensil ==> x-pensil
    call transpose_y_to_x(u2, u1)
    ! call decomp_2d_write_one(1,u1,'u1b.dat')

From 606e01b22ba53335dc35e01acf984ce483239d25 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 20 Jan 2023 13:45:25 +0000
Subject: [PATCH 093/436] Remove SHM from log.f90

---
 src/log.f90 | 95 -----------------------------------------------------
 1 file changed, 95 deletions(-)

diff --git a/src/log.f90 b/src/log.f90
index 9f6933ed..e0799754 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -177,9 +177,6 @@ module subroutine d2d_listing(given_io_unit)
 #ifdef PROFILER
       write (io_unit, *) 'Compile flag PROFILER detected'
 #endif
-#ifdef SHM
-      write (io_unit, *) 'Compile flag SHM detected'
-#endif
 #ifdef EVEN
       write (io_unit, *) 'Compile flag EVEN detected'
 #endif
@@ -192,9 +189,6 @@ module subroutine d2d_listing(given_io_unit)
 #ifdef HALO_DEBUG
       write (io_unit, *) 'Compile flag HALO_DEBUG detected'
 #endif
-#ifdef SHM_DEBUG
-      write (io_unit, *) 'Compile flag SHM_DEBUG detected'
-#endif
 #ifdef _GPU
       write (io_unit, *) 'Compile flag _GPU detected'
 #endif
@@ -218,10 +212,6 @@ module subroutine d2d_listing(given_io_unit)
       call decomp_info_print(ph2, io_unit, "ph2")
       call decomp_info_print(ph3, io_unit, "ph3")
       call decomp_info_print(ph4, io_unit, "ph4")
-#ifdef SHM_DEBUG
-      write (io_unit, *) '==========================================================='
-      call print_smp(io_unit)
-#endif
       write (io_unit, *) '==========================================================='
       write (io_unit, *) '==========================================================='
 
@@ -302,93 +292,8 @@ module subroutine decomp_info_print(d2d, io_unit, d2dname)
       write (io_unit, *) '   y2count : ', d2d%y2count
       write (io_unit, *) '   z2count : ', d2d%z2count
       write (io_unit, *) '   even : ', d2d%even
-#ifdef SHM
-      write (io_unit, *) '   listing of the SHM part is not yet implemented'
-#endif
 #endif
 
    end subroutine decomp_info_print
 
-#ifdef SHM_DEBUG
-
-   ! For debugging, print the shared-memory structure
-   module subroutine print_smp(io_unit)
-
-      implicit none
-
-      ! Argument
-      integer, intent(in) :: io_unit
-
-      ! print out shared-memory information
-      write (io_unit, *) 'I am mpi rank ', nrank, 'Total ranks ', nproc
-      write (io_unit, *) ' '
-      write (io_unit, *) 'Global data size:'
-      write (io_unit, *) 'nx*ny*nz', nx, ny, nz
-      write (io_unit, *) ' '
-      write (io_unit, *) '2D processor grid:'
-      write (io_unit, *) 'p_row*p_col:', dims(1), dims(2)
-      write (io_unit, *) ' '
-      write (io_unit, *) 'Portion of global data held locally:'
-      write (io_unit, *) 'xsize:', xsize
-      write (io_unit, *) 'ysize:', ysize
-      write (io_unit, *) 'zsize:', zsize
-      write (io_unit, *) ' '
-      write (io_unit, *) 'How pensils are to be divided and sent in alltoallv:'
-      write (io_unit, *) 'x1dist:', decomp_main%x1dist
-      write (io_unit, *) 'y1dist:', decomp_main%y1dist
-      write (io_unit, *) 'y2dist:', decomp_main%y2dist
-      write (io_unit, *) 'z2dist:', decomp_main%z2dist
-      write (io_unit, *) ' '
-      write (io_unit, *) '######Shared buffer set up after this point######'
-      write (io_unit, *) ' '
-      write (io_unit, *) 'col communicator detais:'
-      call print_smp_info(decomp_main%COL_INFO, io_unit)
-      write (io_unit, *) ' '
-      write (io_unit, *) 'row communicator detais:'
-      call print_smp_info(decomp_main%ROW_INFO; io_unit)
-      write (io_unit, *) ' '
-      write (io_unit, *) 'Buffer count and displacement of per-core buffers'
-      write (io_unit, *) 'x1cnts:', decomp_main%x1cnts
-      write (io_unit, *) 'y1cnts:', decomp_main%y1cnts
-      write (io_unit, *) 'y2cnts:', decomp_main%y2cnts
-      write (io_unit, *) 'z2cnts:', decomp_main%z2cnts
-      write (io_unit, *) 'x1disp:', decomp_main%x1disp
-      write (io_unit, *) 'y1disp:', decomp_main%y1disp
-      write (io_unit, *) 'y2disp:', decomp_main%y2disp
-      write (io_unit, *) 'z2disp:', decomp_main%z2disp
-      write (io_unit, *) ' '
-      write (io_unit, *) 'Buffer count and displacement of shared buffers'
-      write (io_unit, *) 'x1cnts:', decomp_main%x1cnts_s
-      write (io_unit, *) 'y1cnts:', decomp_main%y1cnts_s
-      write (io_unit, *) 'y2cnts:', decomp_main%y2cnts_s
-      write (io_unit, *) 'z2cnts:', decomp_main%z2cnts_s
-      write (io_unit, *) 'x1disp:', decomp_main%x1disp_s
-      write (io_unit, *) 'y1disp:', decomp_main%y1disp_s
-      write (io_unit, *) 'y2disp:', decomp_main%y2disp_s
-      write (io_unit, *) 'z2disp:', decomp_main%z2disp_s
-
-   end subroutine print_smp
-
-   ! For debugging, print the shared-memory structure
-   module subroutine print_smp_info(s, io_unit)
-
-      implicit none
-
-      ! Argument
-      TYPE(SMP_INFO), intent(in) :: s
-      integer, intent(in) :: io_unit
-
-      write (io_unit, *) 'size of current communicator:', s%NCPU
-      write (io_unit, *) 'rank in current communicator:', s%NODE_ME
-      write (io_unit, *) 'number of SMP-nodes in this communicator:', s%NSMP
-      write (io_unit, *) 'SMP-node id (1 ~ NSMP):', s%SMP_ME
-      write (io_unit, *) 'NCORE - number of cores on this SMP-node', s%NCORE
-      write (io_unit, *) 'core id (1 ~ NCORE):', s%CORE_ME
-      write (io_unit, *) 'maximum no. cores on any SMP-node:', s%MAXCORE
-      write (io_unit, *) 'size of SMP shared memory SND buffer:', s%N_SND
-      write (io_unit, *) 'size of SMP shared memory RCV buffer:', s%N_RCV
-
-   end subroutine print_smp_info
-#endif
-
 end submodule d2d_log

From 917af6d7313524c985eb40f085322cd78e5d105b Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 20 Jan 2023 13:55:01 +0000
Subject: [PATCH 094/436] Remove SHM from transposes

---
 src/transpose_x_to_y.f90 | 90 ----------------------------------------
 src/transpose_y_to_x.f90 | 90 ----------------------------------------
 src/transpose_y_to_z.f90 | 86 --------------------------------------
 src/transpose_z_to_y.f90 | 86 --------------------------------------
 4 files changed, 352 deletions(-)

diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index bd783d7e..094f0698 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -35,11 +35,6 @@ subroutine transpose_x_to_y_real(src, dst, decomp)
      type(ncclResult) :: nccl_stat
 #endif
 
-#ifdef SHM
-     real(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -55,38 +50,16 @@ subroutine transpose_x_to_y_real(src, dst, decomp)
      d3 = SIZE(dst, 3)
 
      ! rearrange source array as send buffer
-#ifdef SHM
-     work1_p = decomp%COL_INFO%SND_P
-     call mem_split_xy_real(src, s1, s2, s3, work1, dims(1), &
-                            decomp%x1dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
                             decomp%x1dist, decomp)
 #else
      call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
                             decomp%x1dist, decomp)
-#endif
-
 #endif
 
      ! define receive buffer
-#ifdef SHM
-     work2_p = decomp%COL_INFO%RCV_P
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-#endif
-
      ! transpose using MPI_ALLTOALL(V)
-#ifdef SHM
-     if (decomp%COL_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
-                           real_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
-                           real_type, decomp%COL_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
-#else
 #ifdef EVEN
      call MPI_ALLTOALL(work1_r, decomp%x1count, &
                        real_type, work2_r, decomp%y1count, &
@@ -123,17 +96,9 @@ subroutine transpose_x_to_y_real(src, dst, decomp)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
-#endif
 #endif
 
      ! rearrange receive buffer
-#ifdef SHM
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_xy_real(work2, d1, d2, d3, dst, dims(1), &
-                            decomp%y1dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
                             decomp%y1dist, decomp)
@@ -142,8 +107,6 @@ subroutine transpose_x_to_y_real(src, dst, decomp)
                             decomp%y1dist, decomp)
 #endif
 
-#endif
-
 #ifdef PROFILER
      if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
 #endif
@@ -170,11 +133,6 @@ subroutine transpose_x_to_y_complex(src, dst, decomp)
      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-#ifdef SHM
-     complex(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -190,38 +148,16 @@ subroutine transpose_x_to_y_complex(src, dst, decomp)
      d3 = SIZE(dst, 3)
 
      ! rearrange source array as send buffer
-#ifdef SHM
-     work1_p = decomp%COL_INFO%SND_P_c
-     call mem_split_xy_complex(src, s1, s2, s3, work1, dims(1), &
-                               decomp%x1dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
                                decomp%x1dist, decomp)
 #else
      call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
                                decomp%x1dist, decomp)
-#endif
-
 #endif
 
      ! define receive buffer
-#ifdef SHM
-     work2_p = decomp%COL_INFO%RCV_P_c
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-#endif
-
      ! transpose using MPI_ALLTOALL(V)
-#ifdef SHM
-     if (decomp%COL_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%x1cnts_s, decomp%x1disp_s, &
-                           complex_type, work2, decomp%y1cnts_s, decomp%y1disp_s, &
-                           complex_type, decomp%COL_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
-#else
 #ifdef EVEN
      call MPI_ALLTOALL(work1_c, decomp%x1count, &
                        complex_type, work2_c, decomp%y1count, &
@@ -241,17 +177,9 @@ subroutine transpose_x_to_y_complex(src, dst, decomp)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
-#endif
 #endif
 
      ! rearrange receive buffer
-#ifdef SHM
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_xy_complex(work2, d1, d2, d3, dst, dims(1), &
-                               decomp%y1dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
                                decomp%y1dist, decomp)
@@ -260,8 +188,6 @@ subroutine transpose_x_to_y_complex(src, dst, decomp)
                                decomp%y1dist, decomp)
 #endif
 
-#endif
-
 #ifdef PROFILER
      if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
 #endif
@@ -295,15 +221,11 @@ subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%x1disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
@@ -349,15 +271,11 @@ subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%x1disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
@@ -403,15 +321,11 @@ subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%y1disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
@@ -457,15 +371,11 @@ subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%y1disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index a742ead6..45e1ba20 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -35,11 +35,6 @@ subroutine transpose_y_to_x_real(src, dst, decomp)
      integer :: col_rank_id, cuda_stat
 #endif
 
-#ifdef SHM
-     real(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -55,38 +50,16 @@ subroutine transpose_y_to_x_real(src, dst, decomp)
      d3 = SIZE(dst, 3)
 
      ! rearrange source array as send buffer
-#ifdef SHM
-     work1_p = decomp%COL_INFO%SND_P
-     call mem_split_yx_real(src, s1, s2, s3, work1, dims(1), &
-                            decomp%y1dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
                             decomp%y1dist, decomp)
 #else
      call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
                             decomp%y1dist, decomp)
-#endif
-
 #endif
 
      ! define receive buffer
-#ifdef SHM
-     work2_p = decomp%COL_INFO%RCV_P
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-#endif
-
      ! transpose using MPI_ALLTOALL(V)
-#ifdef SHM
-     if (decomp%COL_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
-                           real_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
-                           real_type, decomp%COL_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
-#else
 #ifdef EVEN
      call MPI_ALLTOALL(work1_r, decomp%y1count, &
                        real_type, work2_r, decomp%x1count, &
@@ -123,17 +96,9 @@ subroutine transpose_y_to_x_real(src, dst, decomp)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
-#endif
 #endif
 
      ! rearrange receive buffer
-#ifdef SHM
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_yx_real(work2, d1, d2, d3, dst, dims(1), &
-                            decomp%x1dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
                             decomp%x1dist, decomp)
@@ -142,8 +107,6 @@ subroutine transpose_y_to_x_real(src, dst, decomp)
                             decomp%x1dist, decomp)
 #endif
 
-#endif
-
 #ifdef PROFILER
      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
 #endif
@@ -170,11 +133,6 @@ subroutine transpose_y_to_x_complex(src, dst, decomp)
      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-#ifdef SHM
-     complex(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -190,38 +148,16 @@ subroutine transpose_y_to_x_complex(src, dst, decomp)
      d3 = SIZE(dst, 3)
 
      ! rearrange source array as send buffer
-#ifdef SHM
-     work1_p = decomp%COL_INFO%SND_P_c
-     call mem_split_yx_complex(src, s1, s2, s3, work1, dims(1), &
-                               decomp%y1dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
                                decomp%y1dist, decomp)
 #else
      call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
                                decomp%y1dist, decomp)
-#endif
-
 #endif
 
      ! define receive buffer
-#ifdef SHM
-     work2_p = decomp%COL_INFO%RCV_P_c
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-#endif
-
      ! transpose using MPI_ALLTOALL(V)
-#ifdef SHM
-     if (decomp%COL_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%y1cnts_s, decomp%y1disp_s, &
-                           complex_type, work2, decomp%x1cnts_s, decomp%x1disp_s, &
-                           complex_type, decomp%COL_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
-#else
 #ifdef EVEN
      call MPI_ALLTOALL(work1_c, decomp%y1count, &
                        complex_type, work2_c, decomp%x1count, &
@@ -241,17 +177,9 @@ subroutine transpose_y_to_x_complex(src, dst, decomp)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
-#endif
 #endif
 
      ! rearrange receive buffer
-#ifdef SHM
-     call MPI_BARRIER(decomp%COL_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_yx_complex(work2, d1, d2, d3, dst, dims(1), &
-                               decomp%x1dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
                                decomp%x1dist, decomp)
@@ -260,8 +188,6 @@ subroutine transpose_y_to_x_complex(src, dst, decomp)
                                decomp%x1dist, decomp)
 #endif
 
-#endif
-
 #ifdef PROFILER
      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
 #endif
@@ -295,15 +221,11 @@ subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%y1disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
@@ -349,15 +271,11 @@ subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%y1disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
@@ -403,15 +321,11 @@ subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%x1disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
@@ -457,15 +371,11 @@ subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%x1disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 62bc9071..391eac0a 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -38,11 +38,6 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
      integer :: istat
 #endif
 
-#ifdef SHM
-     real(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -58,37 +53,15 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
      d3 = SIZE(dst, 3)
 
      ! rearrange source array as send buffer
-#ifdef SHM
-     work1_p = decomp%ROW_INFO%SND_P
-     call mem_split_yz_real(src, s1, s2, s3, work1, dims(2), &
-                            decomp%y2dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
                             decomp%y2dist, decomp)
 #else
      call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
                             decomp%y2dist, decomp)
-#endif
-
 #endif
 
      ! define receive buffer
-#ifdef SHM
-     work2_p = decomp%ROW_INFO%RCV_P
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-#endif
-
-#ifdef SHM
-     if (decomp%ROW_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
-                           real_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
-                           real_type, decomp%ROW_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
-#else
 #ifdef EVEN
      if (decomp%even) then
         call MPI_ALLTOALL(work1_r, decomp%y2count, &
@@ -132,16 +105,9 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
-#endif
 #endif
 
      ! rearrange receive buffer
-#ifdef SHM
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_yz_real(work2, d1, d2, d3, dst, dims(2), &
-                            decomp%z2dist, decomp)
-#else
 #ifdef EVEN
      if (.not. decomp%even) then
         call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
@@ -155,7 +121,6 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
      istat = cudaMemcpy(dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
 #endif
 
-#endif
 #endif
 
 #ifdef PROFILER
@@ -188,11 +153,6 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
      integer :: istat
 #endif
 
-#ifdef SHM
-     complex(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -208,37 +168,15 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
      d3 = SIZE(dst, 3)
 
      ! rearrange source array as send buffer
-#ifdef SHM
-     work1_p = decomp%ROW_INFO%SND_P_c
-     call mem_split_yz_complex(src, s1, s2, s3, work1, dims(2), &
-                               decomp%y2dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
                                decomp%y2dist, decomp)
 #else
      call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
                                decomp%y2dist, decomp)
-#endif
-
 #endif
 
      ! define receive buffer
-#ifdef SHM
-     work2_p = decomp%ROW_INFO%RCV_P_c
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-#endif
-
-#ifdef SHM
-     if (decomp%ROW_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%y2cnts_s, decomp%y2disp_s, &
-                           complex_type, work2, decomp%z2cnts_s, decomp%z2disp_s, &
-                           complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
-#else
 #ifdef EVEN
      if (decomp%even) then
         call MPI_ALLTOALL(work1_c, decomp%y2count, &
@@ -264,16 +202,9 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
-#endif
 #endif
 
      ! rearrange receive buffer
-#ifdef SHM
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_yz_complex(work2, d1, d2, d3, dst, dims(2), &
-                               decomp%z2dist, decomp)
-#else
 #ifdef EVEN
      if (.not. decomp%even) then
         call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
@@ -287,7 +218,6 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
      istat = cudaMemcpy(dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
 #endif
 
-#endif
 #endif
 
 #ifdef PROFILER
@@ -323,15 +253,11 @@ subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%y2disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
@@ -377,15 +303,11 @@ subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%y2disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
@@ -427,14 +349,10 @@ subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%z2disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
-#endif
 #endif
 
         do k = i1, i2
@@ -472,14 +390,10 @@ subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%z2disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
-#endif
 #endif
 
         do k = i1, i2
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index a2f86844..c7ca6e11 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -38,11 +38,6 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
      integer :: istat
 #endif
 
-#ifdef SHM
-     real(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -58,11 +53,6 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
      d3 = SIZE(dst, 3)
 
      ! rearrange source array as send buffer
-#ifdef SHM
-     work1_p = decomp%ROW_INFO%SND_P
-     call mem_split_zy_real(src, s1, s2, s3, work1, dims(2), &
-                            decomp%z2dist, decomp)
-#else
 #ifdef EVEN
      if (.not. decomp%even) then
         call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
@@ -76,24 +66,9 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
      istat = cudaMemcpy(work1_r_d, src, s1*s2*s3)
 #endif
 
-#endif
 #endif
 
      ! define receive buffer
-#ifdef SHM
-     work2_p = decomp%ROW_INFO%RCV_P
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-#endif
-
-#ifdef SHM
-     if (decomp%ROW_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
-                           real_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
-                           real_type, decomp%ROW_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
-#else
 #ifdef EVEN
      if (decomp%even) then
         call MPI_ALLTOALL(src, decomp%z2count, &
@@ -138,13 +113,6 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
 #endif
 
      ! rearrange receive buffer
-#ifdef SHM
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_zy_real(work2, d1, d2, d3, dst, dims(2), &
-                            decomp%y2dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
                             decomp%y2dist, decomp)
@@ -153,8 +121,6 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
                             decomp%y2dist, decomp)
 #endif
 
-#endif
-
 #ifdef PROFILER
      if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
 #endif
@@ -185,11 +151,6 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
      integer :: istat
 #endif
 
-#ifdef SHM
-     complex(mytype) :: work1(*), work2(*)
-     POINTER(work1_p, work1), (work2_p, work2)  ! Cray pointers
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -205,11 +166,6 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
      d3 = SIZE(dst, 3)
 
      ! rearrange source array as send buffer
-#ifdef SHM
-     work1_p = decomp%ROW_INFO%SND_P_c
-     call mem_split_zy_complex(src, s1, s2, s3, work1, dims(2), &
-                               decomp%z2dist, decomp)
-#else
 #ifdef EVEN
      if (.not. decomp%even) then
         call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
@@ -223,24 +179,9 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
      istat = cudaMemcpy(work1_c_d, src, s1*s2*s3)
 #endif
 
-#endif
 #endif
 
      ! define receive buffer
-#ifdef SHM
-     work2_p = decomp%ROW_INFO%RCV_P_c
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-#endif
-
-#ifdef SHM
-     if (decomp%ROW_INFO%CORE_ME == 1) THEN
-        call MPI_ALLTOALLV(work1, decomp%z2cnts_s, decomp%z2disp_s, &
-                           complex_type, work2, decomp%y2cnts_s, decomp%y2disp_s, &
-                           complex_type, decomp%ROW_INFO%SMP_COMM, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     end if
-#else
 #ifdef EVEN
      if (decomp%even) then
         call MPI_ALLTOALL(src, decomp%z2count, &
@@ -265,19 +206,10 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
                         complex_type, DECOMP_2D_COMM_ROW, ierror)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
-#endif
 
-#endif
 #endif
 
      ! rearrange receive buffer
-#ifdef SHM
-     call MPI_BARRIER(decomp%ROW_INFO%CORE_COMM, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")
-     call mem_merge_zy_complex(work2, d1, d2, d3, dst, dims(2), &
-                               decomp%y2dist, decomp)
-#else
-
 #if defined(_GPU)
      call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
                                decomp%y2dist, decomp)
@@ -286,8 +218,6 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
                                decomp%y2dist, decomp)
 #endif
 
-#endif
-
 #ifdef PROFILER
      if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
 #endif
@@ -319,14 +249,10 @@ subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%z2disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
-#endif
 #endif
 
         do k = i1, i2
@@ -364,14 +290,10 @@ subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%z2disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
-#endif
 #endif
 
         do k = i1, i2
@@ -413,15 +335,11 @@ subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%y2disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
@@ -468,15 +386,11 @@ subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
            i2 = i1 + dist(m) - 1
         end if
 
-#ifdef SHM
-        pos = decomp%y2disp_o(m) + 1
-#else
 #ifdef EVEN
         pos = m*decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
-#endif
 
 #if defined(_GPU)
         istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)

From b3fafc3dd27df1b187afbd4a09facb01dadb605f Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 20 Jan 2023 14:07:01 +0000
Subject: [PATCH 095/436] Remove SHM from main decomp_2d.f90

---
 src/decomp_2d.f90 | 371 ----------------------------------------------
 1 file changed, 371 deletions(-)

diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 79fec912..8b429cb3 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -110,28 +110,6 @@ module decomp_2d
 #endif
 #endif
 
-#ifdef SHM
-   ! derived type to store shared-memory info
-   TYPE, public :: SMP_INFO
-      integer MPI_COMM          ! SMP associated with this communicator
-      integer NODE_ME           ! rank in this communicator
-      integer NCPU              ! size of this communicator
-      integer SMP_COMM          ! communicator for SMP-node masters
-      integer CORE_COMM         ! communicator for cores on SMP-node
-      integer SMP_ME            ! SMP-node id starting from 1 ... NSMP
-      integer NSMP              ! number of SMP-nodes in this communicator
-      integer CORE_ME           ! core id starting from 1 ... NCORE
-      integer NCORE             ! number of cores on this SMP-node
-      integer MAXCORE           ! maximum no. cores on any SMP-node
-      integer N_SND             ! size of SMP shared memory buffer
-      integer N_RCV             ! size of SMP shared memory buffer
-      integer(8) SND_P          ! SNDBUF address (cray pointer), for real
-      integer(8) RCV_P          ! RCVBUF address (cray pointer), for real
-      integer(8) SND_P_c        ! for complex
-      integer(8) RCV_P_c        ! for complex
-   END TYPE SMP_INFO
-#endif
-
    ! derived type to store decomposition info for a given global data size
    TYPE, public :: DECOMP_INFO
       ! staring/ending index and size of data held by current processor
@@ -159,22 +137,6 @@ module decomp_2d
       ! evenly distributed data
       logical :: even
 
-#ifdef SHM
-      ! For shared-memory implementation
-
-      ! one instance of this derived type for each communicator
-      ! shared moemory info, such as which MPI rank belongs to which node
-      TYPE(SMP_INFO) :: ROW_INFO, COL_INFO
-
-      ! shared send/recv buffers for ALLTOALLV
-      integer, allocatable, dimension(:) :: x1cnts_s, y1cnts_s, &
-                                            y2cnts_s, z2cnts_s
-      integer, allocatable, dimension(:) :: x1disp_s, y1disp_s, &
-                                            y2disp_s, z2disp_s
-      ! A copy of original buffer displacement (will be overwriten)
-      integer, allocatable, dimension(:) :: x1disp_o, y1disp_o, &
-                                            y2disp_o, z2disp_o
-#endif
    END TYPE DECOMP_INFO
 
    ! main (default) decomposition information for global size nx*ny*nz
@@ -514,11 +476,6 @@ subroutine decomp_info_init(nx, ny, nz, decomp)
                 decomp%y2disp(0:dims(2) - 1), decomp%z2disp(0:dims(2) - 1))
       call prepare_buffer(decomp)
 
-#ifdef SHM
-      ! prepare shared-memory information if required
-      call decomp_info_init_shm(decomp)
-#endif
-
       ! allocate memory for the MPI_ALLTOALL(V) buffers
       ! define the buffers globally for performance reason
 
@@ -620,21 +577,6 @@ subroutine decomp_info_finalize(decomp)
       if (allocated(decomp%y2disp)) deallocate (decomp%y2disp)
       if (allocated(decomp%z2disp)) deallocate (decomp%z2disp)
 
-#ifdef SHM
-      if (allocated(decomp%x1disp_o)) deallocate (decomp%x1disp_o)
-      if (allocated(decomp%y1disp_o)) deallocate (decomp%y1disp_o)
-      if (allocated(decomp%y2disp_o)) deallocate (decomp%y2disp_o)
-      if (allocated(decomp%z2disp_o)) deallocate (decomp%z2disp_o)
-      if (allocated(decomp%x1cnts_s)) deallocate (decomp%x1cnts_s)
-      if (allocated(decomp%y1cnts_s)) deallocate (decomp%y1cnts_s)
-      if (allocated(decomp%y2cnts_s)) deallocate (decomp%y2cnts_s)
-      if (allocated(decomp%z2cnts_s)) deallocate (decomp%z2cnts_s)
-      if (allocated(decomp%x1disp_s)) deallocate (decomp%x1disp_s)
-      if (allocated(decomp%y1disp_s)) deallocate (decomp%y1disp_s)
-      if (allocated(decomp%y2disp_s)) deallocate (decomp%y2disp_s)
-      if (allocated(decomp%z2disp_s)) deallocate (decomp%z2disp_s)
-#endif
-
       return
    end subroutine decomp_info_finalize
 
@@ -1289,319 +1231,6 @@ subroutine prepare_buffer(decomp)
       return
    end subroutine prepare_buffer
 
-#ifdef SHM
-
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   !  Generate shared-memory information
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine decomp_info_init_shm(decomp)
-
-      implicit none
-
-      TYPE(DECOMP_INFO), intent(INOUT) :: decomp
-
-      ! a copy of old displacement array (will be overwritten by shm code)
-      allocate (decomp%x1disp_o(0:dims(1) - 1), decomp%y1disp_o(0:dims(1) - 1), &
-                decomp%y2disp_o(0:dims(2) - 1), decomp%z2disp_o(0:dims(2) - 1))
-      decomp%x1disp_o = decomp%x1disp
-      decomp%y1disp_o = decomp%y1disp
-      decomp%y2disp_o = decomp%y2disp
-      decomp%z2disp_o = decomp%z2disp
-
-      call prepare_shared_buffer(decomp%ROW_INFO, DECOMP_2D_COMM_ROW, decomp)
-      call prepare_shared_buffer(decomp%COL_INFO, DECOMP_2D_COMM_COL, decomp)
-
-      return
-   end subroutine decomp_info_init_shm
-
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! For shared-memory implementation, prepare send/recv shared buffer
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine prepare_shared_buffer(C, MPI_COMM, decomp)
-
-      implicit none
-
-      TYPE(SMP_INFO) :: C
-      INTEGER :: MPI_COMM
-      TYPE(DECOMP_INFO) :: decomp
-
-      INTEGER, ALLOCATABLE :: KTBL(:, :), NARY(:, :), KTBLALL(:, :)
-      INTEGER MYSMP, MYCORE, COLOR
-
-      integer :: ierror
-
-      C%MPI_COMM = MPI_COMM
-      CALL MPI_COMM_SIZE(MPI_COMM, C%NCPU, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-      CALL MPI_COMM_RANK(MPI_COMM, C%NODE_ME, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_CART_RANK")
-      C%SMP_COMM = MPI_COMM_NULL
-      C%CORE_COMM = MPI_COMM_NULL
-      C%SMP_ME = 0
-      C%NCORE = 0
-      C%CORE_ME = 0
-      C%MAXCORE = 0
-      C%NSMP = 0
-      C%N_SND = 0
-      C%N_RCV = 0
-      C%SND_P = 0
-      C%RCV_P = 0
-      C%SND_P_c = 0
-      C%RCV_P_c = 0
-
-      ! get smp-node map for this communicator and set up smp communicators
-      CALL GET_SMP_MAP(C%MPI_COMM, C%NSMP, MYSMP, &
-                       C%NCORE, MYCORE, C%MAXCORE)
-      C%SMP_ME = MYSMP + 1
-      C%CORE_ME = MYCORE + 1
-      ! - set up inter/intra smp-node communicators
-      COLOR = MYCORE
-      IF (COLOR > 0) COLOR = MPI_UNDEFINED
-      CALL MPI_Comm_split(C%MPI_COMM, COLOR, MYSMP, C%SMP_COMM, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-      CALL MPI_Comm_split(C%MPI_COMM, MYSMP, MYCORE, C%CORE_COMM, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SPLIT")
-      ! - allocate work space
-      ALLOCATE (KTBL(C%MAXCORE, C%NSMP), NARY(C%NCPU, C%NCORE))
-      ALLOCATE (KTBLALL(C%MAXCORE, C%NSMP))
-      ! - set up smp-node/core to node_me lookup table
-      KTBL = 0
-      KTBL(C%CORE_ME, C%SMP_ME) = C%NODE_ME + 1
-      CALL MPI_ALLREDUCE(KTBL, KTBLALL, C%NSMP*C%MAXCORE, MPI_INTEGER, &
-                         MPI_SUM, MPI_COMM, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-      KTBL = KTBLALL
-      !  IF (SUM(KTBL) /= C%NCPU*(C%NCPU+1)/2) &
-      !       CALL MPI_ABORT(...
-
-      ! compute offsets in shared SNDBUF and RCVBUF
-      CALL MAPSET_SMPSHM(C, KTBL, NARY, decomp)
-
-      DEALLOCATE (KTBL, NARY)
-
-      return
-   end subroutine prepare_shared_buffer
-
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Use Ian Bush's FreeIPC to generate shared-memory information
-   !  - system independent solution
-   !  - replacing David Tanqueray's implementation in alloc_shm.c
-   !    (old C code renamed to get_smp_map2)
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine get_smp_map(comm, nnodes, my_node, ncores, my_core, maxcor)
-
-      use FIPC_module
-
-      implicit none
-
-      integer, intent(IN) :: comm
-      integer, intent(OUT) :: nnodes, my_node, ncores, my_core, maxcor
-
-      integer :: intra_comm, extra_comm
-      integer :: ierror
-
-      call FIPC_init(comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_init")
-
-      ! intra_comm: communicator for processes on this shared memory node
-      ! extra_comm: communicator for all rank 0 on each shared memory node
-      call FIPC_ctxt_intra_comm(FIPC_ctxt_world, intra_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_intra_comm")
-      call FIPC_ctxt_extra_comm(FIPC_ctxt_world, extra_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_ctxt_extra_comm")
-
-      call MPI_COMM_SIZE(intra_comm, ncores, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-      call MPI_COMM_RANK(intra_comm, my_core, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-
-      ! only rank 0 on each shared memory node member of extra_comm
-      ! for others extra_comm = MPI_COMM_NULL
-      if (extra_comm /= MPI_COMM_NULL) then
-         call MPI_COMM_SIZE(extra_comm, nnodes, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-         call MPI_COMM_RANK(extra_comm, my_node, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-      end if
-
-      ! other ranks share the same information as their leaders
-      call MPI_BCAST(nnodes, 1, MPI_INTEGER, 0, intra_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
-      call MPI_BCAST(my_node, 1, MPI_INTEGER, 0, intra_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
-
-      ! maxcor
-      call MPI_ALLREDUCE(ncores, maxcor, 1, MPI_INTEGER, MPI_MAX, &
-                         decomp_2d_comm, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-
-      call FIPC_finalize(ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "FIPC_finalize")
-
-      return
-
-   end subroutine get_smp_map
-
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Set up smp-node based shared memory maps
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   SUBROUTINE MAPSET_SMPSHM(C, KTBL, NARY, decomp)
-
-      IMPLICIT NONE
-
-      TYPE(SMP_INFO) C
-      INTEGER KTBL(C%MAXCORE, C%NSMP)
-      INTEGER NARY(C%NCPU, C%NCORE)
-      TYPE(DECOMP_INFO) :: decomp
-
-      INTEGER i, j, k, l, N, PTR, BSIZ, ierror, status, seed
-      character*16 s
-
-      BSIZ = C%N_SND
-
-      ! a - SNDBUF
-      IF (C%MPI_COMM == DECOMP_2D_COMM_COL) THEN
-         ALLOCATE (decomp%x1cnts_s(C%NSMP), decomp%x1disp_s(C%NSMP + 1), &
-                   stat=status)
-         CALL MPI_Allgather(decomp%x1cnts, C%NCPU, MPI_INTEGER, &
-                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-         PTR = 0
-         DO i = 1, C%NSMP
-            decomp%x1disp_s(i) = PTR
-            N = 0
-            DO j = 1, C%MAXCORE
-               k = KTBL(j, i)
-               IF (k > 0) then
-                  DO l = 1, C%NCORE
-                     IF (l == C%CORE_ME) decomp%x1disp_o(k - 1) = PTR
-                     N = N + NARY(k, l)
-                     PTR = PTR + NARY(k, l)
-                  END DO
-               END IF
-            END DO
-            decomp%x1cnts_s(i) = N
-         END DO
-         decomp%x1disp_s(C%NSMP + 1) = PTR
-         IF (PTR > BSIZ) BSIZ = PTR
-
-      ELSE IF (C%MPI_COMM == DECOMP_2D_COMM_ROW) THEN
-         ALLOCATE (decomp%y2cnts_s(C%NSMP), decomp%y2disp_s(C%NSMP + 1), &
-                   stat=status)
-         CALL MPI_Allgather(decomp%y2cnts, C%NCPU, MPI_INTEGER, &
-                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-         PTR = 0
-         DO i = 1, C%NSMP
-            decomp%y2disp_s(i) = PTR
-            N = 0
-            DO j = 1, C%MAXCORE
-               k = KTBL(j, i)
-               IF (k > 0) then
-                  DO l = 1, C%NCORE
-                     IF (l == C%CORE_ME) decomp%y2disp_o(k - 1) = PTR
-                     N = N + NARY(k, l)
-                     PTR = PTR + NARY(k, l)
-                  END DO
-               END IF
-            END DO
-            decomp%y2cnts_s(i) = N
-         END DO
-         decomp%y2disp_s(C%NSMP + 1) = PTR
-         IF (PTR > BSIZ) BSIZ = PTR
-      END IF
-
-      ! b - RCVBUF
-
-      IF (C%MPI_COMM == DECOMP_2D_COMM_COL) THEN
-         ALLOCATE (decomp%y1cnts_s(C%NSMP), decomp%y1disp_s(C%NSMP + 1), &
-                   stat=status)
-         CALL MPI_Allgather(decomp%y1cnts, C%NCPU, MPI_INTEGER, &
-                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-         PTR = 0
-         DO i = 1, C%NSMP
-            decomp%y1disp_s(i) = PTR
-            N = 0
-            DO j = 1, C%NCORE
-               DO l = 1, C%MAXCORE
-                  k = KTBL(l, i)
-                  IF (k > 0) then
-                     IF (j == C%CORE_ME) decomp%y1disp_o(k - 1) = PTR
-                     N = N + NARY(k, j)
-                     PTR = PTR + NARY(k, j)
-                  END IF
-               END DO
-            END DO
-            decomp%y1cnts_s(i) = N
-         END DO
-         decomp%y1disp_s(C%NSMP + 1) = PTR
-         IF (PTR > BSIZ) BSIZ = PTR
-
-      ELSE IF (C%MPI_COMM == DECOMP_2D_COMM_ROW) THEN
-         ALLOCATE (decomp%z2cnts_s(C%NSMP), decomp%z2disp_s(C%NSMP + 1), &
-                   stat=status)
-         CALL MPI_Allgather(decomp%z2cnts, C%NCPU, MPI_INTEGER, &
-                            NARY, C%NCPU, MPI_INTEGER, C%CORE_COMM, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLGATHER")
-         PTR = 0
-         DO i = 1, C%NSMP
-            decomp%z2disp_s(i) = PTR
-            N = 0
-            DO j = 1, C%NCORE
-               DO l = 1, C%MAXCORE
-                  k = KTBL(l, i)
-                  IF (k > 0) then
-                     IF (j == C%CORE_ME) decomp%z2disp_o(k - 1) = PTR
-                     N = N + NARY(k, j)
-                     PTR = PTR + NARY(k, j)
-                  END IF
-               END DO
-            END DO
-            decomp%z2cnts_s(i) = N
-         END DO
-         decomp%z2disp_s(C%NSMP + 1) = PTR
-         IF (PTR > BSIZ) BSIZ = PTR
-
-      END IF
-
-      ! check buffer size and (re)-allocate buffer space if necessary
-      IF (BSIZ > C%N_SND) then
-         IF (C%SND_P /= 0) CALL DEALLOC_SHM(C%SND_P, C%CORE_COMM)
-         ! make sure each rank has unique keys to get shared memory
-         !IF (C%MPI_COMM==DECOMP_2D_COMM_COL) THEN
-         !   seed = nrank+nproc*0+1 ! has to be non-zero
-         !ELSE IF (C%MPI_COMM==DECOMP_2D_COMM_ROW) THEN
-         !   seed = nrank+nproc*1+1
-         !END IF
-         status = 1
-         !CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status, &
-         !     seed)
-         CALL ALLOC_SHM(C%SND_P, BSIZ, real_type, C%CORE_COMM, status)
-         C%N_SND = BSIZ
-
-         IF (C%RCV_P /= 0) CALL DEALLOC_SHM(C%RCV_P, C%CORE_COMM)
-         status = 1
-         CALL ALLOC_SHM(C%RCV_P, BSIZ, real_type, C%CORE_COMM, status)
-         C%N_RCV = BSIZ
-
-         IF (C%SND_P_c /= 0) CALL DEALLOC_SHM(C%SND_P_c, C%CORE_COMM)
-         status = 1
-         CALL ALLOC_SHM(C%SND_P_c, BSIZ, complex_type, C%CORE_COMM, status)
-         C%N_SND = BSIZ
-
-         IF (C%RCV_P_c /= 0) CALL DEALLOC_SHM(C%RCV_P_c, C%CORE_COMM)
-         status = 1
-         CALL ALLOC_SHM(C%RCV_P_c, BSIZ, complex_type, C%CORE_COMM, status)
-         C%N_RCV = BSIZ
-
-      END IF
-
-      RETURN
-   END SUBROUTINE MAPSET_SMPSHM
-
-#endif
-
 #ifdef OCC
    ! For non-blocking communication code, progress the comminication stack
    subroutine transpose_test(handle)

From 79b4a3f3df5c172c501975ffd74bdd0c57f0f417 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Fri, 20 Jan 2023 14:33:37 +0000
Subject: [PATCH 096/436] Reactivate the halo test without GPU execution

---
 examples/Makefile                | 8 ++++++--
 examples/halo_test/halo_test.f90 | 6 ++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index 1643583b..508a0c0b 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,7 +1,7 @@
-.PHONY: init_test test2d fft_physical_x fft_physical_z io_test 
+.PHONY: init_test test2d fft_physical_x fft_physical_z halo_test io_test 
 
 # Just build the examples
-examples: init_test test2d fft_physical_x fft_physical_z io_test
+examples: init_test test2d fft_physical_x fft_physical_z halo_test io_test
 	@echo "Built the examples"
 
 init_test:
@@ -12,6 +12,8 @@ fft_physical_x:
 	$(MAKE) -C $@ all
 fft_physical_z:
 	$(MAKE) -C $@ all
+halo_test:
+	$(MAKE) -C $@ $@
 io_test:
 	$(MAKE) -C $@ all
 
@@ -20,6 +22,7 @@ check:
 	cd test2d; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
+	cd halo_test; $(MAKE) $@
 	cd io_test; $(MAKE) $@
 
 clean:
@@ -27,6 +30,7 @@ clean:
 	cd test2d; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
+	cd halo_test; $(MAKE) $@
 	cd io_test; $(MAKE) $@
 
 export
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 29447a18..6442c395 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -33,9 +33,15 @@ program halo_test
 
    logical :: passing, all_pass
 
+#if defined(_GPU)
+   write(*,*) 'Halo is not currently support on GPU. This test is not performed'
+   return
+#endif
+
    call MPI_INIT(ierror)
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
+
    xlast = xsize(1) - 1
    if (xend(2) == ny) then
       ylast = xsize(2) - 1

From 8d97d0b5d9df978024d80022f299059963d3af69 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 20 Jan 2023 15:24:12 +0000
Subject: [PATCH 097/436] Replace i1/in with ifirst/ilast in halo_test

in is a reserved keyword so shouldn't be used as a variable.
---
 examples/halo_test/halo_test.f90 | 132 +++++++++++++++----------------
 1 file changed, 66 insertions(+), 66 deletions(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 29447a18..e09557ec 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -110,25 +110,25 @@ subroutine test_div_transpose()
 #else
       logical, parameter :: global = .false.
 #endif
-      integer :: i1, in ! I loop start/end
-      integer :: j1, jn ! J loop start/end
-      integer :: k1, kn ! K loop start/end
+      integer :: ifirst, ilast ! I loop start/end
+      integer :: jfirst, jlast ! J loop start/end
+      integer :: kfirst, klast ! K loop start/end
 
       ! du/dx calculated on X-pencil
       call alloc_x(div1, global)
 #ifdef HALO_GLOBAL
-      k1 = xstart(3); kn = xend(3)
-      j1 = xstart(2); jn = xend(2)
+      kfirst = xstart(3); klast = xend(3)
+      jfirst = xstart(2); jlast = xend(2)
 #else
-      k1 = 1; kn = xsize(3)
-      j1 = 1; jn = xsize(2)
+      kfirst = 1; klast = xsize(3)
+      jfirst = 1; jlast = xsize(2)
 #endif
-      i1 = 2; in = xsize(1) - 1
+      ifirst = 2; ilast = xsize(1) - 1
 
       div1 = 0.0_mytype
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
+      do k = kfirst, klast
+         do j = jfirst, jlast
+            do i = ifirst, ilast
                div1(i, j, k) = u1(i + 1, j, k) - u1(i - 1, j, k)
             end do
          end do
@@ -138,20 +138,20 @@ subroutine test_div_transpose()
       call alloc_y(v2, global)
       call alloc_y(wk2, global)
 #ifdef HALO_GLOBAL
-      k1 = ystart(3); kn = yend(3)
-      i1 = ystart(1); in = yend(1)
+      kfirst = ystart(3); klast = yend(3)
+      ifirst = ystart(1); ilast = yend(1)
 #else
-      k1 = 1; kn = ysize(3)
-      i1 = 1; in = ysize(1)
+      kfirst = 1; klast = ysize(3)
+      ifirst = 1; ilast = ysize(1)
 #endif
-      j1 = 2; jn = ysize(2) - 1
+      jfirst = 2; jlast = ysize(2) - 1
 
       call transpose_x_to_y(v1, v2)
       call transpose_x_to_y(div1, wk2)
 
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
+      do k = kfirst, klast
+         do j = jfirst, jlast
+            do i = ifirst, ilast
                wk2(i, j, k) = wk2(i, j, k) + v2(i, j + 1, k) - v2(i, j - 1, k)
             end do
          end do
@@ -162,21 +162,21 @@ subroutine test_div_transpose()
       call alloc_z(w3, global)
       call alloc_z(wk3, global)
 #ifdef HALO_GLOBAL
-      j1 = zstart(2); jn = zend(2)
-      i1 = zstart(1); in = zend(1)
+      jfirst = zstart(2); jlast = zend(2)
+      ifirst = zstart(1); ilast = zend(1)
 #else
-      j1 = 1; jn = zsize(2)
-      i1 = 1; in = zsize(1)
+      jfirst = 1; jlast = zsize(2)
+      ifirst = 1; ilast = zsize(1)
 #endif
-      k1 = 2; kn = zsize(3) - 1
+      kfirst = 2; klast = zsize(3) - 1
 
       call transpose_x_to_y(w1, w2)
       call transpose_y_to_z(w2, w3)
       call transpose_y_to_z(wk2, wk3)
 
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
+      do k = kfirst, klast
+         do j = jfirst, jlast
+            do i = ifirst, ilast
                wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
             end do
          end do
@@ -209,9 +209,9 @@ subroutine test_div_haloX()
 #else
       logical, parameter :: global = .false.
 #endif
-      integer :: i1, in ! I loop start/end
-      integer :: j1, jn ! J loop start/end
-      integer :: k1, kn ! K loop start/end
+      integer :: ifirst, ilast ! I loop start/end
+      integer :: jfirst, jlast ! J loop start/end
+      integer :: kfirst, klast ! K loop start/end
 
       ! Expected sizes
       nx_expected = nx
@@ -223,24 +223,24 @@ subroutine test_div_haloX()
       call update_halo(v1, vh, 1, opt_global=.true., opt_pencil=1)
       call update_halo(w1, wh, 1, opt_global=.true., opt_pencil=1)
 
-      k1 = xstart(3); kn = xend(3)
-      j1 = xstart(2); jn = xend(2)
+      kfirst = xstart(3); klast = xend(3)
+      jfirst = xstart(2); jlast = xend(2)
 #else
       call update_halo(v1, vh, 1, opt_pencil=1)
       call update_halo(w1, wh, 1, opt_pencil=1)
 
-      k1 = 1; kn = xsize(3)
-      j1 = 1; jn = xsize(2)
+      kfirst = 1; klast = xsize(3)
+      jfirst = 1; jlast = xsize(2)
 #endif
-      i1 = 2; in = xsize(1) - 1
+      ifirst = 2; ilast = xsize(1) - 1
 
       call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "X:v")
       call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
 
       div2 = 0.0_mytype
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
+      do k = kfirst, klast
+         do j = jfirst, jlast
+            do i = ifirst, ilast
                div2(i, j, k) = (u1(i + 1, j, k) - u1(i - 1, j, k)) &
                                + (vh(i, j + 1, k) - vh(i, j - 1, k)) &
                                + (wh(i, j, k + 1) - wh(i, j, k - 1))
@@ -267,9 +267,9 @@ subroutine test_div_haloY()
 #else
       logical, parameter :: global = .false.
 #endif
-      integer :: i1, in ! I loop start/end
-      integer :: j1, jn ! J loop start/end
-      integer :: k1, kn ! K loop start/end
+      integer :: ifirst, ilast ! I loop start/end
+      integer :: jfirst, jlast ! J loop start/end
+      integer :: kfirst, klast ! K loop start/end
 
       ! Expected sizes
       nx_expected = ysize(1) + 2
@@ -290,22 +290,22 @@ subroutine test_div_haloY()
 #ifdef HALO_GLOBAL
       call update_halo(u2, uh, 1, opt_global=.true., opt_pencil=2)
       call update_halo(w2, wh, 1, opt_global=.true., opt_pencil=2)
-      k1 = ystart(3); kn = yend(3)
-      i1 = ystart(1); in = yend(1)
+      kfirst = ystart(3); klast = yend(3)
+      ifirst = ystart(1); ilast = yend(1)
 #else
       call update_halo(u2, uh, 1, opt_pencil=2)
       call update_halo(w2, wh, 1, opt_pencil=2)
-      k1 = 1; kn = ysize(3)
-      i1 = 1; in = ysize(1)
+      kfirst = 1; klast = ysize(3)
+      ifirst = 1; ilast = ysize(1)
 #endif
-      j1 = 2; jn = ysize(2) - 1
+      jfirst = 2; jlast = ysize(2) - 1
 
       call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Y:u")
       call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "Y:w")
 
-      do k = k1, kn
-         do j = j1, jn
-            do i = i1, in
+      do k = kfirst, klast
+         do j = jfirst, jlast
+            do i = ifirst, ilast
                wk2(i, j, k) = (uh(i + 1, j, k) - uh(i - 1, j, k)) &
                               + (v2(i, j + 1, k) - v2(i, j - 1, k)) &
                               + (wh(i, j, k + 1) - wh(i, j, k - 1))
@@ -334,9 +334,9 @@ subroutine test_div_haloZ()
 #else
       logical, parameter :: global = .false.
 #endif
-      integer :: i1, in ! I loop start/end
-      integer :: j1, jn ! J loop start/end
-      integer :: k1, kn ! K loop start/end
+      integer :: ifirst, ilast ! I loop start/end
+      integer :: jfirst, jlast ! J loop start/end
+      integer :: kfirst, klast ! K loop start/end
 
       ! Expected sizes
       nx_expected = zsize(1) + 2
@@ -358,20 +358,20 @@ subroutine test_div_haloZ()
       ! du/dx
 #ifdef HALO_GLOBAL
       call update_halo(u3, uh, 1, opt_global=.true., opt_pencil=3)
-      i1 = zstart(1); in = zend(1)
-      j1 = zstart(2); jn = zend(2)
+      ifirst = zstart(1); ilast = zend(1)
+      jfirst = zstart(2); jlast = zend(2)
 #else
       call update_halo(u3, uh, 1, opt_pencil=3)
-      i1 = 1; in = zsize(1)
-      j1 = 1; jn = zsize(2)
+      ifirst = 1; ilast = zsize(1)
+      jfirst = 1; jlast = zsize(2)
 #endif
-      k1 = 2; kn = zsize(3) - 1
+      kfirst = 2; klast = zsize(3) - 1
 
       call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
 
-      do j = j1, jn
-         do i = i1, in
-            do k = k1, kn
+      do j = jfirst, jlast
+         do i = ifirst, ilast
+            do k = kfirst, klast
                wk3(i, j, k) = uh(i + 1, j, k) - uh(i - 1, j, k)
             end do
          end do
@@ -386,18 +386,18 @@ subroutine test_div_haloZ()
 
       call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
 
-      do j = j1, jn
-         do i = i1, in
-            do k = k1, kn
+      do j = jfirst, jlast
+         do i = ifirst, ilast
+            do k = kfirst, klast
                wk3(i, j, k) = wk3(i, j, k) + vh(i, j + 1, k) - vh(i, j - 1, k)
             end do
          end do
       end do
 
       ! dw/dz
-      do j = j1, jn
-         do i = i1, in
-            do k = k1, kn
+      do j = jfirst, jlast
+         do i = ifirst, ilast
+            do k = kfirst, klast
                wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
             end do
          end do

From da7371828c08f18af9074cf8df97ae8a25840dbe Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 20 Jan 2023 15:24:53 +0000
Subject: [PATCH 098/436] Restore testing the halos

Don't run on GPUs, print a message that this is temporary
---
 examples/Makefile           | 8 ++++++--
 examples/halo_test/Makefile | 5 ++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index 1643583b..85286d6b 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,7 +1,7 @@
-.PHONY: init_test test2d fft_physical_x fft_physical_z io_test 
+.PHONY: init_test test2d fft_physical_x fft_physical_z io_test halo_test
 
 # Just build the examples
-examples: init_test test2d fft_physical_x fft_physical_z io_test
+examples: init_test test2d fft_physical_x fft_physical_z io_test halo_test
 	@echo "Built the examples"
 
 init_test:
@@ -14,6 +14,8 @@ fft_physical_z:
 	$(MAKE) -C $@ all
 io_test:
 	$(MAKE) -C $@ all
+halo_test:
+	$(MAKE) -C $@ all
 
 check:
 	cd init_test; $(MAKE) $@
@@ -21,6 +23,7 @@ check:
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
 	cd io_test; $(MAKE) $@
+	cd halo_test; $(MAKE) $@
 
 clean:
 	cd init_test; $(MAKE) $@
@@ -28,5 +31,6 @@ clean:
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
 	cd io_test; $(MAKE) $@
+	cd halo_test; $(MAKE) $@
 
 export
diff --git a/examples/halo_test/Makefile b/examples/halo_test/Makefile
index 72c28439..b39fc632 100644
--- a/examples/halo_test/Makefile
+++ b/examples/halo_test/Makefile
@@ -10,12 +10,15 @@ OBJ = halo_test.o
 NP ?= 1
 MPIRUN ?= mpirun
 
+all: halo_test
+
 halo_test: $(OBJ)
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $(OBJ) $(LIBS)
 
 ifeq ($(PARAMOD),gpu)
 check:
-	$(MPIRUN) -n $(NP) ./bind.sh ./halo_test
+	$(info Halo test temporarily disabled for GPU)
+#	$(MPIRUN) -n $(NP) ./bind.sh ./halo_test
 else
 check:
 	$(MPIRUN) -n $(NP) ./halo_test

From b00440fcfa001467f89e8faffcc800ce06b7e81d Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 14:40:52 +0000
Subject: [PATCH 099/436] Use Xcompact3d CMakeLists.txt to build 2decomp&fft

Currently this appears to correctly build the library, however it
is untested.
---
 CMakeLists.txt     | 259 ++++++++++++++++++++++++++++++++++++++++-----
 src/CMakeLists.txt |  37 +++++++
 2 files changed, 270 insertions(+), 26 deletions(-)
 create mode 100644 src/CMakeLists.txt

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7e7a99ef..571bc7aa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,35 +1,242 @@
-file(GLOB files_decomp decomp_2d.f90
-	               log.f90
-                       glassman.f90)
-include_directories(${CMAKE_SOURCE_DIR}/decomp2d)
+cmake_minimum_required(VERSION 3.12)                        
+cmake_policy(SET CMP0074 NEW)
 
-#if(${FFT_Choice} MATCHES "generic")
-#  file(GLOB files_fft fft_generic.f90)
-#endif(${FFT_Choice} MATCHES "generic")
+project(2decomp LANGUAGES Fortran)
 
-if(FFTW_FOUND)
-  message (STATUS "Compiling using FFTW3")
-  file(GLOB files_fft fft_fftw3.f90)
-else(FFTW_FOUND)
-  message (STATUS "Compiling using Generic FFT")
-  file(GLOB files_fft fft_generic.f90)
-endif(FFTW_FOUND)
+option(ENABLE_OPENACC "Allow user to activate/deactivate OpenACC support" OFF)
+option(ENABLE_CUDA "Allow user to activate/deactivate CUDA support" OFF)
 
+if (ENABLE_CUDA)
+  enable_language(CUDA)
+endif()
 
-set(SRCFILES ${files_decomp} ${files_fft})
+set(AUTHOR "Stefano Rolfo;Charles Moulinec;Paul Bartholomew")
+set(AUTHOR_DETAILS "stefano.rolfo@stfc.ac.uk;charles.moulinec@stfc.ac.uk;p.bartholomew@epcc.ed.ac.uk")
+set(DESCRIPTION "Building 2decomp&fft using cmake")
 
-add_library(decomp2d STATIC ${SRCFILES})
+message(STATUS "building ${PROJECT_NAME}")
+
+include(GNUInstallDirs)
+set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
+set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR})
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/opt" CACHE PATH "..." FORCE)
+endif()
+
+# Add support for CMAKE_DEPENDENT_OPTION
+INCLUDE(CMakeDependentOption)
+INCLUDE(CMakeParseArguments)
+
+# Find the modules included with Xcompact
+#SET(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
+
+# make sure that the default is a RELEASE
+if (NOT CMAKE_BUILD_TYPE)
+  set (CMAKE_BUILD_TYPE RELEASE CACHE STRING
+      "Choose the type of build, options are: None Debug Release."
+      FORCE)
+endif (NOT CMAKE_BUILD_TYPE)
+
+if (ENABLE_OPENACC)
+  include(FindOpenACC)
+  if(OpenACC_Fortran_FOUND)
+    message(STATUS "OpenACC for Fotran Compiler Found, version ${OpenACC_Fortran_VERSION_MAJOR}.${OpenACC_Fortran_VERSION_MINOR}")
+  else()
+    message(ERROR_CRITICAL "No OpenACC support detected")
+  endif()
+  set(OPENACC_TARGET "gpu" CACHE STRING "Target for acceleration (gpu (default) or multicore)")
+  set_property(CACHE OPENACC_TARGET PROPERTY STRINGS gpu multicore)
+endif()
+
+if (OPENACC_TARGET MATCHES "gpu")
+  find_package(CUDAToolkit REQUIRED)
+  if(${CMAKE_VERSION} VERSION_LESS_EQUAL "3.13.4")
+    cuda_select_nvcc_arch_flags(ARCH_FLAGS "Auto") # optional argument for arch to add
+    message(STATUS "ARCH_FLAGS = ${ARCH_FLAGS}")
+    string(REPLACE "-gencode;" "--generate-code=" ARCH_FLAGS "${ARCH_FLAGS}")
+    string(APPEND CMAKE_CUDA_FLAGS "${ARCH_FLAGS}")
+    message(STATUS "ARCH_FLAGS WITH CUDA = ${ARCH_FLAGS}")
+  else()
+    include(FindCUDA/select_compute_arch)
+    CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1)
+    string(STRIP "${INSTALLED_GPU_CCS_1}" INSTALLED_GPU_CCS_2)
+    string(REPLACE " " ";" INSTALLED_GPU_CCS_3 "${INSTALLED_GPU_CCS_2}")
+    string(REPLACE "." "" CUDA_ARCH_LIST "${INSTALLED_GPU_CCS_3}")
+    SET(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_LIST})
+    set_property(GLOBAL PROPERTY CUDA_ARCHITECTURES "${CUDA_ARCH_LIST}")
+    message(STATUS "CUDA_ARCHITECTURES ${CUDA_ARCH_LIST}")
+    list(GET CUDA_ARCH_LIST 0 CUDA_ARCH_FIRST)
+    message(STATUS "CUDA_ARCH_FIRST ${CUDA_ARCH_FIRST}")
+  endif()
+  message(STATUS "CUDA_LIBRARIES ${CUDA_LIBRARIES}")
+endif()
+
+set(NVIDIA_PROFILER "Use the NVidia profiles" ON)
+set(Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER_ID} )
+message(STATUS "COMP ID ${Fortran_COMPILER_NAME}")
+message(STATUS "Fortran compiler name ${Fortran_COMPILER_NAME}")
+message(STATUS "Fortran compiler version ${CMAKE_Fortran_COMPILER_VERSION}")
+if (Fortran_COMPILER_NAME MATCHES "GNU")
+  # gfortran
+  message(STATUS "Setting gfortran flags")
+  set(CMAKE_Fortran_FLAGS "-cpp -funroll-loops -floop-optimize -g -Warray-bounds -fcray-pointer -fbacktrace -ffree-line-length-none")
+  if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
+    message(STATUS "Set New Fortran basic flags")
+    set(CMAKE_Fortran_FLAGS "-cpp -funroll-loops -floop-optimize -g -Warray-bounds -fcray-pointer -fbacktrace -ffree-line-length-none -fallow-argument-mismatch")
+  endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
+  set(CMAKE_Fortran_FLAGS_RELEASE "-funroll-all-loops -fno-f2c -O3")
+  set(CMAKE_Fortran_FLAGS_DEBUG   "-DDEBG -O0 -g")
+elseif (Fortran_COMPILER_NAME MATCHES "Intel")
+  message(STATUS "Setting ifort flags")
+  set(CMAKE_Fortran_FLAGS "-fpp -xHost -heaparrays -safe-cray-ptr -g -traceback")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-O3 -ipo")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g -DDEBG")
+  #set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
+elseif (Fortran_COMPILER_NAME MATCHES "NAG")
+  message(STATUS "Setting nagfor flags")
+  set(CMAKE_Fortran_FLAGS "-fpp")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
+elseif (Fortran_COMPILER_NAME MATCHES "Cray")
+  message(STATUS "Setting cray fortran flags")
+  set(CMAKE_Fortran_FLAGS "-eF -g -N 1023")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
+elseif (Fortran_COMPILER_NAME MATCHES "PGI")
+  message(STATUS "Setting PGI fortran flags")
+  if (ENABLE_OPENACC) 
+    if (OPENACC_TARGET MATCHES "gpu")
+      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=gpu")
+    else(OPENACC_TARGET MATCHES "gpu")
+      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=multicore")
+    endif(OPENACC_TARGET MATCHES "gpu")
+  else()
+    set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
+  endif()
+  #set(CMAKE_Fortran_FLAGS "-fast -cpp -Mfree -Kieee -Minfo=accel -g ")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-fast -O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0")
+elseif (Fortran_COMPILER_NAME MATCHES "NVHPC")
+  message(STATUS "Setting NVHPC fortran flags")
+  if (ENABLE_OPENACC) 
+    if (OPENACC_TARGET MATCHES "gpu")
+      #set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar=gpu -acc -target=gpu -Minstrument")
+      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp -Mfree -Kieee -Minfo=accel -stdpar=gpu -gpu=cc${CUDA_ARCH_FIRST},managed -acc -target=gpu")
+      #add_definitions("-lnvhpcwrapnvtx")
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lnvhpcwrapnvtx")
+    else()
+      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar -acc -target=multicore")
+    endif(OPENACC_TARGET MATCHES "gpu")
+  else()
+    set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
+  endif()
+  set (CMAKE_Fortran_FLAGS_RELEASE "-fast -O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g -DDEBG")
+elseif (Fortran_COMPILER_NAME MATCHES "Fujitsu")
+  message(STATUS "Setting Fujitsu fortran flags")
+  set (CMAKE_Fortran_FLAGS "-Cpp")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG "-O0")
+else (Fortran_COMPILER_NAME MATCHES "GNU")
+  message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
+  message ("Fortran compiler: " ${Fortran_COMPILER_NAME})
+  message ("No optimized Fortran compiler flags are known, we just try -O2...")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-O2")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
+endif (Fortran_COMPILER_NAME MATCHES "GNU")
+
+if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
+  add_definitions("-DDEBG")
+endif (CMAKE_BUILD_TYPE MATCHES "DEBUG")
+
+
+find_package(MPI REQUIRED)
+# Stop if there is no MPI_Fortran_Compiler
+if (MPI_Fortran_COMPILER)
+    message(STATUS "MPI_Fortran_COMPILER found: ${MPI_Fortran_COMPILER}")
+else (MPI_Fortran_COMPILER)
+    message(SEND_ERROR "This application cannot compile without MPI")
+endif(MPI_Fortran_COMPILER)
+# Warning if Include are not found => can be fixed with more recent cmake version
 if (MPI_FOUND)
-  target_link_libraries(decomp2d PRIVATE MPI::MPI_Fortran)
+    message(STATUS "MPI FOUND: ${MPI_FOUND}")
+    include_directories(SYSTEM ${MPI_INCLUDE_PATH})
+    message(STATUS "MPI INCL ALSO FOUND: ${MPI_INCLUDE_PATH}")
+else (MPI_FOUND)
+    message(STATUS "NO MPI include have been found. The executable won't be targeted with MPI include")
+    message(STATUS "Code will compile but performaces can be compromised")
+    message(STATUS "Using a CMake vers > 3.10 should solve the problem")
+    message(STATUS "Alternatively use ccmake to manually set the include if available")
 endif (MPI_FOUND)
 
-if(FFTW_FOUND)
-  target_include_directories(decomp2d PRIVATE ${FFTW_INCLUDE_DIRS})
-  #target_include_directories(decomp2d PRIVATE ${PKG_FFTW_LIBRARY_DIRS})
-endif(FFTW_FOUND)
-
-install(TARGETS decomp2d
-        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 
+execute_process(
+  COMMAND git describe --tag --long --always
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  OUTPUT_VARIABLE GIT_VERSION
+  OUTPUT_STRIP_TRAILING_WHITESPACE
 )
+add_definitions("-DVERSION=\"${GIT_VERSION}\"")
+option(DOUBLE_PRECISION "Build Xcompact with double precision" ON)
+if (DOUBLE_PRECISION)
+  add_definitions("-DDOUBLE_PREC")
+endif()
+
+option(SINGLE_PRECISION_OUTPUT "Build XCompact with output in single precision" OFF)
+if (SINGLE_PRECISION_OUTPUT)
+  add_definitions("-DSAVE_SINGLE")
+endif()
+
+# FFT options
+set(FFT_Choice "generic" CACHE STRING "FFT for XCompact3d project (generic is the default)")
+set_property(CACHE FFT_Choice PROPERTY STRINGS generic fftw mkl)
+
+# Look for fftw is required
+if(${FFT_Choice} MATCHES "fftw")
+  configure_file(cmake/downloadFindFFTW.cmake.in findFFTW-download/CMakeLists.txt)
+  execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+          RESULT_VARIABLE result
+          WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-download )
+  if(result)
+      message(FATAL_ERROR "CMake step for findFFTW failed: ${result}")
+      else()
+      message("CMake step for findFFTW completed (${result}).")
+  endif()
+  execute_process(COMMAND ${CMAKE_COMMAND} --build .
+          RESULT_VARIABLE result
+          WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-download )
+  if(result)
+      message(FATAL_ERROR "Build step for findFFTW failed: ${result}")
+  endif()
+  
+  set(findFFTW_DIR ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-src)
+  
+  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${findFFTW_DIR}")
+  find_package(FFTW)
+  message(STATUS "FFTW_FOUND     : ${FFTW_FOUND}")
+  message(STATUS "FFTW_LIBRARIES : ${FFTW_LIBRARIES}")
+  message(STATUS "FFTW_INCLUDE   : ${FFTW_INCLUDE_DIRS}")
+
+  #add_definitions("-lfftw3 -lfftw3f")
+
+endif(${FFT_Choice} MATCHES "fftw")
+
+# Create a static library for the fft
+add_subdirectory(src)
+
+# # Create an example dir with all input.i3d example files
+# option(BUILD_TESTING "Build with tests" ON)
+# set(test_dir "${PROJECT_BINARY_DIR}/Test")
+# message(STATUS "Before test main ${test_dir}")
+# if (${BUILD_TESTING})
+#   file(MAKE_DIRECTORY ${test_dir})
+#   include(CTest)
+#   message(STATUS "MPI INCL ALSO FOUND: ${MPI_INCLUDE_PATH}")
+#   message(STATUS "MPI EXEC: ${MPIEXEC_EXECUTABLE}")
+#   file(MAKE_DIRECTORY ${test_dir}/App)
+#   add_test(NAME TestAPP COMMAND ${MPIEXEC_EXECUTABLE} -n ${MPIEXEC_MAX_NUMPROCS} ${CMAKE_INSTALL_PREFIX}/bin/xcompact3d WORKING_DIRECTORY ${test_dir}/App)
+#   #if (PFUNIT_FOUND)
+#   #  file(MAKE_DIRECTORY ${test_dir}/Verif_x_00)
+#   #  add_test(NAME Test_dx_00 COMMAND ${MPIEXEC_EXECUTABLE} -n ${MPIEXEC_MAX_NUMPROCS} ${CMAKE_INSTALL_PREFIX}/bin/verif_x_00 WORKING_DIRECTORY ${test_dir}/Verif_x_00)
+#   #endif (PFUNIT_FOUND)
+# endif()
+
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
new file mode 100644
index 00000000..498745f4
--- /dev/null
+++ b/src/CMakeLists.txt
@@ -0,0 +1,37 @@
+file(GLOB files_decomp factor.f90
+	               decomp_2d_constants.f90
+	               decomp_2d.f90
+	               d2d_log.f90
+                       glassman.f90)
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+#if(${FFT_Choice} MATCHES "generic")
+#  file(GLOB files_fft fft_generic.f90)
+#endif(${FFT_Choice} MATCHES "generic")
+
+if(FFTW_FOUND)
+  message (STATUS "Compiling using FFTW3")
+  file(GLOB files_fft fft_fftw3.f90)
+else(FFTW_FOUND)
+  message (STATUS "Compiling using Generic FFT")
+  file(GLOB files_fft fft_generic.f90)
+endif(FFTW_FOUND)
+
+
+set(SRCFILES ${files_decomp} ${files_fft})
+
+add_library(decomp2d STATIC ${SRCFILES})
+if (MPI_FOUND)
+  target_link_libraries(decomp2d PRIVATE MPI::MPI_Fortran)
+endif (MPI_FOUND)
+
+if(FFTW_FOUND)
+  target_include_directories(decomp2d PRIVATE ${FFTW_INCLUDE_DIRS})
+  #target_include_directories(decomp2d PRIVATE ${PKG_FFTW_LIBRARY_DIRS})
+endif(FFTW_FOUND)
+
+install(TARGETS decomp2d
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 
+)

From 8a04ab83f36336ac3e4320af8b015b9e0697a6db Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 14:47:17 +0000
Subject: [PATCH 100/436] Some FFT files were missing from CMakeLists.txt

Testing is (currently) somewhat convoluted:

1) Run CMake build: cmake -B build && make -C build
2) Run standard make build: make
3) Delete libdecomp2d.a: rm libdecomp2d.a
4) Copy CMake library in-place: cp build/lib/libdecomp2d.a ./
5) Build and run examples: make examples && make check
---
 src/CMakeLists.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 498745f4..f311628c 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,8 +1,8 @@
 file(GLOB files_decomp factor.f90
 	               decomp_2d_constants.f90
 	               decomp_2d.f90
-	               d2d_log.f90
-                       glassman.f90)
+		       log.f90
+		       io.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
 #if(${FFT_Choice} MATCHES "generic")
@@ -14,9 +14,9 @@ if(FFTW_FOUND)
   file(GLOB files_fft fft_fftw3.f90)
 else(FFTW_FOUND)
   message (STATUS "Compiling using Generic FFT")
-  file(GLOB files_fft fft_generic.f90)
+  file(GLOB files_fft glassman.f90 fft_generic.f90)
 endif(FFTW_FOUND)
-
+list(APPEND files_fft fft_log.f90)
 
 set(SRCFILES ${files_decomp} ${files_fft})
 

From 92db832e6d5acf40bba9d4a836b1303d497faccc Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 15:35:25 +0000
Subject: [PATCH 101/436] Ensure module files go into mod/

Note, this requires making the mod directory by hand currently
---
 CMakeLists.txt     | 1 +
 src/CMakeLists.txt | 1 +
 2 files changed, 2 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 571bc7aa..8d63f0a5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -19,6 +19,7 @@ message(STATUS "building ${PROJECT_NAME}")
 include(GNUInstallDirs)
 set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
 set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR})
+set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_SOURCE_DIR}/mod)
 if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
   set(CMAKE_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/opt" CACHE PATH "..." FORCE)
 endif()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index f311628c..4fc3e338 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -4,6 +4,7 @@ file(GLOB files_decomp factor.f90
 		       log.f90
 		       io.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
+include_directories(${CMAKE_SOURCE_DIR}/mod)
 
 #if(${FFT_Choice} MATCHES "generic")
 #  file(GLOB files_fft fft_generic.f90)

From 06bf2b213e1b2e8d0c28c5e3ad0ea214a60d92bc Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 15:36:01 +0000
Subject: [PATCH 102/436] Initial testing support

Builds and runs init_test using ctest
---
 CMakeLists.txt                    | 4 ++++
 examples/CMakeLists.txt           | 1 +
 examples/init_test/CMakeLists.txt | 7 +++++++
 3 files changed, 12 insertions(+)
 create mode 100644 examples/CMakeLists.txt
 create mode 100644 examples/init_test/CMakeLists.txt

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8d63f0a5..c278fdf9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.12)
 cmake_policy(SET CMP0074 NEW)
 
 project(2decomp LANGUAGES Fortran)
+enable_testing()
 
 option(ENABLE_OPENACC "Allow user to activate/deactivate OpenACC support" OFF)
 option(ENABLE_CUDA "Allow user to activate/deactivate CUDA support" OFF)
@@ -224,6 +225,9 @@ endif(${FFT_Choice} MATCHES "fftw")
 # Create a static library for the fft
 add_subdirectory(src)
 
+# Add tests
+add_subdirectory(examples)
+
 # # Create an example dir with all input.i3d example files
 # option(BUILD_TESTING "Build with tests" ON)
 # set(test_dir "${PROJECT_BINARY_DIR}/Test")
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
new file mode 100644
index 00000000..0d425761
--- /dev/null
+++ b/examples/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(init_test)
diff --git a/examples/init_test/CMakeLists.txt b/examples/init_test/CMakeLists.txt
new file mode 100644
index 00000000..7e361b50
--- /dev/null
+++ b/examples/init_test/CMakeLists.txt
@@ -0,0 +1,7 @@
+file(GLOB files_test init_test.f90)
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+add_executable(init_test ${files_test})
+include_directories(${CMAKE_SOURCE_DIR}/src)
+target_link_libraries(init_test PRIVATE decomp2d)
+add_test(NAME init_test COMMAND init_test)

From 4d71c33ac36a718fd2ee9be445b96a38feca56e2 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 16:56:04 +0000
Subject: [PATCH 103/436] Add test2d to ctest

---
 examples/CMakeLists.txt        | 1 +
 examples/test2d/CMakeLists.txt | 7 +++++++
 2 files changed, 8 insertions(+)
 create mode 100644 examples/test2d/CMakeLists.txt

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 0d425761..18bcdc7a 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1 +1,2 @@
 add_subdirectory(init_test)
+add_subdirectory(test2d)
diff --git a/examples/test2d/CMakeLists.txt b/examples/test2d/CMakeLists.txt
new file mode 100644
index 00000000..3ebf4821
--- /dev/null
+++ b/examples/test2d/CMakeLists.txt
@@ -0,0 +1,7 @@
+file(GLOB files_test test2d.f90)
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+add_executable(test2d ${files_test})
+include_directories(${CMAKE_SOURCE_DIR}/src)
+target_link_libraries(test2d PRIVATE decomp2d)
+add_test(NAME test2d COMMAND test2d)

From a8862f390075ee502fdf7be287340c9ec6e89f75 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 16:59:33 +0000
Subject: [PATCH 104/436] Add fft_test_x to ctest

---
 examples/CMakeLists.txt                |  1 +
 examples/fft_physical_x/CMakeLists.txt | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)
 create mode 100644 examples/fft_physical_x/CMakeLists.txt

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 18bcdc7a..477b7e05 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,2 +1,3 @@
 add_subdirectory(init_test)
 add_subdirectory(test2d)
+add_subdirectory(fft_physical_x)
diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
new file mode 100644
index 00000000..b2dfd4f0
--- /dev/null
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -0,0 +1,17 @@
+file(GLOB files_fft_c2c fft_c2c_x.f90)
+file(GLOB files_fft_r2c fft_r2c_x.f90)
+file(GLOB files_fft_grid fft_grid_x.f90)
+
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+add_executable(fft_c2c_x ${files_fft_c2c})
+add_executable(fft_r2c_x ${files_fft_r2c})
+add_executable(fft_grid_x ${files_fft_grid})
+
+target_link_libraries(fft_c2c_x PRIVATE decomp2d)
+target_link_libraries(fft_r2c_x PRIVATE decomp2d)
+target_link_libraries(fft_grid_x PRIVATE decomp2d)
+
+add_test(NAME fft_c2c_x COMMAND fft_c2c_x)
+add_test(NAME fft_r2c_x COMMAND fft_r2c_x)
+add_test(NAME fft_grid_x COMMAND fft_grid_x)

From 210e4208a56fd242c0a0b27cad072676aa425546 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 17:00:59 +0000
Subject: [PATCH 105/436] Add fft_physical_z to ctest

---
 examples/CMakeLists.txt                |  1 +
 examples/fft_physical_z/CMakeLists.txt | 13 +++++++++++++
 2 files changed, 14 insertions(+)
 create mode 100644 examples/fft_physical_z/CMakeLists.txt

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 477b7e05..cb1accb2 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,3 +1,4 @@
 add_subdirectory(init_test)
 add_subdirectory(test2d)
 add_subdirectory(fft_physical_x)
+add_subdirectory(fft_physical_z)
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
new file mode 100644
index 00000000..b010e35d
--- /dev/null
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -0,0 +1,13 @@
+file(GLOB files_fft_c2c fft_c2c_z.f90)
+file(GLOB files_fft_r2c fft_r2c_z.f90)
+
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+add_executable(fft_c2c_z ${files_fft_c2c})
+add_executable(fft_r2c_z ${files_fft_r2c})
+
+target_link_libraries(fft_c2c_z PRIVATE decomp2d)
+target_link_libraries(fft_r2c_z PRIVATE decomp2d)
+
+add_test(NAME fft_c2c_z COMMAND fft_c2c_z)
+add_test(NAME fft_r2c_z COMMAND fft_r2c_z)

From cee0612bdfea41d512ecf63b8419a971767328d9 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 17:02:32 +0000
Subject: [PATCH 106/436] Add halo_test to ctest

---
 examples/CMakeLists.txt           | 1 +
 examples/halo_test/CMakeLists.txt | 7 +++++++
 2 files changed, 8 insertions(+)
 create mode 100644 examples/halo_test/CMakeLists.txt

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index cb1accb2..4fcf873d 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -2,3 +2,4 @@ add_subdirectory(init_test)
 add_subdirectory(test2d)
 add_subdirectory(fft_physical_x)
 add_subdirectory(fft_physical_z)
+add_subdirectory(halo_test)
diff --git a/examples/halo_test/CMakeLists.txt b/examples/halo_test/CMakeLists.txt
new file mode 100644
index 00000000..35e6bc31
--- /dev/null
+++ b/examples/halo_test/CMakeLists.txt
@@ -0,0 +1,7 @@
+file(GLOB files_test halo_test.f90)
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+add_executable(halo_test ${files_test})
+include_directories(${CMAKE_SOURCE_DIR}/src)
+target_link_libraries(halo_test PRIVATE decomp2d)
+add_test(NAME halo_test COMMAND halo_test)

From fc037fb6627bcf0b0a5b73febe6069ba45aa219b Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 17:06:47 +0000
Subject: [PATCH 107/436] Add io_test to ctest

---
 examples/CMakeLists.txt         |  1 +
 examples/io_test/CMakeLists.txt | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)
 create mode 100644 examples/io_test/CMakeLists.txt

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 4fcf873d..4241691c 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -3,3 +3,4 @@ add_subdirectory(test2d)
 add_subdirectory(fft_physical_x)
 add_subdirectory(fft_physical_z)
 add_subdirectory(halo_test)
+add_subdirectory(io_test)
diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
new file mode 100644
index 00000000..46ce0da9
--- /dev/null
+++ b/examples/io_test/CMakeLists.txt
@@ -0,0 +1,25 @@
+file(GLOB files_test io_test.f90)
+file(GLOB files_read io_read.f90)
+file(GLOB files_var_test io_var_test.f90)
+file(GLOB files_plane_test io_plane_test.f90)
+file(GLOB files_bench io_bench.f90)
+
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+add_executable(io_test ${files_test})
+add_executable(io_read ${files_read})
+add_executable(io_var_test ${files_var_test})
+add_executable(io_plane_test ${files_plane_test})
+add_executable(io_bench ${files_bench})
+
+target_link_libraries(io_test PRIVATE decomp2d)
+target_link_libraries(io_read PRIVATE decomp2d)
+target_link_libraries(io_var_test PRIVATE decomp2d)
+target_link_libraries(io_plane_test PRIVATE decomp2d)
+target_link_libraries(io_bench PRIVATE decomp2d)
+
+add_test(NAME io_test COMMAND io_test)
+add_test(NAME io_read COMMAND io_read)
+add_test(NAME io_var_test COMMAND io_var_test 0 0)
+add_test(NAME io_plane_test COMMAND io_plane_test)
+add_test(NAME io_bench COMMAND io_bench)

From 12c310d6078f9c807ca8c0f379c24d11cee42c33 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 17:08:09 +0000
Subject: [PATCH 108/436] Cleanup test CMakeLists

---
 examples/halo_test/CMakeLists.txt | 1 -
 examples/init_test/CMakeLists.txt | 1 -
 examples/test2d/CMakeLists.txt    | 1 -
 3 files changed, 3 deletions(-)

diff --git a/examples/halo_test/CMakeLists.txt b/examples/halo_test/CMakeLists.txt
index 35e6bc31..27a17add 100644
--- a/examples/halo_test/CMakeLists.txt
+++ b/examples/halo_test/CMakeLists.txt
@@ -2,6 +2,5 @@ file(GLOB files_test halo_test.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(halo_test ${files_test})
-include_directories(${CMAKE_SOURCE_DIR}/src)
 target_link_libraries(halo_test PRIVATE decomp2d)
 add_test(NAME halo_test COMMAND halo_test)
diff --git a/examples/init_test/CMakeLists.txt b/examples/init_test/CMakeLists.txt
index 7e361b50..be042304 100644
--- a/examples/init_test/CMakeLists.txt
+++ b/examples/init_test/CMakeLists.txt
@@ -2,6 +2,5 @@ file(GLOB files_test init_test.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(init_test ${files_test})
-include_directories(${CMAKE_SOURCE_DIR}/src)
 target_link_libraries(init_test PRIVATE decomp2d)
 add_test(NAME init_test COMMAND init_test)
diff --git a/examples/test2d/CMakeLists.txt b/examples/test2d/CMakeLists.txt
index 3ebf4821..d5be500a 100644
--- a/examples/test2d/CMakeLists.txt
+++ b/examples/test2d/CMakeLists.txt
@@ -2,6 +2,5 @@ file(GLOB files_test test2d.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(test2d ${files_test})
-include_directories(${CMAKE_SOURCE_DIR}/src)
 target_link_libraries(test2d PRIVATE decomp2d)
 add_test(NAME test2d COMMAND test2d)

From d3e9314f9cb5317a5ceba9c6da5d8596163368f1 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 22:23:53 +0000
Subject: [PATCH 109/436] Add DEV build to CMake (GNU only)

---
 CMakeLists.txt | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c278fdf9..b0ae61db 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -35,7 +35,7 @@ INCLUDE(CMakeParseArguments)
 # make sure that the default is a RELEASE
 if (NOT CMAKE_BUILD_TYPE)
   set (CMAKE_BUILD_TYPE RELEASE CACHE STRING
-      "Choose the type of build, options are: None Debug Release."
+      "Choose the type of build, options are: None Dev Debug Release."
       FORCE)
 endif (NOT CMAKE_BUILD_TYPE)
 
@@ -81,13 +81,14 @@ message(STATUS "Fortran compiler version ${CMAKE_Fortran_COMPILER_VERSION}")
 if (Fortran_COMPILER_NAME MATCHES "GNU")
   # gfortran
   message(STATUS "Setting gfortran flags")
-  set(CMAKE_Fortran_FLAGS "-cpp -funroll-loops -floop-optimize -g -Warray-bounds -fcray-pointer -fbacktrace -ffree-line-length-none")
+  set(CMAKE_Fortran_FLAGS "-cpp -std=f2008 -ffree-line-length-none")
   if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
     message(STATUS "Set New Fortran basic flags")
-    set(CMAKE_Fortran_FLAGS "-cpp -funroll-loops -floop-optimize -g -Warray-bounds -fcray-pointer -fbacktrace -ffree-line-length-none -fallow-argument-mismatch")
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch")
   endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
-  set(CMAKE_Fortran_FLAGS_RELEASE "-funroll-all-loops -fno-f2c -O3")
-  set(CMAKE_Fortran_FLAGS_DEBUG   "-DDEBG -O0 -g")
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -march=native")
+  set(CMAKE_Fortran_FLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none")
+  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
 elseif (Fortran_COMPILER_NAME MATCHES "Intel")
   message(STATUS "Setting ifort flags")
   set(CMAKE_Fortran_FLAGS "-fpp -xHost -heaparrays -safe-cray-ptr -g -traceback")

From f97e7b74c41b300ecf673af650a6a7bb5ae43118 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 22:32:37 +0000
Subject: [PATCH 110/436] Compile with Intel

---
 CMakeLists.txt | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b0ae61db..f96effc7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -91,9 +91,10 @@ if (Fortran_COMPILER_NAME MATCHES "GNU")
   set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
 elseif (Fortran_COMPILER_NAME MATCHES "Intel")
   message(STATUS "Setting ifort flags")
-  set(CMAKE_Fortran_FLAGS "-fpp -xHost -heaparrays -safe-cray-ptr -g -traceback")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-O3 -ipo")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g -DDEBG")
+  set(CMAKE_Fortran_FLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback")
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ipo")
+  set(CMAKE_Fortran_FLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG")
+  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -warn all,noexternal")
   #set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
 elseif (Fortran_COMPILER_NAME MATCHES "NAG")
   message(STATUS "Setting nagfor flags")

From 0899848a2596f85a2f7df178c50ca031e58628c4 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 22:32:46 +0000
Subject: [PATCH 111/436] Replace DEBG macro with DEBUG macro

---
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f96effc7..21ee9356 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -135,7 +135,7 @@ elseif (Fortran_COMPILER_NAME MATCHES "NVHPC")
     set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
   endif()
   set (CMAKE_Fortran_FLAGS_RELEASE "-fast -O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g -DDEBG")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g -DDEBUG")
 elseif (Fortran_COMPILER_NAME MATCHES "Fujitsu")
   message(STATUS "Setting Fujitsu fortran flags")
   set (CMAKE_Fortran_FLAGS "-Cpp")
@@ -150,7 +150,7 @@ else (Fortran_COMPILER_NAME MATCHES "GNU")
 endif (Fortran_COMPILER_NAME MATCHES "GNU")
 
 if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
-  add_definitions("-DDEBG")
+  add_definitions("-DDEBUG")
 endif (CMAKE_BUILD_TYPE MATCHES "DEBUG")
 
 

From 7536993d8bf8eb148c2b0fff2a4fcc297d299459 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 21 Jan 2023 22:54:03 +0000
Subject: [PATCH 112/436] Ensure GNU and Intel options appear in ccmake

---
 CMakeLists.txt | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 21ee9356..521fab93 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -81,20 +81,38 @@ message(STATUS "Fortran compiler version ${CMAKE_Fortran_COMPILER_VERSION}")
 if (Fortran_COMPILER_NAME MATCHES "GNU")
   # gfortran
   message(STATUS "Setting gfortran flags")
-  set(CMAKE_Fortran_FLAGS "-cpp -std=f2008 -ffree-line-length-none")
+  set(CMAKE_Fortran_FLAGS "-cpp -std=f2008 -ffree-line-length-none" CACHE STRING
+	  "Baseline FFLAGS"
+	  FORCE)
   if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
     message(STATUS "Set New Fortran basic flags")
-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch")
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch" CACHE STRING
+	    "Baseline FFLAGS"
+	    FORCE)
   endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
-  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -march=native")
-  set(CMAKE_Fortran_FLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none")
-  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -march=native" CACHE STRING
+	  "Release (optimised) FFLAGS"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none" CACHE STRING
+	  "Debugging FFLAGS - don't optimise, and print additional information"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division"
+	  CACHE STRING
+	  "Development build FFLAGS - in addition to debugging build, try and catch coding errors"
+	  FORCE)
 elseif (Fortran_COMPILER_NAME MATCHES "Intel")
   message(STATUS "Setting ifort flags")
-  set(CMAKE_Fortran_FLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback")
-  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ipo")
-  set(CMAKE_Fortran_FLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG")
-  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -warn all,noexternal")
+  set(CMAKE_Fortran_FLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback" CACHE STRING
+	  "Baseline FFLAGS"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ipo" CACHE STRING
+	  "Release (Optimised) FFLAGS"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG" CACHE STRING
+	  "Debugging build FFLAGS - don't optimise, and print additional information"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -warn all,noexternal" CACHE STRING
+	  "Development build FFLAGS - in addition to debugging build, try and catch coding errors")
   #set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
 elseif (Fortran_COMPILER_NAME MATCHES "NAG")
   message(STATUS "Setting nagfor flags")

From cfd5ee4b59ad6d178a7aee97cee29d33d475b175 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sun, 22 Jan 2023 13:15:31 +0000
Subject: [PATCH 113/436] Split CMakeLists.txt into separate files

Makes it easier to understand/modify
---
 CMakeLists.txt        | 200 +-----------------------------------------
 cmake/compilers.cmake | 121 +++++++++++++++++++++++++
 cmake/fft.cmake       |  31 +++++++
 cmake/gpu.cmake       |  37 ++++++++
 cmake/mpi.cmake       |  20 +++++
 5 files changed, 213 insertions(+), 196 deletions(-)
 create mode 100644 cmake/compilers.cmake
 create mode 100644 cmake/fft.cmake
 create mode 100644 cmake/gpu.cmake
 create mode 100644 cmake/mpi.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 521fab93..33b55cbb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -39,208 +39,16 @@ if (NOT CMAKE_BUILD_TYPE)
       FORCE)
 endif (NOT CMAKE_BUILD_TYPE)
 
-if (ENABLE_OPENACC)
-  include(FindOpenACC)
-  if(OpenACC_Fortran_FOUND)
-    message(STATUS "OpenACC for Fotran Compiler Found, version ${OpenACC_Fortran_VERSION_MAJOR}.${OpenACC_Fortran_VERSION_MINOR}")
-  else()
-    message(ERROR_CRITICAL "No OpenACC support detected")
-  endif()
-  set(OPENACC_TARGET "gpu" CACHE STRING "Target for acceleration (gpu (default) or multicore)")
-  set_property(CACHE OPENACC_TARGET PROPERTY STRINGS gpu multicore)
-endif()
-
-if (OPENACC_TARGET MATCHES "gpu")
-  find_package(CUDAToolkit REQUIRED)
-  if(${CMAKE_VERSION} VERSION_LESS_EQUAL "3.13.4")
-    cuda_select_nvcc_arch_flags(ARCH_FLAGS "Auto") # optional argument for arch to add
-    message(STATUS "ARCH_FLAGS = ${ARCH_FLAGS}")
-    string(REPLACE "-gencode;" "--generate-code=" ARCH_FLAGS "${ARCH_FLAGS}")
-    string(APPEND CMAKE_CUDA_FLAGS "${ARCH_FLAGS}")
-    message(STATUS "ARCH_FLAGS WITH CUDA = ${ARCH_FLAGS}")
-  else()
-    include(FindCUDA/select_compute_arch)
-    CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1)
-    string(STRIP "${INSTALLED_GPU_CCS_1}" INSTALLED_GPU_CCS_2)
-    string(REPLACE " " ";" INSTALLED_GPU_CCS_3 "${INSTALLED_GPU_CCS_2}")
-    string(REPLACE "." "" CUDA_ARCH_LIST "${INSTALLED_GPU_CCS_3}")
-    SET(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_LIST})
-    set_property(GLOBAL PROPERTY CUDA_ARCHITECTURES "${CUDA_ARCH_LIST}")
-    message(STATUS "CUDA_ARCHITECTURES ${CUDA_ARCH_LIST}")
-    list(GET CUDA_ARCH_LIST 0 CUDA_ARCH_FIRST)
-    message(STATUS "CUDA_ARCH_FIRST ${CUDA_ARCH_FIRST}")
-  endif()
-  message(STATUS "CUDA_LIBRARIES ${CUDA_LIBRARIES}")
-endif()
-
-set(NVIDIA_PROFILER "Use the NVidia profiles" ON)
-set(Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER_ID} )
-message(STATUS "COMP ID ${Fortran_COMPILER_NAME}")
-message(STATUS "Fortran compiler name ${Fortran_COMPILER_NAME}")
-message(STATUS "Fortran compiler version ${CMAKE_Fortran_COMPILER_VERSION}")
-if (Fortran_COMPILER_NAME MATCHES "GNU")
-  # gfortran
-  message(STATUS "Setting gfortran flags")
-  set(CMAKE_Fortran_FLAGS "-cpp -std=f2008 -ffree-line-length-none" CACHE STRING
-	  "Baseline FFLAGS"
-	  FORCE)
-  if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
-    message(STATUS "Set New Fortran basic flags")
-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch" CACHE STRING
-	    "Baseline FFLAGS"
-	    FORCE)
-  endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
-  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -march=native" CACHE STRING
-	  "Release (optimised) FFLAGS"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none" CACHE STRING
-	  "Debugging FFLAGS - don't optimise, and print additional information"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division"
-	  CACHE STRING
-	  "Development build FFLAGS - in addition to debugging build, try and catch coding errors"
-	  FORCE)
-elseif (Fortran_COMPILER_NAME MATCHES "Intel")
-  message(STATUS "Setting ifort flags")
-  set(CMAKE_Fortran_FLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback" CACHE STRING
-	  "Baseline FFLAGS"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ipo" CACHE STRING
-	  "Release (Optimised) FFLAGS"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG" CACHE STRING
-	  "Debugging build FFLAGS - don't optimise, and print additional information"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -warn all,noexternal" CACHE STRING
-	  "Development build FFLAGS - in addition to debugging build, try and catch coding errors")
-  #set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
-elseif (Fortran_COMPILER_NAME MATCHES "NAG")
-  message(STATUS "Setting nagfor flags")
-  set(CMAKE_Fortran_FLAGS "-fpp")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
-elseif (Fortran_COMPILER_NAME MATCHES "Cray")
-  message(STATUS "Setting cray fortran flags")
-  set(CMAKE_Fortran_FLAGS "-eF -g -N 1023")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
-elseif (Fortran_COMPILER_NAME MATCHES "PGI")
-  message(STATUS "Setting PGI fortran flags")
-  if (ENABLE_OPENACC) 
-    if (OPENACC_TARGET MATCHES "gpu")
-      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=gpu")
-    else(OPENACC_TARGET MATCHES "gpu")
-      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=multicore")
-    endif(OPENACC_TARGET MATCHES "gpu")
-  else()
-    set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
-  endif()
-  #set(CMAKE_Fortran_FLAGS "-fast -cpp -Mfree -Kieee -Minfo=accel -g ")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-fast -O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0")
-elseif (Fortran_COMPILER_NAME MATCHES "NVHPC")
-  message(STATUS "Setting NVHPC fortran flags")
-  if (ENABLE_OPENACC) 
-    if (OPENACC_TARGET MATCHES "gpu")
-      #set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar=gpu -acc -target=gpu -Minstrument")
-      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp -Mfree -Kieee -Minfo=accel -stdpar=gpu -gpu=cc${CUDA_ARCH_FIRST},managed -acc -target=gpu")
-      #add_definitions("-lnvhpcwrapnvtx")
-      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lnvhpcwrapnvtx")
-    else()
-      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar -acc -target=multicore")
-    endif(OPENACC_TARGET MATCHES "gpu")
-  else()
-    set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
-  endif()
-  set (CMAKE_Fortran_FLAGS_RELEASE "-fast -O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g -DDEBUG")
-elseif (Fortran_COMPILER_NAME MATCHES "Fujitsu")
-  message(STATUS "Setting Fujitsu fortran flags")
-  set (CMAKE_Fortran_FLAGS "-Cpp")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG "-O0")
-else (Fortran_COMPILER_NAME MATCHES "GNU")
-  message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
-  message ("Fortran compiler: " ${Fortran_COMPILER_NAME})
-  message ("No optimized Fortran compiler flags are known, we just try -O2...")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-O2")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
-endif (Fortran_COMPILER_NAME MATCHES "GNU")
-
-if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
-  add_definitions("-DDEBUG")
-endif (CMAKE_BUILD_TYPE MATCHES "DEBUG")
-
+include(${CMAKE_SOURCE_DIR}/cmake/gpu.cmake)
 
-find_package(MPI REQUIRED)
-# Stop if there is no MPI_Fortran_Compiler
-if (MPI_Fortran_COMPILER)
-    message(STATUS "MPI_Fortran_COMPILER found: ${MPI_Fortran_COMPILER}")
-else (MPI_Fortran_COMPILER)
-    message(SEND_ERROR "This application cannot compile without MPI")
-endif(MPI_Fortran_COMPILER)
-# Warning if Include are not found => can be fixed with more recent cmake version
-if (MPI_FOUND)
-    message(STATUS "MPI FOUND: ${MPI_FOUND}")
-    include_directories(SYSTEM ${MPI_INCLUDE_PATH})
-    message(STATUS "MPI INCL ALSO FOUND: ${MPI_INCLUDE_PATH}")
-else (MPI_FOUND)
-    message(STATUS "NO MPI include have been found. The executable won't be targeted with MPI include")
-    message(STATUS "Code will compile but performaces can be compromised")
-    message(STATUS "Using a CMake vers > 3.10 should solve the problem")
-    message(STATUS "Alternatively use ccmake to manually set the include if available")
-endif (MPI_FOUND)
+include(${CMAKE_SOURCE_DIR}/cmake/compilers.cmake)
 
-execute_process(
-  COMMAND git describe --tag --long --always
-  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-  OUTPUT_VARIABLE GIT_VERSION
-  OUTPUT_STRIP_TRAILING_WHITESPACE
-)
-add_definitions("-DVERSION=\"${GIT_VERSION}\"")
-option(DOUBLE_PRECISION "Build Xcompact with double precision" ON)
-if (DOUBLE_PRECISION)
-  add_definitions("-DDOUBLE_PREC")
-endif()
-
-option(SINGLE_PRECISION_OUTPUT "Build XCompact with output in single precision" OFF)
-if (SINGLE_PRECISION_OUTPUT)
-  add_definitions("-DSAVE_SINGLE")
-endif()
+include(${CMAKE_SOURCE_DIR}/cmake/mpi.cmake)
 
 # FFT options
 set(FFT_Choice "generic" CACHE STRING "FFT for XCompact3d project (generic is the default)")
 set_property(CACHE FFT_Choice PROPERTY STRINGS generic fftw mkl)
-
-# Look for fftw is required
-if(${FFT_Choice} MATCHES "fftw")
-  configure_file(cmake/downloadFindFFTW.cmake.in findFFTW-download/CMakeLists.txt)
-  execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
-          RESULT_VARIABLE result
-          WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-download )
-  if(result)
-      message(FATAL_ERROR "CMake step for findFFTW failed: ${result}")
-      else()
-      message("CMake step for findFFTW completed (${result}).")
-  endif()
-  execute_process(COMMAND ${CMAKE_COMMAND} --build .
-          RESULT_VARIABLE result
-          WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-download )
-  if(result)
-      message(FATAL_ERROR "Build step for findFFTW failed: ${result}")
-  endif()
-  
-  set(findFFTW_DIR ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-src)
-  
-  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${findFFTW_DIR}")
-  find_package(FFTW)
-  message(STATUS "FFTW_FOUND     : ${FFTW_FOUND}")
-  message(STATUS "FFTW_LIBRARIES : ${FFTW_LIBRARIES}")
-  message(STATUS "FFTW_INCLUDE   : ${FFTW_INCLUDE_DIRS}")
-
-  #add_definitions("-lfftw3 -lfftw3f")
-
-endif(${FFT_Choice} MATCHES "fftw")
+include(${CMAKE_SOURCE_DIR}/cmake/fft.cmake)
 
 # Create a static library for the fft
 add_subdirectory(src)
diff --git a/cmake/compilers.cmake b/cmake/compilers.cmake
new file mode 100644
index 00000000..3d579301
--- /dev/null
+++ b/cmake/compilers.cmake
@@ -0,0 +1,121 @@
+# Compilers CMakeLists
+
+set(Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER_ID} )
+message(STATUS "COMP ID ${Fortran_COMPILER_NAME}")
+message(STATUS "Fortran compiler name ${Fortran_COMPILER_NAME}")
+message(STATUS "Fortran compiler version ${CMAKE_Fortran_COMPILER_VERSION}")
+
+if (Fortran_COMPILER_NAME MATCHES "GNU")
+  # gfortran
+  message(STATUS "Setting gfortran flags")
+  set(CMAKE_Fortran_FLAGS "-cpp -std=f2008 -ffree-line-length-none" CACHE STRING
+	  "Baseline FFLAGS"
+	  FORCE)
+  if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
+    message(STATUS "Set New Fortran basic flags")
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch" CACHE STRING
+	    "Baseline FFLAGS"
+	    FORCE)
+  endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -march=native" CACHE STRING
+	  "Release (optimised) FFLAGS"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none" CACHE STRING
+	  "Debugging FFLAGS - don't optimise, and print additional information"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division"
+	  CACHE STRING
+	  "Development build FFLAGS - in addition to debugging build, try and catch coding errors"
+	  FORCE)
+elseif (Fortran_COMPILER_NAME MATCHES "Intel")
+  message(STATUS "Setting ifort flags")
+  set(CMAKE_Fortran_FLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback" CACHE STRING
+	  "Baseline FFLAGS"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ipo" CACHE STRING
+	  "Release (Optimised) FFLAGS"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG" CACHE STRING
+	  "Debugging build FFLAGS - don't optimise, and print additional information"
+	  FORCE)
+  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -warn all,noexternal" CACHE STRING
+	  "Development build FFLAGS - in addition to debugging build, try and catch coding errors")
+  #set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
+elseif (Fortran_COMPILER_NAME MATCHES "NAG")
+  message(STATUS "Setting nagfor flags")
+  set(CMAKE_Fortran_FLAGS "-fpp")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
+elseif (Fortran_COMPILER_NAME MATCHES "Cray")
+  message(STATUS "Setting cray fortran flags")
+  set(CMAKE_Fortran_FLAGS "-eF -g -N 1023")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
+elseif (Fortran_COMPILER_NAME MATCHES "PGI")
+  message(STATUS "Setting PGI fortran flags")
+  if (ENABLE_OPENACC) 
+    if (OPENACC_TARGET MATCHES "gpu")
+      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=gpu")
+    else(OPENACC_TARGET MATCHES "gpu")
+      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=multicore")
+    endif(OPENACC_TARGET MATCHES "gpu")
+  else()
+    set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
+  endif()
+  #set(CMAKE_Fortran_FLAGS "-fast -cpp -Mfree -Kieee -Minfo=accel -g ")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-fast -O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0")
+elseif (Fortran_COMPILER_NAME MATCHES "NVHPC")
+  message(STATUS "Setting NVHPC fortran flags")
+  if (ENABLE_OPENACC) 
+    if (OPENACC_TARGET MATCHES "gpu")
+      #set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar=gpu -acc -target=gpu -Minstrument")
+      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp -Mfree -Kieee -Minfo=accel -stdpar=gpu -gpu=cc${CUDA_ARCH_FIRST},managed -acc -target=gpu")
+      #add_definitions("-lnvhpcwrapnvtx")
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lnvhpcwrapnvtx")
+    else()
+      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar -acc -target=multicore")
+    endif(OPENACC_TARGET MATCHES "gpu")
+  else()
+    set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
+  endif()
+  set (CMAKE_Fortran_FLAGS_RELEASE "-fast -O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g -DDEBUG")
+elseif (Fortran_COMPILER_NAME MATCHES "Fujitsu")
+  message(STATUS "Setting Fujitsu fortran flags")
+  set (CMAKE_Fortran_FLAGS "-Cpp")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
+  set (CMAKE_Fortran_FLAGS_DEBUG "-O0")
+# elseif (Fortran_COMPILER_NAME MATCHES "Flang")
+#   message(STATUS "Setting Flang flags")
+#   set(CMAKE_Fortran_FLAGS "-cpp -std=f2008" CACHE STRING
+# 	  "Baseline FFLAGS"
+# 	  FORCE)
+else (Fortran_COMPILER_NAME MATCHES "GNU")
+  message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
+  message ("Fortran compiler: " ${Fortran_COMPILER_NAME})
+  message ("No optimized Fortran compiler flags are known, we just try -O2...")
+  set (CMAKE_Fortran_FLAGS_RELEASE "-O2")
+  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
+endif (Fortran_COMPILER_NAME MATCHES "GNU")
+
+if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
+  add_definitions("-DDEBUG")
+endif (CMAKE_BUILD_TYPE MATCHES "DEBUG")
+
+execute_process(
+  COMMAND git describe --tag --long --always
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  OUTPUT_VARIABLE GIT_VERSION
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+add_definitions("-DVERSION=\"${GIT_VERSION}\"")
+option(DOUBLE_PRECISION "Build Xcompact with double precision" ON)
+if (DOUBLE_PRECISION)
+  add_definitions("-DDOUBLE_PREC")
+endif()
+
+option(SINGLE_PRECISION_OUTPUT "Build XCompact with output in single precision" OFF)
+if (SINGLE_PRECISION_OUTPUT)
+  add_definitions("-DSAVE_SINGLE")
+endif()
diff --git a/cmake/fft.cmake b/cmake/fft.cmake
new file mode 100644
index 00000000..4566d6e7
--- /dev/null
+++ b/cmake/fft.cmake
@@ -0,0 +1,31 @@
+# FFT CMakeLists
+
+# Look for fftw if required
+if(${FFT_Choice} MATCHES "fftw")
+  configure_file(cmake/downloadFindFFTW.cmake.in findFFTW-download/CMakeLists.txt)
+  execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+          RESULT_VARIABLE result
+          WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-download )
+  if(result)
+      message(FATAL_ERROR "CMake step for findFFTW failed: ${result}")
+      else()
+      message("CMake step for findFFTW completed (${result}).")
+  endif()
+  execute_process(COMMAND ${CMAKE_COMMAND} --build .
+          RESULT_VARIABLE result
+          WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-download )
+  if(result)
+      message(FATAL_ERROR "Build step for findFFTW failed: ${result}")
+  endif()
+  
+  set(findFFTW_DIR ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-src)
+  
+  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${findFFTW_DIR}")
+  find_package(FFTW)
+  message(STATUS "FFTW_FOUND     : ${FFTW_FOUND}")
+  message(STATUS "FFTW_LIBRARIES : ${FFTW_LIBRARIES}")
+  message(STATUS "FFTW_INCLUDE   : ${FFTW_INCLUDE_DIRS}")
+
+  #add_definitions("-lfftw3 -lfftw3f")
+
+endif(${FFT_Choice} MATCHES "fftw")
diff --git a/cmake/gpu.cmake b/cmake/gpu.cmake
new file mode 100644
index 00000000..858b3a1a
--- /dev/null
+++ b/cmake/gpu.cmake
@@ -0,0 +1,37 @@
+# GPU CMakeLists
+
+if (ENABLE_OPENACC)
+  include(FindOpenACC)
+  if(OpenACC_Fortran_FOUND)
+    message(STATUS "OpenACC for Fotran Compiler Found, version ${OpenACC_Fortran_VERSION_MAJOR}.${OpenACC_Fortran_VERSION_MINOR}")
+  else()
+    message(ERROR_CRITICAL "No OpenACC support detected")
+  endif()
+  set(OPENACC_TARGET "gpu" CACHE STRING "Target for acceleration (gpu (default) or multicore)")
+  set_property(CACHE OPENACC_TARGET PROPERTY STRINGS gpu multicore)
+endif()
+
+if (OPENACC_TARGET MATCHES "gpu")
+  find_package(CUDAToolkit REQUIRED)
+  if(${CMAKE_VERSION} VERSION_LESS_EQUAL "3.13.4")
+    cuda_select_nvcc_arch_flags(ARCH_FLAGS "Auto") # optional argument for arch to add
+    message(STATUS "ARCH_FLAGS = ${ARCH_FLAGS}")
+    string(REPLACE "-gencode;" "--generate-code=" ARCH_FLAGS "${ARCH_FLAGS}")
+    string(APPEND CMAKE_CUDA_FLAGS "${ARCH_FLAGS}")
+    message(STATUS "ARCH_FLAGS WITH CUDA = ${ARCH_FLAGS}")
+  else()
+    include(FindCUDA/select_compute_arch)
+    CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1)
+    string(STRIP "${INSTALLED_GPU_CCS_1}" INSTALLED_GPU_CCS_2)
+    string(REPLACE " " ";" INSTALLED_GPU_CCS_3 "${INSTALLED_GPU_CCS_2}")
+    string(REPLACE "." "" CUDA_ARCH_LIST "${INSTALLED_GPU_CCS_3}")
+    SET(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_LIST})
+    set_property(GLOBAL PROPERTY CUDA_ARCHITECTURES "${CUDA_ARCH_LIST}")
+    message(STATUS "CUDA_ARCHITECTURES ${CUDA_ARCH_LIST}")
+    list(GET CUDA_ARCH_LIST 0 CUDA_ARCH_FIRST)
+    message(STATUS "CUDA_ARCH_FIRST ${CUDA_ARCH_FIRST}")
+  endif()
+  message(STATUS "CUDA_LIBRARIES ${CUDA_LIBRARIES}")
+endif()
+
+set(NVIDIA_PROFILER "Use the NVidia profiles" ON)
diff --git a/cmake/mpi.cmake b/cmake/mpi.cmake
new file mode 100644
index 00000000..64ebba07
--- /dev/null
+++ b/cmake/mpi.cmake
@@ -0,0 +1,20 @@
+# MPI CMakeLists
+
+find_package(MPI REQUIRED)
+# Stop if there is no MPI_Fortran_Compiler
+if (MPI_Fortran_COMPILER)
+    message(STATUS "MPI_Fortran_COMPILER found: ${MPI_Fortran_COMPILER}")
+else (MPI_Fortran_COMPILER)
+    message(SEND_ERROR "This application cannot compile without MPI")
+endif(MPI_Fortran_COMPILER)
+# Warning if Include are not found => can be fixed with more recent cmake version
+if (MPI_FOUND)
+    message(STATUS "MPI FOUND: ${MPI_FOUND}")
+    include_directories(SYSTEM ${MPI_INCLUDE_PATH})
+    message(STATUS "MPI INCL ALSO FOUND: ${MPI_INCLUDE_PATH}")
+else (MPI_FOUND)
+    message(STATUS "NO MPI include have been found. The executable won't be targeted with MPI include")
+    message(STATUS "Code will compile but performaces can be compromised")
+    message(STATUS "Using a CMake vers > 3.10 should solve the problem")
+    message(STATUS "Alternatively use ccmake to manually set the include if available")
+endif (MPI_FOUND)

From dfe5a4d231bf4fa3f3252073f78b3c8916463311 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sun, 22 Jan 2023 14:55:39 +0000
Subject: [PATCH 114/436] Cleanup compilers.cmake

---
 cmake/compilers.cmake | 91 ++++++++++++++++++++-----------------------
 1 file changed, 43 insertions(+), 48 deletions(-)

diff --git a/cmake/compilers.cmake b/cmake/compilers.cmake
index 3d579301..d1e93045 100644
--- a/cmake/compilers.cmake
+++ b/cmake/compilers.cmake
@@ -8,84 +8,66 @@ message(STATUS "Fortran compiler version ${CMAKE_Fortran_COMPILER_VERSION}")
 if (Fortran_COMPILER_NAME MATCHES "GNU")
   # gfortran
   message(STATUS "Setting gfortran flags")
-  set(CMAKE_Fortran_FLAGS "-cpp -std=f2008 -ffree-line-length-none" CACHE STRING
-	  "Baseline FFLAGS"
-	  FORCE)
+  set(D2D_FFLAGS "-cpp -std=f2008 -ffree-line-length-none")
   if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
     message(STATUS "Set New Fortran basic flags")
-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch" CACHE STRING
-	    "Baseline FFLAGS"
-	    FORCE)
+    set(D2D_FFLAGS "${D2D_FLAGS} -fallow-argument-mismatch")
   endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
-  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -march=native" CACHE STRING
-	  "Release (optimised) FFLAGS"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none" CACHE STRING
-	  "Debugging FFLAGS - don't optimise, and print additional information"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division"
-	  CACHE STRING
-	  "Development build FFLAGS - in addition to debugging build, try and catch coding errors"
-	  FORCE)
+  set(D2D_FFLAGS_RELEASE "-O3 -march=native")
+  set(D2D_FFLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none")
+  set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
 elseif (Fortran_COMPILER_NAME MATCHES "Intel")
   message(STATUS "Setting ifort flags")
-  set(CMAKE_Fortran_FLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback" CACHE STRING
-	  "Baseline FFLAGS"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ipo" CACHE STRING
-	  "Release (Optimised) FFLAGS"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG" CACHE STRING
-	  "Debugging build FFLAGS - don't optimise, and print additional information"
-	  FORCE)
-  set(CMAKE_Fortran_FLAGS_DEV     "${CMAKE_Fortran_FLAGS_DEBUG} -warn all,noexternal" CACHE STRING
-	  "Development build FFLAGS - in addition to debugging build, try and catch coding errors")
+  set(D2D_FFLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback")
+  set(D2D_FFLAGS_RELEASE "-O3 -ipo")
+  set(D2D_FFLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG")
+  set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -warn all,noexternal")
   #set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
 elseif (Fortran_COMPILER_NAME MATCHES "NAG")
   message(STATUS "Setting nagfor flags")
-  set(CMAKE_Fortran_FLAGS "-fpp")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
+  set(D2D_FFLAGS "-fpp")
+  set(D2D_FFLAGS_RELEASE "-O3")
+  set(D2D_FFLAGS_DEBUG   "-O0 -g")
 elseif (Fortran_COMPILER_NAME MATCHES "Cray")
   message(STATUS "Setting cray fortran flags")
-  set(CMAKE_Fortran_FLAGS "-eF -g -N 1023")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
+  set(D2D_FFLAGS "-eF -g -N 1023")
+  set(D2D_FFLAGS_RELEASE "-O3")
+  set(D2D_FFLAGS_DEBUG   "-O0 -g")
 elseif (Fortran_COMPILER_NAME MATCHES "PGI")
   message(STATUS "Setting PGI fortran flags")
   if (ENABLE_OPENACC) 
     if (OPENACC_TARGET MATCHES "gpu")
-      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=gpu")
+      set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=gpu")
     else(OPENACC_TARGET MATCHES "gpu")
-      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=multicore")
+      set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=multicore")
     endif(OPENACC_TARGET MATCHES "gpu")
   else()
-    set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
+    set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
   endif()
   #set(CMAKE_Fortran_FLAGS "-fast -cpp -Mfree -Kieee -Minfo=accel -g ")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-fast -O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0")
+  set(D2D_FFLAGS_RELEASE "-fast -O3")
+  set(D2D_FFLAGS_DEBUG   "-O0")
 elseif (Fortran_COMPILER_NAME MATCHES "NVHPC")
   message(STATUS "Setting NVHPC fortran flags")
   if (ENABLE_OPENACC) 
     if (OPENACC_TARGET MATCHES "gpu")
       #set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar=gpu -acc -target=gpu -Minstrument")
-      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp -Mfree -Kieee -Minfo=accel -stdpar=gpu -gpu=cc${CUDA_ARCH_FIRST},managed -acc -target=gpu")
+      set(D2D_FFLAGS "${CMAKE_Fortran_FLAGS} -cpp -Mfree -Kieee -Minfo=accel -stdpar=gpu -gpu=cc${CUDA_ARCH_FIRST},managed -acc -target=gpu")
       #add_definitions("-lnvhpcwrapnvtx")
       set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lnvhpcwrapnvtx")
     else()
-      set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar -acc -target=multicore")
+      set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar -acc -target=multicore")
     endif(OPENACC_TARGET MATCHES "gpu")
   else()
-    set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
+    set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
   endif()
-  set (CMAKE_Fortran_FLAGS_RELEASE "-fast -O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g -DDEBUG")
+  set(D2D_FFLAGS_RELEASE "-fast -O3")
+  set(D2D_FFLAGS_DEBUG   "-O0 -g -DDEBUG")
 elseif (Fortran_COMPILER_NAME MATCHES "Fujitsu")
   message(STATUS "Setting Fujitsu fortran flags")
-  set (CMAKE_Fortran_FLAGS "-Cpp")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-O3")
-  set (CMAKE_Fortran_FLAGS_DEBUG "-O0")
+  set(D2D_FFLAGS "-Cpp")
+  set(D2D_FFLAGS_RELEASE "-O3")
+  set(D2D_FFLAGS_DEBUG   "-O0")
 # elseif (Fortran_COMPILER_NAME MATCHES "Flang")
 #   message(STATUS "Setting Flang flags")
 #   set(CMAKE_Fortran_FLAGS "-cpp -std=f2008" CACHE STRING
@@ -95,10 +77,23 @@ else (Fortran_COMPILER_NAME MATCHES "GNU")
   message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
   message ("Fortran compiler: " ${Fortran_COMPILER_NAME})
   message ("No optimized Fortran compiler flags are known, we just try -O2...")
-  set (CMAKE_Fortran_FLAGS_RELEASE "-O2")
-  set (CMAKE_Fortran_FLAGS_DEBUG   "-O0 -g")
+  set(D2D_FFLAGS_RELEASE "-O2")
+  set(D2D_FFLAGS_DEBUG   "-O0 -g")
 endif (Fortran_COMPILER_NAME MATCHES "GNU")
 
+set(CMAKE_Fortran_FLAGS ${D2D_FFLAGS} CACHE STRING
+  "Baseline compiler FFLAGS"
+  FORCE)
+set(CMAKE_Fortran_FLAGS_RELEASE ${D2D_FFLAGS_RELEASE} CACHE STRING
+  "Additional FFLAGS for Release (optimised) build"
+  FORCE)
+set(CMAKE_Fortran_FLAGS_DEBUG ${D2D_FFLAGS_DEBUG} CACHE STRING
+  "Additional FFLAGS for Debug build"
+  FORCE)
+set(CMAKE_Fortran_FLAGS_DEV ${D2D_FFLAGS_DEV} CACHE STRING
+  "Additional FFLAGS for Dev build"
+  FORCE)
+
 if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
   add_definitions("-DDEBUG")
 endif (CMAKE_BUILD_TYPE MATCHES "DEBUG")

From d90c6656953d2b07f355c49a28c180ba30d8ffee Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sun, 22 Jan 2023 16:36:29 +0000
Subject: [PATCH 115/436] Enable linking MKL via CMake

Based on
https://www.intel.com/content/www/us/en/develop/documentation/onemkl-macos-developer-guide/top/getting-started/cmake-config-for-onemkl.html

Currently failing tests when built against MKL
---
 cmake/fft.cmake    | 2 ++
 src/CMakeLists.txt | 7 +++++++
 2 files changed, 9 insertions(+)

diff --git a/cmake/fft.cmake b/cmake/fft.cmake
index 4566d6e7..0edeb97a 100644
--- a/cmake/fft.cmake
+++ b/cmake/fft.cmake
@@ -28,4 +28,6 @@ if(${FFT_Choice} MATCHES "fftw")
 
   #add_definitions("-lfftw3 -lfftw3f")
 
+elseif(${FFT_Choice} MATCHES "mkl")
+  find_package(MKL CONFIG REQUIRED)
 endif(${FFT_Choice} MATCHES "fftw")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 4fc3e338..1bfb5a85 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -13,6 +13,9 @@ include_directories(${CMAKE_SOURCE_DIR}/mod)
 if(FFTW_FOUND)
   message (STATUS "Compiling using FFTW3")
   file(GLOB files_fft fft_fftw3.f90)
+elseif(MKL_FOUND)
+  message (STATUS "Compiling using MKL")
+  file(GLOB files_fft $ENV{MKLROOT}/include/mkl_dfti.f90 fft_mkl.f90)
 else(FFTW_FOUND)
   message (STATUS "Compiling using Generic FFT")
   file(GLOB files_fft glassman.f90 fft_generic.f90)
@@ -29,6 +32,10 @@ endif (MPI_FOUND)
 if(FFTW_FOUND)
   target_include_directories(decomp2d PRIVATE ${FFTW_INCLUDE_DIRS})
   #target_include_directories(decomp2d PRIVATE ${PKG_FFTW_LIBRARY_DIRS})
+elseif(MKL_FOUND)
+  target_compile_options(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
+  target_include_directories(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
+  target_link_libraries(decomp2d PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif(FFTW_FOUND)
 
 install(TARGETS decomp2d

From adc7f1d98ca110e27de2fd8e892c33ddb768c2e5 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 23 Jan 2023 05:34:53 +0000
Subject: [PATCH 116/436] Remove OCC macro (#140)

---
 src/decomp_2d.f90 | 15 ---------------
 src/log.f90       |  3 ---
 2 files changed, 18 deletions(-)

diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 8b429cb3..63268caa 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -1231,21 +1231,6 @@ subroutine prepare_buffer(decomp)
       return
    end subroutine prepare_buffer
 
-#ifdef OCC
-   ! For non-blocking communication code, progress the comminication stack
-   subroutine transpose_test(handle)
-
-      implicit none
-
-      integer :: handle, ierror
-
-      call NBC_TEST(handle, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "NBC_TEST")
-
-      return
-   end subroutine transpose_test
-#endif
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Transposition routines
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/log.f90 b/src/log.f90
index e0799754..fa9eb90c 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -180,9 +180,6 @@ module subroutine d2d_listing(given_io_unit)
 #ifdef EVEN
       write (io_unit, *) 'Compile flag EVEN detected'
 #endif
-#ifdef OCC
-      write (io_unit, *) 'Compile flag OCC detected'
-#endif
 #ifdef OVERWRITE
       write (io_unit, *) 'Compile flag OVERWRITE detected'
 #endif

From 43fdc94549b5661e475e8d50a10fe4ace1351e86 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 24 Jan 2023 20:56:00 +0000
Subject: [PATCH 117/436] Port across findFFTW

---
 CMakeLists.txt                      |  2 +-
 cmake/fft/downloadFindFFTW.cmake.in | 16 ++++++++++++++++
 cmake/{ => fft}/fft.cmake           |  2 +-
 src/CMakeLists.txt                  |  5 ++++-
 4 files changed, 22 insertions(+), 3 deletions(-)
 create mode 100644 cmake/fft/downloadFindFFTW.cmake.in
 rename cmake/{ => fft}/fft.cmake (91%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 33b55cbb..62a64da1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,7 +48,7 @@ include(${CMAKE_SOURCE_DIR}/cmake/mpi.cmake)
 # FFT options
 set(FFT_Choice "generic" CACHE STRING "FFT for XCompact3d project (generic is the default)")
 set_property(CACHE FFT_Choice PROPERTY STRINGS generic fftw mkl)
-include(${CMAKE_SOURCE_DIR}/cmake/fft.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/fft/fft.cmake)
 
 # Create a static library for the fft
 add_subdirectory(src)
diff --git a/cmake/fft/downloadFindFFTW.cmake.in b/cmake/fft/downloadFindFFTW.cmake.in
new file mode 100644
index 00000000..2df1059b
--- /dev/null
+++ b/cmake/fft/downloadFindFFTW.cmake.in
@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.0.2)
+
+project(findFFTW-download NONE)
+
+include(ExternalProject)
+
+ExternalProject_Add(findFFTW_download
+    GIT_REPOSITORY    "https://github.com/egpbos/findfftw.git"
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND     ""
+    INSTALL_COMMAND   ""
+    TEST_COMMAND      ""
+    SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/findFFTW-src"
+    BINARY_DIR        ""
+    INSTALL_DIR       ""
+)
diff --git a/cmake/fft.cmake b/cmake/fft/fft.cmake
similarity index 91%
rename from cmake/fft.cmake
rename to cmake/fft/fft.cmake
index 0edeb97a..a555f73d 100644
--- a/cmake/fft.cmake
+++ b/cmake/fft/fft.cmake
@@ -2,7 +2,7 @@
 
 # Look for fftw if required
 if(${FFT_Choice} MATCHES "fftw")
-  configure_file(cmake/downloadFindFFTW.cmake.in findFFTW-download/CMakeLists.txt)
+	configure_file(${CMAKE_SOURCE_DIR}/cmake/fft/downloadFindFFTW.cmake.in findFFTW-download/CMakeLists.txt)
   execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
           RESULT_VARIABLE result
           WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-download )
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 1bfb5a85..c864c514 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -30,8 +30,11 @@ if (MPI_FOUND)
 endif (MPI_FOUND)
 
 if(FFTW_FOUND)
+  message(STATUS "FFTW_DOUBLE     : ${FFTW_DOUBLE_LIB}")
+  message(STATUS "FFTW_SINGLE     : ${FFTW_FLOAT_LIB}")
   target_include_directories(decomp2d PRIVATE ${FFTW_INCLUDE_DIRS})
-  #target_include_directories(decomp2d PRIVATE ${PKG_FFTW_LIBRARY_DIRS})
+  target_link_libraries(decomp2d PRIVATE ${FFTW_DOUBLE_LIB})
+  target_link_libraries(decomp2d PRIVATE ${FFTW_FLOAT_LIB})
 elseif(MKL_FOUND)
   target_compile_options(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
   target_include_directories(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)

From 4ad3120fd9989b2ff88f6f4b05137c82fa429733 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 24 Jan 2023 21:00:44 +0000
Subject: [PATCH 118/436] Add License headers for findFFTW

---
 cmake/fft/downloadFindFFTW.cmake.in | 31 +++++++++++++++++++++++++++++
 cmake/fft/fft.cmake                 | 30 ++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)

diff --git a/cmake/fft/downloadFindFFTW.cmake.in b/cmake/fft/downloadFindFFTW.cmake.in
index 2df1059b..28b2b4bb 100644
--- a/cmake/fft/downloadFindFFTW.cmake.in
+++ b/cmake/fft/downloadFindFFTW.cmake.in
@@ -1,3 +1,34 @@
+# downloadFindFFTW.cmake.in
+#
+# From the README at https://github.com/egpbos/findFFTW
+#
+# Copyright (c) 2015, Wenzel Jakob; 2017, Patrick Bos
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 cmake_minimum_required(VERSION 3.0.2)
 
 project(findFFTW-download NONE)
diff --git a/cmake/fft/fft.cmake b/cmake/fft/fft.cmake
index a555f73d..8b5bee95 100644
--- a/cmake/fft/fft.cmake
+++ b/cmake/fft/fft.cmake
@@ -1,6 +1,36 @@
 # FFT CMakeLists
 
 # Look for fftw if required
+# The download findFFTW code is based on the README at
+# https://github.com/egpbos/findFFTW
+#
+# Copyright (c) 2015, Wenzel Jakob; 2017, Patrick Bos
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 if(${FFT_Choice} MATCHES "fftw")
 	configure_file(${CMAKE_SOURCE_DIR}/cmake/fft/downloadFindFFTW.cmake.in findFFTW-download/CMakeLists.txt)
   execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .

From a6bf5078e8aeda194cc42c414e0131e86f7160d1 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 24 Jan 2023 22:10:16 +0000
Subject: [PATCH 119/436] Link FFTW to the fft examples, not decomp2d

---
 examples/fft_physical_x/CMakeLists.txt | 8 ++++++++
 examples/fft_physical_z/CMakeLists.txt | 6 ++++++
 src/CMakeLists.txt                     | 2 --
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index b2dfd4f0..01a9ed10 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -11,6 +11,14 @@ add_executable(fft_grid_x ${files_fft_grid})
 target_link_libraries(fft_c2c_x PRIVATE decomp2d)
 target_link_libraries(fft_r2c_x PRIVATE decomp2d)
 target_link_libraries(fft_grid_x PRIVATE decomp2d)
+if (FFTW_FOUND)
+  target_link_libraries(fft_c2c_x PRIVATE ${FFTW_DOUBLE_LIB})
+  target_link_libraries(fft_c2c_x PRIVATE ${FFTW_FLOAT_LIB})
+  target_link_libraries(fft_r2c_x PRIVATE ${FFTW_DOUBLE_LIB})
+  target_link_libraries(fft_r2c_x PRIVATE ${FFTW_FLOAT_LIB})
+  target_link_libraries(fft_grid_x PRIVATE ${FFTW_DOUBLE_LIB})
+  target_link_libraries(fft_grid_x PRIVATE ${FFTW_FLOAT_LIB})
+endif (FFTW_FOUND)
 
 add_test(NAME fft_c2c_x COMMAND fft_c2c_x)
 add_test(NAME fft_r2c_x COMMAND fft_r2c_x)
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index b010e35d..32335e54 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -8,6 +8,12 @@ add_executable(fft_r2c_z ${files_fft_r2c})
 
 target_link_libraries(fft_c2c_z PRIVATE decomp2d)
 target_link_libraries(fft_r2c_z PRIVATE decomp2d)
+if (FFTW_FOUND)
+  target_link_libraries(fft_c2c_z PRIVATE ${FFTW_DOUBLE_LIB})
+  target_link_libraries(fft_c2c_z PRIVATE ${FFTW_FLOAT_LIB})
+  target_link_libraries(fft_r2c_z PRIVATE ${FFTW_DOUBLE_LIB})
+  target_link_libraries(fft_r2c_z PRIVATE ${FFTW_FLOAT_LIB})
+endif (FFTW_FOUND)
 
 add_test(NAME fft_c2c_z COMMAND fft_c2c_z)
 add_test(NAME fft_r2c_z COMMAND fft_r2c_z)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c864c514..ffca41c1 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -33,8 +33,6 @@ if(FFTW_FOUND)
   message(STATUS "FFTW_DOUBLE     : ${FFTW_DOUBLE_LIB}")
   message(STATUS "FFTW_SINGLE     : ${FFTW_FLOAT_LIB}")
   target_include_directories(decomp2d PRIVATE ${FFTW_INCLUDE_DIRS})
-  target_link_libraries(decomp2d PRIVATE ${FFTW_DOUBLE_LIB})
-  target_link_libraries(decomp2d PRIVATE ${FFTW_FLOAT_LIB})
 elseif(MKL_FOUND)
   target_compile_options(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
   target_include_directories(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)

From d824bb6e6346dd473121b6ca78f88152e0fa10d7 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 24 Jan 2023 22:22:15 +0000
Subject: [PATCH 120/436] Link MKL to examples, not decomp2d

---
 examples/fft_physical_x/CMakeLists.txt | 4 ++++
 examples/fft_physical_z/CMakeLists.txt | 3 +++
 src/CMakeLists.txt                     | 1 -
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index 01a9ed10..3359cba5 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -18,6 +18,10 @@ if (FFTW_FOUND)
   target_link_libraries(fft_r2c_x PRIVATE ${FFTW_FLOAT_LIB})
   target_link_libraries(fft_grid_x PRIVATE ${FFTW_DOUBLE_LIB})
   target_link_libraries(fft_grid_x PRIVATE ${FFTW_FLOAT_LIB})
+elseif (MKL_FOUND)
+  target_link_libraries(fft_c2c_x PUBLIC $<LINK_ONLY:MKL::MKL>)
+  target_link_libraries(fft_r2c_x PUBLIC $<LINK_ONLY:MKL::MKL>)
+  target_link_libraries(fft_grid_x PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
 add_test(NAME fft_c2c_x COMMAND fft_c2c_x)
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index 32335e54..a8c5703a 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -13,6 +13,9 @@ if (FFTW_FOUND)
   target_link_libraries(fft_c2c_z PRIVATE ${FFTW_FLOAT_LIB})
   target_link_libraries(fft_r2c_z PRIVATE ${FFTW_DOUBLE_LIB})
   target_link_libraries(fft_r2c_z PRIVATE ${FFTW_FLOAT_LIB})
+elseif (MKL_FOUND)
+  target_link_libraries(fft_c2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
+  target_link_libraries(fft_r2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
 add_test(NAME fft_c2c_z COMMAND fft_c2c_z)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ffca41c1..29065bbb 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -36,7 +36,6 @@ if(FFTW_FOUND)
 elseif(MKL_FOUND)
   target_compile_options(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
   target_include_directories(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
-  target_link_libraries(decomp2d PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif(FFTW_FOUND)
 
 install(TARGETS decomp2d

From 36d1b5afbe3f0c52b1c105dc8ac4dd6f44c4ba2a Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 27 Jan 2023 10:41:30 +0000
Subject: [PATCH 121/436] Rewrite Building in README

Move from Makefile -> CMake instructions
---
 README.md | 64 +++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 41 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 1de50561..9ae1dd04 100644
--- a/README.md
+++ b/README.md
@@ -2,32 +2,50 @@
 
 ## Building
 
-Different compilers can be set by specifying `CMP`, e.g. `make CMP=intel`
-to build with Intel compilers, see `Makefile` for options.
-
-By default an optimised library will be built, debugging versions of the
-library can be built with `make BUILD=debug`, a development version which 
-additionally sets compile time flags to catch coding errors can be built 
-with `make BUILD=dev` (GNU compilers only currently). The behavior of debug
-and development versions of the library can be changed before the initialization
-using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
+The build system is driven by `cmake`, to generate a build system run
+```
+cmake -B build .
+```
+from the `2decomp&fft` root directory.
+It may be necessary to override the Fortran compiler if the wrong one is picked up (due to, e.g.
+`PATH` ordering), for example to specify the `MPICH` compiler on Ubuntu
+```
+FC=mpif90.mpich cmake -B build .
+```
+This should generate a directory `build/` containing the build system, this can be further
+configured using the `ccmake` utility
+```
+ccmake build/
+```
+and editing as desired, variables that are likely of interest are: `CMAKE_BUILD_TYPE` and `FFT_Choice`;
+additional variables can be shown by entering "advanced mode" by pressing `t`.
+By default a `RELEASE` build will built, other options for `CMAKE_BUILD_TYPE` are `DEBUG` and `DEV` which
+turn on debugging flags and additionally try to catch coding errors at compile time, respectively.
+The behavior of debug and development versions of the library can be changed before the
+initialization using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
 The value provided with the environment variable must be a positive integer below 9999.
 
-On each build of the library (`make`, `make all`) a temporary file `Makefile.settings` with
-all current options (`FFLAGS`, `DEFS`, etc.) will be created, and included
-on subsequent invocations, the user therefore does not need to keep
-specifying options between builds.
-
-To perform a clean build run `make clean` first, this will delete all
-output files, including `Makefile.settings`.
-
-On each build of the library (`make`, `make all`) a temporary file `Makefile.settings` with
-all current options (`FFLAGS`, `DEFS`, etc.) will be created, and included
-on subsequent invocations, the user therefore does not need to keep
-specifying options between builds.
+Once the build system has been configured, build `2decomp&fft` by running
+```
+make -j <nproc> -C build
+```
+appending ```VERBOSE=1``` will display additional information about the build, such as compiler flags.
+After building the library can be tested by running
+```
+make -C build test
+```
+which uses the `ctest` utility.
+Finally, install the library by running
+```
+make -C build install
+```
+which will install the library under `build/opt` by default, this location can be configured using
+`ccmake` as described above and setting the variable `CMAKE_INSTALL_PREFIX`.
 
-To perform a clean build run `make clean` first, this will delete all
-output files, including `Makefile.settings`.
+Occasionally a clean build is required, this can be performed by running
+```
+make -C build clean
+```
 
 ## Testing and examples
 

From f96499dc2ea7eee6905ab29d59b4c80c46cbd07d Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 31 Jan 2023 13:26:46 +0000
Subject: [PATCH 122/436] First porting to GPU of the halo code/example

---
 examples/Makefile                |   2 -
 examples/halo_test/halo_test.f90 | 241 ++++++++++++++++++-------------
 src/halo.f90                     |  16 ++
 src/halo_common.f90              |  32 +++-
 4 files changed, 185 insertions(+), 106 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index 87927396..508a0c0b 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -16,8 +16,6 @@ halo_test:
 	$(MAKE) -C $@ $@
 io_test:
 	$(MAKE) -C $@ all
-halo_test:
-	$(MAKE) -C $@ all
 
 check:
 	cd init_test; $(MAKE) $@
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index f1ef8531..50da3139 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -15,12 +15,11 @@ program halo_test
    integer, parameter :: nx = 171, ny = 132, nz = 113
    integer :: p_row = 0, p_col = 0
 
-   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
-   real(mytype), allocatable, dimension(:, :, :) :: v1, v2, v3
-   real(mytype), allocatable, dimension(:, :, :) :: w1, w2, w3
-   real(mytype), allocatable, dimension(:, :, :) :: wk2, wk3
-   real(mytype), allocatable, dimension(:, :, :) :: uh, vh, wh
-   real(mytype), allocatable, dimension(:, :, :) :: div1, div2, div3, div4
+   real(mytype), allocatable, dimension(:, :, :) :: u1, v1, w1, div1
+   real(mytype), allocatable, dimension(:, :, :) :: u2, v2, w2, div2
+   real(mytype), allocatable, dimension(:, :, :) :: u3, v3, w3, div3
+   real(mytype), allocatable, dimension(:, :, :) :: div, wk2, wk3
+
 
    integer :: i, j, k, ierror, n
 
@@ -33,11 +32,6 @@ program halo_test
 
    logical :: passing, all_pass
 
-#if defined(_GPU)
-   write(*,*) 'Halo is not currently support on GPU. This test is not performed'
-   return
-#endif
-
    call MPI_INIT(ierror)
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
@@ -55,10 +49,12 @@ program halo_test
    end if
 
    call initialise()
+   !$acc data copyin(u1,v1,w1) create(u2,v2,w2,u3,v3,w3,div1,div2,div3,wk2,wk3) copy(div)
    call test_div_transpose()
    call test_div_haloX()
    call test_div_haloY()
    call test_div_haloZ()
+   !$acc end data
 
    if (nrank == 0) then
       write (*, *) '-----------------------------------------------'
@@ -66,8 +62,7 @@ program halo_test
       write (*, *) '==============================================='
    end if
 
-   deallocate (u1, v1, w1, u2, v2, w2, u3, v3, w3)
-   deallocate (div1, div2, div3, div4)
+   call finalize()
 
    call decomp_2d_finalize
 
@@ -77,10 +72,15 @@ program halo_test
 
 contains
 
+   !=====================================================================
+   ! Initialize 
+   !=====================================================================
    subroutine initialise()
 
+      use decomp_2d
+      
       implicit none
-
+   
 #ifdef HALO_GLOBAL
       logical, parameter :: global = .true.
 #else
@@ -91,6 +91,10 @@ subroutine initialise()
       call alloc_x(u1, global)
       call alloc_x(v1, global)
       call alloc_x(w1, global)
+      call alloc_x(div, global)
+      call alloc_x(div1, global)
+      call alloc_x(div2, global)
+      call alloc_x(div3, global)
 
       call random_seed(size=n)
       allocate (seed(n))
@@ -100,10 +104,34 @@ subroutine initialise()
       call random_number(v1)
       call random_number(w1)
 
+      ! Working array used more than once
+      call alloc_y(u2, global)
+      call alloc_y(v2, global)
+      call alloc_y(w2, global)
+      call alloc_y(wk2, global)
+      
+      call alloc_z(u3, global)
+      call alloc_z(v3, global)
+      call alloc_z(w3, global)
+      call alloc_z(wk3, global)
+      
       all_pass = .true.
 
    end subroutine initialise
 
+   !=====================================================================
+   ! Finalize with deallocation of arrays
+   !=====================================================================
+   subroutine finalize()
+
+      implicit none
+
+      deallocate(u1,v1,w1)
+      deallocate(u2,v2,w2)
+      deallocate(u3,v3,w3)
+      deallocate(wk2,wk3)
+
+   end subroutine finalize
    !=====================================================================
    ! Calculate divergence using global transposition
    !=====================================================================
@@ -121,7 +149,6 @@ subroutine test_div_transpose()
       integer :: kfirst, klast ! K loop start/end
 
       ! du/dx calculated on X-pencil
-      call alloc_x(div1, global)
 #ifdef HALO_GLOBAL
       kfirst = xstart(3); klast = xend(3)
       jfirst = xstart(2); jlast = xend(2)
@@ -131,18 +158,20 @@ subroutine test_div_transpose()
 #endif
       ifirst = 2; ilast = xsize(1) - 1
 
-      div1 = 0.0_mytype
+      !$acc kernels default(present)
+      div(:,:,:) = 0.0_mytype
+      !$acc end kernels
+      !$acc kernels default(present)
       do k = kfirst, klast
          do j = jfirst, jlast
             do i = ifirst, ilast
-               div1(i, j, k) = u1(i + 1, j, k) - u1(i - 1, j, k)
+               div(i, j, k) = u1(i + 1, j, k) - u1(i - 1, j, k)
             end do
          end do
       end do
+      !$acc end kernels
 
       ! dv/dy calculated on Y-pencil
-      call alloc_y(v2, global)
-      call alloc_y(wk2, global)
 #ifdef HALO_GLOBAL
       kfirst = ystart(3); klast = yend(3)
       ifirst = ystart(1); ilast = yend(1)
@@ -153,8 +182,9 @@ subroutine test_div_transpose()
       jfirst = 2; jlast = ysize(2) - 1
 
       call transpose_x_to_y(v1, v2)
-      call transpose_x_to_y(div1, wk2)
+      call transpose_x_to_y(div, wk2)
 
+      !$acc kernels default(present)
       do k = kfirst, klast
          do j = jfirst, jlast
             do i = ifirst, ilast
@@ -162,11 +192,9 @@ subroutine test_div_transpose()
             end do
          end do
       end do
+      !$acc end kernels
 
       ! dw/dz calculated on Z-pencil
-      call alloc_y(w2, global)
-      call alloc_z(w3, global)
-      call alloc_z(wk3, global)
 #ifdef HALO_GLOBAL
       jfirst = zstart(2); jlast = zend(2)
       ifirst = zstart(1); ilast = zend(1)
@@ -180,6 +208,7 @@ subroutine test_div_transpose()
       call transpose_y_to_z(w2, w3)
       call transpose_y_to_z(wk2, wk3)
 
+      !$acc kernels default(present)
       do k = kfirst, klast
          do j = jfirst, jlast
             do i = ifirst, ilast
@@ -187,19 +216,19 @@ subroutine test_div_transpose()
             end do
          end do
       end do
+      !$acc end kernels
 
       ! result in X-pencil
       call transpose_z_to_y(wk3, wk2)
-      call transpose_y_to_x(wk2, div1)
+      call transpose_y_to_x(wk2, div)
 
+#ifdef DEBUG
       if (nrank == 0) then
          write (*, *) 'Calculated via global transposition'
-#ifdef DEBUG
-         write (*, *) (div1(i, i, i), i=2, 13)
-#endif
+         !$acc update self(div)
+         write (*, *) (div(i, i, i), i=2, 13)
       end if
-
-      deallocate (v2, w2, w3, wk2, wk3)
+#endif
 
    end subroutine test_div_transpose
 
@@ -210,6 +239,11 @@ subroutine test_div_haloX()
 
       implicit none
 
+      real(mytype), allocatable, dimension(:, :, :) :: vh, wh 
+#if defined(_GPU)
+      attributes(device) :: vh, wh
+#endif
+
 #ifdef HALO_GLOBAL
       logical, parameter :: global = .true.
 #else
@@ -224,7 +258,9 @@ subroutine test_div_haloX()
       ny_expected = xsize(2) + 2
       nz_expected = xsize(3) + 2
 
-      call alloc_x(div2, global)
+      ! Only global arrays defined in initialise needs to be ported
+      ! Halo array are allocated in both host and device in update_halo
+      ! Halo arrays are just removed before being deallocated 
 #ifdef HALO_GLOBAL
       call update_halo(v1, vh, 1, opt_global=.true., opt_pencil=1)
       call update_halo(w1, wh, 1, opt_global=.true., opt_pencil=1)
@@ -243,19 +279,23 @@ subroutine test_div_haloX()
       call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "X:v")
       call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
 
-      div2 = 0.0_mytype
+      !$acc kernels default(present)
+      div1(:,:,:) = 0._mytype
+      !$acc end kernels
+      !$acc kernels default(present)
       do k = kfirst, klast
          do j = jfirst, jlast
             do i = ifirst, ilast
-               div2(i, j, k) = (u1(i + 1, j, k) - u1(i - 1, j, k)) &
-                               + (vh(i, j + 1, k) - vh(i, j - 1, k)) &
-                               + (wh(i, j, k + 1) - wh(i, j, k - 1))
+               div1(i, j, k) = (u1(i + 1, j, k) - u1(i - 1, j, k)) &
+                             + (vh(i, j + 1, k) - vh(i, j - 1, k)) &
+                             + (wh(i, j, k + 1) - wh(i, j, k - 1))
             end do
          end do
       end do
+      !$acc end kernels
 
       ! Compute error
-      call check_err(div2, "X")
+      call check_err(div1, "X")  
 
       deallocate (vh, wh)
 
@@ -267,6 +307,10 @@ end subroutine test_div_haloX
    subroutine test_div_haloY()
 
       implicit none
+      real(mytype), allocatable, dimension(:, :, :) :: uh, wh 
+#if defined(_GPU)
+      attributes(device) :: uh, wh
+#endif
 
 #ifdef HALO_GLOBAL
       logical, parameter :: global = .true.
@@ -282,12 +326,6 @@ subroutine test_div_haloY()
       ny_expected = ny
       nz_expected = ysize(3) + 2
 
-      call alloc_y(u2, global)
-      call alloc_y(v2, global)
-      call alloc_y(w2, global)
-      call alloc_x(div3, global)
-      call alloc_y(wk2, global)
-
       call transpose_x_to_y(u1, u2)
       call transpose_x_to_y(v1, v2)
       call transpose_x_to_y(w1, w2)
@@ -309,22 +347,24 @@ subroutine test_div_haloY()
       call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Y:u")
       call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "Y:w")
 
+      !$acc kernels default(present)
       do k = kfirst, klast
          do j = jfirst, jlast
             do i = ifirst, ilast
                wk2(i, j, k) = (uh(i + 1, j, k) - uh(i - 1, j, k)) &
-                              + (v2(i, j + 1, k) - v2(i, j - 1, k)) &
-                              + (wh(i, j, k + 1) - wh(i, j, k - 1))
+                            + (v2(i, j + 1, k) - v2(i, j - 1, k)) &
+                            + (wh(i, j, k + 1) - wh(i, j, k - 1))
             end do
          end do
       end do
+      !$acc end kernels
 
-      call transpose_y_to_x(wk2, div3)
+      call transpose_y_to_x(wk2, div2)
 
       ! Compute error
-      call check_err(div3, "Y")
+      call check_err(div2, "Y")
 
-      deallocate (uh, wh, wk2)
+      deallocate (uh, wh)
 
    end subroutine test_div_haloY
 
@@ -334,6 +374,10 @@ end subroutine test_div_haloY
    subroutine test_div_haloZ()
 
       implicit none
+      real(mytype), allocatable, dimension(:, :, :) :: uh,vh 
+#if defined(_GPU)
+      attributes(device) :: vh, uh
+#endif
 
 #ifdef HALO_GLOBAL
       logical, parameter :: global = .true.
@@ -349,93 +393,73 @@ subroutine test_div_haloZ()
       ny_expected = zsize(2) + 2
       nz_expected = nz
 
-      call alloc_z(u3, global)
-      call alloc_z(v3, global)
-      call alloc_z(w3, global)
-
       call transpose_y_to_z(u2, u3)
       call transpose_y_to_z(v2, v3)
       call transpose_y_to_z(w2, w3)
 
-      call alloc_x(div4, global)
-      call alloc_y(wk2, global)
-      call alloc_z(wk3, global)
-
       ! du/dx
 #ifdef HALO_GLOBAL
       call update_halo(u3, uh, 1, opt_global=.true., opt_pencil=3)
+      call update_halo(v3, vh, 1, opt_global=.true., opt_pencil=3)
       ifirst = zstart(1); ilast = zend(1)
       jfirst = zstart(2); jlast = zend(2)
 #else
       call update_halo(u3, uh, 1, opt_pencil=3)
+      call update_halo(v3, vh, 1, opt_pencil=3)
       ifirst = 1; ilast = zsize(1)
       jfirst = 1; jlast = zsize(2)
 #endif
       kfirst = 2; klast = zsize(3) - 1
 
       call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
-
-      do j = jfirst, jlast
-         do i = ifirst, ilast
-            do k = kfirst, klast
-               wk3(i, j, k) = uh(i + 1, j, k) - uh(i - 1, j, k)
-            end do
-         end do
-      end do
-
-      ! dv/dy
-#ifdef HALO_GLOBAL
-      call update_halo(v3, vh, 1, opt_global=.true., opt_pencil=3)
-#else
-      call update_halo(v3, vh, 1, opt_pencil=3)
-#endif
-
       call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
 
+      !$acc kernels default(present)
       do j = jfirst, jlast
          do i = ifirst, ilast
             do k = kfirst, klast
-               wk3(i, j, k) = wk3(i, j, k) + vh(i, j + 1, k) - vh(i, j - 1, k)
-            end do
-         end do
-      end do
-
-      ! dw/dz
-      do j = jfirst, jlast
-         do i = ifirst, ilast
-            do k = kfirst, klast
-               wk3(i, j, k) = wk3(i, j, k) + w3(i, j, k + 1) - w3(i, j, k - 1)
+               wk3(i, j, k) = uh(i + 1, j, k) - uh(i - 1, j, k) &
+                            + vh(i, j + 1, k) - vh(i, j - 1, k) &
+                            + w3(i, j, k + 1) - w3(i, j, k - 1)
             end do
          end do
       end do
+      !$acc end kernels
 
       call transpose_z_to_y(wk3, wk2)
-      call transpose_y_to_x(wk2, div4)
+      call transpose_y_to_x(wk2, div3)
 
       ! Compute error
-      call check_err(div4, "Z")
+      call check_err(div3, "Z")
 
-      deallocate (uh, vh, wk2, wk3)
+      deallocate (uh, vh)
    end subroutine test_div_haloZ
-
+   !=====================================================================
+   ! Check the difference between halo and transpose divergence
+   !=====================================================================
    subroutine check_err(divh, pencil)
 
+      implicit none
+      
       real(mytype), dimension(:, :, :), intent(in) :: divh
       character(len=*), intent(in) :: pencil
-
       real(mytype), dimension(:, :, :), allocatable :: tmp
+      real(mytype) :: divmag, error
+#if defined(_GPU)
+      attributes(device) :: tmp
+#endif
 
-      real(mytype) :: divmag
-
-      ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
-      !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
-      !      causes a SEGFAULT. Explicitly computing the difference in a temporary
-      !      array seems to be OK.
       allocate (tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
+
+      !$acc kernels default(present)
       tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - div1(2:xlast, 2:ylast, 2:zlast)
-      err = mag(tmp(2:xlast, 2:ylast, 2:zlast))
-      deallocate (tmp)
-      divmag = mag(div1(2:xlast, 2:ylast, 2:zlast))
+      !$acc end kernels
+      error = mag(tmp)
+      !$acc kernels default(present)
+      tmp(2:xlast, 2:ylast, 2:zlast) = div1(2:xlast, 2:ylast, 2:zlast)
+      !$acc end kernels
+      divmag = mag(tmp)
+      
       if (err < epsilon(divmag)*divmag) then
          passing = .true.
       else
@@ -452,16 +476,34 @@ subroutine check_err(divh, pencil)
          write (*, *) 'Error: ', err, '; Relative: ', err/divmag
          write (*, *) 'Pass: ', passing
       end if
+      deallocate(tmp)
 
    end subroutine check_err
-
+   !=====================================================================
+   ! Compute the magnitude af the ayyays
+   !=====================================================================
    real(mytype) function mag(a)
 
+      implicit none
+
       real(mytype), dimension(:, :, :), intent(in) :: a
+#if defined(_GPU)
+      attributes(device) :: a
+#endif
 
       real(mytype) :: lmag, gmag
 
-      lmag = sum(a(:, :, :)**2)
+      lmag = 0._mytype
+      !$acc parallel loop default(present) collapse(3) reduction(+:lmag)
+      do k = 2, zlast
+         do j = 2, ylast
+            do i = 2, xlast
+               lmag = lmag + a(i,j,k)**2
+            enddo
+         enddo
+      enddo
+      !$acc end parallel
+
       call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
       if (ierror /= 0) then
          call decomp_2d_abort(__FILE__, __LINE__, ierror, &
@@ -471,10 +513,15 @@ real(mytype) function mag(a)
       mag = sqrt(gmag/(nx - 2)/(ny - 2)/(nz - 2))
 
    end function mag
-
+   !=====================================================================
+   ! Check the dimensions of the halo arrays are the one expected
+   !=====================================================================
    subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
 
       real(mytype), dimension(:, :, :), intent(in) :: arrh
+#if defined(_GPU)
+      attributes(device) :: arrh
+#endif
       integer, intent(in) :: nx_expected, ny_expected, nz_expected
       character(len=*), intent(in) :: tag
 
diff --git a/src/halo.f90 b/src/halo.f90
index d0082f5c..3ed61d9e 100644
--- a/src/halo.f90
+++ b/src/halo.f90
@@ -19,6 +19,9 @@ subroutine update_halo_real_short(in, out, level, opt_global, opt_pencil)
      integer, intent(IN) :: level      ! levels of halo cells required
      real(mytype), dimension(:, :, :), intent(IN) :: in
      real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+#if defined(_GPU)
+      attributes(device) :: out
+#endif
      logical, optional :: opt_global
      integer, intent(in), optional :: opt_pencil
 
@@ -33,6 +36,9 @@ subroutine update_halo_real(in, out, level, decomp, opt_global, opt_pencil)
      integer, intent(IN) :: level      ! levels of halo cells required
      real(mytype), dimension(:, :, :), intent(IN) :: in
      real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+#if defined(_GPU)
+      attributes(device) :: out
+#endif
      TYPE(DECOMP_INFO), intent(in) :: decomp
      logical, optional :: opt_global
      integer, intent(in), optional :: opt_pencil
@@ -41,6 +47,8 @@ subroutine update_halo_real(in, out, level, decomp, opt_global, opt_pencil)
 
      ! starting/ending index of array with halo cells
      integer :: xs, ys, zs, xe, ye, ze
+     ! additional start end 
+     integer :: ist, ien, jst, jen, kst, ken
 
      integer :: i, j, k, s1, s2, s3, ierror
      integer :: data_type
@@ -68,6 +76,9 @@ subroutine update_halo_complex_short(in, out, level, opt_global, opt_pencil)
      integer, intent(IN) :: level      ! levels of halo cells required
      complex(mytype), dimension(:, :, :), intent(IN) :: in
      complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+#if defined(_GPU)
+      attributes(device) :: out
+#endif
      logical, optional :: opt_global
      integer, intent(in), optional :: opt_pencil
 
@@ -82,6 +93,9 @@ subroutine update_halo_complex(in, out, level, decomp, opt_global, opt_pencil)
      integer, intent(IN) :: level      ! levels of halo cells required
      complex(mytype), dimension(:, :, :), intent(IN) :: in
      complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
+#if defined(_GPU)
+      attributes(device) :: out
+#endif
      TYPE(DECOMP_INFO), intent(in) :: decomp
      logical, optional :: opt_global
      integer, intent(in), optional :: opt_pencil
@@ -90,6 +104,8 @@ subroutine update_halo_complex(in, out, level, decomp, opt_global, opt_pencil)
 
      ! starting/ending index of array with halo cells
      integer :: xs, ys, zs, xe, ye, ze
+     ! additional start end 
+     integer :: ist, ien, jst, jen, kst, ken
 
      integer :: i, j, k, s1, s2, s3, ierror
      integer :: data_type
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index 13fa5d43..d1071944 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -114,40 +114,55 @@
                             'Invalid data passed to update_halo')
     end if
     allocate (out(xs:xe, ys:ye, zs:ze))
-!    out = -1.0_mytype ! fill the halo for debugging
 
+    !    out = -1.0_mytype ! fill the halo for debugging
+
+    !$acc enter data create(requests,neighbour)
     ! copy input data to output
     if (global) then
        ! using global coordinate
        ! note the input array passed in always has index starting from 1
        ! need to work out the corresponding global index
        if (ipencil == 1) then
-          do k = decomp%xst(3), decomp%xen(3)
-             do j = decomp%xst(2), decomp%xen(2)
+          kst=decomp%xst(3); ken=decomp%xen(3)
+          jst=decomp%xst(2); jen=decomp%xen(2)
+          !$acc kernels default(present)
+          do k = kst, ken
+             do j = jst, jen
                 do i = 1, s1  ! x all local
                    out(i, j, k) = in(i, j - decomp%xst(2) + 1, k - decomp%xst(3) + 1)
                 end do
              end do
           end do
+          !$acc end kernels
        else if (ipencil == 2) then
-          do k = decomp%yst(3), decomp%yen(3)
+          kst=decomp%yst(3); ken=decomp%yen(3)
+          ist=decomp%yst(1); ien=decomp%yen(1)
+          !$acc kernels default(present)
+          do k = kst, ken
              do j = 1, s2  ! y all local
-                do i = decomp%yst(1), decomp%yen(1)
+                do i = ist, ien
                    out(i, j, k) = in(i - decomp%yst(1) + 1, j, k - decomp%yst(3) + 1)
                 end do
              end do
           end do
+          !$acc end kernels
        else if (ipencil == 3) then
+          jst=decomp%zst(2); jen=decomp%zen(2)
+          ist=decomp%xst(1); ien=decomp%xen(1)
+          !$acc kernels default(present)
           do k = 1, s3  ! z all local
-             do j = decomp%zst(2), decomp%zen(2)
-                do i = decomp%zst(1), decomp%zen(1)
+             do j = jst, jen
+                do i = ist, ien
                    out(i, j, k) = in(i - decomp%zst(1) + 1, j - decomp%zst(2) + 1, k)
                 end do
              end do
           end do
+          !$acc end kernels
        end if
     else
        ! not using global coordinate
+       !$acc kernels default(present)
        do k = 1, s3
           do j = 1, s2
              do i = 1, s1
@@ -155,6 +170,8 @@
              end do
           end do
        end do
+       !$acc end kernels
+       !istat = cudaMemcpy(out,in,s1*s2*s3,cudaMemcpyDeviceToDevice)
     end if
 
     ! If needed, define MPI derived data type to pack halo data,
@@ -488,6 +505,7 @@
           end do
        end if
 #endif
+       !$acc exit data delete(neighbour,requests)
 
        ! *** top/bottom ***
        ! all data in local memory already, no halo exchange

From 2d8fb3f635658e84d82a1078f91c4c8c69a67e75 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 31 Jan 2023 20:41:01 +0000
Subject: [PATCH 123/436] Minor formatting change

Make clear difference between commented out code that is a memory
aid, vs OpenACC directive
---
 src/halo_common.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/halo_common.f90 b/src/halo_common.f90
index d1071944..662a4ce5 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -171,7 +171,7 @@
           end do
        end do
        !$acc end kernels
-       !istat = cudaMemcpy(out,in,s1*s2*s3,cudaMemcpyDeviceToDevice)
+       !!! istat = cudaMemcpy(out,in,s1*s2*s3,cudaMemcpyDeviceToDevice)
     end if
 
     ! If needed, define MPI derived data type to pack halo data,

From 5ab1f792ad4bd454d9d660a0c3596d5b92d3e8c5 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 31 Jan 2023 20:44:55 +0000
Subject: [PATCH 124/436] Restore warning about use of tmp array in halo_test

Worth keeping around as apparently tmp array not needed, however
example SEGFAULTS in Intel compilers without
---
 examples/halo_test/halo_test.f90 | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 50da3139..267a8296 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -448,7 +448,10 @@ subroutine check_err(divh, pencil)
 #if defined(_GPU)
       attributes(device) :: tmp
 #endif
-
+      ! XXX: The Intel compiler SEGFAULTs if the array difference is computed inplace
+      !      i.e. mag(divh(2:xlast,2:ylast,2:zlast) - div1(2:xlast,2:ylast,2:zlast))
+      !      causes a SEGFAULT. Explicitly computing the difference in a temporary
+      !      array seems to be OK
       allocate (tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
 
       !$acc kernels default(present)

From 31662409cda0db1c3b7b994b263fe916aab154c1 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 31 Jan 2023 21:18:37 +0000
Subject: [PATCH 125/436] Use MPIEXEC to launch tests under ctest

---
 examples/fft_physical_x/CMakeLists.txt |  8 +++++---
 examples/fft_physical_z/CMakeLists.txt |  4 ++--
 examples/halo_test/CMakeLists.txt      |  2 +-
 examples/init_test/CMakeLists.txt      |  2 +-
 examples/io_test/CMakeLists.txt        | 10 +++++-----
 examples/test2d/CMakeLists.txt         |  2 +-
 6 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index 3359cba5..f7c4dff2 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -24,6 +24,8 @@ elseif (MKL_FOUND)
   target_link_libraries(fft_grid_x PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
-add_test(NAME fft_c2c_x COMMAND fft_c2c_x)
-add_test(NAME fft_r2c_x COMMAND fft_r2c_x)
-add_test(NAME fft_grid_x COMMAND fft_grid_x)
+set(fft_c2c_exec ${MPIEXEC_EXECUTABLE} fft_c2c_x)
+
+add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_c2c_x>)
+add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_r2c_x>)
+add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_grid_x>)
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index a8c5703a..ad169137 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -18,5 +18,5 @@ elseif (MKL_FOUND)
   target_link_libraries(fft_r2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
-add_test(NAME fft_c2c_z COMMAND fft_c2c_z)
-add_test(NAME fft_r2c_z COMMAND fft_r2c_z)
+add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_c2c_z>)
+add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_r2c_z>)
diff --git a/examples/halo_test/CMakeLists.txt b/examples/halo_test/CMakeLists.txt
index 27a17add..3c59d859 100644
--- a/examples/halo_test/CMakeLists.txt
+++ b/examples/halo_test/CMakeLists.txt
@@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(halo_test ${files_test})
 target_link_libraries(halo_test PRIVATE decomp2d)
-add_test(NAME halo_test COMMAND halo_test)
+add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:halo_test>)
diff --git a/examples/init_test/CMakeLists.txt b/examples/init_test/CMakeLists.txt
index be042304..e924cfbf 100644
--- a/examples/init_test/CMakeLists.txt
+++ b/examples/init_test/CMakeLists.txt
@@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(init_test ${files_test})
 target_link_libraries(init_test PRIVATE decomp2d)
-add_test(NAME init_test COMMAND init_test)
+add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:init_test>)
diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 46ce0da9..453364e1 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -18,8 +18,8 @@ target_link_libraries(io_var_test PRIVATE decomp2d)
 target_link_libraries(io_plane_test PRIVATE decomp2d)
 target_link_libraries(io_bench PRIVATE decomp2d)
 
-add_test(NAME io_test COMMAND io_test)
-add_test(NAME io_read COMMAND io_read)
-add_test(NAME io_var_test COMMAND io_var_test 0 0)
-add_test(NAME io_plane_test COMMAND io_plane_test)
-add_test(NAME io_bench COMMAND io_bench)
+add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_test>)
+add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_read>)
+add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_var_test> 0 0)
+add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_plane_test>)
+add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_bench>)
diff --git a/examples/test2d/CMakeLists.txt b/examples/test2d/CMakeLists.txt
index d5be500a..47c0488c 100644
--- a/examples/test2d/CMakeLists.txt
+++ b/examples/test2d/CMakeLists.txt
@@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(test2d ${files_test})
 target_link_libraries(test2d PRIVATE decomp2d)
-add_test(NAME test2d COMMAND test2d)
+add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:test2d>)

From 6b523674cb72aa027c05bc384a17e4b6ec582c0e Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 31 Jan 2023 21:29:44 +0000
Subject: [PATCH 126/436] Set number of processes for tests via CMake/FindMPI

Note this defaults to the maximum number of CPUs detected
---
 examples/fft_physical_x/CMakeLists.txt |  8 +++-----
 examples/fft_physical_z/CMakeLists.txt |  4 ++--
 examples/halo_test/CMakeLists.txt      |  2 +-
 examples/init_test/CMakeLists.txt      |  2 +-
 examples/io_test/CMakeLists.txt        | 10 +++++-----
 examples/test2d/CMakeLists.txt         |  2 +-
 6 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index f7c4dff2..94d34043 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -24,8 +24,6 @@ elseif (MKL_FOUND)
   target_link_libraries(fft_grid_x PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
-set(fft_c2c_exec ${MPIEXEC_EXECUTABLE} fft_c2c_x)
-
-add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_c2c_x>)
-add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_r2c_x>)
-add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_grid_x>)
+add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_x>)
+add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_x>)
+add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_grid_x>)
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index ad169137..ffeb1c64 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -18,5 +18,5 @@ elseif (MKL_FOUND)
   target_link_libraries(fft_r2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
-add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_c2c_z>)
-add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:fft_r2c_z>)
+add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_z>)
+add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_z>)
diff --git a/examples/halo_test/CMakeLists.txt b/examples/halo_test/CMakeLists.txt
index 3c59d859..e6a75a8b 100644
--- a/examples/halo_test/CMakeLists.txt
+++ b/examples/halo_test/CMakeLists.txt
@@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(halo_test ${files_test})
 target_link_libraries(halo_test PRIVATE decomp2d)
-add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:halo_test>)
+add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:halo_test>)
diff --git a/examples/init_test/CMakeLists.txt b/examples/init_test/CMakeLists.txt
index e924cfbf..5c5c0bff 100644
--- a/examples/init_test/CMakeLists.txt
+++ b/examples/init_test/CMakeLists.txt
@@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(init_test ${files_test})
 target_link_libraries(init_test PRIVATE decomp2d)
-add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:init_test>)
+add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:init_test>)
diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 453364e1..e16e1f86 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -18,8 +18,8 @@ target_link_libraries(io_var_test PRIVATE decomp2d)
 target_link_libraries(io_plane_test PRIVATE decomp2d)
 target_link_libraries(io_bench PRIVATE decomp2d)
 
-add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_test>)
-add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_read>)
-add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_var_test> 0 0)
-add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_plane_test>)
-add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:io_bench>)
+add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_test>)
+add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_read>)
+add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_var_test> 0 0)
+add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_plane_test>)
+add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_bench>)
diff --git a/examples/test2d/CMakeLists.txt b/examples/test2d/CMakeLists.txt
index 47c0488c..408afd4c 100644
--- a/examples/test2d/CMakeLists.txt
+++ b/examples/test2d/CMakeLists.txt
@@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(test2d ${files_test})
 target_link_libraries(test2d PRIVATE decomp2d)
-add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} $<TARGET_FILE:test2d>)
+add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:test2d>)

From be8ce9e33153c5a5b7334e2a59b91c8cddc6e81a Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 31 Jan 2023 21:41:37 +0000
Subject: [PATCH 127/436] Pass PROW and PCOL to ctest

Use ccmake to configure
---
 cmake/mpi.cmake                 | 8 ++++++++
 examples/io_test/CMakeLists.txt | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/cmake/mpi.cmake b/cmake/mpi.cmake
index 64ebba07..8e07078f 100644
--- a/cmake/mpi.cmake
+++ b/cmake/mpi.cmake
@@ -18,3 +18,11 @@ else (MPI_FOUND)
     message(STATUS "Using a CMake vers > 3.10 should solve the problem")
     message(STATUS "Alternatively use ccmake to manually set the include if available")
 endif (MPI_FOUND)
+
+# PROW/PCOL options
+# Tests that accept PROW/PCOL as arguments will use these values.
+# To simplify out of the box testing, default to 0.
+set(PROW 0 CACHE STRING
+  "Number of processor rows - PROWxPCOL=NP must be satisfied, 0 for autotuning")
+set(PCOL 0 CACHE STRING
+  "Number of processor rows - PROWxPCOL=NP must be satisfied, 0 for autotuning")
diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index e16e1f86..89c8ff24 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -20,6 +20,6 @@ target_link_libraries(io_bench PRIVATE decomp2d)
 
 add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_test>)
 add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_read>)
-add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_var_test> 0 0)
+add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_var_test> ${PROW} ${PCOL})
 add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_plane_test>)
 add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_bench>)

From 3c4e0e90056aa9fbb445709f1485a0aa53ce3bad Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 2 Feb 2023 10:31:55 +0000
Subject: [PATCH 128/436] Fix typo for GNU>=10 in compiler.cmake

---
 cmake/compilers.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/compilers.cmake b/cmake/compilers.cmake
index d1e93045..a2e16309 100644
--- a/cmake/compilers.cmake
+++ b/cmake/compilers.cmake
@@ -11,7 +11,7 @@ if (Fortran_COMPILER_NAME MATCHES "GNU")
   set(D2D_FFLAGS "-cpp -std=f2008 -ffree-line-length-none")
   if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
     message(STATUS "Set New Fortran basic flags")
-    set(D2D_FFLAGS "${D2D_FLAGS} -fallow-argument-mismatch")
+    set(D2D_FFLAGS "${D2D_FFLAGS} -fallow-argument-mismatch")
   endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
   set(D2D_FFLAGS_RELEASE "-O3 -march=native")
   set(D2D_FFLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none")

From 1d61c3e406310e0b8f34db9fd65bbae03ab14429 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 2 Feb 2023 10:39:58 +0000
Subject: [PATCH 129/436] Remove FORCE from set FFLAGS

This was preventing user overriding settings
---
 cmake/compilers.cmake | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/cmake/compilers.cmake b/cmake/compilers.cmake
index a2e16309..ec3222cd 100644
--- a/cmake/compilers.cmake
+++ b/cmake/compilers.cmake
@@ -82,17 +82,13 @@ else (Fortran_COMPILER_NAME MATCHES "GNU")
 endif (Fortran_COMPILER_NAME MATCHES "GNU")
 
 set(CMAKE_Fortran_FLAGS ${D2D_FFLAGS} CACHE STRING
-  "Baseline compiler FFLAGS"
-  FORCE)
+  "Baseline compiler FFLAGS")
 set(CMAKE_Fortran_FLAGS_RELEASE ${D2D_FFLAGS_RELEASE} CACHE STRING
-  "Additional FFLAGS for Release (optimised) build"
-  FORCE)
+  "Additional FFLAGS for Release (optimised) build")
 set(CMAKE_Fortran_FLAGS_DEBUG ${D2D_FFLAGS_DEBUG} CACHE STRING
-  "Additional FFLAGS for Debug build"
-  FORCE)
+  "Additional FFLAGS for Debug build")
 set(CMAKE_Fortran_FLAGS_DEV ${D2D_FFLAGS_DEV} CACHE STRING
-  "Additional FFLAGS for Dev build"
-  FORCE)
+  "Additional FFLAGS for Dev build")
 
 if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
   add_definitions("-DDEBUG")

From 51c69b337aec483b2472092d3205a5f5d66bf13c Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Sat, 4 Feb 2023 00:41:27 +0000
Subject: [PATCH 130/436] Update CMake with separated files for compilers flags

---
 .gitignore                              |   4 +-
 CMakeLists.txt                          |  46 +++++++---
 cmake/D2D_Compilers.cmake               |  89 +++++++++++++++++++
 cmake/{gpu.cmake => D2D_GPU.cmake}      |   6 +-
 cmake/{mpi.cmake => D2D_MPI.cmake}      |  12 +++
 cmake/compilers.cmake                   | 112 ------------------------
 cmake/compilers/D2D_flags_cray.cmake    |   5 ++
 cmake/compilers/D2D_flags_fujitsu.cmake |   2 +
 cmake/compilers/D2D_flags_gnu.cmake     |  12 +++
 cmake/compilers/D2D_flags_intel.cmake   |   7 ++
 cmake/compilers/D2D_flags_nag.cmake     |   5 ++
 cmake/compilers/D2D_flags_nvidia.cmake  |  37 ++++++++
 cmake/fft/fft.cmake                     |   8 +-
 src/CMakeLists.txt                      |   3 +
 14 files changed, 215 insertions(+), 133 deletions(-)
 create mode 100644 cmake/D2D_Compilers.cmake
 rename cmake/{gpu.cmake => D2D_GPU.cmake} (84%)
 rename cmake/{mpi.cmake => D2D_MPI.cmake} (73%)
 delete mode 100644 cmake/compilers.cmake
 create mode 100644 cmake/compilers/D2D_flags_cray.cmake
 create mode 100644 cmake/compilers/D2D_flags_fujitsu.cmake
 create mode 100644 cmake/compilers/D2D_flags_gnu.cmake
 create mode 100644 cmake/compilers/D2D_flags_intel.cmake
 create mode 100644 cmake/compilers/D2D_flags_nag.cmake
 create mode 100644 cmake/compilers/D2D_flags_nvidia.cmake

diff --git a/.gitignore b/.gitignore
index 8bc244b8..0d2e71b1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,7 +32,7 @@
 *.app
 
 # Cmake
-build
+build*
 
 # Log files
 *.log
@@ -45,4 +45,4 @@ Makefile.settings
 
 # emacs
 .dir-locals.el
-TAGS
\ No newline at end of file
+TAGS
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 62a64da1..bede957e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,12 +4,25 @@ cmake_policy(SET CMP0074 NEW)
 project(2decomp LANGUAGES Fortran)
 enable_testing()
 
-option(ENABLE_OPENACC "Allow user to activate/deactivate OpenACC support" OFF)
-option(ENABLE_CUDA "Allow user to activate/deactivate CUDA support" OFF)
 
-if (ENABLE_CUDA)
-  enable_language(CUDA)
-endif()
+set(BUILD_TARGET "mpi" CACHE STRING "Target for acceleration (mpi (default) or gpu)")
+set_property(CACHE BUILD_TARGET PROPERTY STRINGS mpi gpu)
+
+if (BUILD_TARGET MATCHES "gpu")
+  option(ENABLE_OPENACC "Allow user to activate/deactivate OpenACC support" ON)
+  option(ENABLE_CUDA "Allow user to activate/deactivate CUDA support" ON)
+  option(ENABLE_MANAGED "Allow user to activate/deactivate automatic memory managment from NVHPC" OFF)
+  option(ENABLE_NCCL "Allow user to activate/deactivate Collective Comunication NCCL" ON)
+
+  if (ENABLE_CUDA)
+    message(STATUS "Before enable CUDA")
+    enable_language(CUDA)
+    message(STATUS "After enable CUDA")
+  endif()
+
+endif(BUILD_TARGET MATCHES "gpu")
+
+option(ENABLE_PROFILER "Activate the profiler" OFF)
 
 set(AUTHOR "Stefano Rolfo;Charles Moulinec;Paul Bartholomew")
 set(AUTHOR_DETAILS "stefano.rolfo@stfc.ac.uk;charles.moulinec@stfc.ac.uk;p.bartholomew@epcc.ed.ac.uk")
@@ -29,9 +42,6 @@ endif()
 INCLUDE(CMakeDependentOption)
 INCLUDE(CMakeParseArguments)
 
-# Find the modules included with Xcompact
-#SET(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
-
 # make sure that the default is a RELEASE
 if (NOT CMAKE_BUILD_TYPE)
   set (CMAKE_BUILD_TYPE RELEASE CACHE STRING
@@ -39,16 +49,24 @@ if (NOT CMAKE_BUILD_TYPE)
       FORCE)
 endif (NOT CMAKE_BUILD_TYPE)
 
-include(${CMAKE_SOURCE_DIR}/cmake/gpu.cmake)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake" "${CMAKE_SOURCE_DIR}/cmake/compilers" "${CMAKE_SOURCE_DIR}/cmake/fft")
+
+if (BUILD_TARGET MATCHES "gpu")
+  include(D2D_GPU)
+endif (BUILD_TARGET MATCHES "gpu")
 
-include(${CMAKE_SOURCE_DIR}/cmake/compilers.cmake)
+include(D2D_MPI)
 
-include(${CMAKE_SOURCE_DIR}/cmake/mpi.cmake)
+include(D2D_Compilers)
 
 # FFT options
-set(FFT_Choice "generic" CACHE STRING "FFT for XCompact3d project (generic is the default)")
-set_property(CACHE FFT_Choice PROPERTY STRINGS generic fftw mkl)
-include(${CMAKE_SOURCE_DIR}/cmake/fft/fft.cmake)
+if (BUILD_TARGET MATCHES "gpu")
+  set(FFT_Choice "cufft" CACHE STRING "FFT for XCompact3d project (with GPU cufft is the default)")
+else ()
+  set(FFT_Choice "generic" CACHE STRING "FFT for XCompact3d project (generic is the default)")
+endif (BUILD_TARGET MATCHES "gpu")
+set_property(CACHE FFT_Choice PROPERTY STRINGS generic fftw mkl cufft)
+include(fft)
 
 # Create a static library for the fft
 add_subdirectory(src)
diff --git a/cmake/D2D_Compilers.cmake b/cmake/D2D_Compilers.cmake
new file mode 100644
index 00000000..eb9b2f37
--- /dev/null
+++ b/cmake/D2D_Compilers.cmake
@@ -0,0 +1,89 @@
+# Compilers CMakeLists
+
+set(Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER_ID} )
+message(STATUS "COMP ID ${Fortran_COMPILER_NAME}")
+message(STATUS "Fortran compiler name ${Fortran_COMPILER_NAME}")
+message(STATUS "Fortran compiler version ${CMAKE_Fortran_COMPILER_VERSION}")
+
+
+
+if (Fortran_COMPILER_NAME MATCHES "GNU")
+  # gfortran
+  message(STATUS "Setting gfortran flags")
+  include(D2D_flags_gnu)
+  #set(D2D_FFLAGS "-cpp -std=f2008 -ffree-line-length-none")
+  #if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
+  #  message(STATUS "Set New Fortran basic flags")
+  #  set(D2D_FFLAGS "${D2D_FFLAGS} -fallow-argument-mismatch")
+  #endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
+  #set(D2D_FFLAGS_RELEASE "-O3 -march=native")
+  #set(D2D_FFLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none")
+  #set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
+elseif (Fortran_COMPILER_NAME MATCHES "Intel")
+  message(STATUS "Setting ifort flags")
+  include(D2D_flags_intel)
+elseif (Fortran_COMPILER_NAME MATCHES "NAG")
+  message(STATUS "Setting nagfor flags")
+  include(D2D_flags_nag)
+elseif (Fortran_COMPILER_NAME MATCHES "Cray")
+  message(STATUS "Setting cray fortran flags")
+  include(D2D_flags_cray)
+elseif (Fortran_COMPILER_NAME MATCHES "NVHPC")
+  message(STATUS "Setting NVHPC fortran flags")
+  include(D2D_flags_nvidia)
+# elseif (Fortran_COMPILER_NAME MATCHES "Flang")
+#   message(STATUS "Setting Flang flags")
+#   set(CMAKE_Fortran_FLAGS "-cpp -std=f2008" CACHE STRING
+# 	  "Baseline FFLAGS"
+# 	  FORCE)
+elseif (Fortran_COMPILER_NAME MATCHES "Fujitsu")
+  message(STATUS "Setting Fujitsu fortran flags")
+  set(D2D_FFLAGS "-Cpp")
+  set(D2D_FFLAGS_RELEASE "-O3")
+  set(D2D_FFLAGS_DEBUG   "-O0")
+else (Fortran_COMPILER_NAME MATCHES "GNU")
+  message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
+  message ("Fortran compiler: " ${Fortran_COMPILER_NAME})
+  message ("No optimized Fortran compiler flags are known, we just try -O2...")
+  set(D2D_FFLAGS_RELEASE "-O2")
+  set(D2D_FFLAGS_DEBUG   "-O0 -g")
+endif (Fortran_COMPILER_NAME MATCHES "GNU")
+
+if (NOT FLAGS_SET)
+  set(CMAKE_Fortran_FLAGS ${D2D_FFLAGS} CACHE STRING 
+	"Base FFLAGS for build" FORCE)
+  set(CMAKE_Fortran_FLAGS_RELEASE ${D2D_FFLAGS_RELEASE} CACHE STRING
+  	"Additional FFLAGS for Release (optimised) build" FORCE)
+  set(CMAKE_Fortran_FLAGS_DEBUG ${D2D_FFLAGS_DEBUG} CACHE STRING
+  	"Additional FFLAGS for Debug build" FORCE)
+  set(CMAKE_Fortran_FLAGS_DEV ${D2D_FFLAGS_DEV} CACHE STRING
+  	"Additional FFLAGS for Dev build" FORCE)
+  # Add profiler
+  #if (ENABLE_PROFILER)
+  #  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${D2D_EXE_LINKER_FLAGS}" CACHE STRING 
+  #          "Add profiler to exe" FORCE)
+  #endif(ENABLE_PROFILER)
+
+  set(FLAGS_SET 1 CACHE INTERNAL "Flags are set")
+endif()
+
+if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
+  add_definitions("-DDEBUG")
+endif (CMAKE_BUILD_TYPE MATCHES "DEBUG")
+
+execute_process(
+  COMMAND git describe --tag --long --always
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  OUTPUT_VARIABLE GIT_VERSION
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+add_definitions("-DVERSION=\"${GIT_VERSION}\"")
+option(DOUBLE_PRECISION "Build Xcompact with double precision" ON)
+if (DOUBLE_PRECISION)
+  add_definitions("-DDOUBLE_PREC")
+endif()
+
+option(SINGLE_PRECISION_OUTPUT "Build XCompact with output in single precision" OFF)
+if (SINGLE_PRECISION_OUTPUT)
+  add_definitions("-DSAVE_SINGLE")
+endif()
diff --git a/cmake/gpu.cmake b/cmake/D2D_GPU.cmake
similarity index 84%
rename from cmake/gpu.cmake
rename to cmake/D2D_GPU.cmake
index 858b3a1a..4e43bcdd 100644
--- a/cmake/gpu.cmake
+++ b/cmake/D2D_GPU.cmake
@@ -1,4 +1,5 @@
 # GPU CMakeLists
+message(STATUS "Check GPU")
 
 if (ENABLE_OPENACC)
   include(FindOpenACC)
@@ -7,11 +8,9 @@ if (ENABLE_OPENACC)
   else()
     message(ERROR_CRITICAL "No OpenACC support detected")
   endif()
-  set(OPENACC_TARGET "gpu" CACHE STRING "Target for acceleration (gpu (default) or multicore)")
-  set_property(CACHE OPENACC_TARGET PROPERTY STRINGS gpu multicore)
 endif()
 
-if (OPENACC_TARGET MATCHES "gpu")
+if (ENABLE_CUDA)
   find_package(CUDAToolkit REQUIRED)
   if(${CMAKE_VERSION} VERSION_LESS_EQUAL "3.13.4")
     cuda_select_nvcc_arch_flags(ARCH_FLAGS "Auto") # optional argument for arch to add
@@ -34,4 +33,3 @@ if (OPENACC_TARGET MATCHES "gpu")
   message(STATUS "CUDA_LIBRARIES ${CUDA_LIBRARIES}")
 endif()
 
-set(NVIDIA_PROFILER "Use the NVidia profiles" ON)
diff --git a/cmake/mpi.cmake b/cmake/D2D_MPI.cmake
similarity index 73%
rename from cmake/mpi.cmake
rename to cmake/D2D_MPI.cmake
index 8e07078f..64212c68 100644
--- a/cmake/mpi.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -4,6 +4,18 @@ find_package(MPI REQUIRED)
 # Stop if there is no MPI_Fortran_Compiler
 if (MPI_Fortran_COMPILER)
     message(STATUS "MPI_Fortran_COMPILER found: ${MPI_Fortran_COMPILER}")
+    message(STATUS "MPI_VERSION found: ${MPI_VERSION}")
+    # Try to guess the MPI type to adapt compilation flags if necessary
+    string(FIND "${MPI_Fortran_COMPILER}" "mpich" pos)
+    if(pos GREATER_EQUAL "0")
+      set(FIND_MPICH TRUE)
+      message(STATUS "MPI is MPICH type")
+    endif()
+    string(FIND "${MPI_Fortran_COMPILER}" "openmpi" pos)
+    if(pos GREATER_EQUAL "0")
+      set(FIND_OMPI TRUE)
+      message(STATUS "MPI is openMPI type")
+    endif()
 else (MPI_Fortran_COMPILER)
     message(SEND_ERROR "This application cannot compile without MPI")
 endif(MPI_Fortran_COMPILER)
diff --git a/cmake/compilers.cmake b/cmake/compilers.cmake
deleted file mode 100644
index ec3222cd..00000000
--- a/cmake/compilers.cmake
+++ /dev/null
@@ -1,112 +0,0 @@
-# Compilers CMakeLists
-
-set(Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER_ID} )
-message(STATUS "COMP ID ${Fortran_COMPILER_NAME}")
-message(STATUS "Fortran compiler name ${Fortran_COMPILER_NAME}")
-message(STATUS "Fortran compiler version ${CMAKE_Fortran_COMPILER_VERSION}")
-
-if (Fortran_COMPILER_NAME MATCHES "GNU")
-  # gfortran
-  message(STATUS "Setting gfortran flags")
-  set(D2D_FFLAGS "-cpp -std=f2008 -ffree-line-length-none")
-  if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
-    message(STATUS "Set New Fortran basic flags")
-    set(D2D_FFLAGS "${D2D_FFLAGS} -fallow-argument-mismatch")
-  endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
-  set(D2D_FFLAGS_RELEASE "-O3 -march=native")
-  set(D2D_FFLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none")
-  set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
-elseif (Fortran_COMPILER_NAME MATCHES "Intel")
-  message(STATUS "Setting ifort flags")
-  set(D2D_FFLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback")
-  set(D2D_FFLAGS_RELEASE "-O3 -ipo")
-  set(D2D_FFLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG")
-  set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -warn all,noexternal")
-  #set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
-elseif (Fortran_COMPILER_NAME MATCHES "NAG")
-  message(STATUS "Setting nagfor flags")
-  set(D2D_FFLAGS "-fpp")
-  set(D2D_FFLAGS_RELEASE "-O3")
-  set(D2D_FFLAGS_DEBUG   "-O0 -g")
-elseif (Fortran_COMPILER_NAME MATCHES "Cray")
-  message(STATUS "Setting cray fortran flags")
-  set(D2D_FFLAGS "-eF -g -N 1023")
-  set(D2D_FFLAGS_RELEASE "-O3")
-  set(D2D_FFLAGS_DEBUG   "-O0 -g")
-elseif (Fortran_COMPILER_NAME MATCHES "PGI")
-  message(STATUS "Setting PGI fortran flags")
-  if (ENABLE_OPENACC) 
-    if (OPENACC_TARGET MATCHES "gpu")
-      set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=gpu")
-    else(OPENACC_TARGET MATCHES "gpu")
-      set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -acc -target=multicore")
-    endif(OPENACC_TARGET MATCHES "gpu")
-  else()
-    set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
-  endif()
-  #set(CMAKE_Fortran_FLAGS "-fast -cpp -Mfree -Kieee -Minfo=accel -g ")
-  set(D2D_FFLAGS_RELEASE "-fast -O3")
-  set(D2D_FFLAGS_DEBUG   "-O0")
-elseif (Fortran_COMPILER_NAME MATCHES "NVHPC")
-  message(STATUS "Setting NVHPC fortran flags")
-  if (ENABLE_OPENACC) 
-    if (OPENACC_TARGET MATCHES "gpu")
-      #set(CMAKE_Fortran_FLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar=gpu -acc -target=gpu -Minstrument")
-      set(D2D_FFLAGS "${CMAKE_Fortran_FLAGS} -cpp -Mfree -Kieee -Minfo=accel -stdpar=gpu -gpu=cc${CUDA_ARCH_FIRST},managed -acc -target=gpu")
-      #add_definitions("-lnvhpcwrapnvtx")
-      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lnvhpcwrapnvtx")
-    else()
-      set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g -stdpar -acc -target=multicore")
-    endif(OPENACC_TARGET MATCHES "gpu")
-  else()
-    set(D2D_FFLAGS "-cpp -Mfree -Kieee -Minfo=accel -g")
-  endif()
-  set(D2D_FFLAGS_RELEASE "-fast -O3")
-  set(D2D_FFLAGS_DEBUG   "-O0 -g -DDEBUG")
-elseif (Fortran_COMPILER_NAME MATCHES "Fujitsu")
-  message(STATUS "Setting Fujitsu fortran flags")
-  set(D2D_FFLAGS "-Cpp")
-  set(D2D_FFLAGS_RELEASE "-O3")
-  set(D2D_FFLAGS_DEBUG   "-O0")
-# elseif (Fortran_COMPILER_NAME MATCHES "Flang")
-#   message(STATUS "Setting Flang flags")
-#   set(CMAKE_Fortran_FLAGS "-cpp -std=f2008" CACHE STRING
-# 	  "Baseline FFLAGS"
-# 	  FORCE)
-else (Fortran_COMPILER_NAME MATCHES "GNU")
-  message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
-  message ("Fortran compiler: " ${Fortran_COMPILER_NAME})
-  message ("No optimized Fortran compiler flags are known, we just try -O2...")
-  set(D2D_FFLAGS_RELEASE "-O2")
-  set(D2D_FFLAGS_DEBUG   "-O0 -g")
-endif (Fortran_COMPILER_NAME MATCHES "GNU")
-
-set(CMAKE_Fortran_FLAGS ${D2D_FFLAGS} CACHE STRING
-  "Baseline compiler FFLAGS")
-set(CMAKE_Fortran_FLAGS_RELEASE ${D2D_FFLAGS_RELEASE} CACHE STRING
-  "Additional FFLAGS for Release (optimised) build")
-set(CMAKE_Fortran_FLAGS_DEBUG ${D2D_FFLAGS_DEBUG} CACHE STRING
-  "Additional FFLAGS for Debug build")
-set(CMAKE_Fortran_FLAGS_DEV ${D2D_FFLAGS_DEV} CACHE STRING
-  "Additional FFLAGS for Dev build")
-
-if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
-  add_definitions("-DDEBUG")
-endif (CMAKE_BUILD_TYPE MATCHES "DEBUG")
-
-execute_process(
-  COMMAND git describe --tag --long --always
-  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-  OUTPUT_VARIABLE GIT_VERSION
-  OUTPUT_STRIP_TRAILING_WHITESPACE
-)
-add_definitions("-DVERSION=\"${GIT_VERSION}\"")
-option(DOUBLE_PRECISION "Build Xcompact with double precision" ON)
-if (DOUBLE_PRECISION)
-  add_definitions("-DDOUBLE_PREC")
-endif()
-
-option(SINGLE_PRECISION_OUTPUT "Build XCompact with output in single precision" OFF)
-if (SINGLE_PRECISION_OUTPUT)
-  add_definitions("-DSAVE_SINGLE")
-endif()
diff --git a/cmake/compilers/D2D_flags_cray.cmake b/cmake/compilers/D2D_flags_cray.cmake
new file mode 100644
index 00000000..20b16b50
--- /dev/null
+++ b/cmake/compilers/D2D_flags_cray.cmake
@@ -0,0 +1,5 @@
+#Compilers Flags for Cray
+
+set(D2D_FFLAGS "-eF -g -N 1023")
+set(D2D_FFLAGS_RELEASE "-O3")
+set(D2D_FFLAGS_DEBUG   "-O0 -g")
diff --git a/cmake/compilers/D2D_flags_fujitsu.cmake b/cmake/compilers/D2D_flags_fujitsu.cmake
new file mode 100644
index 00000000..694929d5
--- /dev/null
+++ b/cmake/compilers/D2D_flags_fujitsu.cmake
@@ -0,0 +1,2 @@
+
+ciao
diff --git a/cmake/compilers/D2D_flags_gnu.cmake b/cmake/compilers/D2D_flags_gnu.cmake
new file mode 100644
index 00000000..e4e1ab8b
--- /dev/null
+++ b/cmake/compilers/D2D_flags_gnu.cmake
@@ -0,0 +1,12 @@
+# Flags for GNU compiler
+set(D2D_FFLAGS "-cpp -std=f2008 -ffree-line-length-none")
+if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
+  message(STATUS "Set New Fortran basic flags")
+  set(D2D_FFLAGS "${D2D_FFLAGS} -fallow-argument-mismatch")
+endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
+set(D2D_FFLAGS_RELEASE "-O3 -march=native")
+set(D2D_FFLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none")
+set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
+if (FIND_OMPI)
+  set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEV} -Wimplicit-procedure -Wimplicit-interface")
+endif()
diff --git a/cmake/compilers/D2D_flags_intel.cmake b/cmake/compilers/D2D_flags_intel.cmake
new file mode 100644
index 00000000..88e39ee5
--- /dev/null
+++ b/cmake/compilers/D2D_flags_intel.cmake
@@ -0,0 +1,7 @@
+# Compilers Flags for Intel
+
+set(D2D_FFLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback")
+set(D2D_FFLAGS_RELEASE "-O3 -ipo")
+set(D2D_FFLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG")
+set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -warn all,noexternal")
+#set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
diff --git a/cmake/compilers/D2D_flags_nag.cmake b/cmake/compilers/D2D_flags_nag.cmake
new file mode 100644
index 00000000..313361af
--- /dev/null
+++ b/cmake/compilers/D2D_flags_nag.cmake
@@ -0,0 +1,5 @@
+# Compiler F;ags for NAG
+
+set(D2D_FFLAGS "-fpp")
+set(D2D_FFLAGS_RELEASE "-O3")
+set(D2D_FFLAGS_DEBUG   "-O0 -g")
diff --git a/cmake/compilers/D2D_flags_nvidia.cmake b/cmake/compilers/D2D_flags_nvidia.cmake
new file mode 100644
index 00000000..68d4254c
--- /dev/null
+++ b/cmake/compilers/D2D_flags_nvidia.cmake
@@ -0,0 +1,37 @@
+#Compilers Flags for NVIDIA
+
+set(D2D_FFLAGS "-cpp -Mfree -Kieee")
+set(D2D_FFLAGS_RELEASE "-O3")
+set(D2D_FFLAGS_DEBUG   "-O0 -g -traceback -Mbounds -Mchkptr -Ktrap=fp")
+if (BUILD_TARGET MATCHES "mpi")
+  set(D2D_FFLAGS_RELEASE "${D2D_FFLAGS_RELEASE} -fast -march=native")
+endif (BUILD_TARGET MATCHES "mpi")
+
+if (BUILD_TARGET MATCHES "gpu")
+  set(D2D_FFLAGS "${D2D_FFLAGS} -Minfo=accel -target=gpu")
+  add_definitions("-D_GPU")
+  if (ENABLE_OPENACC)
+    set(D2D_FFLAGS "${D2D_FFLAGS} -acc")
+  endif()
+  if (ENABLE_CUDA)
+    add_definitions("-DUSE_CUDA")
+    set(D2D_FFLAGS "${D2D_FFLAGS} -cuda")
+    # Add Compute Capabilities and memory managemnt 
+    if (ENABLE_MANAGED)
+      set(D2D_FFLAGS "${D2D_FFLAGS} -gpu=cc${CUDA_ARCH_FIRST},managed,lineinfo")
+    else (ENABLE_MANAGED)
+      set(D2D_FFLAGS "${D2D_FFLAGS} -gpu=cc${CUDA_ARCH_FIRST},lineinfo")
+    endif(ENABLE_MANAGED)
+    # Add NCCL cuFFT
+    if (ENABLE_NCCL)
+      add_definitions("-D_NCCL")
+      set(D2D_FFLAGS "${D2D_FFLAGS} -cudalib=nccl,cufft")
+    else(ENABLE_NCCL)
+      set(D2D_FFLAGS "${D2D_FFLAGS} -cudalib=cufft")
+    endif(ENABLE_NCCL)
+  endif(ENABLE_CUDA)
+  # Add profiler
+  #if (ENABLE_PROFILER)
+  #  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lnvhpcwrapnvtx")
+  #endif(ENABLE_PROFILER)
+endif (BUILD_TARGET MATCHES "gpu")
diff --git a/cmake/fft/fft.cmake b/cmake/fft/fft.cmake
index 8b5bee95..a6064c90 100644
--- a/cmake/fft/fft.cmake
+++ b/cmake/fft/fft.cmake
@@ -31,8 +31,9 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+message(STATUS "SET UP FFT")
 if(${FFT_Choice} MATCHES "fftw")
-	configure_file(${CMAKE_SOURCE_DIR}/cmake/fft/downloadFindFFTW.cmake.in findFFTW-download/CMakeLists.txt)
+  configure_file(${CMAKE_SOURCE_DIR}/cmake/fft/downloadFindFFTW.cmake.in findFFTW-download/CMakeLists.txt)
   execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
           RESULT_VARIABLE result
           WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/findFFTW-download )
@@ -60,4 +61,9 @@ if(${FFT_Choice} MATCHES "fftw")
 
 elseif(${FFT_Choice} MATCHES "mkl")
   find_package(MKL CONFIG REQUIRED)
+elseif(${FFT_Choice} MATCHES "cufft")
+  message(STATUS "Enable cuFFT")
+  if (ENABLE_CUDA)
+    set(CUFFT_FOUND TRUE)
+  endif()
 endif(${FFT_Choice} MATCHES "fftw")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 29065bbb..b534a873 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -16,6 +16,9 @@ if(FFTW_FOUND)
 elseif(MKL_FOUND)
   message (STATUS "Compiling using MKL")
   file(GLOB files_fft $ENV{MKLROOT}/include/mkl_dfti.f90 fft_mkl.f90)
+elseif(CUFFT_FOUND)
+  message (STATUS "Compiling using cuFFT")
+  file(GLOB files_fft fft_cufft.f90)
 else(FFTW_FOUND)
   message (STATUS "Compiling using Generic FFT")
   file(GLOB files_fft glassman.f90 fft_generic.f90)

From a67294dea268c440dba05e857703c847e6804b01 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Sat, 4 Feb 2023 10:15:12 +0000
Subject: [PATCH 131/436] Update flags for Fujitsu compiler

---
 cmake/D2D_Compilers.cmake               | 4 +---
 cmake/compilers/D2D_flags_fujitsu.cmake | 5 ++++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cmake/D2D_Compilers.cmake b/cmake/D2D_Compilers.cmake
index eb9b2f37..6a491b1c 100644
--- a/cmake/D2D_Compilers.cmake
+++ b/cmake/D2D_Compilers.cmake
@@ -38,9 +38,7 @@ elseif (Fortran_COMPILER_NAME MATCHES "NVHPC")
 # 	  FORCE)
 elseif (Fortran_COMPILER_NAME MATCHES "Fujitsu")
   message(STATUS "Setting Fujitsu fortran flags")
-  set(D2D_FFLAGS "-Cpp")
-  set(D2D_FFLAGS_RELEASE "-O3")
-  set(D2D_FFLAGS_DEBUG   "-O0")
+  include(D2D_flags_fujitsu)
 else (Fortran_COMPILER_NAME MATCHES "GNU")
   message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
   message ("Fortran compiler: " ${Fortran_COMPILER_NAME})
diff --git a/cmake/compilers/D2D_flags_fujitsu.cmake b/cmake/compilers/D2D_flags_fujitsu.cmake
index 694929d5..5d1675de 100644
--- a/cmake/compilers/D2D_flags_fujitsu.cmake
+++ b/cmake/compilers/D2D_flags_fujitsu.cmake
@@ -1,2 +1,5 @@
+# Compilers flags for Fujitsu
 
-ciao
+  set(D2D_FFLAGS "-Cpp")
+  set(D2D_FFLAGS_RELEASE "-O3")
+  set(D2D_FFLAGS_DEBUG   "-O0")

From 3c1542258d38f50f32c939123f35849990eb7c0d Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Sat, 4 Feb 2023 11:23:29 +0000
Subject: [PATCH 132/436] Update CMake MPI to force default number of ranks to
 1 and fix issue with mpirun

---
 cmake/D2D_MPI.cmake | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/cmake/D2D_MPI.cmake b/cmake/D2D_MPI.cmake
index 64212c68..78d24c4a 100644
--- a/cmake/D2D_MPI.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -24,6 +24,17 @@ if (MPI_FOUND)
     message(STATUS "MPI FOUND: ${MPI_FOUND}")
     include_directories(SYSTEM ${MPI_INCLUDE_PATH})
     message(STATUS "MPI INCL ALSO FOUND: ${MPI_INCLUDE_PATH}")
+    if (NOT MPI_NUMPROCS_SET)
+      message(STATUS "Reset the number of ranks to 1")
+      set(MPIEXEC_MAX_NUMPROCS "1" CACHE STRING
+          "Set the initial value to 1 rank" FORCE)
+      set(MPI_NUMPROCS_SET 1 CACHE INTERNAL "MPI Ranks set")
+      # Force the mpirun to be coherent with the mpifortran
+      string(REPLACE "mpif90" "mpirun" PATH_TO_MPIRUN "${MPI_Fortran_COMPILER}")
+      message(STATUS "Path to mpirun ${PATH_TO_MPIRUN}")
+      set(MPIEXEC_EXECUTABLE "${PATH_TO_MPIRUN}" CACHE STRING
+          "Force MPIRUN to be consistent with MPI_Fortran_COMPILER" FORCE)
+    endif()
 else (MPI_FOUND)
     message(STATUS "NO MPI include have been found. The executable won't be targeted with MPI include")
     message(STATUS "Code will compile but performaces can be compromised")

From 27adabe16a672e91d8390fafc8d93b877455358b Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 4 Feb 2023 14:25:05 +0000
Subject: [PATCH 133/436] Find MPIRUN in case that FC=mpiifort

---
 cmake/D2D_MPI.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmake/D2D_MPI.cmake b/cmake/D2D_MPI.cmake
index 78d24c4a..a583fa0b 100644
--- a/cmake/D2D_MPI.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -30,7 +30,8 @@ if (MPI_FOUND)
           "Set the initial value to 1 rank" FORCE)
       set(MPI_NUMPROCS_SET 1 CACHE INTERNAL "MPI Ranks set")
       # Force the mpirun to be coherent with the mpifortran
-      string(REPLACE "mpif90" "mpirun" PATH_TO_MPIRUN "${MPI_Fortran_COMPILER}")
+      string(REGEX REPLACE "mpif90" "mpirun" PATH_TO_MPIRUN "${MPI_Fortran_COMPILER}")
+      string(REPLACE "mpiifort" "mpirun" PATH_TO_MPIRUN "${PATH_TO_MPIRUN}")
       message(STATUS "Path to mpirun ${PATH_TO_MPIRUN}")
       set(MPIEXEC_EXECUTABLE "${PATH_TO_MPIRUN}" CACHE STRING
           "Force MPIRUN to be consistent with MPI_Fortran_COMPILER" FORCE)

From 939beff82682c446d8117e8fac6ae90735c621c3 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Sun, 5 Feb 2023 23:02:33 +0000
Subject: [PATCH 134/436] Update CMake build to copy binding bash file for GPU
 test

---
 examples/fft_physical_x/CMakeLists.txt | 16 +++++++++++++---
 examples/fft_physical_z/CMakeLists.txt | 13 +++++++++++--
 examples/halo_test/CMakeLists.txt      | 11 ++++++++++-
 examples/init_test/CMakeLists.txt      | 11 ++++++++++-
 examples/io_test/CMakeLists.txt        | 22 +++++++++++++++++-----
 examples/test2d/CMakeLists.txt         | 11 ++++++++++-
 6 files changed, 71 insertions(+), 13 deletions(-)

diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index 94d34043..4fe1ca9e 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -24,6 +24,16 @@ elseif (MKL_FOUND)
   target_link_libraries(fft_grid_x PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
-add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_x>)
-add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_x>)
-add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_grid_x>)
+if (BUILD_TARGET MATCHES "gpu")
+  set(example_dir "${PROJECT_BINARY_DIR}/examples/fft_physical_x")
+  message(STATUS "Example dir ${example_dir}")
+  install(FILES bind.sh DESTINATION ${example_dir})
+  file(COPY bind.sh DESTINATION ${example_dir})
+  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_x>)
+  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_x>)
+  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh  $<TARGET_FILE:fft_grid_x>)
+else ()  
+  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_x>)
+  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_x>)
+  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_grid_x>)
+endif()
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index ffeb1c64..a094a1d5 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -18,5 +18,14 @@ elseif (MKL_FOUND)
   target_link_libraries(fft_r2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
-add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_z>)
-add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_z>)
+if (BUILD_TARGET MATCHES "gpu")
+  set(example_dir "${PROJECT_BINARY_DIR}/examples/fft_physical_z")
+  message(STATUS "Example dir ${example_dir}")
+  install(FILES bind.sh DESTINATION ${example_dir})
+  file(COPY bind.sh DESTINATION ${example_dir})
+  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_z>)
+  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_z>)
+else ()  
+  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_z>)
+  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_z>)
+endif ()
diff --git a/examples/halo_test/CMakeLists.txt b/examples/halo_test/CMakeLists.txt
index e6a75a8b..951e6b42 100644
--- a/examples/halo_test/CMakeLists.txt
+++ b/examples/halo_test/CMakeLists.txt
@@ -3,4 +3,13 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(halo_test ${files_test})
 target_link_libraries(halo_test PRIVATE decomp2d)
-add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:halo_test>)
+
+if (BUILD_TARGET MATCHES "gpu")
+  set(example_dir "${PROJECT_BINARY_DIR}/examples/halo_test")
+  message(STATUS "Example dir ${example_dir}")
+  install(FILES bind.sh DESTINATION ${example_dir})
+  file(COPY bind.sh DESTINATION ${example_dir})
+  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:halo_test>)
+else ()  
+  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:halo_test>)
+endif ()
diff --git a/examples/init_test/CMakeLists.txt b/examples/init_test/CMakeLists.txt
index 5c5c0bff..19deecd3 100644
--- a/examples/init_test/CMakeLists.txt
+++ b/examples/init_test/CMakeLists.txt
@@ -3,4 +3,13 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(init_test ${files_test})
 target_link_libraries(init_test PRIVATE decomp2d)
-add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:init_test>)
+if (BUILD_TARGET MATCHES "gpu")
+  set(example_dir "${PROJECT_BINARY_DIR}/examples/init_test")
+  message(STATUS "Example dir ${example_dir}")
+  install(FILES bind.sh DESTINATION ${example_dir})
+  file(COPY bind.sh DESTINATION ${example_dir})
+  #file(CHMOD bind.sh DESTINATION ${example_dir} [FILE_PERMISSIONS OWNER_EXECUTE])
+  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:init_test>)
+else ()
+  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:init_test>)
+endif()
diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 89c8ff24..3ea8fdf5 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -18,8 +18,20 @@ target_link_libraries(io_var_test PRIVATE decomp2d)
 target_link_libraries(io_plane_test PRIVATE decomp2d)
 target_link_libraries(io_bench PRIVATE decomp2d)
 
-add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_test>)
-add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_read>)
-add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_var_test> ${PROW} ${PCOL})
-add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_plane_test>)
-add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_bench>)
+if (BUILD_TARGET MATCHES "gpu")
+  set(example_dir "${PROJECT_BINARY_DIR}/examples/io_test")
+  message(STATUS "Example dir ${example_dir}")
+  install(FILES bind.sh DESTINATION ${example_dir})
+  file(COPY bind.sh DESTINATION ${example_dir})
+  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_test>)
+  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_read>)
+  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_var_test> ${PROW} ${PCOL})
+  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_plane_test>)
+  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_bench>)
+else ()
+  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_test>)
+  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_read>)
+  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_var_test> ${PROW} ${PCOL})
+  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_plane_test>)
+  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_bench>)
+endif ()
diff --git a/examples/test2d/CMakeLists.txt b/examples/test2d/CMakeLists.txt
index 408afd4c..3647f829 100644
--- a/examples/test2d/CMakeLists.txt
+++ b/examples/test2d/CMakeLists.txt
@@ -3,4 +3,13 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(test2d ${files_test})
 target_link_libraries(test2d PRIVATE decomp2d)
-add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:test2d>)
+
+if (BUILD_TARGET MATCHES "gpu")
+  set(example_dir "${PROJECT_BINARY_DIR}/examples/test2d")
+  message(STATUS "Example dir ${example_dir}")
+  install(FILES bind.sh DESTINATION ${example_dir})
+  file(COPY bind.sh DESTINATION ${example_dir})
+  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:test2d>)
+else ()
+  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:test2d>)
+endif ()

From aeb3633e6c6605442c91129d2e25def61774a835 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Sun, 5 Feb 2023 23:03:40 +0000
Subject: [PATCH 135/436] Reactivate halo test in Makefile

---
 examples/halo_test/Makefile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/halo_test/Makefile b/examples/halo_test/Makefile
index b39fc632..3c46b58d 100644
--- a/examples/halo_test/Makefile
+++ b/examples/halo_test/Makefile
@@ -17,8 +17,7 @@ halo_test: $(OBJ)
 
 ifeq ($(PARAMOD),gpu)
 check:
-	$(info Halo test temporarily disabled for GPU)
-#	$(MPIRUN) -n $(NP) ./bind.sh ./halo_test
+	$(MPIRUN) -n $(NP) ./bind.sh ./halo_test
 else
 check:
 	$(MPIRUN) -n $(NP) ./halo_test

From 0e891b0d07adc064a3de6d030a95378821de2f83 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sun, 12 Feb 2023 21:08:55 +0000
Subject: [PATCH 136/436] Set default cmake options for MKL

By default Intel was using -i8 with MKL - this seems to be causing
us problems later
---
 cmake/fft/fft.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmake/fft/fft.cmake b/cmake/fft/fft.cmake
index a6064c90..dace35eb 100644
--- a/cmake/fft/fft.cmake
+++ b/cmake/fft/fft.cmake
@@ -60,6 +60,8 @@ if(${FFT_Choice} MATCHES "fftw")
   #add_definitions("-lfftw3 -lfftw3f")
 
 elseif(${FFT_Choice} MATCHES "mkl")
+  set(MKL_INTERFACE "lp64")
+  set(MKL_THREADING "sequential")
   find_package(MKL CONFIG REQUIRED)
 elseif(${FFT_Choice} MATCHES "cufft")
   message(STATUS "Enable cuFFT")

From dab2503a1950c82dac8f9a89ba6b72b87449d118 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Mon, 13 Feb 2023 12:54:17 +0100
Subject: [PATCH 137/436] Update README.md (#149)

---
 README.md | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9ae1dd04..7629c11b 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,38 @@
 
 ## Building
 
-The build system is driven by `cmake`, to generate a build system run
+### Makefile
+
+Different compilers can be set by specifying `CMP`, e.g. `make CMP=intel`
+to build with Intel compilers, see `Makefile` for options.
+
+By default an optimised library will be built, debugging versions of the
+library can be built with `make BUILD=debug`, a development version which 
+additionally sets compile time flags to catch coding errors can be built 
+with `make BUILD=dev` (GNU compilers only currently). The behavior of debug
+and development versions of the library can be changed before the initialization
+using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
+The value provided with the environment variable must be a positive integer below 9999.
+
+On each build of the library (`make`, `make all`) a temporary file `Makefile.settings` with
+all current options (`FFLAGS`, `DEFS`, etc.) will be created, and included
+on subsequent invocations, the user therefore does not need to keep
+specifying options between builds.
+
+To perform a clean build run `make clean` first, this will delete all
+output files, including `Makefile.settings`.
+
+On each build of the library (`make`, `make all`) a temporary file `Makefile.settings` with
+all current options (`FFLAGS`, `DEFS`, etc.) will be created, and included
+on subsequent invocations, the user therefore does not need to keep
+specifying options between builds.
+
+To perform a clean build run `make clean` first, this will delete all
+output files, including `Makefile.settings`.
+
+### CMake
+
+The new experimental build system is driven by `cmake`, to generate a build system run
 ```
 cmake -B build .
 ```
@@ -49,6 +80,8 @@ make -C build clean
 
 ## Testing and examples
 
+### Makefile
+
 Various example code to exercise 2decomp functionality can be found under ``examples/``
 and can be built from the top-level directory by executing
 ```
@@ -64,6 +97,14 @@ setting ``MPIRUN``, for example to run tests on 4 ranks using an ``mpiexec`` lau
 make NP=4 MPIRUN=${HOME}/bin/mpiexec check
 ```
 
+### CMake
+
+After building the library can be tested by running
+```
+make -C build test
+```
+which uses the `ctest` utility.
+
 ## GPU compilation
 
 The library can perform multi GPU offoloading using the NVHPC compiler suite for NVIDIA hardware. 

From f25037dca0bd1c051e528b235c850afc4e28b191 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 13 Feb 2023 11:57:36 +0000
Subject: [PATCH 138/436] If you line-break you are not inline! (#146)

---
 src/decomp_2d_init_fin.f90 | 29 +++++++++++++----------------
 src/fft_common.f90         | 14 ++++++--------
 src/fft_fftw3_f03.f90      | 14 ++++++--------
 src/io.f90                 | 36 +++++++++++++++++++++++-------------
 src/io_read_inflow.f90     |  6 ++++--
 src/io_write_plane.inc     |  6 ++++--
 src/log.f90                |  7 +++----
 7 files changed, 59 insertions(+), 53 deletions(-)

diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index 001e7acd..f5afdff2 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -55,10 +55,9 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
      end if
 
      ! Safety check
-     if (MPI_SUCCESS /= 0) call decomp_2d_abort(__FILE__, &
-                                                __LINE__, &
-                                                MPI_SUCCESS, &
-                                                "MPI error check is broken")
+     if (MPI_SUCCESS /= 0) then
+        call decomp_2d_abort(__FILE__, __LINE__, MPI_SUCCESS, "MPI error check is broken")
+     end if
      if (nx <= 0) call decomp_2d_abort(__FILE__, __LINE__, nx, "Invalid value for nx")
      if (ny <= 0) call decomp_2d_abort(__FILE__, __LINE__, ny, "Invalid value for ny")
      if (nz <= 0) call decomp_2d_abort(__FILE__, __LINE__, nz, "Invalid value for nz")
@@ -66,17 +65,15 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
      ! If the external code has not set nrank and nproc
      if (nrank == -1) then
         call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                              __LINE__, &
-                                              ierror, &
-                                              "MPI_COMM_RANK")
+        if (ierror /= 0) then
+           call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+        end if
      end if
      if (nproc == -1) then
         call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                              __LINE__, &
-                                              ierror, &
-                                              "MPI_COMM_SIZE")
+        if (ierror /= 0) then
+           call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+        end if
      end if
 #ifdef DEBUG
      ! Check if a modification of the debug level is needed
@@ -304,10 +301,10 @@ subroutine decomp_2d_debug
 
      ! Conversion to integer if possible
      read (val, '(i4)', iostat=ierror) decomp_debug
-     if (ierror /= 0) call decomp_2d_warning(__FILE__, &
-                                             __LINE__, &
-                                             ierror, &
-                                             "Error when reading DECOMP_2D_DEBUG : "//val)
+     if (ierror /= 0) then
+        call decomp_2d_warning(__FILE__, __LINE__, ierror, &
+             "Error when reading DECOMP_2D_DEBUG : "//val)
+     end if
 
   end subroutine decomp_2d_debug
 #endif
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index a9cf09a3..f8ae3fe9 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -249,10 +249,9 @@ function decomp_2d_fft_get_ph()
 
    type(decomp_info), pointer :: decomp_2d_fft_get_ph
 
-   if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  -1, &
-                                                  'FFT library must be initialised first')
+   if (.not. associated(ph)) then
+      call decomp_2d_abort(__FILE__, __LINE__, -1, 'FFT library must be initialised first')
+   end if
    decomp_2d_fft_get_ph => ph
 
 end function decomp_2d_fft_get_ph
@@ -268,10 +267,9 @@ function decomp_2d_fft_get_sp()
 
    type(decomp_info), pointer :: decomp_2d_fft_get_sp
 
-   if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  -1, &
-                                                  'FFT library must be initialised first')
+   if (.not. associated(ph)) then
+      call decomp_2d_abort(__FILE__, __LINE__, -1, 'FFT library must be initialised first')
+   end if
    decomp_2d_fft_get_sp => sp
 
 end function decomp_2d_fft_get_sp
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 0ac3898e..d7ddc03f 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -274,10 +274,9 @@ function decomp_2d_fft_get_ph()
 
       type(decomp_info), pointer :: decomp_2d_fft_get_ph
 
-      if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                     __LINE__, &
-                                                     -1, &
-                                                     'FFT library must be initialised first')
+      if (.not. associated(ph)) then
+         call decomp_2d_abort(__FILE__, __LINE__, -1, 'FFT library must be initialised first')
+      end if
       decomp_2d_fft_get_ph => ph
 
    end function decomp_2d_fft_get_ph
@@ -293,10 +292,9 @@ function decomp_2d_fft_get_sp()
 
       type(decomp_info), pointer :: decomp_2d_fft_get_sp
 
-      if (.not. associated(ph)) call decomp_2d_abort(__FILE__, &
-                                                     __LINE__, &
-                                                     -1, &
-                                                     'FFT library must be initialised first')
+      if (.not. associated(ph)) then
+         call decomp_2d_abort(__FILE__, __LINE__, -1, 'FFT library must be initialised first')
+      end if
       decomp_2d_fft_get_sp => sp
 
    end function decomp_2d_fft_get_sp
diff --git a/src/io.f90 b/src/io.f90
index b6d137c7..03411390 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -136,9 +136,10 @@ subroutine decomp_2d_io_init()
       adios2_debug_mode = .true.
 
       call adios2_init(adios, trim(config_file), decomp_2d_comm, adios2_debug_mode, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-                                            "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
-
+      if (ierror /= 0) then
+         call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+              "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
+      end if
       engine_live(:) = .false.
 
 #ifdef PROFILER
@@ -164,8 +165,9 @@ subroutine decomp_2d_io_finalise()
       if (decomp_profiler_io) call decomp_profiler_start("io_fin")
 #endif
       call adios2_finalize(adios, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-                                            "adios2_finalize")
+      if (ierror /= 0) then
+         call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_finalize")
+      end if
 #ifdef PROFILER
       if (decomp_profiler_io) call decomp_profiler_end("io_fin")
 #endif
@@ -442,8 +444,10 @@ subroutine adios2_read_one_real(var, engine_name, varname, io_name)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
       call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-      if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-                                                       "ERROR: trying to read variable without registering first! "//trim(varname))
+      if (.not. var_handle%valid) then
+         call decomp_2d_abort(__FILE__, __LINE__, -1, &
+              "ERROR: trying to read variable without registering first! "//trim(varname))
+      end if
 
       call adios2_variable_steps(nsteps, var_handle, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_variable_steps")
@@ -1342,11 +1346,15 @@ subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_n
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
       call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-      if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-                                                       "ERROR: trying to write variable before registering! "//trim(varname))
+      if (.not. var_handle%valid) then
+         call decomp_2d_abort(__FILE__, __LINE__, -1, &
+              "ERROR: trying to write variable before registering! "//trim(varname))
+      end if
 
-      if (idx < 1) call decomp_2d_abort(__FILE__, __LINE__, idx, &
-                                        "You haven't opened "//trim(io_name)//":"//trim(dirname))
+      if (idx < 1) then
+         call decomp_2d_abort(__FILE__, __LINE__, idx, &
+              "You haven't opened "//trim(io_name)//":"//trim(dirname))
+      end if
 
       if (deferred_writes) then
          write_mode = adios2_mode_deferred
@@ -1442,8 +1450,10 @@ subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplan
             call adios2_define_variable(var_handle, io_handle, varname, data_type, &
                                         ndims, int(sizes, kind=8), int(starts, kind=8), int(subsizes, kind=8), &
                                         adios2_constant_dims, ierror)
-            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-                                                  "adios2_define_variable, ERROR registering variable "//trim(varname))
+            if (ierror /= 0) then
+               call decomp_2d_abort(__FILE__, __LINE__, ierror, &
+                    "adios2_define_variable, ERROR registering variable "//trim(varname))
+            end if
          end if
       else
          call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to register variable with invalid IO!")
diff --git a/src/io_read_inflow.f90 b/src/io_read_inflow.f90
index 9484ddcb..a247192b 100644
--- a/src/io_read_inflow.f90
+++ b/src/io_read_inflow.f90
@@ -76,8 +76,10 @@
 if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
 if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-                                                 "trying to write variable before registering! "//trim(varname))
+if (.not. var_handle%valid) then
+   call decomp_2d_abort(__FILE__, __LINE__, -1, &
+        "trying to write variable before registering!  "//trim(varname))
+end if
 
 !! Note - need to use sync mode as we are using a view into the array - unsure how this works with deferred writes
 ! call adios2_set_step_selection(var_handle, int(0, kind=8), int(1, kind=8), ierror)
diff --git a/src/io_write_plane.inc b/src/io_write_plane.inc
index 6965060e..5db42b72 100644
--- a/src/io_write_plane.inc
+++ b/src/io_write_plane.inc
@@ -161,8 +161,10 @@ call adios2_at_io(io_handle, adios, io_name, ierror)
 if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
 call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
 if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
-if (.not. var_handle%valid) call decomp_2d_abort(__FILE__, __LINE__, -1, &
-                                                 "trying to write variable before registering! "//trim(varname))
+if (.not. var_handle%valid) then
+   call decomp_2d_abort(__FILE__, __LINE__, -1, &
+        "trying to write variable before registering!  "//trim(varname))
+end if
 
 !! Note - need to use sync mode as the array for the output plane gets reused.
 call adios2_put(engine_registry(idx), var_handle, wk2d, adios2_mode_sync, ierror)
diff --git a/src/log.f90 b/src/log.f90
index fa9eb90c..0c8bd975 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -228,10 +228,9 @@ module subroutine d2d_listing(given_io_unit)
             call execute_command_line("env", wait=.true.)
          else
             inquire (unit=io_unit, name=fname, iostat=ierror)
-            if (ierror /= 0) call decomp_2d_abort(__FILE__, &
-                                                  __LINE__, &
-                                                  ierror, &
-                                                  "No name for the log file")
+            if (ierror /= 0) then
+               call decomp_2d_abort(__FILE__, __LINE__, ierror, "No name for the log file")
+            end if
             ! Close the IO unit to print the environment variables
             call d2d_listing_close_unit(io_unit)
             call execute_command_line("env >> "//trim(fname), wait=.true.)

From c1c9fce4e8a60e2788dca785ac3ee9639e3d18c3 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 13 Feb 2023 23:52:18 +0000
Subject: [PATCH 139/436] Add OVERWRITE to Makefile

---
 Makefile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Makefile b/Makefile
index c3a9af46..2fe6f535 100644
--- a/Makefile
+++ b/Makefile
@@ -16,6 +16,7 @@ CMP = gcc# intel,gcc,nagfor,cray,nvhpc
 FFT ?= generic# fftw3,fftw3_f03,generic,mkl,cufft
 PARAMOD = mpi # multicore,gpu
 PROFILER ?= none# none, caliper
+INPLACE = no 
 
 BUILD ?= # debug can be used with gcc
 FCFLAGS ?= # user can set default compiler flags
@@ -29,6 +30,10 @@ LIBDECOMP = decomp2d
 AR = ar
 LIBOPT = rcs
 
+ifeq ($(INPLACE),yes)
+  DEFS += -DOVERWRITE
+endif
+
 #######CMP settings###########
 CMPINC = Makefile.compilers
 include $(CMPINC)

From 9ec7c64e806f2863d84bd8a559413ebfb621a195 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 13 Feb 2023 23:53:35 +0000
Subject: [PATCH 140/436] Add OVERWRITE to CMake build

---
 CMakeLists.txt            | 1 +
 cmake/D2D_Compilers.cmake | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bede957e..e3a818b0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,6 +23,7 @@ if (BUILD_TARGET MATCHES "gpu")
 endif(BUILD_TARGET MATCHES "gpu")
 
 option(ENABLE_PROFILER "Activate the profiler" OFF)
+option(ENABLE_INPLACE "Activate the profiler" OFF)
 
 set(AUTHOR "Stefano Rolfo;Charles Moulinec;Paul Bartholomew")
 set(AUTHOR_DETAILS "stefano.rolfo@stfc.ac.uk;charles.moulinec@stfc.ac.uk;p.bartholomew@epcc.ed.ac.uk")
diff --git a/cmake/D2D_Compilers.cmake b/cmake/D2D_Compilers.cmake
index 6a491b1c..37b73741 100644
--- a/cmake/D2D_Compilers.cmake
+++ b/cmake/D2D_Compilers.cmake
@@ -69,6 +69,10 @@ if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
   add_definitions("-DDEBUG")
 endif (CMAKE_BUILD_TYPE MATCHES "DEBUG")
 
+if (ENABLE_INPLACE)
+  add_definitions("-DOVERWRITE")
+endif ()
+
 execute_process(
   COMMAND git describe --tag --long --always
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}

From d0a7e372f3efcc5f5c1759e8906b1b1f64878ab3 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 14 Feb 2023 18:25:49 +0000
Subject: [PATCH 141/436] Add caliper via ENABLE_PROFILERS

Currently the only options are OFF or caliper
---
 CMakeLists.txt            |  4 +++-
 README.md                 |  7 +++++++
 cmake/D2D_Profilers.cmake | 10 ++++++++++
 src/CMakeLists.txt        | 15 ++++++++++++++-
 4 files changed, 34 insertions(+), 2 deletions(-)
 create mode 100644 cmake/D2D_Profilers.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bede957e..1543f9d4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,7 +22,8 @@ if (BUILD_TARGET MATCHES "gpu")
 
 endif(BUILD_TARGET MATCHES "gpu")
 
-option(ENABLE_PROFILER "Activate the profiler" OFF)
+set (ENABLE_PROFILER "OFF" CACHE STRING "Activate/select the profiler")
+set_property(CACHE ENABLE_PROFILER PROPERTY STRINGS OFF caliper)
 
 set(AUTHOR "Stefano Rolfo;Charles Moulinec;Paul Bartholomew")
 set(AUTHOR_DETAILS "stefano.rolfo@stfc.ac.uk;charles.moulinec@stfc.ac.uk;p.bartholomew@epcc.ed.ac.uk")
@@ -58,6 +59,7 @@ endif (BUILD_TARGET MATCHES "gpu")
 include(D2D_MPI)
 
 include(D2D_Compilers)
+include(D2D_Profilers)
 
 # FFT options
 if (BUILD_TARGET MATCHES "gpu")
diff --git a/README.md b/README.md
index 7629c11b..b81a2d6c 100644
--- a/README.md
+++ b/README.md
@@ -129,6 +129,13 @@ Profiling can be activated in the Makefile. Set the variable `PROFILER` to one o
 3. Profile FFT operations (default : true)
 4. Profile decomp_2d init / fin subroutines (default : true)
 
+Profiling can be activated via `cmake` builds, the recommended approach is to run the initial configuration as follows:
+```
+FC=mpif90 CXX=mpicxx cmake -DENABLE_PROFILER=caliper -B build
+```
+where `ENABLE_PROFILER` is set to the profiling tool desired, currently supported values are: `caliper`.
+Note that when using `caliper` a C++ compiler is required as indicated in the above command line.
+
 ## FFT backends
 
 The library provides a built-in FFT engine and supports various FFT backends : FFTW, Intel oneMKL, Nvidia cuFFT. The FFT engine selected during compilation is available through the variable `D2D_FFT_BACKEND` defined in the module `decomp_2d_fft`. The expected value is defined by the integer constants
diff --git a/cmake/D2D_Profilers.cmake b/cmake/D2D_Profilers.cmake
new file mode 100644
index 00000000..fcf73f1d
--- /dev/null
+++ b/cmake/D2D_Profilers.cmake
@@ -0,0 +1,10 @@
+# Profilers CMakeLists
+
+if (ENABLE_PROFILER)
+
+  if (ENABLE_PROFILER MATCHES "caliper")
+    enable_language(CXX)
+    find_package(caliper REQUIRED)
+  endif()
+  
+endif()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index b534a873..b5fca24a 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -25,7 +25,11 @@ else(FFTW_FOUND)
 endif(FFTW_FOUND)
 list(APPEND files_fft fft_log.f90)
 
-set(SRCFILES ${files_decomp} ${files_fft})
+if (ENABLE_PROFILER)
+  file(GLOB prof_files profiler_${ENABLE_PROFILER}.f90)
+endif()
+
+set(SRCFILES ${files_decomp} ${files_fft} ${prof_files})
 
 add_library(decomp2d STATIC ${SRCFILES})
 if (MPI_FOUND)
@@ -41,6 +45,15 @@ elseif(MKL_FOUND)
   target_include_directories(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
 endif(FFTW_FOUND)
 
+if (ENABLE_PROFILER)
+  target_compile_definitions(decomp2d PUBLIC -DPROFILER)
+
+  if (caliper_FOUND)
+    target_include_directories(decomp2d PUBLIC ${caliper_INCLUDE_DIR}/caliper/fortran)
+    target_link_libraries(decomp2d PRIVATE caliper)
+  endif()
+endif()
+
 install(TARGETS decomp2d
         RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

From bac95e62b3c8b94aca2fcb2df40b3f7c65a9eb44 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 14 Feb 2023 18:28:34 +0000
Subject: [PATCH 142/436] Add caliper_DIR to README

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index b81a2d6c..42d9239e 100644
--- a/README.md
+++ b/README.md
@@ -131,6 +131,7 @@ Profiling can be activated in the Makefile. Set the variable `PROFILER` to one o
 
 Profiling can be activated via `cmake` builds, the recommended approach is to run the initial configuration as follows:
 ```
+export caliper_DIR=/path/to/caliper/install/share/cmake/caliper
 FC=mpif90 CXX=mpicxx cmake -DENABLE_PROFILER=caliper -B build
 ```
 where `ENABLE_PROFILER` is set to the profiling tool desired, currently supported values are: `caliper`.

From a1a5b15d71b364aa01eb0a1aa0f7b0416a08ea9f Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 14 Feb 2023 18:36:37 +0000
Subject: [PATCH 143/436] Add HALO_DEBUG to CMakeLists.txt

Have halo debug trigger on rank 0 - not guaranteed to have 5 or
more ranks in which case it would appear to do nothing
---
 CMakeLists.txt      |  1 +
 src/CMakeLists.txt  |  4 ++++
 src/halo_common.f90 | 18 +++++++++---------
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bede957e..ad9e463d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,6 +23,7 @@ if (BUILD_TARGET MATCHES "gpu")
 endif(BUILD_TARGET MATCHES "gpu")
 
 option(ENABLE_PROFILER "Activate the profiler" OFF)
+option(HALO_DEBUG "Enable halo debugging options" OFF)
 
 set(AUTHOR "Stefano Rolfo;Charles Moulinec;Paul Bartholomew")
 set(AUTHOR_DETAILS "stefano.rolfo@stfc.ac.uk;charles.moulinec@stfc.ac.uk;p.bartholomew@epcc.ed.ac.uk")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index b534a873..b33e9d0c 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -41,6 +41,10 @@ elseif(MKL_FOUND)
   target_include_directories(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
 endif(FFTW_FOUND)
 
+if (HALO_DEBUG)
+  target_compile_definitions(decomp2d PUBLIC -DHALO_DEBUG)
+endif()
+
 install(TARGETS decomp2d
         RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index 662a4ce5..8d3dd69f 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -182,7 +182,7 @@
     if (ipencil == 1) then
 
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
+       if (nrank == 0) then
           write (*, *) 'X-pencil input'
           write (*, *) '=============='
           write (*, *) 'Data on a y-z plane is shown'
@@ -236,7 +236,7 @@
        call MPI_TYPE_FREE(halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
+       if (nrank == 0) then
           write (*, *) 'After exchange in Y'
           do j = ye, ys, -1
              write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
@@ -278,7 +278,7 @@
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
+       if (nrank == 0) then
           write (*, *) 'After exchange in Z'
           do j = ye, ys, -1
              write (*, '(10F4.0)') (out(1, j, k), k=zs, ze)
@@ -291,7 +291,7 @@
     else if (ipencil == 2) then
 
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
+       if (nrank == 0) then
           write (*, *) 'Y-pencil input'
           write (*, *) '=============='
           write (*, *) 'Data on a x-z plane is shown'
@@ -342,7 +342,7 @@
        call MPI_TYPE_FREE(halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
+       if (nrank == 0) then
           write (*, *) 'After exchange in X'
           do i = xe, xs, -1
              write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
@@ -387,7 +387,7 @@
        call MPI_WAITALL(4, requests, status, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
+       if (nrank == 0) then
           write (*, *) 'After exchange in Z'
           do i = xe, xs, -1
              write (*, '(10F4.0)') (out(i, 1, k), k=zs, ze)
@@ -400,7 +400,7 @@
     else if (ipencil == 3) then
 
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
+       if (nrank == 0) then
           write (*, *) 'Z-pencil input'
           write (*, *) '=============='
           write (*, *) 'Data on a x-y plane is shown'
@@ -450,7 +450,7 @@
        call MPI_TYPE_FREE(halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
+       if (nrank == 0) then
           write (*, *) 'After exchange in X'
           do i = xe, xs, -1
              write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)
@@ -498,7 +498,7 @@
        call MPI_TYPE_FREE(halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-       if (nrank == 4) then
+       if (nrank == 0) then
           write (*, *) 'After exchange in Y'
           do i = xe, xs, -1
              write (*, '(10F4.0)') (out(i, j, 1), j=ys, ye)

From 66fcde13fed16f61cb78b9123a7e3464f95d46a0 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Wed, 15 Feb 2023 10:33:22 +0000
Subject: [PATCH 144/436] Update fft_c2c_x test to accept input from user at
 runtime

---
 examples/fft_physical_x/fft_c2c_x.f90 | 40 +++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 40d46ff2..31eea76a 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -17,6 +17,8 @@ program fft_c2c_x
    integer :: p_row = 0, p_col = 0
    integer :: resize_domain
    integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
    integer, parameter :: ntest = 10  ! repeat test this times
 
@@ -37,6 +39,44 @@ program fft_c2c_x
    nx = nx_base*resize_domain
    ny = ny_base*resize_domain
    nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

From 9292a310bb0ad5affdbdb94bcc721e4127ee49a5 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Wed, 15 Feb 2023 16:14:54 +0000
Subject: [PATCH 145/436] Run CMake tests in a dedicated working directory

---
 examples/CMakeLists.txt                |  4 ++
 examples/fft_physical_x/CMakeLists.txt | 38 +++++++++++++-----
 examples/fft_physical_z/CMakeLists.txt | 25 ++++++++----
 examples/halo_test/CMakeLists.txt      | 13 ++++---
 examples/init_test/CMakeLists.txt      | 15 ++++----
 examples/io_test/CMakeLists.txt        | 53 +++++++++++++++++++-------
 examples/test2d/CMakeLists.txt         | 13 ++++---
 7 files changed, 110 insertions(+), 51 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 4241691c..4d19c6e2 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,3 +1,7 @@
+# Create a folder directory for testing
+set(test_dir "${PROJECT_BINARY_DIR}/RunTests")
+file(MAKE_DIRECTORY ${test_dir})
+# add subdirectories
 add_subdirectory(init_test)
 add_subdirectory(test2d)
 add_subdirectory(fft_physical_x)
diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index 4fe1ca9e..e994693a 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -24,16 +24,34 @@ elseif (MKL_FOUND)
   target_link_libraries(fft_grid_x PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
+# Run the test(s)
+set(run_dir "${test_dir}/fft_c2c_x")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
-  set(example_dir "${PROJECT_BINARY_DIR}/examples/fft_physical_x")
-  message(STATUS "Example dir ${example_dir}")
-  install(FILES bind.sh DESTINATION ${example_dir})
-  file(COPY bind.sh DESTINATION ${example_dir})
-  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_x>)
-  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_x>)
-  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh  $<TARGET_FILE:fft_grid_x>)
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_x> WORKING_DIRECTORY ${run_dir})
 else ()  
-  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_x>)
-  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_x>)
-  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_grid_x>)
+  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_x> WORKING_DIRECTORY ${run_dir})
+endif()
+#
+set(run_dir "${test_dir}/fft_r2c_x")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+
+  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_x> WORKING_DIRECTORY ${run_dir})
+else ()  
+  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_x> WORKING_DIRECTORY ${run_dir})
+endif()
+#
+set(run_dir "${test_dir}/fft_grid_x")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh  $<TARGET_FILE:fft_grid_x> WORKING_DIRECTORY ${run_dir})
+else ()  
+  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_grid_x> WORKING_DIRECTORY ${run_dir})
 endif()
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index a094a1d5..0ec8aa88 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -18,14 +18,23 @@ elseif (MKL_FOUND)
   target_link_libraries(fft_r2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif (FFTW_FOUND)
 
+# Run the test(s)
+set(run_dir "${test_dir}/fft_c2c_z")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
-  set(example_dir "${PROJECT_BINARY_DIR}/examples/fft_physical_z")
-  message(STATUS "Example dir ${example_dir}")
-  install(FILES bind.sh DESTINATION ${example_dir})
-  file(COPY bind.sh DESTINATION ${example_dir})
-  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_z>)
-  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_z>)
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_z> WORKING_DIRECTORY ${run_dir})
 else ()  
-  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_z>)
-  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_z>)
+  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_z> WORKING_DIRECTORY ${run_dir})
+endif ()
+# 
+set(run_dir "${test_dir}/fft_r2c_z")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_z> WORKING_DIRECTORY ${run_dir})
+else ()  
+  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_z> WORKING_DIRECTORY ${run_dir})
 endif ()
diff --git a/examples/halo_test/CMakeLists.txt b/examples/halo_test/CMakeLists.txt
index 951e6b42..8834fd69 100644
--- a/examples/halo_test/CMakeLists.txt
+++ b/examples/halo_test/CMakeLists.txt
@@ -4,12 +4,13 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 add_executable(halo_test ${files_test})
 target_link_libraries(halo_test PRIVATE decomp2d)
 
+# Run the test(s)
+set(run_dir "${test_dir}/halo_test")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
-  set(example_dir "${PROJECT_BINARY_DIR}/examples/halo_test")
-  message(STATUS "Example dir ${example_dir}")
-  install(FILES bind.sh DESTINATION ${example_dir})
-  file(COPY bind.sh DESTINATION ${example_dir})
-  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:halo_test>)
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:halo_test> WORKING_DIRECTORY ${run_dir})
 else ()  
-  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:halo_test>)
+  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:halo_test> WORKING_DIRECTORY ${run_dir})
 endif ()
diff --git a/examples/init_test/CMakeLists.txt b/examples/init_test/CMakeLists.txt
index 19deecd3..cfc9535d 100644
--- a/examples/init_test/CMakeLists.txt
+++ b/examples/init_test/CMakeLists.txt
@@ -3,13 +3,14 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(init_test ${files_test})
 target_link_libraries(init_test PRIVATE decomp2d)
+
+# Run the test(s)
+set(run_dir "${test_dir}/init_test")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
-  set(example_dir "${PROJECT_BINARY_DIR}/examples/init_test")
-  message(STATUS "Example dir ${example_dir}")
-  install(FILES bind.sh DESTINATION ${example_dir})
-  file(COPY bind.sh DESTINATION ${example_dir})
-  #file(CHMOD bind.sh DESTINATION ${example_dir} [FILE_PERMISSIONS OWNER_EXECUTE])
-  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:init_test>)
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:init_test> WORKING_DIRECTORY ${run_dir})
 else ()
-  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:init_test>)
+  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:init_test> WORKING_DIRECTORY ${run_dir})
 endif()
diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 3ea8fdf5..9a30f83f 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -18,20 +18,45 @@ target_link_libraries(io_var_test PRIVATE decomp2d)
 target_link_libraries(io_plane_test PRIVATE decomp2d)
 target_link_libraries(io_bench PRIVATE decomp2d)
 
+# Run the test(s)
+set(run_dir "${test_dir}/io_test")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
-  set(example_dir "${PROJECT_BINARY_DIR}/examples/io_test")
-  message(STATUS "Example dir ${example_dir}")
-  install(FILES bind.sh DESTINATION ${example_dir})
-  file(COPY bind.sh DESTINATION ${example_dir})
-  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_test>)
-  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_read>)
-  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_var_test> ${PROW} ${PCOL})
-  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_plane_test>)
-  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_bench>)
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_test> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_read> WORKING_DIRECTORY ${run_dir})
 else ()
-  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_test>)
-  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_read>)
-  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_var_test> ${PROW} ${PCOL})
-  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_plane_test>)
-  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_bench>)
+  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_test> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_read> WORKING_DIRECTORY ${run_dir})
+endif ()
+# Run the test(s)
+set(run_dir "${test_dir}/io_var_test")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_var_test> ${PROW} ${PCOL} WORKING_DIRECTORY ${run_dir})
+else ()
+  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_var_test> ${PROW} ${PCOL} WORKING_DIRECTORY ${run_dir})
+endif ()
+# Run the test(s)
+set(run_dir "${test_dir}/io_plane_test")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_plane_test> WORKING_DIRECTORY ${run_dir})
+else ()
+  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_plane_test> WORKING_DIRECTORY ${run_dir})
+endif ()
+# Run the test(s)
+set(run_dir "${test_dir}/io_bench")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_bench> WORKING_DIRECTORY ${run_dir})
+else ()
+  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_bench> WORKING_DIRECTORY ${run_dir})
 endif ()
diff --git a/examples/test2d/CMakeLists.txt b/examples/test2d/CMakeLists.txt
index 3647f829..f364edee 100644
--- a/examples/test2d/CMakeLists.txt
+++ b/examples/test2d/CMakeLists.txt
@@ -4,12 +4,13 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 add_executable(test2d ${files_test})
 target_link_libraries(test2d PRIVATE decomp2d)
 
+# Run the test(s)
+set(run_dir "${test_dir}/test2d")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
-  set(example_dir "${PROJECT_BINARY_DIR}/examples/test2d")
-  message(STATUS "Example dir ${example_dir}")
-  install(FILES bind.sh DESTINATION ${example_dir})
-  file(COPY bind.sh DESTINATION ${example_dir})
-  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:test2d>)
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:test2d> WORKING_DIRECTORY ${run_dir})
 else ()
-  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:test2d>)
+  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:test2d> WORKING_DIRECTORY ${run_dir})
 endif ()

From 90ec6d671aa1b0b01c8ef4bfcfebb0b158819260 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 15 Feb 2023 23:41:24 +0000
Subject: [PATCH 146/436] Set ENABLE_NCCL OFF by default

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 86c9e265..def84c58 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -12,7 +12,7 @@ if (BUILD_TARGET MATCHES "gpu")
   option(ENABLE_OPENACC "Allow user to activate/deactivate OpenACC support" ON)
   option(ENABLE_CUDA "Allow user to activate/deactivate CUDA support" ON)
   option(ENABLE_MANAGED "Allow user to activate/deactivate automatic memory managment from NVHPC" OFF)
-  option(ENABLE_NCCL "Allow user to activate/deactivate Collective Comunication NCCL" ON)
+  option(ENABLE_NCCL "Allow user to activate/deactivate Collective Comunication NCCL" OFF)
 
   if (ENABLE_CUDA)
     message(STATUS "Before enable CUDA")

From ffa3cfa14637f9d3678a4bb517f31ac869121780 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Thu, 16 Feb 2023 00:58:51 +0000
Subject: [PATCH 147/436] Update README with latest CMake instructions

---
 README.md | 66 ++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 43 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 7629c11b..64533af3 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ to build with Intel compilers, see `Makefile` for options.
 By default an optimised library will be built, debugging versions of the
 library can be built with `make BUILD=debug`, a development version which 
 additionally sets compile time flags to catch coding errors can be built 
-with `make BUILD=dev` (GNU compilers only currently). The behavior of debug
+with `make BUILD=dev` (GNU compilers only currently). The behaviour of debug
 and development versions of the library can be changed before the initialization
 using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
 The value provided with the environment variable must be a positive integer below 9999.
@@ -33,49 +33,48 @@ output files, including `Makefile.settings`.
 
 ### CMake
 
-The new experimental build system is driven by `cmake`, to generate a build system run
+The new experimental build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 
 ```
-cmake -B build .
+export FC=my_mpif90
 ```
-from the `2decomp&fft` root directory.
-It may be necessary to override the Fortran compiler if the wrong one is picked up (due to, e.g.
-`PATH` ordering), for example to specify the `MPICH` compiler on Ubuntu
+To generate the build system run
 ```
-FC=mpif90.mpich cmake -B build .
+cmake -S $path_to_sources -B $path_to_build_directory -DOPTION1 -DOPTION2 ...
 ```
-This should generate a directory `build/` containing the build system, this can be further
-configured using the `ccmake` utility
+If the directory do no exist it will be generated and it will contain the configuration files. The configuration can be further
+edited by using the `ccmake` utility as
 ```
-ccmake build/
+ccmake $path_to_build_directory
 ```
 and editing as desired, variables that are likely of interest are: `CMAKE_BUILD_TYPE` and `FFT_Choice`;
 additional variables can be shown by entering "advanced mode" by pressing `t`.
 By default a `RELEASE` build will built, other options for `CMAKE_BUILD_TYPE` are `DEBUG` and `DEV` which
 turn on debugging flags and additionally try to catch coding errors at compile time, respectively.
-The behavior of debug and development versions of the library can be changed before the
+The behaviour of debug and development versions of the library can be changed before the
 initialization using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
 The value provided with the environment variable must be a positive integer below 9999.
 
-Once the build system has been configured, build `2decomp&fft` by running
+Two `BUILD_TARGETS` are available namely `mpi` and `gpu`.  For `mpi` target no additional options should be required. whereas for `gpu` extra options are necessary at the configure stage. Please see section [GPU Compilation](#gpu-compilation)
+
+Once the build system has been configured, you can build `2decomp&fft` by running
 ```
-make -j <nproc> -C build
+cmake --build $path_to_build_directory -j <nproc>
 ```
-appending ```VERBOSE=1``` will display additional information about the build, such as compiler flags.
+appending `-v` will display additional information about the build, such as compiler flags.
+
 After building the library can be tested by running
 ```
-make -C build test
+ctest --test-dir $path_to_build_directory
 ```
-which uses the `ctest` utility.
-Finally, install the library by running
+Options can be added to change the level of verbosity. Finally, the build library can be installed by running 
 ```
-make -C build install
+cmake --install $path_to_build_directory
 ```
-which will install the library under `build/opt` by default, this location can be configured using
-`ccmake` as described above and setting the variable `CMAKE_INSTALL_PREFIX`.
+The default location for `libdecomp2d.a` is `$path_to_build_directory/opt/lib`or  `$path_to_build_directory/opt/lib64` unless the variable `CMAKE_INSTALL_PREFIX` is modified
 
 Occasionally a clean build is required, this can be performed by running
 ```
-make -C build clean
+cmake --build $path_to_build_directory --target clean
 ```
 
 ## Testing and examples
@@ -101,15 +100,16 @@ make NP=4 MPIRUN=${HOME}/bin/mpiexec check
 
 After building the library can be tested by running
 ```
-make -C build test
+ctest --test-dir $path_to_build_directory
 ```
-which uses the `ctest` utility.
+which uses the `ctest` utility. By default tests are performed in serial, but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS`.  For the GPU implementation please be aware that it is based on a single MPI rank per GPU. Therefore to test multiple GPUs, please use the maximum number of available GPUs and not the maximum number of MPI tasks that are available in the system/node. 
 
 ## GPU compilation
 
 The library can perform multi GPU offoloading using the NVHPC compiler suite for NVIDIA hardware. 
 The implementation is based on CUDA-aware MPI and NVIDIA Collective Communication Library (NCCL).
 The FFT is based on cuFFT. 
+### Makefile
 To compile the library for GPU it is possible to execute the following
 ```
 make CMP=nvhpc FFT=cufft PARAMOD=gpu CUFFT_PATH=PATH_TO_NVHPC/Vers/Linux_x86_64/Vers/compilers/ 
@@ -119,7 +119,26 @@ NCCL is not activated by default. If NCCL is installed/required use `NCCL=yes`.
 The current implementation relays also on opeanACC
 and on automatic optimization of `do concurrent` loops.
 By default the compute architecture for the GPU is 80 (i.e. Ampere), to change it use `CCXY=XY` 
+### CMake
+To properly configure for GPU build the following needs to be used 
+```
+cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_TARGET=gpu
+```
+By default CUDA aware MPI will be used together with `cuFFT` for the FFT library. The configure will automatically look for the GPU architecture available on the system. If you are building on a HPC system please use a computing node for the installation. Useful variable to be added are 
+
+ - `-DENABLE_NCCL=yes` to activate the NCCL collectives
+ - `-DENABLE_MANAGED=yes` to activate the automatic memory management form the NVHPC compiler
+If you are getting the following error
+```
+-- The CUDA compiler identification is unknown  
+CMake Error at /usr/share/cmake/Modules/CMakeDetermineCUDACompiler.cmake:633 (message):  
+Failed to detect a default CUDA architecture. 
+```
+It is possible that your default C compiler is too recent and not supported by `nvcc` . You might be able to solve the issue by adding 
+ - `-DCMAKE_CUDA_HOST_COMPILER=$supported_gcc`
  
+ At the moment the supported CUDA host compilers are `gcc11` and earlier. 
+
 ## Profiling
 
 Profiling can be activated in the Makefile. Set the variable `PROFILER` to one of the supported profilers (only `caliper` currently). If using `caliper`, provide the installation path in the variable `CALIPER_PATH`. When the profiling is active, one can tune it before calling `decomp_2d_init` using the subroutine `decomp_profiler_prep`. The input argument for this subroutine is a logical array of size 4. Each input allow activation / deactivation of the profiling as follows :
@@ -248,3 +267,4 @@ make -j
 make test
 make install
 ```
+

From 8405753aff48e8685c8b9104bcdce9c8b97f0720 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Thu, 16 Feb 2023 11:38:20 +0100
Subject: [PATCH 148/436] Update README.md

---
 README.md | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 64533af3..238e68a3 100644
--- a/README.md
+++ b/README.md
@@ -62,10 +62,8 @@ cmake --build $path_to_build_directory -j <nproc>
 ```
 appending `-v` will display additional information about the build, such as compiler flags.
 
-After building the library can be tested by running
-```
-ctest --test-dir $path_to_build_directory
-```
+After building the library can be tested. Please see section [Testting and examples](#testing-and-examples)
+
 Options can be added to change the level of verbosity. Finally, the build library can be installed by running 
 ```
 cmake --install $path_to_build_directory

From 665c5247182327da4798f5f029811928691dc3aa Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Thu, 16 Feb 2023 11:38:37 +0100
Subject: [PATCH 149/436] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 238e68a3..1cb06c83 100644
--- a/README.md
+++ b/README.md
@@ -62,7 +62,7 @@ cmake --build $path_to_build_directory -j <nproc>
 ```
 appending `-v` will display additional information about the build, such as compiler flags.
 
-After building the library can be tested. Please see section [Testting and examples](#testing-and-examples)
+After building the library can be tested. Please see section [Testing and examples](#testing-and-examples)
 
 Options can be added to change the level of verbosity. Finally, the build library can be installed by running 
 ```

From c528d9d583a0207f0452a63fd6b850990af8a213 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Thu, 16 Feb 2023 11:40:42 +0100
Subject: [PATCH 150/436] Update README.md

---
 README.md | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/README.md b/README.md
index 1cb06c83..cbc4a540 100644
--- a/README.md
+++ b/README.md
@@ -223,10 +223,6 @@ This variable is automatically added in GPU builds.
 
 This variable is valid only for GPU builds. The NVIDIA Collective Communication Library (NCCL) implements multi-GPU and multi-node communication primitives optimized for NVIDIA GPUs and Networking.
 
-#### OCC
-
-This variable is not supported
-
 #### SHM
 
 This variable is not supported.

From 479718eb019dbd46cf522c449bcc10a4a7ad8a49 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Thu, 16 Feb 2023 11:48:42 +0000
Subject: [PATCH 151/436] Update sources for all tests to receive inline input

---
 examples/fft_physical_x/fft_grid_x.f90 | 39 +++++++++++
 examples/fft_physical_x/fft_r2c_x.f90  | 40 ++++++++++++
 examples/fft_physical_z/fft_c2c_z.f90  | 41 +++++++++++-
 examples/fft_physical_z/fft_r2c_z.f90  | 41 +++++++++++-
 examples/halo_test/halo_test.f90       | 57 +++++++++++++++-
 examples/init_test/init_test.f90       | 85 +++++++++++++++++++-----
 examples/io_test/io_bench.f90          | 58 ++++++++++++++++-
 examples/io_test/io_plane_test.f90     | 58 +++++++++++++++--
 examples/io_test/io_read.f90           | 65 +++++++++++++++++--
 examples/io_test/io_test.f90           | 62 +++++++++++++++++-
 examples/io_test/io_var_test.f90       | 90 ++++++++++++++++++--------
 examples/test2d/test2d.f90             | 62 ++++++++++++++++--
 12 files changed, 632 insertions(+), 66 deletions(-)

diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 52fe8cd8..c0d12d3e 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -17,6 +17,8 @@ program fft_physical_x
    integer :: p_row = 0, p_col = 0
    integer :: resize_domain
    integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
    integer, parameter :: ntest = 10  ! repeat test this times
 
@@ -37,6 +39,43 @@ program fft_physical_x
    nx = nx_base*resize_domain
    ny = ny_base*resize_domain
    nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
    call decomp_2d_init(nx + 1, ny + 1, nz + 1, p_row, p_col)
 
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 1724ac1f..908e19ea 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -17,6 +17,8 @@ program fft_r2c_x
    integer :: p_row = 0, p_col = 0
    integer :: resize_domain
    integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
    integer, parameter :: ntest = 10  ! repeat test this times
 
@@ -38,6 +40,44 @@ program fft_r2c_x
    nx = nx_base*resize_domain
    ny = ny_base*resize_domain
    nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index f76b0cf0..4e80b6ca 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -12,12 +12,13 @@ program fft_c2c_z
 
    implicit none
 
-   !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
    integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
    integer :: nx, ny, nz
    integer :: p_row = 0, p_col = 0
    integer :: resize_domain
    integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
    integer, parameter :: ntest = 10  ! repeat test this times
 
@@ -38,6 +39,44 @@ program fft_c2c_z
    nx = nx_base*resize_domain
    ny = ny_base*resize_domain
    nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 9f7fbaf9..6e786373 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -12,12 +12,13 @@ program fft_r2c_z
 
    implicit none
 
-   !integer, parameter :: nx_base=4, ny_base=2, nz_base=3
    integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
    integer :: nx, ny, nz
    integer :: p_row = 0, p_col = 0
    integer :: resize_domain
    integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
    integer, parameter :: ntest = 10  ! repeat test this times
 
@@ -39,6 +40,44 @@ program fft_r2c_z
    nx = nx_base*resize_domain
    ny = ny_base*resize_domain
    nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 267a8296..5fed1ba2 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -9,11 +9,22 @@ program halo_test
    use mpi
 
    use decomp_2d
+   use decomp_2d_constants
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
 
    implicit none
 
-   integer, parameter :: nx = 171, ny = 132, nz = 113
+   integer, parameter :: nx_base = 65, ny_base = 48, nz_base = 33
+   integer :: nx, ny, nz
    integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
    real(mytype), allocatable, dimension(:, :, :) :: u1, v1, w1, div1
    real(mytype), allocatable, dimension(:, :, :) :: u2, v2, w2, div2
@@ -33,6 +44,50 @@ program halo_test
    logical :: passing, all_pass
 
    call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
 
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 25953391..46c2d0cb 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -6,24 +6,75 @@ program init_test
 
    use MPI
    use decomp_2d
+   use decomp_2d_constants
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
 
    implicit none
 
-   integer, parameter :: nx = 5
-   integer, parameter :: ny = 6
-   integer, parameter :: nz = 7
-   integer, parameter :: nexpect = nx*ny*nz
-
-   integer :: p_row, p_col
-
-   integer :: ierr
-
-   call MPI_Init(ierr)
-
-   p_row = 0; p_col = 0
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
+   integer :: nexpect 
+   integer :: ierror
+
+   call MPI_Init(ierror)
+
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+   nexpect = nx * ny *nz
    call run(p_row, p_col)
 
-   call MPI_Finalize(ierr)
+   call MPI_Finalize(ierror)
 
 contains
 
@@ -57,16 +108,18 @@ subroutine check_axis(axis)
          sizes = zsize
       else
          sizes = 0
-         print *, "ERROR: unknown axis requested!"
+         if (nrank==0) print *, "ERROR: unknown axis requested!"
          stop 1
       end if
 
       suml = product(sizes)
-      call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierr)
+      call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierror)
 
       if (sumg /= nexpect) then
-         print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
+         if (nrank==0) print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
          stop 1
+      else
+         if (nrank==0) print *, "Init Test pass for axis ", axis
       end if
 
    end subroutine check_axis
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index b1cabb76..d921275f 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -3,11 +3,20 @@ program io_bench
    use decomp_2d
    use decomp_2d_io
    use MPI
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
 
    implicit none
 
-   integer, parameter :: nx = 100, ny = 100, nz = 100
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
    integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
    real(mytype), allocatable, dimension(:, :, :) :: u1
 
@@ -15,8 +24,51 @@ program io_bench
    integer :: ierror
 
    call MPI_INIT(ierror)
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+   
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    call alloc_x(u1, .true.)
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 5a188916..97de5a7d 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -17,10 +17,15 @@ program io_plane_test
 
    implicit none
 
-   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
    integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
-   real(mytype), dimension(nx, ny, nz) :: data1
+   real(mytype), allocatable, dimension(:, :, :) :: data1
    real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
 
    real(mytype), allocatable, dimension(:, :, :) :: work
@@ -31,11 +36,55 @@ program io_plane_test
    logical :: found
 
    call MPI_INIT(ierror)
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+   
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! ***** global data *****
+   allocate(data1(nx,ny,nz))
    m = 1
    do k = 1, nz
       do j = 1, ny
@@ -158,6 +207,7 @@ program io_plane_test
    end if
 
    deallocate (u1, u2, u3)
+   deallocate (data1)
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
 
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 4803cf3c..eaf53e1e 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -4,19 +4,27 @@ program io_read
 
    use decomp_2d
    use decomp_2d_io
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
 
    implicit none
 
-   integer, parameter :: nx = 17, ny = 13, nz = 11
-   ! use different number of processes
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
    integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
 #ifdef COMPLEX_TEST
-   complex(mytype), dimension(nx, ny, nz) :: data1
+   complex(mytype), allocatable, dimension(:, :, :) :: data1
 
    complex(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #else
-   real(mytype), dimension(nx, ny, nz) :: data1
+   real(mytype), allocatable, dimension(:, :, :) :: data1
 
    real(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #endif
@@ -26,11 +34,55 @@ program io_read
    integer :: i, j, k, m, ierror
 
    call MPI_INIT(ierror)
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+   
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! ***** global data *****
+   allocate(data1(nx,ny,nz))
    m = 1
    do k = 1, nz
       do j = 1, ny
@@ -80,6 +132,7 @@ program io_read
    end do
 
    deallocate (u1b, u2b, u3b)
+   deallocate (data1)
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
 
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index e4e5979c..eddb76b6 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -4,19 +4,28 @@ program io_test
 
    use decomp_2d
    use decomp_2d_io
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
 
    implicit none
 
-   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
    integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
 #ifdef COMPLEX_TEST
-   complex(mytype), dimension(nx, ny, nz) :: data1
+   complex(mytype), allocatable, dimension(:, :, :) :: data1
 
    complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
    complex(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
 #else
-   real(mytype), dimension(nx, ny, nz) :: data1
+   real(mytype), allocatable, dimension(:, :, :) :: data1
 
    real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
    real(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
@@ -29,9 +38,55 @@ program io_test
    integer :: xen1, xen2, xen3
 
    call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+   
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! ***** global data *****
+   allocate(data1(nx,ny,nz))
    m = 1
    do k = 1, nz
       do j = 1, ny
@@ -139,6 +194,7 @@ program io_test
 
    deallocate (u1, u2, u3)
    deallocate (u1b, u2b, u3b)
+   deallocate (data1)
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
 
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 961217e1..ca0eb0e0 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -3,21 +3,30 @@
 
 program io_var_test
 
+   use mpi
    use decomp_2d
    use decomp_2d_io
-   use MPI
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
 
    implicit none
 
-   integer, parameter :: nx = 17, ny = 13, nz = 11
-   integer :: p_row, p_col
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
    real(mytype), parameter :: eps = 1.0E-7
 
    ! for global data
-   real(mytype), dimension(nx, ny, nz) :: data1
+   real(mytype), allocatable, dimension(:, :, :) :: data1
    real(mytype), allocatable, dimension(:, :, :) :: data1_large
-   complex(mytype), dimension(nx, ny, nz) :: cdata1
+   complex(mytype), allocatable, dimension(:, :, :) :: cdata1
 
    ! for distributed data
    real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
@@ -39,38 +48,65 @@ program io_var_test
    TYPE(DECOMP_INFO) :: large
 
    integer :: i, j, k, m, ierror, fh
-   character(len=15) :: filename, arg
+   character(len=15) :: filename
    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
 
-   allocate (data1_large(nx*2, ny*2, nz*2))
 
    call MPI_INIT(ierror)
-   call MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierror)
-   call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-
-   ! Defaults
-   p_row = 0
-   p_col = 0
-
-   ! Read commandline input
-   i = command_argument_count()
-   if (i /= 2) then
-      call MPI_ABORT(MPI_COMM_WORLD, 1, ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
    else
-      call get_command_argument(1, arg)
-      read (arg, '(I10)') i
-      p_row = i
-      call get_command_argument(2, arg)
-      read (arg, '(I10)') i
-      p_col = i
-   end if
-
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+   
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! also create a data set over a large domain
    call decomp_info_init(nx*2, ny*2, nz*2, large)
 
    ! initialise global data
+   allocate (data1(nx, ny, nz))
+   allocate (cdata1(nx, ny, nz))
+   allocate (data1_large(nx*2, ny*2, nz*2))
    m = 1
    do k = 1, nz
       do j = 1, ny
@@ -295,9 +331,9 @@ program io_var_test
    deallocate (u1_b, u2_b, u3_b)
    deallocate (u1l_b, u2l_b, u3l_b)
    deallocate (cu1_b, cu2_b, cu3_b)
+   deallocate (data1, cdata1, data1_large)
    call decomp_info_finalize(large)
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
-   deallocate (data1_large)
 
 end program io_var_test
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 5e5e64fb..4989e4fc 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -1,16 +1,24 @@
 program test2d
 
    use mpi
-
    use decomp_2d
-   ! use decomp_2d_io
+   use decomp_2d_constants
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
 
    implicit none
 
-   integer, parameter :: nx = 17, ny = 13, nz = 11
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
    integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
 
-   real(mytype), dimension(nx, ny, nz) :: data1
+   real(mytype), allocatable, dimension(:, :, :) :: data1
 
    real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
 
@@ -27,9 +35,54 @@ program test2d
    error_flag = .false.
    call MPI_INIT(ierror)
    if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_INIT")
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! ***** global data *****
+   allocate(data1(nx,ny,nz))
    m = 1
    do k = 1, nz
       do j = 1, ny
@@ -191,6 +244,7 @@ program test2d
    call MPI_FINALIZE(ierror)
    !$acc end data
    deallocate (u1, u2, u3)
+   deallocate (data1)
 
 end program test2d
 

From 404f92ab95cf9fb31323a21f010d7b665b5b995b Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Thu, 16 Feb 2023 12:11:49 +0000
Subject: [PATCH 152/436] Minor README changes

---
 README.md | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 8e412f1e..e090b037 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@ To generate the build system run
 ```
 cmake -S $path_to_sources -B $path_to_build_directory -DOPTION1 -DOPTION2 ...
 ```
-If the directory do no exist it will be generated and it will contain the configuration files. The configuration can be further
+If the directory does not exist it will be generated and it will contain the configuration files. The configuration can be further
 edited by using the `ccmake` utility as
 ```
 ccmake $path_to_build_directory
@@ -54,7 +54,7 @@ The behaviour of debug and development versions of the library can be changed be
 initialization using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
 The value provided with the environment variable must be a positive integer below 9999.
 
-Two `BUILD_TARGETS` are available namely `mpi` and `gpu`.  For `mpi` target no additional options should be required. whereas for `gpu` extra options are necessary at the configure stage. Please see section [GPU Compilation](#gpu-compilation)
+Two `BUILD_TARGETS` are available namely `mpi` and `gpu`.  For the `mpi` target no additional options should be required. whereas for `gpu` extra options are necessary at the configure stage. Please see section [GPU Compilation](#gpu-compilation)
 
 Once the build system has been configured, you can build `2decomp&fft` by running
 ```
@@ -122,7 +122,7 @@ To properly configure for GPU build the following needs to be used
 ```
 cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_TARGET=gpu
 ```
-By default CUDA aware MPI will be used together with `cuFFT` for the FFT library. The configure will automatically look for the GPU architecture available on the system. If you are building on a HPC system please use a computing node for the installation. Useful variable to be added are 
+By default CUDA aware MPI will be used together with `cuFFT` for the FFT library. The configure will automatically look for the GPU architecture available on the system. If you are building on a HPC system please use a computing node for the installation. Useful variables to be added are 
 
  - `-DENABLE_NCCL=yes` to activate the NCCL collectives
  - `-DENABLE_MANAGED=yes` to activate the automatic memory management form the NVHPC compiler
@@ -149,7 +149,8 @@ Profiling can be activated in the Makefile. Set the variable `PROFILER` to one o
 Profiling can be activated via `cmake` builds, the recommended approach is to run the initial configuration as follows:
 ```
 export caliper_DIR=/path/to/caliper/install/share/cmake/caliper
-FC=mpif90 CXX=mpicxx cmake -DENABLE_PROFILER=caliper -B build
+export CXX=mpicxx
+cmake -S $path_to_sources -B $path_to_build_directory -DENABLE_PROFILER=caliper
 ```
 where `ENABLE_PROFILER` is set to the profiling tool desired, currently supported values are: `caliper`.
 Note that when using `caliper` a C++ compiler is required as indicated in the above command line.
@@ -231,14 +232,6 @@ This variable is automatically added in GPU builds.
 
 This variable is valid only for GPU builds. The NVIDIA Collective Communication Library (NCCL) implements multi-GPU and multi-node communication primitives optimized for NVIDIA GPUs and Networking.
 
-#### SHM
-
-This variable is not supported.
-
-#### SHM_DEBUG
-
-This variable is not supported
-
 ## Optional dependencies
 
 ### FFTW

From db1f9dbbaed63c41e6c936e96e067bbc5cfd1206 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Thu, 16 Feb 2023 12:13:43 +0000
Subject: [PATCH 153/436] Update docstring for ENABLE_INPLACE

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 927d2c59..dd297563 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,7 +24,7 @@ endif(BUILD_TARGET MATCHES "gpu")
 
 set (ENABLE_PROFILER "OFF" CACHE STRING "Activate/select the profiler")
 set_property(CACHE ENABLE_PROFILER PROPERTY STRINGS OFF caliper)
-option(ENABLE_INPLACE "Activate the profiler" OFF)
+option(ENABLE_INPLACE "Enable in-place operation of FFT" OFF)
 option(HALO_DEBUG "Enable halo debugging options" OFF)
 
 set(AUTHOR "Stefano Rolfo;Charles Moulinec;Paul Bartholomew")

From c6179c1cebf3da92b6d326caa2662e22010d1eb5 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Thu, 16 Feb 2023 21:35:19 +0000
Subject: [PATCH 154/436] Update CTest to pass decomposition info

---
 cmake/D2D_MPI.cmake                        | 47 ++++++++++------------
 examples/CMakeLists.txt                    |  3 ++
 examples/cmake/D2D_Set_Decomposition.cmake | 23 +++++++++++
 examples/fft_physical_x/CMakeLists.txt     | 12 +++---
 examples/fft_physical_z/CMakeLists.txt     |  8 ++--
 examples/halo_test/CMakeLists.txt          |  4 +-
 examples/init_test/CMakeLists.txt          |  4 +-
 examples/init_test/init_test.f90           |  2 +
 examples/io_test/CMakeLists.txt            | 20 ++++-----
 examples/test2d/CMakeLists.txt             |  4 +-
 10 files changed, 75 insertions(+), 52 deletions(-)
 create mode 100644 examples/cmake/D2D_Set_Decomposition.cmake

diff --git a/cmake/D2D_MPI.cmake b/cmake/D2D_MPI.cmake
index a583fa0b..c91c798e 100644
--- a/cmake/D2D_MPI.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -21,32 +21,27 @@ else (MPI_Fortran_COMPILER)
 endif(MPI_Fortran_COMPILER)
 # Warning if Include are not found => can be fixed with more recent cmake version
 if (MPI_FOUND)
-    message(STATUS "MPI FOUND: ${MPI_FOUND}")
-    include_directories(SYSTEM ${MPI_INCLUDE_PATH})
-    message(STATUS "MPI INCL ALSO FOUND: ${MPI_INCLUDE_PATH}")
-    if (NOT MPI_NUMPROCS_SET)
-      message(STATUS "Reset the number of ranks to 1")
-      set(MPIEXEC_MAX_NUMPROCS "1" CACHE STRING
-          "Set the initial value to 1 rank" FORCE)
-      set(MPI_NUMPROCS_SET 1 CACHE INTERNAL "MPI Ranks set")
-      # Force the mpirun to be coherent with the mpifortran
-      string(REGEX REPLACE "mpif90" "mpirun" PATH_TO_MPIRUN "${MPI_Fortran_COMPILER}")
-      string(REPLACE "mpiifort" "mpirun" PATH_TO_MPIRUN "${PATH_TO_MPIRUN}")
-      message(STATUS "Path to mpirun ${PATH_TO_MPIRUN}")
-      set(MPIEXEC_EXECUTABLE "${PATH_TO_MPIRUN}" CACHE STRING
-          "Force MPIRUN to be consistent with MPI_Fortran_COMPILER" FORCE)
-    endif()
+  message(STATUS "MPI FOUND: ${MPI_FOUND}")
+  include_directories(SYSTEM ${MPI_INCLUDE_PATH})
+  message(STATUS "MPI INCL ALSO FOUND: ${MPI_INCLUDE_PATH}")
+  if (NOT MPI_NUMPROCS_SET)
+    # Save the Maximim number of MPI ranks on the system
+    set(MAX_NUMPROCS ${MPIEXEC_MAX_NUMPROCS} CACHE STRING "SAVE NRANKS MAX" FORCE)
+    message(STATUS "Reset the number of ranks to 1")
+    set(MPIEXEC_MAX_NUMPROCS "1" CACHE STRING
+        "Set the initial value to 1 rank" FORCE)
+    set(MPI_NUMPROCS_SET 1 CACHE INTERNAL "MPI Ranks set" FORCE)
+    # Force the mpirun to be coherent with the mpifortran
+    string(REGEX REPLACE "mpif90" "mpirun" PATH_TO_MPIRUN "${MPI_Fortran_COMPILER}")
+    string(REPLACE "mpiifort" "mpirun" PATH_TO_MPIRUN "${PATH_TO_MPIRUN}")
+    message(STATUS "Path to mpirun ${PATH_TO_MPIRUN}")
+    set(MPIEXEC_EXECUTABLE "${PATH_TO_MPIRUN}" CACHE STRING
+        "Force MPIRUN to be consistent with MPI_Fortran_COMPILER" FORCE)
+  endif()
 else (MPI_FOUND)
-    message(STATUS "NO MPI include have been found. The executable won't be targeted with MPI include")
-    message(STATUS "Code will compile but performaces can be compromised")
-    message(STATUS "Using a CMake vers > 3.10 should solve the problem")
-    message(STATUS "Alternatively use ccmake to manually set the include if available")
+  message(STATUS "NO MPI include have been found. The executable won't be targeted with MPI include")
+  message(STATUS "Code will compile but performaces can be compromised")
+  message(STATUS "Using a CMake vers > 3.10 should solve the problem")
+  message(STATUS "Alternatively use ccmake to manually set the include if available")
 endif (MPI_FOUND)
 
-# PROW/PCOL options
-# Tests that accept PROW/PCOL as arguments will use these values.
-# To simplify out of the box testing, default to 0.
-set(PROW 0 CACHE STRING
-  "Number of processor rows - PROWxPCOL=NP must be satisfied, 0 for autotuning")
-set(PCOL 0 CACHE STRING
-  "Number of processor rows - PROWxPCOL=NP must be satisfied, 0 for autotuning")
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 4d19c6e2..cdebbf1a 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,3 +1,6 @@
+# Set the decomposition and resolution
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/examples/cmake")
+include(D2D_set_decomposition)
 # Create a folder directory for testing
 set(test_dir "${PROJECT_BINARY_DIR}/RunTests")
 file(MAKE_DIRECTORY ${test_dir})
diff --git a/examples/cmake/D2D_Set_Decomposition.cmake b/examples/cmake/D2D_Set_Decomposition.cmake
new file mode 100644
index 00000000..3e577658
--- /dev/null
+++ b/examples/cmake/D2D_Set_Decomposition.cmake
@@ -0,0 +1,23 @@
+# PROW/PCOL options
+# Tests that accept PROW/PCOL as arguments will use these values.
+# To simplify out of the box testing, default to 0.
+set(PROW 0 CACHE STRING
+  "Number of processor rows - PROWxPCOL=NP must be satisfied, 0 for autotuning")
+set(PCOL 0 CACHE STRING
+  "Number of processor rows - PROWxPCOL=NP must be satisfied, 0 for autotuning")
+# In case decomposition is imposed force number of MPI task to be consistent
+math(EXPR NUMPROCS "${PROW} * ${PCOL}")
+message(STATUS "Computed NRANK: ${NUMPROCS} Max avail ${MAX_NUMPROCS}")
+set(ADD_DECOMP_TO_EXAMPLE FALSE)
+if (NUMPROCS GREATER "0")
+  if (NUMPROCS LESS "${MAX_NUMPROCS}")
+    message(STATUS "Decomposion has been imposed to ${PROW}X${PCOL}: number of MPI tasks to be used is imposed to ${NUMPROCS}")
+    set(MPIEXEC_MAX_NUMPROCS "${NUMPROCS}" CACHE STRING
+            "Force N to be p_row*p_col" FORCE)
+  else ()
+    message(STATUS "The decomposition ${PROW}x${PCOL} cannot be run. Only ${MAX_NUMPROCS} are available. Default testing is performed")
+  endif()
+endif ()
+#string(JOIN " " TEST_ARGUMENTS "${PROW}" "${PCOL}")
+set(TEST_ARGUMENTS "${PROW}" "${PCOL}")
+message(STATUS "Test argument string ${TEST_ARGUMENTS}")
diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index e994693a..61d7543c 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -30,9 +30,9 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_x> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_x> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()  
-  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_x> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_c2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_x> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif()
 #
 set(run_dir "${test_dir}/fft_r2c_x")
@@ -41,9 +41,9 @@ file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
 
-  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_x> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_x> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()  
-  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_x> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_r2c_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_x> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif()
 #
 set(run_dir "${test_dir}/fft_grid_x")
@@ -51,7 +51,7 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh  $<TARGET_FILE:fft_grid_x> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh  $<TARGET_FILE:fft_grid_x> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()  
-  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_grid_x> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_grid_x COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_grid_x> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif()
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index 0ec8aa88..04e25f48 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -24,9 +24,9 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_z> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_c2c_z> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()  
-  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_z> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_c2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_c2c_z> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
 # 
 set(run_dir "${test_dir}/fft_r2c_z")
@@ -34,7 +34,7 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_z> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:fft_r2c_z> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()  
-  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_z> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME fft_r2c_z COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:fft_r2c_z> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
diff --git a/examples/halo_test/CMakeLists.txt b/examples/halo_test/CMakeLists.txt
index 8834fd69..b534612a 100644
--- a/examples/halo_test/CMakeLists.txt
+++ b/examples/halo_test/CMakeLists.txt
@@ -10,7 +10,7 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:halo_test> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:halo_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()  
-  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:halo_test> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME halo_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:halo_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
diff --git a/examples/init_test/CMakeLists.txt b/examples/init_test/CMakeLists.txt
index cfc9535d..dea5ad7f 100644
--- a/examples/init_test/CMakeLists.txt
+++ b/examples/init_test/CMakeLists.txt
@@ -10,7 +10,7 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:init_test> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:init_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()
-  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:init_test> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME init_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:init_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif()
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 46c2d0cb..8f615c97 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -37,10 +37,12 @@ program init_test
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin=command_argument_count()
+   write(*,*) 'nargin ', nargin
    if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
          read(InputFN, *, iostat=status) DecInd
+         write(*,*) 'DecInd ', DecInd
          if (arg.eq.1) then
             p_row = DecInd
          elseif (arg.eq.2) then
diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 9a30f83f..8dada8fe 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -24,11 +24,11 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_test> WORKING_DIRECTORY ${run_dir})
-  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_read> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_read> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()
-  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_test> WORKING_DIRECTORY ${run_dir})
-  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_read> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_read> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
 # Run the test(s)
 set(run_dir "${test_dir}/io_var_test")
@@ -36,9 +36,9 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_var_test> ${PROW} ${PCOL} WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_var_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()
-  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_var_test> ${PROW} ${PCOL} WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_var_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_var_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
 # Run the test(s)
 set(run_dir "${test_dir}/io_plane_test")
@@ -46,9 +46,9 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_plane_test> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_plane_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()
-  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_plane_test> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_plane_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_plane_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
 # Run the test(s)
 set(run_dir "${test_dir}/io_bench")
@@ -56,7 +56,7 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_bench> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_bench> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()
-  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_bench> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_bench> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
diff --git a/examples/test2d/CMakeLists.txt b/examples/test2d/CMakeLists.txt
index f364edee..66dda720 100644
--- a/examples/test2d/CMakeLists.txt
+++ b/examples/test2d/CMakeLists.txt
@@ -10,7 +10,7 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:test2d> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:test2d> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()
-  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:test2d> WORKING_DIRECTORY ${run_dir})
+  add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:test2d> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()

From a1996a4a92a838e45073f7835c9d8350bd41d612 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Thu, 16 Feb 2023 21:53:03 +0000
Subject: [PATCH 155/436] Add to the CTest arguments for test the resolution

---
 examples/CMakeLists.txt                 |  3 ++-
 examples/cmake/D2D_Set_Resolution.cmake | 11 +++++++++++
 examples/init_test/init_test.f90        |  2 --
 3 files changed, 13 insertions(+), 3 deletions(-)
 create mode 100644 examples/cmake/D2D_Set_Resolution.cmake

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index cdebbf1a..a2da4c8e 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,6 +1,7 @@
 # Set the decomposition and resolution
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/examples/cmake")
-include(D2D_set_decomposition)
+include(D2D_Set_Decomposition)
+include(D2D_Set_Resolution)
 # Create a folder directory for testing
 set(test_dir "${PROJECT_BINARY_DIR}/RunTests")
 file(MAKE_DIRECTORY ${test_dir})
diff --git a/examples/cmake/D2D_Set_Resolution.cmake b/examples/cmake/D2D_Set_Resolution.cmake
new file mode 100644
index 00000000..dae567c1
--- /dev/null
+++ b/examples/cmake/D2D_Set_Resolution.cmake
@@ -0,0 +1,11 @@
+# Set the resolution in each direction
+set(NX 0 CACHE STRING "Number of points in X direction, 0 for default resolution")
+set(NY 0 CACHE STRING "Number of points in Y direction, 0 for default resolution")
+set(NZ 0 CACHE STRING "Number of points in Z direction, 0 for default resolution")
+math(EXPR NPOINTS "${NX} * ${NY} * ${NZ}")
+message(STATUS "Total number of points: ${NPOINTS}")
+if (NPOINTS GREATER "0")
+  list(APPEND TEST_ARGUMENTS "${NX}")
+  list(APPEND TEST_ARGUMENTS "${NY}")
+  list(APPEND TEST_ARGUMENTS "${NZ}")
+endif()
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 8f615c97..46c2d0cb 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -37,12 +37,10 @@ program init_test
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin=command_argument_count()
-   write(*,*) 'nargin ', nargin
    if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
          read(InputFN, *, iostat=status) DecInd
-         write(*,*) 'DecInd ', DecInd
          if (arg.eq.1) then
             p_row = DecInd
          elseif (arg.eq.2) then

From e64c924ea4beb5674c84d0968eea1d0abed84551 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 17 Feb 2023 08:08:32 +0000
Subject: [PATCH 156/436] Cmake improvements (#154)

* Build module files in CMAKE_INSTALL_INCLUDEDIR and install them alongside library

* Improve module install process

Previous implementation had worked by luck, now we install the
contents of ${CMAKE_Fortran_MODULE_DIRECTORY}/ to the
${CMAKE_INSTALL_INCLUDEDIR} destination, this should be robust

* Add option to build shared libraries

By default builds static libraries

* Updated README to explain building shared library
---
 CMakeLists.txt     |  3 ++-
 README.md          | 10 +++++++++-
 src/CMakeLists.txt | 19 +++++++++++++------
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dd297563..4be7e08b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -26,6 +26,7 @@ set (ENABLE_PROFILER "OFF" CACHE STRING "Activate/select the profiler")
 set_property(CACHE ENABLE_PROFILER PROPERTY STRINGS OFF caliper)
 option(ENABLE_INPLACE "Enable in-place operation of FFT" OFF)
 option(HALO_DEBUG "Enable halo debugging options" OFF)
+option(BUILD_SHARED_LIBS "Set ON to build a shared library" OFF)
 
 set(AUTHOR "Stefano Rolfo;Charles Moulinec;Paul Bartholomew")
 set(AUTHOR_DETAILS "stefano.rolfo@stfc.ac.uk;charles.moulinec@stfc.ac.uk;p.bartholomew@epcc.ed.ac.uk")
@@ -36,7 +37,7 @@ message(STATUS "building ${PROJECT_NAME}")
 include(GNUInstallDirs)
 set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
 set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR})
-set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_SOURCE_DIR}/mod)
+set(CMAKE_Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR})
 if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
   set(CMAKE_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/opt" CACHE PATH "..." FORCE)
 endif()
diff --git a/README.md b/README.md
index e090b037..fc8a4c76 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,15 @@ Options can be added to change the level of verbosity. Finally, the build librar
 ```
 cmake --install $path_to_build_directory
 ```
-The default location for `libdecomp2d.a` is `$path_to_build_directory/opt/lib`or  `$path_to_build_directory/opt/lib64` unless the variable `CMAKE_INSTALL_PREFIX` is modified
+The default location for `libdecomp2d.a` is `$path_to_build_directory/opt/lib`or  `$path_to_build_directory/opt/lib64` unless the variable `CMAKE_INSTALL_PREFIX` is modified.
+The module files generated by the build process will similarly be installed to `$path_to_build_directory/opt/install`, users of the library should add this to the include paths for their program.
+
+As indicated above, by default a static `libdecomp2d.a` will be compiled, if desired a shared library can be built by setting `BUILD_SHARED_LIBS=ON` either on the command line:
+```
+cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_SHARED_LIBS=ON
+```
+or by editing the configuration using `ccmake`.
+This might be useful for a centralised install supporting multiple users that is upgraded over time.
 
 Occasionally a clean build is required, this can be performed by running
 ```
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 3910d2be..0c23e09b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -4,7 +4,6 @@ file(GLOB files_decomp factor.f90
 		       log.f90
 		       io.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
-include_directories(${CMAKE_SOURCE_DIR}/mod)
 
 #if(${FFT_Choice} MATCHES "generic")
 #  file(GLOB files_fft fft_generic.f90)
@@ -31,7 +30,10 @@ endif()
 
 set(SRCFILES ${files_decomp} ${files_fft} ${prof_files})
 
-add_library(decomp2d STATIC ${SRCFILES})
+add_library(decomp2d ${SRCFILES})
+target_include_directories(decomp2d PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_Fortran_MODULE_DIRECTORY}>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
 if (MPI_FOUND)
   target_link_libraries(decomp2d PRIVATE MPI::MPI_Fortran)
 endif (MPI_FOUND)
@@ -58,7 +60,12 @@ if (HALO_DEBUG)
 endif()
 
 install(TARGETS decomp2d
-        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 
-)
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+
+install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  FILES_MATCHING PATTERN "*.mod")
+install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  FILES_MATCHING PATTERN "*.smod")
+      

From ed3eff4fce2110530f965b8c1fb6908db541fdae Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Fri, 17 Feb 2023 13:24:50 +0000
Subject: [PATCH 157/436] Update README with additional instructions for ctest

---
 README.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index fc8a4c76..29fbcf3d 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,12 @@ After building the library can be tested by running
 ```
 ctest --test-dir $path_to_build_directory
 ```
-which uses the `ctest` utility. By default tests are performed in serial, but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS`.  For the GPU implementation please be aware that it is based on a single MPI rank per GPU. Therefore to test multiple GPUs, please use the maximum number of available GPUs and not the maximum number of MPI tasks that are available in the system/node. 
+which uses the `ctest` utility. By default tests are performed in serial, but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS` under `ccamke` utility.  
+It is also possible to specify the decomposition by setting `PROW` and `PCOL` parameters at the configure stage or using `ccmake`. 
+The configure is consequently forcing the number of MPI tasks `MPIEXEC_MAX_NUMPROCS` to be equal to the product of PROW times PCOL. 
+Mesh resolution can also be imposed using the parameters `NX`, `NY` and `NZ`. 
+
+For the GPU implementation please be aware that it is based on a single MPI rank per GPU. Therefore to test multiple GPUs use the maximum number of available GPUs and not the maximum number of MPI tasks that are available in the system/node. 
 
 ## GPU compilation
 

From 1d0a9c3e30e3729f6c2c94056c5e5d4955283a5b Mon Sep 17 00:00:00 2001
From: rfj82982 <35581400+rfj82982@users.noreply.github.com>
Date: Fri, 17 Feb 2023 13:37:43 +0000
Subject: [PATCH 158/436] Update README.md

Correct some bad English
---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 29fbcf3d..9b8ecb93 100644
--- a/README.md
+++ b/README.md
@@ -108,12 +108,12 @@ After building the library can be tested by running
 ```
 ctest --test-dir $path_to_build_directory
 ```
-which uses the `ctest` utility. By default tests are performed in serial, but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS` under `ccamke` utility.  
+which uses the `ctest` utility. By default tests are performed in serial, but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS` under `ccmake` utility.  
 It is also possible to specify the decomposition by setting `PROW` and `PCOL` parameters at the configure stage or using `ccmake`. 
-The configure is consequently forcing the number of MPI tasks `MPIEXEC_MAX_NUMPROCS` to be equal to the product of PROW times PCOL. 
+Consequently, the configure stage is forcing the number of MPI tasks `MPIEXEC_MAX_NUMPROCS` to be equal to the product of PROW times PCOL. 
 Mesh resolution can also be imposed using the parameters `NX`, `NY` and `NZ`. 
 
-For the GPU implementation please be aware that it is based on a single MPI rank per GPU. Therefore to test multiple GPUs use the maximum number of available GPUs and not the maximum number of MPI tasks that are available in the system/node. 
+For the GPU implementation please be aware that it is based on a single MPI rank per GPU. Therefore, to test multiple GPUs, use the maximum number of available GPUs on the system/node and not the maximum number of MPI tasks. 
 
 ## GPU compilation
 

From 490ee37decf346e1c196cfe245c7d63e8889d575 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 27 Feb 2023 11:35:22 +0000
Subject: [PATCH 159/436] Add timing test for transpose of reals

---
 examples/Makefile                 |   2 +-
 examples/test2d/CMakeLists.txt    |  20 ++-
 examples/test2d/Makefile          |  15 +-
 examples/test2d/timing2d_real.f90 | 289 ++++++++++++++++++++++++++++++
 4 files changed, 318 insertions(+), 8 deletions(-)
 create mode 100644 examples/test2d/timing2d_real.f90

diff --git a/examples/Makefile b/examples/Makefile
index 508a0c0b..546d1377 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -7,7 +7,7 @@ examples: init_test test2d fft_physical_x fft_physical_z halo_test io_test
 init_test:
 	$(MAKE) -C $@ all
 test2d:
-	$(MAKE) -C $@ $@
+	$(MAKE) -C $@ all
 fft_physical_x:
 	$(MAKE) -C $@ all
 fft_physical_z:
diff --git a/examples/test2d/CMakeLists.txt b/examples/test2d/CMakeLists.txt
index 66dda720..39cd5976 100644
--- a/examples/test2d/CMakeLists.txt
+++ b/examples/test2d/CMakeLists.txt
@@ -1,8 +1,13 @@
-file(GLOB files_test test2d.f90)
+file(GLOB files_test2d test2d.f90)
+file(GLOB files_timing2d_real timing2d_real.f90)
+
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
-add_executable(test2d ${files_test})
+add_executable(test2d ${files_test2d})
+add_executable(timing2d_real ${files_timing2d_real})
+
 target_link_libraries(test2d PRIVATE decomp2d)
+target_link_libraries(timing2d_real PRIVATE decomp2d)
 
 # Run the test(s)
 set(run_dir "${test_dir}/test2d")
@@ -14,3 +19,14 @@ if (BUILD_TARGET MATCHES "gpu")
 else ()
   add_test(NAME test2d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:test2d> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
+
+# Run the test(s)
+set(run_dir "${test_dir}/timing2d_real")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME timing2d_real COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:timing2d_real> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+else ()
+  add_test(NAME timing2d_real COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:timing2d_real> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+endif ()
diff --git a/examples/test2d/Makefile b/examples/test2d/Makefile
index 4b798294..fa2ed202 100644
--- a/examples/test2d/Makefile
+++ b/examples/test2d/Makefile
@@ -6,25 +6,30 @@ FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
 FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
 LIBS = -L../../ -l$(LIBDECOMP) $(LFLAGS)
 
-OBJ = test2d.o
-
 NP ?= 1
 MPIRUN ?= mpirun
 
-test2d: $(OBJ)
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $(OBJ) $(LIBS)
+all: test2d timing2d_real
+
+test2d: test2d.o
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
+
+timing2d_real: timing2d_real.o
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
 
 ifeq ($(PARAMOD),gpu)
 check:
 	$(MPIRUN) -n $(NP) ./bind.sh ./test2d
+	$(MPIRUN) -n $(NP) ./bind.sh ./timing2d_real
 else
 check:
 	$(MPIRUN) -n $(NP) ./test2d
+	$(MPIRUN) -n $(NP) ./timing2d_real
 endif
 
 
 clean:
-	rm -f *.o test2d u*.dat *.log
+	rm -f *.o test2d timing2d_real u*.dat *.log
 
 %.o : %.f90
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
new file mode 100644
index 00000000..7df5c7f2
--- /dev/null
+++ b/examples/test2d/timing2d_real.f90
@@ -0,0 +1,289 @@
+program timing2d_real
+
+   use mpi
+   use decomp_2d
+   use decomp_2d_constants
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
+
+   implicit none
+
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
+
+   real(mytype), allocatable, dimension(:, :, :) :: data1
+
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+
+   integer :: i, j, k, m, ierror
+   integer :: xst1, xst2, xst3
+   integer :: xen1, xen2, xen3
+   integer :: yst1, yst2, yst3
+   integer :: yen1, yen2, yen3
+   integer :: zst1, zst2, zst3
+   integer :: zen1, zen2, zen3
+   logical :: error_flag
+
+   real(mytype) :: t1, t2, t3, t4, t5, t6, t7, t8
+   integer :: iter, niter =10
+
+   ! Init
+   error_flag = .false.
+   call MPI_INIT(ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_INIT")
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   allocate(data1(nx,ny,nz))
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
+            data1(i, j, k) = float(m)
+            m = m + 1
+         end do
+      end do
+   end do
+
+   ! Fill the local index
+   xst1 = xstart(1); xen1 = xend(1)
+   xst2 = xstart(2); xen2 = xend(2)
+   xst3 = xstart(3); xen3 = xend(3)
+   yst1 = ystart(1); yen1 = yend(1)
+   yst2 = ystart(2); yen2 = yend(2)
+   yst3 = ystart(3); yen3 = yend(3)
+   zst1 = zstart(1); zen1 = zend(1)
+   zst2 = zstart(2); zen2 = zend(2)
+   zst3 = zstart(3); zen3 = zend(3)
+   
+
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   ! Testing the swap routines
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+   call alloc_x(u1, opt_global=.true.)
+   call alloc_y(u2, opt_global=.true.)
+   call alloc_z(u3, opt_global=.true.)
+
+   !$acc data copyin(data1) copy(u1,u2,u3)
+   ! original x-pensil based data
+   !$acc parallel loop default(present)
+   do k = xst3, xen3
+      do j = xst2, xen2
+         do i = xst1, xen1
+            u1(i, j, k) = data1(i, j, k)
+         end do
+      end do
+   end do
+   !$acc end loop
+
+10 format(15I5)
+
+#ifdef DEBUG
+   if (nrank == 0) then
+      !$acc update self(u1)
+      write (*, *) 'Numbers held on Rank 0'
+      write (*, *) ' '
+      write (*, *) 'X-pencil'
+      write (*, 10) int(u1)
+   end if
+#endif
+
+   ! call decomp_2d_write_one(1,u1,'u1.dat')
+
+   t2=0._mytype
+   t4=0._mytype
+   t6=0._mytype
+   t8=0._mytype
+   do iter=1,niter
+      !!!!!!!!!!!!!!!!!!!!!!!
+      write(*,*) 'ITER ', iter
+      ! x-pensil ==> y-pensil
+      t1 = MPI_WTIME()
+      call transpose_x_to_y(u1, u2)
+      t2 = t2 + MPI_WTIME()-t1
+
+#ifdef DEBUG
+      if (nrank == 0) then
+         !$acc update self(u2)
+         write (*, *) ' '
+         write (*, *) 'Y-pencil'
+         write (*, 10) int(u2)
+      end if
+#endif
+
+      ! call decomp_2d_write_one(2,u2,'u2.dat')
+      ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
+
+      ! also check the transposition this way
+      !$acc parallel loop default(present)
+      do k = yst3, yen3
+         do j = yst2, yen2
+            do i = yst1, yen1
+               if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            end do
+         end do
+      end do
+      !$acc end loop
+      call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+      if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+      if (error_flag) call decomp_2d_abort(1, "error swaping x->y")
+
+      !!!!!!!!!!!!!!!!!!!!!!!
+      ! y-pensil ==> z-pensil
+      t3 = MPI_WTIME()
+      call transpose_y_to_z(u2, u3)
+      t4 = t4 + MPI_WTIME()-t3
+
+#ifdef DEBUG
+      if (nrank == 0) then
+         !$acc update self(u3)
+         write (*, *) ' '
+         write (*, *) 'Z-pencil'
+         write (*, 10) int(u3)
+      end if
+#endif
+
+      ! call decomp_2d_write_one(3,u3,'u3.dat')
+      ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
+
+      !$acc parallel loop default(present)
+      do k = zst3, zen3
+         do j = zst2, zen2
+            do i = zst1, zen1
+               if (abs(u3(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            end do
+         end do
+      end do
+      !$acc end loop
+      call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+      if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+      if (error_flag) call decomp_2d_abort(2, "error swaping y->z")
+
+      !!!!!!!!!!!!!!!!!!!!!!!
+      ! z-pensil ==> y-pensil
+      t5 = MPI_WTIME()
+      call transpose_z_to_y(u3, u2)
+      t6 = t6 + MPI_WTIME()-t5
+      ! call decomp_2d_write_one(2,u2,'u2b.dat')
+
+      !$acc parallel loop default(present)
+      do k = yst3, yen3
+         do j = yst2, yen2
+            do i = yst1, yen1
+               if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            end do
+         end do
+      end do
+      !$acc end loop
+      call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+      if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+      if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
+
+      !!!!!!!!!!!!!!!!!!!!!!!
+      ! y-pensil ==> x-pensil
+      t7 = MPI_WTIME()
+      call transpose_y_to_x(u2, u1)
+      t8 = t8 + MPI_WTIME()-t7
+      ! call decomp_2d_write_one(1,u1,'u1b.dat')
+
+      !$acc parallel loop default(present)
+      do k = xst3, xen3
+         do j = xst2, xen2
+            do i = xst1, xen1
+               if (abs(u1(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            end do
+         end do
+      end do
+      !$acc end loop
+      call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+      if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+      if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
+   enddo
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+   call MPI_ALLREDUCE(t6, t5, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t5 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t8, t7, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t7 = t3/real(nproc, mytype)
+   t8 = t1+t3+t5+t8
+   if (nrank == 0) then
+     write(*,*) 'Time X->Y ', t1
+     write(*,*) 'Time Y->Z ', t3
+     write(*,*) 'Time Z->Y ', t5
+     write(*,*) 'Time Y->X ', t7
+     write(*,*) 'Time TOT  ', t8
+   endif
+
+   if (nrank == 0) then
+      write (*, *) " "
+      write (*, *) "test2d completed"
+      write (*, *) " "
+   end if
+
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   !$acc end data
+   deallocate (u1, u2, u3)
+   deallocate (data1)
+
+end program timing2d_real
+

From bb010031e1db6a786b611a92feb6a059865aa6a6 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 27 Feb 2023 13:09:59 +0000
Subject: [PATCH 160/436] Remove extra write

---
 examples/test2d/timing2d_real.f90 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 7df5c7f2..4f965108 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -148,7 +148,6 @@ program timing2d_real
    t8=0._mytype
    do iter=1,niter
       !!!!!!!!!!!!!!!!!!!!!!!
-      write(*,*) 'ITER ', iter
       ! x-pensil ==> y-pensil
       t1 = MPI_WTIME()
       call transpose_x_to_y(u1, u2)

From b054bf0e588baf41ba2060623a955f9d9776de37 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 27 Feb 2023 14:13:17 +0000
Subject: [PATCH 161/436] Add timing comples transpose

---
 examples/test2d/CMakeLists.txt       |  14 ++
 examples/test2d/Makefile             |   9 +-
 examples/test2d/timing2d_complex.f90 | 288 +++++++++++++++++++++++++++
 3 files changed, 309 insertions(+), 2 deletions(-)
 create mode 100644 examples/test2d/timing2d_complex.f90

diff --git a/examples/test2d/CMakeLists.txt b/examples/test2d/CMakeLists.txt
index 39cd5976..cf9e3d7b 100644
--- a/examples/test2d/CMakeLists.txt
+++ b/examples/test2d/CMakeLists.txt
@@ -1,13 +1,16 @@
 file(GLOB files_test2d test2d.f90)
 file(GLOB files_timing2d_real timing2d_real.f90)
+file(GLOB files_timing2d_complex timing2d_complex.f90)
 
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(test2d ${files_test2d})
 add_executable(timing2d_real ${files_timing2d_real})
+add_executable(timing2d_complex ${files_timing2d_complex})
 
 target_link_libraries(test2d PRIVATE decomp2d)
 target_link_libraries(timing2d_real PRIVATE decomp2d)
+target_link_libraries(timing2d_complex PRIVATE decomp2d)
 
 # Run the test(s)
 set(run_dir "${test_dir}/test2d")
@@ -30,3 +33,14 @@ if (BUILD_TARGET MATCHES "gpu")
 else ()
   add_test(NAME timing2d_real COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:timing2d_real> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
+
+# Run the test(s)
+set(run_dir "${test_dir}/timing2d_complex")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME timing2d_complex COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:timing2d_complex> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+else ()
+  add_test(NAME timing2d_complex COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:timing2d_complex> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+endif ()
diff --git a/examples/test2d/Makefile b/examples/test2d/Makefile
index fa2ed202..eeaae0db 100644
--- a/examples/test2d/Makefile
+++ b/examples/test2d/Makefile
@@ -9,7 +9,7 @@ LIBS = -L../../ -l$(LIBDECOMP) $(LFLAGS)
 NP ?= 1
 MPIRUN ?= mpirun
 
-all: test2d timing2d_real
+all: test2d timing2d_real timing2d_complex
 
 test2d: test2d.o
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
@@ -17,19 +17,24 @@ test2d: test2d.o
 timing2d_real: timing2d_real.o
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
 
+timing2d_complex: timing2d_complex.o
+	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
+
 ifeq ($(PARAMOD),gpu)
 check:
 	$(MPIRUN) -n $(NP) ./bind.sh ./test2d
 	$(MPIRUN) -n $(NP) ./bind.sh ./timing2d_real
+	$(MPIRUN) -n $(NP) ./bind.sh ./timing2d_complex
 else
 check:
 	$(MPIRUN) -n $(NP) ./test2d
 	$(MPIRUN) -n $(NP) ./timing2d_real
+	$(MPIRUN) -n $(NP) ./timing2d_complex
 endif
 
 
 clean:
-	rm -f *.o test2d timing2d_real u*.dat *.log
+	rm -f *.o test2d timing2d_real timing2d_complex u*.dat *.log
 
 %.o : %.f90
 	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
new file mode 100644
index 00000000..284c737e
--- /dev/null
+++ b/examples/test2d/timing2d_complex.f90
@@ -0,0 +1,288 @@
+program timing2d_complex
+
+   use mpi
+   use decomp_2d
+   use decomp_2d_constants
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
+
+   implicit none
+
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
+
+   complex(mytype), allocatable, dimension(:, :, :) :: data1
+
+   complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+
+   integer :: i, j, k, m, ierror
+   integer :: xst1, xst2, xst3
+   integer :: xen1, xen2, xen3
+   integer :: yst1, yst2, yst3
+   integer :: yen1, yen2, yen3
+   integer :: zst1, zst2, zst3
+   integer :: zen1, zen2, zen3
+   logical :: error_flag
+
+   real(mytype) :: t1, t2, t3, t4, t5, t6, t7, t8
+   integer :: iter, niter =10
+
+   ! Init
+   error_flag = .false.
+   call MPI_INIT(ierror)
+   if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_INIT")
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot/4) + 1
+   nx = nx_base*resize_domain
+   ny = ny_base*resize_domain
+   nz = nz_base*resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin=command_argument_count()
+   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read(InputFN, *, iostat=status) DecInd
+         if (arg.eq.1) then
+            p_row = DecInd
+         elseif (arg.eq.2) then
+            p_col = DecInd
+         elseif (arg.eq.3) then
+            nx = DecInd
+         elseif (arg.eq.4) then
+            ny = DecInd
+         elseif (arg.eq.5) then
+            nz = DecInd
+         endif
+      enddo
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank==0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      endif
+   endif
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   ! ***** global data *****
+   allocate(data1(nx,ny,nz))
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
+            data1(i, j, k) = cmplx(float(m),float(m-1))
+            m = m + 1
+         end do
+      end do
+   end do
+
+   ! Fill the local index
+   xst1 = xstart(1); xen1 = xend(1)
+   xst2 = xstart(2); xen2 = xend(2)
+   xst3 = xstart(3); xen3 = xend(3)
+   yst1 = ystart(1); yen1 = yend(1)
+   yst2 = ystart(2); yen2 = yend(2)
+   yst3 = ystart(3); yen3 = yend(3)
+   zst1 = zstart(1); zen1 = zend(1)
+   zst2 = zstart(2); zen2 = zend(2)
+   zst3 = zstart(3); zen3 = zend(3)
+   
+
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   ! Testing the swap routines
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+   call alloc_x(u1, opt_global=.true.)
+   call alloc_y(u2, opt_global=.true.)
+   call alloc_z(u3, opt_global=.true.)
+
+   !$acc data copyin(data1) copy(u1,u2,u3)
+   ! original x-pensil based data
+   !$acc parallel loop default(present)
+   do k = xst3, xen3
+      do j = xst2, xen2
+         do i = xst1, xen1
+            u1(i, j, k) = data1(i, j, k)
+         end do
+      end do
+   end do
+   !$acc end loop
+
+10 format(15I5)
+
+#ifdef DEBUG
+   if (nrank == 0) then
+      !$acc update self(u1)
+      write (*, *) 'Numbers held on Rank 0'
+      write (*, *) ' '
+      write (*, *) 'X-pencil'
+      write (*, 10) int(u1)
+   end if
+#endif
+
+   ! call decomp_2d_write_one(1,u1,'u1.dat')
+
+   t2=0._mytype
+   t4=0._mytype
+   t6=0._mytype
+   t8=0._mytype
+   do iter=1,niter
+      !!!!!!!!!!!!!!!!!!!!!!!
+      ! x-pensil ==> y-pensil
+      t1 = MPI_WTIME()
+      call transpose_x_to_y(u1, u2)
+      t2 = t2 + MPI_WTIME()-t1
+
+#ifdef DEBUG
+      if (nrank == 0) then
+         !$acc update self(u2)
+         write (*, *) ' '
+         write (*, *) 'Y-pencil'
+         write (*, 10) int(u2)
+      end if
+#endif
+
+      ! call decomp_2d_write_one(2,u2,'u2.dat')
+      ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
+
+      ! also check the transposition this way
+      !$acc parallel loop default(present)
+      do k = yst3, yen3
+         do j = yst2, yen2
+            do i = yst1, yen1
+               if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            end do
+         end do
+      end do
+      !$acc end loop
+      call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+      if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+      if (error_flag) call decomp_2d_abort(1, "error swaping x->y")
+
+      !!!!!!!!!!!!!!!!!!!!!!!
+      ! y-pensil ==> z-pensil
+      t3 = MPI_WTIME()
+      call transpose_y_to_z(u2, u3)
+      t4 = t4 + MPI_WTIME()-t3
+
+#ifdef DEBUG
+      if (nrank == 0) then
+         !$acc update self(u3)
+         write (*, *) ' '
+         write (*, *) 'Z-pencil'
+         write (*, 10) int(u3)
+      end if
+#endif
+
+      ! call decomp_2d_write_one(3,u3,'u3.dat')
+      ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
+
+      !$acc parallel loop default(present)
+      do k = zst3, zen3
+         do j = zst2, zen2
+            do i = zst1, zen1
+               if (abs(u3(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            end do
+         end do
+      end do
+      !$acc end loop
+      call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+      if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+      if (error_flag) call decomp_2d_abort(2, "error swaping y->z")
+
+      !!!!!!!!!!!!!!!!!!!!!!!
+      ! z-pensil ==> y-pensil
+      t5 = MPI_WTIME()
+      call transpose_z_to_y(u3, u2)
+      t6 = t6 + MPI_WTIME()-t5
+      ! call decomp_2d_write_one(2,u2,'u2b.dat')
+
+      !$acc parallel loop default(present)
+      do k = yst3, yen3
+         do j = yst2, yen2
+            do i = yst1, yen1
+               if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            end do
+         end do
+      end do
+      !$acc end loop
+      call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+      if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+      if (error_flag) call decomp_2d_abort(3, "error swaping z->y")
+
+      !!!!!!!!!!!!!!!!!!!!!!!
+      ! y-pensil ==> x-pensil
+      t7 = MPI_WTIME()
+      call transpose_y_to_x(u2, u1)
+      t8 = t8 + MPI_WTIME()-t7
+      ! call decomp_2d_write_one(1,u1,'u1b.dat')
+
+      !$acc parallel loop default(present)
+      do k = xst3, xen3
+         do j = xst2, xen2
+            do i = xst1, xen1
+               if (abs(u1(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            end do
+         end do
+      end do
+      !$acc end loop
+      call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
+      if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
+      if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
+   enddo
+
+   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3/real(nproc, mytype)
+   call MPI_ALLREDUCE(t6, t5, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t5 = t1/real(nproc, mytype)
+   call MPI_ALLREDUCE(t8, t7, 1, real_type, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t7 = t3/real(nproc, mytype)
+   t8 = t1+t3+t5+t8
+   if (nrank == 0) then
+     write(*,*) 'Time X->Y ', t1
+     write(*,*) 'Time Y->Z ', t3
+     write(*,*) 'Time Z->Y ', t5
+     write(*,*) 'Time Y->X ', t7
+     write(*,*) 'Time TOT  ', t8
+   endif
+
+   if (nrank == 0) then
+      write (*, *) " "
+      write (*, *) "Complex transpose completed"
+      write (*, *) " "
+   end if
+
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+   !$acc end data
+   deallocate (u1, u2, u3)
+   deallocate (data1)
+
+end program timing2d_complex
+

From 307b7d4ea0c762e26a1b35366eb8046a8d3efa4b Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 27 Feb 2023 14:35:21 +0000
Subject: [PATCH 162/436] Bug correction in timing

---
 examples/test2d/timing2d_complex.f90 | 2 +-
 examples/test2d/timing2d_real.f90    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 284c737e..7441e612 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -263,7 +263,7 @@ program timing2d_complex
    call MPI_ALLREDUCE(t8, t7, 1, real_type, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
    t7 = t3/real(nproc, mytype)
-   t8 = t1+t3+t5+t8
+   t8 = t1+t3+t5+t7
    if (nrank == 0) then
      write(*,*) 'Time X->Y ', t1
      write(*,*) 'Time Y->Z ', t3
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 4f965108..623d0e1c 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -263,7 +263,7 @@ program timing2d_real
    call MPI_ALLREDUCE(t8, t7, 1, real_type, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
    t7 = t3/real(nproc, mytype)
-   t8 = t1+t3+t5+t8
+   t8 = t1+t3+t5+t7
    if (nrank == 0) then
      write(*,*) 'Time X->Y ', t1
      write(*,*) 'Time Y->Z ', t3

From faa0adc5e0d75d90fa86fb63f2f1f77fb66d4a3f Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Wed, 1 Mar 2023 11:18:08 +0100
Subject: [PATCH 163/436] Typo in README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9b8ecb93..ecb4b2f8 100644
--- a/README.md
+++ b/README.md
@@ -223,7 +223,7 @@ This variable is valid for double precision builds only. When it is present, sna
 
 #### PROFILER
 
-This variable is automatically added when selecting the profiler. It activates the profiling sectionsof the code.
+This variable is automatically added when selecting the profiler. It activates the profiling sections of the code.
 
 #### EVEN
 

From 7a6ced7b0beda752787d805c165f4431fa77b9dc Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 2 Mar 2023 11:40:26 +0000
Subject: [PATCH 164/436] Fix timing bug in test2d and remove usage of global
 allocated data1

---
 examples/test2d/test2d.f90           | 53 +++++++++++++-----------
 examples/test2d/timing2d_complex.f90 | 62 ++++++++++++++++------------
 examples/test2d/timing2d_real.f90    | 59 ++++++++++++++------------
 3 files changed, 97 insertions(+), 77 deletions(-)

diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 4989e4fc..ef015ec0 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -22,7 +22,7 @@ program test2d
 
    real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
 
-   integer :: i, j, k, m, ierror
+   integer :: i, j, k, ierror
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
    integer :: yst1, yst2, yst3
@@ -30,6 +30,7 @@ program test2d
    integer :: zst1, zst2, zst3
    integer :: zen1, zen2, zen3
    logical :: error_flag
+   real(mytype) :: m
 
    ! Init
    error_flag = .false.
@@ -81,17 +82,17 @@ program test2d
    endif
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-   ! ***** global data *****
-   allocate(data1(nx,ny,nz))
-   m = 1
-   do k = 1, nz
-      do j = 1, ny
-         do i = 1, nx
-            data1(i, j, k) = float(m)
-            m = m + 1
-         end do
-      end do
-   end do
+   !! ***** global data *****
+   !allocate(data1(nx,ny,nz))
+   !m = 1
+   !do k = 1, nz
+   !   do j = 1, ny
+   !      do i = 1, nx
+   !         data1(i, j, k) = float(m)
+   !         m = m + 1
+   !      end do
+   !   end do
+   !end do
 
    ! Fill the local index
    xst1 = xstart(1); xen1 = xend(1)
@@ -113,13 +114,14 @@ program test2d
    call alloc_y(u2, opt_global=.true.)
    call alloc_z(u3, opt_global=.true.)
 
-   !$acc data copyin(data1) copy(u1,u2,u3)
+   !$acc data copy(u1,u2,u3)
    ! original x-pensil based data
-   !$acc parallel loop default(present)
+   !$acc parallel loop default(present) private(m) 
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            u1(i, j, k) = data1(i, j, k)
+            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            u1(i, j, k) = m
          end do
       end do
    end do
@@ -156,11 +158,12 @@ program test2d
    ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
 
    ! also check the transposition this way
-   !$acc parallel loop default(present)
+   !$acc parallel loop default(present) private(m)
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
-            if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
    end do
@@ -185,11 +188,12 @@ program test2d
    ! call decomp_2d_write_one(3,u3,'u3.dat')
    ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
-   !$acc parallel loop default(present)
+   !$acc parallel loop default(present) private(m)
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            if (abs(u3(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            if (abs(u3(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
    end do
@@ -203,11 +207,12 @@ program test2d
    call transpose_z_to_y(u3, u2)
    ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
-   !$acc parallel loop default(present)
+   !$acc parallel loop default(present) private(m)
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
-            if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
    end do
@@ -221,11 +226,12 @@ program test2d
    call transpose_y_to_x(u2, u1)
    ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
-   !$acc parallel loop default(present)
+   !$acc parallel loop default(present) private(m)
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            if (abs(u1(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            if (abs(u1(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
    end do
@@ -244,7 +250,6 @@ program test2d
    call MPI_FINALIZE(ierror)
    !$acc end data
    deallocate (u1, u2, u3)
-   deallocate (data1)
 
 end program test2d
 
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 7441e612..5f25f83b 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -22,7 +22,7 @@ program timing2d_complex
 
    complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
 
-   integer :: i, j, k, m, ierror
+   integer :: i, j, k, ierror
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
    integer :: yst1, yst2, yst3
@@ -30,6 +30,8 @@ program timing2d_complex
    integer :: zst1, zst2, zst3
    integer :: zen1, zen2, zen3
    logical :: error_flag
+   real(mytype) :: m
+   complex(mytype) :: cm
 
    real(mytype) :: t1, t2, t3, t4, t5, t6, t7, t8
    integer :: iter, niter =10
@@ -84,17 +86,17 @@ program timing2d_complex
    endif
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-   ! ***** global data *****
-   allocate(data1(nx,ny,nz))
-   m = 1
-   do k = 1, nz
-      do j = 1, ny
-         do i = 1, nx
-            data1(i, j, k) = cmplx(float(m),float(m-1))
-            m = m + 1
-         end do
-      end do
-   end do
+   !! ***** global data *****
+   !allocate(data1(nx,ny,nz))
+   !m = 1
+   !do k = 1, nz
+   !   do j = 1, ny
+   !      do i = 1, nx
+   !         data1(i, j, k) = cmplx(float(m),float(m-1))
+   !         m = m + 1
+   !      end do
+   !   end do
+   !end do
 
    ! Fill the local index
    xst1 = xstart(1); xen1 = xend(1)
@@ -116,13 +118,14 @@ program timing2d_complex
    call alloc_y(u2, opt_global=.true.)
    call alloc_z(u3, opt_global=.true.)
 
-   !$acc data copyin(data1) copy(u1,u2,u3)
+   !$acc data copy(u1,u2,u3)
    ! original x-pensil based data
-   !$acc parallel loop default(present)
+   !$acc parallel loop default(present) private(m) 
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            u1(i, j, k) = data1(i, j, k)
+            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            u1(i, j, k) = cmplx(m,real(m-1,mytype))
          end do
       end do
    end do
@@ -166,11 +169,13 @@ program timing2d_complex
       ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
 
       ! also check the transposition this way
-      !$acc parallel loop default(present)
+      !$acc parallel loop default(present) private(m,cm)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               cm = cmplx(m,real(m-1,mytype))
+               if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
       end do
@@ -197,11 +202,13 @@ program timing2d_complex
       ! call decomp_2d_write_one(3,u3,'u3.dat')
       ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
-      !$acc parallel loop default(present)
+      !$acc parallel loop default(present) private(m,cm)
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
-               if (abs(u3(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               cm = cmplx(m,real(m-1,mytype))
+               if (abs(u3(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
       end do
@@ -217,11 +224,13 @@ program timing2d_complex
       t6 = t6 + MPI_WTIME()-t5
       ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
-      !$acc parallel loop default(present)
+      !$acc parallel loop default(present) private(m,cm)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               cm = cmplx(m,real(m-1,mytype))
+               if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
       end do
@@ -237,11 +246,13 @@ program timing2d_complex
       t8 = t8 + MPI_WTIME()-t7
       ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
-      !$acc parallel loop default(present)
+      !$acc parallel loop default(present) private(m,cm)
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1
-               if (abs(u1(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               cm = cmplx(m,real(m-1,mytype))
+               if (abs(u1(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
       end do
@@ -259,10 +270,10 @@ program timing2d_complex
    t3 = t3/real(nproc, mytype)
    call MPI_ALLREDUCE(t6, t5, 1, real_type, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t1/real(nproc, mytype)
+   t5 = t5/real(nproc, mytype)
    call MPI_ALLREDUCE(t8, t7, 1, real_type, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t3/real(nproc, mytype)
+   t7 = t7/real(nproc, mytype)
    t8 = t1+t3+t5+t7
    if (nrank == 0) then
      write(*,*) 'Time X->Y ', t1
@@ -282,7 +293,6 @@ program timing2d_complex
    call MPI_FINALIZE(ierror)
    !$acc end data
    deallocate (u1, u2, u3)
-   deallocate (data1)
 
 end program timing2d_complex
 
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 623d0e1c..935b8477 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -22,7 +22,7 @@ program timing2d_real
 
    real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
 
-   integer :: i, j, k, m, ierror
+   integer :: i, j, k, ierror
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
    integer :: yst1, yst2, yst3
@@ -30,6 +30,7 @@ program timing2d_real
    integer :: zst1, zst2, zst3
    integer :: zen1, zen2, zen3
    logical :: error_flag
+   real(mytype) :: m
 
    real(mytype) :: t1, t2, t3, t4, t5, t6, t7, t8
    integer :: iter, niter =10
@@ -84,17 +85,17 @@ program timing2d_real
    endif
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-   ! ***** global data *****
-   allocate(data1(nx,ny,nz))
-   m = 1
-   do k = 1, nz
-      do j = 1, ny
-         do i = 1, nx
-            data1(i, j, k) = float(m)
-            m = m + 1
-         end do
-      end do
-   end do
+   !! ***** global data *****
+   !allocate(data1(nx,ny,nz))
+   !m = 1
+   !do k = 1, nz
+   !   do j = 1, ny
+   !      do i = 1, nx
+   !         data1(i, j, k) = float(m)
+   !         m = m + 1
+   !      end do
+   !   end do
+   !end do
 
    ! Fill the local index
    xst1 = xstart(1); xen1 = xend(1)
@@ -116,13 +117,14 @@ program timing2d_real
    call alloc_y(u2, opt_global=.true.)
    call alloc_z(u3, opt_global=.true.)
 
-   !$acc data copyin(data1) copy(u1,u2,u3)
+   !$acc data copy(u1,u2,u3)
    ! original x-pensil based data
-   !$acc parallel loop default(present)
+   !$acc parallel loop default(present) private(m) 
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            u1(i, j, k) = data1(i, j, k)
+            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            u1(i, j, k) = m
          end do
       end do
    end do
@@ -166,11 +168,12 @@ program timing2d_real
       ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
 
       ! also check the transposition this way
-      !$acc parallel loop default(present)
+      !$acc parallel loop default(present) private(m)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
       end do
@@ -197,11 +200,12 @@ program timing2d_real
       ! call decomp_2d_write_one(3,u3,'u3.dat')
       ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
-      !$acc parallel loop default(present)
+      !$acc parallel loop default(present) private(m)
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
-               if (abs(u3(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               if (abs(u3(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
       end do
@@ -217,11 +221,12 @@ program timing2d_real
       t6 = t6 + MPI_WTIME()-t5
       ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
-      !$acc parallel loop default(present)
+      !$acc parallel loop default(present) private(m)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               if (abs(u2(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
       end do
@@ -237,11 +242,12 @@ program timing2d_real
       t8 = t8 + MPI_WTIME()-t7
       ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
-      !$acc parallel loop default(present)
+      !$acc parallel loop default(present) private(m)
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1
-               if (abs(u1(i, j, k) - data1(i, j, k)) > 0) error_flag = .true.
+               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               if (abs(u1(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
       end do
@@ -259,10 +265,10 @@ program timing2d_real
    t3 = t3/real(nproc, mytype)
    call MPI_ALLREDUCE(t6, t5, 1, real_type, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t1/real(nproc, mytype)
+   t5 = t5/real(nproc, mytype)
    call MPI_ALLREDUCE(t8, t7, 1, real_type, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t3/real(nproc, mytype)
+   t7 = t7/real(nproc, mytype)
    t8 = t1+t3+t5+t7
    if (nrank == 0) then
      write(*,*) 'Time X->Y ', t1
@@ -274,7 +280,7 @@ program timing2d_real
 
    if (nrank == 0) then
       write (*, *) " "
-      write (*, *) "test2d completed"
+      write (*, *) "Real transpose completed"
       write (*, *) " "
    end if
 
@@ -282,7 +288,6 @@ program timing2d_real
    call MPI_FINALIZE(ierror)
    !$acc end data
    deallocate (u1, u2, u3)
-   deallocate (data1)
 
 end program timing2d_real
 

From cf941d08434f0f98b5b392acf48a10581d93e036 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 2 Mar 2023 12:20:29 +0000
Subject: [PATCH 165/436] Fix NCCL type to allow double/single precision
 transpose

---
 src/decomp_2d.f90        | 6 ++++++
 src/transpose_x_to_y.f90 | 4 ++--
 src/transpose_y_to_x.f90 | 4 ++--
 src/transpose_y_to_z.f90 | 4 ++--
 src/transpose_z_to_y.f90 | 4 ++--
 5 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 63268caa..6fe9e030 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -41,6 +41,9 @@ module decomp_2d
    integer, parameter, public :: mytype_single = KIND(0._real64)
    integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
 #endif
+#if defined(_GPU) && defined(_NCCL)
+   type(ncclDataType) :: ncclType = ncclDouble
+#endif
 #else
    integer, parameter, public :: mytype = KIND(0._real32)
    integer, parameter, public :: real_type = MPI_REAL
@@ -48,6 +51,9 @@ module decomp_2d
    integer, parameter, public :: complex_type = MPI_COMPLEX
    integer, parameter, public :: mytype_single = KIND(0._real32)
    integer, parameter, public :: real_type_single = MPI_REAL
+#if defined(_GPU) && defined(_NCCL)
+   type(ncclDataType) :: ncclType = ncclFloat
+#endif
 #endif
 
    integer, save, public :: mytype_bytes
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index 00125215..547cb9be 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -73,10 +73,10 @@ subroutine transpose_x_to_y_real(src, dst, decomp)
      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
      do col_rank_id = 0, (col_comm_size - 1)
         nccl_stat = ncclSend(work1_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
-                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+                             ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
         nccl_stat = ncclRecv(work2_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
-                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+                             ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
      end do
      nccl_stat = ncclGroupEnd()
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index e04a97e7..e10e3c33 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -73,10 +73,10 @@ subroutine transpose_y_to_x_real(src, dst, decomp)
      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
      do col_rank_id = 0, (col_comm_size - 1)
         nccl_stat = ncclSend(work1_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
-                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+                             ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
         nccl_stat = ncclRecv(work2_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
-                             ncclDouble, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+                             ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
      end do
      nccl_stat = ncclGroupEnd()
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 3ff9b31c..26797c99 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -82,10 +82,10 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
      do row_rank_id = 0, (row_comm_size - 1)
         nccl_stat = ncclSend(work1_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
-                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+                             ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
         nccl_stat = ncclRecv(work2_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
-                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+                             ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
      end do
      nccl_stat = ncclGroupEnd()
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index 90f335d9..aca0cfa8 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -91,10 +91,10 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
      do row_rank_id = 0, (row_comm_size - 1)
         nccl_stat = ncclSend(work1_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
-                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+                             ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
         nccl_stat = ncclRecv(work2_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
-                             ncclDouble, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+                             ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
      end do
      nccl_stat = ncclGroupEnd()

From 20d99bd2c012d11d7edbc31050453567ad24bc12 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 2 Mar 2023 13:21:13 +0100
Subject: [PATCH 166/436] Fix unused variable error

---
 examples/test2d/test2d.f90           | 2 --
 examples/test2d/timing2d_complex.f90 | 2 --
 examples/test2d/timing2d_real.f90    | 2 --
 3 files changed, 6 deletions(-)

diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index ef015ec0..4f54d2f8 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -18,8 +18,6 @@ program test2d
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
-   real(mytype), allocatable, dimension(:, :, :) :: data1
-
    real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
 
    integer :: i, j, k, ierror
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 5f25f83b..37cd989d 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -18,8 +18,6 @@ program timing2d_complex
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
-   complex(mytype), allocatable, dimension(:, :, :) :: data1
-
    complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
 
    integer :: i, j, k, ierror
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 935b8477..7a300f7c 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -18,8 +18,6 @@ program timing2d_real
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
-   real(mytype), allocatable, dimension(:, :, :) :: data1
-
    real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
 
    integer :: i, j, k, ierror

From 7debe4c97cbb54b97486e5ca5763d2d71908249c Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 2 Mar 2023 13:25:56 +0100
Subject: [PATCH 167/436] Fix conversion error

---
 examples/test2d/timing2d_complex.f90 | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 37cd989d..130c82d8 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -123,7 +123,7 @@ program timing2d_complex
       do j = xst2, xen2
          do i = xst1, xen1
             m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-            u1(i, j, k) = cmplx(m,real(m-1,mytype))
+            u1(i, j, k) = cmplx(m,real(m-1,mytype),kind=mytype)
          end do
       end do
    end do
@@ -172,7 +172,7 @@ program timing2d_complex
          do j = yst2, yen2
             do i = yst1, yen1
                m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-               cm = cmplx(m,real(m-1,mytype))
+               cm = cmplx(m,real(m-1,mytype),kind=mytype)
                if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
@@ -205,7 +205,7 @@ program timing2d_complex
          do j = zst2, zen2
             do i = zst1, zen1
                m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-               cm = cmplx(m,real(m-1,mytype))
+               cm = cmplx(m,real(m-1,mytype),kind=mytype)
                if (abs(u3(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
@@ -227,7 +227,7 @@ program timing2d_complex
          do j = yst2, yen2
             do i = yst1, yen1
                m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-               cm = cmplx(m,real(m-1,mytype))
+               cm = cmplx(m,real(m-1,mytype),kind=mytype)
                if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
@@ -249,7 +249,7 @@ program timing2d_complex
          do j = xst2, xen2
             do i = xst1, xen1
                m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-               cm = cmplx(m,real(m-1,mytype))
+               cm = cmplx(m,real(m-1,mytype),kind=mytype)
                if (abs(u1(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do

From 4d4aa9f7071da52591f1c725a5da04b0c6f8bd6f Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Fri, 3 Mar 2023 19:49:49 +0000
Subject: [PATCH 168/436] Add cuMPI module

---
 examples/io_test/io_bench.f90      |   1 +
 examples/io_test/io_plane_test.f90 |   2 +-
 examples/io_test/io_read.f90       |   2 +-
 examples/io_test/io_test.f90       |   2 +-
 examples/io_test/io_var_test.f90   |   1 +
 src/CMakeLists.txt                 |   6 +-
 src/decomp_2d.f90                  | 262 +--------------------------
 src/decomp_2d_constants.f90        | 282 +++++++++++++++++++++++++++++
 src/decomp_2d_cumpi.f90            |  83 +++++++++
 src/decomp_2d_init_fin.f90         |  40 +---
 src/glassman.f90                   |   2 +-
 src/io.f90                         |   3 +-
 src/io_write_every.inc             |   2 +-
 13 files changed, 390 insertions(+), 298 deletions(-)
 create mode 100644 src/decomp_2d_cumpi.f90

diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index d921275f..2a877a88 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -1,5 +1,6 @@
 program io_bench
 
+   use decomp_2d_constants
    use decomp_2d
    use decomp_2d_io
    use MPI
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 97de5a7d..81f9d9d6 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -7,7 +7,7 @@
 program io_plane_test
 
    use mpi
-
+   use decomp_2d_constants
    use decomp_2d
    use decomp_2d_io
 #if defined(_GPU)
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index eaf53e1e..4fd564f3 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -1,7 +1,7 @@
 program io_read
 
    use mpi
-
+   use decomp_2d_constants
    use decomp_2d
    use decomp_2d_io
 #if defined(_GPU)
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index eddb76b6..114614b7 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -1,7 +1,7 @@
 program io_test
 
    use mpi
-
+   use decomp_2d_constants
    use decomp_2d
    use decomp_2d_io
 #if defined(_GPU)
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index ca0eb0e0..c5d35839 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -4,6 +4,7 @@
 program io_var_test
 
    use mpi
+   use decomp_2d_constants
    use decomp_2d
    use decomp_2d_io
 #if defined(_GPU)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 0c23e09b..969cd716 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -5,9 +5,9 @@ file(GLOB files_decomp factor.f90
 		       io.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
-#if(${FFT_Choice} MATCHES "generic")
-#  file(GLOB files_fft fft_generic.f90)
-#endif(${FFT_Choice} MATCHES "generic")
+if(${BUILD_TARGET} MATCHES "gpu")
+  list(APPEND files_decomp decomp_2d_cumpi.f90)
+endif(${BUILD_TARGET} MATCHES "gpu")
 
 if(FFTW_FOUND)
   message (STATUS "Compiling using FFTW3")
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 6fe9e030..8b39d4b7 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -20,6 +20,7 @@ module decomp_2d
    use decomp_2d_constants
 #if defined(_GPU)
    use cudafor
+   use decomp_2d_cumpi
 #if defined(_NCCL)
    use nccl
 #endif
@@ -29,42 +30,10 @@ module decomp_2d
 
    private        ! Make everything private unless declared public
 
-#ifdef DOUBLE_PREC
-   integer, parameter, public :: mytype = KIND(0._real64)
-   integer, parameter, public :: real_type = MPI_DOUBLE_PRECISION
-   integer, parameter, public :: real2_type = MPI_2DOUBLE_PRECISION
-   integer, parameter, public :: complex_type = MPI_DOUBLE_COMPLEX
-#ifdef SAVE_SINGLE
-   integer, parameter, public :: mytype_single = KIND(0._real32)
-   integer, parameter, public :: real_type_single = MPI_REAL
-#else
-   integer, parameter, public :: mytype_single = KIND(0._real64)
-   integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
-#endif
-#if defined(_GPU) && defined(_NCCL)
-   type(ncclDataType) :: ncclType = ncclDouble
-#endif
-#else
-   integer, parameter, public :: mytype = KIND(0._real32)
-   integer, parameter, public :: real_type = MPI_REAL
-   integer, parameter, public :: real2_type = MPI_2REAL
-   integer, parameter, public :: complex_type = MPI_COMPLEX
-   integer, parameter, public :: mytype_single = KIND(0._real32)
-   integer, parameter, public :: real_type_single = MPI_REAL
-#if defined(_GPU) && defined(_NCCL)
-   type(ncclDataType) :: ncclType = ncclFloat
-#endif
-#endif
-
-   integer, save, public :: mytype_bytes
 
    ! some key global variables
    integer, save, public :: nx_global, ny_global, nz_global  ! global size
 
-   integer, save, public :: nrank = -1 ! local MPI rank
-   integer, save, public :: nproc = -1 ! total number of processors
-   integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
-
    ! parameters for 2D Cartesian topology
    integer, save, dimension(2) :: dims, coord
    integer, save, public :: DECOMP_2D_COMM_CART_X = MPI_COMM_NULL
@@ -163,8 +132,6 @@ module decomp_2d
    complex(mytype), allocatable, dimension(:) :: work1_c, work2_c
 
 #if defined(_GPU)
-   real(mytype), allocatable, dimension(:), device :: work1_r_d, work2_r_d
-   complex(mytype), allocatable, dimension(:), device :: work1_c_d, work2_c_d
 
 #if defined(_NCCL)
    integer col_comm_size, row_comm_size
@@ -209,9 +176,9 @@ module decomp_2d
              init_coarser_mesh_statV, fine_to_coarseV, &
              init_coarser_mesh_statP, fine_to_coarseP, &
              alloc_x, alloc_y, alloc_z, &
-             update_halo, decomp_2d_abort, &
-             decomp_2d_warning, get_decomp_info, &
-             decomp_mpi_comm_free, get_decomp_dims, &
+             update_halo, &
+             get_decomp_info, &
+             get_decomp_dims, &
              d2d_listing_get_unit, d2d_listing_close_unit
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -296,19 +263,6 @@ module decomp_2d
       module procedure alloc_z_complex_short
    end interface alloc_z
 
-   interface decomp_2d_abort
-      module procedure decomp_2d_abort_basic
-      module procedure decomp_2d_abort_file_line
-#if defined(_GPU) && defined(_NCCL)
-      module procedure decomp_2d_abort_nccl_basic
-      module procedure decomp_2d_abort_nccl_file_line
-#endif
-   end interface decomp_2d_abort
-
-   interface decomp_2d_warning
-      module procedure decomp_2d_warning_basic
-      module procedure decomp_2d_warning_file_line
-   end interface decomp_2d_warning
 
    interface
 
@@ -376,25 +330,6 @@ end subroutine decomp_profiler_end_char
 
 #include "decomp_2d_init_fin.f90"
 
-   !
-   ! Small wrapper to free a MPI communicator
-   !
-   subroutine decomp_mpi_comm_free(mpi_comm)
-
-      implicit none
-
-      integer, intent(inout) :: mpi_comm
-      integer :: ierror
-
-      ! Return if no MPI comm to free
-      if (mpi_comm == MPI_COMM_NULL) return
-
-      ! Free the provided MPI communicator
-      call MPI_COMM_FREE(mpi_comm, ierror)
-      if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
-      mpi_comm = MPI_COMM_NULL
-
-   end subroutine decomp_mpi_comm_free
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Return the default decomposition object
@@ -499,34 +434,7 @@ subroutine decomp_info_init(nx, ny, nz, decomp)
       if (buf_size > decomp_buf_size) then
          decomp_buf_size = buf_size
 #if defined(_GPU)
-         if (allocated(work1_r_d)) deallocate (work1_r_d)
-         if (allocated(work2_r_d)) deallocate (work2_r_d)
-         if (allocated(work1_c_d)) deallocate (work1_c_d)
-         if (allocated(work2_c_d)) deallocate (work2_c_d)
-         allocate (work1_r_d(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work1_c_d(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work2_r_d(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work2_c_d(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
+         call decomp_2d_cumpi_init(buf_size)
 #endif
          if (allocated(work1_r)) deallocate (work1_r)
          if (allocated(work2_r)) deallocate (work2_r)
@@ -1250,166 +1158,6 @@ end subroutine prepare_buffer
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 #include "halo.f90"
 
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Error handling
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   subroutine decomp_2d_abort_basic(errorcode, msg)
-
-      use iso_fortran_env, only: error_unit
-
-      implicit none
-
-      integer, intent(IN) :: errorcode
-      character(len=*), intent(IN) :: msg
-
-      integer :: ierror
-
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-         write (*, *) 'ERROR MESSAGE: '//msg
-         write (error_unit, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-         write (error_unit, *) 'ERROR MESSAGE: '//msg
-      end if
-      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
-
-   end subroutine decomp_2d_abort_basic
-
-   subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
-
-      use iso_fortran_env, only: error_unit
-
-      implicit none
-
-      integer, intent(IN) :: errorcode, line
-      character(len=*), intent(IN) :: msg, file
-
-      integer :: ierror
-
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT ERROR'
-         write (*, *) '  errorcode:     ', errorcode
-         write (*, *) '  error in file  '//file
-         write (*, *) '           line  ', line
-         write (*, *) '  error message: '//msg
-         write (error_unit, *) '2DECOMP&FFT ERROR'
-         write (error_unit, *) '  errorcode:     ', errorcode
-         write (error_unit, *) '  error in file  '//file
-         write (error_unit, *) '           line  ', line
-         write (error_unit, *) '  error message: '//msg
-      end if
-      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
-
-   end subroutine decomp_2d_abort_file_line
-
-#if defined(_GPU) && defined(_NCCL)
-   !
-   ! This is based on the file "nccl.h" in nvhpc 22.1
-   !
-   function _ncclresult_to_integer(errorcode)
-
-      implicit none
-
-      type(ncclresult), intent(IN) :: errorcode
-      integer :: _ncclresult_to_integer
-
-      if (errorcode == ncclSuccess) then
-         _ncclresult_to_integer = 0
-      elseif (errorcode == ncclUnhandledCudaError) then
-         _ncclresult_to_integer = 1
-      elseif (errorcode == ncclSystemError) then
-         _ncclresult_to_integer = 2
-      elseif (errorcode == ncclInternalError) then
-         _ncclresult_to_integer = 3
-      elseif (errorcode == ncclInvalidArgument) then
-         _ncclresult_to_integer = 4
-      elseif (errorcode == ncclInvalidUsage) then
-         _ncclresult_to_integer = 5
-      elseif (errorcode == ncclNumResults) then
-         _ncclresult_to_integer = 6
-      else
-         _ncclresult_to_integer = -1
-         call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
-                                "NCCL error handling needs some update")
-      end if
-
-   end function _ncclresult_to_integer
-
-   !
-   ! Small wrapper for basic NCCL errors
-   !
-   subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
-
-      implicit none
-
-      type(ncclresult), intent(IN) :: errorcode
-      character(len=*), intent(IN) :: msg
-
-      call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
-                           msg//" "//ncclGetErrorString(errorcode))
-
-   end subroutine decomp_2d_abort_nccl_basic
-
-   !
-   ! Small wrapper for NCCL errors
-   !
-   subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
-
-      implicit none
-
-      type(ncclresult), intent(IN) :: errorcode
-      integer, intent(in) :: line
-      character(len=*), intent(IN) :: msg, file
-
-      call decomp_2d_abort(file, &
-                           line, &
-                           _ncclresult_to_integer(errorcode), &
-                           msg//" "//ncclGetErrorString(errorcode))
-
-   end subroutine decomp_2d_abort_nccl_file_line
-#endif
-
-   subroutine decomp_2d_warning_basic(errorcode, msg)
-
-      use iso_fortran_env, only: error_unit
-
-      implicit none
-
-      integer, intent(IN) :: errorcode
-      character(len=*), intent(IN) :: msg
-
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-         write (*, *) 'ERROR MESSAGE: '//msg
-         write (error_unit, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-         write (error_unit, *) 'ERROR MESSAGE: '//msg
-      end if
-
-   end subroutine decomp_2d_warning_basic
-
-   subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
-
-      use iso_fortran_env, only: error_unit
-
-      implicit none
-
-      integer, intent(IN) :: errorcode, line
-      character(len=*), intent(IN) :: msg, file
-
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT WARNING'
-         write (*, *) '  errorcode:     ', errorcode
-         write (*, *) '  error in file  '//file
-         write (*, *) '           line  ', line
-         write (*, *) '  error message: '//msg
-         write (error_unit, *) '2DECOMP&FFT WARNING'
-         write (error_unit, *) '  errorcode:     ', errorcode
-         write (error_unit, *) '  error in file  '//file
-         write (error_unit, *) '           line  ', line
-         write (error_unit, *) '  error message: '//msg
-      end if
-
-   end subroutine decomp_2d_warning_file_line
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Utility routines to help allocate 3D arrays
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index d1bf438c..04596a3c 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -14,8 +14,48 @@
 
 module decomp_2d_constants
 
+   use mpi
+   use, intrinsic :: iso_fortran_env, only: real32, real64
+#if defined(_GPU) && defined(_NCCL)
+   use nccl 
+#endif
+
    implicit none
 
+   !private 
+ 
+#ifdef DOUBLE_PREC
+   integer, parameter, public :: mytype = KIND(0._real64)
+   integer, parameter, public :: real_type = MPI_DOUBLE_PRECISION
+   integer, parameter, public :: real2_type = MPI_2DOUBLE_PRECISION
+   integer, parameter, public :: complex_type = MPI_DOUBLE_COMPLEX
+#ifdef SAVE_SINGLE
+   integer, parameter, public :: mytype_single = KIND(0._real32)
+   integer, parameter, public :: real_type_single = MPI_REAL
+#else
+   integer, parameter, public :: mytype_single = KIND(0._real64)
+   integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
+#endif
+#if defined(_GPU) && defined(_NCCL)
+   type(ncclDataType) :: ncclType = ncclDouble
+#endif
+#else
+   integer, parameter, public :: mytype = KIND(0._real32)
+   integer, parameter, public :: real_type = MPI_REAL
+   integer, parameter, public :: real2_type = MPI_2REAL
+   integer, parameter, public :: complex_type = MPI_COMPLEX
+   integer, parameter, public :: mytype_single = KIND(0._real32)
+   integer, parameter, public :: real_type_single = MPI_REAL
+#if defined(_GPU) && defined(_NCCL)
+   type(ncclDataType) :: ncclType = ncclFloat
+#endif
+#endif
+
+   integer, save, public :: mytype_bytes
+   ! Global MPI parameters
+   integer, save, public :: nrank = -1 ! local MPI rank
+   integer, save, public :: nproc = -1 ! total number of processors
+   integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
    !
    ! Output for the log can be changed by the external code before calling decomp_2d_init
    !
@@ -81,5 +121,247 @@ module decomp_2d_constants
    integer, parameter, public :: PHYSICAL_IN_X = 1 ! Forward is input in X, output in Z
    integer, parameter, public :: PHYSICAL_IN_Z = 3 ! Forward is input in Z, output in X
 
+   public :: decomp_2d_mpi_init, &
+             decomp_2d_mpi_fin, &
+             decomp_2d_mpi_comm_free, &
+             decomp_2d_abort, &
+             decomp_2d_warning 
+
+   interface decomp_2d_abort
+      module procedure decomp_2d_abort_basic
+      module procedure decomp_2d_abort_file_line
+#if defined(_GPU) && defined(_NCCL)
+      module procedure decomp_2d_abort_nccl_basic
+      module procedure decomp_2d_abort_nccl_file_line
+#endif
+   end interface decomp_2d_abort
+
+   interface decomp_2d_warning
+      module procedure decomp_2d_warning_basic
+      module procedure decomp_2d_warning_file_line
+   end interface decomp_2d_warning
+
+contains
+   !
+   ! Set the main MPI comunicator 
+   ! 
+   subroutine decomp_2d_mpi_init(comm)
+      
+      implicit none
+      
+      integer, intent(in), optional :: comm
+
+      integer :: ierror
+
+      ! Use the provided MPI communicator if present
+      if (present(comm)) then
+         decomp_2d_comm = comm
+      else
+         decomp_2d_comm = MPI_COMM_WORLD
+      end if
+
+      ! If the external code has not set nrank and nproc
+      if (nrank == -1) then
+         call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
+         if (ierror /= 0) then
+            call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+         end if
+      end if
+      if (nproc == -1) then
+         call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
+         if (ierror /= 0) then
+            call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+         end if
+      end if
+   
+   end subroutine decomp_2d_mpi_init
+   !
+   ! Reset the MPI
+   ! 
+   subroutine decomp_2d_mpi_fin
+      
+      implicit none
+
+      nrank = -1
+      nproc = -1
+
+   end subroutine decomp_2d_mpi_fin
+   !
+   ! Small wrapper to free a MPI communicator
+   !
+   subroutine decomp_2d_mpi_comm_free(mpi_comm)
+
+      implicit none
+
+      integer, intent(inout) :: mpi_comm
+      integer :: ierror
+
+      ! Return if no MPI comm to free
+      if (mpi_comm == MPI_COMM_NULL) return
+
+      ! Free the provided MPI communicator
+      call MPI_COMM_FREE(mpi_comm, ierror)
+      if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
+      mpi_comm = MPI_COMM_NULL
+
+   end subroutine decomp_2d_mpi_comm_free
+
+   subroutine decomp_2d_abort_basic(errorcode, msg)
+
+      use iso_fortran_env, only: error_unit
+
+      implicit none
+
+      integer, intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
+
+      integer :: ierror
+
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+         write (*, *) 'ERROR MESSAGE: '//msg
+         write (error_unit, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+         write (error_unit, *) 'ERROR MESSAGE: '//msg
+      end if
+      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
+
+   end subroutine decomp_2d_abort_basic
+
+   subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
+
+      use iso_fortran_env, only: error_unit
+
+      implicit none
+
+      integer, intent(IN) :: errorcode, line
+      character(len=*), intent(IN) :: msg, file
+
+      integer :: ierror
+
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT ERROR'
+         write (*, *) '  errorcode:     ', errorcode
+         write (*, *) '  error in file  '//file
+         write (*, *) '           line  ', line
+         write (*, *) '  error message: '//msg
+         write (error_unit, *) '2DECOMP&FFT ERROR'
+         write (error_unit, *) '  errorcode:     ', errorcode
+         write (error_unit, *) '  error in file  '//file
+         write (error_unit, *) '           line  ', line
+         write (error_unit, *) '  error message: '//msg
+      end if
+      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
+
+   end subroutine decomp_2d_abort_file_line
+
+#if defined(_GPU) && defined(_NCCL)
+   !
+   ! This is based on the file "nccl.h" in nvhpc 22.1
+   !
+   function _ncclresult_to_integer(errorcode)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      integer :: _ncclresult_to_integer
+
+      if (errorcode == ncclSuccess) then
+         _ncclresult_to_integer = 0
+      elseif (errorcode == ncclUnhandledCudaError) then
+         _ncclresult_to_integer = 1
+      elseif (errorcode == ncclSystemError) then
+         _ncclresult_to_integer = 2
+      elseif (errorcode == ncclInternalError) then
+         _ncclresult_to_integer = 3
+      elseif (errorcode == ncclInvalidArgument) then
+         _ncclresult_to_integer = 4
+      elseif (errorcode == ncclInvalidUsage) then
+         _ncclresult_to_integer = 5
+      elseif (errorcode == ncclNumResults) then
+         _ncclresult_to_integer = 6
+      else
+         _ncclresult_to_integer = -1
+         call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
+                                "NCCL error handling needs some update")
+      end if
+
+   end function _ncclresult_to_integer
+   !
+   ! Small wrapper for basic NCCL errors
+   !
+   subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
+
+      call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
+                           msg//" "//ncclGetErrorString(errorcode))
+
+   end subroutine decomp_2d_abort_nccl_basic
+
+   !
+   ! Small wrapper for NCCL errors
+   !
+   subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      integer, intent(in) :: line
+      character(len=*), intent(IN) :: msg, file
+
+      call decomp_2d_abort(file, &
+                           line, &
+                           _ncclresult_to_integer(errorcode), &
+                           msg//" "//ncclGetErrorString(errorcode))
+
+   end subroutine decomp_2d_abort_nccl_file_line
+#endif
+
+   subroutine decomp_2d_warning_basic(errorcode, msg)
+
+      use iso_fortran_env, only: error_unit
+
+      implicit none
+
+      integer, intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
+
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+         write (*, *) 'ERROR MESSAGE: '//msg
+         write (error_unit, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+         write (error_unit, *) 'ERROR MESSAGE: '//msg
+      end if
+
+   end subroutine decomp_2d_warning_basic
+
+   subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
+
+      use iso_fortran_env, only: error_unit
+
+      implicit none
+
+      integer, intent(IN) :: errorcode, line
+      character(len=*), intent(IN) :: msg, file
+
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT WARNING'
+         write (*, *) '  errorcode:     ', errorcode
+         write (*, *) '  error in file  '//file
+         write (*, *) '           line  ', line
+         write (*, *) '  error message: '//msg
+         write (error_unit, *) '2DECOMP&FFT WARNING'
+         write (error_unit, *) '  errorcode:     ', errorcode
+         write (error_unit, *) '  error in file  '//file
+         write (error_unit, *) '           line  ', line
+         write (error_unit, *) '  error message: '//msg
+      end if
+
+   end subroutine decomp_2d_warning_file_line
+
+
 end module decomp_2d_constants
 
diff --git a/src/decomp_2d_cumpi.f90 b/src/decomp_2d_cumpi.f90
new file mode 100644
index 00000000..2dc2b813
--- /dev/null
+++ b/src/decomp_2d_cumpi.f90
@@ -0,0 +1,83 @@
+!=======================================================================
+! This is part of the 2DECOMP&FFT library
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! decomposition. It also implements a highly scalable distributed
+! three-dimensional Fast Fourier Transform (FFT).
+!
+! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
+! Copyright (C) 2022               Science and Technology Facilities Council(STFC)
+!
+!=======================================================================
+
+! Module for the cuda aware MPI
+
+module decomp_2d_cumpi
+
+   use decomp_2d_constants  
+
+   implicit none
+   
+   private        ! Make everything private unless declared public
+   
+   ! Device working arrays
+   real(mytype), allocatable, dimension(:), device , public:: work1_r_d, work2_r_d
+   complex(mytype), allocatable, dimension(:), device, public :: work1_c_d, work2_c_d
+
+   public :: decomp_2d_cumpi_init, &
+             decomp_2d_cumpi_fin
+
+contains
+   !
+   ! init of the arrays
+   ! 
+   subroutine decomp_2d_cumpi_init(buf_size)
+      
+      implicit none
+      
+      integer, intent(in) :: buf_size
+      integer :: status, errorcode
+      
+      if (allocated(work1_r_d)) deallocate (work1_r_d)
+      if (allocated(work2_r_d)) deallocate (work2_r_d)
+      if (allocated(work1_c_d)) deallocate (work1_c_d)
+      if (allocated(work2_c_d)) deallocate (work2_c_d)
+      allocate (work1_r_d(buf_size), STAT=status)
+      if (status /= 0) then
+         errorcode = 2
+         call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                              'Out of memory when allocating 2DECOMP workspace')
+      end if
+      allocate (work1_c_d(buf_size), STAT=status)
+      if (status /= 0) then
+         errorcode = 2
+         call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                              'Out of memory when allocating 2DECOMP workspace')
+      end if
+      allocate (work2_r_d(buf_size), STAT=status)
+      if (status /= 0) then
+         errorcode = 2
+         call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                              'Out of memory when allocating 2DECOMP workspace')
+      end if
+      allocate (work2_c_d(buf_size), STAT=status)
+      if (status /= 0) then
+         errorcode = 2
+         call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
+                              'Out of memory when allocating 2DECOMP workspace')
+      end if
+   
+   end subroutine decomp_2d_cumpi_init
+   !
+   ! init of the arrays
+   ! 
+   subroutine decomp_2d_cumpi_fin
+      
+      implicit none
+
+      deallocate (work1_r_d, work2_r_d, work1_c_d, work2_c_d)
+
+   end subroutine decomp_2d_cumpi_fin
+
+end module decomp_2d_cumpi
+
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index f5afdff2..71dd6e8c 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -47,34 +47,12 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
      if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_init")
 #endif
 
-     ! Use the provided MPI communicator if present
-     if (present(comm)) then
-        decomp_2d_comm = comm
-     else
-        decomp_2d_comm = MPI_COMM_WORLD
-     end if
+     call decomp_2d_mpi_init(comm)
 
-     ! Safety check
-     if (MPI_SUCCESS /= 0) then
-        call decomp_2d_abort(__FILE__, __LINE__, MPI_SUCCESS, "MPI error check is broken")
-     end if
      if (nx <= 0) call decomp_2d_abort(__FILE__, __LINE__, nx, "Invalid value for nx")
      if (ny <= 0) call decomp_2d_abort(__FILE__, __LINE__, ny, "Invalid value for ny")
      if (nz <= 0) call decomp_2d_abort(__FILE__, __LINE__, nz, "Invalid value for nz")
 
-     ! If the external code has not set nrank and nproc
-     if (nrank == -1) then
-        call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
-        if (ierror /= 0) then
-           call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-        end if
-     end if
-     if (nproc == -1) then
-        call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
-        if (ierror /= 0) then
-           call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-        end if
-     end if
 #ifdef DEBUG
      ! Check if a modification of the debug level is needed
      call decomp_2d_debug()
@@ -240,26 +218,24 @@ subroutine decomp_2d_finalize_ref
      if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_fin")
 #endif
 
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_ROW)
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_COL)
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_X)
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Y)
-     call decomp_mpi_comm_free(DECOMP_2D_COMM_CART_Z)
+     call decomp_2d_mpi_comm_free(DECOMP_2D_COMM_ROW)
+     call decomp_2d_mpi_comm_free(DECOMP_2D_COMM_COL)
+     call decomp_2d_mpi_comm_free(DECOMP_2D_COMM_CART_X)
+     call decomp_2d_mpi_comm_free(DECOMP_2D_COMM_CART_Y)
+     call decomp_2d_mpi_comm_free(DECOMP_2D_COMM_CART_Z)
 
      call decomp_info_finalize(decomp_main)
 
      decomp_buf_size = 0
      deallocate (work1_r, work2_r, work1_c, work2_c)
 #if defined(_GPU)
-     deallocate (work1_r_d, work2_r_d, work1_c_d, work2_c_d)
-
+     call decomp_2d_cumpi_fin()
 #if defined(_NCCL)
      nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
 #endif
 #endif
 
-     nrank = -1
-     nproc = -1
+     call decomp_2d_mpi_fin()
 
 #ifdef PROFILER
      if (decomp_profiler_d2d) call decomp_profiler_end("decomp_2d_fin")
diff --git a/src/glassman.f90 b/src/glassman.f90
index d0572101..9e2ca251 100644
--- a/src/glassman.f90
+++ b/src/glassman.f90
@@ -14,7 +14,7 @@
 
 module glassman
 
-   use decomp_2d, only: mytype
+   use decomp_2d_constants, only: mytype
 
    implicit none
 
diff --git a/src/io.f90 b/src/io.f90
index 03411390..308f2c15 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -16,6 +16,7 @@
 module decomp_2d_io
 
    use decomp_2d
+   use decomp_2d_constants
    use MPI
 
 #ifdef ADIOS2
@@ -1760,7 +1761,7 @@ subroutine write_subdomain(ipencil, var, is, ie, js, je, ks, ke, filename)
 
       end if
 
-      call decomp_mpi_comm_free(newcomm)
+      call decomp_2d_mpi_comm_free(newcomm)
 
 #ifdef PROFILER
       if (decomp_profiler_io) call decomp_profiler_end("io_write_subdomain")
diff --git a/src/io_write_every.inc b/src/io_write_every.inc
index 3289264f..f11af06a 100644
--- a/src/io_write_every.inc
+++ b/src/io_write_every.inc
@@ -230,7 +230,7 @@ if (color == 1) then ! only ranks in this group do IO collectively
 
 end if ! color==1
 
-call decomp_mpi_comm_free(newcomm)
+call decomp_2d_mpi_comm_free(newcomm)
 
 call MPI_BARRIER(decomp_2d_comm, ierror)
 if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BARRIER")

From 0fb1427ce0d2d0b528a210e52ade62fc42402409 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 16 Mar 2023 06:51:12 +0000
Subject: [PATCH 169/436] Reinstate check for MPI_SUCCESS in MPI initialisation

---
 src/decomp_2d_constants.f90 | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 04596a3c..8ca30c9d 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -173,6 +173,10 @@ subroutine decomp_2d_mpi_init(comm)
             call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
          end if
       end if
+
+      if (MPI_SUCCESS /= 0) then
+         call decomp_2d_abort(__FILE__, __LINE__, MPI_SUCCESS, "MPI error check is broken")
+      end if
    
    end subroutine decomp_2d_mpi_init
    !

From 32c56cb082d35fa8a98abe07b310c74574031d99 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 16 Mar 2023 12:45:13 +0100
Subject: [PATCH 170/436] Move MPI-related stuff in a dedicated module

---
 Makefile                               |   2 +-
 examples/fft_physical_x/fft_c2c_x.f90  |   1 +
 examples/fft_physical_x/fft_grid_x.f90 |   1 +
 examples/fft_physical_x/fft_r2c_x.f90  |   1 +
 examples/fft_physical_z/fft_c2c_z.f90  |   1 +
 examples/fft_physical_z/fft_r2c_z.f90  |   1 +
 examples/halo_test/halo_test.f90       |   1 +
 examples/init_test/init_test.f90       |   1 +
 examples/io_test/io_bench.f90          |   1 +
 examples/io_test/io_plane_test.f90     |   1 +
 examples/io_test/io_read.f90           |   1 +
 examples/io_test/io_test.f90           |   1 +
 examples/io_test/io_var_test.f90       |   1 +
 examples/test2d/test2d.f90             |   1 +
 examples/test2d/timing2d_complex.f90   |   1 +
 examples/test2d/timing2d_real.f90      |   1 +
 src/CMakeLists.txt                     |   1 +
 src/decomp_2d.f90                      |   2 +-
 src/decomp_2d_constants.f90            | 255 +---------------------
 src/decomp_2d_mpi.f90                  | 280 +++++++++++++++++++++++++
 src/fft_cufft.f90                      |   1 +
 src/fft_fftw3.f90                      |   1 +
 src/fft_fftw3_f03.f90                  |   1 +
 src/fft_generic.f90                    |   1 +
 src/fft_mkl.f90                        |   1 +
 src/io.f90                             |   1 +
 26 files changed, 306 insertions(+), 255 deletions(-)
 create mode 100644 src/decomp_2d_mpi.f90

diff --git a/Makefile b/Makefile
index 2fe6f535..687b6b2e 100644
--- a/Makefile
+++ b/Makefile
@@ -39,7 +39,7 @@ CMPINC = Makefile.compilers
 include $(CMPINC)
 
 ### List of files for the main code
-SRCDECOMP = decomp_2d_constants.f90 factor.f90 decomp_2d.f90 log.f90 io.f90
+SRCDECOMP = decomp_2d_constants.f90 decomp_2d_mpi.f90 factor.f90 decomp_2d.f90 log.f90 io.f90
 
 #######FFT settings##########
 ifeq ($(FFT),fftw3)
diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 31eea76a..453d3aab 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -3,6 +3,7 @@ program fft_c2c_x
    use decomp_2d
    use decomp_2d_fft
    use decomp_2d_constants
+   use decomp_2d_mpi
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index c0d12d3e..af508820 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -3,6 +3,7 @@ program fft_physical_x
    use decomp_2d
    use decomp_2d_fft
    use decomp_2d_constants
+   use decomp_2d_mpi
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 908e19ea..897a2e17 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -3,6 +3,7 @@ program fft_r2c_x
    use decomp_2d
    use decomp_2d_fft
    use decomp_2d_constants
+   use decomp_2d_mpi
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 4e80b6ca..5434f8cd 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -3,6 +3,7 @@ program fft_c2c_z
    use decomp_2d
    use decomp_2d_fft
    use decomp_2d_constants
+   use decomp_2d_mpi
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 6e786373..5671e48a 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -3,6 +3,7 @@ program fft_r2c_z
    use decomp_2d
    use decomp_2d_fft
    use decomp_2d_constants
+   use decomp_2d_mpi
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 5fed1ba2..faf53951 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -10,6 +10,7 @@ program halo_test
 
    use decomp_2d
    use decomp_2d_constants
+   use decomp_2d_mpi
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 46c2d0cb..289ec200 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -7,6 +7,7 @@ program init_test
    use MPI
    use decomp_2d
    use decomp_2d_constants
+   use decomp_2d_mpi
    use MPI
 #if defined(_GPU)
    use cudafor
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index 2a877a88..0f5926ed 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -1,6 +1,7 @@
 program io_bench
 
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d
    use decomp_2d_io
    use MPI
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 81f9d9d6..f362bb9e 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -8,6 +8,7 @@ program io_plane_test
 
    use mpi
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d
    use decomp_2d_io
 #if defined(_GPU)
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 4fd564f3..3eded469 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -2,6 +2,7 @@ program io_read
 
    use mpi
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d
    use decomp_2d_io
 #if defined(_GPU)
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 114614b7..ba4378c0 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -2,6 +2,7 @@ program io_test
 
    use mpi
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d
    use decomp_2d_io
 #if defined(_GPU)
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index c5d35839..2ae5abb5 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -5,6 +5,7 @@ program io_var_test
 
    use mpi
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d
    use decomp_2d_io
 #if defined(_GPU)
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 4f54d2f8..c56f99f8 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -3,6 +3,7 @@ program test2d
    use mpi
    use decomp_2d
    use decomp_2d_constants
+   use decomp_2d_mpi
 #if defined(_GPU)
    use cudafor
    use openacc
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 130c82d8..54063ed0 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -3,6 +3,7 @@ program timing2d_complex
    use mpi
    use decomp_2d
    use decomp_2d_constants
+   use decomp_2d_mpi
 #if defined(_GPU)
    use cudafor
    use openacc
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 7a300f7c..09e04996 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -3,6 +3,7 @@ program timing2d_real
    use mpi
    use decomp_2d
    use decomp_2d_constants
+   use decomp_2d_mpi
 #if defined(_GPU)
    use cudafor
    use openacc
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 969cd716..d26724c3 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,5 +1,6 @@
 file(GLOB files_decomp factor.f90
 	               decomp_2d_constants.f90
+		       decomp_2d_mpi.f90
 	               decomp_2d.f90
 		       log.f90
 		       io.f90)
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 8b39d4b7..06451e05 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -18,6 +18,7 @@ module decomp_2d
    use, intrinsic :: iso_fortran_env, only: real32, real64
    use factor
    use decomp_2d_constants
+   use decomp_2d_mpi
 #if defined(_GPU)
    use cudafor
    use decomp_2d_cumpi
@@ -1164,4 +1165,3 @@ end subroutine prepare_buffer
 #include "alloc.f90"
 
 end module decomp_2d
-
diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 8ca30c9d..19356f6c 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -37,7 +37,7 @@ module decomp_2d_constants
    integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
 #endif
 #if defined(_GPU) && defined(_NCCL)
-   type(ncclDataType) :: ncclType = ncclDouble
+   type(ncclDataType), parameter, public :: ncclType = ncclDouble
 #endif
 #else
    integer, parameter, public :: mytype = KIND(0._real32)
@@ -47,15 +47,10 @@ module decomp_2d_constants
    integer, parameter, public :: mytype_single = KIND(0._real32)
    integer, parameter, public :: real_type_single = MPI_REAL
 #if defined(_GPU) && defined(_NCCL)
-   type(ncclDataType) :: ncclType = ncclFloat
+   type(ncclDataType), parameter, public :: ncclType = ncclFloat
 #endif
 #endif
 
-   integer, save, public :: mytype_bytes
-   ! Global MPI parameters
-   integer, save, public :: nrank = -1 ! local MPI rank
-   integer, save, public :: nproc = -1 ! total number of processors
-   integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
    !
    ! Output for the log can be changed by the external code before calling decomp_2d_init
    !
@@ -121,251 +116,5 @@ module decomp_2d_constants
    integer, parameter, public :: PHYSICAL_IN_X = 1 ! Forward is input in X, output in Z
    integer, parameter, public :: PHYSICAL_IN_Z = 3 ! Forward is input in Z, output in X
 
-   public :: decomp_2d_mpi_init, &
-             decomp_2d_mpi_fin, &
-             decomp_2d_mpi_comm_free, &
-             decomp_2d_abort, &
-             decomp_2d_warning 
-
-   interface decomp_2d_abort
-      module procedure decomp_2d_abort_basic
-      module procedure decomp_2d_abort_file_line
-#if defined(_GPU) && defined(_NCCL)
-      module procedure decomp_2d_abort_nccl_basic
-      module procedure decomp_2d_abort_nccl_file_line
-#endif
-   end interface decomp_2d_abort
-
-   interface decomp_2d_warning
-      module procedure decomp_2d_warning_basic
-      module procedure decomp_2d_warning_file_line
-   end interface decomp_2d_warning
-
-contains
-   !
-   ! Set the main MPI comunicator 
-   ! 
-   subroutine decomp_2d_mpi_init(comm)
-      
-      implicit none
-      
-      integer, intent(in), optional :: comm
-
-      integer :: ierror
-
-      ! Use the provided MPI communicator if present
-      if (present(comm)) then
-         decomp_2d_comm = comm
-      else
-         decomp_2d_comm = MPI_COMM_WORLD
-      end if
-
-      ! If the external code has not set nrank and nproc
-      if (nrank == -1) then
-         call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
-         if (ierror /= 0) then
-            call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-         end if
-      end if
-      if (nproc == -1) then
-         call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
-         if (ierror /= 0) then
-            call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-         end if
-      end if
-
-      if (MPI_SUCCESS /= 0) then
-         call decomp_2d_abort(__FILE__, __LINE__, MPI_SUCCESS, "MPI error check is broken")
-      end if
-   
-   end subroutine decomp_2d_mpi_init
-   !
-   ! Reset the MPI
-   ! 
-   subroutine decomp_2d_mpi_fin
-      
-      implicit none
-
-      nrank = -1
-      nproc = -1
-
-   end subroutine decomp_2d_mpi_fin
-   !
-   ! Small wrapper to free a MPI communicator
-   !
-   subroutine decomp_2d_mpi_comm_free(mpi_comm)
-
-      implicit none
-
-      integer, intent(inout) :: mpi_comm
-      integer :: ierror
-
-      ! Return if no MPI comm to free
-      if (mpi_comm == MPI_COMM_NULL) return
-
-      ! Free the provided MPI communicator
-      call MPI_COMM_FREE(mpi_comm, ierror)
-      if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
-      mpi_comm = MPI_COMM_NULL
-
-   end subroutine decomp_2d_mpi_comm_free
-
-   subroutine decomp_2d_abort_basic(errorcode, msg)
-
-      use iso_fortran_env, only: error_unit
-
-      implicit none
-
-      integer, intent(IN) :: errorcode
-      character(len=*), intent(IN) :: msg
-
-      integer :: ierror
-
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-         write (*, *) 'ERROR MESSAGE: '//msg
-         write (error_unit, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
-         write (error_unit, *) 'ERROR MESSAGE: '//msg
-      end if
-      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
-
-   end subroutine decomp_2d_abort_basic
-
-   subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
-
-      use iso_fortran_env, only: error_unit
-
-      implicit none
-
-      integer, intent(IN) :: errorcode, line
-      character(len=*), intent(IN) :: msg, file
-
-      integer :: ierror
-
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT ERROR'
-         write (*, *) '  errorcode:     ', errorcode
-         write (*, *) '  error in file  '//file
-         write (*, *) '           line  ', line
-         write (*, *) '  error message: '//msg
-         write (error_unit, *) '2DECOMP&FFT ERROR'
-         write (error_unit, *) '  errorcode:     ', errorcode
-         write (error_unit, *) '  error in file  '//file
-         write (error_unit, *) '           line  ', line
-         write (error_unit, *) '  error message: '//msg
-      end if
-      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
-
-   end subroutine decomp_2d_abort_file_line
-
-#if defined(_GPU) && defined(_NCCL)
-   !
-   ! This is based on the file "nccl.h" in nvhpc 22.1
-   !
-   function _ncclresult_to_integer(errorcode)
-
-      implicit none
-
-      type(ncclresult), intent(IN) :: errorcode
-      integer :: _ncclresult_to_integer
-
-      if (errorcode == ncclSuccess) then
-         _ncclresult_to_integer = 0
-      elseif (errorcode == ncclUnhandledCudaError) then
-         _ncclresult_to_integer = 1
-      elseif (errorcode == ncclSystemError) then
-         _ncclresult_to_integer = 2
-      elseif (errorcode == ncclInternalError) then
-         _ncclresult_to_integer = 3
-      elseif (errorcode == ncclInvalidArgument) then
-         _ncclresult_to_integer = 4
-      elseif (errorcode == ncclInvalidUsage) then
-         _ncclresult_to_integer = 5
-      elseif (errorcode == ncclNumResults) then
-         _ncclresult_to_integer = 6
-      else
-         _ncclresult_to_integer = -1
-         call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
-                                "NCCL error handling needs some update")
-      end if
-
-   end function _ncclresult_to_integer
-   !
-   ! Small wrapper for basic NCCL errors
-   !
-   subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
-
-      implicit none
-
-      type(ncclresult), intent(IN) :: errorcode
-      character(len=*), intent(IN) :: msg
-
-      call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
-                           msg//" "//ncclGetErrorString(errorcode))
-
-   end subroutine decomp_2d_abort_nccl_basic
-
-   !
-   ! Small wrapper for NCCL errors
-   !
-   subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
-
-      implicit none
-
-      type(ncclresult), intent(IN) :: errorcode
-      integer, intent(in) :: line
-      character(len=*), intent(IN) :: msg, file
-
-      call decomp_2d_abort(file, &
-                           line, &
-                           _ncclresult_to_integer(errorcode), &
-                           msg//" "//ncclGetErrorString(errorcode))
-
-   end subroutine decomp_2d_abort_nccl_file_line
-#endif
-
-   subroutine decomp_2d_warning_basic(errorcode, msg)
-
-      use iso_fortran_env, only: error_unit
-
-      implicit none
-
-      integer, intent(IN) :: errorcode
-      character(len=*), intent(IN) :: msg
-
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-         write (*, *) 'ERROR MESSAGE: '//msg
-         write (error_unit, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
-         write (error_unit, *) 'ERROR MESSAGE: '//msg
-      end if
-
-   end subroutine decomp_2d_warning_basic
-
-   subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
-
-      use iso_fortran_env, only: error_unit
-
-      implicit none
-
-      integer, intent(IN) :: errorcode, line
-      character(len=*), intent(IN) :: msg, file
-
-      if (nrank == 0) then
-         write (*, *) '2DECOMP&FFT WARNING'
-         write (*, *) '  errorcode:     ', errorcode
-         write (*, *) '  error in file  '//file
-         write (*, *) '           line  ', line
-         write (*, *) '  error message: '//msg
-         write (error_unit, *) '2DECOMP&FFT WARNING'
-         write (error_unit, *) '  errorcode:     ', errorcode
-         write (error_unit, *) '  error in file  '//file
-         write (error_unit, *) '           line  ', line
-         write (error_unit, *) '  error message: '//msg
-      end if
-
-   end subroutine decomp_2d_warning_file_line
-
-
 end module decomp_2d_constants
 
diff --git a/src/decomp_2d_mpi.f90 b/src/decomp_2d_mpi.f90
new file mode 100644
index 00000000..fee81051
--- /dev/null
+++ b/src/decomp_2d_mpi.f90
@@ -0,0 +1,280 @@
+!=======================================================================
+! This is part of the 2DECOMP&FFT library
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! decomposition. It also implements a highly scalable distributed
+! three-dimensional Fast Fourier Transform (FFT).
+!
+! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
+! Copyright (C) 2021               the University of Edinburgh (UoE)
+!
+!=======================================================================
+
+! MPI module for 2decomp&fft library
+
+module decomp_2d_mpi
+
+   use MPI
+   use decomp_2d_constants
+#if defined(_GPU)
+#if defined(_NCCL)
+   use nccl
+#endif
+#endif
+
+   implicit none
+
+   integer, save, public :: mytype_bytes
+   integer, save, public :: nrank = -1 ! local MPI rank
+   integer, save, public :: nproc = -1 ! total number of processors
+   integer, save, public :: decomp_2d_comm = MPI_COMM_NULL ! MPI communicator
+
+   public :: decomp_2d_mpi_init, &
+             decomp_2d_mpi_fin, &
+             decomp_2d_mpi_comm_free, &
+             decomp_2d_abort, &
+             decomp_2d_warning
+
+   interface decomp_2d_abort
+      module procedure decomp_2d_abort_basic
+      module procedure decomp_2d_abort_file_line
+#if defined(_GPU) && defined(_NCCL)
+      module procedure decomp_2d_abort_nccl_basic
+      module procedure decomp_2d_abort_nccl_file_line
+#endif
+   end interface decomp_2d_abort
+
+   interface decomp_2d_warning
+      module procedure decomp_2d_warning_basic
+      module procedure decomp_2d_warning_file_line
+   end interface decomp_2d_warning
+
+contains
+
+   !
+   ! Initialize the MPI module
+   !
+   subroutine decomp_2d_mpi_init(comm)
+
+      implicit none
+
+      integer, intent(in), optional :: comm
+
+      integer :: ierror
+
+      ! Use the provided MPI communicator if present
+      if (present(comm)) then
+         decomp_2d_comm = comm
+      else
+         decomp_2d_comm = MPI_COMM_WORLD
+      end if
+
+      ! If the external code has not set nrank and nproc
+      if (nrank == -1) then
+         call MPI_COMM_RANK(decomp_2d_comm, nrank, ierror)
+         if (ierror /= 0) then
+            call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+         end if
+      end if
+      if (nproc == -1) then
+         call MPI_COMM_SIZE(decomp_2d_comm, nproc, ierror)
+         if (ierror /= 0) then
+            call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+         end if
+      end if
+
+      if (MPI_SUCCESS /= 0) then
+         call decomp_2d_abort(__FILE__, __LINE__, MPI_SUCCESS, "MPI error check is broken")
+      end if
+
+   end subroutine decomp_2d_mpi_init
+
+   !
+   ! Finalize the MPI module
+   !
+   subroutine decomp_2d_mpi_fin
+
+      implicit none
+
+      nrank = -1
+      nproc = -1
+
+   end subroutine decomp_2d_mpi_fin
+
+   !
+   ! Small wrapper to free a MPI communicator
+   !
+   subroutine decomp_2d_mpi_comm_free(mpi_comm)
+
+      implicit none
+
+      integer, intent(inout) :: mpi_comm
+      integer :: ierror
+
+      ! Return if no MPI comm to free
+      if (mpi_comm == MPI_COMM_NULL) return
+
+      ! Free the provided MPI communicator
+      call MPI_COMM_FREE(mpi_comm, ierror)
+!      if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
+      mpi_comm = MPI_COMM_NULL
+
+   end subroutine decomp_2d_mpi_comm_free
+
+   subroutine decomp_2d_abort_basic(errorcode, msg)
+
+      use iso_fortran_env, only: error_unit
+
+      implicit none
+
+      integer, intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
+
+      integer :: ierror
+
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+         write (*, *) 'ERROR MESSAGE: '//msg
+         write (error_unit, *) '2DECOMP&FFT ERROR - errorcode: ', errorcode
+         write (error_unit, *) 'ERROR MESSAGE: '//msg
+      end if
+      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
+
+   end subroutine decomp_2d_abort_basic
+
+   subroutine decomp_2d_abort_file_line(file, line, errorcode, msg)
+
+      use iso_fortran_env, only: error_unit
+
+      implicit none
+
+      integer, intent(IN) :: errorcode, line
+      character(len=*), intent(IN) :: msg, file
+
+      integer :: ierror
+
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT ERROR'
+         write (*, *) '  errorcode:     ', errorcode
+         write (*, *) '  error in file  '//file
+         write (*, *) '           line  ', line
+         write (*, *) '  error message: '//msg
+         write (error_unit, *) '2DECOMP&FFT ERROR'
+         write (error_unit, *) '  errorcode:     ', errorcode
+         write (error_unit, *) '  error in file  '//file
+         write (error_unit, *) '           line  ', line
+         write (error_unit, *) '  error message: '//msg
+      end if
+      call MPI_ABORT(decomp_2d_comm, errorcode, ierror)
+
+   end subroutine decomp_2d_abort_file_line
+
+#if defined(_GPU) && defined(_NCCL)
+   !
+   ! This is based on the file "nccl.h" in nvhpc 22.1
+   !
+   function _ncclresult_to_integer(errorcode)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      integer :: _ncclresult_to_integer
+
+      if (errorcode == ncclSuccess) then
+         _ncclresult_to_integer = 0
+      elseif (errorcode == ncclUnhandledCudaError) then
+         _ncclresult_to_integer = 1
+      elseif (errorcode == ncclSystemError) then
+         _ncclresult_to_integer = 2
+      elseif (errorcode == ncclInternalError) then
+         _ncclresult_to_integer = 3
+      elseif (errorcode == ncclInvalidArgument) then
+         _ncclresult_to_integer = 4
+      elseif (errorcode == ncclInvalidUsage) then
+         _ncclresult_to_integer = 5
+      elseif (errorcode == ncclNumResults) then
+         _ncclresult_to_integer = 6
+      else
+         _ncclresult_to_integer = -1
+         call decomp_2d_warning(__FILE__, __LINE__, _ncclresult_to_integer, &
+                                "NCCL error handling needs some update")
+      end if
+
+   end function _ncclresult_to_integer
+   !
+   ! Small wrapper for basic NCCL errors
+   !
+   subroutine decomp_2d_abort_nccl_basic(errorcode, msg)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
+
+      call decomp_2d_abort(_ncclresult_to_integer(errorcode), &
+                           msg//" "//ncclGetErrorString(errorcode))
+
+   end subroutine decomp_2d_abort_nccl_basic
+
+   !
+   ! Small wrapper for NCCL errors
+   !
+   subroutine decomp_2d_abort_nccl_file_line(file, line, errorcode, msg)
+
+      implicit none
+
+      type(ncclresult), intent(IN) :: errorcode
+      integer, intent(in) :: line
+      character(len=*), intent(IN) :: msg, file
+
+      call decomp_2d_abort(file, &
+                           line, &
+                           _ncclresult_to_integer(errorcode), &
+                           msg//" "//ncclGetErrorString(errorcode))
+
+   end subroutine decomp_2d_abort_nccl_file_line
+#endif
+
+   subroutine decomp_2d_warning_basic(errorcode, msg)
+
+      use iso_fortran_env, only: error_unit
+
+      implicit none
+
+      integer, intent(IN) :: errorcode
+      character(len=*), intent(IN) :: msg
+
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+         write (*, *) 'ERROR MESSAGE: '//msg
+         write (error_unit, *) '2DECOMP&FFT WARNING - errorcode: ', errorcode
+         write (error_unit, *) 'ERROR MESSAGE: '//msg
+      end if
+
+   end subroutine decomp_2d_warning_basic
+
+   subroutine decomp_2d_warning_file_line(file, line, errorcode, msg)
+
+      use iso_fortran_env, only: error_unit
+
+      implicit none
+
+      integer, intent(IN) :: errorcode, line
+      character(len=*), intent(IN) :: msg, file
+
+      if (nrank == 0) then
+         write (*, *) '2DECOMP&FFT WARNING'
+         write (*, *) '  errorcode:     ', errorcode
+         write (*, *) '  error in file  '//file
+         write (*, *) '           line  ', line
+         write (*, *) '  error message: '//msg
+         write (error_unit, *) '2DECOMP&FFT WARNING'
+         write (error_unit, *) '  errorcode:     ', errorcode
+         write (error_unit, *) '  error in file  '//file
+         write (error_unit, *) '           line  ', line
+         write (error_unit, *) '  error message: '//msg
+      end if
+
+   end subroutine decomp_2d_warning_file_line
+
+end module decomp_2d_mpi
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index bbeabde5..6f08dfe3 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -14,6 +14,7 @@
 module decomp_2d_fft
 
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d  ! 2D decomposition module
    use iso_c_binding
    use cudafor
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index 5e9188c1..a9dc689e 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -14,6 +14,7 @@
 module decomp_2d_fft
 
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d  ! 2D decomposition module
    use iso_c_binding
 
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index d7ddc03f..d06475d1 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -15,6 +15,7 @@
 module decomp_2d_fft
 
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d  ! 2D decomposition module
    use, intrinsic :: iso_c_binding
 
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index 047bc9aa..cff64e98 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -15,6 +15,7 @@ module decomp_2d_fft
 
    use iso_c_binding, only: c_f_pointer, c_loc
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d  ! 2D decomposition module
    use glassman
 
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 29fe70ce..be4d419d 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -15,6 +15,7 @@ module decomp_2d_fft
 
    use iso_c_binding, only: c_f_pointer, c_loc
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d  ! 2D decomposition module
    use MKL_DFTI   ! MKL FFT module
 
diff --git a/src/io.f90 b/src/io.f90
index 308f2c15..8855e094 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -17,6 +17,7 @@ module decomp_2d_io
 
    use decomp_2d
    use decomp_2d_constants
+   use decomp_2d_mpi
    use MPI
 
 #ifdef ADIOS2

From 9a40bc3031fe185837f26f42f46b72781c07dccd Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 16 Mar 2023 14:46:47 +0000
Subject: [PATCH 171/436] Bug corrections for NVHPC

---
 examples/init_test/init_test.f90 | 3 +--
 src/decomp_2d_cumpi.f90          | 1 +
 src/fft_log.f90                  | 1 +
 src/log.f90                      | 1 +
 4 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 289ec200..dfe53a0e 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -5,10 +5,9 @@
 program init_test
 
    use MPI
-   use decomp_2d
    use decomp_2d_constants
    use decomp_2d_mpi
-   use MPI
+   use decomp_2d
 #if defined(_GPU)
    use cudafor
    use openacc
diff --git a/src/decomp_2d_cumpi.f90 b/src/decomp_2d_cumpi.f90
index 2dc2b813..59a1199c 100644
--- a/src/decomp_2d_cumpi.f90
+++ b/src/decomp_2d_cumpi.f90
@@ -15,6 +15,7 @@
 module decomp_2d_cumpi
 
    use decomp_2d_constants  
+   use decomp_2d_mpi 
 
    implicit none
    
diff --git a/src/fft_log.f90 b/src/fft_log.f90
index b0608486..45349373 100644
--- a/src/fft_log.f90
+++ b/src/fft_log.f90
@@ -13,6 +13,7 @@
 submodule(decomp_2d_fft) d2d_fft_log
 
    use decomp_2d_constants
+   use decomp_2d_mpi
    use decomp_2d
 
    implicit none
diff --git a/src/log.f90 b/src/log.f90
index 0c8bd975..809d5820 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -13,6 +13,7 @@
 submodule(decomp_2d) d2d_log
 
    use decomp_2d_constants
+   use decomp_2d_mpi
 
    implicit none
 

From b13251642a1eb8d0294786e10f04774540973ba8 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 16 Mar 2023 16:12:14 +0100
Subject: [PATCH 172/436] Minor update

---
 src/decomp_2d_mpi.f90 | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/decomp_2d_mpi.f90 b/src/decomp_2d_mpi.f90
index fee81051..25b824e9 100644
--- a/src/decomp_2d_mpi.f90
+++ b/src/decomp_2d_mpi.f90
@@ -16,10 +16,8 @@ module decomp_2d_mpi
 
    use MPI
    use decomp_2d_constants
-#if defined(_GPU)
-#if defined(_NCCL)
+#if defined(_GPU) && defined(_NCCL)
    use nccl
-#endif
 #endif
 
    implicit none

From 864a1508d4487b92fba7bba1ebe8a98bfb5c893d Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 16 Mar 2023 16:23:41 +0100
Subject: [PATCH 173/436] Add error checks

---
 src/decomp_2d_init_fin.f90 | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index 71dd6e8c..db820db0 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -178,12 +178,17 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
 
      if (nrank == 0) then
         nccl_stat = ncclGetUniqueId(nccl_uid_2decomp)
+     else
+        nccl_stat = ncclSuccess
      end if
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGetUniqueId")
      call MPI_Bcast(nccl_uid_2decomp, int(sizeof(ncclUniqueId)), MPI_BYTE, 0, decomp_2d_comm, ierror)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
 
      nccl_stat = ncclCommInitRank(nccl_comm_2decomp, nproc, nccl_uid_2decomp, nrank)
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclCommInitRank")
      cuda_stat = cudaStreamCreate(cuda_stream_2decomp)
+     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamCreate")
 #endif
 #endif
 
@@ -232,6 +237,7 @@ subroutine decomp_2d_finalize_ref
      call decomp_2d_cumpi_fin()
 #if defined(_NCCL)
      nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
+     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclCommDestroy")
 #endif
 #endif
 

From 8e0cf77f909c9bff2d0c2c32fa75d36c44ce9daa Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 16 Mar 2023 19:22:35 +0000
Subject: [PATCH 174/436] Remove the makefile (#155)

* The grand purging of Makefiles

* Remove Makefile from README

* Move Build test to CMake

* Replace tabs in github action

* Build library/examples separately on github

* Begin porting full CI suite to cmake

* Update CI steps using cmake to reflect README instructions

* Restore clean build when CI builds IFORT/MKL

* Add instructions for building decomp2d from external project

* Ignore Vim swp files

* Use full build for GNU checks with verbose output

* Add Dev flags for Nvidia compilers

* Build with nvidia

* Update Intel cmake build

* Add support to build F2003 FFTW3 with CMake

* Test build FFTW with cmake

* Build test caliper using cmake

* Update building FFTW3 instructions

* Add caliper build instructions for Cmake
---
 .github/workflows/Build.yml            |  12 +-
 .github/workflows/Build_all.yml        |  51 +++++---
 .gitignore                             |   3 +
 CMakeLists.txt                         |   2 +-
 Makefile                               | 168 -------------------------
 Makefile.compilers                     | 124 ------------------
 README.md                              | 121 +++++++++---------
 cmake/compilers/D2D_flags_nvidia.cmake |   1 +
 examples/Makefile                      |  36 ------
 examples/fft_physical_x/Makefile       |  41 ------
 examples/fft_physical_z/Makefile       |  36 ------
 examples/halo_test/Makefile            |  30 -----
 examples/init_test/Makefile            |  30 -----
 examples/io_test/Makefile              |  54 --------
 examples/test2d/Makefile               |  40 ------
 src/CMakeLists.txt                     |   9 +-
 16 files changed, 112 insertions(+), 646 deletions(-)
 delete mode 100644 Makefile
 delete mode 100644 Makefile.compilers
 delete mode 100644 examples/Makefile
 delete mode 100644 examples/fft_physical_x/Makefile
 delete mode 100644 examples/fft_physical_z/Makefile
 delete mode 100644 examples/halo_test/Makefile
 delete mode 100644 examples/init_test/Makefile
 delete mode 100644 examples/io_test/Makefile
 delete mode 100644 examples/test2d/Makefile

diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml
index b0861cce..04ddea08 100644
--- a/.github/workflows/Build.yml
+++ b/.github/workflows/Build.yml
@@ -24,11 +24,17 @@ jobs:
     - name: Install requirements
       run: |
         sudo apt-get update
-        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev
+        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev cmake
 
     # Runs a set of commands using the runners shell
     - name: Compile
-      run: make BUILD=dev
+      run: |
+        export FC=mpif90
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=dev
+        cmake --build build --target decomp2d
 
     - name: Examples
-      run: make examples
+      run: |
+        export FC=mpif90
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=dev
+        cmake --build build
diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 4f1970a6..11d753a4 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -24,11 +24,14 @@ jobs:
     - name: Install requirements
       run: |
         sudo apt-get update
-        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev
+        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev cmake
 
     # Runs a set of commands using the runners shell
     - name: Compile
-      run: make BUILD=dev && make examples
+      run: |
+        export FC=mpif90
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
+        cmake --build build --verbose
 
   bld_gnu_mpich:
     # The type of runner that the job will run on
@@ -43,11 +46,14 @@ jobs:
     - name: Install requirements
       run: |
         sudo apt-get update
-        sudo apt-get install -y gfortran mpich libmpich-dev
+        sudo apt-get install -y gfortran mpich libmpich-dev cmake
 
     # Runs a set of commands using the runners shell
     - name: Compile
-      run: make BUILD=debug && make examples
+      run: |
+        export FC=mpif90
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
+        cmake --build build --verbose
 
   bld_nvidia:
     # The type of runner that the job will run on
@@ -74,8 +80,8 @@ jobs:
     # Runs a set of commands using the runners shell
     - name: CPU version
       run: |
-        PATH=$TMP:$PATH make CMP=nvhpc BUILD=debug
-        PATH=$TMP:$PATH make examples
+        PATH=$TMP:$PATH FC=mpif90 cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
+        PATH=$TMP:$PATH cmake --build build --verbose
 
 ###    - name: GPU version
 ###      run: |
@@ -99,15 +105,25 @@ jobs:
     - name: install
       run: |
         sudo apt-get update
-        sudo apt-get install -y intel-oneapi-common-vars intel-oneapi-compiler-fortran intel-oneapi-mkl intel-oneapi-mkl-devel intel-oneapi-mpi intel-oneapi-mpi-devel
+        sudo apt-get install -y intel-oneapi-common-vars intel-oneapi-compiler-fortran intel-oneapi-mkl intel-oneapi-mkl-devel intel-oneapi-mpi intel-oneapi-mpi-devel cmake
 
     # Runs a set of commands using the runners shell
     - name: Compile
-      run: source /opt/intel/oneapi/setvars.sh && make CMP=intel BUILD=debug && make examples
+      run: |
+        source /opt/intel/oneapi/setvars.sh
+        export FC=mpiifort
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
+        cmake --build build
 
     # Runs a set of commands using the runners shell
     - name: Compile
-      run: source /opt/intel/oneapi/setvars.sh && make clean && make CMP=intel FFT=mkl BUILD=debug && make examples
+      run: |
+        source /opt/intel/oneapi/setvars.sh
+        export FC=mpiifort
+        export MKL_DIR=${MKLROOT}/lib/cmake/mkl
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=mkl
+        cmake --build build --target clean
+        cmake --build build
 
   bld_fftw3:
     # The type of runner that the job will run on
@@ -127,15 +143,15 @@ jobs:
     # Runs a set of commands using the runners shell
     - name: Regular FFTW3
       run: |
-        make clean
-        make BUILD=dev FFT=fftw3 FFTW3_PATH_INCLUDE=/usr/include FFTW3_PATH_LIB=/usr/lib/x86_64-linux-gnu
-        make examples
+        export FC=mpif90
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DFFT_Choice=fftw
+        cmake --build build --verbose
 
     - name: New FFTW3
       run: |
-        make clean
-        make BUILD=dev FFT=fftw3_f03 FFTW3_PATH_INCLUDE=/usr/include FFTW3_PATH_LIB=/usr/lib/x86_64-linux-gnu
-        make examples
+        export FC=mpif90
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=fftw_f03
+        cmake --build build --verbose
 
   bld_caliper:
     # The type of runner that the job will run on
@@ -164,5 +180,6 @@ jobs:
 
     - name: Compile
       run: |
-        make clean
-        make BUILD=dev PROFILER=caliper CALIPER_PATH=./Caliper-2.8.0_bld
+        export caliper_DIR=./Caliper-2.8.0_bld/share/cmake/caliper
+        FC=mpif90 cmake -S . -B build -DENABLE_PROFILER=caliper
+        cmake --build build --verbose
diff --git a/.gitignore b/.gitignore
index 0d2e71b1..743fe033 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,6 @@ Makefile.settings
 # emacs
 .dir-locals.el
 TAGS
+
+# vim
+*.swp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4be7e08b..1e28d8e7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -70,7 +70,7 @@ if (BUILD_TARGET MATCHES "gpu")
 else ()
   set(FFT_Choice "generic" CACHE STRING "FFT for XCompact3d project (generic is the default)")
 endif (BUILD_TARGET MATCHES "gpu")
-set_property(CACHE FFT_Choice PROPERTY STRINGS generic fftw mkl cufft)
+set_property(CACHE FFT_Choice PROPERTY STRINGS generic fftw fftw_f03 mkl cufft)
 include(fft)
 
 # Create a static library for the fft
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 687b6b2e..00000000
--- a/Makefile
+++ /dev/null
@@ -1,168 +0,0 @@
-#=======================================================================
-# Makefile for 2DECOMP&FFT
-#=======================================================================
-# Choose pre-processing options
-#   -DDOUBLE_PREC - use double-precision
-#   -DSAVE_SINGLE - Save 3D data in single-precision
-#   -DDEBG        - debugging
-#   -DOVERWIR     - Enable overwriting optimisation in MKL FFT
-# generate a Git version string
-GIT_VERSION := $(shell git describe --tag --long --always)
-
-DEFS = -DDOUBLE_PREC -DVERSION=\"$(GIT_VERSION)\"
-
-LCL = local# local,lad,sdu,archer
-CMP = gcc# intel,gcc,nagfor,cray,nvhpc
-FFT ?= generic# fftw3,fftw3_f03,generic,mkl,cufft
-PARAMOD = mpi # multicore,gpu
-PROFILER ?= none# none, caliper
-INPLACE = no 
-
-BUILD ?= # debug can be used with gcc
-FCFLAGS ?= # user can set default compiler flags
-LDFLAGS ?= # user can set default linker flags
-FFLAGS = $(FCFLAGS)
-LFLAGS = $(LDFLAGS)
-MODFLAG = -J 
-
-LIBDECOMP = decomp2d
-
-AR = ar
-LIBOPT = rcs
-
-ifeq ($(INPLACE),yes)
-  DEFS += -DOVERWRITE
-endif
-
-#######CMP settings###########
-CMPINC = Makefile.compilers
-include $(CMPINC)
-
-### List of files for the main code
-SRCDECOMP = decomp_2d_constants.f90 decomp_2d_mpi.f90 factor.f90 decomp_2d.f90 log.f90 io.f90
-
-#######FFT settings##########
-ifeq ($(FFT),fftw3)
-  FFTW3_PATH ?= /usr
-  FFTW3_PATH_INCLUDE ?= $(FFTW3_PATH)/include
-  FFTW3_PATH_LIB ?= $(FFTW3_PATH)/lib
-  INC=-I$(FFTW3_PATH_INCLUDE)
-  LIBFFT=-L$(FFTW3_PATH_LIB) -lfftw3 -lfftw3f
-else ifeq ($(FFT),fftw3_f03)
-  FFTW3_PATH ?= /usr
-  FFTW3_PATH_INCLUDE ?= $(FFTW3_PATH)/include
-  FFTW3_PATH_LIB ?= $(FFTW3_PATH)/lib
-  INC=-I$(FFTW3_PATH_INCLUDE)
-  LIBFFT=-L$(FFTW3_PATH_LIB) -lfftw3 -lfftw3f
-else ifeq ($(FFT),generic)
-  SRCDECOMP += ./glassman.f90
-  INC=
-  LIBFFT=
-else ifeq ($(FFT),mkl)
-  SRCDECOMP += $(MKLROOT)/include/mkl_dfti.f90
-  LIBFFT=-Wl,--start-group $(MKLROOT)/lib/intel64/libmkl_intel_lp64.a $(MKLROOT)/lib/intel64/libmkl_sequential.a $(MKLROOT)/lib/intel64/libmkl_core.a -Wl,--end-group -lpthread
-  INC=-I$(MKLROOT)/include
-else ifeq ($(FFT),cufft)
-  CUFFT_PATH ?= $(NVHPC)/Linux_x86_64/$(EBVERSIONNVHPC)/compilers
-  INC=-I$(CUFFT_PATH)/include
-  #LIBFFT=-L$(CUFFT_PATH)/lib64 -Mcudalib=cufft 
-endif
-
-### IO Options ###
-LIBIO :=
-OPTIO :=
-INCIO :=
-ADIOS2DIR :=
-ifeq ($(IO),adios2)
-  ifeq ($(ADIOS2DIR),)
-    $(error Set ADIOS2DIR=/path/to/adios2/install/)
-  endif
-  OPTIO := -DADIOS2 $(OPT)
-  INCIO := $(INC) $(shell $(ADIOS2DIR)/bin/adios2-config --fortran-flags) #$(patsubst $(shell $(ADIOS2DIR)/bin/adios2-config --fortran-libs),,$(shell $(ADIOS2DIR)/bin/adios2-config -f))
-  LIBIO := $(shell $(ADIOS2DIR)/bin/adios2-config --fortran-libs)
-endif
-
-### Add the profiler if needed
-ifneq ($(PROFILER),none)
-  DEFS += -DPROFILER
-endif
-ifeq ($(PROFILER),caliper)
-  CALIPER_PATH ?= xxxxxxxxx/caliper/caliper_2.8.0
-  SRCDECOMP := $(SRCDECOMP) profiler_caliper.f90
-  INC := $(INC) -I$(CALIPER_PATH)/include/caliper/fortran
-  LFLAGS := $(LFLAGS) -L$(CALIPER_PATH)/lib -lcaliper
-endif
-
-#######OPTIONS settings###########
-OPT =
-LINKOPT = $(FFLAGS)
-#-----------------------------------------------------------------------
-# Normally no need to change anything below
-
-OBJDIR = obj
-SRCDIR = src
-DECOMPINC = mod
-FFLAGS += $(MODFLAG)$(DECOMPINC) -I$(DECOMPINC)
-
-SRCDECOMP := $(SRCDECOMP) fft_$(FFT).f90 fft_log.f90
-SRCDECOMP_ = $(patsubst %.f90,$(SRCDIR)/%.f90,$(filter-out %/mkl_dfti.f90,$(SRCDECOMP)))
-SRCDECOMP_ += $(filter %/mkl_dfti.f90,$(SRCDECOMP))
-OBJDECOMP_MKL_ = $(patsubst $(MKLROOT)/include/%.f90,$(OBJDIR)/%.f90,$(filter %/mkl_dfti.f90,$(SRCDECOMP_)))
-OBJDECOMP_MKL = $(OBJDECOMP_MKL_:%.f90=%.o)
-OBJDECOMP = $(SRCDECOMP_:$(SRCDIR)/%.f90=$(OBJDIR)/%.o)
-
-OPT += $(OPTIO)
-INC += $(INCIO)
-
--include Makefile.settings
-
-all: $(DECOMPINC) $(OBJDIR) $(LIBDECOMP)
-
-$(DECOMPINC):
-	mkdir $(DECOMPINC)
-
-$(LIBDECOMP) : Makefile.settings lib$(LIBDECOMP).a
-
-lib$(LIBDECOMP).a: $(OBJDECOMP_MKL) $(OBJDECOMP)
-	$(AR) $(LIBOPT) $@ $^
-
-$(OBJDIR):
-	mkdir $(OBJDIR)
-
-$(OBJDECOMP) : $(OBJDIR)/%.o : $(SRCDIR)/%.f90
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
-
-$(OBJDECOMP_MKL) : $(OBJDIR)/%.o : $(MKLROOT)/include/%.f90
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $(MKLROOT)/include/mkl_dfti.f90 -o $(OBJDIR)/mkl_dfti.o
-
-examples: $(LIBDECOMP)
-	$(MAKE) -C examples
-
-.PHONY: check
-
-check: examples
-	$(MAKE) -C examples $@
-
-.PHONY: clean
-
-clean: clean-examples
-	rm -f $(OBJDIR)/*.o $(DECOMPINC)/*.mod $(DECOMPINC)/*.smod lib$(LIBDECOMP).a
-	rm -f ./*.o ./*.mod ./*.smod # Ensure old files are removed
-	rm -f Makefile.settings
-
-clean-examples:
-	$(MAKE) -C examples clean
-
-.PHONY: Makefile.settings
-
-Makefile.settings:
-	echo "FC = $(FC)" > $@
-	echo "FFLAGS = $(FFLAGS)" >> $@
-	echo "OPT = $(OPT)" >> $@
-	echo "DEFS = $(DEFS)" >> $@
-	echo "INC = $(INC)" >> $@
-	echo "LIBOPT = $(LIBOPT)" >> $@
-	echo "LIBFFT = ${LIBFFT}" >> $@
-	echo "LFLAGS = $(LFLAGS)" >> $@
-
-export
diff --git a/Makefile.compilers b/Makefile.compilers
deleted file mode 100644
index b41599de..00000000
--- a/Makefile.compilers
+++ /dev/null
@@ -1,124 +0,0 @@
-#=======================================================================
-# Makefile for 2DECOMP&FFT compilers
-#=======================================================================
-
-DEBUG_BUILD =
-ifeq ($(BUILD),debug)
-  DEBUG_BUILD = yes
-endif
-ifeq ($(BUILD),dev)
-  DEBUG_BUILD = yes
-endif
-
-ifeq ($(CMP),intel)
-  FC = mpiifort
-
-  FFLAGS += -fpp -std08
-  ifeq ($(DEBUG_BUILD),yes)
-    DEFS += -DDEBUG
-    FFLAGS += -g -O0 -debug extended -traceback
-  else
-    FFLAGS += -O3 -mavx2 -march=core-avx2 -mtune=core-avx2
-  endif
-
-  ifeq ($(BUILD),dev)
-    # Add additional, stricter flags
-    FFLAGS += -warn all,noexternal
-  endif
-
-  MODFLAG = -module # Explicit space
-else ifeq ($(CMP),gcc)
-  FC = mpif90
-  FFLAGS += -cpp -std=f2008
-  ifeq "$(shell expr `gfortran -dumpversion | cut -f1 -d.` \>= 10)" "1"
-    FFLAGS += -fallow-argument-mismatch
-  endif
-  ifeq ($(DEBUG_BUILD),yes)
-    DEFS += -DDEBUG
-    FFLAGS += -g3 -Og
-    FFLAGS += -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none
-  else
-    FFLAGS += -O3 -march=native
-  endif
-
-  ifeq ($(findstring openmpi,$(shell $(FC) -show | grep openmpi)),openmpi)
-    MPI_FAMILY = openmpi
-  else
-    MPI_FAMILY = mpich # Probably
-  endif
-
-  ifeq ($(BUILD),dev)
-    # Add additional, stricter flags
-    FFLAGS += -Wall -Wpedantic
-    ifneq ($(FFT),fftw3)
-      # FFTW3 (legacy) doesn't have interfaces
-      ifeq ($(MPI_FAMILY),openmpi)
-        # MPICH doesn't export interfaces...
-        FFLAGS += -Wimplicit-procedure -Wimplicit-interface
-      endif
-    endif
-    FFLAGS += -Wno-unused-function
-    FFLAGS += -Werror -Wno-integer-division
-  endif
-
-else ifeq ($(CMP),nagfor)
-  FC = mpinagfor
-  FFLAGS += -fpp
-else ifeq ($(CMP),cray)
-  FC = ftn
-  FFLAGS += -eF
-  ifeq ($(DEBUG_BUILD),yes)
-    DEFS += -DDEBUG
-    FFLAGS += -G2
-    FFLAGS += -en
-  else
-    FFLAGS += -g -O3
-  endif
-  FFLAGS += -N1023 -M 296
-else ifeq ($(CMP),nvhpc)
-  FC = mpif90
-  MODFLAG = -module # Explicit space
-  FFLAGS += -cpp
-  ifeq ($(PARAMOD),multicore)
-     FFLAGS += -O3 -Minfo=accel -stdpar -acc -target=multicore
-     LFLAGS += -acc -lnvhpcwrapnvtx
-  else ifeq ($(PARAMOD),gpu)
-     CCXY=80
-     MANAGED=yes
-     ifeq ($(MANAGED),yes)
-       GPUOPT=-gpu=cc${CCXY},managed,lineinfo
-     else
-       GPUOPT=-gpu=cc${CCXY},lineinfo
-     endif
-     FFLAGS += -D_GPU
-     NCCL=no     
-     ifeq ($(NCCL),yes)
-       FFLAGS += -D_NCCL
-     endif
-     FFLAGS += -Mfree -Kieee -Minfo=accel,stdpar ${GPUOPT} -acc -target=gpu -traceback -O3 -DUSE_CUDA -cuda 
-     ifeq ($(NCCL),yes)
-       FFLAGS += -cudalib=cufft,nccl
-     else
-       FFLAGS += -cudalib=cufft
-     endif
-     #FFLAGS += -D_GPU -Mfree -Kieee -Minfo=accel,stdpar -stdpar=gpu -gpu=cc80,managed,lineinfo -acc -target=gpu -traceback -O3 -DUSE_CUDA -cuda -cudalib=cufft
-     #FFLAGS += -Mfree -Kieee -Minfo=accel,stdpar -stdpar=gpu -gpu=cc80,managed,lineinfo -acc -target=gpu -traceback -O3 -DUSE_CUDA -cuda -cudalib=cufft
-     LFLAGS += -acc -lnvhpcwrapnvtx
-  else
-    ifeq ($(DEBUG_BUILD),yes)
-      DEFS += -DDEBUG
-      FFLAGS += -g -O0
-      FFLAGS += -traceback
-      FFLAGS += -Mbounds -Mchkptr
-      FFLAGS += -Ktrap=fp # Trap floating-point errors
-    else
-      FFLAGS += -O3 -march=native
-    endif
-  endif
-  #FFLAGS += -cpp -O3 -Minfo=accel -stdpar -acc -target=multicore
-  #FFLAGS = -cpp -Mfree -Kieee -Minfo=accel -g -acc -target=gpu -fast -O3 -Minstrument
-endif
-
-# Local Variables:
-# mode: makefile
-# End:
diff --git a/README.md b/README.md
index ecb4b2f8..8ed3acca 100644
--- a/README.md
+++ b/README.md
@@ -2,35 +2,6 @@
 
 ## Building
 
-### Makefile
-
-Different compilers can be set by specifying `CMP`, e.g. `make CMP=intel`
-to build with Intel compilers, see `Makefile` for options.
-
-By default an optimised library will be built, debugging versions of the
-library can be built with `make BUILD=debug`, a development version which 
-additionally sets compile time flags to catch coding errors can be built 
-with `make BUILD=dev` (GNU compilers only currently). The behaviour of debug
-and development versions of the library can be changed before the initialization
-using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
-The value provided with the environment variable must be a positive integer below 9999.
-
-On each build of the library (`make`, `make all`) a temporary file `Makefile.settings` with
-all current options (`FFLAGS`, `DEFS`, etc.) will be created, and included
-on subsequent invocations, the user therefore does not need to keep
-specifying options between builds.
-
-To perform a clean build run `make clean` first, this will delete all
-output files, including `Makefile.settings`.
-
-On each build of the library (`make`, `make all`) a temporary file `Makefile.settings` with
-all current options (`FFLAGS`, `DEFS`, etc.) will be created, and included
-on subsequent invocations, the user therefore does not need to keep
-specifying options between builds.
-
-To perform a clean build run `make clean` first, this will delete all
-output files, including `Makefile.settings`.
-
 ### CMake
 
 The new experimental build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 
@@ -85,23 +56,6 @@ cmake --build $path_to_build_directory --target clean
 
 ## Testing and examples
 
-### Makefile
-
-Various example code to exercise 2decomp functionality can be found under ``examples/``
-and can be built from the top-level directory by executing
-```
-make check
-```
-which will (re)build 2decomp&fft as necessary.
-
-By default this will run the tests/examples on a single rank, launched with ``mpirun``.
-The number of ranks can be controlled by setting the ``NP`` variable and the launcher is overridden by
-setting ``MPIRUN``, for example to run tests on 4 ranks using an ``mpiexec`` launcher installed at
-``${HOME}/bin/mpiexec`` the command would be
-```
-make NP=4 MPIRUN=${HOME}/bin/mpiexec check
-```
-
 ### CMake
 
 After building the library can be tested by running
@@ -120,16 +74,7 @@ For the GPU implementation please be aware that it is based on a single MPI rank
 The library can perform multi GPU offoloading using the NVHPC compiler suite for NVIDIA hardware. 
 The implementation is based on CUDA-aware MPI and NVIDIA Collective Communication Library (NCCL).
 The FFT is based on cuFFT. 
-### Makefile
-To compile the library for GPU it is possible to execute the following
-```
-make CMP=nvhpc FFT=cufft PARAMOD=gpu CUFFT_PATH=PATH_TO_NVHPC/Vers/Linux_x86_64/Vers/compilers/ 
-``` 
-The `Makefile` will look for the relative libraries (NVCC, cuFFT, etc) under the `${CUFFT_PATH}/include`
-NCCL is not activated by default. If NCCL is installed/required use `NCCL=yes`. 
-The current implementation relays also on opeanACC
-and on automatic optimization of `do concurrent` loops.
-By default the compute architecture for the GPU is 80 (i.e. Ampere), to change it use `CCXY=XY` 
+
 ### CMake
 To properly configure for GPU build the following needs to be used 
 ```
@@ -152,14 +97,7 @@ It is possible that your default C compiler is too recent and not supported by `
 
 ## Profiling
 
-Profiling can be activated in the Makefile. Set the variable `PROFILER` to one of the supported profilers (only `caliper` currently). If using `caliper`, provide the installation path in the variable `CALIPER_PATH`. When the profiling is active, one can tune it before calling `decomp_2d_init` using the subroutine `decomp_profiler_prep`. The input argument for this subroutine is a logical array of size 4. Each input allow activation / deactivation of the profiling as follows :
-
-1. Profile transpose operations (default : true)
-2. Profile IO operations (default : true)
-3. Profile FFT operations (default : true)
-4. Profile decomp_2d init / fin subroutines (default : true)
-
-Profiling can be activated via `cmake` builds, the recommended approach is to run the initial configuration as follows:
+Profiling can be activated via `cmake` configuration, the recommended approach is to run the initial configuration as follows:
 ```
 export caliper_DIR=/path/to/caliper/install/share/cmake/caliper
 export CXX=mpicxx
@@ -168,6 +106,16 @@ cmake -S $path_to_sources -B $path_to_build_directory -DENABLE_PROFILER=caliper
 where `ENABLE_PROFILER` is set to the profiling tool desired, currently supported values are: `caliper`.
 Note that when using `caliper` a C++ compiler is required as indicated in the above command line.
 
+When the profiling is active, one can tune it before calling `decomp_2d_init` using the subroutine
+`decomp_profiler_prep`. 
+The input argument for this subroutine is a logical array of size 4. 
+Each input allow activation / deactivation of the profiling as follows :
+
+1. Profile transpose operations (default : true)
+2. Profile IO operations (default : true)
+3. Profile FFT operations (default : true)
+4. Profile decomp_2d init / fin subroutines (default : true)
+
 ## FFT backends
 
 The library provides a built-in FFT engine and supports various FFT backends : FFTW, Intel oneMKL, Nvidia cuFFT. The FFT engine selected during compilation is available through the variable `D2D_FFT_BACKEND` defined in the module `decomp_2d_fft`. The expected value is defined by the integer constants
@@ -188,6 +136,37 @@ The external code can use the named variables to check the FFT backend used in a
 - The oneMKL backend supports the OVERWRITE flag and can perform in-place complex 1D fft
 - The cuFFT backend supports the OVERWRITE flag and can perform in-place complex 1D fft
 
+## Linking from external codes
+
+### Codes using Makefiles
+
+When building a code that links 2decomp-fft using a Makefile you will need to add the include and link paths as appropriate (`inlude/` and `link/` under the installation directory, respectively).
+It is also possible to drive the build and installation of 2decomp-fft from a Makefile such as in the following example code
+```
+FC = mpif90
+BUILD = Release
+
+DECOMP_ROOT = /path/to/2decomp-fft
+DECOMP_BUILD_DIR = $(DECOMP_ROOT)/build
+DECOMP_INSTALL_DIR ?= $(DECOMP_BUILD_DIR)/opt # Use default unless set by user
+
+INC += -I$(DECOMP_INSTALL_DIR)/include
+
+# Users build/link targets
+LFLAGS += -L$(DECOMP_INSTALL_DIR)/lib -ldecomp.a
+
+# Building libdecomp.a
+$(DECOMP_INSTALL_DIR)/lib/libdecomp.a:
+	FC=$(FC) cmake -S $(DECOMP_ROOT) -B $(DECOMP_BUILD_DIR) -DCMAKE_BUILD_TYPE=$(BUILD) -DCMAKE_INSTALL_PREFIX=$(DECOMP_INSTALL_DIR)
+	cmake --build $(DECOMP_BUILD_DIR) --target decomp2d
+	cmake --build $(DECOMP_BUILD_DIR) --target install
+
+# Clean libdecomp.a
+clean-decomp:
+	cmake --build $(DECOMP_BUILD_DIR) --target clean
+	rm -f $(DECOMP_INSTALL_DIR)/lib/libdecomp.a
+```
+
 ## Miscellaneous
 
 ### Print the log to a file or to stdout
@@ -261,6 +240,14 @@ make -j check
 make install
 ```
 
+To build `2decomp&fft` against fftw3 first ensure the package configuration for fftw3 is in your `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`, then either specify on the command line when configuring the build
+```
+cmake -S . -B build -DFFT_Choice=<fftw|fftw_f03>
+```
+or modify the build configuration using `ccmake`.
+
+Note the legacy `fftw` interface lacks interface definitions and will fail when stricter compilation flags are used (e.g. when `-DCMAKE_BUILD_TYPE=Dev`) for this it is recommended to use `fftw_f03` which provides proper interfaces.
+
 ### Caliper
 
 The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.9.0 was tested and is supported, version 2.8.0 has also been tested and is still expected to work. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
@@ -276,3 +263,9 @@ make test
 make install
 ```
 
+After installing Caliper ensure to set `caliper_DIR=/path/to/caliper/install/share/cmake/caliper`.
+Following this the `2decomp-fft` build can be configured to use Caliper profiling as
+```
+cmake -S . -B -DENABLE_PROFILING=caliper
+```
+or by modifying the configuration to set `ENABLE_PROFILING=caliper` via `ccmake`.
diff --git a/cmake/compilers/D2D_flags_nvidia.cmake b/cmake/compilers/D2D_flags_nvidia.cmake
index 68d4254c..73a2d1d4 100644
--- a/cmake/compilers/D2D_flags_nvidia.cmake
+++ b/cmake/compilers/D2D_flags_nvidia.cmake
@@ -3,6 +3,7 @@
 set(D2D_FFLAGS "-cpp -Mfree -Kieee")
 set(D2D_FFLAGS_RELEASE "-O3")
 set(D2D_FFLAGS_DEBUG   "-O0 -g -traceback -Mbounds -Mchkptr -Ktrap=fp")
+set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG}")
 if (BUILD_TARGET MATCHES "mpi")
   set(D2D_FFLAGS_RELEASE "${D2D_FFLAGS_RELEASE} -fast -march=native")
 endif (BUILD_TARGET MATCHES "mpi")
diff --git a/examples/Makefile b/examples/Makefile
deleted file mode 100644
index 546d1377..00000000
--- a/examples/Makefile
+++ /dev/null
@@ -1,36 +0,0 @@
-.PHONY: init_test test2d fft_physical_x fft_physical_z halo_test io_test 
-
-# Just build the examples
-examples: init_test test2d fft_physical_x fft_physical_z halo_test io_test
-	@echo "Built the examples"
-
-init_test:
-	$(MAKE) -C $@ all
-test2d:
-	$(MAKE) -C $@ all
-fft_physical_x:
-	$(MAKE) -C $@ all
-fft_physical_z:
-	$(MAKE) -C $@ all
-halo_test:
-	$(MAKE) -C $@ $@
-io_test:
-	$(MAKE) -C $@ all
-
-check:
-	cd init_test; $(MAKE) $@
-	cd test2d; $(MAKE) $@
-	cd fft_physical_x; $(MAKE) $@
-	cd fft_physical_z; $(MAKE) $@
-	cd halo_test; $(MAKE) $@
-	cd io_test; $(MAKE) $@
-
-clean:
-	cd init_test; $(MAKE) $@
-	cd test2d; $(MAKE) $@
-	cd fft_physical_x; $(MAKE) $@
-	cd fft_physical_z; $(MAKE) $@
-	cd halo_test; $(MAKE) $@
-	cd io_test; $(MAKE) $@
-
-export
diff --git a/examples/fft_physical_x/Makefile b/examples/fft_physical_x/Makefile
deleted file mode 100644
index 0ccb3311..00000000
--- a/examples/fft_physical_x/Makefile
+++ /dev/null
@@ -1,41 +0,0 @@
-#include ../../src/Makefile.inc
-
-#INCLUDE = -I../../include
-FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
-FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
-LIBS = -L../../ -l$(LIBDECOMP) $(LIBFFT) $(LFLAGS)
-
-NP ?= 1
-MPIRUN ?= mpirun
-
-all: fft_c2c_x fft_r2c_x fft_grid_x
-
-fft_c2c_x: fft_c2c_x.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-fft_r2c_x: fft_r2c_x.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-fft_grid_x: fft_grid_x.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-ifeq ($(PARAMOD),gpu)
-check:
-	$(MPIRUN) -n $(NP) ./bind.sh ./fft_c2c_x
-	$(MPIRUN) -n $(NP) ./bind.sh ./fft_r2c_x
-	$(MPIRUN) -n $(NP) ./bind.sh ./fft_grid_x
-else
-check:
-	$(MPIRUN) -n $(NP) ./fft_c2c_x
-	$(MPIRUN) -n $(NP) ./fft_r2c_x
-	$(MPIRUN) -n $(NP) ./fft_grid_x
-endif
-
-mem_leak:
-	valgrind --leak-check=full --show-leak-kinds=all $(MPIRUN) -n 1 ./fft_c2c_x 1 1
-
-clean:
-	rm -f *.o fft_c2c_x fft_r2c_x fft_grid_x *.log
-
-%.o : %.f90
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/fft_physical_z/Makefile b/examples/fft_physical_z/Makefile
deleted file mode 100644
index ce3e2aea..00000000
--- a/examples/fft_physical_z/Makefile
+++ /dev/null
@@ -1,36 +0,0 @@
-#include ../../src/Makefile.inc
-
-#INCLUDE = -I../../include
-FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
-FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
-LIBS = -L../../ -l$(LIBDECOMP) $(LIBFFT) $(LFLAGS)
-
-NP ?= 1
-MPIRUN ?= mpirun
-
-all: fft_c2c_z fft_r2c_z 
-
-fft_c2c_z: fft_c2c_z.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-fft_r2c_z: fft_r2c_z.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-ifeq ($(PARAMOD),gpu)
-check:
-	$(MPIRUN) -n $(NP) ./bind.sh ./fft_c2c_z
-	$(MPIRUN) -n $(NP) ./bind.sh ./fft_r2c_z
-else
-check:
-	$(MPIRUN) -n $(NP) ./fft_c2c_z
-	$(MPIRUN) -n $(NP) ./fft_r2c_z
-endif
-
-mem_leak:
-	valgrind --leak-check=full --show-leak-kinds=all $(MPIRUN) -n 1 ./fft_c2c_z 1 1
-
-clean:
-	rm -f *.o fft_c2c_z fft_r2c_z *.log
-
-%.o : %.f90
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/halo_test/Makefile b/examples/halo_test/Makefile
deleted file mode 100644
index 3c46b58d..00000000
--- a/examples/halo_test/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-#include ../../src/Makefile.inc
-
-#INCLUDE = -I../../include
-FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
-FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
-LIBS = -L../../ -l$(LIBDECOMP) $(LFLAGS)
-
-OBJ = halo_test.o
-
-NP ?= 1
-MPIRUN ?= mpirun
-
-all: halo_test
-
-halo_test: $(OBJ)
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $(OBJ) $(LIBS)
-
-ifeq ($(PARAMOD),gpu)
-check:
-	$(MPIRUN) -n $(NP) ./bind.sh ./halo_test
-else
-check:
-	$(MPIRUN) -n $(NP) ./halo_test
-endif
-
-clean:
-	rm -f *.o halo_test *.log
-
-%.o : %.f90
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/init_test/Makefile b/examples/init_test/Makefile
deleted file mode 100644
index 5adedd9c..00000000
--- a/examples/init_test/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
-LIBS = -L../../ -l$(LIBDECOMP) $(LFLAGS)
-
-OBJ = init_test.o
-
-NP ?= 1
-MPIRUN ?= mpirun
-MODDIR ?= $(DECOMPINC)
-
-all: $(MODDIR) init_test
-
-$(MODDIR):
-	mkdir -p $@
-
-init_test: $(OBJ)
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $(OBJ) $(LIBS)
-
-ifeq ($(PARAMOD),gpu)
-check:
-	$(MPIRUN) -n $(NP) ./bind.sh ./init_test
-else
-check:
-	$(MPIRUN) -n $(NP) ./init_test
-endif
-
-clean:
-	rm -f *.o init_test *.log *.mod *.smod $(MODDIR)/*.mod $(MODDIR)/*.smod
-
-%.o : %.f90
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/io_test/Makefile b/examples/io_test/Makefile
deleted file mode 100644
index 3d4a004b..00000000
--- a/examples/io_test/Makefile
+++ /dev/null
@@ -1,54 +0,0 @@
-#include ../../src/Makefile.inc
-
-#INCLUDE = -I../../include
-FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
-FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
-LIBS = -L../../ -l$(LIBDECOMP) $(LFLAGS)
-
-NP ?= 1
-MPIRUN ?= mpirun
-NROW ?= 0
-NCOL ?= 0
-
-all: $(DECOMPINC) io_test io_read io_var_test io_plane_test io_bench
-
-$(DECOMPINC):
-	mkdir $@
-
-io_test: io_test.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-io_read: io_read.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-io_var_test: io_var_test.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-io_plane_test: io_plane_test.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-io_bench: io_bench.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-ifeq ($(PARAMOD),gpu)
-check:
-	$(MPIRUN) -n $(NP) ./bind.sh ./io_test
-	$(MPIRUN) -n $(NP) ./bind.sh ./io_read
-	$(MPIRUN) -n $(NP) ./bind.sh ./io_var_test $(NROW) $(NCOL)
-	$(MPIRUN) -n $(NP) ./bind.sh ./io_plane_test
-	$(MPIRUN) -n $(NP) ./bind.sh ./io_bench
-else
-check:
-	$(MPIRUN) -n $(NP) ./io_test
-	$(MPIRUN) -n $(NP) ./io_read
-	$(MPIRUN) -n $(NP) ./io_var_test $(NROW) $(NCOL)
-	$(MPIRUN) -n $(NP) ./io_plane_test
-	$(MPIRUN) -n $(NP) ./io_bench
-endif
-
-clean:
-	rm -f *.o io_test io_read io_var_test io_plane_test io_bench
-	rm -f *.dat io_var_data.* *.log
-
-%.o : %.f90
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/examples/test2d/Makefile b/examples/test2d/Makefile
deleted file mode 100644
index eeaae0db..00000000
--- a/examples/test2d/Makefile
+++ /dev/null
@@ -1,40 +0,0 @@
-#include ../../src/Makefile.inc
-
-
-#INCLUDE = -I../../include
-FFLAGS := $(subst $(MODFLAG),$(MODFLAG)../../,$(FFLAGS))
-FFLAGS := $(patsubst -I%,-I../../%,$(FFLAGS))
-LIBS = -L../../ -l$(LIBDECOMP) $(LFLAGS)
-
-NP ?= 1
-MPIRUN ?= mpirun
-
-all: test2d timing2d_real timing2d_complex
-
-test2d: test2d.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-timing2d_real: timing2d_real.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-timing2d_complex: timing2d_complex.o
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -o $@ $< $(OBJ) $(LIBS)
-
-ifeq ($(PARAMOD),gpu)
-check:
-	$(MPIRUN) -n $(NP) ./bind.sh ./test2d
-	$(MPIRUN) -n $(NP) ./bind.sh ./timing2d_real
-	$(MPIRUN) -n $(NP) ./bind.sh ./timing2d_complex
-else
-check:
-	$(MPIRUN) -n $(NP) ./test2d
-	$(MPIRUN) -n $(NP) ./timing2d_real
-	$(MPIRUN) -n $(NP) ./timing2d_complex
-endif
-
-
-clean:
-	rm -f *.o test2d timing2d_real timing2d_complex u*.dat *.log
-
-%.o : %.f90
-	$(FC) $(FFLAGS) $(OPT) $(DEFS) $(INC) -c $< -o $@
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index d26724c3..e4ba02e9 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -11,8 +11,13 @@ if(${BUILD_TARGET} MATCHES "gpu")
 endif(${BUILD_TARGET} MATCHES "gpu")
 
 if(FFTW_FOUND)
-  message (STATUS "Compiling using FFTW3")
-  file(GLOB files_fft fft_fftw3.f90)
+  if (${FFT_Choice} MATCHES "fftw_f03")
+    message (STATUS "Compiling using FFTW3 F2003 interface")
+    file(GLOB files_fft fft_fftw3_f03.f90)
+  else()
+    message (STATUS "Compiling using FFTW3 legacy interface")
+    file(GLOB files_fft fft_fftw3.f90)
+  endif()
 elseif(MKL_FOUND)
   message (STATUS "Compiling using MKL")
   file(GLOB files_fft $ENV{MKLROOT}/include/mkl_dfti.f90 fft_mkl.f90)

From a1273b3e6b23b42f6f835371be6eed33bf227e51 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Thu, 16 Mar 2023 22:50:00 +0000
Subject: [PATCH 175/436] CMake is now the build system

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8ed3acca..e16b0532 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
 
 ### CMake
 
-The new experimental build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 
+The build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 
 ```
 export FC=my_mpif90
 ```

From 5a15dc4c97b593ef7f4b10768f56c860d80c3d72 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 17 Mar 2023 05:47:17 +0000
Subject: [PATCH 176/436] Lower strictness of Dev build for GNU10 + MPICH
 (#163)

MPICH doesn't export proper interfaces so need to relax the flags
a little
---
 cmake/compilers/D2D_flags_gnu.cmake | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmake/compilers/D2D_flags_gnu.cmake b/cmake/compilers/D2D_flags_gnu.cmake
index e4e1ab8b..10f4a88c 100644
--- a/cmake/compilers/D2D_flags_gnu.cmake
+++ b/cmake/compilers/D2D_flags_gnu.cmake
@@ -3,10 +3,17 @@ set(D2D_FFLAGS "-cpp -std=f2008 -ffree-line-length-none")
 if (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
   message(STATUS "Set New Fortran basic flags")
   set(D2D_FFLAGS "${D2D_FFLAGS} -fallow-argument-mismatch")
+  set(D2D_GNU10 TRUE)
+else (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
+  set(D2D_GNU10 FALSE)
 endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
 set(D2D_FFLAGS_RELEASE "-O3 -march=native")
 set(D2D_FFLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none")
-set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
+if (FIND_MPICH AND D2D_GNU10)
+  set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wno-unused-function -Wno-integer-division")
+else()
+  set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
+endif()
 if (FIND_OMPI)
   set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEV} -Wimplicit-procedure -Wimplicit-interface")
 endif()

From d5f1571130057dfa31a0116316f23e2e0332790d Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Sat, 18 Mar 2023 16:37:36 +0000
Subject: [PATCH 177/436] Explicitly specify mpi compiler in build workflow

This is needed to find which compiler is being used
---
 .github/workflows/Build_all.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 11d753a4..cb17078e 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -29,7 +29,7 @@ jobs:
     # Runs a set of commands using the runners shell
     - name: Compile
       run: |
-        export FC=mpif90
+        export FC=mpif90.openmpi
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
         cmake --build build --verbose
 
@@ -51,7 +51,7 @@ jobs:
     # Runs a set of commands using the runners shell
     - name: Compile
       run: |
-        export FC=mpif90
+        export FC=mpif90.mpich
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
         cmake --build build --verbose
 

From ab681342cdbfe92d0cd7080917643701814b52ca Mon Sep 17 00:00:00 2001
From: rfj82982 <35581400+rfj82982@users.noreply.github.com>
Date: Sun, 19 Mar 2023 19:44:31 +0000
Subject: [PATCH 178/436] Module nccl (#161)

* Add NCCL module to transpose real

* Create an interface for NCCL transpose real

* Add complex to NCCL Send-Recv

* Bug correction to properly enforce my type on complex number in NCCL complex transpose

* Force all timining to be in double precision

* Include MPI module into NCCL

* Update type for MPI SUM

* Clean up of the code

* fprettify the file decomp_2d_nccl.f90

* Update comments

* Minor cleaning

---------

Co-authored-by: cflag <cedric.flageul@univ-poitiers.fr>
---
 examples/fft_physical_x/fft_c2c_x.f90  |  14 +-
 examples/fft_physical_x/fft_grid_x.f90 |  14 +-
 examples/fft_physical_x/fft_r2c_x.f90  |  14 +-
 examples/fft_physical_z/fft_c2c_z.f90  |  14 +-
 examples/fft_physical_z/fft_r2c_z.f90  |  14 +-
 examples/test2d/timing2d_complex.f90   |  26 +--
 examples/test2d/timing2d_real.f90      |  26 +--
 src/CMakeLists.txt                     |   3 +
 src/decomp_2d.f90                      |  17 +-
 src/decomp_2d_constants.f90            |   6 -
 src/decomp_2d_init_fin.f90             |  49 +---
 src/decomp_2d_nccl.f90                 | 309 +++++++++++++++++++++++++
 src/transpose_x_to_y.f90               |  37 ++-
 src/transpose_y_to_x.f90               |  37 ++-
 src/transpose_y_to_z.f90               |  36 +--
 src/transpose_z_to_y.f90               |  36 +--
 16 files changed, 449 insertions(+), 203 deletions(-)
 create mode 100644 src/decomp_2d_nccl.f90

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 453d3aab..453f7b1b 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -30,7 +30,7 @@ program fft_c2c_x
    integer :: ierror, i, j, k, m
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
-   real(mytype) :: t1, t2, t3, t4
+   double precision :: t1, t2, t3, t4
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -105,8 +105,8 @@ program fft_c2c_x
       end do
    end do
 
-   t2 = 0._mytype
-   t4 = 0._mytype
+   t2 = 0.d0
+   t4 = 0.d0
    !$acc data copyin(in) copy(out)
    do m = 1, ntest
 
@@ -130,12 +130,12 @@ program fft_c2c_x
    ierror = cudaDeviceSynchronize()
 #endif
 
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+   t1 = t1/dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
+   t3 = t3/dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index af508820..2dee619a 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -30,7 +30,7 @@ program fft_physical_x
    integer :: ierror, i, j, k, m
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
-   real(mytype) :: t1, t2, t3, t4
+   double precision :: t1, t2, t3, t4
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -106,8 +106,8 @@ program fft_physical_x
       end do
    end do
 
-   t2 = 0._mytype
-   t4 = 0._mytype
+   t2 = 0.d0
+   t4 = 0.d0
    !$acc data copyin(in) copy(out)
    do m = 1, ntest
 
@@ -131,12 +131,12 @@ program fft_physical_x
    ierror = cudaDeviceSynchronize()
 #endif
 
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+   t1 = t1/dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
+   t3 = t3/dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 897a2e17..6427d6ed 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -31,7 +31,7 @@ program fft_r2c_x
    integer :: ierror, i, j, k, m
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
-   real(mytype) :: t1, t2, t3, t4
+   double precision :: t1, t2, t3, t4
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -105,8 +105,8 @@ program fft_r2c_x
       end do
    end do
 
-   t2 = 0._mytype
-   t4 = 0._mytype
+   t2 = 0.d0
+   t4 = 0.d0
    !$acc data copyin(in_r) copy(out)
    do m = 1, ntest
 
@@ -130,12 +130,12 @@ program fft_r2c_x
    ierror = cudaDeviceSynchronize()
 #endif
 
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+   t1 = t1/dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
+   t3 = t3/dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 5434f8cd..40e9761f 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -30,7 +30,7 @@ program fft_c2c_z
    integer :: ierror, i, j, k, m
    integer :: zst1, zst2, zst3
    integer :: zen1, zen2, zen3
-   real(mytype) :: t1, t2, t3, t4
+   double precision :: t1, t2, t3, t4
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -106,8 +106,8 @@ program fft_c2c_z
       end do
    end do
 
-   t2 = 0._mytype
-   t4 = 0._mytype
+   t2 = 0.d0
+   t4 = 0.d0
    !$acc data copyin(in) copy(out)
    do m = 1, ntest
 
@@ -131,12 +131,12 @@ program fft_c2c_z
    ierror = cudaDeviceSynchronize()
 #endif
 
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+   t1 = t1/dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
+   t3 = t3/dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 5671e48a..297a0a5b 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -31,7 +31,7 @@ program fft_r2c_z
    integer :: ierror, i, j, k, m
    integer :: zst1, zst2, zst3
    integer :: zen1, zen2, zen3
-   real(mytype) :: t1, t2, t3, t4
+   double precision :: t1, t2, t3, t4
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -106,8 +106,8 @@ program fft_r2c_z
       end do
    end do
 
-   t2 = 0._mytype
-   t4 = 0._mytype
+   t2 = 0.d0
+   t4 = 0.d0
    !$acc data copyin(in_r) copy(out)
    do m = 1, ntest
 
@@ -130,12 +130,12 @@ program fft_r2c_z
    ierror = cudaDeviceSynchronize()
 #endif
 
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+   t1 = t1/dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
+   t3 = t3/dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 54063ed0..05f61aa0 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -32,7 +32,7 @@ program timing2d_complex
    real(mytype) :: m
    complex(mytype) :: cm
 
-   real(mytype) :: t1, t2, t3, t4, t5, t6, t7, t8
+   double precision :: t1, t2, t3, t4, t5, t6, t7, t8
    integer :: iter, niter =10
 
    ! Init
@@ -144,10 +144,10 @@ program timing2d_complex
 
    ! call decomp_2d_write_one(1,u1,'u1.dat')
 
-   t2=0._mytype
-   t4=0._mytype
-   t6=0._mytype
-   t8=0._mytype
+   t2=0.d0
+   t4=0.d0
+   t6=0.d0
+   t8=0.d0
    do iter=1,niter
       !!!!!!!!!!!!!!!!!!!!!!!
       ! x-pensil ==> y-pensil
@@ -261,18 +261,18 @@ program timing2d_complex
       if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
    enddo
 
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+   t1 = t1/dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
-   call MPI_ALLREDUCE(t6, t5, 1, real_type, MPI_SUM, &
+   t3 = t3/dble(nproc)
+   call MPI_ALLREDUCE(t6, t5, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t5/real(nproc, mytype)
-   call MPI_ALLREDUCE(t8, t7, 1, real_type, MPI_SUM, &
+   t5 = t5/dble(nproc)
+   call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t7/real(nproc, mytype)
+   t7 = t7/dble(nproc)
    t8 = t1+t3+t5+t7
    if (nrank == 0) then
      write(*,*) 'Time X->Y ', t1
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 09e04996..5eab1de7 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -31,7 +31,7 @@ program timing2d_real
    logical :: error_flag
    real(mytype) :: m
 
-   real(mytype) :: t1, t2, t3, t4, t5, t6, t7, t8
+   double precision :: t1, t2, t3, t4, t5, t6, t7, t8
    integer :: iter, niter =10
 
    ! Init
@@ -143,10 +143,10 @@ program timing2d_real
 
    ! call decomp_2d_write_one(1,u1,'u1.dat')
 
-   t2=0._mytype
-   t4=0._mytype
-   t6=0._mytype
-   t8=0._mytype
+   t2=0.d0
+   t4=0.d0
+   t6=0.d0
+   t8=0.d0
    do iter=1,niter
       !!!!!!!!!!!!!!!!!!!!!!!
       ! x-pensil ==> y-pensil
@@ -256,18 +256,18 @@ program timing2d_real
       if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
    enddo
 
-   call MPI_ALLREDUCE(t2, t1, 1, real_type, MPI_SUM, &
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/real(nproc, mytype)
-   call MPI_ALLREDUCE(t4, t3, 1, real_type, MPI_SUM, &
+   t1 = t1/dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/real(nproc, mytype)
-   call MPI_ALLREDUCE(t6, t5, 1, real_type, MPI_SUM, &
+   t3 = t3/dble(nproc)
+   call MPI_ALLREDUCE(t6, t5, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t5/real(nproc, mytype)
-   call MPI_ALLREDUCE(t8, t7, 1, real_type, MPI_SUM, &
+   t5 = t5/dble(nproc)
+   call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t7/real(nproc, mytype)
+   t7 = t7/dble(nproc)
    t8 = t1+t3+t5+t7
    if (nrank == 0) then
      write(*,*) 'Time X->Y ', t1
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index e4ba02e9..62623547 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -8,6 +8,9 @@ include_directories(${CMAKE_SOURCE_DIR}/src)
 
 if(${BUILD_TARGET} MATCHES "gpu")
   list(APPEND files_decomp decomp_2d_cumpi.f90)
+  if(ENABLE_NCCL)
+    list(APPEND files_decomp decomp_2d_nccl.f90)
+  endif()
 endif(${BUILD_TARGET} MATCHES "gpu")
 
 if(FFTW_FOUND)
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 06451e05..7baaba65 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -24,6 +24,7 @@ module decomp_2d
    use decomp_2d_cumpi
 #if defined(_NCCL)
    use nccl
+   use decomp_2d_nccl 
 #endif
 #endif
 
@@ -80,12 +81,6 @@ module decomp_2d
    integer(kind(D2D_DEBUG_LEVEL_OFF)), public, save :: decomp_debug = D2D_DEBUG_LEVEL_OFF
 #endif
 
-#if defined(_GPU)
-#if defined(_NCCL)
-   integer, save :: row_rank, col_rank
-#endif
-#endif
-
    ! derived type to store decomposition info for a given global data size
    TYPE, public :: DECOMP_INFO
       ! staring/ending index and size of data held by current processor
@@ -132,16 +127,6 @@ module decomp_2d
    real(mytype), allocatable, dimension(:) :: work1_r, work2_r
    complex(mytype), allocatable, dimension(:) :: work1_c, work2_c
 
-#if defined(_GPU)
-
-#if defined(_NCCL)
-   integer col_comm_size, row_comm_size
-   integer, allocatable, dimension(:) :: local_to_global_col, local_to_global_row
-   type(ncclUniqueId) :: nccl_uid_2decomp
-   type(ncclComm) :: nccl_comm_2decomp
-   integer(kind=cuda_stream_kind) :: cuda_stream_2decomp
-#endif
-#endif
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! To define smaller arrays using every several mesh points
    integer, save, dimension(3), public :: xszS, yszS, zszS, xstS, ystS, zstS, xenS, yenS, zenS
diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 19356f6c..97af2ef1 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -36,9 +36,6 @@ module decomp_2d_constants
    integer, parameter, public :: mytype_single = KIND(0._real64)
    integer, parameter, public :: real_type_single = MPI_DOUBLE_PRECISION
 #endif
-#if defined(_GPU) && defined(_NCCL)
-   type(ncclDataType), parameter, public :: ncclType = ncclDouble
-#endif
 #else
    integer, parameter, public :: mytype = KIND(0._real32)
    integer, parameter, public :: real_type = MPI_REAL
@@ -46,9 +43,6 @@ module decomp_2d_constants
    integer, parameter, public :: complex_type = MPI_COMPLEX
    integer, parameter, public :: mytype_single = KIND(0._real32)
    integer, parameter, public :: real_type_single = MPI_REAL
-#if defined(_GPU) && defined(_NCCL)
-   type(ncclDataType), parameter, public :: ncclType = ncclFloat
-#endif
 #endif
 
    !
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index db820db0..94e74e15 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -33,10 +33,6 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
 
      integer :: errorcode, ierror, row, col, iounit
      logical, dimension(2) :: periodic
-#if defined(_GPU) && defined(_NCCL)
-     integer :: cuda_stat
-     type(ncclResult) :: nccl_stat
-#endif
 
 #ifdef PROFILER
      ! Prepare the profiler if it was not already prepared
@@ -153,43 +149,8 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
      if (nrank == 0) write (*, *) 'Padded ALLTOALL optimisation on'
 #endif
 
-#if defined(_GPU)
-#if defined(_NCCL)
-     call MPI_COMM_RANK(DECOMP_2D_COMM_COL, col_rank, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-     call MPI_COMM_RANK(DECOMP_2D_COMM_ROW, row_rank, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
-     call MPI_COMM_SIZE(DECOMP_2D_COMM_COL, col_comm_size, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-     call MPI_COMM_SIZE(DECOMP_2D_COMM_ROW, row_comm_size, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
-
-     allocate (local_to_global_col(col_comm_size), local_to_global_row(row_comm_size))
-
-     local_to_global_col(:) = 0
-     local_to_global_row(:) = 0
-     local_to_global_col(col_rank + 1) = nrank
-     local_to_global_row(row_rank + 1) = nrank
-
-     call mpi_allreduce(MPI_IN_PLACE, local_to_global_col, col_comm_size, MPI_INTEGER, MPI_SUM, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-     call mpi_allreduce(MPI_IN_PLACE, local_to_global_row, row_comm_size, MPI_INTEGER, MPI_SUM, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
-
-     if (nrank == 0) then
-        nccl_stat = ncclGetUniqueId(nccl_uid_2decomp)
-     else
-        nccl_stat = ncclSuccess
-     end if
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGetUniqueId")
-     call MPI_Bcast(nccl_uid_2decomp, int(sizeof(ncclUniqueId)), MPI_BYTE, 0, decomp_2d_comm, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
-
-     nccl_stat = ncclCommInitRank(nccl_comm_2decomp, nproc, nccl_uid_2decomp, nrank)
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclCommInitRank")
-     cuda_stat = cudaStreamCreate(cuda_stream_2decomp)
-     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamCreate")
-#endif
+#if defined(_GPU) && defined(_NCCL)
+     call decomp_2d_nccl_init(DECOMP_2D_COMM_COL,DECOMP_2D_COMM_ROW)
 #endif
 
      !
@@ -215,9 +176,6 @@ end subroutine decomp_2d_init_ref
   subroutine decomp_2d_finalize_ref
 
      implicit none
-#if defined(_GPU) && defined(_NCCL)
-     type(ncclResult) :: nccl_stat
-#endif
 
 #ifdef PROFILER
      if (decomp_profiler_d2d) call decomp_profiler_start("decomp_2d_fin")
@@ -236,8 +194,7 @@ subroutine decomp_2d_finalize_ref
 #if defined(_GPU)
      call decomp_2d_cumpi_fin()
 #if defined(_NCCL)
-     nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclCommDestroy")
+     call decomp_2d_nccl_fin()
 #endif
 #endif
 
diff --git a/src/decomp_2d_nccl.f90 b/src/decomp_2d_nccl.f90
new file mode 100644
index 00000000..65065d23
--- /dev/null
+++ b/src/decomp_2d_nccl.f90
@@ -0,0 +1,309 @@
+!=======================================================================
+! This is part of the 2DECOMP&FFT library
+!
+! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
+! decomposition. It also implements a highly scalable distributed
+! three-dimensional Fast Fourier Transform (FFT).
+!
+! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
+! Copyright (C) 2022               Science and Technology Facilities Council(STFC)
+!
+!=======================================================================
+
+! Module for the cuda aware MPI
+
+module decomp_2d_nccl
+
+   use mpi
+   use decomp_2d_constants
+   use decomp_2d_mpi
+   use decomp_2d_cumpi
+   use nccl
+
+   implicit none
+
+   private        ! Make everything private unless declared public
+
+#ifdef DOUBLE_PREC
+   type(ncclDataType), parameter, public :: ncclType = ncclDouble
+#else
+   type(ncclDataType), parameter, public :: ncclType = ncclFloat
+#endif
+
+   integer, save, public :: row_rank, col_rank
+
+   integer, save, public :: col_comm_size, row_comm_size
+   integer, allocatable, dimension(:), save, public :: local_to_global_col, local_to_global_row
+   type(ncclUniqueId), save, public :: nccl_uid_2decomp
+   type(ncclComm), save, public :: nccl_comm_2decomp
+   integer(kind=cuda_stream_kind), save, public :: cuda_stream_2decomp
+
+   public :: decomp_2d_nccl_init, &
+             decomp_2d_nccl_fin, &
+             decomp_2d_nccl_send_recv_col, &
+             decomp_2d_nccl_send_recv_row
+
+   interface decomp_2d_nccl_send_recv_col
+      module procedure decomp_2d_nccl_send_recv_real_col
+      module procedure decomp_2d_nccl_send_recv_cmplx_col
+   end interface decomp_2d_nccl_send_recv_col
+
+   interface decomp_2d_nccl_send_recv_row
+      module procedure decomp_2d_nccl_send_recv_real_row
+      module procedure decomp_2d_nccl_send_recv_cmplx_row
+   end interface decomp_2d_nccl_send_recv_row
+
+contains
+   !
+   ! init of the arrays
+   !
+   subroutine decomp_2d_nccl_init(COMM_COL, COMM_ROW)
+
+      implicit none
+
+      integer, intent(in) :: COMM_COL, COMM_ROW
+      integer             :: ierror
+      integer :: cuda_stat
+      type(ncclResult) :: nccl_stat
+
+      call MPI_COMM_RANK(COMM_COL, col_rank, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+      call MPI_COMM_RANK(COMM_ROW, row_rank, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_RANK")
+      call MPI_COMM_SIZE(COMM_COL, col_comm_size, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+      call MPI_COMM_SIZE(COMM_ROW, row_comm_size, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_COMM_SIZE")
+
+      allocate (local_to_global_col(col_comm_size), local_to_global_row(row_comm_size))
+
+      local_to_global_col(:) = 0
+      local_to_global_row(:) = 0
+      local_to_global_col(col_rank + 1) = nrank
+      local_to_global_row(row_rank + 1) = nrank
+
+      call mpi_allreduce(MPI_IN_PLACE, local_to_global_col, col_comm_size, MPI_INTEGER, MPI_SUM, COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+      call mpi_allreduce(MPI_IN_PLACE, local_to_global_row, row_comm_size, MPI_INTEGER, MPI_SUM, COMM_ROW, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLREDUCE")
+
+      if (nrank == 0) then
+         nccl_stat = ncclGetUniqueId(nccl_uid_2decomp)
+         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGetUniqueId")
+      end if
+      call MPI_Bcast(nccl_uid_2decomp, int(sizeof(ncclUniqueId)), MPI_BYTE, 0, decomp_2d_comm, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_BCAST")
+
+      nccl_stat = ncclCommInitRank(nccl_comm_2decomp, nproc, nccl_uid_2decomp, nrank)
+      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclCommInitRank")
+      cuda_stat = cudaStreamCreate(cuda_stream_2decomp)
+      if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "cudaStreamCreate")
+
+   end subroutine decomp_2d_nccl_init
+
+   !
+   ! Finalize the module (release nccl communicator)
+   !
+   subroutine decomp_2d_nccl_fin()
+
+      implicit none
+
+      type(ncclResult) :: nccl_stat
+
+      nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
+      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclCommDestroy")
+
+   end subroutine decomp_2d_nccl_fin
+   !
+   ! Send-Recv Real Col
+   !
+   subroutine decomp_2d_nccl_send_recv_real_col(dst_d, &
+                                                src_d, &
+                                                disp_s, &
+                                                cnts_s, &
+                                                disp_r, &
+                                                cnts_r, &
+                                                dime)
+
+      implicit none
+
+      integer, intent(in) :: dime
+      real(mytype), dimension(:), intent(in), device :: src_d
+      real(mytype), dimension(:), intent(out), device :: dst_d
+      integer, dimension(0:dime - 1), intent(in) :: disp_s, cnts_s
+      integer, dimension(0:dime - 1), intent(in) :: disp_r, cnts_r
+
+      integer :: col_rank_id, cuda_stat
+      type(ncclResult) :: nccl_stat
+
+      nccl_stat = ncclGroupStart()
+      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+      do col_rank_id = 0, (col_comm_size - 1)
+         nccl_stat = ncclSend(src_d(disp_s(col_rank_id) + 1), cnts_s(col_rank_id), &
+                              ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
+         nccl_stat = ncclRecv(dst_d(disp_r(col_rank_id) + 1), cnts_r(col_rank_id), &
+                              ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
+      end do
+      nccl_stat = ncclGroupEnd()
+      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+
+   end subroutine decomp_2d_nccl_send_recv_real_col
+   !
+   ! Send-Recv complex
+   !
+   subroutine decomp_2d_nccl_send_recv_cmplx_col(dst_d, &
+                                                 src_d, &
+                                                 disp_s, &
+                                                 cnts_s, &
+                                                 disp_r, &
+                                                 cnts_r, &
+                                                 dime, &
+                                                 buf_size)
+
+      implicit none
+
+      integer, intent(in) :: dime, buf_size
+      complex(mytype), dimension(buf_size), intent(in), device :: src_d
+      complex(mytype), dimension(buf_size), intent(out), device :: dst_d
+      integer, dimension(0:dime - 1), intent(in) :: disp_s, cnts_s
+      integer, dimension(0:dime - 1), intent(in) :: disp_r, cnts_r
+
+      integer :: ii
+
+      ! Send-Recv Real part
+      !$acc kernels default(present)
+      do ii = 1, buf_size
+         work1_r_d(ii) = real(src_d(ii), mytype)
+      end do
+      !$acc end kernels
+      call decomp_2d_nccl_send_recv_col(work2_r_d, &
+                                        work1_r_d, &
+                                        disp_s, &
+                                        cnts_s, &
+                                        disp_r, &
+                                        cnts_r, &
+                                        dime)
+      !$acc kernels default(present)
+      do ii = 1, buf_size
+         dst_d(ii) = cmplx(work2_r_d(ii), 0._mytype, mytype)
+      end do
+      !$acc end kernels
+      ! Send-Recv Immaginary Part
+      !$acc kernels default(present)
+      do ii = 1, buf_size
+         work1_r_d(ii) = aimag(src_d(ii))
+      end do
+      !$acc end kernels
+      call decomp_2d_nccl_send_recv_col(work2_r_d, &
+                                        work1_r_d, &
+                                        disp_s, &
+                                        cnts_s, &
+                                        disp_r, &
+                                        cnts_r, &
+                                        dime)
+      !$acc kernels default(present)
+      do ii = 1, buf_size
+         dst_d(ii) = cmplx(dst_d(ii), work2_r_d(ii), mytype)
+      end do
+      !$acc end kernels
+   end subroutine decomp_2d_nccl_send_recv_cmplx_col
+   !
+   ! Send-Recv Real Row
+   !
+   subroutine decomp_2d_nccl_send_recv_real_row(dst_d, &
+                                                src_d, &
+                                                disp_s, &
+                                                cnts_s, &
+                                                disp_r, &
+                                                cnts_r, &
+                                                dime)
+
+      implicit none
+
+      integer, intent(in) :: dime
+      real(mytype), dimension(:), intent(in), device :: src_d
+      real(mytype), dimension(:), intent(out), device :: dst_d
+      integer, dimension(0:dime - 1), intent(in) :: disp_s, cnts_s
+      integer, dimension(0:dime - 1), intent(in) :: disp_r, cnts_r
+
+      integer :: row_rank_id, cuda_stat
+      type(ncclResult) :: nccl_stat
+
+      nccl_stat = ncclGroupStart()
+      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
+      do row_rank_id = 0, (row_comm_size - 1)
+         nccl_stat = ncclSend(src_d(disp_s(row_rank_id) + 1), cnts_s(row_rank_id), &
+                              ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
+         nccl_stat = ncclRecv(dst_d(disp_r(row_rank_id) + 1), cnts_r(row_rank_id), &
+                              ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
+         if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
+      end do
+      nccl_stat = ncclGroupEnd()
+      if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+
+   end subroutine decomp_2d_nccl_send_recv_real_row
+   !
+   ! Send-Recv complex
+   !
+   subroutine decomp_2d_nccl_send_recv_cmplx_row(dst_d, &
+                                                 src_d, &
+                                                 disp_s, &
+                                                 cnts_s, &
+                                                 disp_r, &
+                                                 cnts_r, &
+                                                 dime, &
+                                                 buf_size)
+
+      implicit none
+
+      integer, intent(in) :: dime, buf_size
+      complex(mytype), dimension(buf_size), intent(in), device :: src_d
+      complex(mytype), dimension(buf_size), intent(out), device :: dst_d
+      integer, dimension(0:dime - 1), intent(in) :: disp_s, cnts_s
+      integer, dimension(0:dime - 1), intent(in) :: disp_r, cnts_r
+
+      integer :: ii
+
+      ! Send-Recv Real part
+      !$acc kernels default(present)
+      do ii = 1, buf_size
+         work1_r_d(ii) = real(src_d(ii), mytype)
+      end do
+      !$acc end kernels
+      call decomp_2d_nccl_send_recv_row(work2_r_d, &
+                                        work1_r_d, &
+                                        disp_s, &
+                                        cnts_s, &
+                                        disp_r, &
+                                        cnts_r, &
+                                        dime)
+      !$acc kernels default(present)
+      do ii = 1, buf_size
+         dst_d(ii) = cmplx(work2_r_d(ii), 0._mytype, mytype)
+      end do
+      !$acc end kernels
+      ! Send-Recv Immaginary Part
+      !$acc kernels default(present)
+      do ii = 1, buf_size
+         work1_r_d(ii) = aimag(src_d(ii))
+      end do
+      !$acc end kernels
+      call decomp_2d_nccl_send_recv_row(work2_r_d, &
+                                        work1_r_d, &
+                                        disp_s, &
+                                        cnts_s, &
+                                        disp_r, &
+                                        cnts_r, &
+                                        dime)
+      !$acc kernels default(present)
+      do ii = 1, buf_size
+         dst_d(ii) = cmplx(dst_d(ii), work2_r_d(ii), mytype)
+      end do
+      !$acc end kernels
+   end subroutine decomp_2d_nccl_send_recv_cmplx_row
+
+end module decomp_2d_nccl
+
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index 547cb9be..ffa17767 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -30,11 +30,6 @@ subroutine transpose_x_to_y_real(src, dst, decomp)
      real(mytype), dimension(:, :, :), intent(OUT) :: dst
      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-#if defined(_GPU) && defined(_NCCL)
-     integer :: col_rank_id, cuda_stat
-     type(ncclResult) :: nccl_stat
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -69,20 +64,13 @@ subroutine transpose_x_to_y_real(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     nccl_stat = ncclGroupStart()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-     do col_rank_id = 0, (col_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
-                             ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
-                             ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-     end do
-     nccl_stat = ncclGroupEnd()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     call decomp_2d_nccl_send_recv_col(work2_r_d,     &
+                                       work1_r_d,     &
+                                       decomp%x1disp, &
+                                       decomp%x1cnts, &
+                                       decomp%y1disp, & 
+                                       decomp%y1cnts, &  
+                                       dims(1)        )
 #else
      call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
                         real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
@@ -166,10 +154,21 @@ subroutine transpose_x_to_y_complex(src, dst, decomp)
 #else
 
 #if defined(_GPU)
+#if defined(_NCCL)
+     call decomp_2d_nccl_send_recv_col(work2_c_d,     &
+                                       work1_c_d,     &
+                                       decomp%x1disp, &
+                                       decomp%x1cnts, &
+                                       decomp%y1disp, & 
+                                       decomp%y1cnts, &  
+                                       dims(1)      , &
+                                       decomp_buf_size)
+#else
      call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
                         complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
                         complex_type, DECOMP_2D_COMM_COL, ierror)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+#endif
 #else
      call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
                         complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index e10e3c33..890629da 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -30,11 +30,6 @@ subroutine transpose_y_to_x_real(src, dst, decomp)
      real(mytype), dimension(:, :, :), intent(OUT) :: dst
      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-#if defined(_GPU) && defined(_NCCL)
-     type(ncclResult) :: nccl_stat
-     integer :: col_rank_id, cuda_stat
-#endif
-
      integer :: s1, s2, s3, d1, d2, d3
      integer :: ierror
 
@@ -69,20 +64,13 @@ subroutine transpose_y_to_x_real(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     nccl_stat = ncclGroupStart()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-     do col_rank_id = 0, (col_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d(decomp%y1disp(col_rank_id) + 1), decomp%y1cnts(col_rank_id), &
-                             ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d(decomp%x1disp(col_rank_id) + 1), decomp%x1cnts(col_rank_id), &
-                             ncclType, local_to_global_col(col_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-     end do
-     nccl_stat = ncclGroupEnd()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     call decomp_2d_nccl_send_recv_col(work2_r_d,     &
+                                       work1_r_d,     &
+                                       decomp%y1disp, &
+                                       decomp%y1cnts, &
+                                       decomp%x1disp, & 
+                                       decomp%x1cnts, &
+                                       dims(1)        )
 #else
      call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
                         real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
@@ -166,10 +154,21 @@ subroutine transpose_y_to_x_complex(src, dst, decomp)
 #else
 
 #if defined(_GPU)
+#if defined(_NCCL)
+     call decomp_2d_nccl_send_recv_col(work2_c_d,     &
+                                       work1_c_d,     &
+                                       decomp%y1disp, &
+                                       decomp%y1cnts, &
+                                       decomp%x1disp, & 
+                                       decomp%x1cnts, &
+                                       dims(1)      , &
+                                       decomp_buf_size )
+#else
      call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
                         complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
                         complex_type, DECOMP_2D_COMM_COL, ierror)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+#endif
 #else
      call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
                         complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 26797c99..5e8d38cb 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -31,10 +31,6 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-#if defined(_NCCL)
-     type(ncclResult) :: nccl_stat
-     integer :: row_rank_id, cuda_stat
-#endif
      integer :: istat
 #endif
 
@@ -78,20 +74,13 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     nccl_stat = ncclGroupStart()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-     do row_rank_id = 0, (row_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
-                             ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
-                             ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-     end do
-     nccl_stat = ncclGroupEnd()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     call decomp_2d_nccl_send_recv_row(work2_r_d,     &
+                                       work1_r_d,     &
+                                       decomp%y2disp, &
+                                       decomp%y2cnts, &
+                                       decomp%z2disp, &
+                                       decomp%z2cnts, & 
+                                       dims(2)        )
 #else
      call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
                         real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
@@ -195,10 +184,21 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
 #else
 
 #if defined(_GPU)
+#if defined(_NCCL)
+     call decomp_2d_nccl_send_recv_row(work2_c_d,     &
+                                       work1_c_d,     &
+                                       decomp%y2disp, &
+                                       decomp%y2cnts, &
+                                       decomp%z2disp, &
+                                       decomp%z2cnts, & 
+                                       dims(2)      , &
+                                       decomp_buf_size)
+#else
      call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
                         complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
                         complex_type, DECOMP_2D_COMM_ROW, ierror)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+#endif
 #else
      call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
                         complex_type, dst, decomp%z2cnts, decomp%z2disp, &
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index aca0cfa8..2b2d1260 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -31,10 +31,6 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-#if defined(_NCCL)
-     type(ncclResult) :: nccl_stat
-     integer :: row_rank_id, cuda_stat
-#endif
      integer :: istat
 #endif
 
@@ -87,20 +83,13 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     nccl_stat = ncclGroupStart()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupStart")
-     do row_rank_id = 0, (row_comm_size - 1)
-        nccl_stat = ncclSend(work1_r_d(decomp%z2disp(row_rank_id) + 1), decomp%z2cnts(row_rank_id), &
-                             ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclSend")
-        nccl_stat = ncclRecv(work2_r_d(decomp%y2disp(row_rank_id) + 1), decomp%y2cnts(row_rank_id), &
-                             ncclType, local_to_global_row(row_rank_id + 1), nccl_comm_2decomp, cuda_stream_2decomp)
-        if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclRecv")
-     end do
-     nccl_stat = ncclGroupEnd()
-     if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-     cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
-     if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
+     call decomp_2d_nccl_send_recv_row(work2_r_d,     &
+                                       work1_r_d,     &
+                                       decomp%z2disp, &
+                                       decomp%z2cnts, &
+                                       decomp%y2disp, &
+                                       decomp%y2cnts, &
+                                       dims(2)        )
 #else
      call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
                         real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
@@ -202,10 +191,21 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
 #else
 
 #if defined(_GPU)
+#if defined(_NCCL)
+     call decomp_2d_nccl_send_recv_row(work2_c_d,     &
+                                       work1_c_d,     &
+                                       decomp%z2disp, &
+                                       decomp%z2cnts, &
+                                       decomp%y2disp, &
+                                       decomp%y2cnts, &
+                                       dims(2)      , &
+                                       decomp_buf_size)
+#else
      call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
                         complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
                         complex_type, DECOMP_2D_COMM_ROW, ierror)
      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+#endif
 #else
      call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
                         complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &

From bb013dae3a74a31f1a5523831373a28328cdfc53 Mon Sep 17 00:00:00 2001
From: rfj82982 <35581400+rfj82982@users.noreply.github.com>
Date: Mon, 20 Mar 2023 07:54:11 +0000
Subject: [PATCH 179/436] Update transpose for slabs (#162)

* Add NCCL module to transpose real

* Create an interface for NCCL transpose real

* Add complex to NCCL Send-Recv

* Bug correction to properly enforce my type on complex number in NCCL complex transpose

* Force all timining to be in double precision

* Modify transpose to use in memory copy in case of slabs

* Remove private for scalar variables
---
 examples/test2d/test2d.f90 | 10 +++----
 src/decomp_2d.f90          | 16 +++++------
 src/transpose_x_to_y.f90   | 53 +++++++++++++++++++++++++++++++++++++
 src/transpose_y_to_x.f90   | 53 +++++++++++++++++++++++++++++++++++++
 src/transpose_y_to_z.f90   | 54 ++++++++++++++++++++++++++++++++++++++
 src/transpose_z_to_y.f90   | 54 ++++++++++++++++++++++++++++++++++++++
 6 files changed, 227 insertions(+), 13 deletions(-)

diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index c56f99f8..9122553f 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -115,7 +115,7 @@ program test2d
 
    !$acc data copy(u1,u2,u3)
    ! original x-pensil based data
-   !$acc parallel loop default(present) private(m) 
+   !$acc parallel loop default(present) collapse(3)
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
@@ -157,7 +157,7 @@ program test2d
    ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
 
    ! also check the transposition this way
-   !$acc parallel loop default(present) private(m)
+   !$acc parallel loop default(present) collapse(3)
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
@@ -187,7 +187,7 @@ program test2d
    ! call decomp_2d_write_one(3,u3,'u3.dat')
    ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
-   !$acc parallel loop default(present) private(m)
+   !$acc parallel loop default(present) collapse(3)
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
@@ -206,7 +206,7 @@ program test2d
    call transpose_z_to_y(u3, u2)
    ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
-   !$acc parallel loop default(present) private(m)
+   !$acc parallel loop default(present) collapse(3)
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
@@ -225,7 +225,7 @@ program test2d
    call transpose_y_to_x(u2, u1)
    ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
-   !$acc parallel loop default(present) private(m)
+   !$acc parallel loop default(present) collapse(3)
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 7baaba65..71018a8b 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -194,30 +194,30 @@ module decomp_2d
    end interface decomp_2d_finalize
 
    interface transpose_x_to_y
-      module procedure transpose_x_to_y_real
+      module procedure transpose_x_to_y_real_long
       module procedure transpose_x_to_y_real_short
-      module procedure transpose_x_to_y_complex
+      module procedure transpose_x_to_y_complex_long
       module procedure transpose_x_to_y_complex_short
    end interface transpose_x_to_y
 
    interface transpose_y_to_z
-      module procedure transpose_y_to_z_real
+      module procedure transpose_y_to_z_real_long
       module procedure transpose_y_to_z_real_short
-      module procedure transpose_y_to_z_complex
+      module procedure transpose_y_to_z_complex_long
       module procedure transpose_y_to_z_complex_short
    end interface transpose_y_to_z
 
    interface transpose_z_to_y
-      module procedure transpose_z_to_y_real
+      module procedure transpose_z_to_y_real_long
       module procedure transpose_z_to_y_real_short
-      module procedure transpose_z_to_y_complex
+      module procedure transpose_z_to_y_complex_long
       module procedure transpose_z_to_y_complex_short
    end interface transpose_z_to_y
 
    interface transpose_y_to_x
-      module procedure transpose_y_to_x_real
+      module procedure transpose_y_to_x_real_long
       module procedure transpose_y_to_x_real_short
-      module procedure transpose_y_to_x_complex
+      module procedure transpose_y_to_x_complex_long
       module procedure transpose_y_to_x_complex_short
    end interface transpose_y_to_x
 
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index ffa17767..eb625b00 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -21,6 +21,33 @@ subroutine transpose_x_to_y_real_short(src, dst)
      call transpose_x_to_y(src, dst, decomp_main)
 
   end subroutine transpose_x_to_y_real_short
+  
+  subroutine transpose_x_to_y_real_long(src, dst,decomp)
+
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+#if defined(_GPU)
+     integer :: istat, nsize
+#endif
+
+     if (dims(1) == 1) then
+#if defined(_GPU)
+        nsize =  product(decomp%xsz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+#else
+        dst=src
+#endif       
+     else
+        call transpose_x_to_y_real(src, dst, decomp)
+     endif
+
+  end subroutine transpose_x_to_y_real_long
 
   subroutine transpose_x_to_y_real(src, dst, decomp)
 
@@ -113,6 +140,32 @@ subroutine transpose_x_to_y_complex_short(src, dst)
 
   end subroutine transpose_x_to_y_complex_short
 
+  subroutine transpose_x_to_y_complex_long(src, dst,decomp)
+
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+#if defined(_GPU)
+     integer :: istat, nsize
+#endif
+     if (dims(1) == 1) then
+#if defined(_GPU)
+        nsize =  product(decomp%xsz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+#else
+        dst=src
+#endif       
+     else
+        call transpose_x_to_y_complex(src, dst, decomp)
+     endif
+
+  end subroutine transpose_x_to_y_complex_long
+  
   subroutine transpose_x_to_y_complex(src, dst, decomp)
 
      implicit none
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index 890629da..eaf9a14a 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -22,6 +22,33 @@ subroutine transpose_y_to_x_real_short(src, dst)
 
   end subroutine transpose_y_to_x_real_short
 
+  subroutine transpose_y_to_x_real_long(src, dst, decomp)
+
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+#if defined(_GPU)
+     integer :: istat, nsize
+#endif
+
+     if (dims(1) == 1) then
+#if defined(_GPU)
+        nsize =  product(decomp%ysz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+#else
+        dst=src
+#endif       
+     else
+        call transpose_y_to_x_real(src, dst, decomp)
+     endif
+
+  end subroutine transpose_y_to_x_real_long
+
   subroutine transpose_y_to_x_real(src, dst, decomp)
 
      implicit none
@@ -113,6 +140,32 @@ subroutine transpose_y_to_x_complex_short(src, dst)
 
   end subroutine transpose_y_to_x_complex_short
 
+  subroutine transpose_y_to_x_complex_long(src, dst,decomp)
+
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+#if defined(_GPU)
+     integer :: istat, nsize
+#endif
+     if (dims(1) == 1) then
+#if defined(_GPU)
+        nsize =  product(decomp%ysz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+#else
+        dst=src
+#endif       
+     else
+        call transpose_y_to_x_complex(src, dst, decomp)
+     endif
+
+  end subroutine transpose_y_to_x_complex_long
+
   subroutine transpose_y_to_x_complex(src, dst, decomp)
 
      implicit none
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 5e8d38cb..d64876f8 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -21,6 +21,33 @@ subroutine transpose_y_to_z_real_short(src, dst)
      call transpose_y_to_z(src, dst, decomp_main)
 
   end subroutine transpose_y_to_z_real_short
+  
+  subroutine transpose_y_to_z_real_long(src, dst, decomp)
+
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+#if defined(_GPU)
+     integer :: istat, nsize
+#endif
+
+     if (dims(2) == 1) then
+#if defined(_GPU)
+        nsize =  product(decomp%ysz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+#else
+        dst=src
+#endif       
+     else
+        call transpose_y_to_z_real(src, dst, decomp)
+     endif
+
+  end subroutine transpose_y_to_z_real_long
 
   subroutine transpose_y_to_z_real(src, dst, decomp)
 
@@ -134,6 +161,33 @@ subroutine transpose_y_to_z_complex_short(src, dst)
 
   end subroutine transpose_y_to_z_complex_short
 
+  subroutine transpose_y_to_z_complex_long(src, dst, decomp)
+
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+#if defined(_GPU)
+     integer :: istat, nsize
+#endif
+
+     if (dims(2) == 1) then
+#if defined(_GPU)
+        nsize =  product(decomp%ysz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+#else
+        dst=src
+#endif       
+     else
+        call transpose_y_to_z_complex(src, dst, decomp)
+     endif
+
+  end subroutine transpose_y_to_z_complex_long
+
   subroutine transpose_y_to_z_complex(src, dst, decomp)
 
      implicit none
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index 2b2d1260..cb25e2ee 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -22,6 +22,33 @@ subroutine transpose_z_to_y_real_short(src, dst)
 
   end subroutine transpose_z_to_y_real_short
 
+  subroutine transpose_z_to_y_real_long(src, dst, decomp)
+
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+#if defined(_GPU)
+     integer :: istat, nsize
+#endif
+
+     if (dims(2) == 1) then
+#if defined(_GPU)
+        nsize =  product(decomp%zsz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+#else
+        dst=src
+#endif       
+     else
+        call transpose_z_to_y_real(src, dst, decomp)
+     endif
+
+  end subroutine transpose_z_to_y_real_long
+ 
   subroutine transpose_z_to_y_real(src, dst, decomp)
 
      implicit none
@@ -131,6 +158,33 @@ subroutine transpose_z_to_y_complex_short(src, dst)
 
   end subroutine transpose_z_to_y_complex_short
 
+  subroutine transpose_z_to_y_complex_long(src, dst, decomp)
+
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+#if defined(_GPU)
+     integer :: istat, nsize
+#endif
+
+     if (dims(2) == 1) then
+#if defined(_GPU)
+        nsize =  product(decomp%zsz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+#else
+        dst=src
+#endif       
+     else
+        call transpose_z_to_y_complex(src, dst, decomp)
+     endif
+
+  end subroutine transpose_z_to_y_complex_long
+
   subroutine transpose_z_to_y_complex(src, dst, decomp)
 
      implicit none

From 1b1a322f0fe3a5a20bf4e234c9db08582d034ba4 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 20 Mar 2023 08:56:21 +0100
Subject: [PATCH 180/436] fprettify --enable-replacements --c-relations
 src/*f90 examples/*/*f90

---
 examples/fft_physical_x/fft_c2c_x.f90  | 26 ++++----
 examples/fft_physical_x/fft_grid_x.f90 | 26 ++++----
 examples/fft_physical_x/fft_r2c_x.f90  | 26 ++++----
 examples/fft_physical_z/fft_c2c_z.f90  | 26 ++++----
 examples/fft_physical_z/fft_r2c_z.f90  | 26 ++++----
 examples/halo_test/halo_test.f90       | 86 +++++++++++++-------------
 examples/init_test/init_test.f90       | 36 +++++------
 examples/io_test/io_bench.f90          | 28 ++++-----
 examples/io_test/io_plane_test.f90     | 30 ++++-----
 examples/io_test/io_read.f90           | 30 ++++-----
 examples/io_test/io_test.f90           | 30 ++++-----
 examples/io_test/io_var_test.f90       | 41 ++++++------
 examples/test2d/test2d.f90             | 37 ++++++-----
 examples/test2d/timing2d_complex.f90   | 85 +++++++++++++------------
 examples/test2d/timing2d_real.f90      | 75 +++++++++++-----------
 src/decomp_2d.f90                      |  5 +-
 src/decomp_2d_constants.f90            |  6 +-
 src/decomp_2d_cumpi.f90                | 24 +++----
 src/decomp_2d_init_fin.f90             |  4 +-
 src/halo.f90                           | 12 ++--
 src/halo_common.f90                    | 12 ++--
 src/io.f90                             | 10 +--
 src/io_read_inflow.f90                 |  2 +-
 src/transpose_x_to_y.f90               | 44 ++++++-------
 src/transpose_y_to_x.f90               | 36 +++++------
 src/transpose_y_to_z.f90               | 34 +++++-----
 src/transpose_z_to_y.f90               | 30 ++++-----
 27 files changed, 409 insertions(+), 418 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 453f7b1b..1291734a 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -42,28 +42,28 @@ program fft_c2c_x
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -75,8 +75,8 @@ program fft_c2c_x
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
+      end if
+   end if
 
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 2dee619a..d51e6cfc 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -42,28 +42,28 @@ program fft_physical_x
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -75,8 +75,8 @@ program fft_physical_x
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
+      end if
+   end if
    call decomp_2d_init(nx + 1, ny + 1, nz + 1, p_row, p_col)
 
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 6427d6ed..9cf55702 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -43,28 +43,28 @@ program fft_r2c_x
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -76,8 +76,8 @@ program fft_r2c_x
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
+      end if
+   end if
 
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 40e9761f..91ba1cbb 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -42,28 +42,28 @@ program fft_c2c_z
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -75,8 +75,8 @@ program fft_c2c_z
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
+      end if
+   end if
 
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 297a0a5b..6cfcda39 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -43,28 +43,28 @@ program fft_r2c_z
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -76,8 +76,8 @@ program fft_r2c_z
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
+      end if
+   end if
 
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index faf53951..ed3558cc 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -32,7 +32,6 @@ program halo_test
    real(mytype), allocatable, dimension(:, :, :) :: u3, v3, w3, div3
    real(mytype), allocatable, dimension(:, :, :) :: div, wk2, wk3
 
-
    integer :: i, j, k, ierror, n
 
    integer, allocatable, dimension(:) :: seed
@@ -54,28 +53,28 @@ program halo_test
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -87,11 +86,10 @@ program halo_test
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
+      end if
+   end if
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-
    xlast = xsize(1) - 1
    if (xend(2) == ny) then
       ylast = xsize(2) - 1
@@ -129,14 +127,14 @@ program halo_test
 contains
 
    !=====================================================================
-   ! Initialize 
+   ! Initialize
    !=====================================================================
    subroutine initialise()
 
       use decomp_2d
-      
+
       implicit none
-   
+
 #ifdef HALO_GLOBAL
       logical, parameter :: global = .true.
 #else
@@ -165,12 +163,12 @@ subroutine initialise()
       call alloc_y(v2, global)
       call alloc_y(w2, global)
       call alloc_y(wk2, global)
-      
+
       call alloc_z(u3, global)
       call alloc_z(v3, global)
       call alloc_z(w3, global)
       call alloc_z(wk3, global)
-      
+
       all_pass = .true.
 
    end subroutine initialise
@@ -182,10 +180,10 @@ subroutine finalize()
 
       implicit none
 
-      deallocate(u1,v1,w1)
-      deallocate(u2,v2,w2)
-      deallocate(u3,v3,w3)
-      deallocate(wk2,wk3)
+      deallocate (u1, v1, w1)
+      deallocate (u2, v2, w2)
+      deallocate (u3, v3, w3)
+      deallocate (wk2, wk3)
 
    end subroutine finalize
    !=====================================================================
@@ -215,7 +213,7 @@ subroutine test_div_transpose()
       ifirst = 2; ilast = xsize(1) - 1
 
       !$acc kernels default(present)
-      div(:,:,:) = 0.0_mytype
+      div(:, :, :) = 0.0_mytype
       !$acc end kernels
       !$acc kernels default(present)
       do k = kfirst, klast
@@ -295,7 +293,7 @@ subroutine test_div_haloX()
 
       implicit none
 
-      real(mytype), allocatable, dimension(:, :, :) :: vh, wh 
+      real(mytype), allocatable, dimension(:, :, :) :: vh, wh
 #if defined(_GPU)
       attributes(device) :: vh, wh
 #endif
@@ -316,7 +314,7 @@ subroutine test_div_haloX()
 
       ! Only global arrays defined in initialise needs to be ported
       ! Halo array are allocated in both host and device in update_halo
-      ! Halo arrays are just removed before being deallocated 
+      ! Halo arrays are just removed before being deallocated
 #ifdef HALO_GLOBAL
       call update_halo(v1, vh, 1, opt_global=.true., opt_pencil=1)
       call update_halo(w1, wh, 1, opt_global=.true., opt_pencil=1)
@@ -336,22 +334,22 @@ subroutine test_div_haloX()
       call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
 
       !$acc kernels default(present)
-      div1(:,:,:) = 0._mytype
+      div1(:, :, :) = 0._mytype
       !$acc end kernels
       !$acc kernels default(present)
       do k = kfirst, klast
          do j = jfirst, jlast
             do i = ifirst, ilast
                div1(i, j, k) = (u1(i + 1, j, k) - u1(i - 1, j, k)) &
-                             + (vh(i, j + 1, k) - vh(i, j - 1, k)) &
-                             + (wh(i, j, k + 1) - wh(i, j, k - 1))
+                               + (vh(i, j + 1, k) - vh(i, j - 1, k)) &
+                               + (wh(i, j, k + 1) - wh(i, j, k - 1))
             end do
          end do
       end do
       !$acc end kernels
 
       ! Compute error
-      call check_err(div1, "X")  
+      call check_err(div1, "X")
 
       deallocate (vh, wh)
 
@@ -363,7 +361,7 @@ end subroutine test_div_haloX
    subroutine test_div_haloY()
 
       implicit none
-      real(mytype), allocatable, dimension(:, :, :) :: uh, wh 
+      real(mytype), allocatable, dimension(:, :, :) :: uh, wh
 #if defined(_GPU)
       attributes(device) :: uh, wh
 #endif
@@ -408,8 +406,8 @@ subroutine test_div_haloY()
          do j = jfirst, jlast
             do i = ifirst, ilast
                wk2(i, j, k) = (uh(i + 1, j, k) - uh(i - 1, j, k)) &
-                            + (v2(i, j + 1, k) - v2(i, j - 1, k)) &
-                            + (wh(i, j, k + 1) - wh(i, j, k - 1))
+                              + (v2(i, j + 1, k) - v2(i, j - 1, k)) &
+                              + (wh(i, j, k + 1) - wh(i, j, k - 1))
             end do
          end do
       end do
@@ -430,7 +428,7 @@ end subroutine test_div_haloY
    subroutine test_div_haloZ()
 
       implicit none
-      real(mytype), allocatable, dimension(:, :, :) :: uh,vh 
+      real(mytype), allocatable, dimension(:, :, :) :: uh, vh
 #if defined(_GPU)
       attributes(device) :: vh, uh
 #endif
@@ -475,8 +473,8 @@ subroutine test_div_haloZ()
          do i = ifirst, ilast
             do k = kfirst, klast
                wk3(i, j, k) = uh(i + 1, j, k) - uh(i - 1, j, k) &
-                            + vh(i, j + 1, k) - vh(i, j - 1, k) &
-                            + w3(i, j, k + 1) - w3(i, j, k - 1)
+                              + vh(i, j + 1, k) - vh(i, j - 1, k) &
+                              + w3(i, j, k + 1) - w3(i, j, k - 1)
             end do
          end do
       end do
@@ -496,7 +494,7 @@ end subroutine test_div_haloZ
    subroutine check_err(divh, pencil)
 
       implicit none
-      
+
       real(mytype), dimension(:, :, :), intent(in) :: divh
       character(len=*), intent(in) :: pencil
       real(mytype), dimension(:, :, :), allocatable :: tmp
@@ -518,7 +516,7 @@ subroutine check_err(divh, pencil)
       tmp(2:xlast, 2:ylast, 2:zlast) = div1(2:xlast, 2:ylast, 2:zlast)
       !$acc end kernels
       divmag = mag(tmp)
-      
+
       if (err < epsilon(divmag)*divmag) then
          passing = .true.
       else
@@ -535,7 +533,7 @@ subroutine check_err(divh, pencil)
          write (*, *) 'Error: ', err, '; Relative: ', err/divmag
          write (*, *) 'Pass: ', passing
       end if
-      deallocate(tmp)
+      deallocate (tmp)
 
    end subroutine check_err
    !=====================================================================
@@ -557,10 +555,10 @@ real(mytype) function mag(a)
       do k = 2, zlast
          do j = 2, ylast
             do i = 2, xlast
-               lmag = lmag + a(i,j,k)**2
-            enddo
-         enddo
-      enddo
+               lmag = lmag + a(i, j, k)**2
+            end do
+         end do
+      end do
       !$acc end parallel
 
       call MPI_Allreduce(lmag, gmag, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index dfe53a0e..0d6630fb 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -22,7 +22,7 @@ program init_test
    integer :: nranks_tot
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
-   integer :: nexpect 
+   integer :: nexpect
    integer :: ierror
 
    call MPI_Init(ierror)
@@ -36,28 +36,28 @@ program init_test
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -69,9 +69,9 @@ program init_test
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
-   nexpect = nx * ny *nz
+      end if
+   end if
+   nexpect = nx*ny*nz
    call run(p_row, p_col)
 
    call MPI_Finalize(ierror)
@@ -108,7 +108,7 @@ subroutine check_axis(axis)
          sizes = zsize
       else
          sizes = 0
-         if (nrank==0) print *, "ERROR: unknown axis requested!"
+         if (nrank == 0) print *, "ERROR: unknown axis requested!"
          stop 1
       end if
 
@@ -116,10 +116,10 @@ subroutine check_axis(axis)
       call MPI_Allreduce(suml, sumg, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierror)
 
       if (sumg /= nexpect) then
-         if (nrank==0) print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
+         if (nrank == 0) print *, "ERROR: got ", sumg, " nodes, expected ", nexpect
          stop 1
       else
-         if (nrank==0) print *, "Init Test pass for axis ", axis
+         if (nrank == 0) print *, "Init Test pass for axis ", axis
       end if
 
    end subroutine check_axis
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index 0f5926ed..0b37e0d0 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -35,28 +35,28 @@ program io_bench
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -68,9 +68,9 @@ program io_bench
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
-   
+      end if
+   end if
+
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    call alloc_x(u1, .true.)
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index f362bb9e..2c86bf56 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -46,28 +46,28 @@ program io_plane_test
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -79,13 +79,13 @@ program io_plane_test
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
-   
+      end if
+   end if
+
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! ***** global data *****
-   allocate(data1(nx,ny,nz))
+   allocate (data1(nx, ny, nz))
    m = 1
    do k = 1, nz
       do j = 1, ny
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 3eded469..d396159a 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -44,28 +44,28 @@ program io_read
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -77,13 +77,13 @@ program io_read
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
-   
+      end if
+   end if
+
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! ***** global data *****
-   allocate(data1(nx,ny,nz))
+   allocate (data1(nx, ny, nz))
    m = 1
    do k = 1, nz
       do j = 1, ny
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index ba4378c0..82b68b80 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -48,28 +48,28 @@ program io_test
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -81,13 +81,13 @@ program io_test
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
-   
+      end if
+   end if
+
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! ***** global data *****
-   allocate(data1(nx,ny,nz))
+   allocate (data1(nx, ny, nz))
    m = 1
    do k = 1, nz
       do j = 1, ny
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 2ae5abb5..1e7e6d4e 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -53,7 +53,6 @@ program io_var_test
    character(len=15) :: filename
    integer(kind=MPI_OFFSET_KIND) :: filesize, disp
 
-
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
@@ -64,28 +63,28 @@ program io_var_test
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -97,9 +96,9 @@ program io_var_test
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
-   
+      end if
+   end if
+
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! also create a data set over a large domain
@@ -154,9 +153,9 @@ program io_var_test
 
    ! distribute the data
    !$acc data copyin(data1,cdata1,data1_large) copy(u1,u2,u3,u1l,u2l,u3l,cu1,cu2,cu3)
-   xst1 = xstart(1); xen1=xend(1)
-   xst2 = xstart(2); xen2=xend(2)
-   xst3 = xstart(3); xen3=xend(3)
+   xst1 = xstart(1); xen1 = xend(1)
+   xst2 = xstart(2); xen2 = xend(2)
+   xst3 = xstart(3); xen3 = xend(3)
    !$acc parallel loop default(present)
    do k = xst3, xen3
       do j = xst2, xen2
@@ -167,9 +166,9 @@ program io_var_test
       end do
    end do
    !$acc end loop
-   xst1 = large%xst(1); xen1=large%xen(1)
-   xst2 = large%xst(2); xen2=large%xen(2)
-   xst3 = large%xst(3); xen3=large%xen(3)
+   xst1 = large%xst(1); xen1 = large%xen(1)
+   xst2 = large%xst(2); xen2 = large%xen(2)
+   xst3 = large%xst(3); xen3 = large%xen(3)
    !$acc parallel loop default(present)
    do k = xst3, xen3
       do j = xst2, xen2
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 9122553f..e903cd29 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -44,28 +44,28 @@ program test2d
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -77,8 +77,8 @@ program test2d
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
+      end if
+   end if
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    !! ***** global data *****
@@ -103,7 +103,6 @@ program test2d
    zst1 = zstart(1); zen1 = zend(1)
    zst2 = zstart(2); zen2 = zend(2)
    zst3 = zstart(3); zen3 = zend(3)
-   
 
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Testing the swap routines
@@ -119,7 +118,7 @@ program test2d
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
             u1(i, j, k) = m
          end do
       end do
@@ -161,7 +160,7 @@ program test2d
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
-            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
             if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
@@ -191,7 +190,7 @@ program test2d
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
             if (abs(u3(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
@@ -210,7 +209,7 @@ program test2d
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
-            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
             if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
@@ -229,7 +228,7 @@ program test2d
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
             if (abs(u1(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 05f61aa0..dc7362ec 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -33,7 +33,7 @@ program timing2d_complex
    complex(mytype) :: cm
 
    double precision :: t1, t2, t3, t4, t5, t6, t7, t8
-   integer :: iter, niter =10
+   integer :: iter, niter = 10
 
    ! Init
    error_flag = .false.
@@ -48,28 +48,28 @@ program timing2d_complex
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -81,8 +81,8 @@ program timing2d_complex
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
+      end if
+   end if
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    !! ***** global data *****
@@ -107,7 +107,6 @@ program timing2d_complex
    zst1 = zstart(1); zen1 = zend(1)
    zst2 = zstart(2); zen2 = zend(2)
    zst3 = zstart(3); zen3 = zend(3)
-   
 
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Testing the swap routines
@@ -119,12 +118,12 @@ program timing2d_complex
 
    !$acc data copy(u1,u2,u3)
    ! original x-pensil based data
-   !$acc parallel loop default(present) private(m) 
+   !$acc parallel loop default(present) private(m)
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-            u1(i, j, k) = cmplx(m,real(m-1,mytype),kind=mytype)
+            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            u1(i, j, k) = cmplx(m, real(m - 1, mytype), kind=mytype)
          end do
       end do
    end do
@@ -144,16 +143,16 @@ program timing2d_complex
 
    ! call decomp_2d_write_one(1,u1,'u1.dat')
 
-   t2=0.d0
-   t4=0.d0
-   t6=0.d0
-   t8=0.d0
-   do iter=1,niter
+   t2 = 0.d0
+   t4 = 0.d0
+   t6 = 0.d0
+   t8 = 0.d0
+   do iter = 1, niter
       !!!!!!!!!!!!!!!!!!!!!!!
       ! x-pensil ==> y-pensil
       t1 = MPI_WTIME()
       call transpose_x_to_y(u1, u2)
-      t2 = t2 + MPI_WTIME()-t1
+      t2 = t2 + MPI_WTIME() - t1
 
 #ifdef DEBUG
       if (nrank == 0) then
@@ -172,8 +171,8 @@ program timing2d_complex
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-               cm = cmplx(m,real(m-1,mytype),kind=mytype)
+               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
@@ -187,7 +186,7 @@ program timing2d_complex
       ! y-pensil ==> z-pensil
       t3 = MPI_WTIME()
       call transpose_y_to_z(u2, u3)
-      t4 = t4 + MPI_WTIME()-t3
+      t4 = t4 + MPI_WTIME() - t3
 
 #ifdef DEBUG
       if (nrank == 0) then
@@ -205,8 +204,8 @@ program timing2d_complex
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
-               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-               cm = cmplx(m,real(m-1,mytype),kind=mytype)
+               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u3(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
@@ -220,15 +219,15 @@ program timing2d_complex
       ! z-pensil ==> y-pensil
       t5 = MPI_WTIME()
       call transpose_z_to_y(u3, u2)
-      t6 = t6 + MPI_WTIME()-t5
+      t6 = t6 + MPI_WTIME() - t5
       ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
       !$acc parallel loop default(present) private(m,cm)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-               cm = cmplx(m,real(m-1,mytype),kind=mytype)
+               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
@@ -242,15 +241,15 @@ program timing2d_complex
       ! y-pensil ==> x-pensil
       t7 = MPI_WTIME()
       call transpose_y_to_x(u2, u1)
-      t8 = t8 + MPI_WTIME()-t7
+      t8 = t8 + MPI_WTIME() - t7
       ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
       !$acc parallel loop default(present) private(m,cm)
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1
-               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
-               cm = cmplx(m,real(m-1,mytype),kind=mytype)
+               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u1(i, j, k) - cm) > 0) error_flag = .true.
             end do
          end do
@@ -259,7 +258,7 @@ program timing2d_complex
       call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
       if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
       if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
-   enddo
+   end do
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
@@ -273,14 +272,14 @@ program timing2d_complex
    call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
    t7 = t7/dble(nproc)
-   t8 = t1+t3+t5+t7
+   t8 = t1 + t3 + t5 + t7
    if (nrank == 0) then
-     write(*,*) 'Time X->Y ', t1
-     write(*,*) 'Time Y->Z ', t3
-     write(*,*) 'Time Z->Y ', t5
-     write(*,*) 'Time Y->X ', t7
-     write(*,*) 'Time TOT  ', t8
-   endif
+      write (*, *) 'Time X->Y ', t1
+      write (*, *) 'Time Y->Z ', t3
+      write (*, *) 'Time Z->Y ', t5
+      write (*, *) 'Time Y->X ', t7
+      write (*, *) 'Time TOT  ', t8
+   end if
 
    if (nrank == 0) then
       write (*, *) " "
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 5eab1de7..9e1c007b 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -32,7 +32,7 @@ program timing2d_real
    real(mytype) :: m
 
    double precision :: t1, t2, t3, t4, t5, t6, t7, t8
-   integer :: iter, niter =10
+   integer :: iter, niter = 10
 
    ! Init
    error_flag = .false.
@@ -47,28 +47,28 @@ program timing2d_real
    nz = nz_base*resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
-   nargin=command_argument_count()
-   if ((nargin==0).or.(nargin==2).or.(nargin==5)) then
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
-         read(InputFN, *, iostat=status) DecInd
-         if (arg.eq.1) then
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
             p_row = DecInd
-         elseif (arg.eq.2) then
+         elseif (arg == 2) then
             p_col = DecInd
-         elseif (arg.eq.3) then
+         elseif (arg == 3) then
             nx = DecInd
-         elseif (arg.eq.4) then
+         elseif (arg == 4) then
             ny = DecInd
-         elseif (arg.eq.5) then
+         elseif (arg == 5) then
             nz = DecInd
-         endif
-      enddo
+         end if
+      end do
    else
       ! nrank not yet computed we need to avoid write
       ! for every rank
       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
-      if (nrank==0) then
+      if (nrank == 0) then
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
@@ -80,8 +80,8 @@ program timing2d_real
          print *, "  5) nz "
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
-      endif
-   endif
+      end if
+   end if
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    !! ***** global data *****
@@ -106,7 +106,6 @@ program timing2d_real
    zst1 = zstart(1); zen1 = zend(1)
    zst2 = zstart(2); zen2 = zend(2)
    zst3 = zstart(3); zen3 = zend(3)
-   
 
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Testing the swap routines
@@ -118,11 +117,11 @@ program timing2d_real
 
    !$acc data copy(u1,u2,u3)
    ! original x-pensil based data
-   !$acc parallel loop default(present) private(m) 
+   !$acc parallel loop default(present) private(m)
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
             u1(i, j, k) = m
          end do
       end do
@@ -143,16 +142,16 @@ program timing2d_real
 
    ! call decomp_2d_write_one(1,u1,'u1.dat')
 
-   t2=0.d0
-   t4=0.d0
-   t6=0.d0
-   t8=0.d0
-   do iter=1,niter
+   t2 = 0.d0
+   t4 = 0.d0
+   t6 = 0.d0
+   t8 = 0.d0
+   do iter = 1, niter
       !!!!!!!!!!!!!!!!!!!!!!!
       ! x-pensil ==> y-pensil
       t1 = MPI_WTIME()
       call transpose_x_to_y(u1, u2)
-      t2 = t2 + MPI_WTIME()-t1
+      t2 = t2 + MPI_WTIME() - t1
 
 #ifdef DEBUG
       if (nrank == 0) then
@@ -171,7 +170,7 @@ program timing2d_real
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
                if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -185,7 +184,7 @@ program timing2d_real
       ! y-pensil ==> z-pensil
       t3 = MPI_WTIME()
       call transpose_y_to_z(u2, u3)
-      t4 = t4 + MPI_WTIME()-t3
+      t4 = t4 + MPI_WTIME() - t3
 
 #ifdef DEBUG
       if (nrank == 0) then
@@ -203,7 +202,7 @@ program timing2d_real
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
-               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
                if (abs(u3(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -217,14 +216,14 @@ program timing2d_real
       ! z-pensil ==> y-pensil
       t5 = MPI_WTIME()
       call transpose_z_to_y(u3, u2)
-      t6 = t6 + MPI_WTIME()-t5
+      t6 = t6 + MPI_WTIME() - t5
       ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
       !$acc parallel loop default(present) private(m)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
                if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -238,14 +237,14 @@ program timing2d_real
       ! y-pensil ==> x-pensil
       t7 = MPI_WTIME()
       call transpose_y_to_x(u2, u1)
-      t8 = t8 + MPI_WTIME()-t7
+      t8 = t8 + MPI_WTIME() - t7
       ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
       !$acc parallel loop default(present) private(m)
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1
-               m = real(i+(j-1)*nx+(k-1)*nx*ny,mytype)
+               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
                if (abs(u1(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -254,7 +253,7 @@ program timing2d_real
       call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
       if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
       if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
-   enddo
+   end do
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
@@ -268,14 +267,14 @@ program timing2d_real
    call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
    t7 = t7/dble(nproc)
-   t8 = t1+t3+t5+t7
+   t8 = t1 + t3 + t5 + t7
    if (nrank == 0) then
-     write(*,*) 'Time X->Y ', t1
-     write(*,*) 'Time Y->Z ', t3
-     write(*,*) 'Time Z->Y ', t5
-     write(*,*) 'Time Y->X ', t7
-     write(*,*) 'Time TOT  ', t8
-   endif
+      write (*, *) 'Time X->Y ', t1
+      write (*, *) 'Time Y->Z ', t3
+      write (*, *) 'Time Z->Y ', t5
+      write (*, *) 'Time Y->X ', t7
+      write (*, *) 'Time TOT  ', t8
+   end if
 
    if (nrank == 0) then
       write (*, *) " "
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 71018a8b..2a505f8d 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -24,7 +24,7 @@ module decomp_2d
    use decomp_2d_cumpi
 #if defined(_NCCL)
    use nccl
-   use decomp_2d_nccl 
+   use decomp_2d_nccl
 #endif
 #endif
 
@@ -32,7 +32,6 @@ module decomp_2d
 
    private        ! Make everything private unless declared public
 
-
    ! some key global variables
    integer, save, public :: nx_global, ny_global, nz_global  ! global size
 
@@ -249,7 +248,6 @@ module decomp_2d
       module procedure alloc_z_complex_short
    end interface alloc_z
 
-
    interface
 
       module function d2d_listing_get_unit()
@@ -316,7 +314,6 @@ end subroutine decomp_profiler_end_char
 
 #include "decomp_2d_init_fin.f90"
 
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Return the default decomposition object
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 97af2ef1..28a3d349 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -17,13 +17,13 @@ module decomp_2d_constants
    use mpi
    use, intrinsic :: iso_fortran_env, only: real32, real64
 #if defined(_GPU) && defined(_NCCL)
-   use nccl 
+   use nccl
 #endif
 
    implicit none
 
-   !private 
- 
+   !private
+
 #ifdef DOUBLE_PREC
    integer, parameter, public :: mytype = KIND(0._real64)
    integer, parameter, public :: real_type = MPI_DOUBLE_PRECISION
diff --git a/src/decomp_2d_cumpi.f90 b/src/decomp_2d_cumpi.f90
index 59a1199c..4e4b54af 100644
--- a/src/decomp_2d_cumpi.f90
+++ b/src/decomp_2d_cumpi.f90
@@ -14,15 +14,15 @@
 
 module decomp_2d_cumpi
 
-   use decomp_2d_constants  
-   use decomp_2d_mpi 
+   use decomp_2d_constants
+   use decomp_2d_mpi
 
    implicit none
-   
+
    private        ! Make everything private unless declared public
-   
+
    ! Device working arrays
-   real(mytype), allocatable, dimension(:), device , public:: work1_r_d, work2_r_d
+   real(mytype), allocatable, dimension(:), device, public:: work1_r_d, work2_r_d
    complex(mytype), allocatable, dimension(:), device, public :: work1_c_d, work2_c_d
 
    public :: decomp_2d_cumpi_init, &
@@ -31,14 +31,14 @@ module decomp_2d_cumpi
 contains
    !
    ! init of the arrays
-   ! 
+   !
    subroutine decomp_2d_cumpi_init(buf_size)
-      
+
       implicit none
-      
+
       integer, intent(in) :: buf_size
       integer :: status, errorcode
-      
+
       if (allocated(work1_r_d)) deallocate (work1_r_d)
       if (allocated(work2_r_d)) deallocate (work2_r_d)
       if (allocated(work1_c_d)) deallocate (work1_c_d)
@@ -67,13 +67,13 @@ subroutine decomp_2d_cumpi_init(buf_size)
          call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
                               'Out of memory when allocating 2DECOMP workspace')
       end if
-   
+
    end subroutine decomp_2d_cumpi_init
    !
    ! init of the arrays
-   ! 
+   !
    subroutine decomp_2d_cumpi_fin
-      
+
       implicit none
 
       deallocate (work1_r_d, work2_r_d, work1_c_d, work2_c_d)
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index 94e74e15..7a8bd916 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -150,7 +150,7 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
 #endif
 
 #if defined(_GPU) && defined(_NCCL)
-     call decomp_2d_nccl_init(DECOMP_2D_COMM_COL,DECOMP_2D_COMM_ROW)
+     call decomp_2d_nccl_init(DECOMP_2D_COMM_COL, DECOMP_2D_COMM_ROW)
 #endif
 
      !
@@ -242,7 +242,7 @@ subroutine decomp_2d_debug
      read (val, '(i4)', iostat=ierror) decomp_debug
      if (ierror /= 0) then
         call decomp_2d_warning(__FILE__, __LINE__, ierror, &
-             "Error when reading DECOMP_2D_DEBUG : "//val)
+                               "Error when reading DECOMP_2D_DEBUG : "//val)
      end if
 
   end subroutine decomp_2d_debug
diff --git a/src/halo.f90 b/src/halo.f90
index 3ed61d9e..16bad6e6 100644
--- a/src/halo.f90
+++ b/src/halo.f90
@@ -20,7 +20,7 @@ subroutine update_halo_real_short(in, out, level, opt_global, opt_pencil)
      real(mytype), dimension(:, :, :), intent(IN) :: in
      real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
 #if defined(_GPU)
-      attributes(device) :: out
+     attributes(device) :: out
 #endif
      logical, optional :: opt_global
      integer, intent(in), optional :: opt_pencil
@@ -37,7 +37,7 @@ subroutine update_halo_real(in, out, level, decomp, opt_global, opt_pencil)
      real(mytype), dimension(:, :, :), intent(IN) :: in
      real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
 #if defined(_GPU)
-      attributes(device) :: out
+     attributes(device) :: out
 #endif
      TYPE(DECOMP_INFO), intent(in) :: decomp
      logical, optional :: opt_global
@@ -47,7 +47,7 @@ subroutine update_halo_real(in, out, level, decomp, opt_global, opt_pencil)
 
      ! starting/ending index of array with halo cells
      integer :: xs, ys, zs, xe, ye, ze
-     ! additional start end 
+     ! additional start end
      integer :: ist, ien, jst, jen, kst, ken
 
      integer :: i, j, k, s1, s2, s3, ierror
@@ -77,7 +77,7 @@ subroutine update_halo_complex_short(in, out, level, opt_global, opt_pencil)
      complex(mytype), dimension(:, :, :), intent(IN) :: in
      complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
 #if defined(_GPU)
-      attributes(device) :: out
+     attributes(device) :: out
 #endif
      logical, optional :: opt_global
      integer, intent(in), optional :: opt_pencil
@@ -94,7 +94,7 @@ subroutine update_halo_complex(in, out, level, decomp, opt_global, opt_pencil)
      complex(mytype), dimension(:, :, :), intent(IN) :: in
      complex(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
 #if defined(_GPU)
-      attributes(device) :: out
+     attributes(device) :: out
 #endif
      TYPE(DECOMP_INFO), intent(in) :: decomp
      logical, optional :: opt_global
@@ -104,7 +104,7 @@ subroutine update_halo_complex(in, out, level, decomp, opt_global, opt_pencil)
 
      ! starting/ending index of array with halo cells
      integer :: xs, ys, zs, xe, ye, ze
-     ! additional start end 
+     ! additional start end
      integer :: ist, ien, jst, jen, kst, ken
 
      integer :: i, j, k, s1, s2, s3, ierror
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index 8d3dd69f..bf21f10f 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -124,8 +124,8 @@
        ! note the input array passed in always has index starting from 1
        ! need to work out the corresponding global index
        if (ipencil == 1) then
-          kst=decomp%xst(3); ken=decomp%xen(3)
-          jst=decomp%xst(2); jen=decomp%xen(2)
+          kst = decomp%xst(3); ken = decomp%xen(3)
+          jst = decomp%xst(2); jen = decomp%xen(2)
           !$acc kernels default(present)
           do k = kst, ken
              do j = jst, jen
@@ -136,8 +136,8 @@
           end do
           !$acc end kernels
        else if (ipencil == 2) then
-          kst=decomp%yst(3); ken=decomp%yen(3)
-          ist=decomp%yst(1); ien=decomp%yen(1)
+          kst = decomp%yst(3); ken = decomp%yen(3)
+          ist = decomp%yst(1); ien = decomp%yen(1)
           !$acc kernels default(present)
           do k = kst, ken
              do j = 1, s2  ! y all local
@@ -148,8 +148,8 @@
           end do
           !$acc end kernels
        else if (ipencil == 3) then
-          jst=decomp%zst(2); jen=decomp%zen(2)
-          ist=decomp%xst(1); ien=decomp%xen(1)
+          jst = decomp%zst(2); jen = decomp%zen(2)
+          ist = decomp%xst(1); ien = decomp%xen(1)
           !$acc kernels default(present)
           do k = 1, s3  ! z all local
              do j = jst, jen
diff --git a/src/io.f90 b/src/io.f90
index 8855e094..594e7b69 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -140,7 +140,7 @@ subroutine decomp_2d_io_init()
       call adios2_init(adios, trim(config_file), decomp_2d_comm, adios2_debug_mode, ierror)
       if (ierror /= 0) then
          call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-              "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
+                              "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")
       end if
       engine_live(:) = .false.
 
@@ -448,7 +448,7 @@ subroutine adios2_read_one_real(var, engine_name, varname, io_name)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
       if (.not. var_handle%valid) then
          call decomp_2d_abort(__FILE__, __LINE__, -1, &
-              "ERROR: trying to read variable without registering first! "//trim(varname))
+                              "ERROR: trying to read variable without registering first! "//trim(varname))
       end if
 
       call adios2_variable_steps(nsteps, var_handle, ierror)
@@ -1350,12 +1350,12 @@ subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_n
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
       if (.not. var_handle%valid) then
          call decomp_2d_abort(__FILE__, __LINE__, -1, &
-              "ERROR: trying to write variable before registering! "//trim(varname))
+                              "ERROR: trying to write variable before registering! "//trim(varname))
       end if
 
       if (idx < 1) then
          call decomp_2d_abort(__FILE__, __LINE__, idx, &
-              "You haven't opened "//trim(io_name)//":"//trim(dirname))
+                              "You haven't opened "//trim(io_name)//":"//trim(dirname))
       end if
 
       if (deferred_writes) then
@@ -1454,7 +1454,7 @@ subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplan
                                         adios2_constant_dims, ierror)
             if (ierror /= 0) then
                call decomp_2d_abort(__FILE__, __LINE__, ierror, &
-                    "adios2_define_variable, ERROR registering variable "//trim(varname))
+                                    "adios2_define_variable, ERROR registering variable "//trim(varname))
             end if
          end if
       else
diff --git a/src/io_read_inflow.f90 b/src/io_read_inflow.f90
index a247192b..162510c7 100644
--- a/src/io_read_inflow.f90
+++ b/src/io_read_inflow.f90
@@ -78,7 +78,7 @@
 if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
 if (.not. var_handle%valid) then
    call decomp_2d_abort(__FILE__, __LINE__, -1, &
-        "trying to write variable before registering!  "//trim(varname))
+                        "trying to write variable before registering!  "//trim(varname))
 end if
 
 !! Note - need to use sync mode as we are using a view into the array - unsure how this works with deferred writes
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index eb625b00..9cae71f0 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -21,8 +21,8 @@ subroutine transpose_x_to_y_real_short(src, dst)
      call transpose_x_to_y(src, dst, decomp_main)
 
   end subroutine transpose_x_to_y_real_short
-  
-  subroutine transpose_x_to_y_real_long(src, dst,decomp)
+
+  subroutine transpose_x_to_y_real_long(src, dst, decomp)
 
      implicit none
 
@@ -35,17 +35,17 @@ subroutine transpose_x_to_y_real_long(src, dst,decomp)
 
      if (dims(1) == 1) then
 #if defined(_GPU)
-        nsize =  product(decomp%xsz)
+        nsize = product(decomp%xsz)
         !$acc host_data use_device(src,dst)
         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst=src
-#endif       
+        dst = src
+#endif
      else
         call transpose_x_to_y_real(src, dst, decomp)
-     endif
+     end if
 
   end subroutine transpose_x_to_y_real_long
 
@@ -91,13 +91,13 @@ subroutine transpose_x_to_y_real(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_col(work2_r_d,     &
-                                       work1_r_d,     &
+     call decomp_2d_nccl_send_recv_col(work2_r_d, &
+                                       work1_r_d, &
                                        decomp%x1disp, &
                                        decomp%x1cnts, &
-                                       decomp%y1disp, & 
-                                       decomp%y1cnts, &  
-                                       dims(1)        )
+                                       decomp%y1disp, &
+                                       decomp%y1cnts, &
+                                       dims(1))
 #else
      call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
                         real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
@@ -140,7 +140,7 @@ subroutine transpose_x_to_y_complex_short(src, dst)
 
   end subroutine transpose_x_to_y_complex_short
 
-  subroutine transpose_x_to_y_complex_long(src, dst,decomp)
+  subroutine transpose_x_to_y_complex_long(src, dst, decomp)
 
      implicit none
 
@@ -152,20 +152,20 @@ subroutine transpose_x_to_y_complex_long(src, dst,decomp)
 #endif
      if (dims(1) == 1) then
 #if defined(_GPU)
-        nsize =  product(decomp%xsz)
+        nsize = product(decomp%xsz)
         !$acc host_data use_device(src,dst)
         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst=src
-#endif       
+        dst = src
+#endif
      else
         call transpose_x_to_y_complex(src, dst, decomp)
-     endif
+     end if
 
   end subroutine transpose_x_to_y_complex_long
-  
+
   subroutine transpose_x_to_y_complex(src, dst, decomp)
 
      implicit none
@@ -208,13 +208,13 @@ subroutine transpose_x_to_y_complex(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_col(work2_c_d,     &
-                                       work1_c_d,     &
+     call decomp_2d_nccl_send_recv_col(work2_c_d, &
+                                       work1_c_d, &
                                        decomp%x1disp, &
                                        decomp%x1cnts, &
-                                       decomp%y1disp, & 
-                                       decomp%y1cnts, &  
-                                       dims(1)      , &
+                                       decomp%y1disp, &
+                                       decomp%y1cnts, &
+                                       dims(1), &
                                        decomp_buf_size)
 #else
      call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index eaf9a14a..db2e439a 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -35,17 +35,17 @@ subroutine transpose_y_to_x_real_long(src, dst, decomp)
 
      if (dims(1) == 1) then
 #if defined(_GPU)
-        nsize =  product(decomp%ysz)
+        nsize = product(decomp%ysz)
         !$acc host_data use_device(src,dst)
         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst=src
-#endif       
+        dst = src
+#endif
      else
         call transpose_y_to_x_real(src, dst, decomp)
-     endif
+     end if
 
   end subroutine transpose_y_to_x_real_long
 
@@ -91,13 +91,13 @@ subroutine transpose_y_to_x_real(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_col(work2_r_d,     &
-                                       work1_r_d,     &
+     call decomp_2d_nccl_send_recv_col(work2_r_d, &
+                                       work1_r_d, &
                                        decomp%y1disp, &
                                        decomp%y1cnts, &
-                                       decomp%x1disp, & 
+                                       decomp%x1disp, &
                                        decomp%x1cnts, &
-                                       dims(1)        )
+                                       dims(1))
 #else
      call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
                         real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
@@ -140,7 +140,7 @@ subroutine transpose_y_to_x_complex_short(src, dst)
 
   end subroutine transpose_y_to_x_complex_short
 
-  subroutine transpose_y_to_x_complex_long(src, dst,decomp)
+  subroutine transpose_y_to_x_complex_long(src, dst, decomp)
 
      implicit none
 
@@ -152,17 +152,17 @@ subroutine transpose_y_to_x_complex_long(src, dst,decomp)
 #endif
      if (dims(1) == 1) then
 #if defined(_GPU)
-        nsize =  product(decomp%ysz)
+        nsize = product(decomp%ysz)
         !$acc host_data use_device(src,dst)
         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst=src
-#endif       
+        dst = src
+#endif
      else
         call transpose_y_to_x_complex(src, dst, decomp)
-     endif
+     end if
 
   end subroutine transpose_y_to_x_complex_long
 
@@ -208,14 +208,14 @@ subroutine transpose_y_to_x_complex(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_col(work2_c_d,     &
-                                       work1_c_d,     &
+     call decomp_2d_nccl_send_recv_col(work2_c_d, &
+                                       work1_c_d, &
                                        decomp%y1disp, &
                                        decomp%y1cnts, &
-                                       decomp%x1disp, & 
+                                       decomp%x1disp, &
                                        decomp%x1cnts, &
-                                       dims(1)      , &
-                                       decomp_buf_size )
+                                       dims(1), &
+                                       decomp_buf_size)
 #else
      call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
                         complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index d64876f8..7d28cb84 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -21,7 +21,7 @@ subroutine transpose_y_to_z_real_short(src, dst)
      call transpose_y_to_z(src, dst, decomp_main)
 
   end subroutine transpose_y_to_z_real_short
-  
+
   subroutine transpose_y_to_z_real_long(src, dst, decomp)
 
      implicit none
@@ -35,17 +35,17 @@ subroutine transpose_y_to_z_real_long(src, dst, decomp)
 
      if (dims(2) == 1) then
 #if defined(_GPU)
-        nsize =  product(decomp%ysz)
+        nsize = product(decomp%ysz)
         !$acc host_data use_device(src,dst)
         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst=src
-#endif       
+        dst = src
+#endif
      else
         call transpose_y_to_z_real(src, dst, decomp)
-     endif
+     end if
 
   end subroutine transpose_y_to_z_real_long
 
@@ -101,13 +101,13 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_row(work2_r_d,     &
-                                       work1_r_d,     &
+     call decomp_2d_nccl_send_recv_row(work2_r_d, &
+                                       work1_r_d, &
                                        decomp%y2disp, &
                                        decomp%y2cnts, &
                                        decomp%z2disp, &
-                                       decomp%z2cnts, & 
-                                       dims(2)        )
+                                       decomp%z2cnts, &
+                                       dims(2))
 #else
      call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
                         real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
@@ -174,17 +174,17 @@ subroutine transpose_y_to_z_complex_long(src, dst, decomp)
 
      if (dims(2) == 1) then
 #if defined(_GPU)
-        nsize =  product(decomp%ysz)
+        nsize = product(decomp%ysz)
         !$acc host_data use_device(src,dst)
         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst=src
-#endif       
+        dst = src
+#endif
      else
         call transpose_y_to_z_complex(src, dst, decomp)
-     endif
+     end if
 
   end subroutine transpose_y_to_z_complex_long
 
@@ -239,13 +239,13 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_row(work2_c_d,     &
-                                       work1_c_d,     &
+     call decomp_2d_nccl_send_recv_row(work2_c_d, &
+                                       work1_c_d, &
                                        decomp%y2disp, &
                                        decomp%y2cnts, &
                                        decomp%z2disp, &
-                                       decomp%z2cnts, & 
-                                       dims(2)      , &
+                                       decomp%z2cnts, &
+                                       dims(2), &
                                        decomp_buf_size)
 #else
      call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index cb25e2ee..c408696f 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -35,20 +35,20 @@ subroutine transpose_z_to_y_real_long(src, dst, decomp)
 
      if (dims(2) == 1) then
 #if defined(_GPU)
-        nsize =  product(decomp%zsz)
+        nsize = product(decomp%zsz)
         !$acc host_data use_device(src,dst)
         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst=src
-#endif       
+        dst = src
+#endif
      else
         call transpose_z_to_y_real(src, dst, decomp)
-     endif
+     end if
 
   end subroutine transpose_z_to_y_real_long
- 
+
   subroutine transpose_z_to_y_real(src, dst, decomp)
 
      implicit none
@@ -110,13 +110,13 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_row(work2_r_d,     &
-                                       work1_r_d,     &
+     call decomp_2d_nccl_send_recv_row(work2_r_d, &
+                                       work1_r_d, &
                                        decomp%z2disp, &
                                        decomp%z2cnts, &
                                        decomp%y2disp, &
                                        decomp%y2cnts, &
-                                       dims(2)        )
+                                       dims(2))
 #else
      call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
                         real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
@@ -171,17 +171,17 @@ subroutine transpose_z_to_y_complex_long(src, dst, decomp)
 
      if (dims(2) == 1) then
 #if defined(_GPU)
-        nsize =  product(decomp%zsz)
+        nsize = product(decomp%zsz)
         !$acc host_data use_device(src,dst)
         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst=src
-#endif       
+        dst = src
+#endif
      else
         call transpose_z_to_y_complex(src, dst, decomp)
-     endif
+     end if
 
   end subroutine transpose_z_to_y_complex_long
 
@@ -246,13 +246,13 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_row(work2_c_d,     &
-                                       work1_c_d,     &
+     call decomp_2d_nccl_send_recv_row(work2_c_d, &
+                                       work1_c_d, &
                                        decomp%z2disp, &
                                        decomp%z2cnts, &
                                        decomp%y2disp, &
                                        decomp%y2cnts, &
-                                       dims(2)      , &
+                                       dims(2), &
                                        decomp_buf_size)
 #else
      call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &

From 16db23379f0de7b2ea30946aeec22ead5e97fceb Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 20 Mar 2023 12:53:27 +0000
Subject: [PATCH 181/436] Run fprettify --enable-replacements --c-relations -w
 3 src/*f90 examples/*/*f90

This ensures multiplication/division operators also get whitespace
---
 examples/fft_physical_x/fft_c2c_x.f90  |  38 +++----
 examples/fft_physical_x/fft_grid_x.f90 |  38 +++----
 examples/fft_physical_x/fft_r2c_x.f90  |  24 ++---
 examples/fft_physical_z/fft_c2c_z.f90  |  38 +++----
 examples/fft_physical_z/fft_r2c_z.f90  |  24 ++---
 examples/halo_test/halo_test.f90       |  14 +--
 examples/init_test/init_test.f90       |  10 +-
 examples/io_test/io_bench.f90          |   8 +-
 examples/io_test/io_plane_test.f90     |  26 ++---
 examples/io_test/io_read.f90           |  10 +-
 examples/io_test/io_test.f90           |  10 +-
 examples/io_test/io_var_test.f90       |  18 ++--
 examples/test2d/test2d.f90             |  18 ++--
 examples/test2d/timing2d_complex.f90   |  26 ++---
 examples/test2d/timing2d_real.f90      |  26 ++---
 src/decomp_2d.f90                      | 136 ++++++++++++-------------
 src/decomp_2d_init_fin.f90             |   6 +-
 src/factor.f90                         |  16 +--
 src/fft_common.f90                     |   4 +-
 src/fft_cufft.f90                      |  98 +++++++++---------
 src/fft_fftw3.f90                      |  48 ++++-----
 src/fft_fftw3_f03.f90                  |  88 ++++++++--------
 src/fft_generic.f90                    |   8 +-
 src/fft_mkl.f90                        |  14 +--
 src/glassman.f90                       |  16 +--
 src/halo_common.f90                    |  22 ++--
 src/io.f90                             |  44 ++++----
 src/io_read_inflow.f90                 |  14 +--
 src/io_write_outflow.f90               |  14 +--
 src/transpose_x_to_y.f90               |  16 +--
 src/transpose_y_to_x.f90               |  16 +--
 src/transpose_y_to_z.f90               |  16 +--
 src/transpose_z_to_y.f90               |  16 +--
 33 files changed, 460 insertions(+), 460 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 1291734a..d692f8b9 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -36,10 +36,10 @@ program fft_c2c_x
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -97,8 +97,8 @@ program fft_c2c_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             in(i, j, k) = cmplx(dr, di, mytype)
          end do
@@ -122,7 +122,7 @@ program fft_c2c_x
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      in = in / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
       !$acc end kernels
 
    end do
@@ -132,10 +132,10 @@ program fft_c2c_x
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -143,31 +143,31 @@ program fft_c2c_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             dr = dr - real(in(i, j, k), mytype)
             di = di - aimag(in(i, j, k))
-            error = error + sqrt(dr*dr + di*di)
+            error = error + sqrt(dr * dr + di * di)
          end do
       end do
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
-      n1 = n1**(1._mytype/3._mytype)
+      n1 = real(nx, mytype) * real(ny, mytype) * real(nz, mytype)
+      n1 = n1**(1._mytype / 3._mytype)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops*3._mytype*n1**2
-      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+      flops = flops * 3._mytype * n1**2
+      flops = 2._mytype * flops / ((t1 + t3) / real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops / 1000._mytype**3
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index d51e6cfc..1410fa15 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -36,10 +36,10 @@ program fft_physical_x
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -98,8 +98,8 @@ program fft_physical_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             in(i, j, k) = cmplx(dr, di, mytype)
          end do
@@ -123,7 +123,7 @@ program fft_physical_x
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      in = in / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
       !$acc end kernels
 
    end do
@@ -133,10 +133,10 @@ program fft_physical_x
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -144,31 +144,31 @@ program fft_physical_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             dr = dr - real(in(i, j, k), mytype)
             di = di - aimag(in(i, j, k))
-            error = error + sqrt(dr*dr + di*di)
+            error = error + sqrt(dr * dr + di * di)
          end do
       end do
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
-      n1 = n1**(1._mytype/3._mytype)
+      n1 = real(nx, mytype) * real(ny, mytype) * real(nz, mytype)
+      n1 = n1**(1._mytype / 3._mytype)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops*3._mytype*n1**2
-      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+      flops = flops * 3._mytype * n1**2
+      flops = 2._mytype * flops / ((t1 + t3) / real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops / 1000._mytype**3
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 9cf55702..a887ef0e 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -37,10 +37,10 @@ program fft_r2c_x
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -99,8 +99,8 @@ program fft_r2c_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            in_r(i, j, k) = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                            / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
          end do
       end do
    end do
@@ -122,7 +122,7 @@ program fft_r2c_x
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in_r = in_r/(real(nx, mytype)*real(ny, mytype)*real(nz, mytype))
+      in_r = in_r / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
       !$acc end kernels
 
    end do
@@ -132,10 +132,10 @@ program fft_r2c_x
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -143,15 +143,15 @@ program fft_r2c_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             error = error + abs(in_r(i, j, k) - dr)
          end do
       end do
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== r2c/c2r interface ====='
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 91ba1cbb..347b5123 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -36,10 +36,10 @@ program fft_c2c_z
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -98,8 +98,8 @@ program fft_c2c_z
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             in(i, j, k) = cmplx(dr, di, mytype)
          end do
@@ -123,7 +123,7 @@ program fft_c2c_z
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      in = in / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
       !$acc end kernels
 
    end do
@@ -133,10 +133,10 @@ program fft_c2c_z
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -144,31 +144,31 @@ program fft_c2c_z
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             dr = dr - real(in(i, j, k), mytype)
             di = di - aimag(in(i, j, k))
-            error = error + sqrt(dr*dr + di*di)
+            error = error + sqrt(dr * dr + di * di)
          end do
       end do
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
-      n1 = n1**(1._mytype/3._mytype)
+      n1 = real(nx, mytype) * real(ny, mytype) * real(nz, mytype)
+      n1 = n1**(1._mytype / 3._mytype)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops*3._mytype*n1**2
-      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+      flops = flops * 3._mytype * n1**2
+      flops = 2._mytype * flops / ((t1 + t3) / real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops / 1000._mytype**3
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 6cfcda39..2798a63a 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -37,10 +37,10 @@ program fft_r2c_z
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -100,8 +100,8 @@ program fft_r2c_z
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            in_r(i, j, k) = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                            / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
          end do
       end do
    end do
@@ -122,7 +122,7 @@ program fft_r2c_z
       t4 = t4 + MPI_WTIME() - t3
 
       !$acc kernels
-      in_r = in_r/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      in_r = in_r / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
       !$acc end kernels
 
    end do
@@ -132,10 +132,10 @@ program fft_r2c_z
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -143,8 +143,8 @@ program fft_r2c_z
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             error = error + abs(in_r(i, j, k) - dr)
             !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
          end do
@@ -154,7 +154,7 @@ program fft_r2c_z
 !10 format('in_r final ', I2,1x,I2,1x,I2,1x,I2,1x,F12.6,1x,F12.6)
 
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== r2c/c2r interface ====='
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index ed3558cc..1eb93d85 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -47,10 +47,10 @@ program halo_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -517,7 +517,7 @@ subroutine check_err(divh, pencil)
       !$acc end kernels
       divmag = mag(tmp)
 
-      if (err < epsilon(divmag)*divmag) then
+      if (err < epsilon(divmag) * divmag) then
          passing = .true.
       else
          passing = .false.
@@ -530,7 +530,7 @@ subroutine check_err(divh, pencil)
 #ifdef DEBUG
          write (*, *) (divh(i, i, i), i=2, 13)
 #endif
-         write (*, *) 'Error: ', err, '; Relative: ', err/divmag
+         write (*, *) 'Error: ', err, '; Relative: ', err / divmag
          write (*, *) 'Pass: ', passing
       end if
       deallocate (tmp)
@@ -567,7 +567,7 @@ real(mytype) function mag(a)
                               "halo_test::mag::MPI_Allreduce")
       end if
 
-      mag = sqrt(gmag/(nx - 2)/(ny - 2)/(nz - 2))
+      mag = sqrt(gmag / (nx - 2) / (ny - 2) / (nz - 2))
 
    end function mag
    !=====================================================================
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 0d6630fb..7f932266 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -30,10 +30,10 @@ program init_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -71,7 +71,7 @@ program init_test
          print *, "will be used"
       end if
    end if
-   nexpect = nx*ny*nz
+   nexpect = nx * ny * nz
    call run(p_row, p_col)
 
    call MPI_Finalize(ierror)
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index 0b37e0d0..34b2d29f 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -29,10 +29,10 @@ program io_bench
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 2c86bf56..0dac2e7f 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -40,10 +40,10 @@ program io_plane_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -124,17 +124,17 @@ program io_plane_test
    !$acc update self(u3)
    !$acc end data
    ! X-pencil data
-   call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 1, nx / 2, '.', 'x_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 2, ny / 2, '.', 'x_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 3, nz / 2, '.', 'x_pencil-z_plane.dat', 'test')
    ! Y-pencil data
-   call decomp_2d_write_plane(2, u2, 2, ny/2, '.', 'y_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(2, u2, 3, nz/2, '.', 'y_pencil-z_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 2, ny / 2, '.', 'y_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 1, nx / 2, '.', 'y_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 3, nz / 2, '.', 'y_pencil-z_plane.dat', 'test')
    ! Z-pencil data
-   call decomp_2d_write_plane(3, u3, 1, nx/2, '.', 'z_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(3, u3, 2, ny/2, '.', 'z_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(3, u3, 3, nz/2, '.', 'z_pencil-z_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 1, nx / 2, '.', 'z_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 2, ny / 2, '.', 'z_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 3, nz / 2, '.', 'z_pencil-z_plane.dat', 'test')
    ! Attemp to read the files
    if (nrank == 0) then
       inquire (iolength=iol) data1(1, 1, 1)
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index d396159a..8b2ddc1c 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -38,10 +38,10 @@ program io_read
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -89,7 +89,7 @@ program io_read
       do j = 1, ny
          do i = 1, nx
 #ifdef COMPLEX_TEST
-            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
+            data1(i, j, k) = cmplx(real(m, mytype), real(nx * ny * nz - m, mytype))
 #else
             data1(i, j, k) = real(m, mytype)
 #endif
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 82b68b80..b4f85195 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -42,10 +42,10 @@ program io_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -93,7 +93,7 @@ program io_test
       do j = 1, ny
          do i = 1, nx
 #ifdef COMPLEX_TEST
-            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
+            data1(i, j, k) = cmplx(real(m, mytype), real(nx * ny * nz - m, mytype))
 #else
             data1(i, j, k) = real(m, mytype)
 #endif
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 1e7e6d4e..1b253c9f 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -57,10 +57,10 @@ program io_var_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -102,12 +102,12 @@ program io_var_test
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! also create a data set over a large domain
-   call decomp_info_init(nx*2, ny*2, nz*2, large)
+   call decomp_info_init(nx * 2, ny * 2, nz * 2, large)
 
    ! initialise global data
    allocate (data1(nx, ny, nz))
    allocate (cdata1(nx, ny, nz))
-   allocate (data1_large(nx*2, ny*2, nz*2))
+   allocate (data1_large(nx * 2, ny * 2, nz * 2))
    m = 1
    do k = 1, nz
       do j = 1, ny
@@ -120,9 +120,9 @@ program io_var_test
    end do
 
    m = 1
-   do k = 1, nz*2
-      do j = 1, ny*2
-         do i = 1, nx*2
+   do k = 1, nz * 2
+      do j = 1, ny * 2
+         do i = 1, nx * 2
             data1_large(i, j, k) = real(m, mytype)
             m = m + 1
          end do
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index e903cd29..1535e944 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -38,10 +38,10 @@ program test2d
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -118,7 +118,7 @@ program test2d
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             u1(i, j, k) = m
          end do
       end do
@@ -160,7 +160,7 @@ program test2d
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
@@ -190,7 +190,7 @@ program test2d
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             if (abs(u3(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
@@ -209,7 +209,7 @@ program test2d
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
@@ -228,7 +228,7 @@ program test2d
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             if (abs(u1(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index dc7362ec..6735e2f9 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -42,10 +42,10 @@ program timing2d_complex
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -122,7 +122,7 @@ program timing2d_complex
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             u1(i, j, k) = cmplx(m, real(m - 1, mytype), kind=mytype)
          end do
       end do
@@ -171,7 +171,7 @@ program timing2d_complex
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
@@ -204,7 +204,7 @@ program timing2d_complex
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u3(i, j, k) - cm) > 0) error_flag = .true.
             end do
@@ -226,7 +226,7 @@ program timing2d_complex
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
@@ -248,7 +248,7 @@ program timing2d_complex
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u1(i, j, k) - cm) > 0) error_flag = .true.
             end do
@@ -262,16 +262,16 @@ program timing2d_complex
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
    call MPI_ALLREDUCE(t6, t5, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t5/dble(nproc)
+   t5 = t5 / dble(nproc)
    call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t7/dble(nproc)
+   t7 = t7 / dble(nproc)
    t8 = t1 + t3 + t5 + t7
    if (nrank == 0) then
       write (*, *) 'Time X->Y ', t1
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 9e1c007b..1f9e216b 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -41,10 +41,10 @@ program timing2d_real
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -121,7 +121,7 @@ program timing2d_real
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             u1(i, j, k) = m
          end do
       end do
@@ -170,7 +170,7 @@ program timing2d_real
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -202,7 +202,7 @@ program timing2d_real
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                if (abs(u3(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -223,7 +223,7 @@ program timing2d_real
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -244,7 +244,7 @@ program timing2d_real
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                if (abs(u1(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -257,16 +257,16 @@ program timing2d_real
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
    call MPI_ALLREDUCE(t6, t5, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t5/dble(nproc)
+   t5 = t5 / dble(nproc)
    call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t7/dble(nproc)
+   t7 = t7 / dble(nproc)
    t8 = t1 + t3 + t5 + t7
    if (nrank == 0) then
       write (*, *) 'Time X->Y ', t1
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 2a505f8d..8f9d6ea8 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -403,13 +403,13 @@ subroutine decomp_info_init(nx, ny, nz, decomp)
       ! allocate memory for the MPI_ALLTOALL(V) buffers
       ! define the buffers globally for performance reason
 
-      buf_size = max(decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3), &
-                     max(decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3), &
-                         decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)))
+      buf_size = max(decomp%xsz(1) * decomp%xsz(2) * decomp%xsz(3), &
+                     max(decomp%ysz(1) * decomp%ysz(2) * decomp%ysz(3), &
+                         decomp%zsz(1) * decomp%zsz(2) * decomp%zsz(3)))
 #ifdef EVEN
       ! padded alltoall optimisation may need larger buffer space
       buf_size = max(buf_size, &
-                     max(decomp%x1count*dims(1), decomp%y2count*dims(2)))
+                     max(decomp%x1count * dims(1), decomp%y2count * dims(2)))
 #endif
 
       ! check if additional memory is required
@@ -502,39 +502,39 @@ subroutine init_coarser_mesh_statS(i_skip, j_skip, k_skip, from1)
 
       do i = 1, 3
          if (from1) then
-            xstS(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            xstS(i) = (xstart(i) + skip(i) - 1) / skip(i)
             if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstS(i) = xstS(i) + 1
-            xenS(i) = (xend(i) + skip(i) - 1)/skip(i)
+            xenS(i) = (xend(i) + skip(i) - 1) / skip(i)
          else
-            xstS(i) = xstart(i)/skip(i)
+            xstS(i) = xstart(i) / skip(i)
             if (mod(xstart(i), skip(i)) /= 0) xstS(i) = xstS(i) + 1
-            xenS(i) = xend(i)/skip(i)
+            xenS(i) = xend(i) / skip(i)
          end if
          xszS(i) = xenS(i) - xstS(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            ystS(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            ystS(i) = (ystart(i) + skip(i) - 1) / skip(i)
             if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystS(i) = ystS(i) + 1
-            yenS(i) = (yend(i) + skip(i) - 1)/skip(i)
+            yenS(i) = (yend(i) + skip(i) - 1) / skip(i)
          else
-            ystS(i) = ystart(i)/skip(i)
+            ystS(i) = ystart(i) / skip(i)
             if (mod(ystart(i), skip(i)) /= 0) ystS(i) = ystS(i) + 1
-            yenS(i) = yend(i)/skip(i)
+            yenS(i) = yend(i) / skip(i)
          end if
          yszS(i) = yenS(i) - ystS(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            zstS(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            zstS(i) = (zstart(i) + skip(i) - 1) / skip(i)
             if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstS(i) = zstS(i) + 1
-            zenS(i) = (zend(i) + skip(i) - 1)/skip(i)
+            zenS(i) = (zend(i) + skip(i) - 1) / skip(i)
          else
-            zstS(i) = zstart(i)/skip(i)
+            zstS(i) = zstart(i) / skip(i)
             if (mod(zstart(i), skip(i)) /= 0) zstS(i) = zstS(i) + 1
-            zenS(i) = zend(i)/skip(i)
+            zenS(i) = zend(i) / skip(i)
          end if
          zszS(i) = zenS(i) - zstS(i) + 1
       end do
@@ -567,39 +567,39 @@ subroutine init_coarser_mesh_statV(i_skip, j_skip, k_skip, from1)
 
       do i = 1, 3
          if (from1) then
-            xstV(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            xstV(i) = (xstart(i) + skip(i) - 1) / skip(i)
             if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstV(i) = xstV(i) + 1
-            xenV(i) = (xend(i) + skip(i) - 1)/skip(i)
+            xenV(i) = (xend(i) + skip(i) - 1) / skip(i)
          else
-            xstV(i) = xstart(i)/skip(i)
+            xstV(i) = xstart(i) / skip(i)
             if (mod(xstart(i), skip(i)) /= 0) xstV(i) = xstV(i) + 1
-            xenV(i) = xend(i)/skip(i)
+            xenV(i) = xend(i) / skip(i)
          end if
          xszV(i) = xenV(i) - xstV(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            ystV(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            ystV(i) = (ystart(i) + skip(i) - 1) / skip(i)
             if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystV(i) = ystV(i) + 1
-            yenV(i) = (yend(i) + skip(i) - 1)/skip(i)
+            yenV(i) = (yend(i) + skip(i) - 1) / skip(i)
          else
-            ystV(i) = ystart(i)/skip(i)
+            ystV(i) = ystart(i) / skip(i)
             if (mod(ystart(i), skip(i)) /= 0) ystV(i) = ystV(i) + 1
-            yenV(i) = yend(i)/skip(i)
+            yenV(i) = yend(i) / skip(i)
          end if
          yszV(i) = yenV(i) - ystV(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            zstV(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            zstV(i) = (zstart(i) + skip(i) - 1) / skip(i)
             if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstV(i) = zstV(i) + 1
-            zenV(i) = (zend(i) + skip(i) - 1)/skip(i)
+            zenV(i) = (zend(i) + skip(i) - 1) / skip(i)
          else
-            zstV(i) = zstart(i)/skip(i)
+            zstV(i) = zstart(i) / skip(i)
             if (mod(zstart(i), skip(i)) /= 0) zstV(i) = zstV(i) + 1
-            zenV(i) = zend(i)/skip(i)
+            zenV(i) = zend(i) / skip(i)
          end if
          zszV(i) = zenV(i) - zstV(i) + 1
       end do
@@ -632,39 +632,39 @@ subroutine init_coarser_mesh_statP(i_skip, j_skip, k_skip, from1)
 
       do i = 1, 3
          if (from1) then
-            xstP(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            xstP(i) = (xstart(i) + skip(i) - 1) / skip(i)
             if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstP(i) = xstP(i) + 1
-            xenP(i) = (xend(i) + skip(i) - 1)/skip(i)
+            xenP(i) = (xend(i) + skip(i) - 1) / skip(i)
          else
-            xstP(i) = xstart(i)/skip(i)
+            xstP(i) = xstart(i) / skip(i)
             if (mod(xstart(i), skip(i)) /= 0) xstP(i) = xstP(i) + 1
-            xenP(i) = xend(i)/skip(i)
+            xenP(i) = xend(i) / skip(i)
          end if
          xszP(i) = xenP(i) - xstP(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            ystP(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            ystP(i) = (ystart(i) + skip(i) - 1) / skip(i)
             if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystP(i) = ystP(i) + 1
-            yenP(i) = (yend(i) + skip(i) - 1)/skip(i)
+            yenP(i) = (yend(i) + skip(i) - 1) / skip(i)
          else
-            ystP(i) = ystart(i)/skip(i)
+            ystP(i) = ystart(i) / skip(i)
             if (mod(ystart(i), skip(i)) /= 0) ystP(i) = ystP(i) + 1
-            yenP(i) = yend(i)/skip(i)
+            yenP(i) = yend(i) / skip(i)
          end if
          yszP(i) = yenP(i) - ystP(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            zstP(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            zstP(i) = (zstart(i) + skip(i) - 1) / skip(i)
             if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstP(i) = zstP(i) + 1
-            zenP(i) = (zend(i) + skip(i) - 1)/skip(i)
+            zenP(i) = (zend(i) + skip(i) - 1) / skip(i)
          else
-            zstP(i) = zstart(i)/skip(i)
+            zstP(i) = zstart(i) / skip(i)
             if (mod(zstart(i), skip(i)) /= 0) zstP(i) = zstP(i) + 1
-            zenP(i) = zend(i)/skip(i)
+            zenP(i) = zend(i) / skip(i)
          end if
          zszP(i) = zenP(i) - zstP(i) + 1
       end do
@@ -692,7 +692,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = xstS(3), xenS(3)
                do j = xstS(2), xenS(2)
                   do i = xstS(1), xenS(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipS + 1, (j - 1) * jskipS + 1, (k - 1) * kskipS + 1)
                   end do
                end do
             end do
@@ -700,7 +700,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = xstS(3), xenS(3)
                do j = xstS(2), xenS(2)
                   do i = xstS(1), xenS(1)
-                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                     wk(i, j, k) = wk2(i * iskipS, j * jskipS, k * kskipS)
                   end do
                end do
             end do
@@ -714,7 +714,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = ystS(3), yenS(3)
                do j = ystS(2), yenS(2)
                   do i = ystS(1), yenS(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipS + 1, (j - 1) * jskipS + 1, (k - 1) * kskipS + 1)
                   end do
                end do
             end do
@@ -722,7 +722,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = ystS(3), yenS(3)
                do j = ystS(2), yenS(2)
                   do i = ystS(1), yenS(1)
-                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                     wk(i, j, k) = wk2(i * iskipS, j * jskipS, k * kskipS)
                   end do
                end do
             end do
@@ -736,7 +736,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = zstS(3), zenS(3)
                do j = zstS(2), zenS(2)
                   do i = zstS(1), zenS(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipS + 1, (j - 1) * jskipS + 1, (k - 1) * kskipS + 1)
                   end do
                end do
             end do
@@ -744,7 +744,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = zstS(3), zenS(3)
                do j = zstS(2), zenS(2)
                   do i = zstS(1), zenS(1)
-                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                     wk(i, j, k) = wk2(i * iskipS, j * jskipS, k * kskipS)
                   end do
                end do
             end do
@@ -777,7 +777,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = xstV(3), xenV(3)
                do j = xstV(2), xenV(2)
                   do i = xstV(1), xenV(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipV + 1, (j - 1) * jskipV + 1, (k - 1) * kskipV + 1)
                   end do
                end do
             end do
@@ -785,7 +785,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = xstV(3), xenV(3)
                do j = xstV(2), xenV(2)
                   do i = xstV(1), xenV(1)
-                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                     wk(i, j, k) = wk2(i * iskipV, j * jskipV, k * kskipV)
                   end do
                end do
             end do
@@ -799,7 +799,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = ystV(3), yenV(3)
                do j = ystV(2), yenV(2)
                   do i = ystV(1), yenV(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipV + 1, (j - 1) * jskipV + 1, (k - 1) * kskipV + 1)
                   end do
                end do
             end do
@@ -807,7 +807,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = ystV(3), yenV(3)
                do j = ystV(2), yenV(2)
                   do i = ystV(1), yenV(1)
-                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                     wk(i, j, k) = wk2(i * iskipV, j * jskipV, k * kskipV)
                   end do
                end do
             end do
@@ -821,7 +821,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = zstV(3), zenV(3)
                do j = zstV(2), zenV(2)
                   do i = zstV(1), zenV(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipV + 1, (j - 1) * jskipV + 1, (k - 1) * kskipV + 1)
                   end do
                end do
             end do
@@ -829,7 +829,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = zstV(3), zenV(3)
                do j = zstV(2), zenV(2)
                   do i = zstV(1), zenV(1)
-                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                     wk(i, j, k) = wk2(i * iskipV, j * jskipV, k * kskipV)
                   end do
                end do
             end do
@@ -862,7 +862,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = xstP(3), xenP(3)
                do j = xstP(2), xenP(2)
                   do i = xstP(1), xenP(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipP + 1, (j - 1) * jskipP + 1, (k - 1) * kskipP + 1)
                   end do
                end do
             end do
@@ -870,7 +870,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = xstP(3), xenP(3)
                do j = xstP(2), xenP(2)
                   do i = xstP(1), xenP(1)
-                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                     wk(i, j, k) = wk2(i * iskipP, j * jskipP, k * kskipP)
                   end do
                end do
             end do
@@ -884,7 +884,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = ystP(3), yenP(3)
                do j = ystP(2), yenP(2)
                   do i = ystP(1), yenP(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipP + 1, (j - 1) * jskipP + 1, (k - 1) * kskipP + 1)
                   end do
                end do
             end do
@@ -892,7 +892,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = ystP(3), yenP(3)
                do j = ystP(2), yenP(2)
                   do i = ystP(1), yenP(1)
-                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                     wk(i, j, k) = wk2(i * iskipP, j * jskipP, k * kskipP)
                   end do
                end do
             end do
@@ -906,7 +906,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = zstP(3), zenP(3)
                do j = zstP(2), zenP(2)
                   do i = zstP(1), zenP(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipP + 1, (j - 1) * jskipP + 1, (k - 1) * kskipP + 1)
                   end do
                end do
             end do
@@ -914,7 +914,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = zstP(3), zenP(3)
                do j = zstP(2), zenP(2)
                   do i = zstP(1), zenP(1)
-                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                     wk(i, j, k) = wk2(i * iskipP, j * jskipP, k * kskipP)
                   end do
                end do
             end do
@@ -1005,8 +1005,8 @@ subroutine distribute(data1, proc, st, en, sz)
       integer data1, proc, st(0:proc - 1), en(0:proc - 1), sz(0:proc - 1)
       integer i, size1, nl, nu
 
-      size1 = data1/proc
-      nu = data1 - size1*proc
+      size1 = data1 / proc
+      nu = data1 - size1 * proc
       nl = proc - nu
       st(0) = 1
       sz(0) = size1
@@ -1082,8 +1082,8 @@ subroutine prepare_buffer(decomp)
       ! MPI_ALLTOALLV buffer information
 
       do i = 0, dims(1) - 1
-         decomp%x1cnts(i) = decomp%x1dist(i)*decomp%xsz(2)*decomp%xsz(3)
-         decomp%y1cnts(i) = decomp%ysz(1)*decomp%y1dist(i)*decomp%ysz(3)
+         decomp%x1cnts(i) = decomp%x1dist(i) * decomp%xsz(2) * decomp%xsz(3)
+         decomp%y1cnts(i) = decomp%ysz(1) * decomp%y1dist(i) * decomp%ysz(3)
          if (i == 0) then
             decomp%x1disp(i) = 0  ! displacement is 0-based index
             decomp%y1disp(i) = 0
@@ -1094,8 +1094,8 @@ subroutine prepare_buffer(decomp)
       end do
 
       do i = 0, dims(2) - 1
-         decomp%y2cnts(i) = decomp%ysz(1)*decomp%y2dist(i)*decomp%ysz(3)
-         decomp%z2cnts(i) = decomp%zsz(1)*decomp%zsz(2)*decomp%z2dist(i)
+         decomp%y2cnts(i) = decomp%ysz(1) * decomp%y2dist(i) * decomp%ysz(3)
+         decomp%z2cnts(i) = decomp%zsz(1) * decomp%zsz(2) * decomp%z2dist(i)
          if (i == 0) then
             decomp%y2disp(i) = 0  ! displacement is 0-based index
             decomp%z2disp(i) = 0
@@ -1117,12 +1117,12 @@ subroutine prepare_buffer(decomp)
       ! For unevenly distributed data, pad smaller messages. Note the
       ! last blocks along pencils always get assigned more mesh points
       ! for X <=> Y transposes
-      decomp%x1count = decomp%x1dist(dims(1) - 1)* &
-                       decomp%y1dist(dims(1) - 1)*decomp%xsz(3)
+      decomp%x1count = decomp%x1dist(dims(1) - 1) * &
+                       decomp%y1dist(dims(1) - 1) * decomp%xsz(3)
       decomp%y1count = decomp%x1count
       ! for Y <=> Z transposes
-      decomp%y2count = decomp%y2dist(dims(2) - 1)* &
-                       decomp%z2dist(dims(2) - 1)*decomp%zsz(1)
+      decomp%y2count = decomp%y2dist(dims(2) - 1) * &
+                       decomp%z2dist(dims(2) - 1) * decomp%zsz(1)
       decomp%z2count = decomp%y2count
 
       return
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index 7a8bd916..ff4c6c5e 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -74,7 +74,7 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
         p_row = row
         p_col = col
      else
-        if (nproc /= p_row*p_col) then
+        if (nproc /= p_row * p_col) then
            errorcode = 1
            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
                                 'Invalid 2D processor grid - nproc /= p_row*p_col')
@@ -268,11 +268,11 @@ subroutine best_2d_grid(iproc, best_p_row, best_p_col)
      call findfactor(iproc, factors, nfact)
      if (nrank == 0) write (*, *) 'factors: ', (factors(i), i=1, nfact)
 
-     i_best = nfact/2 + 1
+     i_best = nfact / 2 + 1
      col = factors(i_best)
 
      best_p_col = col
-     best_p_row = iproc/col
+     best_p_row = iproc / col
      if (nrank == 0) print *, 'p_row x p_col', best_p_row, best_p_col
      if ((best_p_col == 1) .and. (nrank == 0)) then
         print *, 'WARNING: current 2D DECOMP set-up might not work'
diff --git a/src/factor.f90 b/src/factor.f90
index 5f7e530a..941c8378 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -35,7 +35,7 @@ subroutine findfactor(num, factors, nfact)
       m = int(sqrt(real(num)))
       nfact = 1
       do i = 1, m
-         if (num/i*i == num) then
+         if (num / i * i == num) then
             factors(nfact) = i
             nfact = nfact + 1
          end if
@@ -44,15 +44,15 @@ subroutine findfactor(num, factors, nfact)
 
       ! derive those > sqrt(num)
       if (factors(nfact)**2 /= num) then
-         do i = nfact + 1, 2*nfact
-            factors(i) = num/factors(2*nfact - i + 1)
+         do i = nfact + 1, 2 * nfact
+            factors(i) = num / factors(2 * nfact - i + 1)
          end do
-         nfact = nfact*2
+         nfact = nfact * 2
       else
-         do i = nfact + 1, 2*nfact - 1
-            factors(i) = num/factors(2*nfact - i)
+         do i = nfact + 1, 2 * nfact - 1
+            factors(i) = num / factors(2 * nfact - i)
          end do
-         nfact = nfact*2 - 1
+         nfact = nfact * 2 - 1
       end if
 
       return
@@ -76,7 +76,7 @@ subroutine primefactors(num, factors, nfact)
          if (mod(n, i) == 0) then
             factors(nfact) = i
             nfact = nfact + 1
-            n = n/i
+            n = n / i
          else
             i = i + 1
          end if
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index f8ae3fe9..34632a40 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -128,9 +128,9 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       call decomp_info_init(nx, ny, nz, ph)
    end if
    if (format == PHYSICAL_IN_X) then
-      call decomp_info_init(nx/2 + 1, ny, nz, sp)
+      call decomp_info_init(nx / 2 + 1, ny, nz, sp)
    else if (format == PHYSICAL_IN_Z) then
-      call decomp_info_init(nx, ny, nz/2 + 1, sp)
+      call decomp_info_init(nx, ny, nz / 2 + 1, sp)
    else
       call decomp_2d_abort(__FILE__, __LINE__, format, "Invalid value for format")
    end if
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 6f08dfe3..1776d601 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -65,7 +65,7 @@ subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
       istat = cufftMakePlanMany(plan1, 1, decomp%xsz(1), &
                                 decomp%xsz(1), 1, decomp%xsz(1), &
                                 decomp%xsz(1), 1, decomp%xsz(1), &
-                                cufft_type, decomp%xsz(2)*decomp%xsz(3), worksize)
+                                cufft_type, decomp%xsz(2) * decomp%xsz(3), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine c2c_1m_x_plan
@@ -117,9 +117,9 @@ subroutine c2c_1m_z_plan(plan1, decomp, cufft_type, worksize)
       istat = cufftSetAutoAllocation(plan1, 0)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
       istat = cufftMakePlanMany(plan1, 1, decomp%zsz(3), &
-                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
-                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
-                                cufft_type, decomp%zsz(1)*decomp%zsz(2), worksize)
+                                decomp%zsz(3), decomp%zsz(1) * decomp%zsz(2), 1, &
+                                decomp%zsz(3), decomp%zsz(1) * decomp%zsz(2), 1, &
+                                cufft_type, decomp%zsz(1) * decomp%zsz(2), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine c2c_1m_z_plan
@@ -146,7 +146,7 @@ subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
       istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
                                 decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                 decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
+                                cufft_type, decomp_ph%xsz(2) * decomp_ph%xsz(3), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine r2c_1m_x_plan
@@ -173,7 +173,7 @@ subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
       istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
                                 decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                 decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
+                                cufft_type, decomp_ph%xsz(2) * decomp_ph%xsz(3), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine c2r_1m_x_plan
@@ -198,9 +198,9 @@ subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
       istat = cufftSetAutoAllocation(plan1, 0)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
       istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
-                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
-                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
-                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
+                                decomp_ph%zsz(3), decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, &
+                                decomp_sp%zsz(3), decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, &
+                                cufft_type, decomp_ph%zsz(1) * decomp_ph%zsz(2), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine r2c_1m_z_plan
@@ -225,9 +225,9 @@ subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
       istat = cufftSetAutoAllocation(plan1, 0)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
       istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
-                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
-                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
-                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
+                                decomp_sp%zsz(3), decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, &
+                                decomp_ph%zsz(3), decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, &
+                                cufft_type, decomp_ph%zsz(1) * decomp_ph%zsz(2), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine c2r_1m_z_plan
@@ -369,7 +369,7 @@ subroutine init_fft_engine
 
       end if
 #endif
-      cufft_ws = cufft_ws/sizeof(1._mytype)
+      cufft_ws = cufft_ws / sizeof(1._mytype)
       allocate (cufft_workspace(cufft_ws))
       do j = 1, 3
          do i = -1, 2
@@ -732,9 +732,9 @@ subroutine fft_3d_r2c(in_r, out_c)
          !call nvtxStartRange("Z r2c_1m_z")
 #ifdef DEBUG
          dim3d = shape(in_r)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_r(i, j, k))
                end do
             end do
@@ -753,9 +753,9 @@ subroutine fft_3d_r2c(in_r, out_c)
 #ifdef DEBUG
             write (*, *) 'c2c_1m_y'
             dim3d = shape(wk2_r2c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
                         aimag(wk2_r2c(i, j, k))
                   end do
@@ -774,9 +774,9 @@ subroutine fft_3d_r2c(in_r, out_c)
 #ifdef DEBUG
             write (*, *) 'c2c_1m_y2'
             dim3d = shape(out_c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
                         aimag(out_c(i, j, k))
                   end do
@@ -799,9 +799,9 @@ subroutine fft_3d_r2c(in_r, out_c)
 #ifdef DEBUG
          write (*, *) 'c2c_1m_x'
          dim3d = shape(out_c)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
                      aimag(out_c(i, j, k))
                end do
@@ -874,9 +874,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back Init c2c_1m_x line 788'
          dim3d = shape(in_c)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
                      aimag(in_c(i, j, k))
                end do
@@ -891,9 +891,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back c2c_1m_x overwrite line 804'
          dim3d = shape(in_c)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
                      aimag(in_c(i, j, k))
                end do
@@ -913,9 +913,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back2 c2c_1m_x line 821'
          dim3d = shape(wk1)
-         do k = 1, dim3d(3), dim3d(1)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(1) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
                      aimag(wk1(i, j, k))
                end do
@@ -937,9 +937,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
             write (*, *) 'Back c2c_1m_y line 844'
             dim3d = shape(wk2_r2c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
                         aimag(wk2_r2c(i, j, k))
                   end do
@@ -954,9 +954,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
             write (*, *) 'Back2 c2c_1m_y line 860'
             dim3d = shape(in_c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
                         aimag(in_c(i, j, k))
                   end do
@@ -970,9 +970,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
             write (*, *) 'Back3 c2c_1m_y line 875'
             dim3d = shape(wk1)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
                         aimag(wk1(i, j, k))
                   end do
@@ -997,9 +997,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back2 after tr_y2z'
          dim3d = shape(wk13)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk13(i, j, k)), &
                      aimag(wk13(i, j, k))
                end do
@@ -1012,9 +1012,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back2 c2r_1m_z out_r line 902'
          dim3d = shape(out_r)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_r(i, j, k))
                end do
             end do
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index a9dc689e..ced99dbd 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -65,12 +65,12 @@ subroutine c2c_1m_x_plan(plan1, decomp, isign)
 
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                               decomp%xsz(2) * decomp%xsz(3), a1, decomp%xsz(1), 1, &
                                decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
                                isign, plan_type)
 #else
       call sfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                               decomp%xsz(2) * decomp%xsz(3), a1, decomp%xsz(1), 1, &
                                decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
                                isign, plan_type)
 #endif
@@ -126,14 +126,14 @@ subroutine c2c_1m_z_plan(plan1, decomp, isign)
 
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+                               decomp%zsz(1) * decomp%zsz(2), a1, decomp%zsz(3), &
+                               decomp%zsz(1) * decomp%zsz(2), 1, a1, decomp%zsz(3), &
+                               decomp%zsz(1) * decomp%zsz(2), 1, isign, plan_type)
 #else
       call sfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+                               decomp%zsz(1) * decomp%zsz(2), a1, decomp%zsz(3), &
+                               decomp%zsz(1) * decomp%zsz(2), 1, a1, decomp%zsz(3), &
+                               decomp%zsz(1) * decomp%zsz(2), 1, isign, plan_type)
 #endif
 
       deallocate (a1)
@@ -157,12 +157,12 @@ subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
       allocate (a2(decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)))
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                   decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
                                    decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                    plan_type)
 #else
       call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                   decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
                                    decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                    plan_type)
 #endif
@@ -187,12 +187,12 @@ subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
       allocate (a2(decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)))
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                   decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
                                    decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                    plan_type)
 #else
       call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                   decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
                                    decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                    plan_type)
 #endif
@@ -217,14 +217,14 @@ subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
       allocate (a2(decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)))
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, plan_type)
 #else
       call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, plan_type)
 #endif
       deallocate (a1, a2)
 
@@ -248,14 +248,14 @@ subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
 
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, plan_type)
 #else
       call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, plan_type)
 #endif
       deallocate (a1, a2)
 
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index d06475d1..4f0ceb46 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -156,9 +156,9 @@ subroutine fft_init_general(pencil, nx, ny, nz)
          call decomp_info_init(nx, ny, nz, ph)
       end if
       if (format == PHYSICAL_IN_X) then
-         call decomp_info_init(nx/2 + 1, ny, nz, sp)
+         call decomp_info_init(nx / 2 + 1, ny, nz, sp)
       else if (format == PHYSICAL_IN_Z) then
-         call decomp_info_init(nx, ny, nz/2 + 1, sp)
+         call decomp_info_init(nx, ny, nz / 2 + 1, sp)
       else
          call decomp_2d_abort(__FILE__, __LINE__, format, "Invalid value for format")
       end if
@@ -167,7 +167,7 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       ! Allocate the workspace fo intermediate y-pencil data
       ! The largest memory block needed is the one for c2c transforms
       !
-      sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)
+      sz = ph%ysz(1) * ph%ysz(2) * ph%ysz(3)
       wk2_c2c_p = fftw_alloc_complex(sz)
       call c_f_pointer(wk2_c2c_p, wk2_c2c, [ph%ysz(1), ph%ysz(2), ph%ysz(3)])
       !
@@ -186,11 +186,11 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       ! transpose_y_to_z(wk2_r2c, wk13, sp)
       !
       if (format == PHYSICAL_IN_X) then
-         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+         sz = sp%xsz(1) * sp%xsz(2) * sp%xsz(3)
          wk13_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk13_p, wk13, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
       else if (format == PHYSICAL_IN_Z) then
-         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+         sz = sp%zsz(1) * sp%zsz(2) * sp%zsz(3)
          wk13_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk13_p, wk13, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
       end if
@@ -319,19 +319,19 @@ subroutine c2c_1m_x_plan(plan1, decomp, isign)
       type(C_PTR) :: a1_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)
+      sz = decomp%xsz(1) * decomp%xsz(2) * decomp%xsz(3)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
       call c_f_pointer(a1_p, a1o, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft(1, decomp%xsz(1), &
-                                 decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                                 decomp%xsz(2) * decomp%xsz(3), a1, decomp%xsz(1), 1, &
                                  decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
                                  isign, plan_type)
 #else
       plan1 = fftwf_plan_many_dft(1, decomp%xsz(1), &
-                                  decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                                  decomp%xsz(2) * decomp%xsz(3), a1, decomp%xsz(1), 1, &
                                   decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
                                   isign, plan_type)
 #endif
@@ -362,7 +362,7 @@ subroutine c2c_1m_y_plan(plan1, decomp, isign)
 
       ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
       ! done one Z-plane at a time. So plan for 2D data sets here.
-      sz = decomp%ysz(1)*decomp%ysz(2)
+      sz = decomp%ysz(1) * decomp%ysz(2)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, [decomp%ysz(1), decomp%ysz(2)])
       call c_f_pointer(a1_p, a1o, [decomp%ysz(1), decomp%ysz(2)])
@@ -401,21 +401,21 @@ subroutine c2c_1m_z_plan(plan1, decomp, isign)
       type(C_PTR) :: a1_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)
+      sz = decomp%zsz(1) * decomp%zsz(2) * decomp%zsz(3)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
       call c_f_pointer(a1_p, a1o, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft(1, decomp%zsz(3), &
-                                 decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                                 decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-                                 decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+                                 decomp%zsz(1) * decomp%zsz(2), a1, decomp%zsz(3), &
+                                 decomp%zsz(1) * decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+                                 decomp%zsz(1) * decomp%zsz(2), 1, isign, plan_type)
 #else
       plan1 = fftwf_plan_many_dft(1, decomp%zsz(3), &
-                                  decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                                  decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-                                  decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+                                  decomp%zsz(1) * decomp%zsz(2), a1, decomp%zsz(3), &
+                                  decomp%zsz(1) * decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+                                  decomp%zsz(1) * decomp%zsz(2), 1, isign, plan_type)
 #endif
 
       call fftw_free(a1_p)
@@ -437,23 +437,23 @@ subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
       type(C_PTR) :: a1_p, a2_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+      sz = decomp_ph%xsz(1) * decomp_ph%xsz(2) * decomp_ph%xsz(3)
       a1_p = fftw_alloc_real(sz)
       call c_f_pointer(a1_p, a1, &
                        [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
-      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+      sz = decomp_sp%xsz(1) * decomp_sp%xsz(2) * decomp_sp%xsz(3)
       a2_p = fftw_alloc_complex(sz)
       call c_f_pointer(a2_p, a2, &
                        [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                     decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
                                      decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                      plan_type)
 #else
       plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                      decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
                                       decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                       plan_type)
 #endif
@@ -478,23 +478,23 @@ subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
       type(C_PTR) :: a1_p, a2_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+      sz = decomp_sp%xsz(1) * decomp_sp%xsz(2) * decomp_sp%xsz(3)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, &
                        [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
-      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+      sz = decomp_ph%xsz(1) * decomp_ph%xsz(2) * decomp_ph%xsz(3)
       a2_p = fftw_alloc_real(sz)
       call c_f_pointer(a2_p, a2, &
                        [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                     decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
                                      decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                      plan_type)
 #else
       plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                      decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
                                       decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                       plan_type)
 #endif
@@ -519,25 +519,25 @@ subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
       type(C_PTR) :: a1_p, a2_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+      sz = decomp_ph%zsz(1) * decomp_ph%zsz(2) * decomp_ph%zsz(3)
       a1_p = fftw_alloc_real(sz)
       call c_f_pointer(a1_p, a1, &
                        [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
-      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+      sz = decomp_sp%zsz(1) * decomp_sp%zsz(2) * decomp_sp%zsz(3)
       a2_p = fftw_alloc_complex(sz)
       call c_f_pointer(a2_p, a2, &
                        [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+                                     decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                     decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, plan_type)
 #else
       plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+                                      decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                      decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, plan_type)
 #endif
 
       call fftw_free(a1_p)
@@ -560,25 +560,25 @@ subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
       type(C_PTR) :: a1_p, a2_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+      sz = decomp_sp%zsz(1) * decomp_sp%zsz(2) * decomp_sp%zsz(3)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, &
                        [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
-      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+      sz = decomp_ph%zsz(1) * decomp_ph%zsz(2) * decomp_ph%zsz(3)
       a2_p = fftw_alloc_real(sz)
       call c_f_pointer(a2_p, a2, &
                        [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+                                     decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                     decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, plan_type)
 #else
       plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+                                      decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                      decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, plan_type)
 #endif
 
       call fftw_free(a1_p)
@@ -823,7 +823,7 @@ subroutine fft_3d_c2c(in, out, isign)
 #ifdef OVERWRITE
          call c2c_1m_x(in, plan(isign, 1))
 #else
-         sz = ph%xsz(1)*ph%xsz(2)*ph%xsz(3)
+         sz = ph%xsz(1) * ph%xsz(2) * ph%xsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [ph%xsz(1), ph%xsz(2), ph%xsz(3)])
          wk1 = in
@@ -867,7 +867,7 @@ subroutine fft_3d_c2c(in, out, isign)
 #ifdef OVERWRITE
          call c2c_1m_z(in, plan(isign, 3))
 #else
-         sz = ph%zsz(1)*ph%zsz(2)*ph%zsz(3)
+         sz = ph%zsz(1) * ph%zsz(2) * ph%zsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [ph%zsz(1), ph%zsz(2), ph%zsz(3)])
          wk1 = in
@@ -1003,7 +1003,7 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef OVERWRITE
          call c2c_1m_z(in_c, plan(2, 3))
 #else
-         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+         sz = sp%zsz(1) * sp%zsz(2) * sp%zsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
          wk1 = in_c
@@ -1032,7 +1032,7 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef OVERWRITE
          call c2c_1m_x(in_c, plan(2, 1))
 #else
-         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+         sz = sp%xsz(1) * sp%xsz(2) * sp%xsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
          wk1 = in_c
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index cff64e98..c07c52e2 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -260,7 +260,7 @@ subroutine c2r_1m_x(input, output)
          do j = 1, d2
             ! Glassman's FFT is c2c only,
             ! needing some pre- and post-processing for c2r
-            do i = 1, d1/2 + 1
+            do i = 1, d1 / 2 + 1
                buf(i) = input(i, j, k)
             end do
             ! expanding to a full-size complex array
@@ -270,7 +270,7 @@ subroutine c2r_1m_x(input, output)
             ! For even N, the storage is:
             !  1, 2, ...... N/2  , N/2+1
             !     N, ...... N/2+2  again a(i) conjugate of a(N+2-i)
-            do i = d1/2 + 2, d1
+            do i = d1 / 2 + 2, d1
                buf(i) = conjg(buf(d1 + 2 - i))
             end do
             call spcfft(buf, d1, 1, scratch)
@@ -305,10 +305,10 @@ subroutine c2r_1m_z(input, output)
       !$acc parallel loop gang vector collapse(2) private(buf, scratch)
       do j = 1, d2
          do i = 1, d1
-            do k = 1, d3/2 + 1
+            do k = 1, d3 / 2 + 1
                buf(k) = input(i, j, k)
             end do
-            do k = d3/2 + 2, d3
+            do k = d3 / 2 + 2, d3
                buf(k) = conjg(buf(d3 + 2 - k))
             end do
             call spcfft(buf, d3, 1, scratch)
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index be4d419d..cf63924d 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -90,7 +90,7 @@ subroutine c2c_1m_x_plan(desc, decomp)
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp%xsz(2)*decomp%xsz(3))
+                            decomp%xsz(2) * decomp%xsz(3))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
 #ifdef OVERWRITE
       status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_INPLACE)
@@ -178,14 +178,14 @@ subroutine c2c_1m_z_plan(desc, decomp)
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp%zsz(1)*decomp%zsz(2))
+                            decomp%zsz(1) * decomp%zsz(2))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       strides(1) = 0
-      strides(2) = decomp%zsz(1)*decomp%zsz(2)
+      strides(2) = decomp%zsz(1) * decomp%zsz(2)
       status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
@@ -218,7 +218,7 @@ subroutine r2c_1m_x_plan(desc, decomp_ph, decomp_sp, direction)
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp_ph%xsz(2)*decomp_ph%xsz(3))
+                            decomp_ph%xsz(2) * decomp_ph%xsz(3))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
                             DFTI_COMPLEX_COMPLEX)
@@ -268,7 +268,7 @@ subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp_ph%zsz(1)*decomp_ph%zsz(2))
+                            decomp_ph%zsz(1) * decomp_ph%zsz(2))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
                             DFTI_COMPLEX_COMPLEX)
@@ -280,14 +280,14 @@ subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       strides(1) = 0
-      strides(2) = decomp_ph%zsz(1)*decomp_ph%zsz(2)
+      strides(2) = decomp_ph%zsz(1) * decomp_ph%zsz(2)
       if (direction == -1) then
          status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
       else if (direction == 1) then
          status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
       end if
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      strides(2) = decomp_sp%zsz(1)*decomp_sp%zsz(2)
+      strides(2) = decomp_sp%zsz(1) * decomp_sp%zsz(2)
       if (direction == -1) then
          status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
       else if (direction == 1) then
diff --git a/src/glassman.f90 b/src/glassman.f90
index 9e2ca251..bd0e9ae6 100644
--- a/src/glassman.f90
+++ b/src/glassman.f90
@@ -52,12 +52,12 @@ PURE SUBROUTINE SPCFFT(U, N, ISIGN, WORK)
       INU = .TRUE.
 
       DO WHILE (B > 1)
-         A = C*A
+         A = C * A
          C = 2
          DO WHILE (MOD(B, C) /= 0)
             C = C + 1
          END DO
-         B = B/C
+         B = B / C
          IF (INU) THEN
             CALL SPCPFT(A, B, C, U, WORK, ISIGN)
          ELSE
@@ -90,7 +90,7 @@ PURE SUBROUTINE SPCPFT(A, B, C, UIN, UOUT, ISIGN)
       COMPLEX(mytype), INTENT(OUT) :: UOUT(B, A, C)
       COMPLEX(mytype) :: DELTA, OMEGA, SUM
 
-      ANGLE = 6.28318530717958_mytype/REAL(A*C, kind=mytype)
+      ANGLE = 6.28318530717958_mytype / REAL(A * C, kind=mytype)
       OMEGA = CMPLX(1.0, 0.0, kind=mytype)
 
       IF (ISIGN == 1) THEN
@@ -105,11 +105,11 @@ PURE SUBROUTINE SPCPFT(A, B, C, UIN, UOUT, ISIGN)
                SUM = UIN(IB, C, IA)
                DO JCR = 2, C
                   JC = C + 1 - JCR
-                  SUM = UIN(IB, JC, IA) + OMEGA*SUM
+                  SUM = UIN(IB, JC, IA) + OMEGA * SUM
                END DO
                UOUT(IB, IA, IC) = SUM
             END DO
-            OMEGA = DELTA*OMEGA
+            OMEGA = DELTA * OMEGA
          END DO
       END DO
 
@@ -148,7 +148,7 @@ subroutine glassman_3d_r2c(in_r, nx, ny, nz, out_c)
             end do
             call spcfft(buf, nx, -1, scratch)
             ! simply drop the redundant part of the complex output
-            do i = 1, nx/2 + 1
+            do i = 1, nx / 2 + 1
                out_c(i, j, k) = buf(i)
             end do
          end do
@@ -156,7 +156,7 @@ subroutine glassman_3d_r2c(in_r, nx, ny, nz, out_c)
 
       ! ===== 1D FFTs in Y =====
       do k = 1, nz
-         do i = 1, nx/2 + 1
+         do i = 1, nx / 2 + 1
             do j = 1, ny
                buf(j) = out_c(i, j, k)
             end do
@@ -169,7 +169,7 @@ subroutine glassman_3d_r2c(in_r, nx, ny, nz, out_c)
 
       ! ===== 1D FFTs in Z =====
       do j = 1, ny
-         do i = 1, nx/2 + 1
+         do i = 1, nx / 2 + 1
             do k = 1, nz
                buf(k) = out_c(i, j, k)
             end do
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index bf21f10f..2148b9f0 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -203,9 +203,9 @@
        else
           tag_n = coord(1) + 1
        end if
-       icount = s3 + 2*level
-       ilength = level*s1
-       ijump = s1*(s2 + 2*level)
+       icount = s3 + 2 * level
+       ilength = level * s1
+       ijump = s1 * (s2 + 2 * level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
                             data_type, halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
@@ -254,7 +254,7 @@
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s1*(s2 + 2*level))*level
+       icount = (s1 * (s2 + 2 * level)) * level
        ! receive from bottom
        call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
                       neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
@@ -309,9 +309,9 @@
        else
           tag_e = coord(1) + 1
        end if
-       icount = s2*(s3 + 2*level)
+       icount = s2 * (s3 + 2 * level)
        ilength = level
-       ijump = s1 + 2*level
+       ijump = s1 + 2 * level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
                             data_type, halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
@@ -363,7 +363,7 @@
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s2*(s1 + 2*level))*level
+       icount = (s2 * (s1 + 2 * level)) * level
        ! receive from bottom
        call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
                       neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
@@ -417,9 +417,9 @@
        else
           tag_e = coord(1) + 1
        end if
-       icount = (s2 + 2*level)*s3
+       icount = (s2 + 2 * level) * s3
        ilength = level
-       ijump = s1 + 2*level
+       ijump = s1 + 2 * level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
                             data_type, halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
@@ -466,8 +466,8 @@
           tag_n = coord(2) + 1
        end if
        icount = s3
-       ilength = level*(s1 + 2*level)
-       ijump = (s1 + 2*level)*(s2 + 2*level)
+       ilength = level * (s1 + 2 * level)
+       ijump = (s1 + 2 * level) * (s2 + 2 * level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
                             data_type, halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
diff --git a/src/io.f90 b/src/io.f90
index 594e7b69..6096a1c1 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -355,14 +355,14 @@ subroutine read_one_real(ipencil, var, dirname, varname, io_name, opt_decomp, re
          if (read_reduce_prec) then
             allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
             call MPI_FILE_READ_ALL(fh, varsingle, &
-                                   subsizes(1)*subsizes(2)*subsizes(3), &
+                                   subsizes(1) * subsizes(2) * subsizes(3), &
                                    data_type, MPI_STATUS_IGNORE, ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
             var = real(varsingle, mytype)
             deallocate (varsingle)
          else
             call MPI_FILE_READ_ALL(fh, var, &
-                                   subsizes(1)*subsizes(2)*subsizes(3), &
+                                   subsizes(1) * subsizes(2) * subsizes(3), &
                                    data_type, MPI_STATUS_IGNORE, ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
          end if
@@ -370,9 +370,9 @@ subroutine read_one_real(ipencil, var, dirname, varname, io_name, opt_decomp, re
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
          disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                *int(disp_bytes, kind=MPI_OFFSET_KIND)
+                * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                * int(disp_bytes, kind=MPI_OFFSET_KIND)
       end associate
 
       if (opened_new) then
@@ -668,7 +668,7 @@ subroutine write_scalar_real(fh, disp, n, var)
                               MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine write_scalar_real
@@ -696,8 +696,8 @@ subroutine write_scalar_complex(fh, disp, n, var)
                               MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-             *2_MPI_OFFSET_KIND
+             * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+             * 2_MPI_OFFSET_KIND
 
       return
    end subroutine write_scalar_complex
@@ -727,7 +727,7 @@ subroutine write_scalar_integer(fh, disp, n, var)
       call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+             * int(m, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine write_scalar_integer
@@ -757,7 +757,7 @@ subroutine write_scalar_logical(fh, disp, n, var)
       call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+             * int(m, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine write_scalar_logical
@@ -787,7 +787,7 @@ subroutine read_scalar_real(fh, disp, n, var)
                              MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine read_scalar_real
@@ -810,8 +810,8 @@ subroutine read_scalar_complex(fh, disp, n, var)
                              MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-             *2_MPI_OFFSET_KIND
+             * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+             * 2_MPI_OFFSET_KIND
 
       return
    end subroutine read_scalar_complex
@@ -836,7 +836,7 @@ subroutine read_scalar_integer(fh, disp, n, var)
       call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+             * int(m, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine read_scalar_integer
@@ -861,7 +861,7 @@ subroutine read_scalar_logical(fh, disp, n, var)
       call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+             * int(m, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine read_scalar_logical
@@ -1312,22 +1312,22 @@ subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_n
                                 newtype, 'native', MPI_INFO_NULL, ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
          call MPI_FILE_WRITE_ALL(fh_registry(idx), varsingle, &
-                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 subsizes(1) * subsizes(2) * subsizes(3), &
                                  real_type_single, MPI_STATUS_IGNORE, ierror)
       else
          call MPI_FILE_SET_VIEW(fh_registry(idx), fh_disp(idx), real_type, &
                                 newtype, 'native', MPI_INFO_NULL, ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
          call MPI_FILE_WRITE_ALL(fh_registry(idx), var, &
-                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 subsizes(1) * subsizes(2) * subsizes(3), &
                                  real_type, MPI_STATUS_IGNORE, ierror)
       end if
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
 
       fh_disp(idx) = fh_disp(idx) + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                     *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                     *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                     *int(disp_bytes, kind=MPI_OFFSET_KIND)
+                     * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                     * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                     * int(disp_bytes, kind=MPI_OFFSET_KIND)
 
       if (opened_new) then
          call decomp_2d_close_io(io_name, full_io_name)
@@ -1537,7 +1537,7 @@ subroutine mpiio_write_real_probe(ipencil, var, filename, nlength)
                              newtype, 'native', MPI_INFO_NULL, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
       call MPI_FILE_WRITE_ALL(fh, var, &
-                              subsizes(1)*subsizes(2)*subsizes(3)*subsizes(4), &
+                              subsizes(1) * subsizes(2) * subsizes(3) * subsizes(4), &
                               real_type, MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
       call MPI_FILE_CLOSE(fh, ierror)
@@ -1750,7 +1750,7 @@ subroutine write_subdomain(ipencil, var, is, ie, js, je, ks, ke, filename)
                                 newtype, 'native', MPI_INFO_NULL, ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
          call MPI_FILE_WRITE_ALL(fh, wk, &
-                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 subsizes(1) * subsizes(2) * subsizes(3), &
                                  data_type, MPI_STATUS_IGNORE, ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
          call MPI_FILE_CLOSE(fh, ierror)
diff --git a/src/io_read_inflow.f90 b/src/io_read_inflow.f90
index 162510c7..75fa9488 100644
--- a/src/io_read_inflow.f90
+++ b/src/io_read_inflow.f90
@@ -50,7 +50,7 @@
                           newtype, 'native', MPI_INFO_NULL, ierror)
    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
    call MPI_FILE_READ_ALL(fh, var, &
-                          subsizes(1)*subsizes(2)*subsizes(3), &
+                          subsizes(1) * subsizes(2) * subsizes(3), &
                           data_type, MPI_STATUS_IGNORE, ierror)
    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
    call MPI_TYPE_FREE(newtype, ierror)
@@ -58,13 +58,13 @@
 
    ! update displacement for the next read operation
    disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-          *int(sizes(2), kind=MPI_OFFSET_KIND) &
-          *int(sizes(3), kind=MPI_OFFSET_KIND) &
-          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+          * int(sizes(2), kind=MPI_OFFSET_KIND) &
+          * int(sizes(3), kind=MPI_OFFSET_KIND) &
+          * int(mytype_bytes, kind=MPI_OFFSET_KIND)
    if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                         * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                         * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                         * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
 associate (vnm => varname) ! Silence unused argument
diff --git a/src/io_write_outflow.f90 b/src/io_write_outflow.f90
index fe126584..9e9e4f98 100644
--- a/src/io_write_outflow.f90
+++ b/src/io_write_outflow.f90
@@ -50,7 +50,7 @@
                           newtype, 'native', MPI_INFO_NULL, ierror)
    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
    call MPI_FILE_WRITE_ALL(fh, var, &
-                           subsizes(1)*subsizes(2)*subsizes(3), &
+                           subsizes(1) * subsizes(2) * subsizes(3), &
                            data_type, MPI_STATUS_IGNORE, ierror)
    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
    call MPI_TYPE_FREE(newtype, ierror)
@@ -58,13 +58,13 @@
 
    ! update displacement for the next write operation
    disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-          *int(sizes(2), kind=MPI_OFFSET_KIND) &
-          *int(sizes(3), kind=MPI_OFFSET_KIND) &
-          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+          * int(sizes(2), kind=MPI_OFFSET_KIND) &
+          * int(sizes(3), kind=MPI_OFFSET_KIND) &
+          * int(mytype_bytes, kind=MPI_OFFSET_KIND)
    if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                         * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                         * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                         * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
 associate (vnm => varname) ! Silence unused dummy argument
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index 9cae71f0..ea8f4a74 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -274,14 +274,14 @@ subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -326,14 +326,14 @@ subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -378,14 +378,14 @@ subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -430,14 +430,14 @@ subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index db2e439a..2b1a1efd 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -274,14 +274,14 @@ subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -326,14 +326,14 @@ subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -378,14 +378,14 @@ subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -430,14 +430,14 @@ subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 7d28cb84..cca04207 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -136,7 +136,7 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
 #if defined(_GPU)
      !If one of the array in cuda call is not device we need to add acc host_data
      !$acc host_data use_device(dst)
-     istat = cudaMemcpy(dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
+     istat = cudaMemcpy(dst, work2_r_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -274,7 +274,7 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
 
 #if defined(_GPU)
      !$acc host_data use_device(dst)
-     istat = cudaMemcpy(dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
+     istat = cudaMemcpy(dst, work2_c_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -315,14 +315,14 @@ subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -367,14 +367,14 @@ subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -415,7 +415,7 @@ subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
 #endif
@@ -456,7 +456,7 @@ subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
 #endif
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index c408696f..35d6714e 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -87,7 +87,7 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
 
 #if defined(_GPU)
      !$acc host_data use_device(src)
-     istat = cudaMemcpy(work1_r_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice)
+     istat = cudaMemcpy(work1_r_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -223,7 +223,7 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
 
 #if defined(_GPU)
      !$acc host_data use_device(src)
-     istat = cudaMemcpy(work1_c_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice)
+     istat = cudaMemcpy(work1_c_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -310,7 +310,7 @@ subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
 #endif
@@ -351,7 +351,7 @@ subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
 #endif
@@ -396,14 +396,14 @@ subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -449,14 +449,14 @@ subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else

From c0d47379ffe216b4c8f563f74dc4ba2e003f5750 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 20 Mar 2023 13:07:34 +0000
Subject: [PATCH 182/436] Add script to automate running fprettify

---
 scripts/format.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 scripts/format.sh

diff --git a/scripts/format.sh b/scripts/format.sh
new file mode 100644
index 00000000..cd0f2803
--- /dev/null
+++ b/scripts/format.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env sh
+#
+# format.sh
+#
+# Runs the prettifier over 2decomp sources.
+
+# fprettify options
+# --enable-replacements : Replaces relational operators, e.g. .lt. -> <
+# --c-relations : Use C-style relational operators, e.g. <=
+# -w 3 : Whitespace preset - operators, print/read, *, /, +, -
+fprettify --enable-replacements --c-relations -w 3 src/*f90 examples/*/*f90

From 1df10e1bef453688203da7469fd389659611dff6 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 20 Mar 2023 13:17:17 +0000
Subject: [PATCH 183/436] Add format target to build system

Runs the prettifier using a script for consistency
---
 CMakeLists.txt    | 4 ++++
 scripts/format.sh | 6 +++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1e28d8e7..8c48354b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -79,6 +79,10 @@ add_subdirectory(src)
 # Add tests
 add_subdirectory(examples)
 
+# Add a prettify target
+add_custom_target(format sh ${CMAKE_SOURCE_DIR}/scripts/format.sh
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
 # # Create an example dir with all input.i3d example files
 # option(BUILD_TESTING "Build with tests" ON)
 # set(test_dir "${PROJECT_BINARY_DIR}/Test")
diff --git a/scripts/format.sh b/scripts/format.sh
index cd0f2803..5c202a4d 100644
--- a/scripts/format.sh
+++ b/scripts/format.sh
@@ -4,8 +4,12 @@
 #
 # Runs the prettifier over 2decomp sources.
 
+# Find 2decomp root
+SCRDIR=$( dirname -- "$0"; )
+D2DDIR=${SCRDIR}/../
+
 # fprettify options
 # --enable-replacements : Replaces relational operators, e.g. .lt. -> <
 # --c-relations : Use C-style relational operators, e.g. <=
 # -w 3 : Whitespace preset - operators, print/read, *, /, +, -
-fprettify --enable-replacements --c-relations -w 3 src/*f90 examples/*/*f90
+fprettify --enable-replacements --c-relations -w 3 ${D2DDIR}/src/*f90 ${D2DDIR}/examples/*/*f90

From 3671ee13e5d924f794955024f50aa56d56bba91d Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 20 Mar 2023 13:19:53 +0000
Subject: [PATCH 184/436] Add fprettify to README

---
 README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.md b/README.md
index e16b0532..cbb4242c 100644
--- a/README.md
+++ b/README.md
@@ -224,6 +224,11 @@ This variable is automatically added in GPU builds.
 
 This variable is valid only for GPU builds. The NVIDIA Collective Communication Library (NCCL) implements multi-GPU and multi-node communication primitives optimized for NVIDIA GPUs and Networking.
 
+### Code formatting
+
+The code is formatted using the `fprettify` program (available via `pip`), to ensure consistency of use there is a script file `scripts/format.sh` which will run `fprettify` across the 2decomp&fft source, you can also use the `format` build target to run the script.
+It is recommended that you should format the code before making a pull request.
+
 ## Optional dependencies
 
 ### FFTW

From 205836254028bef42bcf5ac345245e6ac5daf729 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 20 Mar 2023 13:53:37 +0000
Subject: [PATCH 185/436] Run fprettify --enable-replacements --c-relations -w
 3 src/*f90 examples/*/*f90 (#167)

* Run fprettify --enable-replacements --c-relations -w 3 src/*f90 examples/*/*f90

This ensures multiplication/division operators also get whitespace

* Add script to automate running fprettify

* Add format target to build system

Runs the prettifier using a script for consistency

* Add fprettify to README
---
 CMakeLists.txt                         |   4 +
 README.md                              |   5 +
 examples/fft_physical_x/fft_c2c_x.f90  |  38 +++----
 examples/fft_physical_x/fft_grid_x.f90 |  38 +++----
 examples/fft_physical_x/fft_r2c_x.f90  |  24 ++---
 examples/fft_physical_z/fft_c2c_z.f90  |  38 +++----
 examples/fft_physical_z/fft_r2c_z.f90  |  24 ++---
 examples/halo_test/halo_test.f90       |  14 +--
 examples/init_test/init_test.f90       |  10 +-
 examples/io_test/io_bench.f90          |   8 +-
 examples/io_test/io_plane_test.f90     |  26 ++---
 examples/io_test/io_read.f90           |  10 +-
 examples/io_test/io_test.f90           |  10 +-
 examples/io_test/io_var_test.f90       |  18 ++--
 examples/test2d/test2d.f90             |  18 ++--
 examples/test2d/timing2d_complex.f90   |  26 ++---
 examples/test2d/timing2d_real.f90      |  26 ++---
 scripts/format.sh                      |  15 +++
 src/decomp_2d.f90                      | 136 ++++++++++++-------------
 src/decomp_2d_init_fin.f90             |   6 +-
 src/factor.f90                         |  16 +--
 src/fft_common.f90                     |   4 +-
 src/fft_cufft.f90                      |  98 +++++++++---------
 src/fft_fftw3.f90                      |  48 ++++-----
 src/fft_fftw3_f03.f90                  |  88 ++++++++--------
 src/fft_generic.f90                    |   8 +-
 src/fft_mkl.f90                        |  14 +--
 src/glassman.f90                       |  16 +--
 src/halo_common.f90                    |  22 ++--
 src/io.f90                             |  44 ++++----
 src/io_read_inflow.f90                 |  14 +--
 src/io_write_outflow.f90               |  14 +--
 src/transpose_x_to_y.f90               |  16 +--
 src/transpose_y_to_x.f90               |  16 +--
 src/transpose_y_to_z.f90               |  16 +--
 src/transpose_z_to_y.f90               |  16 +--
 36 files changed, 484 insertions(+), 460 deletions(-)
 create mode 100644 scripts/format.sh

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1e28d8e7..8c48354b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -79,6 +79,10 @@ add_subdirectory(src)
 # Add tests
 add_subdirectory(examples)
 
+# Add a prettify target
+add_custom_target(format sh ${CMAKE_SOURCE_DIR}/scripts/format.sh
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
 # # Create an example dir with all input.i3d example files
 # option(BUILD_TESTING "Build with tests" ON)
 # set(test_dir "${PROJECT_BINARY_DIR}/Test")
diff --git a/README.md b/README.md
index e16b0532..cbb4242c 100644
--- a/README.md
+++ b/README.md
@@ -224,6 +224,11 @@ This variable is automatically added in GPU builds.
 
 This variable is valid only for GPU builds. The NVIDIA Collective Communication Library (NCCL) implements multi-GPU and multi-node communication primitives optimized for NVIDIA GPUs and Networking.
 
+### Code formatting
+
+The code is formatted using the `fprettify` program (available via `pip`), to ensure consistency of use there is a script file `scripts/format.sh` which will run `fprettify` across the 2decomp&fft source, you can also use the `format` build target to run the script.
+It is recommended that you should format the code before making a pull request.
+
 ## Optional dependencies
 
 ### FFTW
diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 1291734a..d692f8b9 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -36,10 +36,10 @@ program fft_c2c_x
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -97,8 +97,8 @@ program fft_c2c_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             in(i, j, k) = cmplx(dr, di, mytype)
          end do
@@ -122,7 +122,7 @@ program fft_c2c_x
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      in = in / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
       !$acc end kernels
 
    end do
@@ -132,10 +132,10 @@ program fft_c2c_x
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -143,31 +143,31 @@ program fft_c2c_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             dr = dr - real(in(i, j, k), mytype)
             di = di - aimag(in(i, j, k))
-            error = error + sqrt(dr*dr + di*di)
+            error = error + sqrt(dr * dr + di * di)
          end do
       end do
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
-      n1 = n1**(1._mytype/3._mytype)
+      n1 = real(nx, mytype) * real(ny, mytype) * real(nz, mytype)
+      n1 = n1**(1._mytype / 3._mytype)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops*3._mytype*n1**2
-      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+      flops = flops * 3._mytype * n1**2
+      flops = 2._mytype * flops / ((t1 + t3) / real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops / 1000._mytype**3
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index d51e6cfc..1410fa15 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -36,10 +36,10 @@ program fft_physical_x
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -98,8 +98,8 @@ program fft_physical_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             in(i, j, k) = cmplx(dr, di, mytype)
          end do
@@ -123,7 +123,7 @@ program fft_physical_x
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      in = in / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
       !$acc end kernels
 
    end do
@@ -133,10 +133,10 @@ program fft_physical_x
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -144,31 +144,31 @@ program fft_physical_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             dr = dr - real(in(i, j, k), mytype)
             di = di - aimag(in(i, j, k))
-            error = error + sqrt(dr*dr + di*di)
+            error = error + sqrt(dr * dr + di * di)
          end do
       end do
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
-      n1 = n1**(1._mytype/3._mytype)
+      n1 = real(nx, mytype) * real(ny, mytype) * real(nz, mytype)
+      n1 = n1**(1._mytype / 3._mytype)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops*3._mytype*n1**2
-      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+      flops = flops * 3._mytype * n1**2
+      flops = 2._mytype * flops / ((t1 + t3) / real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops / 1000._mytype**3
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 9cf55702..a887ef0e 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -37,10 +37,10 @@ program fft_r2c_x
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -99,8 +99,8 @@ program fft_r2c_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            in_r(i, j, k) = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                            / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
          end do
       end do
    end do
@@ -122,7 +122,7 @@ program fft_r2c_x
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in_r = in_r/(real(nx, mytype)*real(ny, mytype)*real(nz, mytype))
+      in_r = in_r / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
       !$acc end kernels
 
    end do
@@ -132,10 +132,10 @@ program fft_r2c_x
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -143,15 +143,15 @@ program fft_r2c_x
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             error = error + abs(in_r(i, j, k) - dr)
          end do
       end do
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== r2c/c2r interface ====='
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 91ba1cbb..347b5123 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -36,10 +36,10 @@ program fft_c2c_z
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -98,8 +98,8 @@ program fft_c2c_z
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             in(i, j, k) = cmplx(dr, di, mytype)
          end do
@@ -123,7 +123,7 @@ program fft_c2c_z
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in = in/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      in = in / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
       !$acc end kernels
 
    end do
@@ -133,10 +133,10 @@ program fft_c2c_z
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -144,31 +144,31 @@ program fft_c2c_z
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             di = dr
             dr = dr - real(in(i, j, k), mytype)
             di = di - aimag(in(i, j, k))
-            error = error + sqrt(dr*dr + di*di)
+            error = error + sqrt(dr * dr + di * di)
          end do
       end do
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype)*real(ny, mytype)*real(nz, mytype)
-      n1 = n1**(1._mytype/3._mytype)
+      n1 = real(nx, mytype) * real(ny, mytype) * real(nz, mytype)
+      n1 = n1**(1._mytype / 3._mytype)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype*n1*log(n1)/log(2.0_mytype)
+      flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops*3._mytype*n1**2
-      flops = 2._mytype*flops/((t1 + t3)/real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops/1000._mytype**3
+      flops = flops * 3._mytype * n1**2
+      flops = 2._mytype * flops / ((t1 + t3) / real(NTEST, mytype))
+      write (*, *) 'GFLOPS : ', flops / 1000._mytype**3
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 6cfcda39..2798a63a 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -37,10 +37,10 @@ program fft_r2c_z
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -100,8 +100,8 @@ program fft_r2c_z
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            in_r(i, j, k) = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                            /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            in_r(i, j, k) = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                            / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
          end do
       end do
    end do
@@ -122,7 +122,7 @@ program fft_r2c_z
       t4 = t4 + MPI_WTIME() - t3
 
       !$acc kernels
-      in_r = in_r/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+      in_r = in_r / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
       !$acc end kernels
 
    end do
@@ -132,10 +132,10 @@ program fft_r2c_z
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
 
    ! checking accuracy
    error = 0._mytype
@@ -143,8 +143,8 @@ program fft_r2c_z
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            dr = real(i, mytype)/real(nx, mytype)*real(j, mytype) &
-                 /real(ny, mytype)*real(k, mytype)/real(nz, mytype)
+            dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
+                 / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             error = error + abs(in_r(i, j, k) - dr)
             !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
          end do
@@ -154,7 +154,7 @@ program fft_r2c_z
 !10 format('in_r final ', I2,1x,I2,1x,I2,1x,I2,1x,F12.6,1x,F12.6)
 
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all/real(nx, mytype)/real(ny, mytype)/real(nz, mytype)
+   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
 
    if (nrank == 0) then
       write (*, *) '===== r2c/c2r interface ====='
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index ed3558cc..1eb93d85 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -47,10 +47,10 @@ program halo_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -517,7 +517,7 @@ subroutine check_err(divh, pencil)
       !$acc end kernels
       divmag = mag(tmp)
 
-      if (err < epsilon(divmag)*divmag) then
+      if (err < epsilon(divmag) * divmag) then
          passing = .true.
       else
          passing = .false.
@@ -530,7 +530,7 @@ subroutine check_err(divh, pencil)
 #ifdef DEBUG
          write (*, *) (divh(i, i, i), i=2, 13)
 #endif
-         write (*, *) 'Error: ', err, '; Relative: ', err/divmag
+         write (*, *) 'Error: ', err, '; Relative: ', err / divmag
          write (*, *) 'Pass: ', passing
       end if
       deallocate (tmp)
@@ -567,7 +567,7 @@ real(mytype) function mag(a)
                               "halo_test::mag::MPI_Allreduce")
       end if
 
-      mag = sqrt(gmag/(nx - 2)/(ny - 2)/(nz - 2))
+      mag = sqrt(gmag / (nx - 2) / (ny - 2) / (nz - 2))
 
    end function mag
    !=====================================================================
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 0d6630fb..7f932266 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -30,10 +30,10 @@ program init_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -71,7 +71,7 @@ program init_test
          print *, "will be used"
       end if
    end if
-   nexpect = nx*ny*nz
+   nexpect = nx * ny * nz
    call run(p_row, p_col)
 
    call MPI_Finalize(ierror)
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index 0b37e0d0..34b2d29f 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -29,10 +29,10 @@ program io_bench
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 2c86bf56..0dac2e7f 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -40,10 +40,10 @@ program io_plane_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -124,17 +124,17 @@ program io_plane_test
    !$acc update self(u3)
    !$acc end data
    ! X-pencil data
-   call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 1, nx / 2, '.', 'x_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 2, ny / 2, '.', 'x_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(1, u1, 3, nz / 2, '.', 'x_pencil-z_plane.dat', 'test')
    ! Y-pencil data
-   call decomp_2d_write_plane(2, u2, 2, ny/2, '.', 'y_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(2, u2, 3, nz/2, '.', 'y_pencil-z_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 2, ny / 2, '.', 'y_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 1, nx / 2, '.', 'y_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 3, nz / 2, '.', 'y_pencil-z_plane.dat', 'test')
    ! Z-pencil data
-   call decomp_2d_write_plane(3, u3, 1, nx/2, '.', 'z_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(3, u3, 2, ny/2, '.', 'z_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(3, u3, 3, nz/2, '.', 'z_pencil-z_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 1, nx / 2, '.', 'z_pencil-x_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 2, ny / 2, '.', 'z_pencil-y_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 3, nz / 2, '.', 'z_pencil-z_plane.dat', 'test')
    ! Attemp to read the files
    if (nrank == 0) then
       inquire (iolength=iol) data1(1, 1, 1)
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index d396159a..8b2ddc1c 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -38,10 +38,10 @@ program io_read
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -89,7 +89,7 @@ program io_read
       do j = 1, ny
          do i = 1, nx
 #ifdef COMPLEX_TEST
-            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
+            data1(i, j, k) = cmplx(real(m, mytype), real(nx * ny * nz - m, mytype))
 #else
             data1(i, j, k) = real(m, mytype)
 #endif
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 82b68b80..b4f85195 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -42,10 +42,10 @@ program io_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -93,7 +93,7 @@ program io_test
       do j = 1, ny
          do i = 1, nx
 #ifdef COMPLEX_TEST
-            data1(i, j, k) = cmplx(real(m, mytype), real(nx*ny*nz - m, mytype))
+            data1(i, j, k) = cmplx(real(m, mytype), real(nx * ny * nz - m, mytype))
 #else
             data1(i, j, k) = real(m, mytype)
 #endif
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 1e7e6d4e..1b253c9f 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -57,10 +57,10 @@ program io_var_test
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -102,12 +102,12 @@ program io_var_test
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
    ! also create a data set over a large domain
-   call decomp_info_init(nx*2, ny*2, nz*2, large)
+   call decomp_info_init(nx * 2, ny * 2, nz * 2, large)
 
    ! initialise global data
    allocate (data1(nx, ny, nz))
    allocate (cdata1(nx, ny, nz))
-   allocate (data1_large(nx*2, ny*2, nz*2))
+   allocate (data1_large(nx * 2, ny * 2, nz * 2))
    m = 1
    do k = 1, nz
       do j = 1, ny
@@ -120,9 +120,9 @@ program io_var_test
    end do
 
    m = 1
-   do k = 1, nz*2
-      do j = 1, ny*2
-         do i = 1, nx*2
+   do k = 1, nz * 2
+      do j = 1, ny * 2
+         do i = 1, nx * 2
             data1_large(i, j, k) = real(m, mytype)
             m = m + 1
          end do
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index e903cd29..1535e944 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -38,10 +38,10 @@ program test2d
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -118,7 +118,7 @@ program test2d
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             u1(i, j, k) = m
          end do
       end do
@@ -160,7 +160,7 @@ program test2d
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
@@ -190,7 +190,7 @@ program test2d
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             if (abs(u3(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
@@ -209,7 +209,7 @@ program test2d
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
@@ -228,7 +228,7 @@ program test2d
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             if (abs(u1(i, j, k) - m) > 0) error_flag = .true.
          end do
       end do
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index dc7362ec..6735e2f9 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -42,10 +42,10 @@ program timing2d_complex
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -122,7 +122,7 @@ program timing2d_complex
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             u1(i, j, k) = cmplx(m, real(m - 1, mytype), kind=mytype)
          end do
       end do
@@ -171,7 +171,7 @@ program timing2d_complex
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
@@ -204,7 +204,7 @@ program timing2d_complex
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u3(i, j, k) - cm) > 0) error_flag = .true.
             end do
@@ -226,7 +226,7 @@ program timing2d_complex
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u2(i, j, k) - cm) > 0) error_flag = .true.
             end do
@@ -248,7 +248,7 @@ program timing2d_complex
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                cm = cmplx(m, real(m - 1, mytype), kind=mytype)
                if (abs(u1(i, j, k) - cm) > 0) error_flag = .true.
             end do
@@ -262,16 +262,16 @@ program timing2d_complex
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
    call MPI_ALLREDUCE(t6, t5, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t5/dble(nproc)
+   t5 = t5 / dble(nproc)
    call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t7/dble(nproc)
+   t7 = t7 / dble(nproc)
    t8 = t1 + t3 + t5 + t7
    if (nrank == 0) then
       write (*, *) 'Time X->Y ', t1
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 9e1c007b..1f9e216b 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -41,10 +41,10 @@ program timing2d_real
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
-   resize_domain = int(nranks_tot/4) + 1
-   nx = nx_base*resize_domain
-   ny = ny_base*resize_domain
-   nz = nz_base*resize_domain
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
@@ -121,7 +121,7 @@ program timing2d_real
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
-            m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+            m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
             u1(i, j, k) = m
          end do
       end do
@@ -170,7 +170,7 @@ program timing2d_real
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -202,7 +202,7 @@ program timing2d_real
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                if (abs(u3(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -223,7 +223,7 @@ program timing2d_real
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                if (abs(u2(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -244,7 +244,7 @@ program timing2d_real
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1
-               m = real(i + (j - 1)*nx + (k - 1)*nx*ny, mytype)
+               m = real(i + (j - 1) * nx + (k - 1) * nx * ny, mytype)
                if (abs(u1(i, j, k) - m) > 0) error_flag = .true.
             end do
          end do
@@ -257,16 +257,16 @@ program timing2d_real
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1/dble(nproc)
+   t1 = t1 / dble(nproc)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3/dble(nproc)
+   t3 = t3 / dble(nproc)
    call MPI_ALLREDUCE(t6, t5, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t5/dble(nproc)
+   t5 = t5 / dble(nproc)
    call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t7/dble(nproc)
+   t7 = t7 / dble(nproc)
    t8 = t1 + t3 + t5 + t7
    if (nrank == 0) then
       write (*, *) 'Time X->Y ', t1
diff --git a/scripts/format.sh b/scripts/format.sh
new file mode 100644
index 00000000..5c202a4d
--- /dev/null
+++ b/scripts/format.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+#
+# format.sh
+#
+# Runs the prettifier over 2decomp sources.
+
+# Find 2decomp root
+SCRDIR=$( dirname -- "$0"; )
+D2DDIR=${SCRDIR}/../
+
+# fprettify options
+# --enable-replacements : Replaces relational operators, e.g. .lt. -> <
+# --c-relations : Use C-style relational operators, e.g. <=
+# -w 3 : Whitespace preset - operators, print/read, *, /, +, -
+fprettify --enable-replacements --c-relations -w 3 ${D2DDIR}/src/*f90 ${D2DDIR}/examples/*/*f90
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 2a505f8d..8f9d6ea8 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -403,13 +403,13 @@ subroutine decomp_info_init(nx, ny, nz, decomp)
       ! allocate memory for the MPI_ALLTOALL(V) buffers
       ! define the buffers globally for performance reason
 
-      buf_size = max(decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3), &
-                     max(decomp%ysz(1)*decomp%ysz(2)*decomp%ysz(3), &
-                         decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)))
+      buf_size = max(decomp%xsz(1) * decomp%xsz(2) * decomp%xsz(3), &
+                     max(decomp%ysz(1) * decomp%ysz(2) * decomp%ysz(3), &
+                         decomp%zsz(1) * decomp%zsz(2) * decomp%zsz(3)))
 #ifdef EVEN
       ! padded alltoall optimisation may need larger buffer space
       buf_size = max(buf_size, &
-                     max(decomp%x1count*dims(1), decomp%y2count*dims(2)))
+                     max(decomp%x1count * dims(1), decomp%y2count * dims(2)))
 #endif
 
       ! check if additional memory is required
@@ -502,39 +502,39 @@ subroutine init_coarser_mesh_statS(i_skip, j_skip, k_skip, from1)
 
       do i = 1, 3
          if (from1) then
-            xstS(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            xstS(i) = (xstart(i) + skip(i) - 1) / skip(i)
             if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstS(i) = xstS(i) + 1
-            xenS(i) = (xend(i) + skip(i) - 1)/skip(i)
+            xenS(i) = (xend(i) + skip(i) - 1) / skip(i)
          else
-            xstS(i) = xstart(i)/skip(i)
+            xstS(i) = xstart(i) / skip(i)
             if (mod(xstart(i), skip(i)) /= 0) xstS(i) = xstS(i) + 1
-            xenS(i) = xend(i)/skip(i)
+            xenS(i) = xend(i) / skip(i)
          end if
          xszS(i) = xenS(i) - xstS(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            ystS(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            ystS(i) = (ystart(i) + skip(i) - 1) / skip(i)
             if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystS(i) = ystS(i) + 1
-            yenS(i) = (yend(i) + skip(i) - 1)/skip(i)
+            yenS(i) = (yend(i) + skip(i) - 1) / skip(i)
          else
-            ystS(i) = ystart(i)/skip(i)
+            ystS(i) = ystart(i) / skip(i)
             if (mod(ystart(i), skip(i)) /= 0) ystS(i) = ystS(i) + 1
-            yenS(i) = yend(i)/skip(i)
+            yenS(i) = yend(i) / skip(i)
          end if
          yszS(i) = yenS(i) - ystS(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            zstS(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            zstS(i) = (zstart(i) + skip(i) - 1) / skip(i)
             if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstS(i) = zstS(i) + 1
-            zenS(i) = (zend(i) + skip(i) - 1)/skip(i)
+            zenS(i) = (zend(i) + skip(i) - 1) / skip(i)
          else
-            zstS(i) = zstart(i)/skip(i)
+            zstS(i) = zstart(i) / skip(i)
             if (mod(zstart(i), skip(i)) /= 0) zstS(i) = zstS(i) + 1
-            zenS(i) = zend(i)/skip(i)
+            zenS(i) = zend(i) / skip(i)
          end if
          zszS(i) = zenS(i) - zstS(i) + 1
       end do
@@ -567,39 +567,39 @@ subroutine init_coarser_mesh_statV(i_skip, j_skip, k_skip, from1)
 
       do i = 1, 3
          if (from1) then
-            xstV(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            xstV(i) = (xstart(i) + skip(i) - 1) / skip(i)
             if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstV(i) = xstV(i) + 1
-            xenV(i) = (xend(i) + skip(i) - 1)/skip(i)
+            xenV(i) = (xend(i) + skip(i) - 1) / skip(i)
          else
-            xstV(i) = xstart(i)/skip(i)
+            xstV(i) = xstart(i) / skip(i)
             if (mod(xstart(i), skip(i)) /= 0) xstV(i) = xstV(i) + 1
-            xenV(i) = xend(i)/skip(i)
+            xenV(i) = xend(i) / skip(i)
          end if
          xszV(i) = xenV(i) - xstV(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            ystV(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            ystV(i) = (ystart(i) + skip(i) - 1) / skip(i)
             if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystV(i) = ystV(i) + 1
-            yenV(i) = (yend(i) + skip(i) - 1)/skip(i)
+            yenV(i) = (yend(i) + skip(i) - 1) / skip(i)
          else
-            ystV(i) = ystart(i)/skip(i)
+            ystV(i) = ystart(i) / skip(i)
             if (mod(ystart(i), skip(i)) /= 0) ystV(i) = ystV(i) + 1
-            yenV(i) = yend(i)/skip(i)
+            yenV(i) = yend(i) / skip(i)
          end if
          yszV(i) = yenV(i) - ystV(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            zstV(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            zstV(i) = (zstart(i) + skip(i) - 1) / skip(i)
             if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstV(i) = zstV(i) + 1
-            zenV(i) = (zend(i) + skip(i) - 1)/skip(i)
+            zenV(i) = (zend(i) + skip(i) - 1) / skip(i)
          else
-            zstV(i) = zstart(i)/skip(i)
+            zstV(i) = zstart(i) / skip(i)
             if (mod(zstart(i), skip(i)) /= 0) zstV(i) = zstV(i) + 1
-            zenV(i) = zend(i)/skip(i)
+            zenV(i) = zend(i) / skip(i)
          end if
          zszV(i) = zenV(i) - zstV(i) + 1
       end do
@@ -632,39 +632,39 @@ subroutine init_coarser_mesh_statP(i_skip, j_skip, k_skip, from1)
 
       do i = 1, 3
          if (from1) then
-            xstP(i) = (xstart(i) + skip(i) - 1)/skip(i)
+            xstP(i) = (xstart(i) + skip(i) - 1) / skip(i)
             if (mod(xstart(i) + skip(i) - 1, skip(i)) /= 0) xstP(i) = xstP(i) + 1
-            xenP(i) = (xend(i) + skip(i) - 1)/skip(i)
+            xenP(i) = (xend(i) + skip(i) - 1) / skip(i)
          else
-            xstP(i) = xstart(i)/skip(i)
+            xstP(i) = xstart(i) / skip(i)
             if (mod(xstart(i), skip(i)) /= 0) xstP(i) = xstP(i) + 1
-            xenP(i) = xend(i)/skip(i)
+            xenP(i) = xend(i) / skip(i)
          end if
          xszP(i) = xenP(i) - xstP(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            ystP(i) = (ystart(i) + skip(i) - 1)/skip(i)
+            ystP(i) = (ystart(i) + skip(i) - 1) / skip(i)
             if (mod(ystart(i) + skip(i) - 1, skip(i)) /= 0) ystP(i) = ystP(i) + 1
-            yenP(i) = (yend(i) + skip(i) - 1)/skip(i)
+            yenP(i) = (yend(i) + skip(i) - 1) / skip(i)
          else
-            ystP(i) = ystart(i)/skip(i)
+            ystP(i) = ystart(i) / skip(i)
             if (mod(ystart(i), skip(i)) /= 0) ystP(i) = ystP(i) + 1
-            yenP(i) = yend(i)/skip(i)
+            yenP(i) = yend(i) / skip(i)
          end if
          yszP(i) = yenP(i) - ystP(i) + 1
       end do
 
       do i = 1, 3
          if (from1) then
-            zstP(i) = (zstart(i) + skip(i) - 1)/skip(i)
+            zstP(i) = (zstart(i) + skip(i) - 1) / skip(i)
             if (mod(zstart(i) + skip(i) - 1, skip(i)) /= 0) zstP(i) = zstP(i) + 1
-            zenP(i) = (zend(i) + skip(i) - 1)/skip(i)
+            zenP(i) = (zend(i) + skip(i) - 1) / skip(i)
          else
-            zstP(i) = zstart(i)/skip(i)
+            zstP(i) = zstart(i) / skip(i)
             if (mod(zstart(i), skip(i)) /= 0) zstP(i) = zstP(i) + 1
-            zenP(i) = zend(i)/skip(i)
+            zenP(i) = zend(i) / skip(i)
          end if
          zszP(i) = zenP(i) - zstP(i) + 1
       end do
@@ -692,7 +692,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = xstS(3), xenS(3)
                do j = xstS(2), xenS(2)
                   do i = xstS(1), xenS(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipS + 1, (j - 1) * jskipS + 1, (k - 1) * kskipS + 1)
                   end do
                end do
             end do
@@ -700,7 +700,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = xstS(3), xenS(3)
                do j = xstS(2), xenS(2)
                   do i = xstS(1), xenS(1)
-                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                     wk(i, j, k) = wk2(i * iskipS, j * jskipS, k * kskipS)
                   end do
                end do
             end do
@@ -714,7 +714,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = ystS(3), yenS(3)
                do j = ystS(2), yenS(2)
                   do i = ystS(1), yenS(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipS + 1, (j - 1) * jskipS + 1, (k - 1) * kskipS + 1)
                   end do
                end do
             end do
@@ -722,7 +722,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = ystS(3), yenS(3)
                do j = ystS(2), yenS(2)
                   do i = ystS(1), yenS(1)
-                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                     wk(i, j, k) = wk2(i * iskipS, j * jskipS, k * kskipS)
                   end do
                end do
             end do
@@ -736,7 +736,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = zstS(3), zenS(3)
                do j = zstS(2), zenS(2)
                   do i = zstS(1), zenS(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipS + 1, (j - 1)*jskipS + 1, (k - 1)*kskipS + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipS + 1, (j - 1) * jskipS + 1, (k - 1) * kskipS + 1)
                   end do
                end do
             end do
@@ -744,7 +744,7 @@ subroutine fine_to_coarseS(ipencil, var_fine, var_coarse)
             do k = zstS(3), zenS(3)
                do j = zstS(2), zenS(2)
                   do i = zstS(1), zenS(1)
-                     wk(i, j, k) = wk2(i*iskipS, j*jskipS, k*kskipS)
+                     wk(i, j, k) = wk2(i * iskipS, j * jskipS, k * kskipS)
                   end do
                end do
             end do
@@ -777,7 +777,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = xstV(3), xenV(3)
                do j = xstV(2), xenV(2)
                   do i = xstV(1), xenV(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipV + 1, (j - 1) * jskipV + 1, (k - 1) * kskipV + 1)
                   end do
                end do
             end do
@@ -785,7 +785,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = xstV(3), xenV(3)
                do j = xstV(2), xenV(2)
                   do i = xstV(1), xenV(1)
-                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                     wk(i, j, k) = wk2(i * iskipV, j * jskipV, k * kskipV)
                   end do
                end do
             end do
@@ -799,7 +799,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = ystV(3), yenV(3)
                do j = ystV(2), yenV(2)
                   do i = ystV(1), yenV(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipV + 1, (j - 1) * jskipV + 1, (k - 1) * kskipV + 1)
                   end do
                end do
             end do
@@ -807,7 +807,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = ystV(3), yenV(3)
                do j = ystV(2), yenV(2)
                   do i = ystV(1), yenV(1)
-                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                     wk(i, j, k) = wk2(i * iskipV, j * jskipV, k * kskipV)
                   end do
                end do
             end do
@@ -821,7 +821,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = zstV(3), zenV(3)
                do j = zstV(2), zenV(2)
                   do i = zstV(1), zenV(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipV + 1, (j - 1)*jskipV + 1, (k - 1)*kskipV + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipV + 1, (j - 1) * jskipV + 1, (k - 1) * kskipV + 1)
                   end do
                end do
             end do
@@ -829,7 +829,7 @@ subroutine fine_to_coarseV(ipencil, var_fine, var_coarse)
             do k = zstV(3), zenV(3)
                do j = zstV(2), zenV(2)
                   do i = zstV(1), zenV(1)
-                     wk(i, j, k) = wk2(i*iskipV, j*jskipV, k*kskipV)
+                     wk(i, j, k) = wk2(i * iskipV, j * jskipV, k * kskipV)
                   end do
                end do
             end do
@@ -862,7 +862,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = xstP(3), xenP(3)
                do j = xstP(2), xenP(2)
                   do i = xstP(1), xenP(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipP + 1, (j - 1) * jskipP + 1, (k - 1) * kskipP + 1)
                   end do
                end do
             end do
@@ -870,7 +870,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = xstP(3), xenP(3)
                do j = xstP(2), xenP(2)
                   do i = xstP(1), xenP(1)
-                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                     wk(i, j, k) = wk2(i * iskipP, j * jskipP, k * kskipP)
                   end do
                end do
             end do
@@ -884,7 +884,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = ystP(3), yenP(3)
                do j = ystP(2), yenP(2)
                   do i = ystP(1), yenP(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipP + 1, (j - 1) * jskipP + 1, (k - 1) * kskipP + 1)
                   end do
                end do
             end do
@@ -892,7 +892,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = ystP(3), yenP(3)
                do j = ystP(2), yenP(2)
                   do i = ystP(1), yenP(1)
-                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                     wk(i, j, k) = wk2(i * iskipP, j * jskipP, k * kskipP)
                   end do
                end do
             end do
@@ -906,7 +906,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = zstP(3), zenP(3)
                do j = zstP(2), zenP(2)
                   do i = zstP(1), zenP(1)
-                     wk(i, j, k) = wk2((i - 1)*iskipP + 1, (j - 1)*jskipP + 1, (k - 1)*kskipP + 1)
+                     wk(i, j, k) = wk2((i - 1) * iskipP + 1, (j - 1) * jskipP + 1, (k - 1) * kskipP + 1)
                   end do
                end do
             end do
@@ -914,7 +914,7 @@ subroutine fine_to_coarseP(ipencil, var_fine, var_coarse)
             do k = zstP(3), zenP(3)
                do j = zstP(2), zenP(2)
                   do i = zstP(1), zenP(1)
-                     wk(i, j, k) = wk2(i*iskipP, j*jskipP, k*kskipP)
+                     wk(i, j, k) = wk2(i * iskipP, j * jskipP, k * kskipP)
                   end do
                end do
             end do
@@ -1005,8 +1005,8 @@ subroutine distribute(data1, proc, st, en, sz)
       integer data1, proc, st(0:proc - 1), en(0:proc - 1), sz(0:proc - 1)
       integer i, size1, nl, nu
 
-      size1 = data1/proc
-      nu = data1 - size1*proc
+      size1 = data1 / proc
+      nu = data1 - size1 * proc
       nl = proc - nu
       st(0) = 1
       sz(0) = size1
@@ -1082,8 +1082,8 @@ subroutine prepare_buffer(decomp)
       ! MPI_ALLTOALLV buffer information
 
       do i = 0, dims(1) - 1
-         decomp%x1cnts(i) = decomp%x1dist(i)*decomp%xsz(2)*decomp%xsz(3)
-         decomp%y1cnts(i) = decomp%ysz(1)*decomp%y1dist(i)*decomp%ysz(3)
+         decomp%x1cnts(i) = decomp%x1dist(i) * decomp%xsz(2) * decomp%xsz(3)
+         decomp%y1cnts(i) = decomp%ysz(1) * decomp%y1dist(i) * decomp%ysz(3)
          if (i == 0) then
             decomp%x1disp(i) = 0  ! displacement is 0-based index
             decomp%y1disp(i) = 0
@@ -1094,8 +1094,8 @@ subroutine prepare_buffer(decomp)
       end do
 
       do i = 0, dims(2) - 1
-         decomp%y2cnts(i) = decomp%ysz(1)*decomp%y2dist(i)*decomp%ysz(3)
-         decomp%z2cnts(i) = decomp%zsz(1)*decomp%zsz(2)*decomp%z2dist(i)
+         decomp%y2cnts(i) = decomp%ysz(1) * decomp%y2dist(i) * decomp%ysz(3)
+         decomp%z2cnts(i) = decomp%zsz(1) * decomp%zsz(2) * decomp%z2dist(i)
          if (i == 0) then
             decomp%y2disp(i) = 0  ! displacement is 0-based index
             decomp%z2disp(i) = 0
@@ -1117,12 +1117,12 @@ subroutine prepare_buffer(decomp)
       ! For unevenly distributed data, pad smaller messages. Note the
       ! last blocks along pencils always get assigned more mesh points
       ! for X <=> Y transposes
-      decomp%x1count = decomp%x1dist(dims(1) - 1)* &
-                       decomp%y1dist(dims(1) - 1)*decomp%xsz(3)
+      decomp%x1count = decomp%x1dist(dims(1) - 1) * &
+                       decomp%y1dist(dims(1) - 1) * decomp%xsz(3)
       decomp%y1count = decomp%x1count
       ! for Y <=> Z transposes
-      decomp%y2count = decomp%y2dist(dims(2) - 1)* &
-                       decomp%z2dist(dims(2) - 1)*decomp%zsz(1)
+      decomp%y2count = decomp%y2dist(dims(2) - 1) * &
+                       decomp%z2dist(dims(2) - 1) * decomp%zsz(1)
       decomp%z2count = decomp%y2count
 
       return
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index 7a8bd916..ff4c6c5e 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -74,7 +74,7 @@ subroutine decomp_2d_init_ref(nx, ny, nz, p_row, p_col, periodic_bc, comm)
         p_row = row
         p_col = col
      else
-        if (nproc /= p_row*p_col) then
+        if (nproc /= p_row * p_col) then
            errorcode = 1
            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
                                 'Invalid 2D processor grid - nproc /= p_row*p_col')
@@ -268,11 +268,11 @@ subroutine best_2d_grid(iproc, best_p_row, best_p_col)
      call findfactor(iproc, factors, nfact)
      if (nrank == 0) write (*, *) 'factors: ', (factors(i), i=1, nfact)
 
-     i_best = nfact/2 + 1
+     i_best = nfact / 2 + 1
      col = factors(i_best)
 
      best_p_col = col
-     best_p_row = iproc/col
+     best_p_row = iproc / col
      if (nrank == 0) print *, 'p_row x p_col', best_p_row, best_p_col
      if ((best_p_col == 1) .and. (nrank == 0)) then
         print *, 'WARNING: current 2D DECOMP set-up might not work'
diff --git a/src/factor.f90 b/src/factor.f90
index 5f7e530a..941c8378 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -35,7 +35,7 @@ subroutine findfactor(num, factors, nfact)
       m = int(sqrt(real(num)))
       nfact = 1
       do i = 1, m
-         if (num/i*i == num) then
+         if (num / i * i == num) then
             factors(nfact) = i
             nfact = nfact + 1
          end if
@@ -44,15 +44,15 @@ subroutine findfactor(num, factors, nfact)
 
       ! derive those > sqrt(num)
       if (factors(nfact)**2 /= num) then
-         do i = nfact + 1, 2*nfact
-            factors(i) = num/factors(2*nfact - i + 1)
+         do i = nfact + 1, 2 * nfact
+            factors(i) = num / factors(2 * nfact - i + 1)
          end do
-         nfact = nfact*2
+         nfact = nfact * 2
       else
-         do i = nfact + 1, 2*nfact - 1
-            factors(i) = num/factors(2*nfact - i)
+         do i = nfact + 1, 2 * nfact - 1
+            factors(i) = num / factors(2 * nfact - i)
          end do
-         nfact = nfact*2 - 1
+         nfact = nfact * 2 - 1
       end if
 
       return
@@ -76,7 +76,7 @@ subroutine primefactors(num, factors, nfact)
          if (mod(n, i) == 0) then
             factors(nfact) = i
             nfact = nfact + 1
-            n = n/i
+            n = n / i
          else
             i = i + 1
          end if
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index f8ae3fe9..34632a40 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -128,9 +128,9 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       call decomp_info_init(nx, ny, nz, ph)
    end if
    if (format == PHYSICAL_IN_X) then
-      call decomp_info_init(nx/2 + 1, ny, nz, sp)
+      call decomp_info_init(nx / 2 + 1, ny, nz, sp)
    else if (format == PHYSICAL_IN_Z) then
-      call decomp_info_init(nx, ny, nz/2 + 1, sp)
+      call decomp_info_init(nx, ny, nz / 2 + 1, sp)
    else
       call decomp_2d_abort(__FILE__, __LINE__, format, "Invalid value for format")
    end if
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 6f08dfe3..1776d601 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -65,7 +65,7 @@ subroutine c2c_1m_x_plan(plan1, decomp, cufft_type, worksize)
       istat = cufftMakePlanMany(plan1, 1, decomp%xsz(1), &
                                 decomp%xsz(1), 1, decomp%xsz(1), &
                                 decomp%xsz(1), 1, decomp%xsz(1), &
-                                cufft_type, decomp%xsz(2)*decomp%xsz(3), worksize)
+                                cufft_type, decomp%xsz(2) * decomp%xsz(3), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine c2c_1m_x_plan
@@ -117,9 +117,9 @@ subroutine c2c_1m_z_plan(plan1, decomp, cufft_type, worksize)
       istat = cufftSetAutoAllocation(plan1, 0)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
       istat = cufftMakePlanMany(plan1, 1, decomp%zsz(3), &
-                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
-                                decomp%zsz(3), decomp%zsz(1)*decomp%zsz(2), 1, &
-                                cufft_type, decomp%zsz(1)*decomp%zsz(2), worksize)
+                                decomp%zsz(3), decomp%zsz(1) * decomp%zsz(2), 1, &
+                                decomp%zsz(3), decomp%zsz(1) * decomp%zsz(2), 1, &
+                                cufft_type, decomp%zsz(1) * decomp%zsz(2), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine c2c_1m_z_plan
@@ -146,7 +146,7 @@ subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
       istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
                                 decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                 decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
-                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
+                                cufft_type, decomp_ph%xsz(2) * decomp_ph%xsz(3), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine r2c_1m_x_plan
@@ -173,7 +173,7 @@ subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
       istat = cufftMakePlanMany(plan1, 1, decomp_ph%xsz(1), &
                                 decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                 decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
-                                cufft_type, decomp_ph%xsz(2)*decomp_ph%xsz(3), worksize)
+                                cufft_type, decomp_ph%xsz(2) * decomp_ph%xsz(3), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine c2r_1m_x_plan
@@ -198,9 +198,9 @@ subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp, cufft_type, worksize)
       istat = cufftSetAutoAllocation(plan1, 0)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
       istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
-                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
-                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
-                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
+                                decomp_ph%zsz(3), decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, &
+                                decomp_sp%zsz(3), decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, &
+                                cufft_type, decomp_ph%zsz(1) * decomp_ph%zsz(2), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine r2c_1m_z_plan
@@ -225,9 +225,9 @@ subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph, cufft_type, worksize)
       istat = cufftSetAutoAllocation(plan1, 0)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftSetAutoAllocation")
       istat = cufftMakePlanMany(plan1, 1, decomp_ph%zsz(3), &
-                                decomp_sp%zsz(3), decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, &
-                                decomp_ph%zsz(3), decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, &
-                                cufft_type, decomp_ph%zsz(1)*decomp_ph%zsz(2), worksize)
+                                decomp_sp%zsz(3), decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, &
+                                decomp_ph%zsz(3), decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, &
+                                cufft_type, decomp_ph%zsz(1) * decomp_ph%zsz(2), worksize)
       if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cufftMakePlanMany")
 
    end subroutine c2r_1m_z_plan
@@ -369,7 +369,7 @@ subroutine init_fft_engine
 
       end if
 #endif
-      cufft_ws = cufft_ws/sizeof(1._mytype)
+      cufft_ws = cufft_ws / sizeof(1._mytype)
       allocate (cufft_workspace(cufft_ws))
       do j = 1, 3
          do i = -1, 2
@@ -732,9 +732,9 @@ subroutine fft_3d_r2c(in_r, out_c)
          !call nvtxStartRange("Z r2c_1m_z")
 #ifdef DEBUG
          dim3d = shape(in_r)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_r(i, j, k))
                end do
             end do
@@ -753,9 +753,9 @@ subroutine fft_3d_r2c(in_r, out_c)
 #ifdef DEBUG
             write (*, *) 'c2c_1m_y'
             dim3d = shape(wk2_r2c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
                         aimag(wk2_r2c(i, j, k))
                   end do
@@ -774,9 +774,9 @@ subroutine fft_3d_r2c(in_r, out_c)
 #ifdef DEBUG
             write (*, *) 'c2c_1m_y2'
             dim3d = shape(out_c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
                         aimag(out_c(i, j, k))
                   end do
@@ -799,9 +799,9 @@ subroutine fft_3d_r2c(in_r, out_c)
 #ifdef DEBUG
          write (*, *) 'c2c_1m_x'
          dim3d = shape(out_c)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_c(i, j, k)), &
                      aimag(out_c(i, j, k))
                end do
@@ -874,9 +874,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back Init c2c_1m_x line 788'
          dim3d = shape(in_c)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
                      aimag(in_c(i, j, k))
                end do
@@ -891,9 +891,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back c2c_1m_x overwrite line 804'
          dim3d = shape(in_c)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
                      aimag(in_c(i, j, k))
                end do
@@ -913,9 +913,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back2 c2c_1m_x line 821'
          dim3d = shape(wk1)
-         do k = 1, dim3d(3), dim3d(1)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(1) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
                      aimag(wk1(i, j, k))
                end do
@@ -937,9 +937,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
             write (*, *) 'Back c2c_1m_y line 844'
             dim3d = shape(wk2_r2c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk2_r2c(i, j, k)), &
                         aimag(wk2_r2c(i, j, k))
                   end do
@@ -954,9 +954,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
             write (*, *) 'Back2 c2c_1m_y line 860'
             dim3d = shape(in_c)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(in_c(i, j, k)), &
                         aimag(in_c(i, j, k))
                   end do
@@ -970,9 +970,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
             write (*, *) 'Back3 c2c_1m_y line 875'
             dim3d = shape(wk1)
-            do k = 1, dim3d(3), dim3d(3)/8
-               do j = 1, dim3d(2), dim3d(2)/8
-                  do i = 1, dim3d(1), dim3d(1)/8
+            do k = 1, dim3d(3), dim3d(3) / 8
+               do j = 1, dim3d(2), dim3d(2) / 8
+                  do i = 1, dim3d(1), dim3d(1) / 8
                      print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk1(i, j, k)), &
                         aimag(wk1(i, j, k))
                   end do
@@ -997,9 +997,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back2 after tr_y2z'
          dim3d = shape(wk13)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(wk13(i, j, k)), &
                      aimag(wk13(i, j, k))
                end do
@@ -1012,9 +1012,9 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef DEBUG
          write (*, *) 'Back2 c2r_1m_z out_r line 902'
          dim3d = shape(out_r)
-         do k = 1, dim3d(3), dim3d(3)/8
-            do j = 1, dim3d(2), dim3d(2)/8
-               do i = 1, dim3d(1), dim3d(1)/8
+         do k = 1, dim3d(3), dim3d(3) / 8
+            do j = 1, dim3d(2), dim3d(2) / 8
+               do i = 1, dim3d(1), dim3d(1) / 8
                   print "(i3,1x,i3,1x,i3,1x,e12.5,1x,e12.5)", i, j, k, real(out_r(i, j, k))
                end do
             end do
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index a9dc689e..ced99dbd 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -65,12 +65,12 @@ subroutine c2c_1m_x_plan(plan1, decomp, isign)
 
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                               decomp%xsz(2) * decomp%xsz(3), a1, decomp%xsz(1), 1, &
                                decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
                                isign, plan_type)
 #else
       call sfftw_plan_many_dft(plan1, 1, decomp%xsz(1), &
-                               decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                               decomp%xsz(2) * decomp%xsz(3), a1, decomp%xsz(1), 1, &
                                decomp%xsz(1), a1, decomp%xsz(1), 1, decomp%xsz(1), &
                                isign, plan_type)
 #endif
@@ -126,14 +126,14 @@ subroutine c2c_1m_z_plan(plan1, decomp, isign)
 
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+                               decomp%zsz(1) * decomp%zsz(2), a1, decomp%zsz(3), &
+                               decomp%zsz(1) * decomp%zsz(2), 1, a1, decomp%zsz(3), &
+                               decomp%zsz(1) * decomp%zsz(2), 1, isign, plan_type)
 #else
       call sfftw_plan_many_dft(plan1, 1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, a1, decomp%zsz(3), &
-                               decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+                               decomp%zsz(1) * decomp%zsz(2), a1, decomp%zsz(3), &
+                               decomp%zsz(1) * decomp%zsz(2), 1, a1, decomp%zsz(3), &
+                               decomp%zsz(1) * decomp%zsz(2), 1, isign, plan_type)
 #endif
 
       deallocate (a1)
@@ -157,12 +157,12 @@ subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
       allocate (a2(decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)))
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                   decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
                                    decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                    plan_type)
 #else
       call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                   decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
                                    decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                    plan_type)
 #endif
@@ -187,12 +187,12 @@ subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
       allocate (a2(decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)))
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                   decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
                                    decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                    plan_type)
 #else
       call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%xsz(1), &
-                                   decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                   decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
                                    decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                    plan_type)
 #endif
@@ -217,14 +217,14 @@ subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
       allocate (a2(decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)))
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, plan_type)
 #else
       call sfftw_plan_many_dft_r2c(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, plan_type)
 #endif
       deallocate (a1, a2)
 
@@ -248,14 +248,14 @@ subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
 
 #ifdef DOUBLE_PREC
       call dfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, plan_type)
 #else
       call sfftw_plan_many_dft_c2r(plan1, 1, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                   decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                   decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                   decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                   decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, plan_type)
 #endif
       deallocate (a1, a2)
 
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index d06475d1..4f0ceb46 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -156,9 +156,9 @@ subroutine fft_init_general(pencil, nx, ny, nz)
          call decomp_info_init(nx, ny, nz, ph)
       end if
       if (format == PHYSICAL_IN_X) then
-         call decomp_info_init(nx/2 + 1, ny, nz, sp)
+         call decomp_info_init(nx / 2 + 1, ny, nz, sp)
       else if (format == PHYSICAL_IN_Z) then
-         call decomp_info_init(nx, ny, nz/2 + 1, sp)
+         call decomp_info_init(nx, ny, nz / 2 + 1, sp)
       else
          call decomp_2d_abort(__FILE__, __LINE__, format, "Invalid value for format")
       end if
@@ -167,7 +167,7 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       ! Allocate the workspace fo intermediate y-pencil data
       ! The largest memory block needed is the one for c2c transforms
       !
-      sz = ph%ysz(1)*ph%ysz(2)*ph%ysz(3)
+      sz = ph%ysz(1) * ph%ysz(2) * ph%ysz(3)
       wk2_c2c_p = fftw_alloc_complex(sz)
       call c_f_pointer(wk2_c2c_p, wk2_c2c, [ph%ysz(1), ph%ysz(2), ph%ysz(3)])
       !
@@ -186,11 +186,11 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       ! transpose_y_to_z(wk2_r2c, wk13, sp)
       !
       if (format == PHYSICAL_IN_X) then
-         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+         sz = sp%xsz(1) * sp%xsz(2) * sp%xsz(3)
          wk13_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk13_p, wk13, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
       else if (format == PHYSICAL_IN_Z) then
-         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+         sz = sp%zsz(1) * sp%zsz(2) * sp%zsz(3)
          wk13_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk13_p, wk13, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
       end if
@@ -319,19 +319,19 @@ subroutine c2c_1m_x_plan(plan1, decomp, isign)
       type(C_PTR) :: a1_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)
+      sz = decomp%xsz(1) * decomp%xsz(2) * decomp%xsz(3)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
       call c_f_pointer(a1_p, a1o, [decomp%xsz(1), decomp%xsz(2), decomp%xsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft(1, decomp%xsz(1), &
-                                 decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                                 decomp%xsz(2) * decomp%xsz(3), a1, decomp%xsz(1), 1, &
                                  decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
                                  isign, plan_type)
 #else
       plan1 = fftwf_plan_many_dft(1, decomp%xsz(1), &
-                                  decomp%xsz(2)*decomp%xsz(3), a1, decomp%xsz(1), 1, &
+                                  decomp%xsz(2) * decomp%xsz(3), a1, decomp%xsz(1), 1, &
                                   decomp%xsz(1), a1o, decomp%xsz(1), 1, decomp%xsz(1), &
                                   isign, plan_type)
 #endif
@@ -362,7 +362,7 @@ subroutine c2c_1m_y_plan(plan1, decomp, isign)
 
       ! Due to memory pattern of 3D arrays, 1D FFTs along Y have to be
       ! done one Z-plane at a time. So plan for 2D data sets here.
-      sz = decomp%ysz(1)*decomp%ysz(2)
+      sz = decomp%ysz(1) * decomp%ysz(2)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, [decomp%ysz(1), decomp%ysz(2)])
       call c_f_pointer(a1_p, a1o, [decomp%ysz(1), decomp%ysz(2)])
@@ -401,21 +401,21 @@ subroutine c2c_1m_z_plan(plan1, decomp, isign)
       type(C_PTR) :: a1_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp%zsz(1)*decomp%zsz(2)*decomp%zsz(3)
+      sz = decomp%zsz(1) * decomp%zsz(2) * decomp%zsz(3)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
       call c_f_pointer(a1_p, a1o, [decomp%zsz(1), decomp%zsz(2), decomp%zsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft(1, decomp%zsz(3), &
-                                 decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                                 decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-                                 decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+                                 decomp%zsz(1) * decomp%zsz(2), a1, decomp%zsz(3), &
+                                 decomp%zsz(1) * decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+                                 decomp%zsz(1) * decomp%zsz(2), 1, isign, plan_type)
 #else
       plan1 = fftwf_plan_many_dft(1, decomp%zsz(3), &
-                                  decomp%zsz(1)*decomp%zsz(2), a1, decomp%zsz(3), &
-                                  decomp%zsz(1)*decomp%zsz(2), 1, a1o, decomp%zsz(3), &
-                                  decomp%zsz(1)*decomp%zsz(2), 1, isign, plan_type)
+                                  decomp%zsz(1) * decomp%zsz(2), a1, decomp%zsz(3), &
+                                  decomp%zsz(1) * decomp%zsz(2), 1, a1o, decomp%zsz(3), &
+                                  decomp%zsz(1) * decomp%zsz(2), 1, isign, plan_type)
 #endif
 
       call fftw_free(a1_p)
@@ -437,23 +437,23 @@ subroutine r2c_1m_x_plan(plan1, decomp_ph, decomp_sp)
       type(C_PTR) :: a1_p, a2_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+      sz = decomp_ph%xsz(1) * decomp_ph%xsz(2) * decomp_ph%xsz(3)
       a1_p = fftw_alloc_real(sz)
       call c_f_pointer(a1_p, a1, &
                        [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
-      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+      sz = decomp_sp%xsz(1) * decomp_sp%xsz(2) * decomp_sp%xsz(3)
       a2_p = fftw_alloc_complex(sz)
       call c_f_pointer(a2_p, a2, &
                        [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                     decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
                                      decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                      plan_type)
 #else
       plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%xsz(1), &
-                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
+                                      decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_ph%xsz(1), 1, &
                                       decomp_ph%xsz(1), a2, decomp_sp%xsz(1), 1, decomp_sp%xsz(1), &
                                       plan_type)
 #endif
@@ -478,23 +478,23 @@ subroutine c2r_1m_x_plan(plan1, decomp_sp, decomp_ph)
       type(C_PTR) :: a1_p, a2_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp_sp%xsz(1)*decomp_sp%xsz(2)*decomp_sp%xsz(3)
+      sz = decomp_sp%xsz(1) * decomp_sp%xsz(2) * decomp_sp%xsz(3)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, &
                        [decomp_sp%xsz(1), decomp_sp%xsz(2), decomp_sp%xsz(3)])
-      sz = decomp_ph%xsz(1)*decomp_ph%xsz(2)*decomp_ph%xsz(3)
+      sz = decomp_ph%xsz(1) * decomp_ph%xsz(2) * decomp_ph%xsz(3)
       a2_p = fftw_alloc_real(sz)
       call c_f_pointer(a2_p, a2, &
                        [decomp_ph%xsz(1), decomp_ph%xsz(2), decomp_ph%xsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-                                     decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                     decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
                                      decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                      plan_type)
 #else
       plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%xsz(1), &
-                                      decomp_ph%xsz(2)*decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
+                                      decomp_ph%xsz(2) * decomp_ph%xsz(3), a1, decomp_sp%xsz(1), 1, &
                                       decomp_sp%xsz(1), a2, decomp_ph%xsz(1), 1, decomp_ph%xsz(1), &
                                       plan_type)
 #endif
@@ -519,25 +519,25 @@ subroutine r2c_1m_z_plan(plan1, decomp_ph, decomp_sp)
       type(C_PTR) :: a1_p, a2_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+      sz = decomp_ph%zsz(1) * decomp_ph%zsz(2) * decomp_ph%zsz(3)
       a1_p = fftw_alloc_real(sz)
       call c_f_pointer(a1_p, a1, &
                        [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
-      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+      sz = decomp_sp%zsz(1) * decomp_sp%zsz(2) * decomp_sp%zsz(3)
       a2_p = fftw_alloc_complex(sz)
       call c_f_pointer(a2_p, a2, &
                        [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+                                     decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                     decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, plan_type)
 #else
       plan1 = fftwf_plan_many_dft_r2c(1, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
-                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, plan_type)
+                                      decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, a2, decomp_sp%zsz(3), &
+                                      decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, plan_type)
 #endif
 
       call fftw_free(a1_p)
@@ -560,25 +560,25 @@ subroutine c2r_1m_z_plan(plan1, decomp_sp, decomp_ph)
       type(C_PTR) :: a1_p, a2_p
       integer(C_SIZE_T) :: sz
 
-      sz = decomp_sp%zsz(1)*decomp_sp%zsz(2)*decomp_sp%zsz(3)
+      sz = decomp_sp%zsz(1) * decomp_sp%zsz(2) * decomp_sp%zsz(3)
       a1_p = fftw_alloc_complex(sz)
       call c_f_pointer(a1_p, a1, &
                        [decomp_sp%zsz(1), decomp_sp%zsz(2), decomp_sp%zsz(3)])
-      sz = decomp_ph%zsz(1)*decomp_ph%zsz(2)*decomp_ph%zsz(3)
+      sz = decomp_ph%zsz(1) * decomp_ph%zsz(2) * decomp_ph%zsz(3)
       a2_p = fftw_alloc_real(sz)
       call c_f_pointer(a2_p, a2, &
                        [decomp_ph%zsz(1), decomp_ph%zsz(2), decomp_ph%zsz(3)])
 
 #ifdef DOUBLE_PREC
       plan1 = fftw_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                     decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                     decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+                                     decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                     decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                     decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, plan_type)
 #else
       plan1 = fftwf_plan_many_dft_c2r(1, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
-                                      decomp_sp%zsz(1)*decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
-                                      decomp_ph%zsz(1)*decomp_ph%zsz(2), 1, plan_type)
+                                      decomp_ph%zsz(1) * decomp_ph%zsz(2), a1, decomp_sp%zsz(3), &
+                                      decomp_sp%zsz(1) * decomp_sp%zsz(2), 1, a2, decomp_ph%zsz(3), &
+                                      decomp_ph%zsz(1) * decomp_ph%zsz(2), 1, plan_type)
 #endif
 
       call fftw_free(a1_p)
@@ -823,7 +823,7 @@ subroutine fft_3d_c2c(in, out, isign)
 #ifdef OVERWRITE
          call c2c_1m_x(in, plan(isign, 1))
 #else
-         sz = ph%xsz(1)*ph%xsz(2)*ph%xsz(3)
+         sz = ph%xsz(1) * ph%xsz(2) * ph%xsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [ph%xsz(1), ph%xsz(2), ph%xsz(3)])
          wk1 = in
@@ -867,7 +867,7 @@ subroutine fft_3d_c2c(in, out, isign)
 #ifdef OVERWRITE
          call c2c_1m_z(in, plan(isign, 3))
 #else
-         sz = ph%zsz(1)*ph%zsz(2)*ph%zsz(3)
+         sz = ph%zsz(1) * ph%zsz(2) * ph%zsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [ph%zsz(1), ph%zsz(2), ph%zsz(3)])
          wk1 = in
@@ -1003,7 +1003,7 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef OVERWRITE
          call c2c_1m_z(in_c, plan(2, 3))
 #else
-         sz = sp%zsz(1)*sp%zsz(2)*sp%zsz(3)
+         sz = sp%zsz(1) * sp%zsz(2) * sp%zsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [sp%zsz(1), sp%zsz(2), sp%zsz(3)])
          wk1 = in_c
@@ -1032,7 +1032,7 @@ subroutine fft_3d_c2r(in_c, out_r)
 #ifdef OVERWRITE
          call c2c_1m_x(in_c, plan(2, 1))
 #else
-         sz = sp%xsz(1)*sp%xsz(2)*sp%xsz(3)
+         sz = sp%xsz(1) * sp%xsz(2) * sp%xsz(3)
          wk1_p = fftw_alloc_complex(sz)
          call c_f_pointer(wk1_p, wk1, [sp%xsz(1), sp%xsz(2), sp%xsz(3)])
          wk1 = in_c
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index cff64e98..c07c52e2 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -260,7 +260,7 @@ subroutine c2r_1m_x(input, output)
          do j = 1, d2
             ! Glassman's FFT is c2c only,
             ! needing some pre- and post-processing for c2r
-            do i = 1, d1/2 + 1
+            do i = 1, d1 / 2 + 1
                buf(i) = input(i, j, k)
             end do
             ! expanding to a full-size complex array
@@ -270,7 +270,7 @@ subroutine c2r_1m_x(input, output)
             ! For even N, the storage is:
             !  1, 2, ...... N/2  , N/2+1
             !     N, ...... N/2+2  again a(i) conjugate of a(N+2-i)
-            do i = d1/2 + 2, d1
+            do i = d1 / 2 + 2, d1
                buf(i) = conjg(buf(d1 + 2 - i))
             end do
             call spcfft(buf, d1, 1, scratch)
@@ -305,10 +305,10 @@ subroutine c2r_1m_z(input, output)
       !$acc parallel loop gang vector collapse(2) private(buf, scratch)
       do j = 1, d2
          do i = 1, d1
-            do k = 1, d3/2 + 1
+            do k = 1, d3 / 2 + 1
                buf(k) = input(i, j, k)
             end do
-            do k = d3/2 + 2, d3
+            do k = d3 / 2 + 2, d3
                buf(k) = conjg(buf(d3 + 2 - k))
             end do
             call spcfft(buf, d3, 1, scratch)
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index be4d419d..cf63924d 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -90,7 +90,7 @@ subroutine c2c_1m_x_plan(desc, decomp)
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp%xsz(2)*decomp%xsz(3))
+                            decomp%xsz(2) * decomp%xsz(3))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
 #ifdef OVERWRITE
       status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_INPLACE)
@@ -178,14 +178,14 @@ subroutine c2c_1m_z_plan(desc, decomp)
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp%zsz(1)*decomp%zsz(2))
+                            decomp%zsz(1) * decomp%zsz(2))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_INPUT_DISTANCE, 1)
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       strides(1) = 0
-      strides(2) = decomp%zsz(1)*decomp%zsz(2)
+      strides(2) = decomp%zsz(1) * decomp%zsz(2)
       status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
@@ -218,7 +218,7 @@ subroutine r2c_1m_x_plan(desc, decomp_ph, decomp_sp, direction)
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp_ph%xsz(2)*decomp_ph%xsz(3))
+                            decomp_ph%xsz(2) * decomp_ph%xsz(3))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
                             DFTI_COMPLEX_COMPLEX)
@@ -268,7 +268,7 @@ subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
 #endif
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiCreateDescriptor")
       status = DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, &
-                            decomp_ph%zsz(1)*decomp_ph%zsz(2))
+                            decomp_ph%zsz(1) * decomp_ph%zsz(2))
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       status = DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, &
                             DFTI_COMPLEX_COMPLEX)
@@ -280,14 +280,14 @@ subroutine r2c_1m_z_plan(desc, decomp_ph, decomp_sp, direction)
       status = DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, 1)
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
       strides(1) = 0
-      strides(2) = decomp_ph%zsz(1)*decomp_ph%zsz(2)
+      strides(2) = decomp_ph%zsz(1) * decomp_ph%zsz(2)
       if (direction == -1) then
          status = DftiSetValue(desc, DFTI_INPUT_STRIDES, strides)
       else if (direction == 1) then
          status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
       end if
       if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "DftiSetValue")
-      strides(2) = decomp_sp%zsz(1)*decomp_sp%zsz(2)
+      strides(2) = decomp_sp%zsz(1) * decomp_sp%zsz(2)
       if (direction == -1) then
          status = DftiSetValue(desc, DFTI_OUTPUT_STRIDES, strides)
       else if (direction == 1) then
diff --git a/src/glassman.f90 b/src/glassman.f90
index 9e2ca251..bd0e9ae6 100644
--- a/src/glassman.f90
+++ b/src/glassman.f90
@@ -52,12 +52,12 @@ PURE SUBROUTINE SPCFFT(U, N, ISIGN, WORK)
       INU = .TRUE.
 
       DO WHILE (B > 1)
-         A = C*A
+         A = C * A
          C = 2
          DO WHILE (MOD(B, C) /= 0)
             C = C + 1
          END DO
-         B = B/C
+         B = B / C
          IF (INU) THEN
             CALL SPCPFT(A, B, C, U, WORK, ISIGN)
          ELSE
@@ -90,7 +90,7 @@ PURE SUBROUTINE SPCPFT(A, B, C, UIN, UOUT, ISIGN)
       COMPLEX(mytype), INTENT(OUT) :: UOUT(B, A, C)
       COMPLEX(mytype) :: DELTA, OMEGA, SUM
 
-      ANGLE = 6.28318530717958_mytype/REAL(A*C, kind=mytype)
+      ANGLE = 6.28318530717958_mytype / REAL(A * C, kind=mytype)
       OMEGA = CMPLX(1.0, 0.0, kind=mytype)
 
       IF (ISIGN == 1) THEN
@@ -105,11 +105,11 @@ PURE SUBROUTINE SPCPFT(A, B, C, UIN, UOUT, ISIGN)
                SUM = UIN(IB, C, IA)
                DO JCR = 2, C
                   JC = C + 1 - JCR
-                  SUM = UIN(IB, JC, IA) + OMEGA*SUM
+                  SUM = UIN(IB, JC, IA) + OMEGA * SUM
                END DO
                UOUT(IB, IA, IC) = SUM
             END DO
-            OMEGA = DELTA*OMEGA
+            OMEGA = DELTA * OMEGA
          END DO
       END DO
 
@@ -148,7 +148,7 @@ subroutine glassman_3d_r2c(in_r, nx, ny, nz, out_c)
             end do
             call spcfft(buf, nx, -1, scratch)
             ! simply drop the redundant part of the complex output
-            do i = 1, nx/2 + 1
+            do i = 1, nx / 2 + 1
                out_c(i, j, k) = buf(i)
             end do
          end do
@@ -156,7 +156,7 @@ subroutine glassman_3d_r2c(in_r, nx, ny, nz, out_c)
 
       ! ===== 1D FFTs in Y =====
       do k = 1, nz
-         do i = 1, nx/2 + 1
+         do i = 1, nx / 2 + 1
             do j = 1, ny
                buf(j) = out_c(i, j, k)
             end do
@@ -169,7 +169,7 @@ subroutine glassman_3d_r2c(in_r, nx, ny, nz, out_c)
 
       ! ===== 1D FFTs in Z =====
       do j = 1, ny
-         do i = 1, nx/2 + 1
+         do i = 1, nx / 2 + 1
             do k = 1, nz
                buf(k) = out_c(i, j, k)
             end do
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index bf21f10f..2148b9f0 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -203,9 +203,9 @@
        else
           tag_n = coord(1) + 1
        end if
-       icount = s3 + 2*level
-       ilength = level*s1
-       ijump = s1*(s2 + 2*level)
+       icount = s3 + 2 * level
+       ilength = level * s1
+       ijump = s1 * (s2 + 2 * level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
                             data_type, halo12, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
@@ -254,7 +254,7 @@
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s1*(s2 + 2*level))*level
+       icount = (s1 * (s2 + 2 * level)) * level
        ! receive from bottom
        call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
                       neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
@@ -309,9 +309,9 @@
        else
           tag_e = coord(1) + 1
        end if
-       icount = s2*(s3 + 2*level)
+       icount = s2 * (s3 + 2 * level)
        ilength = level
-       ijump = s1 + 2*level
+       ijump = s1 + 2 * level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
                             data_type, halo21, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
@@ -363,7 +363,7 @@
        else
           tag_t = coord(2) + 1
        end if
-       icount = (s2*(s1 + 2*level))*level
+       icount = (s2 * (s1 + 2 * level)) * level
        ! receive from bottom
        call MPI_IRECV(out(xs, ys, zs), icount, data_type, &
                       neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
@@ -417,9 +417,9 @@
        else
           tag_e = coord(1) + 1
        end if
-       icount = (s2 + 2*level)*s3
+       icount = (s2 + 2 * level) * s3
        ilength = level
-       ijump = s1 + 2*level
+       ijump = s1 + 2 * level
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
                             data_type, halo31, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
@@ -466,8 +466,8 @@
           tag_n = coord(2) + 1
        end if
        icount = s3
-       ilength = level*(s1 + 2*level)
-       ijump = (s1 + 2*level)*(s2 + 2*level)
+       ilength = level * (s1 + 2 * level)
+       ijump = (s1 + 2 * level) * (s2 + 2 * level)
        call MPI_TYPE_VECTOR(icount, ilength, ijump, &
                             data_type, halo32, ierror)
        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
diff --git a/src/io.f90 b/src/io.f90
index 594e7b69..6096a1c1 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -355,14 +355,14 @@ subroutine read_one_real(ipencil, var, dirname, varname, io_name, opt_decomp, re
          if (read_reduce_prec) then
             allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
             call MPI_FILE_READ_ALL(fh, varsingle, &
-                                   subsizes(1)*subsizes(2)*subsizes(3), &
+                                   subsizes(1) * subsizes(2) * subsizes(3), &
                                    data_type, MPI_STATUS_IGNORE, ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
             var = real(varsingle, mytype)
             deallocate (varsingle)
          else
             call MPI_FILE_READ_ALL(fh, var, &
-                                   subsizes(1)*subsizes(2)*subsizes(3), &
+                                   subsizes(1) * subsizes(2) * subsizes(3), &
                                    data_type, MPI_STATUS_IGNORE, ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
          end if
@@ -370,9 +370,9 @@ subroutine read_one_real(ipencil, var, dirname, varname, io_name, opt_decomp, re
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
          disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                *int(disp_bytes, kind=MPI_OFFSET_KIND)
+                * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                * int(disp_bytes, kind=MPI_OFFSET_KIND)
       end associate
 
       if (opened_new) then
@@ -668,7 +668,7 @@ subroutine write_scalar_real(fh, disp, n, var)
                               MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine write_scalar_real
@@ -696,8 +696,8 @@ subroutine write_scalar_complex(fh, disp, n, var)
                               MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-             *2_MPI_OFFSET_KIND
+             * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+             * 2_MPI_OFFSET_KIND
 
       return
    end subroutine write_scalar_complex
@@ -727,7 +727,7 @@ subroutine write_scalar_integer(fh, disp, n, var)
       call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+             * int(m, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine write_scalar_integer
@@ -757,7 +757,7 @@ subroutine write_scalar_logical(fh, disp, n, var)
       call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+             * int(m, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine write_scalar_logical
@@ -787,7 +787,7 @@ subroutine read_scalar_real(fh, disp, n, var)
                              MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+             * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine read_scalar_real
@@ -810,8 +810,8 @@ subroutine read_scalar_complex(fh, disp, n, var)
                              MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(mytype_bytes, kind=MPI_OFFSET_KIND) &
-             *2_MPI_OFFSET_KIND
+             * int(mytype_bytes, kind=MPI_OFFSET_KIND) &
+             * 2_MPI_OFFSET_KIND
 
       return
    end subroutine read_scalar_complex
@@ -836,7 +836,7 @@ subroutine read_scalar_integer(fh, disp, n, var)
       call MPI_TYPE_SIZE(MPI_INTEGER, m, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+             * int(m, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine read_scalar_integer
@@ -861,7 +861,7 @@ subroutine read_scalar_logical(fh, disp, n, var)
       call MPI_TYPE_SIZE(MPI_LOGICAL, m, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
       disp = disp + int(n, kind=MPI_OFFSET_KIND) &
-             *int(m, kind=MPI_OFFSET_KIND)
+             * int(m, kind=MPI_OFFSET_KIND)
 
       return
    end subroutine read_scalar_logical
@@ -1312,22 +1312,22 @@ subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_n
                                 newtype, 'native', MPI_INFO_NULL, ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
          call MPI_FILE_WRITE_ALL(fh_registry(idx), varsingle, &
-                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 subsizes(1) * subsizes(2) * subsizes(3), &
                                  real_type_single, MPI_STATUS_IGNORE, ierror)
       else
          call MPI_FILE_SET_VIEW(fh_registry(idx), fh_disp(idx), real_type, &
                                 newtype, 'native', MPI_INFO_NULL, ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
          call MPI_FILE_WRITE_ALL(fh_registry(idx), var, &
-                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 subsizes(1) * subsizes(2) * subsizes(3), &
                                  real_type, MPI_STATUS_IGNORE, ierror)
       end if
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
 
       fh_disp(idx) = fh_disp(idx) + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                     *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                     *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                     *int(disp_bytes, kind=MPI_OFFSET_KIND)
+                     * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                     * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                     * int(disp_bytes, kind=MPI_OFFSET_KIND)
 
       if (opened_new) then
          call decomp_2d_close_io(io_name, full_io_name)
@@ -1537,7 +1537,7 @@ subroutine mpiio_write_real_probe(ipencil, var, filename, nlength)
                              newtype, 'native', MPI_INFO_NULL, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
       call MPI_FILE_WRITE_ALL(fh, var, &
-                              subsizes(1)*subsizes(2)*subsizes(3)*subsizes(4), &
+                              subsizes(1) * subsizes(2) * subsizes(3) * subsizes(4), &
                               real_type, MPI_STATUS_IGNORE, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
       call MPI_FILE_CLOSE(fh, ierror)
@@ -1750,7 +1750,7 @@ subroutine write_subdomain(ipencil, var, is, ie, js, je, ks, ke, filename)
                                 newtype, 'native', MPI_INFO_NULL, ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
          call MPI_FILE_WRITE_ALL(fh, wk, &
-                                 subsizes(1)*subsizes(2)*subsizes(3), &
+                                 subsizes(1) * subsizes(2) * subsizes(3), &
                                  data_type, MPI_STATUS_IGNORE, ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
          call MPI_FILE_CLOSE(fh, ierror)
diff --git a/src/io_read_inflow.f90 b/src/io_read_inflow.f90
index 162510c7..75fa9488 100644
--- a/src/io_read_inflow.f90
+++ b/src/io_read_inflow.f90
@@ -50,7 +50,7 @@
                           newtype, 'native', MPI_INFO_NULL, ierror)
    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
    call MPI_FILE_READ_ALL(fh, var, &
-                          subsizes(1)*subsizes(2)*subsizes(3), &
+                          subsizes(1) * subsizes(2) * subsizes(3), &
                           data_type, MPI_STATUS_IGNORE, ierror)
    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_READ_ALL")
    call MPI_TYPE_FREE(newtype, ierror)
@@ -58,13 +58,13 @@
 
    ! update displacement for the next read operation
    disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-          *int(sizes(2), kind=MPI_OFFSET_KIND) &
-          *int(sizes(3), kind=MPI_OFFSET_KIND) &
-          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+          * int(sizes(2), kind=MPI_OFFSET_KIND) &
+          * int(sizes(3), kind=MPI_OFFSET_KIND) &
+          * int(mytype_bytes, kind=MPI_OFFSET_KIND)
    if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                         * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                         * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                         * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
 associate (vnm => varname) ! Silence unused argument
diff --git a/src/io_write_outflow.f90 b/src/io_write_outflow.f90
index fe126584..9e9e4f98 100644
--- a/src/io_write_outflow.f90
+++ b/src/io_write_outflow.f90
@@ -50,7 +50,7 @@
                           newtype, 'native', MPI_INFO_NULL, ierror)
    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
    call MPI_FILE_WRITE_ALL(fh, var, &
-                           subsizes(1)*subsizes(2)*subsizes(3), &
+                           subsizes(1) * subsizes(2) * subsizes(3), &
                            data_type, MPI_STATUS_IGNORE, ierror)
    if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_WRITE_ALL")
    call MPI_TYPE_FREE(newtype, ierror)
@@ -58,13 +58,13 @@
 
    ! update displacement for the next write operation
    disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-          *int(sizes(2), kind=MPI_OFFSET_KIND) &
-          *int(sizes(3), kind=MPI_OFFSET_KIND) &
-          *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+          * int(sizes(2), kind=MPI_OFFSET_KIND) &
+          * int(sizes(3), kind=MPI_OFFSET_KIND) &
+          * int(mytype_bytes, kind=MPI_OFFSET_KIND)
    if (data_type == complex_type) disp = disp + int(sizes(1), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(2), kind=MPI_OFFSET_KIND) &
-                                         *int(sizes(3), kind=MPI_OFFSET_KIND) &
-                                         *int(mytype_bytes, kind=MPI_OFFSET_KIND)
+                                         * int(sizes(2), kind=MPI_OFFSET_KIND) &
+                                         * int(sizes(3), kind=MPI_OFFSET_KIND) &
+                                         * int(mytype_bytes, kind=MPI_OFFSET_KIND)
 end associate
 
 associate (vnm => varname) ! Silence unused dummy argument
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index 9cae71f0..ea8f4a74 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -274,14 +274,14 @@ subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -326,14 +326,14 @@ subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -378,14 +378,14 @@ subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -430,14 +430,14 @@ subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index db2e439a..2b1a1efd 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -274,14 +274,14 @@ subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -326,14 +326,14 @@ subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -378,14 +378,14 @@ subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -430,14 +430,14 @@ subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2*n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 7d28cb84..cca04207 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -136,7 +136,7 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
 #if defined(_GPU)
      !If one of the array in cuda call is not device we need to add acc host_data
      !$acc host_data use_device(dst)
-     istat = cudaMemcpy(dst, work2_r_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
+     istat = cudaMemcpy(dst, work2_r_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -274,7 +274,7 @@ subroutine transpose_y_to_z_complex(src, dst, decomp)
 
 #if defined(_GPU)
      !$acc host_data use_device(dst)
-     istat = cudaMemcpy(dst, work2_c_d, d1*d2*d3, cudaMemcpyDeviceToDevice)
+     istat = cudaMemcpy(dst, work2_c_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -315,14 +315,14 @@ subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -367,14 +367,14 @@ subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1*(i2 - i1 + 1), in(1, i1, 1), n1*n2, n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -415,7 +415,7 @@ subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
 #endif
@@ -456,7 +456,7 @@ subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
 #endif
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index c408696f..35d6714e 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -87,7 +87,7 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
 
 #if defined(_GPU)
      !$acc host_data use_device(src)
-     istat = cudaMemcpy(work1_r_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice)
+     istat = cudaMemcpy(work1_r_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -223,7 +223,7 @@ subroutine transpose_z_to_y_complex(src, dst, decomp)
 
 #if defined(_GPU)
      !$acc host_data use_device(src)
-     istat = cudaMemcpy(work1_c_d, src, s1*s2*s3, cudaMemcpyDeviceToDevice)
+     istat = cudaMemcpy(work1_c_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
      !$acc end host_data
      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
@@ -310,7 +310,7 @@ subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
 #endif
@@ -351,7 +351,7 @@ subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
         pos = decomp%z2disp(m) + 1
 #endif
@@ -396,14 +396,14 @@ subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
@@ -449,14 +449,14 @@ subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
         end if
 
 #ifdef EVEN
-        pos = m*decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
         !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1*n2, in(pos), n1*(i2 - i1 + 1), n1*(i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
         !$acc end host_data
         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else

From 73a75fc30d4e0d9badf3ab0f95886b2b86b386cd Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 20 Mar 2023 13:58:39 +0000
Subject: [PATCH 186/436] Add config file for fprettify

---
 scripts/.fprettifyrc | 5 +++++
 scripts/format.sh    | 7 ++-----
 2 files changed, 7 insertions(+), 5 deletions(-)
 create mode 100644 scripts/.fprettifyrc

diff --git a/scripts/.fprettifyrc b/scripts/.fprettifyrc
new file mode 100644
index 00000000..80316756
--- /dev/null
+++ b/scripts/.fprettifyrc
@@ -0,0 +1,5 @@
+# Configuration options for fprettify
+
+whitespace = 3             # Whitespace preset - operators, print/read, *, /, +, -
+enable-replacements = true # Replaces relational operators, e.g. .lt. -> <
+c-relations = true         # Use C-style relational operators, e.g. <=
diff --git a/scripts/format.sh b/scripts/format.sh
index 5c202a4d..50658aaf 100644
--- a/scripts/format.sh
+++ b/scripts/format.sh
@@ -8,8 +8,5 @@
 SCRDIR=$( dirname -- "$0"; )
 D2DDIR=${SCRDIR}/../
 
-# fprettify options
-# --enable-replacements : Replaces relational operators, e.g. .lt. -> <
-# --c-relations : Use C-style relational operators, e.g. <=
-# -w 3 : Whitespace preset - operators, print/read, *, /, +, -
-fprettify --enable-replacements --c-relations -w 3 ${D2DDIR}/src/*f90 ${D2DDIR}/examples/*/*f90
+# Run fprettify with config file scripts/.fprettifyrc
+fprettify -c ${SCRDIR}/.fprettifyrc ${D2DDIR}/src/*f90 ${D2DDIR}/examples/*/*f90

From 432f7b728b7102f31882bb0de8960492d80b2a1d Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 20 Mar 2023 17:22:33 +0100
Subject: [PATCH 187/436] Print major and minor version number

---
 src/decomp_2d_constants.f90 | 9 +++++++++
 src/log.f90                 | 7 +++++++
 2 files changed, 16 insertions(+)

diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 28a3d349..1398716d 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -110,5 +110,14 @@ module decomp_2d_constants
    integer, parameter, public :: PHYSICAL_IN_X = 1 ! Forward is input in X, output in Z
    integer, parameter, public :: PHYSICAL_IN_Z = 3 ! Forward is input in Z, output in X
 
+   !
+   ! Major and minor version number
+   !
+   integer, parameter :: D2D_MAJOR = 2
+   integer, parameter :: D2D_MINOR = 0
+   logical, parameter :: D2D_RELEASE = .false.
+   character, parameter :: D2D_LOG_MAJOR = char(48 + D2D_MAJOR)
+   character, parameter :: D2D_LOG_MINOR = char(48 + D2D_MINOR)
+
 end module decomp_2d_constants
 
diff --git a/src/log.f90 b/src/log.f90
index 809d5820..13a4db0a 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -136,6 +136,13 @@ module subroutine d2d_listing(given_io_unit)
       write (io_unit, *) '=================== Decomp2D - log ========================'
       write (io_unit, *) '==========================================================='
 
+      ! Major and minor version number
+      if (D2D_RELEASE) then
+         write (io_unit, *) 'Release ', D2D_LOG_MAJOR, '.', D2D_LOG_MINOR
+      else
+         write (io_unit, *) 'Release ', D2D_LOG_MAJOR, '.', D2D_LOG_MINOR, '.alpha'
+      end if
+
       ! Git hash if available
 #if defined(VERSION)
       write (io_unit, *) 'Git version        : ', VERSION

From 6fa32d92e7ae7f9b57b00807ad30512b3afb1178 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 20 Mar 2023 19:23:46 +0100
Subject: [PATCH 188/436] Minor update

---
 src/decomp_2d_constants.f90 | 2 --
 src/log.f90                 | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 1398716d..6944878c 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -116,8 +116,6 @@ module decomp_2d_constants
    integer, parameter :: D2D_MAJOR = 2
    integer, parameter :: D2D_MINOR = 0
    logical, parameter :: D2D_RELEASE = .false.
-   character, parameter :: D2D_LOG_MAJOR = char(48 + D2D_MAJOR)
-   character, parameter :: D2D_LOG_MINOR = char(48 + D2D_MINOR)
 
 end module decomp_2d_constants
 
diff --git a/src/log.f90 b/src/log.f90
index 13a4db0a..4525bc52 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -138,9 +138,9 @@ module subroutine d2d_listing(given_io_unit)
 
       ! Major and minor version number
       if (D2D_RELEASE) then
-         write (io_unit, *) 'Release ', D2D_LOG_MAJOR, '.', D2D_LOG_MINOR
+         write (io_unit, "(A,I0,A,I0)") ' Release ', D2D_MAJOR, '.', D2D_MINOR
       else
-         write (io_unit, *) 'Release ', D2D_LOG_MAJOR, '.', D2D_LOG_MINOR, '.alpha'
+         write (io_unit, "(A,I0,A,I0,A)") ' Release ', D2D_MAJOR, '.', D2D_MINOR, '.alpha'
       end if
 
       ! Git hash if available

From 2c11ab408ecd812156a00af3783053d797920323 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 31 Mar 2023 09:53:30 +0100
Subject: [PATCH 189/436] Add initial CMake package configuration

---
 CMakeLists.txt                        |  4 +++-
 cmake/2decomp-config-version.cmake.in |  5 +++++
 cmake/2decomp-config.cmake.in         | 14 ++++++++++++++
 src/CMakeLists.txt                    | 18 ++++++++++++++++++
 4 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 cmake/2decomp-config-version.cmake.in
 create mode 100644 cmake/2decomp-config.cmake.in

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8c48354b..bfeca3e8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,7 +1,9 @@
 cmake_minimum_required(VERSION 3.12)                        
 cmake_policy(SET CMP0074 NEW)
 
-project(2decomp LANGUAGES Fortran)
+project(2decomp
+  LANGUAGES Fortran)
+set(version 2.0.0)
 enable_testing()
 
 
diff --git a/cmake/2decomp-config-version.cmake.in b/cmake/2decomp-config-version.cmake.in
new file mode 100644
index 00000000..95bb9ec2
--- /dev/null
+++ b/cmake/2decomp-config-version.cmake.in
@@ -0,0 +1,5 @@
+#
+# Versioning file for 2decomp&fft
+#
+
+set(PACKAGE_VERSION "@version@")
diff --git a/cmake/2decomp-config.cmake.in b/cmake/2decomp-config.cmake.in
new file mode 100644
index 00000000..4438351b
--- /dev/null
+++ b/cmake/2decomp-config.cmake.in
@@ -0,0 +1,14 @@
+#
+# 2decomp CMake configuration.
+# This is used by other packages to configure themselves against 2decomp&fft.
+#
+
+# Compute installation prefix relative to this file
+get_filename_component(_dir "${CMAKE_CURRENT_LIST_FILE}" PATH)
+get_filename_component(_prefix "${_dir}/../.." ABSOLUTE)
+
+# Import the targets
+include("${_prefix}/lib/2decomp-targets.cmake")
+
+
+
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 62623547..f47695b7 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -69,6 +69,7 @@ if (HALO_DEBUG)
 endif()
 
 install(TARGETS decomp2d
+  EXPORT decomp2d-targets
   RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
   LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
   ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
@@ -78,3 +79,20 @@ install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/ DESTINATION ${CMAKE_INSTALL
 install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
   FILES_MATCHING PATTERN "*.smod")
       
+## Packaging
+install(EXPORT decomp2d-targets
+  DESTINATION ${CMAKE_INSTALL_LIBDIR})
+
+configure_file(
+  ${CMAKE_SOURCE_DIR}/cmake/2decomp-config.cmake.in
+  ${PROJECT_BINARY_DIR}/cmake/2decomp-config.cmake @ONLY
+)
+configure_file(
+  ${CMAKE_SOURCE_DIR}/cmake/2decomp-config-version.cmake.in
+  ${PROJECT_BINARY_DIR}/cmake/2decomp-config-version.cmake
+)
+
+install(FILES
+  ${PROJECT_BINARY_DIR}/cmake/2decomp-config.cmake
+  ${PROJECT_BINARY_DIR}/cmake/2decomp-config-version.cmake
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/2decomp)

From d27958239afbc05f5cc31064ae69172f88193bac Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 31 Mar 2023 10:52:28 +0100
Subject: [PATCH 190/436] Rename project 2decomp -> decomp2d

CMake requires directories/files/environment variables of the form
${PROJECT_NAME}-X and 2decomp appears to be invalid as an environment
variable.
---
 CMakeLists.txt                                     |  2 +-
 ...n.cmake.in => decomp2d-config-version.cmake.in} |  0
 ...mp-config.cmake.in => decomp2d-config.cmake.in} |  4 ++--
 src/CMakeLists.txt                                 | 14 +++++++-------
 4 files changed, 10 insertions(+), 10 deletions(-)
 rename cmake/{2decomp-config-version.cmake.in => decomp2d-config-version.cmake.in} (100%)
 rename cmake/{2decomp-config.cmake.in => decomp2d-config.cmake.in} (76%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bfeca3e8..3d6290c7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.12)                        
 cmake_policy(SET CMP0074 NEW)
 
-project(2decomp
+project(decomp2d
   LANGUAGES Fortran)
 set(version 2.0.0)
 enable_testing()
diff --git a/cmake/2decomp-config-version.cmake.in b/cmake/decomp2d-config-version.cmake.in
similarity index 100%
rename from cmake/2decomp-config-version.cmake.in
rename to cmake/decomp2d-config-version.cmake.in
diff --git a/cmake/2decomp-config.cmake.in b/cmake/decomp2d-config.cmake.in
similarity index 76%
rename from cmake/2decomp-config.cmake.in
rename to cmake/decomp2d-config.cmake.in
index 4438351b..5a06051e 100644
--- a/cmake/2decomp-config.cmake.in
+++ b/cmake/decomp2d-config.cmake.in
@@ -8,7 +8,7 @@ get_filename_component(_dir "${CMAKE_CURRENT_LIST_FILE}" PATH)
 get_filename_component(_prefix "${_dir}/../.." ABSOLUTE)
 
 # Import the targets
-include("${_prefix}/lib/2decomp-targets.cmake")
-
+include("${_prefix}/lib/decomp2d-targets.cmake")
 
+set(2decomp_INCLUDE_DIR "${_prefix}/include")
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index f47695b7..8b8a9391 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -84,15 +84,15 @@ install(EXPORT decomp2d-targets
   DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
 configure_file(
-  ${CMAKE_SOURCE_DIR}/cmake/2decomp-config.cmake.in
-  ${PROJECT_BINARY_DIR}/cmake/2decomp-config.cmake @ONLY
+  ${CMAKE_SOURCE_DIR}/cmake/decomp2d-config.cmake.in
+  ${PROJECT_BINARY_DIR}/cmake/decomp2d-config.cmake @ONLY
 )
 configure_file(
-  ${CMAKE_SOURCE_DIR}/cmake/2decomp-config-version.cmake.in
-  ${PROJECT_BINARY_DIR}/cmake/2decomp-config-version.cmake
+  ${CMAKE_SOURCE_DIR}/cmake/decomp2d-config-version.cmake.in
+  ${PROJECT_BINARY_DIR}/cmake/decomp2d-config-version.cmake
 )
 
 install(FILES
-  ${PROJECT_BINARY_DIR}/cmake/2decomp-config.cmake
-  ${PROJECT_BINARY_DIR}/cmake/2decomp-config-version.cmake
-  DESTINATION ${CMAKE_INSTALL_LIBDIR}/2decomp)
+  ${PROJECT_BINARY_DIR}/cmake/decomp2d-config.cmake
+  ${PROJECT_BINARY_DIR}/cmake/decomp2d-config-version.cmake
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/decomp2d)

From f60b9f642c6a89a0a4bb79549b476eb2933c7cfa Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 31 Mar 2023 12:22:17 +0100
Subject: [PATCH 191/436] Add in-build support for cmake packages

---
 cmake/decomp2d-build_config.cmake.in | 3 +++
 src/CMakeLists.txt                   | 8 ++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 cmake/decomp2d-build_config.cmake.in

diff --git a/cmake/decomp2d-build_config.cmake.in b/cmake/decomp2d-build_config.cmake.in
new file mode 100644
index 00000000..8c4daec3
--- /dev/null
+++ b/cmake/decomp2d-build_config.cmake.in
@@ -0,0 +1,3 @@
+include("@PROJECT_BINARY_DIR@/decomp2d-targets.cmake")
+
+set(decomp2d_INCLUDE_DIRS "@CMAKE_Fortran_MODULE_DIRECTORY@")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 8b8a9391..165506aa 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -96,3 +96,11 @@ install(FILES
   ${PROJECT_BINARY_DIR}/cmake/decomp2d-config.cmake
   ${PROJECT_BINARY_DIR}/cmake/decomp2d-config-version.cmake
   DESTINATION ${CMAKE_INSTALL_LIBDIR}/decomp2d)
+
+# Make 2decomp usable from build tree.
+export(TARGETS decomp2d
+  FILE decomp2d-targets.cmake)
+configure_file(
+  ${CMAKE_SOURCE_DIR}/cmake/decomp2d-build_config.cmake.in
+  ${PROJECT_BINARY_DIR}/decomp2d-config.cmake @ONLY
+)

From 485ad558a11deda747e6743aae0b2bfc94abb70b Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 31 Mar 2023 12:31:07 +0100
Subject: [PATCH 192/436] Ensure decomp2d-targets.cmake goes to correct
 location

---
 src/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 165506aa..1b187eeb 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -99,7 +99,7 @@ install(FILES
 
 # Make 2decomp usable from build tree.
 export(TARGETS decomp2d
-  FILE decomp2d-targets.cmake)
+  FILE ${PROJECT_BINARY_DIR}/decomp2d-targets.cmake)
 configure_file(
   ${CMAKE_SOURCE_DIR}/cmake/decomp2d-build_config.cmake.in
   ${PROJECT_BINARY_DIR}/decomp2d-config.cmake @ONLY

From 39204459eb6ce344843867df675e0f267f53e060 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 3 Apr 2023 17:02:42 +0100
Subject: [PATCH 193/436] Update README to include example of Makefile code to
 add decomp2d-fft as external library

---
 README.md | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cbb4242c..fc6adf7c 100644
--- a/README.md
+++ b/README.md
@@ -141,6 +141,17 @@ The external code can use the named variables to check the FFT backend used in a
 ### Codes using Makefiles
 
 When building a code that links 2decomp-fft using a Makefile you will need to add the include and link paths as appropriate (`inlude/` and `link/` under the installation directory, respectively).
+```
+DECOMP_ROOT = /path/to/2decomp-fft
+DECOMP_BUILD_DIR = $(DECOMP_ROOT)/build
+DECOMP_INSTALL_DIR ?= $(DECOMP_BUILD_DIR)/opt # Use default unless set by user
+
+INC += -I$(DECOMP_INSTALL_DIR)/include
+
+# Users build/link targets
+LFLAGS += -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
+
+```
 It is also possible to drive the build and installation of 2decomp-fft from a Makefile such as in the following example code
 ```
 FC = mpif90
@@ -153,7 +164,7 @@ DECOMP_INSTALL_DIR ?= $(DECOMP_BUILD_DIR)/opt # Use default unless set by user
 INC += -I$(DECOMP_INSTALL_DIR)/include
 
 # Users build/link targets
-LFLAGS += -L$(DECOMP_INSTALL_DIR)/lib -ldecomp.a
+LFLAGS += -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
 
 # Building libdecomp.a
 $(DECOMP_INSTALL_DIR)/lib/libdecomp.a:

From c0e8281a1f81e5128f48053c5c3ce0e3f1e02ec6 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 10 Apr 2023 11:02:00 +0200
Subject: [PATCH 194/436] Update CMake scripts and README to help linking
 against fftw3.

---
 README.md                              |  5 +++--
 examples/fft_physical_x/CMakeLists.txt | 15 +++++++++------
 examples/fft_physical_z/CMakeLists.txt | 11 +++++++----
 src/CMakeLists.txt                     | 11 +++++++++--
 4 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index cbb4242c..274b38c7 100644
--- a/README.md
+++ b/README.md
@@ -244,10 +244,11 @@ make -j
 make -j check
 make install
 ```
+Please note that the resulting build is not compatible with CMake ([fftw3#130][https://github.com/FFTW/fftw3/issues/130]). As a workaround, one can open the file `xxxxxxx/fftw3/fftw-3.3.10_bld/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line `include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")`.
 
-To build `2decomp&fft` against fftw3 first ensure the package configuration for fftw3 is in your `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`, then either specify on the command line when configuring the build
+To build `2decomp&fft` against fftw3, one can provide the package configuration for fftw3 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`. One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. Then either specify on the command line when configuring the build
 ```
-cmake -S . -B build -DFFT_Choice=<fftw|fftw_f03>
+cmake -S . -B build -DFFT_Choice=<fftw|fftw_f03> -DFFTW_ROOT=/path/to/fftw3/install
 ```
 or modify the build configuration using `ccmake`.
 
diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index 61d7543c..a0b92408 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -12,12 +12,15 @@ target_link_libraries(fft_c2c_x PRIVATE decomp2d)
 target_link_libraries(fft_r2c_x PRIVATE decomp2d)
 target_link_libraries(fft_grid_x PRIVATE decomp2d)
 if (FFTW_FOUND)
-  target_link_libraries(fft_c2c_x PRIVATE ${FFTW_DOUBLE_LIB})
-  target_link_libraries(fft_c2c_x PRIVATE ${FFTW_FLOAT_LIB})
-  target_link_libraries(fft_r2c_x PRIVATE ${FFTW_DOUBLE_LIB})
-  target_link_libraries(fft_r2c_x PRIVATE ${FFTW_FLOAT_LIB})
-  target_link_libraries(fft_grid_x PRIVATE ${FFTW_DOUBLE_LIB})
-  target_link_libraries(fft_grid_x PRIVATE ${FFTW_FLOAT_LIB})
+  if (FFTW_DOUBLE_LIB_FOUND)
+    target_link_libraries(fft_c2c_x PRIVATE ${FFTW_DOUBLE_LIB})
+    target_link_libraries(fft_r2c_x PRIVATE ${FFTW_DOUBLE_LIB})
+    target_link_libraries(fft_grid_x PRIVATE ${FFTW_DOUBLE_LIB})
+  elseif (FFTW_FLOAT_LIB_FOUND)
+    target_link_libraries(fft_c2c_x PRIVATE ${FFTW_FLOAT_LIB})
+    target_link_libraries(fft_r2c_x PRIVATE ${FFTW_FLOAT_LIB})
+    target_link_libraries(fft_grid_x PRIVATE ${FFTW_FLOAT_LIB})
+  endif (FFTW_DOUBLE_LIB_FOUND)
 elseif (MKL_FOUND)
   target_link_libraries(fft_c2c_x PUBLIC $<LINK_ONLY:MKL::MKL>)
   target_link_libraries(fft_r2c_x PUBLIC $<LINK_ONLY:MKL::MKL>)
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index 04e25f48..f23ae7cd 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -9,10 +9,13 @@ add_executable(fft_r2c_z ${files_fft_r2c})
 target_link_libraries(fft_c2c_z PRIVATE decomp2d)
 target_link_libraries(fft_r2c_z PRIVATE decomp2d)
 if (FFTW_FOUND)
-  target_link_libraries(fft_c2c_z PRIVATE ${FFTW_DOUBLE_LIB})
-  target_link_libraries(fft_c2c_z PRIVATE ${FFTW_FLOAT_LIB})
-  target_link_libraries(fft_r2c_z PRIVATE ${FFTW_DOUBLE_LIB})
-  target_link_libraries(fft_r2c_z PRIVATE ${FFTW_FLOAT_LIB})
+  if (FFTW_DOUBLE_LIB_FOUND)
+    target_link_libraries(fft_c2c_z PRIVATE ${FFTW_DOUBLE_LIB})
+    target_link_libraries(fft_r2c_z PRIVATE ${FFTW_DOUBLE_LIB})
+  elseif (FFTW_FLOAT_LIB_FOUND)
+    target_link_libraries(fft_c2c_z PRIVATE ${FFTW_FLOAT_LIB})
+    target_link_libraries(fft_r2c_z PRIVATE ${FFTW_FLOAT_LIB})
+  endif (FFTW_DOUBLE_LIB_FOUND)
 elseif (MKL_FOUND)
   target_link_libraries(fft_c2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
   target_link_libraries(fft_r2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 62623547..113009c5 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -48,8 +48,15 @@ if (MPI_FOUND)
 endif (MPI_FOUND)
 
 if(FFTW_FOUND)
-  message(STATUS "FFTW_DOUBLE     : ${FFTW_DOUBLE_LIB}")
-  message(STATUS "FFTW_SINGLE     : ${FFTW_FLOAT_LIB}")
+  if (FFTW_DOUBLE_LIB_FOUND)
+    message(STATUS "FFTW_DOUBLE     : ${FFTW_DOUBLE_LIB}")
+  endif (FFTW_DOUBLE_LIB_FOUND)
+  if (FFTW_FLOAT_LIB_FOUND)
+    message(STATUS "FFTW_SINGLE     : ${FFTW_FLOAT_LIB}")
+  endif (FFTW_FLOAT_LIB_FOUND)
+  if (NOT (FFTW_DOUBLE_LIB_FOUND OR FFTW_FLOAT_LIB_FOUND))
+    message(FATAL_ERROR "CMake step for FFTW failed")
+  endif (NOT (FFTW_DOUBLE_LIB_FOUND OR FFTW_FLOAT_LIB_FOUND))
   target_include_directories(decomp2d PRIVATE ${FFTW_INCLUDE_DIRS})
 elseif(MKL_FOUND)
   target_compile_options(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)

From 1c375d0066ccee04b30565facda7fdfbdc0faac0 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Mon, 10 Apr 2023 11:07:32 +0200
Subject: [PATCH 195/436] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 274b38c7..29692631 100644
--- a/README.md
+++ b/README.md
@@ -244,7 +244,7 @@ make -j
 make -j check
 make install
 ```
-Please note that the resulting build is not compatible with CMake ([fftw3#130][https://github.com/FFTW/fftw3/issues/130]). As a workaround, one can open the file `xxxxxxx/fftw3/fftw-3.3.10_bld/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line `include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")`.
+Please note that the resulting build is not compatible with CMake (https://github.com/FFTW/fftw3/issues/130). As a workaround, one can open the file `xxxxxxx/fftw3/fftw-3.3.10_bld/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line `include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")`.
 
 To build `2decomp&fft` against fftw3, one can provide the package configuration for fftw3 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`. One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. Then either specify on the command line when configuring the build
 ```

From de544cdc9463377227b02ddc60a3e2c63d5b166f Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Mon, 10 Apr 2023 11:09:32 +0200
Subject: [PATCH 196/436] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 29692631..7204ea53 100644
--- a/README.md
+++ b/README.md
@@ -244,7 +244,7 @@ make -j
 make -j check
 make install
 ```
-Please note that the resulting build is not compatible with CMake (https://github.com/FFTW/fftw3/issues/130). As a workaround, one can open the file `xxxxxxx/fftw3/fftw-3.3.10_bld/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line `include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")`.
+Please note that the resulting build is not compatible with CMake (https://github.com/FFTW/fftw3/issues/130). As a workaround, one can open the file `/path/to/fftw3/install/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line `include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")`.
 
 To build `2decomp&fft` against fftw3, one can provide the package configuration for fftw3 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`. One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. Then either specify on the command line when configuring the build
 ```

From 8b90562cf43c5bc87efe0913fa6d48b280261aa3 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Mon, 10 Apr 2023 11:10:51 +0200
Subject: [PATCH 197/436] Update README.md

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7204ea53..e06fc07a 100644
--- a/README.md
+++ b/README.md
@@ -244,7 +244,9 @@ make -j
 make -j check
 make install
 ```
-Please note that the resulting build is not compatible with CMake (https://github.com/FFTW/fftw3/issues/130). As a workaround, one can open the file `/path/to/fftw3/install/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line `include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")`.
+Please note that the resulting build is not compatible with CMake (https://github.com/FFTW/fftw3/issues/130). As a workaround, one can open the file `/path/to/fftw3/install/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line
+```include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")
+```
 
 To build `2decomp&fft` against fftw3, one can provide the package configuration for fftw3 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`. One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. Then either specify on the command line when configuring the build
 ```

From 0778d3cb8fb04912363da1e2abcb8bd6d7cc1dae Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 10 Apr 2023 11:14:16 +0200
Subject: [PATCH 198/436] Update README

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index e06fc07a..2c162c48 100644
--- a/README.md
+++ b/README.md
@@ -194,11 +194,11 @@ This variable is automatically added in debug and dev builds. Extra information
 
 #### DOUBLE_PREC
 
-When this variable is not present, the library uses single precision. When it is present, the library uses double precision
+When this variable is not present, the library uses single precision. When it is present, the library uses double precision. This preprocessor variable is driven by the CMake on/off variable `DOUBLE_PRECISION`.
 
 #### SAVE_SINGLE
 
-This variable is valid for double precision builds only. When it is present, snapshots are written in single precision.
+This variable is valid for double precision builds only. When it is present, snapshots are written in single precision. This preprocessor variable is driven by the CMake on/off variable `SINGLE_PRECISION_OUTPUT`.
 
 #### PROFILER
 
@@ -210,7 +210,7 @@ This preprocessor variable is not valid for GPU builds. It leads to padded allto
 
 #### OVERWRITE
 
-This variable leads to overwrite the input array when computing FFT. The support of this flag does not always correspond to in-place transforms, depending on the FFT backend selected, as described above.
+This variable leads to overwrite the input array when computing FFT. The support of this flag does not always correspond to in-place transforms, depending on the FFT backend selected, as described above. This preprocessor variable is driven by the CMake on/off variable `ENABLE_INPLACE`.
 
 #### HALO_DEBUG
 

From 3d39f0bee06faf44558c43740cf51ea80d1e5201 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Mon, 10 Apr 2023 11:16:07 +0200
Subject: [PATCH 199/436] Update README.md

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2c162c48..501b16f5 100644
--- a/README.md
+++ b/README.md
@@ -245,7 +245,8 @@ make -j check
 make install
 ```
 Please note that the resulting build is not compatible with CMake (https://github.com/FFTW/fftw3/issues/130). As a workaround, one can open the file `/path/to/fftw3/install/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line
-```include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")
+```
+include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")
 ```
 
 To build `2decomp&fft` against fftw3, one can provide the package configuration for fftw3 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`. One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. Then either specify on the command line when configuring the build

From 0d212dcfa12b35de7a8990348f1eeb1c3420f0cd Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Mon, 10 Apr 2023 13:11:21 +0200
Subject: [PATCH 200/436] Update README.md

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 501b16f5..0435d87b 100644
--- a/README.md
+++ b/README.md
@@ -259,14 +259,14 @@ Note the legacy `fftw` interface lacks interface definitions and will fail when
 
 ### Caliper
 
-The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.9.0 was tested and is supported, version 2.8.0 has also been tested and is still expected to work. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
+The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.9.1 was tested and is supported, version 2.8.0 has also been tested and is still expected to work. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
 
 ```
 git clone https://github.com/LLNL/Caliper.git caliper_github
 cd caliper_github
-git checkout v2.9.0
+git checkout v2.9.1
 mkdir build && cd build
-cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../caliper_build_2.9.0 -DWITH_FORTRAN=yes -DWITH_MPI=yes -DBUILD_TESTING=yes ../
+cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../caliper_build_2.9.1 -DWITH_FORTRAN=yes -DWITH_MPI=yes -DBUILD_TESTING=yes ../
 make -j
 make test
 make install

From 181b05e0aec992aee730e092357db4f5f2a76033 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Mon, 10 Apr 2023 13:16:31 +0200
Subject: [PATCH 201/436] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0435d87b..f4f804f1 100644
--- a/README.md
+++ b/README.md
@@ -214,7 +214,7 @@ This variable leads to overwrite the input array when computing FFT. The support
 
 #### HALO_DEBUG
 
-This variable is used to debug the halo operations.
+This variable is used to debug the halo operations. This preprocessor variable is driven by the CMake on/off variable `HALO_DEBUG`.
 
 #### _GPU
 

From 01a7d6e243a3f577a079157baf42885c245c4e3d Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 10 Apr 2023 18:16:37 +0200
Subject: [PATCH 202/436] Fix CMake examples when using fftw

---
 examples/fft_physical_x/CMakeLists.txt | 5 +++--
 examples/fft_physical_z/CMakeLists.txt | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index a0b92408..abc2013b 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -16,11 +16,12 @@ if (FFTW_FOUND)
     target_link_libraries(fft_c2c_x PRIVATE ${FFTW_DOUBLE_LIB})
     target_link_libraries(fft_r2c_x PRIVATE ${FFTW_DOUBLE_LIB})
     target_link_libraries(fft_grid_x PRIVATE ${FFTW_DOUBLE_LIB})
-  elseif (FFTW_FLOAT_LIB_FOUND)
+  endif (FFTW_DOUBLE_LIB_FOUND)
+  if (FFTW_FLOAT_LIB_FOUND)
     target_link_libraries(fft_c2c_x PRIVATE ${FFTW_FLOAT_LIB})
     target_link_libraries(fft_r2c_x PRIVATE ${FFTW_FLOAT_LIB})
     target_link_libraries(fft_grid_x PRIVATE ${FFTW_FLOAT_LIB})
-  endif (FFTW_DOUBLE_LIB_FOUND)
+  endif (FFTW_FLOAT_LIB_FOUND)
 elseif (MKL_FOUND)
   target_link_libraries(fft_c2c_x PUBLIC $<LINK_ONLY:MKL::MKL>)
   target_link_libraries(fft_r2c_x PUBLIC $<LINK_ONLY:MKL::MKL>)
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index f23ae7cd..964cbf8a 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -12,10 +12,11 @@ if (FFTW_FOUND)
   if (FFTW_DOUBLE_LIB_FOUND)
     target_link_libraries(fft_c2c_z PRIVATE ${FFTW_DOUBLE_LIB})
     target_link_libraries(fft_r2c_z PRIVATE ${FFTW_DOUBLE_LIB})
-  elseif (FFTW_FLOAT_LIB_FOUND)
+  endif (FFTW_DOUBLE_LIB_FOUND)
+  if (FFTW_FLOAT_LIB_FOUND)
     target_link_libraries(fft_c2c_z PRIVATE ${FFTW_FLOAT_LIB})
     target_link_libraries(fft_r2c_z PRIVATE ${FFTW_FLOAT_LIB})
-  endif (FFTW_DOUBLE_LIB_FOUND)
+  endif (FFTW_FLOAT_LIB_FOUND)
 elseif (MKL_FOUND)
   target_link_libraries(fft_c2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
   target_link_libraries(fft_r2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)

From cfe15bc9699d96dc2e6c3cdedaba1916ce0c07d0 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 18 Apr 2023 14:40:05 +0100
Subject: [PATCH 203/436] Update README with new instructions on how to link
 against 2decomp-fft

---
 README.md | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index fc6adf7c..1622daf1 100644
--- a/README.md
+++ b/README.md
@@ -149,8 +149,27 @@ DECOMP_INSTALL_DIR ?= $(DECOMP_BUILD_DIR)/opt # Use default unless set by user
 INC += -I$(DECOMP_INSTALL_DIR)/include
 
 # Users build/link targets
-LFLAGS += -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
+LIBS = -L$(DECOMP_INSTALL_DIR)/lib64 -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
 
+OBJ = my_exec.o
+
+my_exec: $(OBJ)
+	$(F90) -o $@ $(OBJ) $(LIBS)
+
+```
+In case 2decomp-fft has been compiled with an external FFT, such as FFTW3, `LIBS` should also contain the following 
+```
+FFTW3_PATH=/my_path_to_FFTW/lib
+LIBFFT=-L$(FFTW3_PATH) -lfftw3 -lfftw3f
+LIBS += $(LIBFFT)
+```
+In case of 2decomp-fft compiled for GPU with NVHPC, linking against cuFFT is mandatory 
+```
+LIBS += -cudalib=cufft
+```
+In case of NCCL the follwoing is required 
+```
+LIBS += -cudalib=cufft,nccl 
 ```
 It is also possible to drive the build and installation of 2decomp-fft from a Makefile such as in the following example code
 ```
@@ -164,7 +183,7 @@ DECOMP_INSTALL_DIR ?= $(DECOMP_BUILD_DIR)/opt # Use default unless set by user
 INC += -I$(DECOMP_INSTALL_DIR)/include
 
 # Users build/link targets
-LFLAGS += -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
+LIBS = -L$(DECOMP_INSTALL_DIR)/lib64 -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
 
 # Building libdecomp.a
 $(DECOMP_INSTALL_DIR)/lib/libdecomp.a:

From 3b2c5aea7173ee2e4d6452a3c6d57702fd6c4dac Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 18 Apr 2023 18:02:49 +0100
Subject: [PATCH 204/436] Fix minor typos in README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2e3b28c0..38ee1c9d 100644
--- a/README.md
+++ b/README.md
@@ -140,7 +140,7 @@ The external code can use the named variables to check the FFT backend used in a
 
 ### Codes using Makefiles
 
-When building a code that links 2decomp-fft using a Makefile you will need to add the include and link paths as appropriate (`inlude/` and `link/` under the installation directory, respectively).
+When building a code that links 2decomp-fft using a Makefile you will need to add the include and link paths as appropriate (`inlude/` and `lib/` under the installation directory, respectively).
 ```
 DECOMP_ROOT = /path/to/2decomp-fft
 DECOMP_BUILD_DIR = $(DECOMP_ROOT)/build
@@ -167,7 +167,7 @@ In case of 2decomp-fft compiled for GPU with NVHPC, linking against cuFFT is man
 ```
 LIBS += -cudalib=cufft
 ```
-In case of NCCL the follwoing is required 
+In case of NCCL the following is required 
 ```
 LIBS += -cudalib=cufft,nccl 
 ```

From 7269e3c9991a5f26a693a23a8d4c0d5a020057fb Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Sat, 22 Apr 2023 14:58:21 +0100
Subject: [PATCH 205/436] Deactivate testing as default option

---
 CMakeLists.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3d6290c7..8617167f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -79,7 +79,10 @@ include(fft)
 add_subdirectory(src)
 
 # Add tests
-add_subdirectory(examples)
+option(BUILD_TESTING "Build with tests" OFF)
+if (${BUILD_TESTING})
+  add_subdirectory(examples)
+endif (${BUILD_TESTING})
 
 # Add a prettify target
 add_custom_target(format sh ${CMAKE_SOURCE_DIR}/scripts/format.sh

From eeb695ec199cd8d33fe664baa0705c30a343e94a Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Sat, 22 Apr 2023 15:23:33 +0100
Subject: [PATCH 206/436] Update README with CTest activation option

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 38ee1c9d..0cf890ac 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,8 @@ cmake --build $path_to_build_directory --target clean
 ## Testing and examples
 
 ### CMake
-
+By default build of the test is deactivated. To activate the testing the option `-DBUILD_TESTING=ON` can be added or 
+alternativey the option can be activated in the GUI interface `ccmake`.
 After building the library can be tested by running
 ```
 ctest --test-dir $path_to_build_directory

From a9aa608fb7f5a1d0cbbef36033b088ad74cba13c Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Sat, 22 Apr 2023 15:33:16 +0100
Subject: [PATCH 207/436] Update the github workflows to enable testing

---
 .github/workflows/Build.yml     |  4 ++--
 .github/workflows/Build_all.yml | 14 +++++++-------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml
index 04ddea08..29210335 100644
--- a/.github/workflows/Build.yml
+++ b/.github/workflows/Build.yml
@@ -30,11 +30,11 @@ jobs:
     - name: Compile
       run: |
         export FC=mpif90
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=dev
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=dev -DBUILD_TESTING=ON
         cmake --build build --target decomp2d
 
     - name: Examples
       run: |
         export FC=mpif90
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=dev
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=dev -DBUILD_TESTING=ON
         cmake --build build
diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index cb17078e..deda9a20 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -30,7 +30,7 @@ jobs:
     - name: Compile
       run: |
         export FC=mpif90.openmpi
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON
         cmake --build build --verbose
 
   bld_gnu_mpich:
@@ -52,7 +52,7 @@ jobs:
     - name: Compile
       run: |
         export FC=mpif90.mpich
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=On
         cmake --build build --verbose
 
   bld_nvidia:
@@ -112,7 +112,7 @@ jobs:
       run: |
         source /opt/intel/oneapi/setvars.sh
         export FC=mpiifort
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON
         cmake --build build
 
     # Runs a set of commands using the runners shell
@@ -121,7 +121,7 @@ jobs:
         source /opt/intel/oneapi/setvars.sh
         export FC=mpiifort
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=mkl
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=mkl -DBUILD_TESTING=ON
         cmake --build build --target clean
         cmake --build build
 
@@ -144,13 +144,13 @@ jobs:
     - name: Regular FFTW3
       run: |
         export FC=mpif90
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DFFT_Choice=fftw
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DFFT_Choice=fftw -DBUILD_TESTING=ON
         cmake --build build --verbose
 
     - name: New FFTW3
       run: |
         export FC=mpif90
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=fftw_f03
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=fftw_f03 -DBUILD_TESTING=ON
         cmake --build build --verbose
 
   bld_caliper:
@@ -181,5 +181,5 @@ jobs:
     - name: Compile
       run: |
         export caliper_DIR=./Caliper-2.8.0_bld/share/cmake/caliper
-        FC=mpif90 cmake -S . -B build -DENABLE_PROFILER=caliper
+        FC=mpif90 cmake -S . -B build -DENABLE_PROFILER=caliper -DBUILD_TESTING=ON
         cmake --build build --verbose

From ad4934d20050d251629c8cfbef21834c120aa66c Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Sat, 22 Apr 2023 15:52:37 +0100
Subject: [PATCH 208/436] Use BUILD_TESTING=ON

Not sure if capitalisation actually matters...
---
 .github/workflows/Build_all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index deda9a20..ee476f6f 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -52,7 +52,7 @@ jobs:
     - name: Compile
       run: |
         export FC=mpif90.mpich
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=On
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON
         cmake --build build --verbose
 
   bld_nvidia:

From b4278833eb0961a87c041353c79fc1a0a3e83a93 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Sat, 22 Apr 2023 21:14:39 +0100
Subject: [PATCH 209/436] Link FFT and MKL with decomp2d and not with Example
 exec

---
 examples/fft_physical_x/CMakeLists.txt | 16 ----------------
 examples/fft_physical_z/CMakeLists.txt | 13 -------------
 src/CMakeLists.txt                     |  3 +++
 3 files changed, 3 insertions(+), 29 deletions(-)

diff --git a/examples/fft_physical_x/CMakeLists.txt b/examples/fft_physical_x/CMakeLists.txt
index abc2013b..ca6c65c2 100644
--- a/examples/fft_physical_x/CMakeLists.txt
+++ b/examples/fft_physical_x/CMakeLists.txt
@@ -11,22 +11,6 @@ add_executable(fft_grid_x ${files_fft_grid})
 target_link_libraries(fft_c2c_x PRIVATE decomp2d)
 target_link_libraries(fft_r2c_x PRIVATE decomp2d)
 target_link_libraries(fft_grid_x PRIVATE decomp2d)
-if (FFTW_FOUND)
-  if (FFTW_DOUBLE_LIB_FOUND)
-    target_link_libraries(fft_c2c_x PRIVATE ${FFTW_DOUBLE_LIB})
-    target_link_libraries(fft_r2c_x PRIVATE ${FFTW_DOUBLE_LIB})
-    target_link_libraries(fft_grid_x PRIVATE ${FFTW_DOUBLE_LIB})
-  endif (FFTW_DOUBLE_LIB_FOUND)
-  if (FFTW_FLOAT_LIB_FOUND)
-    target_link_libraries(fft_c2c_x PRIVATE ${FFTW_FLOAT_LIB})
-    target_link_libraries(fft_r2c_x PRIVATE ${FFTW_FLOAT_LIB})
-    target_link_libraries(fft_grid_x PRIVATE ${FFTW_FLOAT_LIB})
-  endif (FFTW_FLOAT_LIB_FOUND)
-elseif (MKL_FOUND)
-  target_link_libraries(fft_c2c_x PUBLIC $<LINK_ONLY:MKL::MKL>)
-  target_link_libraries(fft_r2c_x PUBLIC $<LINK_ONLY:MKL::MKL>)
-  target_link_libraries(fft_grid_x PUBLIC $<LINK_ONLY:MKL::MKL>)
-endif (FFTW_FOUND)
 
 # Run the test(s)
 set(run_dir "${test_dir}/fft_c2c_x")
diff --git a/examples/fft_physical_z/CMakeLists.txt b/examples/fft_physical_z/CMakeLists.txt
index 964cbf8a..05e79264 100644
--- a/examples/fft_physical_z/CMakeLists.txt
+++ b/examples/fft_physical_z/CMakeLists.txt
@@ -8,19 +8,6 @@ add_executable(fft_r2c_z ${files_fft_r2c})
 
 target_link_libraries(fft_c2c_z PRIVATE decomp2d)
 target_link_libraries(fft_r2c_z PRIVATE decomp2d)
-if (FFTW_FOUND)
-  if (FFTW_DOUBLE_LIB_FOUND)
-    target_link_libraries(fft_c2c_z PRIVATE ${FFTW_DOUBLE_LIB})
-    target_link_libraries(fft_r2c_z PRIVATE ${FFTW_DOUBLE_LIB})
-  endif (FFTW_DOUBLE_LIB_FOUND)
-  if (FFTW_FLOAT_LIB_FOUND)
-    target_link_libraries(fft_c2c_z PRIVATE ${FFTW_FLOAT_LIB})
-    target_link_libraries(fft_r2c_z PRIVATE ${FFTW_FLOAT_LIB})
-  endif (FFTW_FLOAT_LIB_FOUND)
-elseif (MKL_FOUND)
-  target_link_libraries(fft_c2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
-  target_link_libraries(fft_r2c_z PUBLIC $<LINK_ONLY:MKL::MKL>)
-endif (FFTW_FOUND)
 
 # Run the test(s)
 set(run_dir "${test_dir}/fft_c2c_z")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 153bf0e4..e8a84b59 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -50,9 +50,11 @@ endif (MPI_FOUND)
 if(FFTW_FOUND)
   if (FFTW_DOUBLE_LIB_FOUND)
     message(STATUS "FFTW_DOUBLE     : ${FFTW_DOUBLE_LIB}")
+    target_link_libraries(decomp2d PRIVATE ${FFTW_DOUBLE_LIB})
   endif (FFTW_DOUBLE_LIB_FOUND)
   if (FFTW_FLOAT_LIB_FOUND)
     message(STATUS "FFTW_SINGLE     : ${FFTW_FLOAT_LIB}")
+    target_link_libraries(decomp2d PRIVATE ${FFTW_FLOAT_LIB})
   endif (FFTW_FLOAT_LIB_FOUND)
   if (NOT (FFTW_DOUBLE_LIB_FOUND OR FFTW_FLOAT_LIB_FOUND))
     message(FATAL_ERROR "CMake step for FFTW failed")
@@ -61,6 +63,7 @@ if(FFTW_FOUND)
 elseif(MKL_FOUND)
   target_compile_options(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
   target_include_directories(decomp2d PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
+  target_link_libraries(decomp2d PUBLIC $<LINK_ONLY:MKL::MKL>)
 endif(FFTW_FOUND)
 
 if (ENABLE_PROFILER)

From 6b4df8305255d30373b85cb3ef883eb25682df47 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Sun, 23 Apr 2023 19:04:51 +0100
Subject: [PATCH 210/436] Add the search also for lib64 for CMake include

---
 cmake/decomp2d-config.cmake.in | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cmake/decomp2d-config.cmake.in b/cmake/decomp2d-config.cmake.in
index 5a06051e..f519615d 100644
--- a/cmake/decomp2d-config.cmake.in
+++ b/cmake/decomp2d-config.cmake.in
@@ -8,7 +8,13 @@ get_filename_component(_dir "${CMAKE_CURRENT_LIST_FILE}" PATH)
 get_filename_component(_prefix "${_dir}/../.." ABSOLUTE)
 
 # Import the targets
-include("${_prefix}/lib/decomp2d-targets.cmake")
+if (EXISTS ${_prefix}/lib )
+  message(STATUS "Found decomp2d under lib")
+  include("${_prefix}/lib/decomp2d-targets.cmake")
+else()
+  message(STATUS "Not Found decomp2d under lib, we'll use lib64")
+  include("${_prefix}/lib64/decomp2d-targets.cmake")
+endif()
 
 set(2decomp_INCLUDE_DIR "${_prefix}/include")
 

From 2c293b2a23ebd9e7a5e1850035f83dc17c659fd7 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Sun, 23 Apr 2023 19:06:58 +0100
Subject: [PATCH 211/436] Remove uncessary write in cuFFT file

---
 src/fft_cufft.f90 | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 1776d601..125854d4 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -278,7 +278,6 @@ subroutine init_fft_engine
       else if (format == PHYSICAL_IN_Z) then
 
          ! For C2C transforms
-         write (*, *) 'Create the plans'
          call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_Z2Z, ws)
          cufft_ws = max(cufft_ws, ws)
          call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_Z2Z, ws)
@@ -339,7 +338,6 @@ subroutine init_fft_engine
       else if (format == PHYSICAL_IN_Z) then
 
          ! For C2C transforms
-         write (*, *) 'Create the plans on rank ', nproc
          call c2c_1m_z_plan(plan(-1, 3), ph, CUFFT_C2C, ws)
          cufft_ws = max(cufft_ws, ws)
          call c2c_1m_y_plan(plan(-1, 2), ph, CUFFT_C2C, ws)

From 37cdf16d5fa80f0c66730cfd2f569b0ae9ddf91e Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 25 Apr 2023 09:11:25 +0200
Subject: [PATCH 212/436] Update decomp_2d_mpi.f90

Add missing check on the value returned by `MPI_COMM_FREE`
---
 src/decomp_2d_mpi.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/decomp_2d_mpi.f90 b/src/decomp_2d_mpi.f90
index 25b824e9..0e478625 100644
--- a/src/decomp_2d_mpi.f90
+++ b/src/decomp_2d_mpi.f90
@@ -114,7 +114,7 @@ subroutine decomp_2d_mpi_comm_free(mpi_comm)
 
       ! Free the provided MPI communicator
       call MPI_COMM_FREE(mpi_comm, ierror)
-!      if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
+      if (ierror /= 0) call decomp_2d_warning(__FILE__, __LINE__, ierror, "MPI_COMM_FREE")
       mpi_comm = MPI_COMM_NULL
 
    end subroutine decomp_2d_mpi_comm_free

From be8452a77f3dec3f2843a28c362849c322ead91a Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 25 Apr 2023 10:47:15 +0200
Subject: [PATCH 213/436] Update decomp_2d_init_fin.f90 (#180)

---
 src/decomp_2d_init_fin.f90 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index ff4c6c5e..dd220a95 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -15,6 +15,8 @@
   !   INPUT:
   !     nx, ny, nz   - global data dimension
   !     p_row, p_col - 2D processor grid
+  !     periodic_bc  - optional, periodicity flag for halo operations
+  !     comm         - optional, MPI communicator, default MPI_COMM_WORLD
   !   OUTPUT:
   !     all internal data structures initialised properly
   !     library ready to use

From 627a4724aae47b4a76b084f5ee65d233836cdd88 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 25 Apr 2023 12:21:27 +0200
Subject: [PATCH 214/436] Update Build_all.yml

Clean the regular FFTW3 build before launching the new FFTW3 build
---
 .github/workflows/Build_all.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index ee476f6f..2f87c151 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -146,6 +146,7 @@ jobs:
         export FC=mpif90
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DFFT_Choice=fftw -DBUILD_TESTING=ON
         cmake --build build --verbose
+        cmake --build build --target clean
 
     - name: New FFTW3
       run: |

From 14a3afeb21ca591cb73e1b6dced70919e55c7019 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 25 Apr 2023 14:42:57 +0200
Subject: [PATCH 215/436] Typo in README.md

`ENABLE_PROFILING` was used instead of `ENABLE_PROFILER`
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 0cf890ac..2e864093 100644
--- a/README.md
+++ b/README.md
@@ -306,6 +306,6 @@ make install
 After installing Caliper ensure to set `caliper_DIR=/path/to/caliper/install/share/cmake/caliper`.
 Following this the `2decomp-fft` build can be configured to use Caliper profiling as
 ```
-cmake -S . -B -DENABLE_PROFILING=caliper
+cmake -S . -B -DENABLE_PROFILER=caliper
 ```
-or by modifying the configuration to set `ENABLE_PROFILING=caliper` via `ccmake`.
+or by modifying the configuration to set `ENABLE_PROFILER=caliper` via `ccmake`.

From fb97446c9f3bbec2136f2aa4420feec19e916aa8 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Wed, 26 Apr 2023 15:50:15 +0100
Subject: [PATCH 216/436] Add section on versioning to README

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index 2e864093..c962ca3a 100644
--- a/README.md
+++ b/README.md
@@ -309,3 +309,9 @@ Following this the `2decomp-fft` build can be configured to use Caliper profilin
 cmake -S . -B -DENABLE_PROFILER=caliper
 ```
 or by modifying the configuration to set `ENABLE_PROFILER=caliper` via `ccmake`.
+
+### Versioning
+
+The development of `2decomp&fft` occurs on Github, with release versions on the `main` branch.
+New features will be implemented on the `develop` branch and merged into `main` once a new release is ready.
+For example, starting from `v2.0.0` the `main` branch will only be updated to receive fixes giving `v2.0.1`, etc. until the next release (either `v2.1.0` or `v3.0.0` depending on the magnitude of the change is ready).

From 177c538d520e40e1c36830ea6e506e373cad757d Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Wed, 26 Apr 2023 15:50:56 +0100
Subject: [PATCH 217/436] Formatting

---
 README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index c962ca3a..c0fe97a4 100644
--- a/README.md
+++ b/README.md
@@ -313,5 +313,8 @@ or by modifying the configuration to set `ENABLE_PROFILER=caliper` via `ccmake`.
 ### Versioning
 
 The development of `2decomp&fft` occurs on Github, with release versions on the `main` branch.
-New features will be implemented on the `develop` branch and merged into `main` once a new release is ready.
-For example, starting from `v2.0.0` the `main` branch will only be updated to receive fixes giving `v2.0.1`, etc. until the next release (either `v2.1.0` or `v3.0.0` depending on the magnitude of the change is ready).
+New features will be implemented on the `develop` branch and merged into `main` once a new release
+is ready.
+For example, starting from `v2.0.0` the `main` branch will only be updated to receive fixes giving
+`v2.0.1`, etc. until the next release (either `v2.1.0` or `v3.0.0` depending on the magnitude of the
+change is ready).

From 9e67d86887bc3343ba967e78772fbfad1948a61c Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 27 Apr 2023 10:08:52 +0100
Subject: [PATCH 218/436] Add instructions about ccmake for GPU build

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c0fe97a4..b6d59030 100644
--- a/README.md
+++ b/README.md
@@ -63,7 +63,7 @@ After building the library can be tested by running
 ```
 ctest --test-dir $path_to_build_directory
 ```
-which uses the `ctest` utility. By default tests are performed in serial, but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS` under `ccmake` utility.  
+which uses the `ctest` utility. By default tests are performed in serial, but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS` under `ccmake` utility.
 It is also possible to specify the decomposition by setting `PROW` and `PCOL` parameters at the configure stage or using `ccmake`. 
 Consequently, the configure stage is forcing the number of MPI tasks `MPIEXEC_MAX_NUMPROCS` to be equal to the product of PROW times PCOL. 
 Mesh resolution can also be imposed using the parameters `NX`, `NY` and `NZ`. 
@@ -81,6 +81,8 @@ To properly configure for GPU build the following needs to be used
 ```
 cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_TARGET=gpu
 ```
+Note, further configuration can be performed using `ccmake`, however the initial configuration of GPU builds must include the `-DBUILD_TARGET=gpu` flag as shown above.
+
 By default CUDA aware MPI will be used together with `cuFFT` for the FFT library. The configure will automatically look for the GPU architecture available on the system. If you are building on a HPC system please use a computing node for the installation. Useful variables to be added are 
 
  - `-DENABLE_NCCL=yes` to activate the NCCL collectives

From 5b1765d6b68ecd0de5109a6b0bd8128cb00aa7a4 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 27 Apr 2023 10:36:42 +0100
Subject: [PATCH 219/436] Begin splitting README into README+INSTALL

---
 INSTALL.md | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 README.md  | 136 ++++++++-------------------------------------------
 2 files changed, 162 insertions(+), 115 deletions(-)
 create mode 100644 INSTALL.md

diff --git a/INSTALL.md b/INSTALL.md
new file mode 100644
index 00000000..cd9ed3e1
--- /dev/null
+++ b/INSTALL.md
@@ -0,0 +1,141 @@
+# Bulding and installing 2decomp-fft
+
+## Building
+
+The build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 
+```
+export FC=my_mpif90
+```
+To generate the build system run
+```
+cmake -S $path_to_sources -B $path_to_build_directory -DOPTION1 -DOPTION2 ...
+```
+If the directory does not exist it will be generated and it will contain the configuration files. The configuration can be further
+edited by using the `ccmake` utility as
+```
+ccmake $path_to_build_directory
+```
+and editing as desired, variables that are likely of interest are: `CMAKE_BUILD_TYPE` and `FFT_Choice`;
+additional variables can be shown by entering "advanced mode" by pressing `t`.
+By default a `RELEASE` build will built, other options for `CMAKE_BUILD_TYPE` are `DEBUG` and `DEV` which
+turn on debugging flags and additionally try to catch coding errors at compile time, respectively.
+The behaviour of debug and development versions of the library can be changed before the
+initialization using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
+The value provided with the environment variable must be a positive integer below 9999.
+
+Two `BUILD_TARGETS` are available namely `mpi` and `gpu`.  For the `mpi` target no additional options should be required. whereas for `gpu` extra options are necessary at the configure stage. Please see section [GPU Compilation](#gpu-compilation)
+
+Once the build system has been configured, you can build `2decomp&fft` by running
+```
+cmake --build $path_to_build_directory -j <nproc>
+```
+appending `-v` will display additional information about the build, such as compiler flags.
+
+After building the library can be tested. Please see section [Testing and examples](#testing-and-examples)
+
+Options can be added to change the level of verbosity. Finally, the build library can be installed by running 
+```
+cmake --install $path_to_build_directory
+```
+The default location for `libdecomp2d.a` is `$path_to_build_directory/opt/lib`or  `$path_to_build_directory/opt/lib64` unless the variable `CMAKE_INSTALL_PREFIX` is modified.
+The module files generated by the build process will similarly be installed to `$path_to_build_directory/opt/install`, users of the library should add this to the include paths for their program.
+
+As indicated above, by default a static `libdecomp2d.a` will be compiled, if desired a shared library can be built by setting `BUILD_SHARED_LIBS=ON` either on the command line:
+```
+cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_SHARED_LIBS=ON
+```
+or by editing the configuration using `ccmake`.
+This might be useful for a centralised install supporting multiple users that is upgraded over time.
+
+Occasionally a clean build is required, this can be performed by running
+```
+cmake --build $path_to_build_directory --target clean
+```
+
+## GPU compilation
+
+The library can perform multi GPU offoloading using the NVHPC compiler suite for NVIDIA hardware. 
+The implementation is based on CUDA-aware MPI and NVIDIA Collective Communication Library (NCCL).
+The FFT is based on cuFFT. 
+
+To properly configure for GPU build the following needs to be used 
+```
+cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_TARGET=gpu
+```
+Note, further configuration can be performed using `ccmake`, however the initial configuration of GPU builds must include the `-DBUILD_TARGET=gpu` flag as shown above.
+
+By default CUDA aware MPI will be used together with `cuFFT` for the FFT library. The configure will automatically look for the GPU architecture available on the system. If you are building on a HPC system please use a computing node for the installation. Useful variables to be added are 
+
+ - `-DENABLE_NCCL=yes` to activate the NCCL collectives
+ - `-DENABLE_MANAGED=yes` to activate the automatic memory management form the NVHPC compiler
+If you are getting the following error
+```
+-- The CUDA compiler identification is unknown  
+CMake Error at /usr/share/cmake/Modules/CMakeDetermineCUDACompiler.cmake:633 (message):  
+Failed to detect a default CUDA architecture. 
+```
+It is possible that your default C compiler is too recent and not supported by `nvcc` . You might be able to solve the issue by adding 
+ - `-DCMAKE_CUDA_HOST_COMPILER=$supported_gcc`
+ 
+ At the moment the supported CUDA host compilers are `gcc11` and earlier. 
+
+## Linking from external codes
+
+### Codes using Makefiles
+
+When building a code that links 2decomp-fft using a Makefile you will need to add the include and link paths as appropriate (`inlude/` and `lib/` under the installation directory, respectively).
+```
+DECOMP_ROOT = /path/to/2decomp-fft
+DECOMP_BUILD_DIR = $(DECOMP_ROOT)/build
+DECOMP_INSTALL_DIR ?= $(DECOMP_BUILD_DIR)/opt # Use default unless set by user
+
+INC += -I$(DECOMP_INSTALL_DIR)/include
+
+# Users build/link targets
+LIBS = -L$(DECOMP_INSTALL_DIR)/lib64 -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
+
+OBJ = my_exec.o
+
+my_exec: $(OBJ)
+	$(F90) -o $@ $(OBJ) $(LIBS)
+
+```
+In case 2decomp-fft has been compiled with an external FFT, such as FFTW3, `LIBS` should also contain the following 
+```
+FFTW3_PATH=/my_path_to_FFTW/lib
+LIBFFT=-L$(FFTW3_PATH) -lfftw3 -lfftw3f
+LIBS += $(LIBFFT)
+```
+In case of 2decomp-fft compiled for GPU with NVHPC, linking against cuFFT is mandatory 
+```
+LIBS += -cudalib=cufft
+```
+In case of NCCL the following is required 
+```
+LIBS += -cudalib=cufft,nccl 
+```
+It is also possible to drive the build and installation of 2decomp-fft from a Makefile such as in the following example code
+```
+FC = mpif90
+BUILD = Release
+
+DECOMP_ROOT = /path/to/2decomp-fft
+DECOMP_BUILD_DIR = $(DECOMP_ROOT)/build
+DECOMP_INSTALL_DIR ?= $(DECOMP_BUILD_DIR)/opt # Use default unless set by user
+
+INC += -I$(DECOMP_INSTALL_DIR)/include
+
+# Users build/link targets
+LIBS = -L$(DECOMP_INSTALL_DIR)/lib64 -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
+
+# Building libdecomp.a
+$(DECOMP_INSTALL_DIR)/lib/libdecomp.a:
+	FC=$(FC) cmake -S $(DECOMP_ROOT) -B $(DECOMP_BUILD_DIR) -DCMAKE_BUILD_TYPE=$(BUILD) -DCMAKE_INSTALL_PREFIX=$(DECOMP_INSTALL_DIR)
+	cmake --build $(DECOMP_BUILD_DIR) --target decomp2d
+	cmake --build $(DECOMP_BUILD_DIR) --target install
+
+# Clean libdecomp.a
+clean-decomp:
+	cmake --build $(DECOMP_BUILD_DIR) --target clean
+	rm -f $(DECOMP_INSTALL_DIR)/lib/libdecomp.a
+```
diff --git a/README.md b/README.md
index b6d59030..23a11f0e 100644
--- a/README.md
+++ b/README.md
@@ -1,31 +1,25 @@
 # 2decomp-fft
 
-## Building
+This README contains basic instructions for building and installing the `2decomp&fft` library, more
+detailed instructions can be found in [INSTALL.md](INSTALL.md).
 
-### CMake
+## Building
 
 The build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 
 ```
 export FC=my_mpif90
 ```
-To generate the build system run
+The build system can then be generated by running
 ```
 cmake -S $path_to_sources -B $path_to_build_directory -DOPTION1 -DOPTION2 ...
 ```
-If the directory does not exist it will be generated and it will contain the configuration files. The configuration can be further
-edited by using the `ccmake` utility as
+for many users a configuration line
 ```
-ccmake $path_to_build_directory
+cmake -S . -B build
 ```
-and editing as desired, variables that are likely of interest are: `CMAKE_BUILD_TYPE` and `FFT_Choice`;
-additional variables can be shown by entering "advanced mode" by pressing `t`.
-By default a `RELEASE` build will built, other options for `CMAKE_BUILD_TYPE` are `DEBUG` and `DEV` which
-turn on debugging flags and additionally try to catch coding errors at compile time, respectively.
-The behaviour of debug and development versions of the library can be changed before the
-initialization using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
-The value provided with the environment variable must be a positive integer below 9999.
-
-Two `BUILD_TARGETS` are available namely `mpi` and `gpu`.  For the `mpi` target no additional options should be required. whereas for `gpu` extra options are necessary at the configure stage. Please see section [GPU Compilation](#gpu-compilation)
+run from the `2decomp&fft` root directory will be sufficient.
+If the build directory does not exist it will be generated and it will contain the configuration files.
+By default a `RELEASE` build will built for CPU using MPI and the `generic` FFT backend included with `2decomp&fft`, please see [INSTALL.md](INSTALL.md) for instructions on changing the build, including debugging builds, building for GPUs and selecting external FFT libraries.
 
 Once the build system has been configured, you can build `2decomp&fft` by running
 ```
@@ -35,29 +29,30 @@ appending `-v` will display additional information about the build, such as comp
 
 After building the library can be tested. Please see section [Testing and examples](#testing-and-examples)
 
-Options can be added to change the level of verbosity. Finally, the build library can be installed by running 
+Finally, the build library can be installed by running 
 ```
 cmake --install $path_to_build_directory
 ```
 The default location for `libdecomp2d.a` is `$path_to_build_directory/opt/lib`or  `$path_to_build_directory/opt/lib64` unless the variable `CMAKE_INSTALL_PREFIX` is modified.
 The module files generated by the build process will similarly be installed to `$path_to_build_directory/opt/install`, users of the library should add this to the include paths for their program.
 
-As indicated above, by default a static `libdecomp2d.a` will be compiled, if desired a shared library can be built by setting `BUILD_SHARED_LIBS=ON` either on the command line:
-```
-cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_SHARED_LIBS=ON
-```
-or by editing the configuration using `ccmake`.
-This might be useful for a centralised install supporting multiple users that is upgraded over time.
-
 Occasionally a clean build is required, this can be performed by running
 ```
 cmake --build $path_to_build_directory --target clean
 ```
 
+### GPU compilation
+
+The library can perform multi GPU offoloading using the NVHPC compiler suite for NVIDIA hardware. 
+The implementation is based on CUDA-aware MPI and NVIDIA Collective Communication Library (NCCL).
+The FFT is based on cuFFT.
+
+For details of how to configure `2decomp&fft` for GPU offload, see the GPU compilation section in
+[INSTALL.md](INSTALL.md).
+
 ## Testing and examples
 
-### CMake
-By default build of the test is deactivated. To activate the testing the option `-DBUILD_TESTING=ON` can be added or 
+By default building of the tests is deactivated. To activate the testing the option `-DBUILD_TESTING=ON` can be added or 
 alternativey the option can be activated in the GUI interface `ccmake`.
 After building the library can be tested by running
 ```
@@ -65,39 +60,11 @@ ctest --test-dir $path_to_build_directory
 ```
 which uses the `ctest` utility. By default tests are performed in serial, but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS` under `ccmake` utility.
 It is also possible to specify the decomposition by setting `PROW` and `PCOL` parameters at the configure stage or using `ccmake`. 
-Consequently, the configure stage is forcing the number of MPI tasks `MPIEXEC_MAX_NUMPROCS` to be equal to the product of PROW times PCOL. 
+During the configure stage users should ensure that the number of MPI tasks `MPIEXEC_MAX_NUMPROCS` is equal to the product of PROW times PCOL. 
 Mesh resolution can also be imposed using the parameters `NX`, `NY` and `NZ`. 
 
 For the GPU implementation please be aware that it is based on a single MPI rank per GPU. Therefore, to test multiple GPUs, use the maximum number of available GPUs on the system/node and not the maximum number of MPI tasks. 
 
-## GPU compilation
-
-The library can perform multi GPU offoloading using the NVHPC compiler suite for NVIDIA hardware. 
-The implementation is based on CUDA-aware MPI and NVIDIA Collective Communication Library (NCCL).
-The FFT is based on cuFFT. 
-
-### CMake
-To properly configure for GPU build the following needs to be used 
-```
-cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_TARGET=gpu
-```
-Note, further configuration can be performed using `ccmake`, however the initial configuration of GPU builds must include the `-DBUILD_TARGET=gpu` flag as shown above.
-
-By default CUDA aware MPI will be used together with `cuFFT` for the FFT library. The configure will automatically look for the GPU architecture available on the system. If you are building on a HPC system please use a computing node for the installation. Useful variables to be added are 
-
- - `-DENABLE_NCCL=yes` to activate the NCCL collectives
- - `-DENABLE_MANAGED=yes` to activate the automatic memory management form the NVHPC compiler
-If you are getting the following error
-```
--- The CUDA compiler identification is unknown  
-CMake Error at /usr/share/cmake/Modules/CMakeDetermineCUDACompiler.cmake:633 (message):  
-Failed to detect a default CUDA architecture. 
-```
-It is possible that your default C compiler is too recent and not supported by `nvcc` . You might be able to solve the issue by adding 
- - `-DCMAKE_CUDA_HOST_COMPILER=$supported_gcc`
- 
- At the moment the supported CUDA host compilers are `gcc11` and earlier. 
-
 ## Profiling
 
 Profiling can be activated via `cmake` configuration, the recommended approach is to run the initial configuration as follows:
@@ -139,67 +106,6 @@ The external code can use the named variables to check the FFT backend used in a
 - The oneMKL backend supports the OVERWRITE flag and can perform in-place complex 1D fft
 - The cuFFT backend supports the OVERWRITE flag and can perform in-place complex 1D fft
 
-## Linking from external codes
-
-### Codes using Makefiles
-
-When building a code that links 2decomp-fft using a Makefile you will need to add the include and link paths as appropriate (`inlude/` and `lib/` under the installation directory, respectively).
-```
-DECOMP_ROOT = /path/to/2decomp-fft
-DECOMP_BUILD_DIR = $(DECOMP_ROOT)/build
-DECOMP_INSTALL_DIR ?= $(DECOMP_BUILD_DIR)/opt # Use default unless set by user
-
-INC += -I$(DECOMP_INSTALL_DIR)/include
-
-# Users build/link targets
-LIBS = -L$(DECOMP_INSTALL_DIR)/lib64 -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
-
-OBJ = my_exec.o
-
-my_exec: $(OBJ)
-	$(F90) -o $@ $(OBJ) $(LIBS)
-
-```
-In case 2decomp-fft has been compiled with an external FFT, such as FFTW3, `LIBS` should also contain the following 
-```
-FFTW3_PATH=/my_path_to_FFTW/lib
-LIBFFT=-L$(FFTW3_PATH) -lfftw3 -lfftw3f
-LIBS += $(LIBFFT)
-```
-In case of 2decomp-fft compiled for GPU with NVHPC, linking against cuFFT is mandatory 
-```
-LIBS += -cudalib=cufft
-```
-In case of NCCL the following is required 
-```
-LIBS += -cudalib=cufft,nccl 
-```
-It is also possible to drive the build and installation of 2decomp-fft from a Makefile such as in the following example code
-```
-FC = mpif90
-BUILD = Release
-
-DECOMP_ROOT = /path/to/2decomp-fft
-DECOMP_BUILD_DIR = $(DECOMP_ROOT)/build
-DECOMP_INSTALL_DIR ?= $(DECOMP_BUILD_DIR)/opt # Use default unless set by user
-
-INC += -I$(DECOMP_INSTALL_DIR)/include
-
-# Users build/link targets
-LIBS = -L$(DECOMP_INSTALL_DIR)/lib64 -L$(DECOMP_INSTALL_DIR)/lib -ldecomp2d
-
-# Building libdecomp.a
-$(DECOMP_INSTALL_DIR)/lib/libdecomp.a:
-	FC=$(FC) cmake -S $(DECOMP_ROOT) -B $(DECOMP_BUILD_DIR) -DCMAKE_BUILD_TYPE=$(BUILD) -DCMAKE_INSTALL_PREFIX=$(DECOMP_INSTALL_DIR)
-	cmake --build $(DECOMP_BUILD_DIR) --target decomp2d
-	cmake --build $(DECOMP_BUILD_DIR) --target install
-
-# Clean libdecomp.a
-clean-decomp:
-	cmake --build $(DECOMP_BUILD_DIR) --target clean
-	rm -f $(DECOMP_INSTALL_DIR)/lib/libdecomp.a
-```
-
 ## Miscellaneous
 
 ### Print the log to a file or to stdout

From f378ef6c3ca6cb2d4def473ea436471e1d490e15 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 28 Apr 2023 10:19:32 +0100
Subject: [PATCH 220/436] Attempt fix caliper CI

---
 .github/workflows/Build_all.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 2f87c151..7833bea5 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -175,12 +175,13 @@ jobs:
         wget --no-check-certificate https://github.com/LLNL/Caliper/archive/refs/tags/v2.8.0.tar.gz
         tar xzf v2.8.0.tar.gz && cd Caliper-2.8.0
         mkdir build && cd build
+        echo ${PWD}
         cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../Caliper-2.8.0_bld -DWITH_FORTRAN=yes -DWITH_MPI=yes ../
         make -j && make install
         cd ../../
 
     - name: Compile
       run: |
-        export caliper_DIR=./Caliper-2.8.0_bld/share/cmake/caliper
+        export caliper_DIR=${PWD}/Caliper-2.8.0_bld/share/cmake/caliper
         FC=mpif90 cmake -S . -B build -DENABLE_PROFILER=caliper -DBUILD_TESTING=ON
         cmake --build build --verbose

From 96376fa9d9ad6a630a7ae3b4f75434e6a2f6f243 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 28 Apr 2023 10:23:34 +0100
Subject: [PATCH 221/436] Fixing Caliper CI - print directories

---
 .github/workflows/Build_all.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 7833bea5..280f5896 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -172,16 +172,18 @@ jobs:
     # Runs a set of commands using the runners shell
     - name: Compile caliper
       run: |
+        echo ${PWD}
+        echo ${HOME}
         wget --no-check-certificate https://github.com/LLNL/Caliper/archive/refs/tags/v2.8.0.tar.gz
         tar xzf v2.8.0.tar.gz && cd Caliper-2.8.0
         mkdir build && cd build
         echo ${PWD}
-        cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../Caliper-2.8.0_bld -DWITH_FORTRAN=yes -DWITH_MPI=yes ../
+        cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=${HOME}/Caliper-2.8.0_bld -DWITH_FORTRAN=yes -DWITH_MPI=yes ../
         make -j && make install
         cd ../../
 
     - name: Compile
       run: |
-        export caliper_DIR=${PWD}/Caliper-2.8.0_bld/share/cmake/caliper
+        export caliper_DIR=${HOME}/Caliper-2.8.0_bld/share/cmake/caliper
         FC=mpif90 cmake -S . -B build -DENABLE_PROFILER=caliper -DBUILD_TESTING=ON
         cmake --build build --verbose

From 207dd28bcdb9170f7d250cdd45de3a3c8d360809 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 29 Apr 2023 10:05:13 +0200
Subject: [PATCH 222/436] Add a guide for contributions

---
 Contribute.md | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 Contribute.md

diff --git a/Contribute.md b/Contribute.md
new file mode 100644
index 00000000..131a5b49
--- /dev/null
+++ b/Contribute.md
@@ -0,0 +1,32 @@
+1. You want to contribute but have no idea how ? Please refer to the [Get started](#get-started) section.
+2. You have identified a bug ? Please refer to the [Bug](#bug) section.
+3. You have improved a part of the code or have developed a new functionality ? Please refer to the [Advanced](#advanced-contribution) section.
+
+## Get started
+
+## Bug
+
+It appears that you have identified a bug in the 2decomp library.
+If you are not sure this is really a bug in the library, you should go to the [discussions](https://github.com/2decomp-fft/2decomp-fft/discussions) section and open a new discussion.
+Otherwise, follow the steps below.
+
+Firstly, try to reproduce the error with a debug build of the library, a small problem size and a small number of MPI ranks.
+It makes bug-hunting much easier.
+Unfortunately, it is not always possible.
+At least, please try to reproduce the bug on another machine with another compiler.
+
+Secondly, if you have modified the source code of the 2decomp library, you must reproduce the bug without the modifications in 2decomp.
+The development team will only provide support for sections of code available in the present repository.
+
+Thirdly, you must provide a minimal working example.
+The program using 2decomp and exposing the bug should be relatively small.
+The development team will not provide support if the program exposing the bug is very long.
+The programs available in the examples section are a good starting point for a minimal working example.
+
+At this stage, you probably did your best to simplify the problem at hand.
+Open a issue and select the bug report template.
+Provide a meaningful title and do your best to complete all the sections of the template and provide the version of the library, the version of the compiler, the version of the MPI / FFT library, ...
+If you think you have a fix for the bug, please expose it inside the issue.
+It is recommended to wait for feedback before opening a pull-request.
+
+## Advanced contribution

From 11631a29c069139ec5bc7eec69bc6414464074f3 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 29 Apr 2023 10:16:22 +0200
Subject: [PATCH 223/436] Update Contribute.md

---
 Contribute.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Contribute.md b/Contribute.md
index 131a5b49..e9258879 100644
--- a/Contribute.md
+++ b/Contribute.md
@@ -12,6 +12,8 @@ Otherwise, follow the steps below.
 
 Firstly, try to reproduce the error with a debug build of the library, a small problem size and a small number of MPI ranks.
 It makes bug-hunting much easier.
+Please note that for a debug build, the log contains all the environment variables.
+Use it to hunt the bug but think twice before sharing it as it can expose sensitive and personal information.
 Unfortunately, it is not always possible.
 At least, please try to reproduce the bug on another machine with another compiler.
 
@@ -25,7 +27,7 @@ The programs available in the examples section are a good starting point for a m
 
 At this stage, you probably did your best to simplify the problem at hand.
 Open a issue and select the bug report template.
-Provide a meaningful title and do your best to complete all the sections of the template and provide the version of the library, the version of the compiler, the version of the MPI / FFT library, ...
+Provide a meaningful title, do your best to complete all the sections of the template and provide the version of the compiler, the version of the MPI / FFT library, ...
 If you think you have a fix for the bug, please expose it inside the issue.
 It is recommended to wait for feedback before opening a pull-request.
 

From 5677e1f96c0280d3d2f7fd3e3e78ae3fba8b86d3 Mon Sep 17 00:00:00 2001
From: rfj82982 <35581400+rfj82982@users.noreply.github.com>
Date: Sat, 29 Apr 2023 12:05:01 +0100
Subject: [PATCH 224/436] Add CMake variable to set manually CODA Architecture
 (#185)

Co-authored-by: not populated <rfje01@cirrus-login2.ib0.icexa.epcc.ed.ac.uk>
---
 README.md                              |  1 +
 cmake/D2D_GPU.cmake                    | 44 ++++++++++++++++----------
 cmake/compilers/D2D_flags_nvidia.cmake |  4 +--
 3 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index 2e864093..d4c285c5 100644
--- a/README.md
+++ b/README.md
@@ -85,6 +85,7 @@ By default CUDA aware MPI will be used together with `cuFFT` for the FFT library
 
  - `-DENABLE_NCCL=yes` to activate the NCCL collectives
  - `-DENABLE_MANAGED=yes` to activate the automatic memory management form the NVHPC compiler
+ - `-DSET_CUDA_ARCH=XY` to set the CUDA compute architecture (default is CC70, Volta V100)
 If you are getting the following error
 ```
 -- The CUDA compiler identification is unknown  
diff --git a/cmake/D2D_GPU.cmake b/cmake/D2D_GPU.cmake
index 4e43bcdd..cd535696 100644
--- a/cmake/D2D_GPU.cmake
+++ b/cmake/D2D_GPU.cmake
@@ -12,24 +12,34 @@ endif()
 
 if (ENABLE_CUDA)
   find_package(CUDAToolkit REQUIRED)
-  if(${CMAKE_VERSION} VERSION_LESS_EQUAL "3.13.4")
-    cuda_select_nvcc_arch_flags(ARCH_FLAGS "Auto") # optional argument for arch to add
-    message(STATUS "ARCH_FLAGS = ${ARCH_FLAGS}")
-    string(REPLACE "-gencode;" "--generate-code=" ARCH_FLAGS "${ARCH_FLAGS}")
-    string(APPEND CMAKE_CUDA_FLAGS "${ARCH_FLAGS}")
-    message(STATUS "ARCH_FLAGS WITH CUDA = ${ARCH_FLAGS}")
+  if (NOT SET_CUDA_ARCH)
+    set(SET_CUDA_ARCH 1 CACHE INTERNAL "Set CUDA Architecture" FORCE)
+    set(CUDA_ARCH_COMP 70 CACHE STRING "Set CUDA Computing Architecture")
+    if(${CMAKE_VERSION} VERSION_LESS_EQUAL "3.13.4")
+      cuda_select_nvcc_arch_flags(ARCH_FLAGS "Auto") # optional argument for arch to add
+      message(STATUS "ARCH_FLAGS = ${ARCH_FLAGS}")
+      string(REPLACE "-gencode;" "--generate-code=" ARCH_FLAGS "${ARCH_FLAGS}")
+      string(APPEND CMAKE_CUDA_FLAGS "${ARCH_FLAGS}")
+      message(STATUS "ARCH_FLAGS WITH CUDA = ${ARCH_FLAGS}")
+    else()
+      include(FindCUDA/select_compute_arch)
+      CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1)
+      string(STRIP "${INSTALLED_GPU_CCS_1}" INSTALLED_GPU_CCS_2)
+      string(REPLACE " " ";" INSTALLED_GPU_CCS_3 "${INSTALLED_GPU_CCS_2}")
+      string(REPLACE "." "" CUDA_ARCH_LIST "${INSTALLED_GPU_CCS_3}")
+      SET(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_LIST})
+      set_property(GLOBAL PROPERTY CUDA_ARCHITECTURES "${CUDA_ARCH_LIST}")
+      message(STATUS "CUDA_ARCHITECTURES ${CUDA_ARCH_LIST}")
+      list(GET CUDA_ARCH_LIST 0 CUDA_ARCH_FIRST)
+      message(STATUS "CUDA_ARCH_FIRST ${CUDA_ARCH_FIRST}")
+    endif()
+    message(STATUS "CUDA_LIBRARIES ${CUDA_LIBRARIES}")
+    if(${CUDA_ARCH_FIRST} GREATER ${CUDA_ARCH_COMP})
+      set(CUDA_ARCH_COMP ${CUDA_ARCH_FIRST} CACHE)
+    endif()
   else()
-    include(FindCUDA/select_compute_arch)
-    CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1)
-    string(STRIP "${INSTALLED_GPU_CCS_1}" INSTALLED_GPU_CCS_2)
-    string(REPLACE " " ";" INSTALLED_GPU_CCS_3 "${INSTALLED_GPU_CCS_2}")
-    string(REPLACE "." "" CUDA_ARCH_LIST "${INSTALLED_GPU_CCS_3}")
-    SET(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_LIST})
-    set_property(GLOBAL PROPERTY CUDA_ARCHITECTURES "${CUDA_ARCH_LIST}")
-    message(STATUS "CUDA_ARCHITECTURES ${CUDA_ARCH_LIST}")
-    list(GET CUDA_ARCH_LIST 0 CUDA_ARCH_FIRST)
-    message(STATUS "CUDA_ARCH_FIRST ${CUDA_ARCH_FIRST}")
+    set(CUDA_ARCH_COMP ${SET_CUDA_ARCH})
   endif()
-  message(STATUS "CUDA_LIBRARIES ${CUDA_LIBRARIES}")
+  message(STATUS "CUDA_COMP ${CUDA_ARCH_COMP}")
 endif()
 
diff --git a/cmake/compilers/D2D_flags_nvidia.cmake b/cmake/compilers/D2D_flags_nvidia.cmake
index 73a2d1d4..53e7b87a 100644
--- a/cmake/compilers/D2D_flags_nvidia.cmake
+++ b/cmake/compilers/D2D_flags_nvidia.cmake
@@ -19,9 +19,9 @@ if (BUILD_TARGET MATCHES "gpu")
     set(D2D_FFLAGS "${D2D_FFLAGS} -cuda")
     # Add Compute Capabilities and memory managemnt 
     if (ENABLE_MANAGED)
-      set(D2D_FFLAGS "${D2D_FFLAGS} -gpu=cc${CUDA_ARCH_FIRST},managed,lineinfo")
+	    set(D2D_FFLAGS "${D2D_FFLAGS} -gpu=cc${CUDA_ARCH_COMP},managed,lineinfo")
     else (ENABLE_MANAGED)
-      set(D2D_FFLAGS "${D2D_FFLAGS} -gpu=cc${CUDA_ARCH_FIRST},lineinfo")
+	    set(D2D_FFLAGS "${D2D_FFLAGS} -gpu=cc${CUDA_ARCH_COMP},lineinfo")
     endif(ENABLE_MANAGED)
     # Add NCCL cuFFT
     if (ENABLE_NCCL)

From 5a7ef0a00ba9f03244214f1680be3168bebfa964 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 29 Apr 2023 14:10:44 +0200
Subject: [PATCH 225/436] Minor update

---
 Contribute.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Contribute.md b/Contribute.md
index e9258879..de9b6820 100644
--- a/Contribute.md
+++ b/Contribute.md
@@ -12,9 +12,9 @@ Otherwise, follow the steps below.
 
 Firstly, try to reproduce the error with a debug build of the library, a small problem size and a small number of MPI ranks.
 It makes bug-hunting much easier.
+Unfortunately, it is not always possible.
 Please note that for a debug build, the log contains all the environment variables.
 Use it to hunt the bug but think twice before sharing it as it can expose sensitive and personal information.
-Unfortunately, it is not always possible.
 At least, please try to reproduce the bug on another machine with another compiler.
 
 Secondly, if you have modified the source code of the 2decomp library, you must reproduce the bug without the modifications in 2decomp.

From 839845ac06e2256ad56e13b523af550b624fbe0e Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 29 Apr 2023 14:55:27 +0200
Subject: [PATCH 226/436] Update Contribute.md

---
 Contribute.md | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/Contribute.md b/Contribute.md
index de9b6820..cd887052 100644
--- a/Contribute.md
+++ b/Contribute.md
@@ -1,9 +1,13 @@
 1. You want to contribute but have no idea how ? Please refer to the [Get started](#get-started) section.
 2. You have identified a bug ? Please refer to the [Bug](#bug) section.
-3. You have improved a part of the code or have developed a new functionality ? Please refer to the [Advanced](#advanced-contribution) section.
+3. You have improved a part of the code or have developed a new functionality ? Please refer to the [advanced](#advanced-contribution) section.
 
 ## Get started
 
+The recommended strategy to contribute is to start with a [discussion](https://github.com/2decomp-fft/2decomp-fft/discussions) or to pick a existing issue.
+To modify or experiment with the code, fork the 2decomp github repository and commit changes in a dedicated branch of your fork.
+When the modification is ready for review, one can open a pull request as exposed in the [advanced](#advanced-contribution) section below.
+
 ## Bug
 
 It appears that you have identified a bug in the 2decomp library.
@@ -29,6 +33,22 @@ At this stage, you probably did your best to simplify the problem at hand.
 Open a issue and select the bug report template.
 Provide a meaningful title, do your best to complete all the sections of the template and provide the version of the compiler, the version of the MPI / FFT library, ...
 If you think you have a fix for the bug, please expose it inside the issue.
-It is recommended to wait for feedback before opening a pull-request.
+It is recommended to wait for feedback before opening a pull request.
 
 ## Advanced contribution
+
+One should read this section before opening a pull request.
+To fix a bug, please open a issue and use the bug report template first.
+To improve a part of the code or develop a new functionality, please open a issue and use the feature request template first.
+If you are not sure about your contribution, open a [discussion](https://github.com/2decomp-fft/2decomp-fft/discussions) first.
+
+The code in a pull-request should be formatted using the `fprettify` program.
+See the code sample in the `scripts` folder.
+
+Pull requests must be focused, small, coherent and have a detailed description.
+The longer the pull request, the harder the review.
+Please empathise with your fellow contributors who are going to spend time reviewing your code.
+
+As long as the pull request is open for discussion and not ready for merging, convert it to draft.
+Whenever it is ready for merging, convert it to a regular pull request.
+Please note that reviewers might push modifications directly to your branch.

From 62ca0eb97942fbaa39f6a52b27f03b26fb5fbbda Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 2 May 2023 08:57:55 +0200
Subject: [PATCH 227/436] Provide required / tested versions for various
 requirements and optional libraries (#189)

---
 README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/README.md b/README.md
index d4c285c5..a338d1fa 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,15 @@
 
 ## Building
 
+The library 2decomp is a Fortran library compatible with the Fortran 2008 standard.
+It requires a MPI library compatible with MPI-2.0 with extended Fortran support.
+The following [optional libraries](#optional-dependencies) can be used :
+
+- FFTW3, version 3.3.10 was tested
+- Intel oneMKL (oneAPI Math Kernel Library), version 2023.0.0 was tested
+- Nvidia GPU-related libraries, NVHPC version 22.7 and CUDA version 11.8 were tested
+- Caliper, version 2.9.1 was tested
+
 ### CMake
 
 The build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 

From dd69c6059f7c89c93f90d1fab406b6e376e533e4 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 2 May 2023 10:09:32 +0200
Subject: [PATCH 228/436] Update CMake to enable the EVEN preprocessor flag.
 Update the code to hide EVEN-specific variables. (#191)

---
 CMakeLists.txt            |  1 +
 README.md                 |  2 +-
 cmake/D2D_Compilers.cmake |  8 ++++++++
 src/decomp_2d.f90         | 25 +++++++++++++------------
 src/log.f90               |  2 ++
 5 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8617167f..7e8fcb4e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -28,6 +28,7 @@ set (ENABLE_PROFILER "OFF" CACHE STRING "Activate/select the profiler")
 set_property(CACHE ENABLE_PROFILER PROPERTY STRINGS OFF caliper)
 option(ENABLE_INPLACE "Enable in-place operation of FFT" OFF)
 option(HALO_DEBUG "Enable halo debugging options" OFF)
+option(EVEN "Padded alltoall transpose operations" OFF)
 option(BUILD_SHARED_LIBS "Set ON to build a shared library" OFF)
 
 set(AUTHOR "Stefano Rolfo;Charles Moulinec;Paul Bartholomew")
diff --git a/README.md b/README.md
index a338d1fa..090dc91f 100644
--- a/README.md
+++ b/README.md
@@ -247,7 +247,7 @@ This variable is automatically added when selecting the profiler. It activates t
 
 #### EVEN
 
-This preprocessor variable is not valid for GPU builds. It leads to padded alltoall operations.
+This preprocessor variable is not valid for GPU builds. It leads to padded alltoall operations. This preprocessor variable is driven by the CMake on/off variable `EVEN`.
 
 #### OVERWRITE
 
diff --git a/cmake/D2D_Compilers.cmake b/cmake/D2D_Compilers.cmake
index 37b73741..6369274f 100644
--- a/cmake/D2D_Compilers.cmake
+++ b/cmake/D2D_Compilers.cmake
@@ -73,6 +73,14 @@ if (ENABLE_INPLACE)
   add_definitions("-DOVERWRITE")
 endif ()
 
+# Padded MPI alltoall transpose operations (invalid for GPU)
+if (EVEN)
+  if (BUILD_TARGET MATCHES "gpu")
+    message(FATAL_ERROR "The GPU build is not compatible with padded alltoall")
+  endif ()
+  add_definitions("-DEVEN")
+endif ()
+
 execute_process(
   COMMAND git describe --tag --long --always
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 8f9d6ea8..7bf849f7 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -100,12 +100,12 @@ module decomp_2d
       integer, allocatable, dimension(:) :: &
          x1disp, y1disp, y2disp, z2disp
 
-      ! buffer counts for MPI_ALLTOALL: either for evenly distributed data
-      ! or for padded-alltoall
+#ifdef EVEN
+      ! buffer counts for MPI_ALLTOALL for padded-alltoall
       integer :: x1count, y1count, y2count, z2count
-
       ! evenly distributed data
       logical :: even
+#endif
 
    END TYPE DECOMP_INFO
 
@@ -373,13 +373,6 @@ subroutine decomp_info_init(nx, ny, nz, decomp)
                               'min(ny,nz) >= p_col')
       end if
 
-      if (mod(nx, dims(1)) == 0 .and. mod(ny, dims(1)) == 0 .and. &
-          mod(ny, dims(2)) == 0 .and. mod(nz, dims(2)) == 0) then
-         decomp%even = .true.
-      else
-         decomp%even = .false.
-      end if
-
       ! distribute mesh points
       allocate (decomp%x1dist(0:dims(1) - 1), decomp%y1dist(0:dims(1) - 1), &
                 decomp%y2dist(0:dims(2) - 1), decomp%z2dist(0:dims(2) - 1))
@@ -406,10 +399,18 @@ subroutine decomp_info_init(nx, ny, nz, decomp)
       buf_size = max(decomp%xsz(1) * decomp%xsz(2) * decomp%xsz(3), &
                      max(decomp%ysz(1) * decomp%ysz(2) * decomp%ysz(3), &
                          decomp%zsz(1) * decomp%zsz(2) * decomp%zsz(3)))
+
 #ifdef EVEN
       ! padded alltoall optimisation may need larger buffer space
       buf_size = max(buf_size, &
                      max(decomp%x1count * dims(1), decomp%y2count * dims(2)))
+      ! evenly distributed data ?
+      if (mod(nx, dims(1)) == 0 .and. mod(ny, dims(1)) == 0 .and. &
+          mod(ny, dims(2)) == 0 .and. mod(nz, dims(2)) == 0) then
+         decomp%even = .true.
+      else
+         decomp%even = .false.
+      end if
 #endif
 
       ! check if additional memory is required
@@ -1106,7 +1107,7 @@ subroutine prepare_buffer(decomp)
       end do
 
       ! MPI_ALLTOALL buffer information
-
+#ifdef EVEN
       ! For evenly distributed data, following is an easier implementation.
       ! But it should be covered by the more general formulation below.
       !decomp%x1count = decomp%xsz(1)*decomp%xsz(2)*decomp%xsz(3)/dims(1)
@@ -1124,8 +1125,8 @@ subroutine prepare_buffer(decomp)
       decomp%y2count = decomp%y2dist(dims(2) - 1) * &
                        decomp%z2dist(dims(2) - 1) * decomp%zsz(1)
       decomp%z2count = decomp%y2count
+#endif
 
-      return
    end subroutine prepare_buffer
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/log.f90 b/src/log.f90
index 4525bc52..dde647d5 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -291,11 +291,13 @@ module subroutine decomp_info_print(d2d, io_unit, d2dname)
       write (io_unit, *) '   y1disp : ', d2d%y1disp
       write (io_unit, *) '   y2disp : ', d2d%y2disp
       write (io_unit, *) '   z2disp : ', d2d%z2disp
+#ifdef EVEN
       write (io_unit, *) '   x1count : ', d2d%x1count
       write (io_unit, *) '   y1count : ', d2d%y1count
       write (io_unit, *) '   y2count : ', d2d%y2count
       write (io_unit, *) '   z2count : ', d2d%z2count
       write (io_unit, *) '   even : ', d2d%even
+#endif
 #endif
 
    end subroutine decomp_info_print

From 2d229cb51e683d502c111e8b4b8fb4628318ee58 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Thu, 4 May 2023 11:58:58 +0200
Subject: [PATCH 229/436] Update Build_all.yml

Update CI to increase code coverage.
---
 .github/workflows/Build_all.yml | 38 ++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 280f5896..cd534c85 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -33,6 +33,38 @@ jobs:
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON
         cmake --build build --verbose
 
+    # Runs a set of commands using the runners shell
+    - name: Single precision
+      run: |
+        export FC=mpif90.openmpi
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DDOUBLE_PRECISION=OFF
+        cmake --build build --target clean
+        cmake --build build --verbose
+
+    # Runs a set of commands using the runners shell
+    - name: Double precision with single precision IO
+      run: |
+        export FC=mpif90.openmpi
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DSINGLE_PRECISION_OUTPUT=ON
+        cmake --build build --target clean
+        cmake --build build --verbose
+
+    # Runs a set of commands using the runners shell
+    - name: Padded alltoall transpose operations
+      run: |
+        export FC=mpif90.openmpi
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DEVEN=ON
+        cmake --build build --target clean
+        cmake --build build --verbose
+
+    # Runs a set of commands using the runners shell
+    - name: Padded alltoall transpose operations
+      run: |
+        export FC=mpif90.openmpi
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DHALO_DEBUG=ON
+        cmake --build build --target clean
+        cmake --build build --verbose
+
   bld_gnu_mpich:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
@@ -80,7 +112,7 @@ jobs:
     # Runs a set of commands using the runners shell
     - name: CPU version
       run: |
-        PATH=$TMP:$PATH FC=mpif90 cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev
+        PATH=$TMP:$PATH FC=mpif90 cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON
         PATH=$TMP:$PATH cmake --build build --verbose
 
 ###    - name: GPU version
@@ -144,7 +176,7 @@ jobs:
     - name: Regular FFTW3
       run: |
         export FC=mpif90
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DFFT_Choice=fftw -DBUILD_TESTING=ON
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=fftw -DBUILD_TESTING=ON
         cmake --build build --verbose
         cmake --build build --target clean
 
@@ -185,5 +217,5 @@ jobs:
     - name: Compile
       run: |
         export caliper_DIR=${HOME}/Caliper-2.8.0_bld/share/cmake/caliper
-        FC=mpif90 cmake -S . -B build -DENABLE_PROFILER=caliper -DBUILD_TESTING=ON
+        FC=mpif90 cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DENABLE_PROFILER=caliper -DBUILD_TESTING=ON
         cmake --build build --verbose

From f5ecfb1f762c12b7237bfc86854110fd01a37dbc Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 7 May 2023 11:18:53 +0200
Subject: [PATCH 230/436] Add ADIOS2 support in the CMake build

---
 CMakeLists.txt            |  2 ++
 README.md                 | 25 +++++++++++++++++++++++++
 cmake/D2D_Compilers.cmake | 13 +++++++++++++
 cmake/D2D_MPI.cmake       |  7 +++++++
 src/CMakeLists.txt        |  6 ++++++
 5 files changed, 53 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7e8fcb4e..cac2651f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -30,6 +30,8 @@ option(ENABLE_INPLACE "Enable in-place operation of FFT" OFF)
 option(HALO_DEBUG "Enable halo debugging options" OFF)
 option(EVEN "Padded alltoall transpose operations" OFF)
 option(BUILD_SHARED_LIBS "Set ON to build a shared library" OFF)
+set (IO_BACKEND "mpi" CACHE STRING "Default IO backend (mpi (default) or adios2)")
+set_property(CACHE IO_BACKEND PROPERTY STRINGS mpi adios2)
 
 set(AUTHOR "Stefano Rolfo;Charles Moulinec;Paul Bartholomew")
 set(AUTHOR_DETAILS "stefano.rolfo@stfc.ac.uk;charles.moulinec@stfc.ac.uk;p.bartholomew@epcc.ed.ac.uk")
diff --git a/README.md b/README.md
index 090dc91f..49e649e8 100644
--- a/README.md
+++ b/README.md
@@ -6,6 +6,7 @@ The library 2decomp is a Fortran library compatible with the Fortran 2008 standa
 It requires a MPI library compatible with MPI-2.0 with extended Fortran support.
 The following [optional libraries](#optional-dependencies) can be used :
 
+- ADIOS2, version 2.9.0 was tested
 - FFTW3, version 3.3.10 was tested
 - Intel oneMKL (oneAPI Math Kernel Library), version 2023.0.0 was tested
 - Nvidia GPU-related libraries, NVHPC version 22.7 and CUDA version 11.8 were tested
@@ -229,6 +230,10 @@ Before calling `decomp_2d_init`, the external code can modify the variable `deco
 
 ### List of preprocessor variables
 
+#### ADIOS2
+
+This variable is automatically added in builds with the adios2 IO backend.
+
 #### DEBUG
 
 This variable is automatically added in debug and dev builds. Extra information is printed when it is present.
@@ -272,6 +277,26 @@ It is recommended that you should format the code before making a pull request.
 
 ## Optional dependencies
 
+### ADIOS2
+
+The library [adios2](https://adios2.readthedocs.io/en/latest/) can be used as a backend for IO. The version 2.9.0 was tested, is supported and can be downloaded [here](https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.9.0.tar.gz). Below are build instructions for the library. However, it is recommended to use the one provided by the administrators of the computing centre if available.
+
+```
+wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.9.0.tar.gz
+tar xzf v2.9.0.tar.gz
+mkdir 2.9.0_tmp && cd 2.9.0_tmp
+CC=mpicc CXX=mpicxx FC=mpif90 cmake -S ../ADIOS2-2.9.0 -DCMAKE_INSTALL_PREFIX=../2.9.0_bld
+make -j
+make -j test
+make -j install
+```
+
+To build `2decomp&fft` with the adios2 IO backend, one can provide the package configuration for adios2 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/adios2/install/lib/cmake/adios2`. One can also provide the option `-Dadios2_DIR=/path/to/adios2/install/lib/cmake/adios2`. Then either specify on the command line when configuring the build
+```
+cmake -S . -B ./build -DIO_BACKEND=adios2 -Dadios2_DIR=/path/to/adios2/install/lib/cmake/adios2
+```
+or modify the build configuration using `ccmake`. Please note that the support for ADIOS2 is not complete. Currently, for a given IO operation, when the ADIOS2 backend is not supported, the MPI backend is used.
+
 ### FFTW
 
 The library [fftw](http://www.fftw.org/index.html) can be used as a backend for the FFT engine. The version 3.3.10 was tested, is supported and can be downloaded [here](http://www.fftw.org/download.html). Please note that one should build fftw and decomp2d against the same compilers. For build instructions, please check [here](http://www.fftw.org/fftw3_doc/Installation-on-Unix.html). Below is a suggestion for the compilation of the library in double precision (add `--enable-single` for a single precision build):
diff --git a/cmake/D2D_Compilers.cmake b/cmake/D2D_Compilers.cmake
index 6369274f..731a073f 100644
--- a/cmake/D2D_Compilers.cmake
+++ b/cmake/D2D_Compilers.cmake
@@ -97,3 +97,16 @@ option(SINGLE_PRECISION_OUTPUT "Build XCompact with output in single precision"
 if (SINGLE_PRECISION_OUTPUT)
   add_definitions("-DSAVE_SINGLE")
 endif()
+
+if (IO_BACKEND MATCHES "mpi")
+  message(STATUS "Using mpi (default) IO backend")
+elseif (IO_BACKEND MATCHES "adios2")
+  message(STATUS "Using ADIOS2 IO backend")
+  find_package(adios2 REQUIRED)
+  if (NOT ADIOS2_HAVE_MPI)
+    message(FATAL_ERROR "MPI support is missing in the provided ADIOS2 build")
+  endif (NOT ADIOS2_HAVE_MPI)
+  if (NOT ADIOS2_HAVE_Fortran)
+    message(FATAL_ERROR "Fortran support is missing in the provided ADIOS2 build")
+  endif (NOT ADIOS2_HAVE_Fortran)
+endif (IO_BACKEND MATCHES "mpi")
diff --git a/cmake/D2D_MPI.cmake b/cmake/D2D_MPI.cmake
index c91c798e..2d745355 100644
--- a/cmake/D2D_MPI.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -1,6 +1,13 @@
 # MPI CMakeLists
 
 find_package(MPI REQUIRED)
+
+# adios2 IO backend requires C and C++ MPI components
+if (IO_BACKEND MATCHES "adios2")
+    enable_language(C CXX)
+    find_package(MPI REQUIRED COMPONENTS C CXX)
+endif (IO_BACKEND MATCHES "adios2")
+
 # Stop if there is no MPI_Fortran_Compiler
 if (MPI_Fortran_COMPILER)
     message(STATUS "MPI_Fortran_COMPILER found: ${MPI_Fortran_COMPILER}")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index e8a84b59..1bb364ad 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -74,10 +74,16 @@ if (ENABLE_PROFILER)
     target_link_libraries(decomp2d PRIVATE caliper)
   endif()
 endif()
+
 if (HALO_DEBUG)
   target_compile_definitions(decomp2d PUBLIC -DHALO_DEBUG)
 endif()
 
+if (IO_BACKEND MATCHES "adios2")
+  add_definitions("-DADIOS2")
+  target_link_libraries(decomp2d PRIVATE adios2::fortran_mpi adios2::fortran)
+endif (IO_BACKEND MATCHES "adios2")
+
 install(TARGETS decomp2d
   EXPORT decomp2d-targets
   RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}

From 6d81324a904fd1e7154ed85ff7afd8547811217a Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 7 May 2023 11:24:07 +0200
Subject: [PATCH 231/436] Add adios2 in the CI

---
 .github/workflows/Build_all.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 280f5896..fb8de0de 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -187,3 +187,23 @@ jobs:
         export caliper_DIR=${HOME}/Caliper-2.8.0_bld/share/cmake/caliper
         FC=mpif90 cmake -S . -B build -DENABLE_PROFILER=caliper -DBUILD_TESTING=ON
         cmake --build build --verbose
+
+  bld_adios:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest                                                          
+   
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - uses: actions/checkout@v3
+   
+    # Runs a single command using the runners shell                      
+    - name: Install requirements
+      run: | 
+        sudo apt-get update
+        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev libadios-bin libadios-dev libadios-openmpi-dev cmake gcc g++
+
+    - name: Compile             
+      run: |
+        FC=mpif90 cmake -S . -B build -DIO_BACKEND=adios2 -DBUILD_TESTING=ON
+        cmake --build build --verbose

From 483864ed78532dc2418a39878e5f9e327d9e7ca1 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 7 May 2023 11:39:26 +0200
Subject: [PATCH 232/436] Fix CI

---
 .github/workflows/Build_all.yml | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index fb8de0de..483474d6 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -172,8 +172,6 @@ jobs:
     # Runs a set of commands using the runners shell
     - name: Compile caliper
       run: |
-        echo ${PWD}
-        echo ${HOME}
         wget --no-check-certificate https://github.com/LLNL/Caliper/archive/refs/tags/v2.8.0.tar.gz
         tar xzf v2.8.0.tar.gz && cd Caliper-2.8.0
         mkdir build && cd build
@@ -201,9 +199,21 @@ jobs:
     - name: Install requirements
       run: | 
         sudo apt-get update
-        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev libadios-bin libadios-dev libadios-openmpi-dev cmake gcc g++
+        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev cmake gcc g++
+
+    # Runs a set of commands using the runners shell
+    - name: Compile caliper
+      run: | 
+        echo ${PWD}
+        echo ${HOME}
+        wget --no-check-certificate https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.9.0.tar.gz
+        tar xzf v2.9.0.tar.gz
+        mkdir adios2_2.9.0_tmp && cd adios2_2.9.0_tmp
+        CC=mpicc CXX=mpicxx FC=mpif90 cmake -S ../ADIOS2-2.9.0 -DCMAKE_INSTALL_PREFIX=${HOME}/adios2_2.9.0_bld
+        make -j && make -j install
+        cd ../
 
     - name: Compile             
       run: |
-        FC=mpif90 cmake -S . -B build -DIO_BACKEND=adios2 -DBUILD_TESTING=ON
+        FC=mpif90 cmake -S . -B build -DIO_BACKEND=adios2 -Dadios2_DIR=${HOME}/adios2_2.9.0_bld/lib/cmake/adios2 -DBUILD_TESTING=ON
         cmake --build build --verbose

From c9e6b0dbbfc22df5b2ce013735ea2b8ca9366e67 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 7 May 2023 12:02:04 +0200
Subject: [PATCH 233/436] CI : Try to compile adios2 faster

---
 .github/workflows/Build_all.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 483474d6..175b8d07 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -186,7 +186,7 @@ jobs:
         FC=mpif90 cmake -S . -B build -DENABLE_PROFILER=caliper -DBUILD_TESTING=ON
         cmake --build build --verbose
 
-  bld_adios:
+  bld_adios2:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest                                                          
    
@@ -202,14 +202,14 @@ jobs:
         sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev cmake gcc g++
 
     # Runs a set of commands using the runners shell
-    - name: Compile caliper
+    - name: Compile adios2
       run: | 
         echo ${PWD}
         echo ${HOME}
         wget --no-check-certificate https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.9.0.tar.gz
         tar xzf v2.9.0.tar.gz
         mkdir adios2_2.9.0_tmp && cd adios2_2.9.0_tmp
-        CC=mpicc CXX=mpicxx FC=mpif90 cmake -S ../ADIOS2-2.9.0 -DCMAKE_INSTALL_PREFIX=${HOME}/adios2_2.9.0_bld
+        CC=mpicc CXX=mpicxx FC=mpif90 cmake -S ../ADIOS2-2.9.0 -DCMAKE_INSTALL_PREFIX=${HOME}/adios2_2.9.0_bld -DBUILD_SHARED_LIBS=on -DADIOS2_BUILD_EXAMPLES=off -DBUILD_TESTING=off -DCMAKE_BUILD_TYPE=Debug -DADIOS2_USE_MPI=on -DADIOS2_USE_ZeroMQ=off -DADIOS2_USE_HDF5=off -DADIOS2_USE_Python=off -DADIOS2_USE_Fortran=on -DADIOS2_USE_SST=off -DADIOS2_USE_BZip2=off -DADIOS2_USE_ZFP=off -DADIOS2_USE_SZ=off -DADIOS2_USE_MGARD=off -DADIOS2_USE_PNG=off -DADIOS2_USE_IME=off
         make -j && make -j install
         cd ../
 

From 8924d1e22764325b0974e78e6bad84f07cc3499e Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 7 May 2023 12:43:11 +0200
Subject: [PATCH 234/436] CI : try to use a docker image for adios2

---
 .github/workflows/Build_all.yml | 24 ++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 175b8d07..69fb57af 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -188,32 +188,16 @@ jobs:
 
   bld_adios2:
     # The type of runner that the job will run on
-    runs-on: ubuntu-latest                                                          
+    runs-on: ubuntu-latest
+    container:
+      image: ornladios/adios2:release_29-ubuntu-bionic
    
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v3
    
-    # Runs a single command using the runners shell                      
-    - name: Install requirements
-      run: | 
-        sudo apt-get update
-        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev cmake gcc g++
-
-    # Runs a set of commands using the runners shell
-    - name: Compile adios2
-      run: | 
-        echo ${PWD}
-        echo ${HOME}
-        wget --no-check-certificate https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.9.0.tar.gz
-        tar xzf v2.9.0.tar.gz
-        mkdir adios2_2.9.0_tmp && cd adios2_2.9.0_tmp
-        CC=mpicc CXX=mpicxx FC=mpif90 cmake -S ../ADIOS2-2.9.0 -DCMAKE_INSTALL_PREFIX=${HOME}/adios2_2.9.0_bld -DBUILD_SHARED_LIBS=on -DADIOS2_BUILD_EXAMPLES=off -DBUILD_TESTING=off -DCMAKE_BUILD_TYPE=Debug -DADIOS2_USE_MPI=on -DADIOS2_USE_ZeroMQ=off -DADIOS2_USE_HDF5=off -DADIOS2_USE_Python=off -DADIOS2_USE_Fortran=on -DADIOS2_USE_SST=off -DADIOS2_USE_BZip2=off -DADIOS2_USE_ZFP=off -DADIOS2_USE_SZ=off -DADIOS2_USE_MGARD=off -DADIOS2_USE_PNG=off -DADIOS2_USE_IME=off
-        make -j && make -j install
-        cd ../
-
     - name: Compile             
       run: |
-        FC=mpif90 cmake -S . -B build -DIO_BACKEND=adios2 -Dadios2_DIR=${HOME}/adios2_2.9.0_bld/lib/cmake/adios2 -DBUILD_TESTING=ON
+        FC=mpif90 cmake -S . -B build -DIO_BACKEND=adios2 -DBUILD_TESTING=ON
         cmake --build build --verbose

From 94267060382c303565679de150b6718ded017fdf Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 7 May 2023 12:45:39 +0200
Subject: [PATCH 235/436] CI : remove adios2

---
 .github/workflows/Build_all.yml | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 69fb57af..adc9b3ac 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -185,19 +185,3 @@ jobs:
         export caliper_DIR=${HOME}/Caliper-2.8.0_bld/share/cmake/caliper
         FC=mpif90 cmake -S . -B build -DENABLE_PROFILER=caliper -DBUILD_TESTING=ON
         cmake --build build --verbose
-
-  bld_adios2:
-    # The type of runner that the job will run on
-    runs-on: ubuntu-latest
-    container:
-      image: ornladios/adios2:release_29-ubuntu-bionic
-   
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v3
-   
-    - name: Compile             
-      run: |
-        FC=mpif90 cmake -S . -B build -DIO_BACKEND=adios2 -DBUILD_TESTING=ON
-        cmake --build build --verbose

From 3160fc138247ebbfcb78cb3b524a589c0d232feb Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Sun, 7 May 2023 16:33:54 +0200
Subject: [PATCH 236/436] Minor update

Fix name for the CI test with halo_debug active
---
 .github/workflows/Build_all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index cd534c85..34fac423 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -58,7 +58,7 @@ jobs:
         cmake --build build --verbose
 
     # Runs a set of commands using the runners shell
-    - name: Padded alltoall transpose operations
+    - name: Debug flag for halo operations
       run: |
         export FC=mpif90.openmpi
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DHALO_DEBUG=ON

From d3caf496a67e49dfd9ae2d9707f5f654c7f4735a Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 10 May 2023 16:49:46 +0200
Subject: [PATCH 237/436] Fix examples for single precision builds with dev
 flags

---
 examples/fft_physical_x/fft_c2c_x.f90  | 16 ++++++++--------
 examples/fft_physical_x/fft_grid_x.f90 | 16 ++++++++--------
 examples/fft_physical_z/fft_c2c_z.f90  | 16 ++++++++--------
 3 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index d692f8b9..d1edd4d0 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -26,11 +26,11 @@ program fft_c2c_x
    type(decomp_info), pointer :: ph => null()
    complex(mytype), allocatable, dimension(:, :, :) :: in, out
 
-   real(mytype) :: dr, di, error, err_all, n1, flops
+   real(mytype) :: dr, di, error, err_all
    integer :: ierror, i, j, k, m
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
-   double precision :: t1, t2, t3, t4
+   double precision :: n1, flops, t1, t2, t3, t4
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -160,14 +160,14 @@ program fft_c2c_x
       write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype) * real(ny, mytype) * real(nz, mytype)
-      n1 = n1**(1._mytype / 3._mytype)
+      n1 = real(nx) * real(ny) * real(nz)
+      n1 = n1**(1.d0 / 3.d0)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
+      flops = 5.d0 * n1 * log(n1) / log(2.d0)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops * 3._mytype * n1**2
-      flops = 2._mytype * flops / ((t1 + t3) / real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops / 1000._mytype**3
+      flops = flops * 3.d0 * n1**2
+      flops = 2.d0 * flops / ((t1 + t3) / real(NTEST))
+      write (*, *) 'GFLOPS : ', flops / 1000.d0**3
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 1410fa15..eca12d4c 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -26,11 +26,11 @@ program fft_physical_x
    type(decomp_info), pointer :: ph => null()
    complex(mytype), allocatable, dimension(:, :, :) :: in, out
 
-   real(mytype) :: dr, di, error, err_all, n1, flops
+   real(mytype) :: dr, di, error, err_all
    integer :: ierror, i, j, k, m
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
-   double precision :: t1, t2, t3, t4
+   double precision :: n1, flops, t1, t2, t3, t4
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -161,14 +161,14 @@ program fft_physical_x
       write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype) * real(ny, mytype) * real(nz, mytype)
-      n1 = n1**(1._mytype / 3._mytype)
+      n1 = real(nx) * real(ny) * real(nz)
+      n1 = n1**(1.d0 / 3.d0)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
+      flops = 5.d0 * n1 * log(n1) / log(2.d0)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops * 3._mytype * n1**2
-      flops = 2._mytype * flops / ((t1 + t3) / real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops / 1000._mytype**3
+      flops = flops * 3.d0 * n1**2
+      flops = 2.d0 * flops / ((t1 + t3) / real(NTEST))
+      write (*, *) 'GFLOPS : ', flops / 1000.d0**3
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 347b5123..3b56a414 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -26,11 +26,11 @@ program fft_c2c_z
    type(decomp_info), pointer :: ph => null()
    complex(mytype), allocatable, dimension(:, :, :) :: in, out
 
-   real(mytype) :: dr, di, error, err_all, n1, flops
+   real(mytype) :: dr, di, error, err_all
    integer :: ierror, i, j, k, m
    integer :: zst1, zst2, zst3
    integer :: zen1, zen2, zen3
-   double precision :: t1, t2, t3, t4
+   double precision :: n1, flops, t1, t2, t3, t4
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -161,14 +161,14 @@ program fft_c2c_z
       write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
-      n1 = real(nx, mytype) * real(ny, mytype) * real(nz, mytype)
-      n1 = n1**(1._mytype / 3._mytype)
+      n1 = real(nx) * real(ny) * real(nz)
+      n1 = n1**(1.d0 / 3.d0)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
-      flops = 5._mytype * n1 * log(n1) / log(2.0_mytype)
+      flops = 5.d0 * n1 * log(n1) / log(2.d0)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
-      flops = flops * 3._mytype * n1**2
-      flops = 2._mytype * flops / ((t1 + t3) / real(NTEST, mytype))
-      write (*, *) 'GFLOPS : ', flops / 1000._mytype**3
+      flops = flops * 3.d0 * n1**2
+      flops = 2.d0 * flops / ((t1 + t3) / real(NTEST))
+      write (*, *) 'GFLOPS : ', flops / 1000.d0**3
    end if
    !$acc end data
 

From 2e1f04f54ca268b9726373eb3a8385b3cd6b9e1e Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Thu, 11 May 2023 07:19:46 +0200
Subject: [PATCH 238/436] Update Build_all.yml

Update the CI script to restore the default flags
---
 .github/workflows/Build_all.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 34fac423..13b8e04b 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -37,32 +37,32 @@ jobs:
     - name: Single precision
       run: |
         export FC=mpif90.openmpi
+        rm -rf ./build
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DDOUBLE_PRECISION=OFF
-        cmake --build build --target clean
         cmake --build build --verbose
 
     # Runs a set of commands using the runners shell
     - name: Double precision with single precision IO
       run: |
         export FC=mpif90.openmpi
+        rm -rf ./build
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DSINGLE_PRECISION_OUTPUT=ON
-        cmake --build build --target clean
         cmake --build build --verbose
 
     # Runs a set of commands using the runners shell
     - name: Padded alltoall transpose operations
       run: |
         export FC=mpif90.openmpi
+        rm -rf ./build
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DEVEN=ON
-        cmake --build build --target clean
         cmake --build build --verbose
 
     # Runs a set of commands using the runners shell
     - name: Debug flag for halo operations
       run: |
         export FC=mpif90.openmpi
+        rm -rf ./build
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DHALO_DEBUG=ON
-        cmake --build build --target clean
         cmake --build build --verbose
 
   bld_gnu_mpich:
@@ -145,7 +145,7 @@ jobs:
         source /opt/intel/oneapi/setvars.sh
         export FC=mpiifort
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON
-        cmake --build build
+        cmake --build build --verbose
 
     # Runs a set of commands using the runners shell
     - name: Compile
@@ -153,9 +153,9 @@ jobs:
         source /opt/intel/oneapi/setvars.sh
         export FC=mpiifort
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
+        rm -rf ./build
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=mkl -DBUILD_TESTING=ON
-        cmake --build build --target clean
-        cmake --build build
+        cmake --build build --verbose
 
   bld_fftw3:
     # The type of runner that the job will run on
@@ -178,11 +178,11 @@ jobs:
         export FC=mpif90
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=fftw -DBUILD_TESTING=ON
         cmake --build build --verbose
-        cmake --build build --target clean
 
     - name: New FFTW3
       run: |
         export FC=mpif90
+        rm -rf ./build
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=fftw_f03 -DBUILD_TESTING=ON
         cmake --build build --verbose
 

From 9d3b7f99b62d37eadf4704944b91d1bcbf74769a Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Thu, 11 May 2023 10:29:39 +0200
Subject: [PATCH 239/436] Add a list of contributors

---
 CONTRIBUTORS | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 CONTRIBUTORS

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
new file mode 100644
index 00000000..1135ecdc
--- /dev/null
+++ b/CONTRIBUTORS
@@ -0,0 +1,10 @@
+Alphabetic order:
+
+Paul Bartholomew (EPCC, UK)
+Cedric Flageul (Poitiers, France)
+Sylvain Laizet (ICL, UK)
+Thibault Lestang (ICL, UK)
+Ning Li (NAG, UK)
+Charles Moulinec (STFC, UK)
+Stefano Rolfo (STFC, UK)
+Filippo Spiga (NVIDIA, UK)

From cc8b192ba1e252d05a19543c4a2211dbadb3058e Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Fri, 12 May 2023 07:10:10 +0200
Subject: [PATCH 240/436] Update D2D_MPI.cmake

Specify the Fortran component is required
---
 cmake/D2D_MPI.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/D2D_MPI.cmake b/cmake/D2D_MPI.cmake
index 2d745355..3b675cc3 100644
--- a/cmake/D2D_MPI.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -1,6 +1,6 @@
 # MPI CMakeLists
 
-find_package(MPI REQUIRED)
+find_package(MPI REQUIRED COMPONENTS Fortran)
 
 # adios2 IO backend requires C and C++ MPI components
 if (IO_BACKEND MATCHES "adios2")

From e85638fca69eda8e370a8b563230716ed02574ee Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Fri, 12 May 2023 07:36:16 +0200
Subject: [PATCH 241/436] Update D2D_Compilers.cmake

Error message when CMake variable IO_BACKEND is not valid
---
 cmake/D2D_Compilers.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmake/D2D_Compilers.cmake b/cmake/D2D_Compilers.cmake
index 731a073f..42a68c30 100644
--- a/cmake/D2D_Compilers.cmake
+++ b/cmake/D2D_Compilers.cmake
@@ -109,4 +109,6 @@ elseif (IO_BACKEND MATCHES "adios2")
   if (NOT ADIOS2_HAVE_Fortran)
     message(FATAL_ERROR "Fortran support is missing in the provided ADIOS2 build")
   endif (NOT ADIOS2_HAVE_Fortran)
+else (IO_BACKEND MATCHES "mpi")
+  message(FATAL_ERROR "Invalid value for CMake variable IO_BACKEND")
 endif (IO_BACKEND MATCHES "mpi")

From 792fd4c415d263e814d489ee9b97823b588217c2 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 23 May 2023 13:41:40 +0100
Subject: [PATCH 242/436] Some minor text changes to Contribute.md

---
 Contribute.md | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/Contribute.md b/Contribute.md
index cd887052..38b2327f 100644
--- a/Contribute.md
+++ b/Contribute.md
@@ -1,12 +1,14 @@
+# Contributing
+
 1. You want to contribute but have no idea how ? Please refer to the [Get started](#get-started) section.
 2. You have identified a bug ? Please refer to the [Bug](#bug) section.
 3. You have improved a part of the code or have developed a new functionality ? Please refer to the [advanced](#advanced-contribution) section.
 
 ## Get started
 
-The recommended strategy to contribute is to start with a [discussion](https://github.com/2decomp-fft/2decomp-fft/discussions) or to pick a existing issue.
+The recommended strategy to contribute is to start with a [discussion](https://github.com/2decomp-fft/2decomp-fft/discussions) or to pick an existing issue.
 To modify or experiment with the code, fork the 2decomp github repository and commit changes in a dedicated branch of your fork.
-When the modification is ready for review, one can open a pull request as exposed in the [advanced](#advanced-contribution) section below.
+When the modification is ready for review, one can open a pull request as described in the [advanced](#advanced-contribution) section below.
 
 ## Bug
 
@@ -16,21 +18,21 @@ Otherwise, follow the steps below.
 
 Firstly, try to reproduce the error with a debug build of the library, a small problem size and a small number of MPI ranks.
 It makes bug-hunting much easier.
-Unfortunately, it is not always possible.
+Unfortunately, this is not always possible.
 Please note that for a debug build, the log contains all the environment variables.
 Use it to hunt the bug but think twice before sharing it as it can expose sensitive and personal information.
 At least, please try to reproduce the bug on another machine with another compiler.
 
 Secondly, if you have modified the source code of the 2decomp library, you must reproduce the bug without the modifications in 2decomp.
-The development team will only provide support for sections of code available in the present repository.
+The development team can only provide support for sections of code available in the present repository.
 
 Thirdly, you must provide a minimal working example.
 The program using 2decomp and exposing the bug should be relatively small.
 The development team will not provide support if the program exposing the bug is very long.
-The programs available in the examples section are a good starting point for a minimal working example.
+The programs available in the examples section are a good starting point for a minimal working example, ideally you could contribute the minimal working example to the existing examples.
 
 At this stage, you probably did your best to simplify the problem at hand.
-Open a issue and select the bug report template.
+Open an issue and select the bug report template.
 Provide a meaningful title, do your best to complete all the sections of the template and provide the version of the compiler, the version of the MPI / FFT library, ...
 If you think you have a fix for the bug, please expose it inside the issue.
 It is recommended to wait for feedback before opening a pull request.
@@ -38,9 +40,10 @@ It is recommended to wait for feedback before opening a pull request.
 ## Advanced contribution
 
 One should read this section before opening a pull request.
-To fix a bug, please open a issue and use the bug report template first.
-To improve a part of the code or develop a new functionality, please open a issue and use the feature request template first.
+To fix a bug, please open an issue and use the bug report template first.
+To improve a part of the code or develop a new functionality, please open an issue and use the feature request template first.
 If you are not sure about your contribution, open a [discussion](https://github.com/2decomp-fft/2decomp-fft/discussions) first.
+This helps raise awareness of the work and prevents repeated effort from occurring.
 
 The code in a pull-request should be formatted using the `fprettify` program.
 See the code sample in the `scripts` folder.
@@ -51,4 +54,4 @@ Please empathise with your fellow contributors who are going to spend time revie
 
 As long as the pull request is open for discussion and not ready for merging, convert it to draft.
 Whenever it is ready for merging, convert it to a regular pull request.
-Please note that reviewers might push modifications directly to your branch.
+Please note that reviewers might push modifications directly to your branch or request changes.

From 11af245ebf9e69b298d3d42cc1e5cc6801e03142 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 25 May 2023 15:58:22 +0100
Subject: [PATCH 243/436] Bug correction for CMake variable controlling CUDA
 architecture for GPU

---
 cmake/D2D_GPU.cmake | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/cmake/D2D_GPU.cmake b/cmake/D2D_GPU.cmake
index cd535696..e173a8c2 100644
--- a/cmake/D2D_GPU.cmake
+++ b/cmake/D2D_GPU.cmake
@@ -14,7 +14,7 @@ if (ENABLE_CUDA)
   find_package(CUDAToolkit REQUIRED)
   if (NOT SET_CUDA_ARCH)
     set(SET_CUDA_ARCH 1 CACHE INTERNAL "Set CUDA Architecture" FORCE)
-    set(CUDA_ARCH_COMP 70 CACHE STRING "Set CUDA Computing Architecture")
+    set(CUDA_ARCH_TEST 70 )
     if(${CMAKE_VERSION} VERSION_LESS_EQUAL "3.13.4")
       cuda_select_nvcc_arch_flags(ARCH_FLAGS "Auto") # optional argument for arch to add
       message(STATUS "ARCH_FLAGS = ${ARCH_FLAGS}")
@@ -30,12 +30,13 @@ if (ENABLE_CUDA)
       SET(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_LIST})
       set_property(GLOBAL PROPERTY CUDA_ARCHITECTURES "${CUDA_ARCH_LIST}")
       message(STATUS "CUDA_ARCHITECTURES ${CUDA_ARCH_LIST}")
-      list(GET CUDA_ARCH_LIST 0 CUDA_ARCH_FIRST)
-      message(STATUS "CUDA_ARCH_FIRST ${CUDA_ARCH_FIRST}")
+      list(GET CUDA_ARCH_LIST 0 CUDA_ARCH_AUTO)
+      message(STATUS "CUDA_ARCH_AUTO ${CUDA_ARCH_AUTO}")
     endif()
-    message(STATUS "CUDA_LIBRARIES ${CUDA_LIBRARIES}")
-    if(${CUDA_ARCH_FIRST} GREATER ${CUDA_ARCH_COMP})
-      set(CUDA_ARCH_COMP ${CUDA_ARCH_FIRST} CACHE)
+    if(${CUDA_ARCH_AUTO} GREATER ${CUDA_ARCH_TEST})
+      set(CUDA_ARCH_COMP ${CUDA_ARCH_AUTO} CACHE STRING "Set CUDA Computing Architecture")
+    else()
+      set(CUDA_ARCH_COMP ${CUDA_ARCH_TEST} CACHE STRING "Set CUDA Computing Architecture")
     endif()
   else()
     set(CUDA_ARCH_COMP ${SET_CUDA_ARCH})

From bc3a45aeef8df3dce9ada1c6bbcbfb842070ee55 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 27 May 2023 12:39:34 +0200
Subject: [PATCH 244/436] Move the transpose routines in dedicated submodules

---
 src/CMakeLists.txt       |  6 ++-
 src/decomp_2d.f90        | 96 ++++++++++++++++++++++++++++++----------
 src/transpose_x_to_y.f90 | 15 +++++--
 src/transpose_y_to_x.f90 | 15 +++++--
 src/transpose_y_to_z.f90 | 15 +++++--
 src/transpose_z_to_y.f90 | 15 +++++--
 6 files changed, 121 insertions(+), 41 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index e8a84b59..d39cd212 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -3,7 +3,11 @@ file(GLOB files_decomp factor.f90
 		       decomp_2d_mpi.f90
 	               decomp_2d.f90
 		       log.f90
-		       io.f90)
+		       io.f90
+		       transpose_x_to_y.f90
+		       transpose_y_to_x.f90
+		       transpose_y_to_z.f90
+		       transpose_z_to_y.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
 if(${BUILD_TARGET} MATCHES "gpu")
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 7bf849f7..1ccd7a44 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -193,31 +193,87 @@ module decomp_2d
    end interface decomp_2d_finalize
 
    interface transpose_x_to_y
-      module procedure transpose_x_to_y_real_long
-      module procedure transpose_x_to_y_real_short
-      module procedure transpose_x_to_y_complex_long
-      module procedure transpose_x_to_y_complex_short
+      module subroutine transpose_x_to_y_real_long(src, dst, decomp)
+         real(mytype), dimension(:, :, :), intent(IN) :: src
+         real(mytype), dimension(:, :, :), intent(OUT) :: dst
+         TYPE(DECOMP_INFO), intent(IN) :: decomp
+      end subroutine transpose_x_to_y_real_long
+      module subroutine transpose_x_to_y_real_short(src, dst)
+         real(mytype), dimension(:, :, :), intent(IN) :: src
+         real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      end subroutine transpose_x_to_y_real_short
+      module subroutine transpose_x_to_y_complex_long(src, dst, decomp)
+         complex(mytype), dimension(:, :, :), intent(IN) :: src
+         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+         TYPE(DECOMP_INFO), intent(IN) :: decomp
+      end subroutine transpose_x_to_y_complex_long
+      module subroutine transpose_x_to_y_complex_short(src, dst)
+         complex(mytype), dimension(:, :, :), intent(IN) :: src
+         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      end subroutine transpose_x_to_y_complex_short
    end interface transpose_x_to_y
 
    interface transpose_y_to_z
-      module procedure transpose_y_to_z_real_long
-      module procedure transpose_y_to_z_real_short
-      module procedure transpose_y_to_z_complex_long
-      module procedure transpose_y_to_z_complex_short
+      module subroutine transpose_y_to_z_real_long(src, dst, decomp)
+         real(mytype), dimension(:, :, :), intent(IN) :: src
+         real(mytype), dimension(:, :, :), intent(OUT) :: dst
+         TYPE(DECOMP_INFO), intent(IN) :: decomp
+      end subroutine transpose_y_to_z_real_long
+      module subroutine transpose_y_to_z_real_short(src, dst)
+         real(mytype), dimension(:, :, :), intent(IN) :: src
+         real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      end subroutine transpose_y_to_z_real_short
+      module subroutine transpose_y_to_z_complex_long(src, dst, decomp)
+         complex(mytype), dimension(:, :, :), intent(IN) :: src
+         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+         TYPE(DECOMP_INFO), intent(IN) :: decomp
+      end subroutine transpose_y_to_z_complex_long
+      module subroutine transpose_y_to_z_complex_short(src, dst)
+         complex(mytype), dimension(:, :, :), intent(IN) :: src
+         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      end subroutine transpose_y_to_z_complex_short
    end interface transpose_y_to_z
 
    interface transpose_z_to_y
-      module procedure transpose_z_to_y_real_long
-      module procedure transpose_z_to_y_real_short
-      module procedure transpose_z_to_y_complex_long
-      module procedure transpose_z_to_y_complex_short
+      module subroutine transpose_z_to_y_real_long(src, dst, decomp)
+         real(mytype), dimension(:, :, :), intent(IN) :: src
+         real(mytype), dimension(:, :, :), intent(OUT) :: dst
+         TYPE(DECOMP_INFO), intent(IN) :: decomp
+      end subroutine transpose_z_to_y_real_long
+      module subroutine transpose_z_to_y_real_short(src, dst)
+         real(mytype), dimension(:, :, :), intent(IN) :: src
+         real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      end subroutine transpose_z_to_y_real_short
+      module subroutine transpose_z_to_y_complex_long(src, dst, decomp)
+         complex(mytype), dimension(:, :, :), intent(IN) :: src
+         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+         TYPE(DECOMP_INFO), intent(IN) :: decomp
+      end subroutine transpose_z_to_y_complex_long
+      module subroutine transpose_z_to_y_complex_short(src, dst)
+         complex(mytype), dimension(:, :, :), intent(IN) :: src
+         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      end subroutine transpose_z_to_y_complex_short
    end interface transpose_z_to_y
 
    interface transpose_y_to_x
-      module procedure transpose_y_to_x_real_long
-      module procedure transpose_y_to_x_real_short
-      module procedure transpose_y_to_x_complex_long
-      module procedure transpose_y_to_x_complex_short
+      module subroutine transpose_y_to_x_real_long(src, dst, decomp)
+         real(mytype), dimension(:, :, :), intent(IN) :: src
+         real(mytype), dimension(:, :, :), intent(OUT) :: dst
+         TYPE(DECOMP_INFO), intent(IN) :: decomp
+      end subroutine transpose_y_to_x_real_long
+      module subroutine transpose_y_to_x_real_short(src, dst)
+         real(mytype), dimension(:, :, :), intent(IN) :: src
+         real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      end subroutine transpose_y_to_x_real_short
+      module subroutine transpose_y_to_x_complex_long(src, dst, decomp)
+         complex(mytype), dimension(:, :, :), intent(IN) :: src
+         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+         TYPE(DECOMP_INFO), intent(IN) :: decomp
+      end subroutine transpose_y_to_x_complex_long
+      module subroutine transpose_y_to_x_complex_short(src, dst)
+         complex(mytype), dimension(:, :, :), intent(IN) :: src
+         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      end subroutine transpose_y_to_x_complex_short
    end interface transpose_y_to_x
 
    interface update_halo
@@ -1129,14 +1185,6 @@ subroutine prepare_buffer(decomp)
 
    end subroutine prepare_buffer
 
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-   ! Transposition routines
-  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-#include "transpose_x_to_y.f90"
-#include "transpose_y_to_z.f90"
-#include "transpose_z_to_y.f90"
-#include "transpose_y_to_x.f90"
-
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Halo cell support
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index ea8f4a74..871f485d 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -10,8 +10,13 @@
 !=======================================================================
 
 ! This file contains the routines that transpose data from X to Y pencil
+submodule(decomp_2d) d2d_transpose_x_to_y
 
-  subroutine transpose_x_to_y_real_short(src, dst)
+  implicit none
+
+contains
+
+  module subroutine transpose_x_to_y_real_short(src, dst)
 
      implicit none
 
@@ -22,7 +27,7 @@ subroutine transpose_x_to_y_real_short(src, dst)
 
   end subroutine transpose_x_to_y_real_short
 
-  subroutine transpose_x_to_y_real_long(src, dst, decomp)
+  module subroutine transpose_x_to_y_real_long(src, dst, decomp)
 
      implicit none
 
@@ -129,7 +134,7 @@ subroutine transpose_x_to_y_real(src, dst, decomp)
      return
   end subroutine transpose_x_to_y_real
 
-  subroutine transpose_x_to_y_complex_short(src, dst)
+  module subroutine transpose_x_to_y_complex_short(src, dst)
 
      implicit none
 
@@ -140,7 +145,7 @@ subroutine transpose_x_to_y_complex_short(src, dst)
 
   end subroutine transpose_x_to_y_complex_short
 
-  subroutine transpose_x_to_y_complex_long(src, dst, decomp)
+  module subroutine transpose_x_to_y_complex_long(src, dst, decomp)
 
      implicit none
 
@@ -454,3 +459,5 @@ subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
      return
   end subroutine mem_merge_xy_complex
+
+end submodule d2d_transpose_x_to_y
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index 2b1a1efd..1bd9f8ac 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -10,8 +10,13 @@
 !=======================================================================
 
 ! This file contains the routines that transpose data from Y to X pencil
+submodule(decomp_2d) d2d_transpose_y_to_x
 
-  subroutine transpose_y_to_x_real_short(src, dst)
+  implicit none
+
+contains
+
+  module subroutine transpose_y_to_x_real_short(src, dst)
 
      implicit none
 
@@ -22,7 +27,7 @@ subroutine transpose_y_to_x_real_short(src, dst)
 
   end subroutine transpose_y_to_x_real_short
 
-  subroutine transpose_y_to_x_real_long(src, dst, decomp)
+  module subroutine transpose_y_to_x_real_long(src, dst, decomp)
 
      implicit none
 
@@ -129,7 +134,7 @@ subroutine transpose_y_to_x_real(src, dst, decomp)
      return
   end subroutine transpose_y_to_x_real
 
-  subroutine transpose_y_to_x_complex_short(src, dst)
+  module subroutine transpose_y_to_x_complex_short(src, dst)
 
      implicit none
 
@@ -140,7 +145,7 @@ subroutine transpose_y_to_x_complex_short(src, dst)
 
   end subroutine transpose_y_to_x_complex_short
 
-  subroutine transpose_y_to_x_complex_long(src, dst, decomp)
+  module subroutine transpose_y_to_x_complex_long(src, dst, decomp)
 
      implicit none
 
@@ -454,3 +459,5 @@ subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
      return
   end subroutine mem_merge_yx_complex
+
+end submodule d2d_transpose_y_to_x
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index cca04207..6ecc4f55 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -10,8 +10,13 @@
 !=======================================================================
 
 ! This file contains the routines that transpose data from Y to Z pencil
+submodule(decomp_2d) d2d_transpose_y_to_z
 
-  subroutine transpose_y_to_z_real_short(src, dst)
+  implicit none
+
+contains
+
+  module subroutine transpose_y_to_z_real_short(src, dst)
 
      implicit none
 
@@ -22,7 +27,7 @@ subroutine transpose_y_to_z_real_short(src, dst)
 
   end subroutine transpose_y_to_z_real_short
 
-  subroutine transpose_y_to_z_real_long(src, dst, decomp)
+  module subroutine transpose_y_to_z_real_long(src, dst, decomp)
 
      implicit none
 
@@ -150,7 +155,7 @@ subroutine transpose_y_to_z_real(src, dst, decomp)
      return
   end subroutine transpose_y_to_z_real
 
-  subroutine transpose_y_to_z_complex_short(src, dst)
+  module subroutine transpose_y_to_z_complex_short(src, dst)
 
      implicit none
 
@@ -161,7 +166,7 @@ subroutine transpose_y_to_z_complex_short(src, dst)
 
   end subroutine transpose_y_to_z_complex_short
 
-  subroutine transpose_y_to_z_complex_long(src, dst, decomp)
+  module subroutine transpose_y_to_z_complex_long(src, dst, decomp)
 
      implicit none
 
@@ -473,3 +478,5 @@ subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
      return
   end subroutine mem_merge_yz_complex
+
+end submodule d2d_transpose_y_to_z
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index 35d6714e..b1314d54 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -10,8 +10,13 @@
 !=======================================================================
 
 ! This file contains the routines that transpose data from Z to Y pencil
+submodule(decomp_2d) d2d_transpose_z_to_y
 
-  subroutine transpose_z_to_y_real_short(src, dst)
+  implicit none
+
+contains
+
+  module subroutine transpose_z_to_y_real_short(src, dst)
 
      implicit none
 
@@ -22,7 +27,7 @@ subroutine transpose_z_to_y_real_short(src, dst)
 
   end subroutine transpose_z_to_y_real_short
 
-  subroutine transpose_z_to_y_real_long(src, dst, decomp)
+  module subroutine transpose_z_to_y_real_long(src, dst, decomp)
 
      implicit none
 
@@ -147,7 +152,7 @@ subroutine transpose_z_to_y_real(src, dst, decomp)
      return
   end subroutine transpose_z_to_y_real
 
-  subroutine transpose_z_to_y_complex_short(src, dst)
+  module subroutine transpose_z_to_y_complex_short(src, dst)
 
      implicit none
 
@@ -158,7 +163,7 @@ subroutine transpose_z_to_y_complex_short(src, dst)
 
   end subroutine transpose_z_to_y_complex_short
 
-  subroutine transpose_z_to_y_complex_long(src, dst, decomp)
+  module subroutine transpose_z_to_y_complex_long(src, dst, decomp)
 
      implicit none
 
@@ -473,3 +478,5 @@ subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
      return
   end subroutine mem_merge_zy_complex
+
+end submodule d2d_transpose_z_to_y

From 4eac5d5279df1e84182ade8aad86bc0b954cbfd8 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 27 May 2023 12:40:30 +0200
Subject: [PATCH 245/436] fprettify

---
 src/transpose_x_to_y.f90 | 576 ++++++++++++++++++------------------
 src/transpose_y_to_x.f90 | 576 ++++++++++++++++++------------------
 src/transpose_y_to_z.f90 | 618 +++++++++++++++++++-------------------
 src/transpose_z_to_y.f90 | 622 +++++++++++++++++++--------------------
 4 files changed, 1196 insertions(+), 1196 deletions(-)

diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index 871f485d..cf368d65 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -12,452 +12,452 @@
 ! This file contains the routines that transpose data from X to Y pencil
 submodule(decomp_2d) d2d_transpose_x_to_y
 
-  implicit none
+   implicit none
 
 contains
 
-  module subroutine transpose_x_to_y_real_short(src, dst)
+   module subroutine transpose_x_to_y_real_short(src, dst)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-     call transpose_x_to_y(src, dst, decomp_main)
+      call transpose_x_to_y(src, dst, decomp_main)
 
-  end subroutine transpose_x_to_y_real_short
+   end subroutine transpose_x_to_y_real_short
 
-  module subroutine transpose_x_to_y_real_long(src, dst, decomp)
+   module subroutine transpose_x_to_y_real_long(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     integer :: istat, nsize
+      integer :: istat, nsize
 #endif
 
-     if (dims(1) == 1) then
+      if (dims(1) == 1) then
 #if defined(_GPU)
-        nsize = product(decomp%xsz)
-        !$acc host_data use_device(src,dst)
-        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+         nsize = product(decomp%xsz)
+         !$acc host_data use_device(src,dst)
+         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst = src
+         dst = src
 #endif
-     else
-        call transpose_x_to_y_real(src, dst, decomp)
-     end if
+      else
+         call transpose_x_to_y_real(src, dst, decomp)
+      end if
 
-  end subroutine transpose_x_to_y_real_long
+   end subroutine transpose_x_to_y_real_long
 
-  subroutine transpose_x_to_y_real(src, dst, decomp)
+   subroutine transpose_x_to_y_real(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     Real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      Real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+      integer :: s1, s2, s3, d1, d2, d3
+      integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
+      if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+      s1 = SIZE(src, 1)
+      s2 = SIZE(src, 2)
+      s3 = SIZE(src, 3)
+      d1 = SIZE(dst, 1)
+      d2 = SIZE(dst, 2)
+      d3 = SIZE(dst, 3)
 
-     ! rearrange source array as send buffer
+      ! rearrange source array as send buffer
 #if defined(_GPU)
-     call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
-                            decomp%x1dist, decomp)
+      call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
+                             decomp%x1dist, decomp)
 #else
-     call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
-                            decomp%x1dist, decomp)
+      call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
+                             decomp%x1dist, decomp)
 #endif
 
-     ! define receive buffer
-     ! transpose using MPI_ALLTOALL(V)
+      ! define receive buffer
+      ! transpose using MPI_ALLTOALL(V)
 #ifdef EVEN
-     call MPI_ALLTOALL(work1_r, decomp%x1count, &
-                       real_type, work2_r, decomp%y1count, &
-                       real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+      call MPI_ALLTOALL(work1_r, decomp%x1count, &
+                        real_type, work2_r, decomp%y1count, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_col(work2_r_d, &
-                                       work1_r_d, &
-                                       decomp%x1disp, &
-                                       decomp%x1cnts, &
-                                       decomp%y1disp, &
-                                       decomp%y1cnts, &
-                                       dims(1))
+      call decomp_2d_nccl_send_recv_col(work2_r_d, &
+                                        work1_r_d, &
+                                        decomp%x1disp, &
+                                        decomp%x1cnts, &
+                                        decomp%y1disp, &
+                                        decomp%y1cnts, &
+                                        dims(1))
 #else
-     call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
-                        real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
+                         real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
+                         real_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
-                        real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
+                         real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
+                         real_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-     ! rearrange receive buffer
+      ! rearrange receive buffer
 #if defined(_GPU)
-     call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-                            decomp%y1dist, decomp)
+      call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+                             decomp%y1dist, decomp)
 #else
-     call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
-                            decomp%y1dist, decomp)
+      call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
+                             decomp%y1dist, decomp)
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
+      if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
 #endif
 
-     return
-  end subroutine transpose_x_to_y_real
+      return
+   end subroutine transpose_x_to_y_real
 
-  module subroutine transpose_x_to_y_complex_short(src, dst)
+   module subroutine transpose_x_to_y_complex_short(src, dst)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-     call transpose_x_to_y(src, dst, decomp_main)
+      call transpose_x_to_y(src, dst, decomp_main)
 
-  end subroutine transpose_x_to_y_complex_short
+   end subroutine transpose_x_to_y_complex_short
 
-  module subroutine transpose_x_to_y_complex_long(src, dst, decomp)
+   module subroutine transpose_x_to_y_complex_long(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     integer :: istat, nsize
+      integer :: istat, nsize
 #endif
-     if (dims(1) == 1) then
+      if (dims(1) == 1) then
 #if defined(_GPU)
-        nsize = product(decomp%xsz)
-        !$acc host_data use_device(src,dst)
-        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+         nsize = product(decomp%xsz)
+         !$acc host_data use_device(src,dst)
+         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst = src
+         dst = src
 #endif
-     else
-        call transpose_x_to_y_complex(src, dst, decomp)
-     end if
+      else
+         call transpose_x_to_y_complex(src, dst, decomp)
+      end if
 
-  end subroutine transpose_x_to_y_complex_long
+   end subroutine transpose_x_to_y_complex_long
 
-  subroutine transpose_x_to_y_complex(src, dst, decomp)
+   subroutine transpose_x_to_y_complex(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+      integer :: s1, s2, s3, d1, d2, d3
+      integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
+      if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+      s1 = SIZE(src, 1)
+      s2 = SIZE(src, 2)
+      s3 = SIZE(src, 3)
+      d1 = SIZE(dst, 1)
+      d2 = SIZE(dst, 2)
+      d3 = SIZE(dst, 3)
 
-     ! rearrange source array as send buffer
+      ! rearrange source array as send buffer
 #if defined(_GPU)
-     call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-                               decomp%x1dist, decomp)
+      call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+                                decomp%x1dist, decomp)
 #else
-     call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
-                               decomp%x1dist, decomp)
+      call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
+                                decomp%x1dist, decomp)
 #endif
 
-     ! define receive buffer
-     ! transpose using MPI_ALLTOALL(V)
+      ! define receive buffer
+      ! transpose using MPI_ALLTOALL(V)
 #ifdef EVEN
-     call MPI_ALLTOALL(work1_c, decomp%x1count, &
-                       complex_type, work2_c, decomp%y1count, &
-                       complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+      call MPI_ALLTOALL(work1_c, decomp%x1count, &
+                        complex_type, work2_c, decomp%y1count, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_col(work2_c_d, &
-                                       work1_c_d, &
-                                       decomp%x1disp, &
-                                       decomp%x1cnts, &
-                                       decomp%y1disp, &
-                                       decomp%y1cnts, &
-                                       dims(1), &
-                                       decomp_buf_size)
+      call decomp_2d_nccl_send_recv_col(work2_c_d, &
+                                        work1_c_d, &
+                                        decomp%x1disp, &
+                                        decomp%x1cnts, &
+                                        decomp%y1disp, &
+                                        decomp%y1cnts, &
+                                        dims(1), &
+                                        decomp_buf_size)
 #else
-     call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
-                        complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
+                         complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
+                         complex_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
-                        complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
+                         complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
+                         complex_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-     ! rearrange receive buffer
+      ! rearrange receive buffer
 #if defined(_GPU)
-     call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-                               decomp%y1dist, decomp)
+      call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+                                decomp%y1dist, decomp)
 #else
-     call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
-                               decomp%y1dist, decomp)
+      call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
+                                decomp%y1dist, decomp)
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
+      if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
 #endif
 
-     return
-  end subroutine transpose_x_to_y_complex
+      return
+   end subroutine transpose_x_to_y_complex
 
-  subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     real(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+      real(mytype), dimension(*), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+      attributes(device) :: out
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%x1count + 1
+         pos = m * decomp%x1count + 1
 #else
-        pos = decomp%x1disp(m) + 1
+         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(in)
+         istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = 1, n2
-              do i = i1, i2
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = 1, n2
+               do i = i1, i2
+                  out(pos) = in(i, j, k)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_split_xy_real
+      return
+   end subroutine mem_split_xy_real
 
-  subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     complex(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+      complex(mytype), dimension(*), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+      attributes(device) :: out
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%x1count + 1
+         pos = m * decomp%x1count + 1
 #else
-        pos = decomp%x1disp(m) + 1
+         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(in)
+         istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = 1, n2
-              do i = i1, i2
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = 1, n2
+               do i = i1, i2
+                  out(pos) = in(i, j, k)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_split_xy_complex
+      return
+   end subroutine mem_split_xy_complex
 
-  subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(*), intent(IN) :: in
-     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      real(mytype), dimension(*), intent(IN) :: in
+      real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+      attributes(device) :: in
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%y1count + 1
+         pos = m * decomp%y1count + 1
 #else
-        pos = decomp%y1disp(m) + 1
+         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(out)
+         istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = i1, i2
+               do i = 1, n1
+                  out(i, j, k) = in(pos)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_merge_xy_real
+      return
+   end subroutine mem_merge_xy_real
 
-  subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(*), intent(IN) :: in
-     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      complex(mytype), dimension(*), intent(IN) :: in
+      complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+      attributes(device) :: in
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%y1count + 1
+         pos = m * decomp%y1count + 1
 #else
-        pos = decomp%y1disp(m) + 1
+         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(out)
+         istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = i1, i2
+               do i = 1, n1
+                  out(i, j, k) = in(pos)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_merge_xy_complex
+      return
+   end subroutine mem_merge_xy_complex
 
 end submodule d2d_transpose_x_to_y
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index 1bd9f8ac..eefb9498 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -12,452 +12,452 @@
 ! This file contains the routines that transpose data from Y to X pencil
 submodule(decomp_2d) d2d_transpose_y_to_x
 
-  implicit none
+   implicit none
 
 contains
 
-  module subroutine transpose_y_to_x_real_short(src, dst)
+   module subroutine transpose_y_to_x_real_short(src, dst)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-     call transpose_y_to_x(src, dst, decomp_main)
+      call transpose_y_to_x(src, dst, decomp_main)
 
-  end subroutine transpose_y_to_x_real_short
+   end subroutine transpose_y_to_x_real_short
 
-  module subroutine transpose_y_to_x_real_long(src, dst, decomp)
+   module subroutine transpose_y_to_x_real_long(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     integer :: istat, nsize
+      integer :: istat, nsize
 #endif
 
-     if (dims(1) == 1) then
+      if (dims(1) == 1) then
 #if defined(_GPU)
-        nsize = product(decomp%ysz)
-        !$acc host_data use_device(src,dst)
-        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+         nsize = product(decomp%ysz)
+         !$acc host_data use_device(src,dst)
+         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst = src
+         dst = src
 #endif
-     else
-        call transpose_y_to_x_real(src, dst, decomp)
-     end if
+      else
+         call transpose_y_to_x_real(src, dst, decomp)
+      end if
 
-  end subroutine transpose_y_to_x_real_long
+   end subroutine transpose_y_to_x_real_long
 
-  subroutine transpose_y_to_x_real(src, dst, decomp)
+   subroutine transpose_y_to_x_real(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+      integer :: s1, s2, s3, d1, d2, d3
+      integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
+      if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+      s1 = SIZE(src, 1)
+      s2 = SIZE(src, 2)
+      s3 = SIZE(src, 3)
+      d1 = SIZE(dst, 1)
+      d2 = SIZE(dst, 2)
+      d3 = SIZE(dst, 3)
 
-     ! rearrange source array as send buffer
+      ! rearrange source array as send buffer
 #if defined(_GPU)
-     call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
-                            decomp%y1dist, decomp)
+      call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
+                             decomp%y1dist, decomp)
 #else
-     call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
-                            decomp%y1dist, decomp)
+      call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
+                             decomp%y1dist, decomp)
 #endif
 
-     ! define receive buffer
-     ! transpose using MPI_ALLTOALL(V)
+      ! define receive buffer
+      ! transpose using MPI_ALLTOALL(V)
 #ifdef EVEN
-     call MPI_ALLTOALL(work1_r, decomp%y1count, &
-                       real_type, work2_r, decomp%x1count, &
-                       real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+      call MPI_ALLTOALL(work1_r, decomp%y1count, &
+                        real_type, work2_r, decomp%x1count, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_col(work2_r_d, &
-                                       work1_r_d, &
-                                       decomp%y1disp, &
-                                       decomp%y1cnts, &
-                                       decomp%x1disp, &
-                                       decomp%x1cnts, &
-                                       dims(1))
+      call decomp_2d_nccl_send_recv_col(work2_r_d, &
+                                        work1_r_d, &
+                                        decomp%y1disp, &
+                                        decomp%y1cnts, &
+                                        decomp%x1disp, &
+                                        decomp%x1cnts, &
+                                        dims(1))
 #else
-     call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
-                        real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
+                         real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
+                         real_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
-                        real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
+                         real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
+                         real_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-     ! rearrange receive buffer
+      ! rearrange receive buffer
 #if defined(_GPU)
-     call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-                            decomp%x1dist, decomp)
+      call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+                             decomp%x1dist, decomp)
 #else
-     call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
-                            decomp%x1dist, decomp)
+      call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
+                             decomp%x1dist, decomp)
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
+      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
 #endif
 
-     return
-  end subroutine transpose_y_to_x_real
+      return
+   end subroutine transpose_y_to_x_real
 
-  module subroutine transpose_y_to_x_complex_short(src, dst)
+   module subroutine transpose_y_to_x_complex_short(src, dst)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-     call transpose_y_to_x(src, dst, decomp_main)
+      call transpose_y_to_x(src, dst, decomp_main)
 
-  end subroutine transpose_y_to_x_complex_short
+   end subroutine transpose_y_to_x_complex_short
 
-  module subroutine transpose_y_to_x_complex_long(src, dst, decomp)
+   module subroutine transpose_y_to_x_complex_long(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     integer :: istat, nsize
+      integer :: istat, nsize
 #endif
-     if (dims(1) == 1) then
+      if (dims(1) == 1) then
 #if defined(_GPU)
-        nsize = product(decomp%ysz)
-        !$acc host_data use_device(src,dst)
-        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+         nsize = product(decomp%ysz)
+         !$acc host_data use_device(src,dst)
+         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst = src
+         dst = src
 #endif
-     else
-        call transpose_y_to_x_complex(src, dst, decomp)
-     end if
+      else
+         call transpose_y_to_x_complex(src, dst, decomp)
+      end if
 
-  end subroutine transpose_y_to_x_complex_long
+   end subroutine transpose_y_to_x_complex_long
 
-  subroutine transpose_y_to_x_complex(src, dst, decomp)
+   subroutine transpose_y_to_x_complex(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+      integer :: s1, s2, s3, d1, d2, d3
+      integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
+      if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+      s1 = SIZE(src, 1)
+      s2 = SIZE(src, 2)
+      s3 = SIZE(src, 3)
+      d1 = SIZE(dst, 1)
+      d2 = SIZE(dst, 2)
+      d3 = SIZE(dst, 3)
 
-     ! rearrange source array as send buffer
+      ! rearrange source array as send buffer
 #if defined(_GPU)
-     call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-                               decomp%y1dist, decomp)
+      call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+                                decomp%y1dist, decomp)
 #else
-     call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
-                               decomp%y1dist, decomp)
+      call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
+                                decomp%y1dist, decomp)
 #endif
 
-     ! define receive buffer
-     ! transpose using MPI_ALLTOALL(V)
+      ! define receive buffer
+      ! transpose using MPI_ALLTOALL(V)
 #ifdef EVEN
-     call MPI_ALLTOALL(work1_c, decomp%y1count, &
-                       complex_type, work2_c, decomp%x1count, &
-                       complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+      call MPI_ALLTOALL(work1_c, decomp%y1count, &
+                        complex_type, work2_c, decomp%x1count, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_col(work2_c_d, &
-                                       work1_c_d, &
-                                       decomp%y1disp, &
-                                       decomp%y1cnts, &
-                                       decomp%x1disp, &
-                                       decomp%x1cnts, &
-                                       dims(1), &
-                                       decomp_buf_size)
+      call decomp_2d_nccl_send_recv_col(work2_c_d, &
+                                        work1_c_d, &
+                                        decomp%y1disp, &
+                                        decomp%y1cnts, &
+                                        decomp%x1disp, &
+                                        decomp%x1cnts, &
+                                        dims(1), &
+                                        decomp_buf_size)
 #else
-     call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
-                        complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
+                         complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
+                         complex_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
-                        complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
+                         complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
+                         complex_type, DECOMP_2D_COMM_COL, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-     ! rearrange receive buffer
+      ! rearrange receive buffer
 #if defined(_GPU)
-     call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-                               decomp%x1dist, decomp)
+      call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+                                decomp%x1dist, decomp)
 #else
-     call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
-                               decomp%x1dist, decomp)
+      call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
+                                decomp%x1dist, decomp)
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
+      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
 #endif
 
-     return
-  end subroutine transpose_y_to_x_complex
+      return
+   end subroutine transpose_y_to_x_complex
 
-  subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     real(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+      real(mytype), dimension(*), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+      attributes(device) :: out
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%y1count + 1
+         pos = m * decomp%y1count + 1
 #else
-        pos = decomp%y1disp(m) + 1
+         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(in)
+         istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = i1, i2
+               do i = 1, n1
+                  out(pos) = in(i, j, k)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_split_yx_real
+      return
+   end subroutine mem_split_yx_real
 
-  subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     complex(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+      complex(mytype), dimension(*), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+      attributes(device) :: out
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%y1count + 1
+         pos = m * decomp%y1count + 1
 #else
-        pos = decomp%y1disp(m) + 1
+         pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(in)
+         istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = i1, i2
+               do i = 1, n1
+                  out(pos) = in(i, j, k)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_split_yx_complex
+      return
+   end subroutine mem_split_yx_complex
 
-  subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(*), intent(IN) :: in
-     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      real(mytype), dimension(*), intent(IN) :: in
+      real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+      attributes(device) :: in
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%x1count + 1
+         pos = m * decomp%x1count + 1
 #else
-        pos = decomp%x1disp(m) + 1
+         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(out)
+         istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = 1, n2
-              do i = i1, i2
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = 1, n2
+               do i = i1, i2
+                  out(i, j, k) = in(pos)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_merge_yx_real
+      return
+   end subroutine mem_merge_yx_real
 
-  subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(*), intent(IN) :: in
-     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      complex(mytype), dimension(*), intent(IN) :: in
+      complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+      attributes(device) :: in
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%x1count + 1
+         pos = m * decomp%x1count + 1
 #else
-        pos = decomp%x1disp(m) + 1
+         pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(out)
+         istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = 1, n2
-              do i = i1, i2
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = 1, n2
+               do i = i1, i2
+                  out(i, j, k) = in(pos)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_merge_yx_complex
+      return
+   end subroutine mem_merge_yx_complex
 
 end submodule d2d_transpose_y_to_x
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 6ecc4f55..19f244ad 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -12,471 +12,471 @@
 ! This file contains the routines that transpose data from Y to Z pencil
 submodule(decomp_2d) d2d_transpose_y_to_z
 
-  implicit none
+   implicit none
 
 contains
 
-  module subroutine transpose_y_to_z_real_short(src, dst)
+   module subroutine transpose_y_to_z_real_short(src, dst)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-     call transpose_y_to_z(src, dst, decomp_main)
+      call transpose_y_to_z(src, dst, decomp_main)
 
-  end subroutine transpose_y_to_z_real_short
+   end subroutine transpose_y_to_z_real_short
 
-  module subroutine transpose_y_to_z_real_long(src, dst, decomp)
+   module subroutine transpose_y_to_z_real_long(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     integer :: istat, nsize
+      integer :: istat, nsize
 #endif
 
-     if (dims(2) == 1) then
+      if (dims(2) == 1) then
 #if defined(_GPU)
-        nsize = product(decomp%ysz)
-        !$acc host_data use_device(src,dst)
-        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+         nsize = product(decomp%ysz)
+         !$acc host_data use_device(src,dst)
+         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst = src
+         dst = src
 #endif
-     else
-        call transpose_y_to_z_real(src, dst, decomp)
-     end if
+      else
+         call transpose_y_to_z_real(src, dst, decomp)
+      end if
 
-  end subroutine transpose_y_to_z_real_long
+   end subroutine transpose_y_to_z_real_long
 
-  subroutine transpose_y_to_z_real(src, dst, decomp)
+   subroutine transpose_y_to_z_real(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-     integer :: istat
+      integer :: istat
 #endif
 
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+      integer :: s1, s2, s3, d1, d2, d3
+      integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
+      if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+      s1 = SIZE(src, 1)
+      s2 = SIZE(src, 2)
+      s3 = SIZE(src, 3)
+      d1 = SIZE(dst, 1)
+      d2 = SIZE(dst, 2)
+      d3 = SIZE(dst, 3)
 
-     ! rearrange source array as send buffer
+      ! rearrange source array as send buffer
 #if defined(_GPU)
-     call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
-                            decomp%y2dist, decomp)
+      call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
+                             decomp%y2dist, decomp)
 #else
-     call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
-                            decomp%y2dist, decomp)
+      call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
+                             decomp%y2dist, decomp)
 #endif
 
-     ! define receive buffer
+      ! define receive buffer
 #ifdef EVEN
-     if (decomp%even) then
-        call MPI_ALLTOALL(work1_r, decomp%y2count, &
-                          real_type, dst, decomp%z2count, &
-                          real_type, DECOMP_2D_COMM_ROW, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-     else
-        call MPI_ALLTOALL(work1_r, decomp%y2count, &
-                          real_type, work2_r, decomp%z2count, &
-                          real_type, DECOMP_2D_COMM_ROW, ierror)
-        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
-     end if
+      if (decomp%even) then
+         call MPI_ALLTOALL(work1_r, decomp%y2count, &
+                           real_type, dst, decomp%z2count, &
+                           real_type, DECOMP_2D_COMM_ROW, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      else
+         call MPI_ALLTOALL(work1_r, decomp%y2count, &
+                           real_type, work2_r, decomp%z2count, &
+                           real_type, DECOMP_2D_COMM_ROW, ierror)
+         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+      end if
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_row(work2_r_d, &
-                                       work1_r_d, &
-                                       decomp%y2disp, &
-                                       decomp%y2cnts, &
-                                       decomp%z2disp, &
-                                       decomp%z2cnts, &
-                                       dims(2))
+      call decomp_2d_nccl_send_recv_row(work2_r_d, &
+                                        work1_r_d, &
+                                        decomp%y2disp, &
+                                        decomp%y2cnts, &
+                                        decomp%z2disp, &
+                                        decomp%z2cnts, &
+                                        dims(2))
 #else
-     call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
-                        real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
-                        real_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
+                         real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
+                         real_type, DECOMP_2D_COMM_ROW, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
-                        real_type, dst, decomp%z2cnts, decomp%z2disp, &
-                        real_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
+                         real_type, dst, decomp%z2cnts, decomp%z2disp, &
+                         real_type, DECOMP_2D_COMM_ROW, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-     ! rearrange receive buffer
+      ! rearrange receive buffer
 #ifdef EVEN
-     if (.not. decomp%even) then
-        call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
-                               decomp%z2dist, decomp)
-     end if
+      if (.not. decomp%even) then
+         call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
+                                decomp%z2dist, decomp)
+      end if
 #else
-     ! note the receive buffer is already in natural (i,j,k) order
-     ! so no merge operation needed
+      ! note the receive buffer is already in natural (i,j,k) order
+      ! so no merge operation needed
 
 #if defined(_GPU)
-     !If one of the array in cuda call is not device we need to add acc host_data
-     !$acc host_data use_device(dst)
-     istat = cudaMemcpy(dst, work2_r_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
-     !$acc end host_data
-     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+      !If one of the array in cuda call is not device we need to add acc host_data
+      !$acc host_data use_device(dst)
+      istat = cudaMemcpy(dst, work2_r_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
+      !$acc end host_data
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
+      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
 #endif
 
-     return
-  end subroutine transpose_y_to_z_real
+      return
+   end subroutine transpose_y_to_z_real
 
-  module subroutine transpose_y_to_z_complex_short(src, dst)
+   module subroutine transpose_y_to_z_complex_short(src, dst)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-     call transpose_y_to_z(src, dst, decomp_main)
+      call transpose_y_to_z(src, dst, decomp_main)
 
-  end subroutine transpose_y_to_z_complex_short
+   end subroutine transpose_y_to_z_complex_short
 
-  module subroutine transpose_y_to_z_complex_long(src, dst, decomp)
+   module subroutine transpose_y_to_z_complex_long(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     integer :: istat, nsize
+      integer :: istat, nsize
 #endif
 
-     if (dims(2) == 1) then
+      if (dims(2) == 1) then
 #if defined(_GPU)
-        nsize = product(decomp%ysz)
-        !$acc host_data use_device(src,dst)
-        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+         nsize = product(decomp%ysz)
+         !$acc host_data use_device(src,dst)
+         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst = src
+         dst = src
 #endif
-     else
-        call transpose_y_to_z_complex(src, dst, decomp)
-     end if
+      else
+         call transpose_y_to_z_complex(src, dst, decomp)
+      end if
 
-  end subroutine transpose_y_to_z_complex_long
+   end subroutine transpose_y_to_z_complex_long
 
-  subroutine transpose_y_to_z_complex(src, dst, decomp)
+   subroutine transpose_y_to_z_complex(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-     integer :: istat
+      integer :: istat
 #endif
 
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+      integer :: s1, s2, s3, d1, d2, d3
+      integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
+      if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+      s1 = SIZE(src, 1)
+      s2 = SIZE(src, 2)
+      s3 = SIZE(src, 3)
+      d1 = SIZE(dst, 1)
+      d2 = SIZE(dst, 2)
+      d3 = SIZE(dst, 3)
 
-     ! rearrange source array as send buffer
+      ! rearrange source array as send buffer
 #if defined(_GPU)
-     call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
-                               decomp%y2dist, decomp)
+      call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
+                                decomp%y2dist, decomp)
 #else
-     call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
-                               decomp%y2dist, decomp)
+      call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
+                                decomp%y2dist, decomp)
 #endif
 
-     ! define receive buffer
+      ! define receive buffer
 #ifdef EVEN
-     if (decomp%even) then
-        call MPI_ALLTOALL(work1_c, decomp%y2count, &
-                          complex_type, dst, decomp%z2count, &
-                          complex_type, DECOMP_2D_COMM_ROW, ierror)
-     else
-        call MPI_ALLTOALL(work1_c, decomp%y2count, &
-                          complex_type, work2_c, decomp%z2count, &
-                          complex_type, DECOMP_2D_COMM_ROW, ierror)
-     end if
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+      if (decomp%even) then
+         call MPI_ALLTOALL(work1_c, decomp%y2count, &
+                           complex_type, dst, decomp%z2count, &
+                           complex_type, DECOMP_2D_COMM_ROW, ierror)
+      else
+         call MPI_ALLTOALL(work1_c, decomp%y2count, &
+                           complex_type, work2_c, decomp%z2count, &
+                           complex_type, DECOMP_2D_COMM_ROW, ierror)
+      end if
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_row(work2_c_d, &
-                                       work1_c_d, &
-                                       decomp%y2disp, &
-                                       decomp%y2cnts, &
-                                       decomp%z2disp, &
-                                       decomp%z2cnts, &
-                                       dims(2), &
-                                       decomp_buf_size)
+      call decomp_2d_nccl_send_recv_row(work2_c_d, &
+                                        work1_c_d, &
+                                        decomp%y2disp, &
+                                        decomp%y2cnts, &
+                                        decomp%z2disp, &
+                                        decomp%z2cnts, &
+                                        dims(2), &
+                                        decomp_buf_size)
 #else
-     call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
-                        complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
-                        complex_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
+                         complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
+                         complex_type, DECOMP_2D_COMM_ROW, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
-                        complex_type, dst, decomp%z2cnts, decomp%z2disp, &
-                        complex_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
+                         complex_type, dst, decomp%z2cnts, decomp%z2disp, &
+                         complex_type, DECOMP_2D_COMM_ROW, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-     ! rearrange receive buffer
+      ! rearrange receive buffer
 #ifdef EVEN
-     if (.not. decomp%even) then
-        call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
-                                  decomp%z2dist, decomp)
-     end if
+      if (.not. decomp%even) then
+         call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
+                                   decomp%z2dist, decomp)
+      end if
 #else
-     ! note the receive buffer is already in natural (i,j,k) order
-     ! so no merge operation needed
+      ! note the receive buffer is already in natural (i,j,k) order
+      ! so no merge operation needed
 
 #if defined(_GPU)
-     !$acc host_data use_device(dst)
-     istat = cudaMemcpy(dst, work2_c_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
-     !$acc end host_data
-     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+      !$acc host_data use_device(dst)
+      istat = cudaMemcpy(dst, work2_c_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
+      !$acc end host_data
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
+      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
 #endif
 
-     return
-  end subroutine transpose_y_to_z_complex
+      return
+   end subroutine transpose_y_to_z_complex
 
-  subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     real(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+      real(mytype), dimension(*), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+      attributes(device) :: out
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%y2count + 1
+         pos = m * decomp%y2count + 1
 #else
-        pos = decomp%y2disp(m) + 1
+         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(in)
+         istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = i1, i2
+               do i = 1, n1
+                  out(pos) = in(i, j, k)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_split_yz_real
+      return
+   end subroutine mem_split_yz_real
 
-  subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     complex(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+      complex(mytype), dimension(*), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: out
-     integer :: istat
+      attributes(device) :: out
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%y2count + 1
+         pos = m * decomp%y2count + 1
 #else
-        pos = decomp%y2disp(m) + 1
+         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(in)
-        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(in)
+         istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = i1, i2
+               do i = 1, n1
+                  out(pos) = in(i, j, k)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_split_yz_complex
+      return
+   end subroutine mem_split_yz_complex
 
-  subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(*), intent(IN) :: in
-     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      real(mytype), dimension(*), intent(IN) :: in
+      real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%z2count + 1
+         pos = m * decomp%z2count + 1
 #else
-        pos = decomp%z2disp(m) + 1
+         pos = decomp%z2disp(m) + 1
 #endif
 
-        do k = i1, i2
-           do j = 1, n2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-     end do
-
-     return
-  end subroutine mem_merge_yz_real
-
-  subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
-
-     implicit none
-
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(*), intent(IN) :: in
-     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
-
-     integer :: i, j, k, m, i1, i2, pos
-
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+         do k = i1, i2
+            do j = 1, n2
+               do i = 1, n1
+                  out(i, j, k) = in(pos)
+                  pos = pos + 1
+               end do
+            end do
+         end do
+      end do
+
+      return
+   end subroutine mem_merge_yz_real
+
+   subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+
+      implicit none
+
+      integer, intent(IN) :: n1, n2, n3
+      complex(mytype), dimension(*), intent(IN) :: in
+      complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+
+      integer :: i, j, k, m, i1, i2, pos
+
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%z2count + 1
+         pos = m * decomp%z2count + 1
 #else
-        pos = decomp%z2disp(m) + 1
+         pos = decomp%z2disp(m) + 1
 #endif
 
-        do k = i1, i2
-           do j = 1, n2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-     end do
-
-     return
-  end subroutine mem_merge_yz_complex
+         do k = i1, i2
+            do j = 1, n2
+               do i = 1, n1
+                  out(i, j, k) = in(pos)
+                  pos = pos + 1
+               end do
+            end do
+         end do
+      end do
+
+      return
+   end subroutine mem_merge_yz_complex
 
 end submodule d2d_transpose_y_to_z
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index b1314d54..af77904b 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -12,471 +12,471 @@
 ! This file contains the routines that transpose data from Z to Y pencil
 submodule(decomp_2d) d2d_transpose_z_to_y
 
-  implicit none
+   implicit none
 
 contains
 
-  module subroutine transpose_z_to_y_real_short(src, dst)
+   module subroutine transpose_z_to_y_real_short(src, dst)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-     call transpose_z_to_y(src, dst, decomp_main)
+      call transpose_z_to_y(src, dst, decomp_main)
 
-  end subroutine transpose_z_to_y_real_short
+   end subroutine transpose_z_to_y_real_short
 
-  module subroutine transpose_z_to_y_real_long(src, dst, decomp)
+   module subroutine transpose_z_to_y_real_long(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     integer :: istat, nsize
+      integer :: istat, nsize
 #endif
 
-     if (dims(2) == 1) then
+      if (dims(2) == 1) then
 #if defined(_GPU)
-        nsize = product(decomp%zsz)
-        !$acc host_data use_device(src,dst)
-        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+         nsize = product(decomp%zsz)
+         !$acc host_data use_device(src,dst)
+         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst = src
+         dst = src
 #endif
-     else
-        call transpose_z_to_y_real(src, dst, decomp)
-     end if
+      else
+         call transpose_z_to_y_real(src, dst, decomp)
+      end if
 
-  end subroutine transpose_z_to_y_real_long
+   end subroutine transpose_z_to_y_real_long
 
-  subroutine transpose_z_to_y_real(src, dst, decomp)
+   subroutine transpose_z_to_y_real(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     real(mytype), dimension(:, :, :), intent(IN) :: src
-     real(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      real(mytype), dimension(:, :, :), intent(IN) :: src
+      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-     integer :: istat
+      integer :: istat
 #endif
 
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+      integer :: s1, s2, s3, d1, d2, d3
+      integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
+      if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+      s1 = SIZE(src, 1)
+      s2 = SIZE(src, 2)
+      s3 = SIZE(src, 3)
+      d1 = SIZE(dst, 1)
+      d2 = SIZE(dst, 2)
+      d3 = SIZE(dst, 3)
 
-     ! rearrange source array as send buffer
+      ! rearrange source array as send buffer
 #ifdef EVEN
-     if (.not. decomp%even) then
-        call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
-                               decomp%z2dist, decomp)
-     end if
+      if (.not. decomp%even) then
+         call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
+                                decomp%z2dist, decomp)
+      end if
 #else
-     ! note the src array is suitable to be a send buffer
-     ! so no split operation needed
+      ! note the src array is suitable to be a send buffer
+      ! so no split operation needed
 
 #if defined(_GPU)
-     !$acc host_data use_device(src)
-     istat = cudaMemcpy(work1_r_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
-     !$acc end host_data
-     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+      !$acc host_data use_device(src)
+      istat = cudaMemcpy(work1_r_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
+      !$acc end host_data
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
 
-     ! define receive buffer
+      ! define receive buffer
 #ifdef EVEN
-     if (decomp%even) then
-        call MPI_ALLTOALL(src, decomp%z2count, &
-                          real_type, work2_r, decomp%y2count, &
-                          real_type, DECOMP_2D_COMM_ROW, ierror)
-     else
-        call MPI_ALLTOALL(work1_r, decomp%z2count, &
-                          real_type, work2_r, decomp%y2count, &
-                          real_type, DECOMP_2D_COMM_ROW, ierror)
-     end if
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+      if (decomp%even) then
+         call MPI_ALLTOALL(src, decomp%z2count, &
+                           real_type, work2_r, decomp%y2count, &
+                           real_type, DECOMP_2D_COMM_ROW, ierror)
+      else
+         call MPI_ALLTOALL(work1_r, decomp%z2count, &
+                           real_type, work2_r, decomp%y2count, &
+                           real_type, DECOMP_2D_COMM_ROW, ierror)
+      end if
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_row(work2_r_d, &
-                                       work1_r_d, &
-                                       decomp%z2disp, &
-                                       decomp%z2cnts, &
-                                       decomp%y2disp, &
-                                       decomp%y2cnts, &
-                                       dims(2))
+      call decomp_2d_nccl_send_recv_row(work2_r_d, &
+                                        work1_r_d, &
+                                        decomp%z2disp, &
+                                        decomp%z2cnts, &
+                                        decomp%y2disp, &
+                                        decomp%y2cnts, &
+                                        dims(2))
 #else
-     call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
-                        real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
-                        real_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
+                         real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
+                         real_type, DECOMP_2D_COMM_ROW, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-                        real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
-                        real_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+                         real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
+                         real_type, DECOMP_2D_COMM_ROW, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #endif
 
-     ! rearrange receive buffer
+      ! rearrange receive buffer
 #if defined(_GPU)
-     call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
-                            decomp%y2dist, decomp)
+      call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
+                             decomp%y2dist, decomp)
 #else
-     call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
-                            decomp%y2dist, decomp)
+      call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
+                             decomp%y2dist, decomp)
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
+      if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
 #endif
 
-     return
-  end subroutine transpose_z_to_y_real
+      return
+   end subroutine transpose_z_to_y_real
 
-  module subroutine transpose_z_to_y_complex_short(src, dst)
+   module subroutine transpose_z_to_y_complex_short(src, dst)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-     call transpose_z_to_y(src, dst, decomp_main)
+      call transpose_z_to_y(src, dst, decomp_main)
 
-  end subroutine transpose_z_to_y_complex_short
+   end subroutine transpose_z_to_y_complex_short
 
-  module subroutine transpose_z_to_y_complex_long(src, dst, decomp)
+   module subroutine transpose_z_to_y_complex_long(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     integer :: istat, nsize
+      integer :: istat, nsize
 #endif
 
-     if (dims(2) == 1) then
+      if (dims(2) == 1) then
 #if defined(_GPU)
-        nsize = product(decomp%zsz)
-        !$acc host_data use_device(src,dst)
-        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+         nsize = product(decomp%zsz)
+         !$acc host_data use_device(src,dst)
+         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-        dst = src
+         dst = src
 #endif
-     else
-        call transpose_z_to_y_complex(src, dst, decomp)
-     end if
+      else
+         call transpose_z_to_y_complex(src, dst, decomp)
+      end if
 
-  end subroutine transpose_z_to_y_complex_long
+   end subroutine transpose_z_to_y_complex_long
 
-  subroutine transpose_z_to_y_complex(src, dst, decomp)
+   subroutine transpose_z_to_y_complex(src, dst, decomp)
 
-     implicit none
+      implicit none
 
-     complex(mytype), dimension(:, :, :), intent(IN) :: src
-     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      complex(mytype), dimension(:, :, :), intent(IN) :: src
+      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-     integer :: istat
+      integer :: istat
 #endif
 
-     integer :: s1, s2, s3, d1, d2, d3
-     integer :: ierror
+      integer :: s1, s2, s3, d1, d2, d3
+      integer :: ierror
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
+      if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
 #endif
 
-     s1 = SIZE(src, 1)
-     s2 = SIZE(src, 2)
-     s3 = SIZE(src, 3)
-     d1 = SIZE(dst, 1)
-     d2 = SIZE(dst, 2)
-     d3 = SIZE(dst, 3)
+      s1 = SIZE(src, 1)
+      s2 = SIZE(src, 2)
+      s3 = SIZE(src, 3)
+      d1 = SIZE(dst, 1)
+      d2 = SIZE(dst, 2)
+      d3 = SIZE(dst, 3)
 
-     ! rearrange source array as send buffer
+      ! rearrange source array as send buffer
 #ifdef EVEN
-     if (.not. decomp%even) then
-        call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
-                                  decomp%z2dist, decomp)
-     end if
+      if (.not. decomp%even) then
+         call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
+                                   decomp%z2dist, decomp)
+      end if
 #else
-     ! note the src array is suitable to be a send buffer
-     ! so no split operation needed
+      ! note the src array is suitable to be a send buffer
+      ! so no split operation needed
 
 #if defined(_GPU)
-     !$acc host_data use_device(src)
-     istat = cudaMemcpy(work1_c_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
-     !$acc end host_data
-     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+      !$acc host_data use_device(src)
+      istat = cudaMemcpy(work1_c_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
+      !$acc end host_data
+      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
 
-     ! define receive buffer
+      ! define receive buffer
 #ifdef EVEN
-     if (decomp%even) then
-        call MPI_ALLTOALL(src, decomp%z2count, &
-                          complex_type, work2_c, decomp%y2count, &
-                          complex_type, DECOMP_2D_COMM_ROW, ierror)
-     else
-        call MPI_ALLTOALL(work1_c, decomp%z2count, &
-                          complex_type, work2_c, decomp%y2count, &
-                          complex_type, DECOMP_2D_COMM_ROW, ierror)
-     end if
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+      if (decomp%even) then
+         call MPI_ALLTOALL(src, decomp%z2count, &
+                           complex_type, work2_c, decomp%y2count, &
+                           complex_type, DECOMP_2D_COMM_ROW, ierror)
+      else
+         call MPI_ALLTOALL(work1_c, decomp%z2count, &
+                           complex_type, work2_c, decomp%y2count, &
+                           complex_type, DECOMP_2D_COMM_ROW, ierror)
+      end if
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-     call decomp_2d_nccl_send_recv_row(work2_c_d, &
-                                       work1_c_d, &
-                                       decomp%z2disp, &
-                                       decomp%z2cnts, &
-                                       decomp%y2disp, &
-                                       decomp%y2cnts, &
-                                       dims(2), &
-                                       decomp_buf_size)
+      call decomp_2d_nccl_send_recv_row(work2_c_d, &
+                                        work1_c_d, &
+                                        decomp%z2disp, &
+                                        decomp%z2cnts, &
+                                        decomp%y2disp, &
+                                        decomp%y2cnts, &
+                                        dims(2), &
+                                        decomp_buf_size)
 #else
-     call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
-                        complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
-                        complex_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
+                         complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
+                         complex_type, DECOMP_2D_COMM_ROW, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-                        complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
-                        complex_type, DECOMP_2D_COMM_ROW, ierror)
-     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+      call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+                         complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
+                         complex_type, DECOMP_2D_COMM_ROW, ierror)
+      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-     ! rearrange receive buffer
+      ! rearrange receive buffer
 #if defined(_GPU)
-     call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
-                               decomp%y2dist, decomp)
+      call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
+                                decomp%y2dist, decomp)
 #else
-     call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
-                               decomp%y2dist, decomp)
+      call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
+                                decomp%y2dist, decomp)
 #endif
 
 #ifdef PROFILER
-     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
+      if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
 #endif
 
-     return
-  end subroutine transpose_z_to_y_complex
+      return
+   end subroutine transpose_z_to_y_complex
 
-  ! pack/unpack ALLTOALL(V) buffers
+   ! pack/unpack ALLTOALL(V) buffers
 
-  subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     real(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+      real(mytype), dimension(*), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%z2count + 1
+         pos = m * decomp%z2count + 1
 #else
-        pos = decomp%z2disp(m) + 1
+         pos = decomp%z2disp(m) + 1
 #endif
 
-        do k = i1, i2
-           do j = 1, n2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-     end do
-
-     return
-  end subroutine mem_split_zy_real
-
-  subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
-
-     implicit none
-
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-     complex(mytype), dimension(*), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
-
-     integer :: i, j, k, m, i1, i2, pos
-
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+         do k = i1, i2
+            do j = 1, n2
+               do i = 1, n1
+                  out(pos) = in(i, j, k)
+                  pos = pos + 1
+               end do
+            end do
+         end do
+      end do
+
+      return
+   end subroutine mem_split_zy_real
+
+   subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+
+      implicit none
+
+      integer, intent(IN) :: n1, n2, n3
+      complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+      complex(mytype), dimension(*), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
+
+      integer :: i, j, k, m, i1, i2, pos
+
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%z2count + 1
+         pos = m * decomp%z2count + 1
 #else
-        pos = decomp%z2disp(m) + 1
+         pos = decomp%z2disp(m) + 1
 #endif
 
-        do k = i1, i2
-           do j = 1, n2
-              do i = 1, n1
-                 out(pos) = in(i, j, k)
-                 pos = pos + 1
-              end do
-           end do
-        end do
-     end do
-
-     return
-  end subroutine mem_split_zy_complex
-
-  subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
-
-     implicit none
-
-     integer, intent(IN) :: n1, n2, n3
-     real(mytype), dimension(*), intent(IN) :: in
-     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+         do k = i1, i2
+            do j = 1, n2
+               do i = 1, n1
+                  out(pos) = in(i, j, k)
+                  pos = pos + 1
+               end do
+            end do
+         end do
+      end do
+
+      return
+   end subroutine mem_split_zy_complex
+
+   subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
+
+      implicit none
+
+      integer, intent(IN) :: n1, n2, n3
+      real(mytype), dimension(*), intent(IN) :: in
+      real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+      attributes(device) :: in
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%y2count + 1
+         pos = m * decomp%y2count + 1
 #else
-        pos = decomp%y2disp(m) + 1
+         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(out)
+         istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = i1, i2
+               do i = 1, n1
+                  out(i, j, k) = in(pos)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_merge_zy_real
+      return
+   end subroutine mem_merge_zy_real
 
-  subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+   subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-     implicit none
+      implicit none
 
-     integer, intent(IN) :: n1, n2, n3
-     complex(mytype), dimension(*), intent(IN) :: in
-     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-     integer, intent(IN) :: iproc
-     integer, dimension(0:iproc - 1), intent(IN) :: dist
-     TYPE(DECOMP_INFO), intent(IN) :: decomp
+      integer, intent(IN) :: n1, n2, n3
+      complex(mytype), dimension(*), intent(IN) :: in
+      complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+      integer, intent(IN) :: iproc
+      integer, dimension(0:iproc - 1), intent(IN) :: dist
+      TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-     attributes(device) :: in
-     integer :: istat
+      attributes(device) :: in
+      integer :: istat
 #endif
 
-     integer :: i, j, k, m, i1, i2, pos
+      integer :: i, j, k, m, i1, i2, pos
 
-     do m = 0, iproc - 1
-        if (m == 0) then
-           i1 = 1
-           i2 = dist(0)
-        else
-           i1 = i2 + 1
-           i2 = i1 + dist(m) - 1
-        end if
+      do m = 0, iproc - 1
+         if (m == 0) then
+            i1 = 1
+            i2 = dist(0)
+         else
+            i1 = i2 + 1
+            i2 = i1 + dist(m) - 1
+         end if
 
 #ifdef EVEN
-        pos = m * decomp%y2count + 1
+         pos = m * decomp%y2count + 1
 #else
-        pos = decomp%y2disp(m) + 1
+         pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
-        !$acc host_data use_device(out)
-        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-        !$acc end host_data
-        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+         !$acc host_data use_device(out)
+         istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+         !$acc end host_data
+         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-        do k = 1, n3
-           do j = i1, i2
-              do i = 1, n1
-                 out(i, j, k) = in(pos)
-                 pos = pos + 1
-              end do
-           end do
-        end do
+         do k = 1, n3
+            do j = i1, i2
+               do i = 1, n1
+                  out(i, j, k) = in(pos)
+                  pos = pos + 1
+               end do
+            end do
+         end do
 #endif
-     end do
+      end do
 
-     return
-  end subroutine mem_merge_zy_complex
+      return
+   end subroutine mem_merge_zy_complex
 
 end submodule d2d_transpose_z_to_y

From 5269b24771b87dcef2889a3c5f48ea4fbcfbac52 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <ptb0890@gmail.com>
Date: Sat, 27 May 2023 12:08:35 +0100
Subject: [PATCH 246/436] Revert "Move transpose subroutines in dedicated
 submodules"

---
 src/CMakeLists.txt       |   6 +-
 src/decomp_2d.f90        |  96 ++----
 src/transpose_x_to_y.f90 | 581 ++++++++++++++++++------------------
 src/transpose_y_to_x.f90 | 581 ++++++++++++++++++------------------
 src/transpose_y_to_z.f90 | 623 +++++++++++++++++++-------------------
 src/transpose_z_to_y.f90 | 627 +++++++++++++++++++--------------------
 6 files changed, 1217 insertions(+), 1297 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index d39cd212..e8a84b59 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -3,11 +3,7 @@ file(GLOB files_decomp factor.f90
 		       decomp_2d_mpi.f90
 	               decomp_2d.f90
 		       log.f90
-		       io.f90
-		       transpose_x_to_y.f90
-		       transpose_y_to_x.f90
-		       transpose_y_to_z.f90
-		       transpose_z_to_y.f90)
+		       io.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
 if(${BUILD_TARGET} MATCHES "gpu")
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 1ccd7a44..7bf849f7 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -193,87 +193,31 @@ module decomp_2d
    end interface decomp_2d_finalize
 
    interface transpose_x_to_y
-      module subroutine transpose_x_to_y_real_long(src, dst, decomp)
-         real(mytype), dimension(:, :, :), intent(IN) :: src
-         real(mytype), dimension(:, :, :), intent(OUT) :: dst
-         TYPE(DECOMP_INFO), intent(IN) :: decomp
-      end subroutine transpose_x_to_y_real_long
-      module subroutine transpose_x_to_y_real_short(src, dst)
-         real(mytype), dimension(:, :, :), intent(IN) :: src
-         real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      end subroutine transpose_x_to_y_real_short
-      module subroutine transpose_x_to_y_complex_long(src, dst, decomp)
-         complex(mytype), dimension(:, :, :), intent(IN) :: src
-         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-         TYPE(DECOMP_INFO), intent(IN) :: decomp
-      end subroutine transpose_x_to_y_complex_long
-      module subroutine transpose_x_to_y_complex_short(src, dst)
-         complex(mytype), dimension(:, :, :), intent(IN) :: src
-         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      end subroutine transpose_x_to_y_complex_short
+      module procedure transpose_x_to_y_real_long
+      module procedure transpose_x_to_y_real_short
+      module procedure transpose_x_to_y_complex_long
+      module procedure transpose_x_to_y_complex_short
    end interface transpose_x_to_y
 
    interface transpose_y_to_z
-      module subroutine transpose_y_to_z_real_long(src, dst, decomp)
-         real(mytype), dimension(:, :, :), intent(IN) :: src
-         real(mytype), dimension(:, :, :), intent(OUT) :: dst
-         TYPE(DECOMP_INFO), intent(IN) :: decomp
-      end subroutine transpose_y_to_z_real_long
-      module subroutine transpose_y_to_z_real_short(src, dst)
-         real(mytype), dimension(:, :, :), intent(IN) :: src
-         real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      end subroutine transpose_y_to_z_real_short
-      module subroutine transpose_y_to_z_complex_long(src, dst, decomp)
-         complex(mytype), dimension(:, :, :), intent(IN) :: src
-         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-         TYPE(DECOMP_INFO), intent(IN) :: decomp
-      end subroutine transpose_y_to_z_complex_long
-      module subroutine transpose_y_to_z_complex_short(src, dst)
-         complex(mytype), dimension(:, :, :), intent(IN) :: src
-         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      end subroutine transpose_y_to_z_complex_short
+      module procedure transpose_y_to_z_real_long
+      module procedure transpose_y_to_z_real_short
+      module procedure transpose_y_to_z_complex_long
+      module procedure transpose_y_to_z_complex_short
    end interface transpose_y_to_z
 
    interface transpose_z_to_y
-      module subroutine transpose_z_to_y_real_long(src, dst, decomp)
-         real(mytype), dimension(:, :, :), intent(IN) :: src
-         real(mytype), dimension(:, :, :), intent(OUT) :: dst
-         TYPE(DECOMP_INFO), intent(IN) :: decomp
-      end subroutine transpose_z_to_y_real_long
-      module subroutine transpose_z_to_y_real_short(src, dst)
-         real(mytype), dimension(:, :, :), intent(IN) :: src
-         real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      end subroutine transpose_z_to_y_real_short
-      module subroutine transpose_z_to_y_complex_long(src, dst, decomp)
-         complex(mytype), dimension(:, :, :), intent(IN) :: src
-         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-         TYPE(DECOMP_INFO), intent(IN) :: decomp
-      end subroutine transpose_z_to_y_complex_long
-      module subroutine transpose_z_to_y_complex_short(src, dst)
-         complex(mytype), dimension(:, :, :), intent(IN) :: src
-         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      end subroutine transpose_z_to_y_complex_short
+      module procedure transpose_z_to_y_real_long
+      module procedure transpose_z_to_y_real_short
+      module procedure transpose_z_to_y_complex_long
+      module procedure transpose_z_to_y_complex_short
    end interface transpose_z_to_y
 
    interface transpose_y_to_x
-      module subroutine transpose_y_to_x_real_long(src, dst, decomp)
-         real(mytype), dimension(:, :, :), intent(IN) :: src
-         real(mytype), dimension(:, :, :), intent(OUT) :: dst
-         TYPE(DECOMP_INFO), intent(IN) :: decomp
-      end subroutine transpose_y_to_x_real_long
-      module subroutine transpose_y_to_x_real_short(src, dst)
-         real(mytype), dimension(:, :, :), intent(IN) :: src
-         real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      end subroutine transpose_y_to_x_real_short
-      module subroutine transpose_y_to_x_complex_long(src, dst, decomp)
-         complex(mytype), dimension(:, :, :), intent(IN) :: src
-         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-         TYPE(DECOMP_INFO), intent(IN) :: decomp
-      end subroutine transpose_y_to_x_complex_long
-      module subroutine transpose_y_to_x_complex_short(src, dst)
-         complex(mytype), dimension(:, :, :), intent(IN) :: src
-         complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      end subroutine transpose_y_to_x_complex_short
+      module procedure transpose_y_to_x_real_long
+      module procedure transpose_y_to_x_real_short
+      module procedure transpose_y_to_x_complex_long
+      module procedure transpose_y_to_x_complex_short
    end interface transpose_y_to_x
 
    interface update_halo
@@ -1185,6 +1129,14 @@ subroutine prepare_buffer(decomp)
 
    end subroutine prepare_buffer
 
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   ! Transposition routines
+  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+#include "transpose_x_to_y.f90"
+#include "transpose_y_to_z.f90"
+#include "transpose_z_to_y.f90"
+#include "transpose_y_to_x.f90"
+
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! Halo cell support
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index cf368d65..ea8f4a74 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -10,454 +10,447 @@
 !=======================================================================
 
 ! This file contains the routines that transpose data from X to Y pencil
-submodule(decomp_2d) d2d_transpose_x_to_y
 
-   implicit none
+  subroutine transpose_x_to_y_real_short(src, dst)
 
-contains
+     implicit none
 
-   module subroutine transpose_x_to_y_real_short(src, dst)
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-      implicit none
+     call transpose_x_to_y(src, dst, decomp_main)
 
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+  end subroutine transpose_x_to_y_real_short
 
-      call transpose_x_to_y(src, dst, decomp_main)
+  subroutine transpose_x_to_y_real_long(src, dst, decomp)
 
-   end subroutine transpose_x_to_y_real_short
+     implicit none
 
-   module subroutine transpose_x_to_y_real_long(src, dst, decomp)
-
-      implicit none
-
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      integer :: istat, nsize
+     integer :: istat, nsize
 #endif
 
-      if (dims(1) == 1) then
+     if (dims(1) == 1) then
 #if defined(_GPU)
-         nsize = product(decomp%xsz)
-         !$acc host_data use_device(src,dst)
-         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+        nsize = product(decomp%xsz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-         dst = src
+        dst = src
 #endif
-      else
-         call transpose_x_to_y_real(src, dst, decomp)
-      end if
+     else
+        call transpose_x_to_y_real(src, dst, decomp)
+     end if
 
-   end subroutine transpose_x_to_y_real_long
+  end subroutine transpose_x_to_y_real_long
 
-   subroutine transpose_x_to_y_real(src, dst, decomp)
+  subroutine transpose_x_to_y_real(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      Real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     Real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: s1, s2, s3, d1, d2, d3
-      integer :: ierror
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_r")
 #endif
 
-      s1 = SIZE(src, 1)
-      s2 = SIZE(src, 2)
-      s3 = SIZE(src, 3)
-      d1 = SIZE(dst, 1)
-      d2 = SIZE(dst, 2)
-      d3 = SIZE(dst, 3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-      ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #if defined(_GPU)
-      call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
-                             decomp%x1dist, decomp)
+     call mem_split_xy_real(src, s1, s2, s3, work1_r_d, dims(1), &
+                            decomp%x1dist, decomp)
 #else
-      call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
-                             decomp%x1dist, decomp)
+     call mem_split_xy_real(src, s1, s2, s3, work1_r, dims(1), &
+                            decomp%x1dist, decomp)
 #endif
 
-      ! define receive buffer
-      ! transpose using MPI_ALLTOALL(V)
+     ! define receive buffer
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef EVEN
-      call MPI_ALLTOALL(work1_r, decomp%x1count, &
-                        real_type, work2_r, decomp%y1count, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_r, decomp%x1count, &
+                       real_type, work2_r, decomp%y1count, &
+                       real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-      call decomp_2d_nccl_send_recv_col(work2_r_d, &
-                                        work1_r_d, &
-                                        decomp%x1disp, &
-                                        decomp%x1cnts, &
-                                        decomp%y1disp, &
-                                        decomp%y1cnts, &
-                                        dims(1))
+     call decomp_2d_nccl_send_recv_col(work2_r_d, &
+                                       work1_r_d, &
+                                       decomp%x1disp, &
+                                       decomp%x1cnts, &
+                                       decomp%y1disp, &
+                                       decomp%y1cnts, &
+                                       dims(1))
 #else
-      call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
-                         real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
-                         real_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%x1cnts, decomp%x1disp, &
+                        real_type, work2_r_d, decomp%y1cnts, decomp%y1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-      call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
-                         real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
-                         real_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r, decomp%x1cnts, decomp%x1disp, &
+                        real_type, work2_r, decomp%y1cnts, decomp%y1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-      ! rearrange receive buffer
+     ! rearrange receive buffer
 #if defined(_GPU)
-      call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-                             decomp%y1dist, decomp)
+     call mem_merge_xy_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+                            decomp%y1dist, decomp)
 #else
-      call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
-                             decomp%y1dist, decomp)
+     call mem_merge_xy_real(work2_r, d1, d2, d3, dst, dims(1), &
+                            decomp%y1dist, decomp)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_r")
 #endif
 
-      return
-   end subroutine transpose_x_to_y_real
+     return
+  end subroutine transpose_x_to_y_real
 
-   module subroutine transpose_x_to_y_complex_short(src, dst)
+  subroutine transpose_x_to_y_complex_short(src, dst)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-      call transpose_x_to_y(src, dst, decomp_main)
+     call transpose_x_to_y(src, dst, decomp_main)
 
-   end subroutine transpose_x_to_y_complex_short
+  end subroutine transpose_x_to_y_complex_short
 
-   module subroutine transpose_x_to_y_complex_long(src, dst, decomp)
+  subroutine transpose_x_to_y_complex_long(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      integer :: istat, nsize
+     integer :: istat, nsize
 #endif
-      if (dims(1) == 1) then
+     if (dims(1) == 1) then
 #if defined(_GPU)
-         nsize = product(decomp%xsz)
-         !$acc host_data use_device(src,dst)
-         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+        nsize = product(decomp%xsz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-         dst = src
+        dst = src
 #endif
-      else
-         call transpose_x_to_y_complex(src, dst, decomp)
-      end if
+     else
+        call transpose_x_to_y_complex(src, dst, decomp)
+     end if
 
-   end subroutine transpose_x_to_y_complex_long
+  end subroutine transpose_x_to_y_complex_long
 
-   subroutine transpose_x_to_y_complex(src, dst, decomp)
+  subroutine transpose_x_to_y_complex(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: s1, s2, s3, d1, d2, d3
-      integer :: ierror
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_x_y_c")
 #endif
 
-      s1 = SIZE(src, 1)
-      s2 = SIZE(src, 2)
-      s3 = SIZE(src, 3)
-      d1 = SIZE(dst, 1)
-      d2 = SIZE(dst, 2)
-      d3 = SIZE(dst, 3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-      ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #if defined(_GPU)
-      call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-                                decomp%x1dist, decomp)
+     call mem_split_xy_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+                               decomp%x1dist, decomp)
 #else
-      call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
-                                decomp%x1dist, decomp)
+     call mem_split_xy_complex(src, s1, s2, s3, work1_c, dims(1), &
+                               decomp%x1dist, decomp)
 #endif
 
-      ! define receive buffer
-      ! transpose using MPI_ALLTOALL(V)
+     ! define receive buffer
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef EVEN
-      call MPI_ALLTOALL(work1_c, decomp%x1count, &
-                        complex_type, work2_c, decomp%y1count, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_c, decomp%x1count, &
+                       complex_type, work2_c, decomp%y1count, &
+                       complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-      call decomp_2d_nccl_send_recv_col(work2_c_d, &
-                                        work1_c_d, &
-                                        decomp%x1disp, &
-                                        decomp%x1cnts, &
-                                        decomp%y1disp, &
-                                        decomp%y1cnts, &
-                                        dims(1), &
-                                        decomp_buf_size)
+     call decomp_2d_nccl_send_recv_col(work2_c_d, &
+                                       work1_c_d, &
+                                       decomp%x1disp, &
+                                       decomp%x1cnts, &
+                                       decomp%y1disp, &
+                                       decomp%y1cnts, &
+                                       dims(1), &
+                                       decomp_buf_size)
 #else
-      call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
-                         complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
-                         complex_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, work2_c_d, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-      call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
-                         complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
-                         complex_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, work2_c, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-      ! rearrange receive buffer
+     ! rearrange receive buffer
 #if defined(_GPU)
-      call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-                                decomp%y1dist, decomp)
+     call mem_merge_xy_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+                               decomp%y1dist, decomp)
 #else
-      call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
-                                decomp%y1dist, decomp)
+     call mem_merge_xy_complex(work2_c, d1, d2, d3, dst, dims(1), &
+                               decomp%y1dist, decomp)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_x_y_c")
 #endif
 
-      return
-   end subroutine transpose_x_to_y_complex
+     return
+  end subroutine transpose_x_to_y_complex
 
-   subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_split_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-      real(mytype), dimension(*), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: out
-      integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
-         pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(in)
-         istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = 1, n2
-               do i = i1, i2
-                  out(pos) = in(i, j, k)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
+     end do
 
-      return
-   end subroutine mem_split_xy_real
+     return
+  end subroutine mem_split_xy_real
 
-   subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_split_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-      complex(mytype), dimension(*), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: out
-      integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
-         pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(in)
-         istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D(out(pos), i2 - i1 + 1, in(i1, 1, 1), n1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = 1, n2
-               do i = i1, i2
-                  out(pos) = in(i, j, k)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
+     end do
 
-      return
-   end subroutine mem_split_xy_complex
+     return
+  end subroutine mem_split_xy_complex
 
-   subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_merge_xy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      real(mytype), dimension(*), intent(IN) :: in
-      real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: in
-      integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
-         pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(out)
-         istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = i1, i2
-               do i = 1, n1
-                  out(i, j, k) = in(pos)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
+     end do
 
-      return
-   end subroutine mem_merge_xy_real
+     return
+  end subroutine mem_merge_xy_real
 
-   subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_merge_xy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      complex(mytype), dimension(*), intent(IN) :: in
-      complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: in
-      integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
-         pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(out)
-         istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = i1, i2
-               do i = 1, n1
-                  out(i, j, k) = in(pos)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
-
-      return
-   end subroutine mem_merge_xy_complex
+     end do
 
-end submodule d2d_transpose_x_to_y
+     return
+  end subroutine mem_merge_xy_complex
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index eefb9498..2b1a1efd 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -10,454 +10,447 @@
 !=======================================================================
 
 ! This file contains the routines that transpose data from Y to X pencil
-submodule(decomp_2d) d2d_transpose_y_to_x
 
-   implicit none
+  subroutine transpose_y_to_x_real_short(src, dst)
 
-contains
+     implicit none
 
-   module subroutine transpose_y_to_x_real_short(src, dst)
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-      implicit none
+     call transpose_y_to_x(src, dst, decomp_main)
 
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+  end subroutine transpose_y_to_x_real_short
 
-      call transpose_y_to_x(src, dst, decomp_main)
+  subroutine transpose_y_to_x_real_long(src, dst, decomp)
 
-   end subroutine transpose_y_to_x_real_short
+     implicit none
 
-   module subroutine transpose_y_to_x_real_long(src, dst, decomp)
-
-      implicit none
-
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      integer :: istat, nsize
+     integer :: istat, nsize
 #endif
 
-      if (dims(1) == 1) then
+     if (dims(1) == 1) then
 #if defined(_GPU)
-         nsize = product(decomp%ysz)
-         !$acc host_data use_device(src,dst)
-         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+        nsize = product(decomp%ysz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-         dst = src
+        dst = src
 #endif
-      else
-         call transpose_y_to_x_real(src, dst, decomp)
-      end if
+     else
+        call transpose_y_to_x_real(src, dst, decomp)
+     end if
 
-   end subroutine transpose_y_to_x_real_long
+  end subroutine transpose_y_to_x_real_long
 
-   subroutine transpose_y_to_x_real(src, dst, decomp)
+  subroutine transpose_y_to_x_real(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: s1, s2, s3, d1, d2, d3
-      integer :: ierror
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_r")
 #endif
 
-      s1 = SIZE(src, 1)
-      s2 = SIZE(src, 2)
-      s3 = SIZE(src, 3)
-      d1 = SIZE(dst, 1)
-      d2 = SIZE(dst, 2)
-      d3 = SIZE(dst, 3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-      ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #if defined(_GPU)
-      call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
-                             decomp%y1dist, decomp)
+     call mem_split_yx_real(src, s1, s2, s3, work1_r_d, dims(1), &
+                            decomp%y1dist, decomp)
 #else
-      call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
-                             decomp%y1dist, decomp)
+     call mem_split_yx_real(src, s1, s2, s3, work1_r, dims(1), &
+                            decomp%y1dist, decomp)
 #endif
 
-      ! define receive buffer
-      ! transpose using MPI_ALLTOALL(V)
+     ! define receive buffer
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef EVEN
-      call MPI_ALLTOALL(work1_r, decomp%y1count, &
-                        real_type, work2_r, decomp%x1count, &
-                        real_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_r, decomp%y1count, &
+                       real_type, work2_r, decomp%x1count, &
+                       real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-      call decomp_2d_nccl_send_recv_col(work2_r_d, &
-                                        work1_r_d, &
-                                        decomp%y1disp, &
-                                        decomp%y1cnts, &
-                                        decomp%x1disp, &
-                                        decomp%x1cnts, &
-                                        dims(1))
+     call decomp_2d_nccl_send_recv_col(work2_r_d, &
+                                       work1_r_d, &
+                                       decomp%y1disp, &
+                                       decomp%y1cnts, &
+                                       decomp%x1disp, &
+                                       decomp%x1cnts, &
+                                       dims(1))
 #else
-      call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
-                         real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
-                         real_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%y1cnts, decomp%y1disp, &
+                        real_type, work2_r_d, decomp%x1cnts, decomp%x1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-      call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
-                         real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
-                         real_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r, decomp%y1cnts, decomp%y1disp, &
+                        real_type, work2_r, decomp%x1cnts, decomp%x1disp, &
+                        real_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-      ! rearrange receive buffer
+     ! rearrange receive buffer
 #if defined(_GPU)
-      call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
-                             decomp%x1dist, decomp)
+     call mem_merge_yx_real(work2_r_d, d1, d2, d3, dst, dims(1), &
+                            decomp%x1dist, decomp)
 #else
-      call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
-                             decomp%x1dist, decomp)
+     call mem_merge_yx_real(work2_r, d1, d2, d3, dst, dims(1), &
+                            decomp%x1dist, decomp)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_r")
 #endif
 
-      return
-   end subroutine transpose_y_to_x_real
+     return
+  end subroutine transpose_y_to_x_real
 
-   module subroutine transpose_y_to_x_complex_short(src, dst)
+  subroutine transpose_y_to_x_complex_short(src, dst)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-      call transpose_y_to_x(src, dst, decomp_main)
+     call transpose_y_to_x(src, dst, decomp_main)
 
-   end subroutine transpose_y_to_x_complex_short
+  end subroutine transpose_y_to_x_complex_short
 
-   module subroutine transpose_y_to_x_complex_long(src, dst, decomp)
+  subroutine transpose_y_to_x_complex_long(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      integer :: istat, nsize
+     integer :: istat, nsize
 #endif
-      if (dims(1) == 1) then
+     if (dims(1) == 1) then
 #if defined(_GPU)
-         nsize = product(decomp%ysz)
-         !$acc host_data use_device(src,dst)
-         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+        nsize = product(decomp%ysz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-         dst = src
+        dst = src
 #endif
-      else
-         call transpose_y_to_x_complex(src, dst, decomp)
-      end if
+     else
+        call transpose_y_to_x_complex(src, dst, decomp)
+     end if
 
-   end subroutine transpose_y_to_x_complex_long
+  end subroutine transpose_y_to_x_complex_long
 
-   subroutine transpose_y_to_x_complex(src, dst, decomp)
+  subroutine transpose_y_to_x_complex(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: s1, s2, s3, d1, d2, d3
-      integer :: ierror
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_x_c")
 #endif
 
-      s1 = SIZE(src, 1)
-      s2 = SIZE(src, 2)
-      s3 = SIZE(src, 3)
-      d1 = SIZE(dst, 1)
-      d2 = SIZE(dst, 2)
-      d3 = SIZE(dst, 3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-      ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #if defined(_GPU)
-      call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
-                                decomp%y1dist, decomp)
+     call mem_split_yx_complex(src, s1, s2, s3, work1_c_d, dims(1), &
+                               decomp%y1dist, decomp)
 #else
-      call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
-                                decomp%y1dist, decomp)
+     call mem_split_yx_complex(src, s1, s2, s3, work1_c, dims(1), &
+                               decomp%y1dist, decomp)
 #endif
 
-      ! define receive buffer
-      ! transpose using MPI_ALLTOALL(V)
+     ! define receive buffer
+     ! transpose using MPI_ALLTOALL(V)
 #ifdef EVEN
-      call MPI_ALLTOALL(work1_c, decomp%y1count, &
-                        complex_type, work2_c, decomp%x1count, &
-                        complex_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     call MPI_ALLTOALL(work1_c, decomp%y1count, &
+                       complex_type, work2_c, decomp%x1count, &
+                       complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-      call decomp_2d_nccl_send_recv_col(work2_c_d, &
-                                        work1_c_d, &
-                                        decomp%y1disp, &
-                                        decomp%y1cnts, &
-                                        decomp%x1disp, &
-                                        decomp%x1cnts, &
-                                        dims(1), &
-                                        decomp_buf_size)
+     call decomp_2d_nccl_send_recv_col(work2_c_d, &
+                                       work1_c_d, &
+                                       decomp%y1disp, &
+                                       decomp%y1cnts, &
+                                       decomp%x1disp, &
+                                       decomp%x1cnts, &
+                                       dims(1), &
+                                       decomp_buf_size)
 #else
-      call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
-                         complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
-                         complex_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, work2_c_d, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-      call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
-                         complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
-                         complex_type, DECOMP_2D_COMM_COL, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c, decomp%y1cnts, decomp%y1disp, &
+                        complex_type, work2_c, decomp%x1cnts, decomp%x1disp, &
+                        complex_type, DECOMP_2D_COMM_COL, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-      ! rearrange receive buffer
+     ! rearrange receive buffer
 #if defined(_GPU)
-      call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
-                                decomp%x1dist, decomp)
+     call mem_merge_yx_complex(work2_c_d, d1, d2, d3, dst, dims(1), &
+                               decomp%x1dist, decomp)
 #else
-      call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
-                                decomp%x1dist, decomp)
+     call mem_merge_yx_complex(work2_c, d1, d2, d3, dst, dims(1), &
+                               decomp%x1dist, decomp)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_x_c")
 #endif
 
-      return
-   end subroutine transpose_y_to_x_complex
+     return
+  end subroutine transpose_y_to_x_complex
 
-   subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_split_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-      real(mytype), dimension(*), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: out
-      integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
-         pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(in)
-         istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = i1, i2
-               do i = 1, n1
-                  out(pos) = in(i, j, k)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
+     end do
 
-      return
-   end subroutine mem_split_yx_real
+     return
+  end subroutine mem_split_yx_real
 
-   subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_split_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-      complex(mytype), dimension(*), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: out
-      integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%y1count + 1
+        pos = m * decomp%y1count + 1
 #else
-         pos = decomp%y1disp(m) + 1
+        pos = decomp%y1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(in)
-         istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = i1, i2
-               do i = 1, n1
-                  out(pos) = in(i, j, k)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
+     end do
 
-      return
-   end subroutine mem_split_yx_complex
+     return
+  end subroutine mem_split_yx_complex
 
-   subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_merge_yx_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      real(mytype), dimension(*), intent(IN) :: in
-      real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: in
-      integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
-         pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(out)
-         istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = 1, n2
-               do i = i1, i2
-                  out(i, j, k) = in(pos)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
+     end do
 
-      return
-   end subroutine mem_merge_yx_real
+     return
+  end subroutine mem_merge_yx_real
 
-   subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_merge_yx_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      complex(mytype), dimension(*), intent(IN) :: in
-      complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: in
-      integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%x1count + 1
+        pos = m * decomp%x1count + 1
 #else
-         pos = decomp%x1disp(m) + 1
+        pos = decomp%x1disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(out)
-         istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D(out(i1, 1, 1), n1, in(pos), i2 - i1 + 1, i2 - i1 + 1, n2 * n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = 1, n2
-               do i = i1, i2
-                  out(i, j, k) = in(pos)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = 1, n2
+              do i = i1, i2
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
-
-      return
-   end subroutine mem_merge_yx_complex
+     end do
 
-end submodule d2d_transpose_y_to_x
+     return
+  end subroutine mem_merge_yx_complex
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index 19f244ad..cca04207 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -10,473 +10,466 @@
 !=======================================================================
 
 ! This file contains the routines that transpose data from Y to Z pencil
-submodule(decomp_2d) d2d_transpose_y_to_z
 
-   implicit none
+  subroutine transpose_y_to_z_real_short(src, dst)
 
-contains
+     implicit none
 
-   module subroutine transpose_y_to_z_real_short(src, dst)
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-      implicit none
+     call transpose_y_to_z(src, dst, decomp_main)
 
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+  end subroutine transpose_y_to_z_real_short
 
-      call transpose_y_to_z(src, dst, decomp_main)
+  subroutine transpose_y_to_z_real_long(src, dst, decomp)
 
-   end subroutine transpose_y_to_z_real_short
+     implicit none
 
-   module subroutine transpose_y_to_z_real_long(src, dst, decomp)
-
-      implicit none
-
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      integer :: istat, nsize
+     integer :: istat, nsize
 #endif
 
-      if (dims(2) == 1) then
+     if (dims(2) == 1) then
 #if defined(_GPU)
-         nsize = product(decomp%ysz)
-         !$acc host_data use_device(src,dst)
-         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+        nsize = product(decomp%ysz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-         dst = src
+        dst = src
 #endif
-      else
-         call transpose_y_to_z_real(src, dst, decomp)
-      end if
+     else
+        call transpose_y_to_z_real(src, dst, decomp)
+     end if
 
-   end subroutine transpose_y_to_z_real_long
+  end subroutine transpose_y_to_z_real_long
 
-   subroutine transpose_y_to_z_real(src, dst, decomp)
+  subroutine transpose_y_to_z_real(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-      integer :: istat
+     integer :: istat
 #endif
 
-      integer :: s1, s2, s3, d1, d2, d3
-      integer :: ierror
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_r")
 #endif
 
-      s1 = SIZE(src, 1)
-      s2 = SIZE(src, 2)
-      s3 = SIZE(src, 3)
-      d1 = SIZE(dst, 1)
-      d2 = SIZE(dst, 2)
-      d3 = SIZE(dst, 3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-      ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #if defined(_GPU)
-      call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
-                             decomp%y2dist, decomp)
+     call mem_split_yz_real(src, s1, s2, s3, work1_r_d, dims(2), &
+                            decomp%y2dist, decomp)
 #else
-      call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
-                             decomp%y2dist, decomp)
+     call mem_split_yz_real(src, s1, s2, s3, work1_r, dims(2), &
+                            decomp%y2dist, decomp)
 #endif
 
-      ! define receive buffer
+     ! define receive buffer
 #ifdef EVEN
-      if (decomp%even) then
-         call MPI_ALLTOALL(work1_r, decomp%y2count, &
-                           real_type, dst, decomp%z2count, &
-                           real_type, DECOMP_2D_COMM_ROW, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
-      else
-         call MPI_ALLTOALL(work1_r, decomp%y2count, &
-                           real_type, work2_r, decomp%z2count, &
-                           real_type, DECOMP_2D_COMM_ROW, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
-      end if
+     if (decomp%even) then
+        call MPI_ALLTOALL(work1_r, decomp%y2count, &
+                          real_type, dst, decomp%z2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     else
+        call MPI_ALLTOALL(work1_r, decomp%y2count, &
+                          real_type, work2_r, decomp%z2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+        if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     end if
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-      call decomp_2d_nccl_send_recv_row(work2_r_d, &
-                                        work1_r_d, &
-                                        decomp%y2disp, &
-                                        decomp%y2cnts, &
-                                        decomp%z2disp, &
-                                        decomp%z2cnts, &
-                                        dims(2))
+     call decomp_2d_nccl_send_recv_row(work2_r_d, &
+                                       work1_r_d, &
+                                       decomp%y2disp, &
+                                       decomp%y2cnts, &
+                                       decomp%z2disp, &
+                                       decomp%z2cnts, &
+                                       dims(2))
 #else
-      call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
-                         real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
-                         real_type, DECOMP_2D_COMM_ROW, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%y2cnts, decomp%y2disp, &
+                        real_type, work2_r_d, decomp%z2cnts, decomp%z2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-      call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
-                         real_type, dst, decomp%z2cnts, decomp%z2disp, &
-                         real_type, DECOMP_2D_COMM_ROW, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r, decomp%y2cnts, decomp%y2disp, &
+                        real_type, dst, decomp%z2cnts, decomp%z2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-      ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef EVEN
-      if (.not. decomp%even) then
-         call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
-                                decomp%z2dist, decomp)
-      end if
+     if (.not. decomp%even) then
+        call mem_merge_yz_real(work2_r, d1, d2, d3, dst, dims(2), &
+                               decomp%z2dist, decomp)
+     end if
 #else
-      ! note the receive buffer is already in natural (i,j,k) order
-      ! so no merge operation needed
+     ! note the receive buffer is already in natural (i,j,k) order
+     ! so no merge operation needed
 
 #if defined(_GPU)
-      !If one of the array in cuda call is not device we need to add acc host_data
-      !$acc host_data use_device(dst)
-      istat = cudaMemcpy(dst, work2_r_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
-      !$acc end host_data
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+     !If one of the array in cuda call is not device we need to add acc host_data
+     !$acc host_data use_device(dst)
+     istat = cudaMemcpy(dst, work2_r_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
+     !$acc end host_data
+     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_r")
 #endif
 
-      return
-   end subroutine transpose_y_to_z_real
+     return
+  end subroutine transpose_y_to_z_real
 
-   module subroutine transpose_y_to_z_complex_short(src, dst)
+  subroutine transpose_y_to_z_complex_short(src, dst)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-      call transpose_y_to_z(src, dst, decomp_main)
+     call transpose_y_to_z(src, dst, decomp_main)
 
-   end subroutine transpose_y_to_z_complex_short
+  end subroutine transpose_y_to_z_complex_short
 
-   module subroutine transpose_y_to_z_complex_long(src, dst, decomp)
+  subroutine transpose_y_to_z_complex_long(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      integer :: istat, nsize
+     integer :: istat, nsize
 #endif
 
-      if (dims(2) == 1) then
+     if (dims(2) == 1) then
 #if defined(_GPU)
-         nsize = product(decomp%ysz)
-         !$acc host_data use_device(src,dst)
-         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+        nsize = product(decomp%ysz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-         dst = src
+        dst = src
 #endif
-      else
-         call transpose_y_to_z_complex(src, dst, decomp)
-      end if
+     else
+        call transpose_y_to_z_complex(src, dst, decomp)
+     end if
 
-   end subroutine transpose_y_to_z_complex_long
+  end subroutine transpose_y_to_z_complex_long
 
-   subroutine transpose_y_to_z_complex(src, dst, decomp)
+  subroutine transpose_y_to_z_complex(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-      integer :: istat
+     integer :: istat
 #endif
 
-      integer :: s1, s2, s3, d1, d2, d3
-      integer :: ierror
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_y_z_c")
 #endif
 
-      s1 = SIZE(src, 1)
-      s2 = SIZE(src, 2)
-      s3 = SIZE(src, 3)
-      d1 = SIZE(dst, 1)
-      d2 = SIZE(dst, 2)
-      d3 = SIZE(dst, 3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-      ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #if defined(_GPU)
-      call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
-                                decomp%y2dist, decomp)
+     call mem_split_yz_complex(src, s1, s2, s3, work1_c_d, dims(2), &
+                               decomp%y2dist, decomp)
 #else
-      call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
-                                decomp%y2dist, decomp)
+     call mem_split_yz_complex(src, s1, s2, s3, work1_c, dims(2), &
+                               decomp%y2dist, decomp)
 #endif
 
-      ! define receive buffer
+     ! define receive buffer
 #ifdef EVEN
-      if (decomp%even) then
-         call MPI_ALLTOALL(work1_c, decomp%y2count, &
-                           complex_type, dst, decomp%z2count, &
-                           complex_type, DECOMP_2D_COMM_ROW, ierror)
-      else
-         call MPI_ALLTOALL(work1_c, decomp%y2count, &
-                           complex_type, work2_c, decomp%z2count, &
-                           complex_type, DECOMP_2D_COMM_ROW, ierror)
-      end if
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     if (decomp%even) then
+        call MPI_ALLTOALL(work1_c, decomp%y2count, &
+                          complex_type, dst, decomp%z2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     else
+        call MPI_ALLTOALL(work1_c, decomp%y2count, &
+                          complex_type, work2_c, decomp%z2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-      call decomp_2d_nccl_send_recv_row(work2_c_d, &
-                                        work1_c_d, &
-                                        decomp%y2disp, &
-                                        decomp%y2cnts, &
-                                        decomp%z2disp, &
-                                        decomp%z2cnts, &
-                                        dims(2), &
-                                        decomp_buf_size)
+     call decomp_2d_nccl_send_recv_row(work2_c_d, &
+                                       work1_c_d, &
+                                       decomp%y2disp, &
+                                       decomp%y2cnts, &
+                                       decomp%z2disp, &
+                                       decomp%z2cnts, &
+                                       dims(2), &
+                                       decomp_buf_size)
 #else
-      call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
-                         complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
-                         complex_type, DECOMP_2D_COMM_ROW, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, work2_c_d, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-      call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
-                         complex_type, dst, decomp%z2cnts, decomp%z2disp, &
-                         complex_type, DECOMP_2D_COMM_ROW, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, dst, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-      ! rearrange receive buffer
+     ! rearrange receive buffer
 #ifdef EVEN
-      if (.not. decomp%even) then
-         call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
-                                   decomp%z2dist, decomp)
-      end if
+     if (.not. decomp%even) then
+        call mem_merge_yz_complex(work2_c, d1, d2, d3, dst, dims(2), &
+                                  decomp%z2dist, decomp)
+     end if
 #else
-      ! note the receive buffer is already in natural (i,j,k) order
-      ! so no merge operation needed
+     ! note the receive buffer is already in natural (i,j,k) order
+     ! so no merge operation needed
 
 #if defined(_GPU)
-      !$acc host_data use_device(dst)
-      istat = cudaMemcpy(dst, work2_c_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
-      !$acc end host_data
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+     !$acc host_data use_device(dst)
+     istat = cudaMemcpy(dst, work2_c_d, d1 * d2 * d3, cudaMemcpyDeviceToDevice)
+     !$acc end host_data
+     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_y_z_c")
 #endif
 
-      return
-   end subroutine transpose_y_to_z_complex
+     return
+  end subroutine transpose_y_to_z_complex
 
-   subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_split_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-      real(mytype), dimension(*), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: out
-      integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
-         pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(in)
-         istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = i1, i2
-               do i = 1, n1
-                  out(pos) = in(i, j, k)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
+     end do
 
-      return
-   end subroutine mem_split_yz_real
+     return
+  end subroutine mem_split_yz_real
 
-   subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_split_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-      complex(mytype), dimension(*), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: out
-      integer :: istat
+     attributes(device) :: out
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
-         pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(in)
-         istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(in)
+        istat = cudaMemcpy2D(out(pos), n1 * (i2 - i1 + 1), in(1, i1, 1), n1 * n2, n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = i1, i2
-               do i = 1, n1
-                  out(pos) = in(i, j, k)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
+     end do
 
-      return
-   end subroutine mem_split_yz_complex
+     return
+  end subroutine mem_split_yz_complex
 
-   subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_merge_yz_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      real(mytype), dimension(*), intent(IN) :: in
-      real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
-         pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 
-         do k = i1, i2
-            do j = 1, n2
-               do i = 1, n1
-                  out(i, j, k) = in(pos)
-                  pos = pos + 1
-               end do
-            end do
-         end do
-      end do
-
-      return
-   end subroutine mem_merge_yz_real
-
-   subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
-
-      implicit none
-
-      integer, intent(IN) :: n1, n2, n3
-      complex(mytype), dimension(*), intent(IN) :: in
-      complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-
-      integer :: i, j, k, m, i1, i2, pos
-
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
+
+     return
+  end subroutine mem_merge_yz_real
+
+  subroutine mem_merge_yz_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+
+     implicit none
+
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
-         pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 
-         do k = i1, i2
-            do j = 1, n2
-               do i = 1, n1
-                  out(i, j, k) = in(pos)
-                  pos = pos + 1
-               end do
-            end do
-         end do
-      end do
-
-      return
-   end subroutine mem_merge_yz_complex
-
-end submodule d2d_transpose_y_to_z
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
+
+     return
+  end subroutine mem_merge_yz_complex
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index af77904b..35d6714e 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -10,473 +10,466 @@
 !=======================================================================
 
 ! This file contains the routines that transpose data from Z to Y pencil
-submodule(decomp_2d) d2d_transpose_z_to_y
 
-   implicit none
+  subroutine transpose_z_to_y_real_short(src, dst)
 
-contains
+     implicit none
 
-   module subroutine transpose_z_to_y_real_short(src, dst)
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-      implicit none
+     call transpose_z_to_y(src, dst, decomp_main)
 
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
+  end subroutine transpose_z_to_y_real_short
 
-      call transpose_z_to_y(src, dst, decomp_main)
+  subroutine transpose_z_to_y_real_long(src, dst, decomp)
 
-   end subroutine transpose_z_to_y_real_short
+     implicit none
 
-   module subroutine transpose_z_to_y_real_long(src, dst, decomp)
-
-      implicit none
-
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      integer :: istat, nsize
+     integer :: istat, nsize
 #endif
 
-      if (dims(2) == 1) then
+     if (dims(2) == 1) then
 #if defined(_GPU)
-         nsize = product(decomp%zsz)
-         !$acc host_data use_device(src,dst)
-         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+        nsize = product(decomp%zsz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-         dst = src
+        dst = src
 #endif
-      else
-         call transpose_z_to_y_real(src, dst, decomp)
-      end if
+     else
+        call transpose_z_to_y_real(src, dst, decomp)
+     end if
 
-   end subroutine transpose_z_to_y_real_long
+  end subroutine transpose_z_to_y_real_long
 
-   subroutine transpose_z_to_y_real(src, dst, decomp)
+  subroutine transpose_z_to_y_real(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      real(mytype), dimension(:, :, :), intent(IN) :: src
-      real(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     real(mytype), dimension(:, :, :), intent(IN) :: src
+     real(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-      integer :: istat
+     integer :: istat
 #endif
 
-      integer :: s1, s2, s3, d1, d2, d3
-      integer :: ierror
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_r")
 #endif
 
-      s1 = SIZE(src, 1)
-      s2 = SIZE(src, 2)
-      s3 = SIZE(src, 3)
-      d1 = SIZE(dst, 1)
-      d2 = SIZE(dst, 2)
-      d3 = SIZE(dst, 3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-      ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef EVEN
-      if (.not. decomp%even) then
-         call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
-                                decomp%z2dist, decomp)
-      end if
+     if (.not. decomp%even) then
+        call mem_split_zy_real(src, s1, s2, s3, work1_r, dims(2), &
+                               decomp%z2dist, decomp)
+     end if
 #else
-      ! note the src array is suitable to be a send buffer
-      ! so no split operation needed
+     ! note the src array is suitable to be a send buffer
+     ! so no split operation needed
 
 #if defined(_GPU)
-      !$acc host_data use_device(src)
-      istat = cudaMemcpy(work1_r_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
-      !$acc end host_data
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+     !$acc host_data use_device(src)
+     istat = cudaMemcpy(work1_r_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
+     !$acc end host_data
+     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
 
-      ! define receive buffer
+     ! define receive buffer
 #ifdef EVEN
-      if (decomp%even) then
-         call MPI_ALLTOALL(src, decomp%z2count, &
-                           real_type, work2_r, decomp%y2count, &
-                           real_type, DECOMP_2D_COMM_ROW, ierror)
-      else
-         call MPI_ALLTOALL(work1_r, decomp%z2count, &
-                           real_type, work2_r, decomp%y2count, &
-                           real_type, DECOMP_2D_COMM_ROW, ierror)
-      end if
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     if (decomp%even) then
+        call MPI_ALLTOALL(src, decomp%z2count, &
+                          real_type, work2_r, decomp%y2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+     else
+        call MPI_ALLTOALL(work1_r, decomp%z2count, &
+                          real_type, work2_r, decomp%y2count, &
+                          real_type, DECOMP_2D_COMM_ROW, ierror)
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-      call decomp_2d_nccl_send_recv_row(work2_r_d, &
-                                        work1_r_d, &
-                                        decomp%z2disp, &
-                                        decomp%z2cnts, &
-                                        decomp%y2disp, &
-                                        decomp%y2cnts, &
-                                        dims(2))
+     call decomp_2d_nccl_send_recv_row(work2_r_d, &
+                                       work1_r_d, &
+                                       decomp%z2disp, &
+                                       decomp%z2cnts, &
+                                       decomp%y2disp, &
+                                       decomp%y2cnts, &
+                                       dims(2))
 #else
-      call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
-                         real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
-                         real_type, DECOMP_2D_COMM_ROW, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_r_d, decomp%z2cnts, decomp%z2disp, &
+                        real_type, work2_r_d, decomp%y2cnts, decomp%y2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-      call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-                         real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
-                         real_type, DECOMP_2D_COMM_ROW, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+                        real_type, work2_r, decomp%y2cnts, decomp%y2disp, &
+                        real_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #endif
 
-      ! rearrange receive buffer
+     ! rearrange receive buffer
 #if defined(_GPU)
-      call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
-                             decomp%y2dist, decomp)
+     call mem_merge_zy_real(work2_r_d, d1, d2, d3, dst, dims(2), &
+                            decomp%y2dist, decomp)
 #else
-      call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
-                             decomp%y2dist, decomp)
+     call mem_merge_zy_real(work2_r, d1, d2, d3, dst, dims(2), &
+                            decomp%y2dist, decomp)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_r")
 #endif
 
-      return
-   end subroutine transpose_z_to_y_real
+     return
+  end subroutine transpose_z_to_y_real
 
-   module subroutine transpose_z_to_y_complex_short(src, dst)
+  subroutine transpose_z_to_y_complex_short(src, dst)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
 
-      call transpose_z_to_y(src, dst, decomp_main)
+     call transpose_z_to_y(src, dst, decomp_main)
 
-   end subroutine transpose_z_to_y_complex_short
+  end subroutine transpose_z_to_y_complex_short
 
-   module subroutine transpose_z_to_y_complex_long(src, dst, decomp)
+  subroutine transpose_z_to_y_complex_long(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      integer :: istat, nsize
+     integer :: istat, nsize
 #endif
 
-      if (dims(2) == 1) then
+     if (dims(2) == 1) then
 #if defined(_GPU)
-         nsize = product(decomp%zsz)
-         !$acc host_data use_device(src,dst)
-         istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
+        nsize = product(decomp%zsz)
+        !$acc host_data use_device(src,dst)
+        istat = cudaMemcpy(dst, src, nsize, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy")
 #else
-         dst = src
+        dst = src
 #endif
-      else
-         call transpose_z_to_y_complex(src, dst, decomp)
-      end if
+     else
+        call transpose_z_to_y_complex(src, dst, decomp)
+     end if
 
-   end subroutine transpose_z_to_y_complex_long
+  end subroutine transpose_z_to_y_complex_long
 
-   subroutine transpose_z_to_y_complex(src, dst, decomp)
+  subroutine transpose_z_to_y_complex(src, dst, decomp)
 
-      implicit none
+     implicit none
 
-      complex(mytype), dimension(:, :, :), intent(IN) :: src
-      complex(mytype), dimension(:, :, :), intent(OUT) :: dst
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     complex(mytype), dimension(:, :, :), intent(IN) :: src
+     complex(mytype), dimension(:, :, :), intent(OUT) :: dst
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-      integer :: istat
+     integer :: istat
 #endif
 
-      integer :: s1, s2, s3, d1, d2, d3
-      integer :: ierror
+     integer :: s1, s2, s3, d1, d2, d3
+     integer :: ierror
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_start("transp_z_y_c")
 #endif
 
-      s1 = SIZE(src, 1)
-      s2 = SIZE(src, 2)
-      s3 = SIZE(src, 3)
-      d1 = SIZE(dst, 1)
-      d2 = SIZE(dst, 2)
-      d3 = SIZE(dst, 3)
+     s1 = SIZE(src, 1)
+     s2 = SIZE(src, 2)
+     s3 = SIZE(src, 3)
+     d1 = SIZE(dst, 1)
+     d2 = SIZE(dst, 2)
+     d3 = SIZE(dst, 3)
 
-      ! rearrange source array as send buffer
+     ! rearrange source array as send buffer
 #ifdef EVEN
-      if (.not. decomp%even) then
-         call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
-                                   decomp%z2dist, decomp)
-      end if
+     if (.not. decomp%even) then
+        call mem_split_zy_complex(src, s1, s2, s3, work1_c, dims(2), &
+                                  decomp%z2dist, decomp)
+     end if
 #else
-      ! note the src array is suitable to be a send buffer
-      ! so no split operation needed
+     ! note the src array is suitable to be a send buffer
+     ! so no split operation needed
 
 #if defined(_GPU)
-      !$acc host_data use_device(src)
-      istat = cudaMemcpy(work1_c_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
-      !$acc end host_data
-      if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+     !$acc host_data use_device(src)
+     istat = cudaMemcpy(work1_c_d, src, s1 * s2 * s3, cudaMemcpyDeviceToDevice)
+     !$acc end host_data
+     if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #endif
 
 #endif
 
-      ! define receive buffer
+     ! define receive buffer
 #ifdef EVEN
-      if (decomp%even) then
-         call MPI_ALLTOALL(src, decomp%z2count, &
-                           complex_type, work2_c, decomp%y2count, &
-                           complex_type, DECOMP_2D_COMM_ROW, ierror)
-      else
-         call MPI_ALLTOALL(work1_c, decomp%z2count, &
-                           complex_type, work2_c, decomp%y2count, &
-                           complex_type, DECOMP_2D_COMM_ROW, ierror)
-      end if
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
+     if (decomp%even) then
+        call MPI_ALLTOALL(src, decomp%z2count, &
+                          complex_type, work2_c, decomp%y2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     else
+        call MPI_ALLTOALL(work1_c, decomp%z2count, &
+                          complex_type, work2_c, decomp%y2count, &
+                          complex_type, DECOMP_2D_COMM_ROW, ierror)
+     end if
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALL")
 #else
 
 #if defined(_GPU)
 #if defined(_NCCL)
-      call decomp_2d_nccl_send_recv_row(work2_c_d, &
-                                        work1_c_d, &
-                                        decomp%z2disp, &
-                                        decomp%z2cnts, &
-                                        decomp%y2disp, &
-                                        decomp%y2cnts, &
-                                        dims(2), &
-                                        decomp_buf_size)
+     call decomp_2d_nccl_send_recv_row(work2_c_d, &
+                                       work1_c_d, &
+                                       decomp%z2disp, &
+                                       decomp%z2cnts, &
+                                       decomp%y2disp, &
+                                       decomp%y2cnts, &
+                                       dims(2), &
+                                       decomp_buf_size)
 #else
-      call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
-                         complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
-                         complex_type, DECOMP_2D_COMM_ROW, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(work1_c_d, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, work2_c_d, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 #else
-      call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
-                         complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
-                         complex_type, DECOMP_2D_COMM_ROW, ierror)
-      if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
+     call MPI_ALLTOALLV(src, decomp%z2cnts, decomp%z2disp, &
+                        complex_type, work2_c, decomp%y2cnts, decomp%y2disp, &
+                        complex_type, DECOMP_2D_COMM_ROW, ierror)
+     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ALLTOALLV")
 #endif
 
 #endif
 
-      ! rearrange receive buffer
+     ! rearrange receive buffer
 #if defined(_GPU)
-      call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
-                                decomp%y2dist, decomp)
+     call mem_merge_zy_complex(work2_c_d, d1, d2, d3, dst, dims(2), &
+                               decomp%y2dist, decomp)
 #else
-      call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
-                                decomp%y2dist, decomp)
+     call mem_merge_zy_complex(work2_c, d1, d2, d3, dst, dims(2), &
+                               decomp%y2dist, decomp)
 #endif
 
 #ifdef PROFILER
-      if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
+     if (decomp_profiler_transpose) call decomp_profiler_end("transp_z_y_c")
 #endif
 
-      return
-   end subroutine transpose_z_to_y_complex
+     return
+  end subroutine transpose_z_to_y_complex
 
-   ! pack/unpack ALLTOALL(V) buffers
+  ! pack/unpack ALLTOALL(V) buffers
 
-   subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_split_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      real(mytype), dimension(n1, n2, n3), intent(IN) :: in
-      real(mytype), dimension(*), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     real(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
-         pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 
-         do k = i1, i2
-            do j = 1, n2
-               do i = 1, n1
-                  out(pos) = in(i, j, k)
-                  pos = pos + 1
-               end do
-            end do
-         end do
-      end do
-
-      return
-   end subroutine mem_split_zy_real
-
-   subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
-
-      implicit none
-
-      integer, intent(IN) :: n1, n2, n3
-      complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
-      complex(mytype), dimension(*), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
-
-      integer :: i, j, k, m, i1, i2, pos
-
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
+
+     return
+  end subroutine mem_split_zy_real
+
+  subroutine mem_split_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+
+     implicit none
+
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(n1, n2, n3), intent(IN) :: in
+     complex(mytype), dimension(*), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
+
+     integer :: i, j, k, m, i1, i2, pos
+
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%z2count + 1
+        pos = m * decomp%z2count + 1
 #else
-         pos = decomp%z2disp(m) + 1
+        pos = decomp%z2disp(m) + 1
 #endif
 
-         do k = i1, i2
-            do j = 1, n2
-               do i = 1, n1
-                  out(pos) = in(i, j, k)
-                  pos = pos + 1
-               end do
-            end do
-         end do
-      end do
-
-      return
-   end subroutine mem_split_zy_complex
-
-   subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
-
-      implicit none
-
-      integer, intent(IN) :: n1, n2, n3
-      real(mytype), dimension(*), intent(IN) :: in
-      real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+        do k = i1, i2
+           do j = 1, n2
+              do i = 1, n1
+                 out(pos) = in(i, j, k)
+                 pos = pos + 1
+              end do
+           end do
+        end do
+     end do
+
+     return
+  end subroutine mem_split_zy_complex
+
+  subroutine mem_merge_zy_real(in, n1, n2, n3, out, iproc, dist, decomp)
+
+     implicit none
+
+     integer, intent(IN) :: n1, n2, n3
+     real(mytype), dimension(*), intent(IN) :: in
+     real(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 #if defined(_GPU)
-      attributes(device) :: in
-      integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
-         pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(out)
-         istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = i1, i2
-               do i = 1, n1
-                  out(i, j, k) = in(pos)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
+     end do
 
-      return
-   end subroutine mem_merge_zy_real
+     return
+  end subroutine mem_merge_zy_real
 
-   subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
+  subroutine mem_merge_zy_complex(in, n1, n2, n3, out, iproc, dist, decomp)
 
-      implicit none
+     implicit none
 
-      integer, intent(IN) :: n1, n2, n3
-      complex(mytype), dimension(*), intent(IN) :: in
-      complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
-      integer, intent(IN) :: iproc
-      integer, dimension(0:iproc - 1), intent(IN) :: dist
-      TYPE(DECOMP_INFO), intent(IN) :: decomp
+     integer, intent(IN) :: n1, n2, n3
+     complex(mytype), dimension(*), intent(IN) :: in
+     complex(mytype), dimension(n1, n2, n3), intent(OUT) :: out
+     integer, intent(IN) :: iproc
+     integer, dimension(0:iproc - 1), intent(IN) :: dist
+     TYPE(DECOMP_INFO), intent(IN) :: decomp
 
 #if defined(_GPU)
-      attributes(device) :: in
-      integer :: istat
+     attributes(device) :: in
+     integer :: istat
 #endif
 
-      integer :: i, j, k, m, i1, i2, pos
+     integer :: i, j, k, m, i1, i2, pos
 
-      do m = 0, iproc - 1
-         if (m == 0) then
-            i1 = 1
-            i2 = dist(0)
-         else
-            i1 = i2 + 1
-            i2 = i1 + dist(m) - 1
-         end if
+     do m = 0, iproc - 1
+        if (m == 0) then
+           i1 = 1
+           i2 = dist(0)
+        else
+           i1 = i2 + 1
+           i2 = i1 + dist(m) - 1
+        end if
 
 #ifdef EVEN
-         pos = m * decomp%y2count + 1
+        pos = m * decomp%y2count + 1
 #else
-         pos = decomp%y2disp(m) + 1
+        pos = decomp%y2disp(m) + 1
 #endif
 
 #if defined(_GPU)
-         !$acc host_data use_device(out)
-         istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
-         !$acc end host_data
-         if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
+        !$acc host_data use_device(out)
+        istat = cudaMemcpy2D(out(1, i1, 1), n1 * n2, in(pos), n1 * (i2 - i1 + 1), n1 * (i2 - i1 + 1), n3, cudaMemcpyDeviceToDevice)
+        !$acc end host_data
+        if (istat /= 0) call decomp_2d_abort(__FILE__, __LINE__, istat, "cudaMemcpy2D")
 #else
-         do k = 1, n3
-            do j = i1, i2
-               do i = 1, n1
-                  out(i, j, k) = in(pos)
-                  pos = pos + 1
-               end do
-            end do
-         end do
+        do k = 1, n3
+           do j = i1, i2
+              do i = 1, n1
+                 out(i, j, k) = in(pos)
+                 pos = pos + 1
+              end do
+           end do
+        end do
 #endif
-      end do
-
-      return
-   end subroutine mem_merge_zy_complex
+     end do
 
-end submodule d2d_transpose_z_to_y
+     return
+  end subroutine mem_merge_zy_complex

From d7282306e95bdf20c0f11e1987f44c5e0bee4fae Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Sat, 27 May 2023 19:08:43 +0200
Subject: [PATCH 247/436] Increase the size of the log file

The file returned can contain the absolute path and can be relatively long.
---
 src/log.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/log.f90 b/src/log.f90
index dde647d5..55b0694f 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -116,7 +116,7 @@ module subroutine d2d_listing(given_io_unit)
       integer :: io_unit
       integer :: version, subversion, ierror
 #ifdef DEBUG
-      character(len=64) :: fname
+      character(len=512) :: fname
 #endif
 
       ! Output log if needed

From dd4f7cf74465e235f43a21e4a93375a0a55feade Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sun, 28 May 2023 17:09:23 +0100
Subject: [PATCH 248/436] Moving build details to INSTALL

These aren't necessarily needed in the top-level README
---
 INSTALL.md | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 README.md  | 101 ++---------------------------------------------------
 2 files changed, 104 insertions(+), 98 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index cd9ed3e1..49774227 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -139,3 +139,104 @@ clean-decomp:
 	cmake --build $(DECOMP_BUILD_DIR) --target clean
 	rm -f $(DECOMP_INSTALL_DIR)/lib/libdecomp.a
 ```
+
+## Profiling
+
+Profiling can be activated via `cmake` configuration, the recommended approach is to run the initial configuration as follows:
+```
+export caliper_DIR=/path/to/caliper/install/share/cmake/caliper
+export CXX=mpicxx
+cmake -S $path_to_sources -B $path_to_build_directory -DENABLE_PROFILER=caliper
+```
+where `ENABLE_PROFILER` is set to the profiling tool desired, currently supported values are: `caliper`.
+Note that when using `caliper` a C++ compiler is required as indicated in the above command line.
+
+## Miscellaneous
+
+### List of preprocessor variables
+
+#### DEBUG
+
+This variable is automatically added in debug and dev builds. Extra information is printed when it is present.
+
+#### DOUBLE_PREC
+
+When this variable is not present, the library uses single precision. When it is present, the library uses double precision. This preprocessor variable is driven by the CMake on/off variable `DOUBLE_PRECISION`.
+
+#### SAVE_SINGLE
+
+This variable is valid for double precision builds only. When it is present, snapshots are written in single precision. This preprocessor variable is driven by the CMake on/off variable `SINGLE_PRECISION_OUTPUT`.
+
+#### PROFILER
+
+This variable is automatically added when selecting the profiler. It activates the profiling sections of the code.
+
+#### EVEN
+
+This preprocessor variable is not valid for GPU builds. It leads to padded alltoall operations. This preprocessor variable is driven by the CMake on/off variable `EVEN`.
+
+#### OVERWRITE
+
+This variable leads to overwrite the input array when computing FFT. The support of this flag does not always correspond to in-place transforms, depending on the FFT backend selected, as described above. This preprocessor variable is driven by the CMake on/off variable `ENABLE_INPLACE`.
+
+#### HALO_DEBUG
+
+This variable is used to debug the halo operations. This preprocessor variable is driven by the CMake on/off variable `HALO_DEBUG`.
+
+#### _GPU
+
+This variable is automatically added in GPU builds.
+
+#### _NCCL
+
+This variable is valid only for GPU builds. The NVIDIA Collective Communication Library (NCCL) implements multi-GPU and multi-node communication primitives optimized for NVIDIA GPUs and Networking.
+
+## Optional dependencies
+
+### FFTW
+
+The library [fftw](http://www.fftw.org/index.html) can be used as a backend for the FFT engine. The version 3.3.10 was tested, is supported and can be downloaded [here](http://www.fftw.org/download.html). Please note that one should build fftw and decomp2d against the same compilers. For build instructions, please check [here](http://www.fftw.org/fftw3_doc/Installation-on-Unix.html). Below is a suggestion for the compilation of the library in double precision (add `--enable-single` for a single precision build):
+
+```
+wget http://www.fftw.org/fftw-3.3.10.tar.gz
+tar xzf fftw-3.3.10.tar.gz
+mkdir fftw-3.3.10_tmp && cd fftw-3.3.10_tmp
+../fftw-3.3.10/configure --prefix=xxxxxxx/fftw3/fftw-3.3.10_bld --enable-shared
+make -j
+make -j check
+make install
+```
+Please note that the resulting build is not compatible with CMake (https://github.com/FFTW/fftw3/issues/130). As a workaround, one can open the file `/path/to/fftw3/install/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line
+```
+include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")
+```
+
+To build `2decomp&fft` against fftw3, one can provide the package configuration for fftw3 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`. One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. Then either specify on the command line when configuring the build
+```
+cmake -S . -B build -DFFT_Choice=<fftw|fftw_f03> -DFFTW_ROOT=/path/to/fftw3/install
+```
+or modify the build configuration using `ccmake`.
+
+Note the legacy `fftw` interface lacks interface definitions and will fail when stricter compilation flags are used (e.g. when `-DCMAKE_BUILD_TYPE=Dev`) for this it is recommended to use `fftw_f03` which provides proper interfaces.
+
+### Caliper
+
+The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.9.1 was tested and is supported, version 2.8.0 has also been tested and is still expected to work. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
+
+```
+git clone https://github.com/LLNL/Caliper.git caliper_github
+cd caliper_github
+git checkout v2.9.1
+mkdir build && cd build
+cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../caliper_build_2.9.1 -DWITH_FORTRAN=yes -DWITH_MPI=yes -DBUILD_TESTING=yes ../
+make -j
+make test
+make install
+```
+
+After installing Caliper ensure to set `caliper_DIR=/path/to/caliper/install/share/cmake/caliper`.
+Following this the `2decomp-fft` build can be configured to use Caliper profiling as
+```
+cmake -S . -B -DENABLE_PROFILER=caliper
+```
+or by modifying the configuration to set `ENABLE_PROFILER=caliper` via `ccmake`.
diff --git a/README.md b/README.md
index 63a00b0f..552c8c28 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,6 @@
 This README contains basic instructions for building and installing the `2decomp&fft` library, more
 detailed instructions can be found in [INSTALL.md](INSTALL.md).
 
-
 ## Building
 
 The build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 
@@ -66,17 +65,11 @@ Mesh resolution can also be imposed using the parameters `NX`, `NY` and `NZ`.
 
 For the GPU implementation please be aware that it is based on a single MPI rank per GPU. Therefore, to test multiple GPUs, use the maximum number of available GPUs on the system/node and not the maximum number of MPI tasks. 
 
-
 ## Profiling
 
-Profiling can be activated via `cmake` configuration, the recommended approach is to run the initial configuration as follows:
-```
-export caliper_DIR=/path/to/caliper/install/share/cmake/caliper
-export CXX=mpicxx
-cmake -S $path_to_sources -B $path_to_build_directory -DENABLE_PROFILER=caliper
-```
-where `ENABLE_PROFILER` is set to the profiling tool desired, currently supported values are: `caliper`.
-Note that when using `caliper` a C++ compiler is required as indicated in the above command line.
+The `2decomp&fft` library has integrated profiling support via external libraries, see the Profiling
+section of [INSTALL.md](INSTALL.md) for instructions on configuring a profiling build.
+Currently, support for profiling is provided by the `caliper` library.
 
 When the profiling is active, one can tune it before calling `decomp_2d_init` using the subroutine
 `decomp_profiler_prep`. 
@@ -127,99 +120,11 @@ The default value used is `D2D_LOG_TOFILE` for the default build and `D2D_LOG_TO
 
 Before calling `decomp_2d_init`, the external code can modify the variable `decomp_debug` to change the debug level. The user can also modify this variable using the environment variable `DECOMP_2D_DEBUG`. Please note that the environment variable is processed only for debug builds. The expected value for the variable `decomp_debug` is some integer between 0 and 6, bounds included.
 
-### List of preprocessor variables
-
-#### DEBUG
-
-This variable is automatically added in debug and dev builds. Extra information is printed when it is present.
-
-#### DOUBLE_PREC
-
-When this variable is not present, the library uses single precision. When it is present, the library uses double precision. This preprocessor variable is driven by the CMake on/off variable `DOUBLE_PRECISION`.
-
-#### SAVE_SINGLE
-
-This variable is valid for double precision builds only. When it is present, snapshots are written in single precision. This preprocessor variable is driven by the CMake on/off variable `SINGLE_PRECISION_OUTPUT`.
-
-#### PROFILER
-
-This variable is automatically added when selecting the profiler. It activates the profiling sections of the code.
-
-#### EVEN
-
-This preprocessor variable is not valid for GPU builds. It leads to padded alltoall operations. This preprocessor variable is driven by the CMake on/off variable `EVEN`.
-
-#### OVERWRITE
-
-This variable leads to overwrite the input array when computing FFT. The support of this flag does not always correspond to in-place transforms, depending on the FFT backend selected, as described above. This preprocessor variable is driven by the CMake on/off variable `ENABLE_INPLACE`.
-
-#### HALO_DEBUG
-
-This variable is used to debug the halo operations. This preprocessor variable is driven by the CMake on/off variable `HALO_DEBUG`.
-
-#### _GPU
-
-This variable is automatically added in GPU builds.
-
-#### _NCCL
-
-This variable is valid only for GPU builds. The NVIDIA Collective Communication Library (NCCL) implements multi-GPU and multi-node communication primitives optimized for NVIDIA GPUs and Networking.
-
 ### Code formatting
 
 The code is formatted using the `fprettify` program (available via `pip`), to ensure consistency of use there is a script file `scripts/format.sh` which will run `fprettify` across the 2decomp&fft source, you can also use the `format` build target to run the script.
 It is recommended that you should format the code before making a pull request.
 
-## Optional dependencies
-
-### FFTW
-
-The library [fftw](http://www.fftw.org/index.html) can be used as a backend for the FFT engine. The version 3.3.10 was tested, is supported and can be downloaded [here](http://www.fftw.org/download.html). Please note that one should build fftw and decomp2d against the same compilers. For build instructions, please check [here](http://www.fftw.org/fftw3_doc/Installation-on-Unix.html). Below is a suggestion for the compilation of the library in double precision (add `--enable-single` for a single precision build):
-
-```
-wget http://www.fftw.org/fftw-3.3.10.tar.gz
-tar xzf fftw-3.3.10.tar.gz
-mkdir fftw-3.3.10_tmp && cd fftw-3.3.10_tmp
-../fftw-3.3.10/configure --prefix=xxxxxxx/fftw3/fftw-3.3.10_bld --enable-shared
-make -j
-make -j check
-make install
-```
-Please note that the resulting build is not compatible with CMake (https://github.com/FFTW/fftw3/issues/130). As a workaround, one can open the file `/path/to/fftw3/install/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line
-```
-include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")
-```
-
-To build `2decomp&fft` against fftw3, one can provide the package configuration for fftw3 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`. One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. Then either specify on the command line when configuring the build
-```
-cmake -S . -B build -DFFT_Choice=<fftw|fftw_f03> -DFFTW_ROOT=/path/to/fftw3/install
-```
-or modify the build configuration using `ccmake`.
-
-Note the legacy `fftw` interface lacks interface definitions and will fail when stricter compilation flags are used (e.g. when `-DCMAKE_BUILD_TYPE=Dev`) for this it is recommended to use `fftw_f03` which provides proper interfaces.
-
-### Caliper
-
-The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.9.1 was tested and is supported, version 2.8.0 has also been tested and is still expected to work. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
-
-```
-git clone https://github.com/LLNL/Caliper.git caliper_github
-cd caliper_github
-git checkout v2.9.1
-mkdir build && cd build
-cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../caliper_build_2.9.1 -DWITH_FORTRAN=yes -DWITH_MPI=yes -DBUILD_TESTING=yes ../
-make -j
-make test
-make install
-```
-
-After installing Caliper ensure to set `caliper_DIR=/path/to/caliper/install/share/cmake/caliper`.
-Following this the `2decomp-fft` build can be configured to use Caliper profiling as
-```
-cmake -S . -B -DENABLE_PROFILER=caliper
-```
-or by modifying the configuration to set `ENABLE_PROFILER=caliper` via `ccmake`.
-
 ### Versioning
 
 The development of `2decomp&fft` occurs on Github, with release versions on the `main` branch.

From b355ee03901512dff7035212415c2113bce2d029 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 1 Jun 2023 09:47:42 +0100
Subject: [PATCH 249/436] Correct error in reference value used in halo_test

Test was using the X-pencil value which was allocated in the program global scope.

Reference value is now passed explicitly to the error check
---
 examples/halo_test/halo_test.f90 | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 1eb93d85..9344590e 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -349,7 +349,7 @@ subroutine test_div_haloX()
       !$acc end kernels
 
       ! Compute error
-      call check_err(div1, "X")
+      call check_err(div1, div, "X")
 
       deallocate (vh, wh)
 
@@ -416,7 +416,7 @@ subroutine test_div_haloY()
       call transpose_y_to_x(wk2, div2)
 
       ! Compute error
-      call check_err(div2, "Y")
+      call check_err(div2, div, "Y")
 
       deallocate (uh, wh)
 
@@ -484,18 +484,19 @@ subroutine test_div_haloZ()
       call transpose_y_to_x(wk2, div3)
 
       ! Compute error
-      call check_err(div3, "Z")
+      call check_err(div3, div, "Z")
 
       deallocate (uh, vh)
    end subroutine test_div_haloZ
    !=====================================================================
    ! Check the difference between halo and transpose divergence
    !=====================================================================
-   subroutine check_err(divh, pencil)
+   subroutine check_err(divh, divref, pencil)
 
       implicit none
 
       real(mytype), dimension(:, :, :), intent(in) :: divh
+      real(mytype), dimension(:, :, :), intent(in) :: divref
       character(len=*), intent(in) :: pencil
       real(mytype), dimension(:, :, :), allocatable :: tmp
       real(mytype) :: divmag, error
@@ -509,11 +510,11 @@ subroutine check_err(divh, pencil)
       allocate (tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
 
       !$acc kernels default(present)
-      tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - div1(2:xlast, 2:ylast, 2:zlast)
+      tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - divref(2:xlast, 2:ylast, 2:zlast)
       !$acc end kernels
       error = mag(tmp)
       !$acc kernels default(present)
-      tmp(2:xlast, 2:ylast, 2:zlast) = div1(2:xlast, 2:ylast, 2:zlast)
+      tmp(2:xlast, 2:ylast, 2:zlast) = divref(2:xlast, 2:ylast, 2:zlast)
       !$acc end kernels
       divmag = mag(tmp)
 

From 811f7d983665cebee57927d6d49ad0186ab3cef3 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 1 Jun 2023 09:53:44 +0100
Subject: [PATCH 250/436] Make div1,2,3 local to their test subroutines

Prevents accidentally using global values again
---
 examples/halo_test/halo_test.f90 | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 9344590e..3d94645d 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -27,9 +27,9 @@ program halo_test
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
-   real(mytype), allocatable, dimension(:, :, :) :: u1, v1, w1, div1
-   real(mytype), allocatable, dimension(:, :, :) :: u2, v2, w2, div2
-   real(mytype), allocatable, dimension(:, :, :) :: u3, v3, w3, div3
+   real(mytype), allocatable, dimension(:, :, :) :: u1, v1, w1
+   real(mytype), allocatable, dimension(:, :, :) :: u2, v2, w2
+   real(mytype), allocatable, dimension(:, :, :) :: u3, v3, w3
    real(mytype), allocatable, dimension(:, :, :) :: div, wk2, wk3
 
    integer :: i, j, k, ierror, n
@@ -146,9 +146,6 @@ subroutine initialise()
       call alloc_x(v1, global)
       call alloc_x(w1, global)
       call alloc_x(div, global)
-      call alloc_x(div1, global)
-      call alloc_x(div2, global)
-      call alloc_x(div3, global)
 
       call random_seed(size=n)
       allocate (seed(n))
@@ -293,6 +290,7 @@ subroutine test_div_haloX()
 
       implicit none
 
+      real(mytype), allocatable, dimension(:, :, :) :: div1
       real(mytype), allocatable, dimension(:, :, :) :: vh, wh
 #if defined(_GPU)
       attributes(device) :: vh, wh
@@ -307,6 +305,8 @@ subroutine test_div_haloX()
       integer :: jfirst, jlast ! J loop start/end
       integer :: kfirst, klast ! K loop start/end
 
+      call alloc_x(div1, global)
+
       ! Expected sizes
       nx_expected = nx
       ny_expected = xsize(2) + 2
@@ -351,7 +351,7 @@ subroutine test_div_haloX()
       ! Compute error
       call check_err(div1, div, "X")
 
-      deallocate (vh, wh)
+      deallocate (vh, wh, div1)
 
    end subroutine test_div_haloX
 
@@ -361,6 +361,8 @@ end subroutine test_div_haloX
    subroutine test_div_haloY()
 
       implicit none
+
+      real(mytype), allocatable, dimension(:, :, :) :: div2
       real(mytype), allocatable, dimension(:, :, :) :: uh, wh
 #if defined(_GPU)
       attributes(device) :: uh, wh
@@ -375,6 +377,8 @@ subroutine test_div_haloY()
       integer :: jfirst, jlast ! J loop start/end
       integer :: kfirst, klast ! K loop start/end
 
+      call alloc_x(div2, global)
+      
       ! Expected sizes
       nx_expected = ysize(1) + 2
       ny_expected = ny
@@ -418,7 +422,7 @@ subroutine test_div_haloY()
       ! Compute error
       call check_err(div2, div, "Y")
 
-      deallocate (uh, wh)
+      deallocate (uh, wh, div2)
 
    end subroutine test_div_haloY
 
@@ -428,6 +432,9 @@ end subroutine test_div_haloY
    subroutine test_div_haloZ()
 
       implicit none
+
+      real(mytype), allocatable, dimension(:, :, :) :: div3
+      
       real(mytype), allocatable, dimension(:, :, :) :: uh, vh
 #if defined(_GPU)
       attributes(device) :: vh, uh
@@ -442,6 +449,8 @@ subroutine test_div_haloZ()
       integer :: jfirst, jlast ! J loop start/end
       integer :: kfirst, klast ! K loop start/end
 
+      call alloc_x(div3, global)
+
       ! Expected sizes
       nx_expected = zsize(1) + 2
       ny_expected = zsize(2) + 2
@@ -486,7 +495,7 @@ subroutine test_div_haloZ()
       ! Compute error
       call check_err(div3, div, "Z")
 
-      deallocate (uh, vh)
+      deallocate (uh, vh, div3)
    end subroutine test_div_haloZ
    !=====================================================================
    ! Check the difference between halo and transpose divergence

From 5db5744446601d97b6ca5d6b70272b84cbfbdf2e Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 1 Jun 2023 10:46:16 +0100
Subject: [PATCH 251/436] Add openACC data copies

---
 examples/halo_test/halo_test.f90 | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 3d94645d..d45eff16 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -103,7 +103,7 @@ program halo_test
    end if
 
    call initialise()
-   !$acc data copyin(u1,v1,w1) create(u2,v2,w2,u3,v3,w3,div1,div2,div3,wk2,wk3) copy(div)
+   !$acc data copyin(u1,v1,w1) create(u2,v2,w2,u3,v3,w3,wk2,wk3) copy(div)
    call test_div_transpose()
    call test_div_haloX()
    call test_div_haloY()
@@ -333,6 +333,7 @@ subroutine test_div_haloX()
       call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "X:v")
       call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "X:w")
 
+      !$acc data copy(div1)
       !$acc kernels default(present)
       div1(:, :, :) = 0._mytype
       !$acc end kernels
@@ -350,6 +351,7 @@ subroutine test_div_haloX()
 
       ! Compute error
       call check_err(div1, div, "X")
+      !$acc end data
 
       deallocate (vh, wh, div1)
 
@@ -405,6 +407,7 @@ subroutine test_div_haloY()
       call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Y:u")
       call test_halo_size(wh, nx_expected, ny_expected, nz_expected, "Y:w")
 
+      !$acc data copy(div2)
       !$acc kernels default(present)
       do k = kfirst, klast
          do j = jfirst, jlast
@@ -421,6 +424,7 @@ subroutine test_div_haloY()
 
       ! Compute error
       call check_err(div2, div, "Y")
+      !$acc end data
 
       deallocate (uh, wh, div2)
 
@@ -477,6 +481,7 @@ subroutine test_div_haloZ()
       call test_halo_size(uh, nx_expected, ny_expected, nz_expected, "Z:u")
       call test_halo_size(vh, nx_expected, ny_expected, nz_expected, "Z:v")
 
+      !$acc data copy(div3)
       !$acc kernels default(present)
       do j = jfirst, jlast
          do i = ifirst, ilast
@@ -494,6 +499,7 @@ subroutine test_div_haloZ()
 
       ! Compute error
       call check_err(div3, div, "Z")
+      !$acc end data
 
       deallocate (uh, vh, div3)
    end subroutine test_div_haloZ

From 3394db2d617b68d7e9a7f9f82a9bbd4460fcb630 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Fri, 2 Jun 2023 08:58:12 +0100
Subject: [PATCH 252/436] Update adios2_init to new interface

---
 src/io.f90 | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/io.f90 b/src/io.f90
index 6096a1c1..d0d753a5 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -125,7 +125,6 @@ subroutine decomp_2d_io_init()
 
 #ifdef ADIOS2
       integer :: ierror
-      logical :: adios2_debug_mode
       character(len=80) :: config_file = "adios2_config.xml"
 #endif
 
@@ -134,10 +133,7 @@ subroutine decomp_2d_io_init()
       if (decomp_profiler_io) call decomp_profiler_start("io_init")
 #endif
 
-  !! TODO: make this a runtime-option
-      adios2_debug_mode = .true.
-
-      call adios2_init(adios, trim(config_file), decomp_2d_comm, adios2_debug_mode, ierror)
+      call adios2_init(adios, trim(config_file), decomp_2d_comm, ierror)
       if (ierror /= 0) then
          call decomp_2d_abort(__FILE__, __LINE__, ierror, &
                               "Error initialising ADIOS2 - is "//trim(config_file)//" present and valid?")

From 887938756e6dfbb43077fd567c61e73a93bdf24e Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Mon, 5 Jun 2023 06:31:29 +0200
Subject: [PATCH 253/436] Minor update

---
 src/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 1bb364ad..54347ded 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -80,7 +80,7 @@ if (HALO_DEBUG)
 endif()
 
 if (IO_BACKEND MATCHES "adios2")
-  add_definitions("-DADIOS2")
+  target_compile_definitions(decomp2d PUBLIC -DADIOS2)
   target_link_libraries(decomp2d PRIVATE adios2::fortran_mpi adios2::fortran)
 endif (IO_BACKEND MATCHES "adios2")
 

From 078ad59cf64a70849dc22830d7f0e8ffe302b9a4 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 13 Jun 2023 16:23:36 +0100
Subject: [PATCH 254/436] Testing ierr when checking variable%valid in adios2
 is wrong

The variable%valid will be false if the variable is not registered,
however it seems that ADIOS2 will also return a non-zero ierr in this
case. As far as I can tell the sensible approach is to check this value
only if the variable is valid
---
 src/io.f90 | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/io.f90 b/src/io.f90
index d0d753a5..151a131b 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -1427,19 +1427,18 @@ subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplan
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_at_io "//trim(io_name))
       if (io_handle%valid) then
          call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
-         if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
          if (.not. var_handle%valid) then
-          !! New variable
+            !! New variable
             if (nrank == 0) then
                print *, "Registering variable for IO: ", varname
             end if
 
             ! Need to set the ADIOS2 data type
             if (type == kind(0._real64)) then
-             !! Double
+               !! Double
                data_type = adios2_type_dp
             else if (type == kind(0._real32)) then
-             !! Single
+               !! Single
                data_type = adios2_type_real
             else
                call decomp_2d_abort(__FILE__, __LINE__, -1, "Trying to write unknown data type!")
@@ -1452,6 +1451,11 @@ subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplan
                call decomp_2d_abort(__FILE__, __LINE__, ierror, &
                                     "adios2_define_variable, ERROR registering variable "//trim(varname))
             end if
+         else
+            ! This probably can't happen, however if the inquiry to a variable returns a NULL
+            ! pointer an exception is returned in ierr. As the point is to check the existence of
+            ! the variable we are already checking if it is valid or not.
+            if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_inquire_variable "//trim(varname))
          end if
       else
          call decomp_2d_abort(__FILE__, __LINE__, -1, "trying to register variable with invalid IO!")

From 71332eec9c2414a359558bece24ddb2fc41c374d Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 13 Jun 2023 16:27:41 +0100
Subject: [PATCH 255/436] Update io_test to run with ADIOS2

---
 examples/io_test/CMakeLists.txt    |  3 ++
 examples/io_test/adios2_config.xml | 11 +++++++
 examples/io_test/io_test.f90       | 49 +++++++++++++++++++++++++-----
 3 files changed, 56 insertions(+), 7 deletions(-)
 create mode 100644 examples/io_test/adios2_config.xml

diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 8dada8fe..7b0025d5 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -32,6 +32,7 @@ else ()
 endif ()
 # Run the test(s)
 set(run_dir "${test_dir}/io_var_test")
+file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
 message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
@@ -42,6 +43,7 @@ else ()
 endif ()
 # Run the test(s)
 set(run_dir "${test_dir}/io_plane_test")
+file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
 message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
@@ -52,6 +54,7 @@ else ()
 endif ()
 # Run the test(s)
 set(run_dir "${test_dir}/io_bench")
+file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
 message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
diff --git a/examples/io_test/adios2_config.xml b/examples/io_test/adios2_config.xml
new file mode 100644
index 00000000..756f2ce2
--- /dev/null
+++ b/examples/io_test/adios2_config.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0"?>
+<adios-config>
+  <io name="test-io">
+    <engine type="BP4">
+    </engine>
+    <transport type="File">
+      <parameter key="Library" value="fstream"/>
+      <parameter key="ProfileUnits" value="Milliseconds"/>
+    </transport>
+  </io>
+</adios-config>
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index b4f85195..fa956d1b 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -34,10 +34,14 @@ program io_test
 
    real(mytype), parameter :: eps = 1.0E-7_mytype
 
+   character(len=*), parameter :: io_name = "test-io"
+
    integer :: i, j, k, m, ierror
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
 
+   integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
+   
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
@@ -86,6 +90,12 @@ program io_test
 
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
+   call decomp_2d_io_init()
+   call decomp_2d_init_io(io_name)
+   call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
+   
    ! ***** global data *****
    allocate (data1(nx, ny, nz))
    m = 1
@@ -132,16 +142,41 @@ program io_test
    !$acc update self(u2)
    !$acc update self(u3)
    !$acc end data
-
+   
    ! write to disk
-   call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
-   call decomp_2d_write_one(2, u2, '.', 'u2.dat', 0, 'test')
-   call decomp_2d_write_one(3, u3, '.', 'u3.dat', 0, 'test')
+#ifndef ADIOS2
+   if (nrank == 0) then
+      inquire(file="out", exist=dir_exists)
+      if (.not. dir_exists) then
+         call system("mkdir out 2> /dev/null")
+      end if
+   end if
+#endif
+   
+#ifdef ADIOS2
+   call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
+   call decomp_2d_start_io(io_name, "out")
+#endif
+   call decomp_2d_write_one(1, u1, 'out', 'u1.dat', 0, io_name)
+   call decomp_2d_write_one(2, u2, 'out', 'u2.dat', 0, io_name)
+   call decomp_2d_write_one(3, u3, 'out', 'u3.dat', 0, io_name)
+#ifdef ADIOS2
+   call decomp_2d_end_io(io_name, "out")
+   call decomp_2d_close_io(io_name, "out")
+#endif
 
    ! read back to different arrays
-   call decomp_2d_read_one(1, u1b, '.', 'u1.dat', 'test', reduce_prec=.false.)
-   call decomp_2d_read_one(2, u2b, '.', 'u2.dat', 'test', reduce_prec=.false.)
-   call decomp_2d_read_one(3, u3b, '.', 'u3.dat', 'test', reduce_prec=.false.)
+#ifdef ADIOS2
+   call decomp_2d_open_io(io_name, "out", decomp_2d_read_mode)
+   call decomp_2d_start_io(io_name, "out")
+#endif
+   call decomp_2d_read_one(1, u1b, 'out', 'u1.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(2, u2b, 'out', 'u2.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(3, u3b, 'out', 'u3.dat', io_name, reduce_prec=.false.)
+#ifdef ADIOS2
+   call decomp_2d_end_io(io_name, "out")
+   call decomp_2d_close_io(io_name, "out")
+#endif
 
    ! compare
    do k = xstart(3), xend(3)

From 6bb6f82e6575d3604a8372da67b85d7c7ad5ed3e Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 13 Jun 2023 16:33:13 +0100
Subject: [PATCH 256/436] Fix previous commit build failure

Note this breaks io_read
---
 examples/io_test/io_test.f90 | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index fa956d1b..4b582831 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -40,6 +40,10 @@ program io_test
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
 
+#ifndef ADIOS2
+   logical :: dir_exists
+#endif
+
    integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
    
    call MPI_INIT(ierror)
@@ -148,7 +152,7 @@ program io_test
    if (nrank == 0) then
       inquire(file="out", exist=dir_exists)
       if (.not. dir_exists) then
-         call system("mkdir out 2> /dev/null")
+         call execute_command_line("mkdir out 2> /dev/null")
       end if
    end if
 #endif

From 25a1c2a6519d43ece0ec7e4805698b318aef251f Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 13 Jun 2023 16:35:18 +0100
Subject: [PATCH 257/436] Fix io_read in MPIIO

---
 examples/io_test/io_read.f90 | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 8b2ddc1c..9f2f0b9a 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -32,6 +32,8 @@ program io_read
 
    real(mytype), parameter :: eps = 1.0E-7_mytype
 
+   character(len=*), parameter :: io_name = "test-io"
+
    integer :: i, j, k, m, ierror
 
    call MPI_INIT(ierror)
@@ -103,9 +105,9 @@ program io_read
    call alloc_z(u3b, .true.)
 
    ! read back to different arrays
-   call decomp_2d_read_one(1, u1b, '.', 'u1.dat', 'test', reduce_prec=.false.)
-   call decomp_2d_read_one(2, u2b, '.', 'u2.dat', 'test', reduce_prec=.false.)
-   call decomp_2d_read_one(3, u3b, '.', 'u3.dat', 'test', reduce_prec=.false.)
+   call decomp_2d_read_one(1, u1b, 'out', 'u1.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(2, u2b, 'out', 'u2.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(3, u3b, 'out', 'u3.dat', io_name, reduce_prec=.false.)
 
    ! Check against the global data array
    do k = xstart(3), xend(3)

From 437bf02c203f1e241906f6787e2e9b51094126ff Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 13 Jun 2023 16:38:25 +0100
Subject: [PATCH 258/436] Add safety check that the output directory exists for
 the io_read test

---
 examples/io_test/io_read.f90 | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 9f2f0b9a..4eadccb5 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -33,7 +33,10 @@ program io_read
    real(mytype), parameter :: eps = 1.0E-7_mytype
 
    character(len=*), parameter :: io_name = "test-io"
-
+#ifndef ADIOS2
+   logical ::dir_exists
+#endif
+   
    integer :: i, j, k, m, ierror
 
    call MPI_INIT(ierror)
@@ -84,6 +87,15 @@ program io_read
 
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
+#ifndef ADIOS2
+   if (nrank == 0) then
+      inquire(file="out", exist=dir_exists)
+      if (.not. dir_exists) then
+         call decomp_2d_abort(1, "Error, directory 'out' must exist before running io_read test case!")
+      end if
+   end if
+#endif
+
    ! ***** global data *****
    allocate (data1(nx, ny, nz))
    m = 1

From 0d0230b641a5b6d35ee26d67b8d7956f2c322f47 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 13 Jun 2023 16:40:45 +0100
Subject: [PATCH 259/436] io_read passing ADIOS2

---
 examples/io_test/io_read.f90 | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index 4eadccb5..e4281c3f 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -39,6 +39,8 @@ program io_read
    
    integer :: i, j, k, m, ierror
 
+   integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
+
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
@@ -96,6 +98,12 @@ program io_read
    end if
 #endif
 
+   call decomp_2d_io_init()
+   call decomp_2d_init_io(io_name)
+   call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
+
    ! ***** global data *****
    allocate (data1(nx, ny, nz))
    m = 1
@@ -117,9 +125,17 @@ program io_read
    call alloc_z(u3b, .true.)
 
    ! read back to different arrays
+#ifdef ADIOS2
+   call decomp_2d_open_io(io_name, "out", decomp_2d_read_mode)
+   call decomp_2d_start_io(io_name, "out")
+#endif
    call decomp_2d_read_one(1, u1b, 'out', 'u1.dat', io_name, reduce_prec=.false.)
    call decomp_2d_read_one(2, u2b, 'out', 'u2.dat', io_name, reduce_prec=.false.)
    call decomp_2d_read_one(3, u3b, 'out', 'u3.dat', io_name, reduce_prec=.false.)
+#ifdef ADIOS2
+   call decomp_2d_end_io(io_name, "out")
+   call decomp_2d_close_io(io_name, "out")
+#endif
 
    ! Check against the global data array
    do k = xstart(3), xend(3)

From 415d7d83cb5e99bac8d4ec19f0a21bcdc94ba2bb Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 13 Jun 2023 17:22:31 +0100
Subject: [PATCH 260/436] Update io_plane test case to run with ADIOS2

Note there is no read-back of the planes for ADIOS2 as there is
no read_plane subroutine in 2decomp
---
 examples/io_test/io_plane_test.f90 | 44 ++++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 9 deletions(-)

diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 0dac2e7f..54360454 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -36,6 +36,9 @@ program io_plane_test
    integer :: xen1, xen2, xen3
    logical :: found
 
+   character(len=*), parameter :: io_name = "test-io"
+   integer, parameter :: output2D = 0
+   
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
@@ -84,6 +87,18 @@ program io_plane_test
 
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
+   call decomp_2d_io_init()
+   call decomp_2d_init_io(io_name)
+   call decomp_2d_register_variable(io_name, "x_pencil-x_plane.dat", 1, 0, 1, mytype)
+   call decomp_2d_register_variable(io_name, "x_pencil-y_plane.dat", 1, 0, 2, mytype)
+   call decomp_2d_register_variable(io_name, "x_pencil-z_plane.dat", 1, 0, 3, mytype)
+   call decomp_2d_register_variable(io_name, "y_pencil-x_plane.dat", 2, 0, 1, mytype)
+   call decomp_2d_register_variable(io_name, "y_pencil-y_plane.dat", 2, 0, 2, mytype)
+   call decomp_2d_register_variable(io_name, "y_pencil-z_plane.dat", 2, 0, 3, mytype)
+   call decomp_2d_register_variable(io_name, "z_pencil-x_plane.dat", 3, 0, 1, mytype)
+   call decomp_2d_register_variable(io_name, "z_pencil-y_plane.dat", 3, 0, 2, mytype)
+   call decomp_2d_register_variable(io_name, "z_pencil-z_plane.dat", 3, 0, 3, mytype)
+
    ! ***** global data *****
    allocate (data1(nx, ny, nz))
    m = 1
@@ -124,17 +139,27 @@ program io_plane_test
    !$acc update self(u3)
    !$acc end data
    ! X-pencil data
-   call decomp_2d_write_plane(1, u1, 1, nx / 2, '.', 'x_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(1, u1, 2, ny / 2, '.', 'x_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(1, u1, 3, nz / 2, '.', 'x_pencil-z_plane.dat', 'test')
+#ifdef ADIOS2
+   call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
+   call decomp_2d_start_io(io_name, "out")
+#endif
+   call decomp_2d_write_plane(1, u1, 1, nx / 2, 'out', 'x_pencil-x_plane.dat', io_name)
+   call decomp_2d_write_plane(1, u1, 2, ny / 2, 'out', 'x_pencil-y_plane.dat', io_name)
+   call decomp_2d_write_plane(1, u1, 3, nz / 2, 'out', 'x_pencil-z_plane.dat', io_name)
    ! Y-pencil data
-   call decomp_2d_write_plane(2, u2, 2, ny / 2, '.', 'y_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(2, u2, 1, nx / 2, '.', 'y_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(2, u2, 3, nz / 2, '.', 'y_pencil-z_plane.dat', 'test')
+   call decomp_2d_write_plane(2, u2, 2, ny / 2, 'out', 'y_pencil-y_plane.dat', io_name)
+   call decomp_2d_write_plane(2, u2, 1, nx / 2, 'out', 'y_pencil-x_plane.dat', io_name)
+   call decomp_2d_write_plane(2, u2, 3, nz / 2, 'out', 'y_pencil-z_plane.dat', io_name)
    ! Z-pencil data
-   call decomp_2d_write_plane(3, u3, 1, nx / 2, '.', 'z_pencil-x_plane.dat', 'test')
-   call decomp_2d_write_plane(3, u3, 2, ny / 2, '.', 'z_pencil-y_plane.dat', 'test')
-   call decomp_2d_write_plane(3, u3, 3, nz / 2, '.', 'z_pencil-z_plane.dat', 'test')
+   call decomp_2d_write_plane(3, u3, 1, nx / 2, 'out', 'z_pencil-x_plane.dat', io_name)
+   call decomp_2d_write_plane(3, u3, 2, ny / 2, 'out', 'z_pencil-y_plane.dat', io_name)
+   call decomp_2d_write_plane(3, u3, 3, nz / 2, 'out', 'z_pencil-z_plane.dat', io_name)
+#ifdef ADIOS2
+   call decomp_2d_end_io(io_name, "out")
+   call decomp_2d_close_io(io_name, "out")
+#endif
+
+#ifndef ADIOS2
    ! Attemp to read the files
    if (nrank == 0) then
       inquire (iolength=iol) data1(1, 1, 1)
@@ -206,6 +231,7 @@ program io_plane_test
       end if
 
    end if
+#endif
 
    deallocate (u1, u2, u3)
    deallocate (data1)

From 8fcd6595615490361094788679402ffb993b7e15 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 13 Jun 2023 17:27:02 +0100
Subject: [PATCH 261/436] Fix read-back section of io_plane_test for MPIIO

---
 examples/io_test/io_plane_test.f90 | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 54360454..2958ec43 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -165,10 +165,10 @@ program io_plane_test
       inquire (iolength=iol) data1(1, 1, 1)
 
       ! X-plane
-      inquire (file='x_pencil-x_plane.dat', exist=found)
+      inquire (file='./out/x_pencil-x_plane.dat', exist=found)
       if (found) then
          allocate (work(1, ny, nz))
-         open (10, FILE='x_pencil-x_plane.dat', FORM='unformatted', &
+         open (10, FILE='./out/x_pencil-x_plane.dat', FORM='unformatted', &
                ACCESS='DIRECT', RECL=iol)
          m = 1
          do k = 1, nz
@@ -187,10 +187,10 @@ program io_plane_test
       end if
 
       ! Y-plane
-      inquire (file='x_pencil-y_plane.dat', exist=found)
+      inquire (file='./out/x_pencil-y_plane.dat', exist=found)
       if (found) then
          allocate (work(nx, 1, nz))
-         open (10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
+         open (10, FILE='./out/x_pencil-y_plane.dat', FORM='unformatted', &
                ACCESS='DIRECT', RECL=iol)
          m = 1
          do k = 1, nz
@@ -209,10 +209,10 @@ program io_plane_test
       end if
 
       ! Z-plane
-      inquire (file='x_pencil-z_plane.dat', exist=found)
+      inquire (file='./out/x_pencil-z_plane.dat', exist=found)
       if (found) then
          allocate (work(nx, ny, 1))
-         open (10, FILE='x_pencil-z_plane.dat', FORM='unformatted', &
+         open (10, FILE='./out/x_pencil-z_plane.dat', FORM='unformatted', &
                ACCESS='DIRECT', RECL=iol)
          m = 1
          do j = 1, ny

From cd5a34253f86af61844f73d89142f9558e9e4981 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 13 Jun 2023 22:04:23 +0100
Subject: [PATCH 262/436] If compiler wrapper doesn't give MPI name try to use
 -show

As a fallback option, use the -show argument to MPI to try and find
the MPI family.

Default is "Unknown"

Currently this is only important for gfortran > 10 and dev builds
with only OpenMPI supplying the necessary modules to turn up the
warning/error flags fully. If the MPI family is MPICH or Unknown
then less strict flags are used.
---
 cmake/D2D_MPI.cmake                 | 22 ++++++++++++++++++++--
 cmake/compilers/D2D_flags_gnu.cmake |  4 ++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/cmake/D2D_MPI.cmake b/cmake/D2D_MPI.cmake
index c91c798e..d84a75bc 100644
--- a/cmake/D2D_MPI.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -1,6 +1,7 @@
 # MPI CMakeLists
 
 find_package(MPI REQUIRED)
+set(D2D_MPI_FAMILY "Unknown")
 # Stop if there is no MPI_Fortran_Compiler
 if (MPI_Fortran_COMPILER)
     message(STATUS "MPI_Fortran_COMPILER found: ${MPI_Fortran_COMPILER}")
@@ -8,14 +9,31 @@ if (MPI_Fortran_COMPILER)
     # Try to guess the MPI type to adapt compilation flags if necessary
     string(FIND "${MPI_Fortran_COMPILER}" "mpich" pos)
     if(pos GREATER_EQUAL "0")
-      set(FIND_MPICH TRUE)
+      set(D2D_MPI_FAMILY "MPICH")
       message(STATUS "MPI is MPICH type")
     endif()
     string(FIND "${MPI_Fortran_COMPILER}" "openmpi" pos)
     if(pos GREATER_EQUAL "0")
-      set(FIND_OMPI TRUE)
+      set(D2D_MPI_FAMILY "OMPI")
       message(STATUS "MPI is openMPI type")
     endif()
+
+    if (${D2D_MPI_FAMILY} STREQUAL "Unknown")
+      execute_process(COMMAND ${MPI_Fortran_COMPILER} "-show"
+	OUTPUT_VARIABLE mpi_show)
+      
+      string(FIND "${mpi_show}" "openmpi" pos)
+      if(pos GREATER_EQUAL "0")
+	set(D2D_MPI_FAMILY "OMPI")
+      endif()
+
+      string(FIND "${mpi_show}" "mpich" pos)
+      if(pos GREATER_EQUAL "0")
+	set(D2D_MPI_FAMILY "MPICH")
+      endif()
+    endif()
+
+    message(STATUS "MPI Compiler family: ${D2D_MPI_FAMILY}")
 else (MPI_Fortran_COMPILER)
     message(SEND_ERROR "This application cannot compile without MPI")
 endif(MPI_Fortran_COMPILER)
diff --git a/cmake/compilers/D2D_flags_gnu.cmake b/cmake/compilers/D2D_flags_gnu.cmake
index 10f4a88c..81b453ca 100644
--- a/cmake/compilers/D2D_flags_gnu.cmake
+++ b/cmake/compilers/D2D_flags_gnu.cmake
@@ -9,11 +9,11 @@ else (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
 endif (CMAKE_Fortran_COMPILER_VERSION GREATER_EQUAL "10")
 set(D2D_FFLAGS_RELEASE "-O3 -march=native")
 set(D2D_FFLAGS_DEBUG   "-DDEBUG -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none")
-if (FIND_MPICH AND D2D_GNU10)
+if (((${D2D_MPI_FAMILY} STREQUAL "MPICH") OR (${D2D_MPI_FAMILY} STREQUAL "Unknown")) AND D2D_GNU10)
   set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wno-unused-function -Wno-integer-division")
 else()
   set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
 endif()
-if (FIND_OMPI)
+if (${D2D_MPI_FAMILY} STREQUAL "OMPI")
   set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEV} -Wimplicit-procedure -Wimplicit-interface")
 endif()

From ba111cf72140d54696ae0d25ad519d51ccc98659 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 13 Jun 2023 22:15:05 +0100
Subject: [PATCH 263/436] Silence errors when MPI compiler doesn't support
 -show

For example, the Cray compiler wrappers. In this case the MPI
family will be set to "Unknown"
---
 cmake/D2D_MPI.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmake/D2D_MPI.cmake b/cmake/D2D_MPI.cmake
index d84a75bc..3fe581a3 100644
--- a/cmake/D2D_MPI.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -20,7 +20,8 @@ if (MPI_Fortran_COMPILER)
 
     if (${D2D_MPI_FAMILY} STREQUAL "Unknown")
       execute_process(COMMAND ${MPI_Fortran_COMPILER} "-show"
-	OUTPUT_VARIABLE mpi_show)
+	OUTPUT_VARIABLE mpi_show
+	ERROR_QUIET)
       
       string(FIND "${mpi_show}" "openmpi" pos)
       if(pos GREATER_EQUAL "0")

From 247ca0f7ab2c7678ab1a1c0074f6fc2280ae7ce2 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Wed, 14 Jun 2023 05:15:24 +0100
Subject: [PATCH 264/436] Silence Cray warning about multiple module uses
 (#210)

If module A uses MPI (for example) and module B uses modules A and
MPI then this raises a warning. It seems safe to ignore so this
silences it
---
 cmake/compilers/D2D_flags_cray.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/compilers/D2D_flags_cray.cmake b/cmake/compilers/D2D_flags_cray.cmake
index 20b16b50..26dbf71c 100644
--- a/cmake/compilers/D2D_flags_cray.cmake
+++ b/cmake/compilers/D2D_flags_cray.cmake
@@ -1,5 +1,5 @@
 #Compilers Flags for Cray
 
-set(D2D_FFLAGS "-eF -g -N 1023")
+set(D2D_FFLAGS "-eF -g -N 1023 -M878")
 set(D2D_FFLAGS_RELEASE "-O3")
 set(D2D_FFLAGS_DEBUG   "-O0 -g")

From 0c05eebe46dc9aa9f294f33279ec403dca1c6fb2 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Wed, 14 Jun 2023 06:28:05 +0200
Subject: [PATCH 265/436] Remove some double comments and fprettify

---
 examples/io_test/io_plane_test.f90 |  2 +-
 examples/io_test/io_read.f90       |  4 ++--
 examples/io_test/io_test.f90       | 10 +++++-----
 src/io.f90                         | 22 +++++++++++-----------
 4 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 2958ec43..2a6da5bd 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -38,7 +38,7 @@ program io_plane_test
 
    character(len=*), parameter :: io_name = "test-io"
    integer, parameter :: output2D = 0
-   
+
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index e4281c3f..f1254d11 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -36,7 +36,7 @@ program io_read
 #ifndef ADIOS2
    logical ::dir_exists
 #endif
-   
+
    integer :: i, j, k, m, ierror
 
    integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
@@ -91,7 +91,7 @@ program io_read
 
 #ifndef ADIOS2
    if (nrank == 0) then
-      inquire(file="out", exist=dir_exists)
+      inquire (file="out", exist=dir_exists)
       if (.not. dir_exists) then
          call decomp_2d_abort(1, "Error, directory 'out' must exist before running io_read test case!")
       end if
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 4b582831..e1810cb0 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -45,7 +45,7 @@ program io_test
 #endif
 
    integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
-   
+
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
@@ -99,7 +99,7 @@ program io_test
    call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
    call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
    call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
-   
+
    ! ***** global data *****
    allocate (data1(nx, ny, nz))
    m = 1
@@ -146,17 +146,17 @@ program io_test
    !$acc update self(u2)
    !$acc update self(u3)
    !$acc end data
-   
+
    ! write to disk
 #ifndef ADIOS2
    if (nrank == 0) then
-      inquire(file="out", exist=dir_exists)
+      inquire (file="out", exist=dir_exists)
       if (.not. dir_exists) then
          call execute_command_line("mkdir out 2> /dev/null")
       end if
    end if
 #endif
-   
+
 #ifdef ADIOS2
    call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
    call decomp_2d_start_io(io_name, "out")
diff --git a/src/io.f90 b/src/io.f90
index 151a131b..9096ca72 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -1118,7 +1118,7 @@ subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
       end if
 
       if (icoarse == 0) then
-       !! Use full fields
+         ! Use full fields
 
          if (present(opt_decomp)) then
             decomp = opt_decomp
@@ -1242,7 +1242,7 @@ subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_n
       if (decomp_profiler_io) call decomp_profiler_start("mpiio_write_real_coarse")
 #endif
 
-    !! Set defaults
+      ! Set defaults
       write_reduce_prec = .true.
       if (present(opt_deferred_writes)) then
          deferred_writes = opt_deferred_writes
@@ -1265,7 +1265,7 @@ subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_n
       end if
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_SIZE")
 
-    !! Use original MPIIO writers
+      ! Use original MPIIO writers
       if (present(opt_decomp)) then
          call coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
       else
@@ -1428,17 +1428,17 @@ subroutine decomp_2d_register_variable(io_name, varname, ipencil, icoarse, iplan
       if (io_handle%valid) then
          call adios2_inquire_variable(var_handle, io_handle, varname, ierror)
          if (.not. var_handle%valid) then
-            !! New variable
+            ! New variable
             if (nrank == 0) then
                print *, "Registering variable for IO: ", varname
             end if
 
             ! Need to set the ADIOS2 data type
             if (type == kind(0._real64)) then
-               !! Double
+               ! Double
                data_type = adios2_type_dp
             else if (type == kind(0._real32)) then
-               !! Single
+               ! Single
                data_type = adios2_type_real
             else
                call decomp_2d_abort(__FILE__, __LINE__, -1, "Trying to write unknown data type!")
@@ -1828,7 +1828,7 @@ subroutine decomp_2d_open_io(io_name, io_dir, mode)
 
       idx = get_io_idx(io_name, io_dir)
       if (idx < 1) then
-       !! New io destination
+         ! New io destination
          if (nreg_io < MAX_IOH) then
             nreg_io = nreg_io + 1
             do idx = 1, MAX_IOH
@@ -1842,7 +1842,7 @@ subroutine decomp_2d_open_io(io_name, io_dir, mode)
             names_ptr(idx) = full_name
 
             if (mode == decomp_2d_write_mode) then
-             !! Setup writers
+               ! Setup writers
 #ifndef ADIOS2
                filesize = 0_MPI_OFFSET_KIND
                fh_disp(idx) = 0_MPI_OFFSET_KIND
@@ -1851,7 +1851,7 @@ subroutine decomp_2d_open_io(io_name, io_dir, mode)
                access_mode = adios2_mode_write
 #endif
             else if (mode == decomp_2d_read_mode) then
-             !! Setup readers
+               ! Setup readers
 #ifndef ADIOS2
                fh_disp(idx) = 0_MPI_OFFSET_KIND
                access_mode = MPI_MODE_RDONLY
@@ -1871,14 +1871,14 @@ subroutine decomp_2d_open_io(io_name, io_dir, mode)
                stop
             end if
 
-          !! Open IO
+            ! Open IO
 #ifndef ADIOS2
             call MPI_FILE_OPEN(decomp_2d_comm, io_dir, &
                                access_mode, MPI_INFO_NULL, &
                                fh_registry(idx), ierror)
             if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_OPEN")
             if (mode == decomp_2d_write_mode) then
-             !! Guarantee overwriting
+               ! Guarantee overwriting
                call MPI_FILE_SET_SIZE(fh_registry(idx), filesize, ierror)
                if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_SIZE")
             end if

From 6a90549430646f207e51850d2ad50fc95cf1250f Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Wed, 14 Jun 2023 08:50:47 +0100
Subject: [PATCH 266/436] Remove unused variable

---
 examples/io_test/io_plane_test.f90 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 2a6da5bd..b0009400 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -37,7 +37,6 @@ program io_plane_test
    logical :: found
 
    character(len=*), parameter :: io_name = "test-io"
-   integer, parameter :: output2D = 0
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks

From e41f78ccf26b2a580b257773dd28adc6f0a36690 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Thu, 15 Jun 2023 09:57:16 +0100
Subject: [PATCH 267/436] Add missing copy of ADIOS2 config file for io_test

---
 examples/io_test/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 7b0025d5..2f5b5ac6 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -20,6 +20,7 @@ target_link_libraries(io_bench PRIVATE decomp2d)
 
 # Run the test(s)
 set(run_dir "${test_dir}/io_test")
+file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
 message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")

From 81e91e7801d6e07632a4fdb2e6ba08a4306a1cb8 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 20 Jun 2023 09:21:59 +0000
Subject: [PATCH 268/436] Add target link with ADIOS2 for the IO examples

---
 CMakeLists.txt                  |  2 ++
 examples/io_test/CMakeLists.txt | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index cac2651f..3a01f01d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -5,6 +5,8 @@ project(decomp2d
   LANGUAGES Fortran)
 set(version 2.0.0)
 enable_testing()
+# Can be useful to also activate CXX, sometimes is needed by packages
+enable_language(CXX)
 
 
 set(BUILD_TARGET "mpi" CACHE STRING "Target for acceleration (mpi (default) or gpu)")
diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 2f5b5ac6..46a35692 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -13,10 +13,30 @@ add_executable(io_plane_test ${files_plane_test})
 add_executable(io_bench ${files_bench})
 
 target_link_libraries(io_test PRIVATE decomp2d)
+if (IO_BACKEND MATCHES "adios2")
+  target_compile_definitions(io_test PUBLIC -DADIOS2)
+  target_link_libraries(io_test PRIVATE adios2::fortran_mpi adios2::fortran)
+endif (IO_BACKEND MATCHES "adios2")
 target_link_libraries(io_read PRIVATE decomp2d)
+if (IO_BACKEND MATCHES "adios2")
+  target_compile_definitions(io_read PUBLIC -DADIOS2)
+  target_link_libraries(io_read PRIVATE adios2::fortran_mpi adios2::fortran)
+endif (IO_BACKEND MATCHES "adios2")
 target_link_libraries(io_var_test PRIVATE decomp2d)
+if (IO_BACKEND MATCHES "adios2")
+  target_compile_definitions(io_var_test PUBLIC -DADIOS2)
+  target_link_libraries(io_var_test PRIVATE adios2::fortran_mpi adios2::fortran)
+endif (IO_BACKEND MATCHES "adios2")
 target_link_libraries(io_plane_test PRIVATE decomp2d)
+if (IO_BACKEND MATCHES "adios2")
+  target_compile_definitions(io_plane_test PUBLIC -DADIOS2)
+  target_link_libraries(io_plane_test PRIVATE adios2::fortran_mpi adios2::fortran)
+endif (IO_BACKEND MATCHES "adios2")
 target_link_libraries(io_bench PRIVATE decomp2d)
+if (IO_BACKEND MATCHES "adios2")
+  target_compile_definitions(io_bench PUBLIC -DADIOS2)
+  target_link_libraries(io_bench PRIVATE adios2::fortran_mpi adios2::fortran)
+endif (IO_BACKEND MATCHES "adios2")
 
 # Run the test(s)
 set(run_dir "${test_dir}/io_test")

From 6f5f5189c61bb12246d1e7fcab5872dfd326476d Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 20 Jun 2023 09:57:23 +0000
Subject: [PATCH 269/436] Enable C and CXX languages at the beginning of the
 build

---
 CMakeLists.txt      | 2 +-
 cmake/D2D_MPI.cmake | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3a01f01d..6ffed8e8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@ project(decomp2d
 set(version 2.0.0)
 enable_testing()
 # Can be useful to also activate CXX, sometimes is needed by packages
-enable_language(CXX)
+enable_language(C CXX)
 
 
 set(BUILD_TARGET "mpi" CACHE STRING "Target for acceleration (mpi (default) or gpu)")
diff --git a/cmake/D2D_MPI.cmake b/cmake/D2D_MPI.cmake
index 3b675cc3..61a32099 100644
--- a/cmake/D2D_MPI.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -4,7 +4,6 @@ find_package(MPI REQUIRED COMPONENTS Fortran)
 
 # adios2 IO backend requires C and C++ MPI components
 if (IO_BACKEND MATCHES "adios2")
-    enable_language(C CXX)
     find_package(MPI REQUIRED COMPONENTS C CXX)
 endif (IO_BACKEND MATCHES "adios2")
 

From 4f4ebf57ca6dcd79c7632e5e9c8dc5f8fa6eb39e Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 20 Jun 2023 15:05:19 +0000
Subject: [PATCH 270/436] Update target link for ADIOS2 as PUBLIC

---
 examples/io_test/CMakeLists.txt | 20 --------------------
 src/CMakeLists.txt              |  2 +-
 2 files changed, 1 insertion(+), 21 deletions(-)

diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 46a35692..2f5b5ac6 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -13,30 +13,10 @@ add_executable(io_plane_test ${files_plane_test})
 add_executable(io_bench ${files_bench})
 
 target_link_libraries(io_test PRIVATE decomp2d)
-if (IO_BACKEND MATCHES "adios2")
-  target_compile_definitions(io_test PUBLIC -DADIOS2)
-  target_link_libraries(io_test PRIVATE adios2::fortran_mpi adios2::fortran)
-endif (IO_BACKEND MATCHES "adios2")
 target_link_libraries(io_read PRIVATE decomp2d)
-if (IO_BACKEND MATCHES "adios2")
-  target_compile_definitions(io_read PUBLIC -DADIOS2)
-  target_link_libraries(io_read PRIVATE adios2::fortran_mpi adios2::fortran)
-endif (IO_BACKEND MATCHES "adios2")
 target_link_libraries(io_var_test PRIVATE decomp2d)
-if (IO_BACKEND MATCHES "adios2")
-  target_compile_definitions(io_var_test PUBLIC -DADIOS2)
-  target_link_libraries(io_var_test PRIVATE adios2::fortran_mpi adios2::fortran)
-endif (IO_BACKEND MATCHES "adios2")
 target_link_libraries(io_plane_test PRIVATE decomp2d)
-if (IO_BACKEND MATCHES "adios2")
-  target_compile_definitions(io_plane_test PUBLIC -DADIOS2)
-  target_link_libraries(io_plane_test PRIVATE adios2::fortran_mpi adios2::fortran)
-endif (IO_BACKEND MATCHES "adios2")
 target_link_libraries(io_bench PRIVATE decomp2d)
-if (IO_BACKEND MATCHES "adios2")
-  target_compile_definitions(io_bench PUBLIC -DADIOS2)
-  target_link_libraries(io_bench PRIVATE adios2::fortran_mpi adios2::fortran)
-endif (IO_BACKEND MATCHES "adios2")
 
 # Run the test(s)
 set(run_dir "${test_dir}/io_test")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 54347ded..2e80684a 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -81,7 +81,7 @@ endif()
 
 if (IO_BACKEND MATCHES "adios2")
   target_compile_definitions(decomp2d PUBLIC -DADIOS2)
-  target_link_libraries(decomp2d PRIVATE adios2::fortran_mpi adios2::fortran)
+  target_link_libraries(decomp2d PUBLIC adios2::fortran_mpi adios2::fortran)
 endif (IO_BACKEND MATCHES "adios2")
 
 install(TARGETS decomp2d

From 9170109da04411adfd7b72b8965778fafae74d7e Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 22 Jun 2023 10:19:59 +0000
Subject: [PATCH 271/436] Safety check on io_read test pointing to file
 existance rather than directory. Intel return false if inquire function point
 to a directory in any case

---
 examples/io_test/io_read.f90 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index f1254d11..bd9061f9 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -34,7 +34,7 @@ program io_read
 
    character(len=*), parameter :: io_name = "test-io"
 #ifndef ADIOS2
-   logical ::dir_exists
+   logical ::file_exists
 #endif
 
    integer :: i, j, k, m, ierror
@@ -91,8 +91,8 @@ program io_read
 
 #ifndef ADIOS2
    if (nrank == 0) then
-      inquire (file="out", exist=dir_exists)
-      if (.not. dir_exists) then
+      inquire (file="out/u1.dat", exist=file_exists)
+      if (.not. file_exists) then
          call decomp_2d_abort(1, "Error, directory 'out' must exist before running io_read test case!")
       end if
    end if

From fc64650ada3df1c4f2914418276c70fac0c77f04 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 22 Jun 2023 10:37:48 +0000
Subject: [PATCH 272/436] Differentiate the flags between new IntelLLVM and old
 Intel ifort

---
 cmake/compilers/D2D_flags_intel.cmake | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/cmake/compilers/D2D_flags_intel.cmake b/cmake/compilers/D2D_flags_intel.cmake
index 88e39ee5..f502d3af 100644
--- a/cmake/compilers/D2D_flags_intel.cmake
+++ b/cmake/compilers/D2D_flags_intel.cmake
@@ -1,7 +1,12 @@
 # Compilers Flags for Intel
-
-set(D2D_FFLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback")
-set(D2D_FFLAGS_RELEASE "-O3 -ipo")
+# Check is the compiler is the new ifx based on LLVM or the old ifort
+if (Fortran_COMPILER_NAME MATCHES "IntelLLVM")
+  set(D2D_FFLAGS "-fpp -std08")
+  set(D2D_FFLAGS_RELEASE "-O2")
+else (Fortran_COMPILER_NAME MATCHES "IntelLLVM")
+  #set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
+  set(D2D_FFLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback")
+  set(D2D_FFLAGS_RELEASE "-O3 -ipo")
+endif (Fortran_COMPILER_NAME MATCHES "IntelLLVM")
 set(D2D_FFLAGS_DEBUG   "-g -O0 -debug extended -traceback -DDEBUG")
 set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -warn all,noexternal")
-#set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")

From c5503f92382253ddbb6221e4d762faf3ee2c1a73 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 22 Jun 2023 11:09:57 +0000
Subject: [PATCH 273/436] Update find factor function to cast int as double and
 not real. IntelLLVM has an issue to compute sqrt(1.0)

---
 src/factor.f90 | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/factor.f90 b/src/factor.f90
index 941c8378..82b28515 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -32,9 +32,12 @@ subroutine findfactor(num, factors, nfact)
       integer :: i, m
 
       ! find the factors <= sqrt(num)
-      m = int(sqrt(real(num)))
+      ! Cast the int as double to make sure of the correct result of sqrt
+      ! IntelLLVM got an issue with 1.0 but not with 1.d0 
+      m = int(sqrt(real(num,8)))
       nfact = 1
       do i = 1, m
+         print *, 'Resu if ',i,(num / i * i)
          if (num / i * i == num) then
             factors(nfact) = i
             nfact = nfact + 1

From 06ced3ce60f835e868a2a86d7d7a8f670edbf2f8 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 22 Jun 2023 11:12:50 +0000
Subject: [PATCH 274/436] Activate C and CXX languages only in case of ADIOS2.

---
 CMakeLists.txt | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6ffed8e8..ec6f4e5b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -5,8 +5,10 @@ project(decomp2d
   LANGUAGES Fortran)
 set(version 2.0.0)
 enable_testing()
-# Can be useful to also activate CXX, sometimes is needed by packages
-enable_language(C CXX)
+if (IO_BACKEND MATCHES "adios2")
+  # Can be useful to also activate CXX, sometimes is needed by packages
+  enable_language(C CXX)
+endif (IO_BACKEND MATCHES "adios2")
 
 
 set(BUILD_TARGET "mpi" CACHE STRING "Target for acceleration (mpi (default) or gpu)")

From dd6da963a77758b56cd1d7d66c3626cf40548a0a Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Fri, 23 Jun 2023 08:02:10 +0000
Subject: [PATCH 275/436] Remove unnecessary print

---
 src/factor.f90 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/factor.f90 b/src/factor.f90
index 82b28515..8f4060ae 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -37,7 +37,6 @@ subroutine findfactor(num, factors, nfact)
       m = int(sqrt(real(num,8)))
       nfact = 1
       do i = 1, m
-         print *, 'Resu if ',i,(num / i * i)
          if (num / i * i == num) then
             factors(nfact) = i
             nfact = nfact + 1

From 09e1892d3deaad80c39ba49cd1753248082fd7b7 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 23 Jun 2023 10:19:15 +0100
Subject: [PATCH 276/436] Extend output check for io_read test

Check that all required data exists.
---
 examples/io_test/io_read.f90 | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/examples/io_test/io_read.f90 b/examples/io_test/io_read.f90
index bd9061f9..bc05b379 100644
--- a/examples/io_test/io_read.f90
+++ b/examples/io_test/io_read.f90
@@ -34,7 +34,7 @@ program io_read
 
    character(len=*), parameter :: io_name = "test-io"
 #ifndef ADIOS2
-   logical ::file_exists
+   logical ::file_exists1, file_exists2, file_exists3
 #endif
 
    integer :: i, j, k, m, ierror
@@ -91,9 +91,11 @@ program io_read
 
 #ifndef ADIOS2
    if (nrank == 0) then
-      inquire (file="out/u1.dat", exist=file_exists)
-      if (.not. file_exists) then
-         call decomp_2d_abort(1, "Error, directory 'out' must exist before running io_read test case!")
+      inquire (file="out/u1.dat", exist=file_exists1)
+      inquire (file="out/u2.dat", exist=file_exists2)
+      inquire (file="out/u3.dat", exist=file_exists3)
+      if (.not. (file_exists1 .and. file_exists2 .and. file_exists3)) then
+         call decomp_2d_abort(1, "Error, data 'out/u<1,2,3>.dat' must exist before running io_read test case!")
       end if
    end if
 #endif

From 5da5371e805fec25cfd99235db72167ed9bede7d Mon Sep 17 00:00:00 2001
From: rfj82982 <35581400+rfj82982@users.noreply.github.com>
Date: Fri, 23 Jun 2023 13:00:57 +0100
Subject: [PATCH 277/436] Update INSTALL.md to include installation information
 for Intel

---
 INSTALL.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/INSTALL.md b/INSTALL.md
index 7ae22d91..49fd5097 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -163,6 +163,22 @@ Note that when using `caliper` a C++ compiler is required as indicated in the ab
 
 ## Miscellaneous
 
+### Compiling with Intel oneAPI
+In order to compile with the MKL libraries the following environmental variable needs to be set up
+```
+export MKL_DIR=${MKLROOT}/lib/cmake/mkl
+```
+and activate set the `FFT_Choice=mkl`
+To use the new IntelLLVM compiler you can 
+```
+export export FC="mpiifort -fc=ifx"
+```
+and in case of compilatio with ADIOS2 also the `C` and `CXX` compilers needs to be set-up
+```
+export CXX="mpiicpc -cxx=icpx"
+export CC="mpiicc -cc=icx"
+```
+
 ### List of preprocessor variables
 
 #### ADIOS2

From 7ad1e9abed914355e37a2fc0f6c84bcab91124ce Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 23 Jun 2023 13:58:00 +0100
Subject: [PATCH 278/436] Update INSTALL.md

---
 INSTALL.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 49fd5097..dc9a13db 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -164,16 +164,18 @@ Note that when using `caliper` a C++ compiler is required as indicated in the ab
 ## Miscellaneous
 
 ### Compiling with Intel oneAPI
+
 In order to compile with the MKL libraries the following environmental variable needs to be set up
 ```
 export MKL_DIR=${MKLROOT}/lib/cmake/mkl
 ```
-and activate set the `FFT_Choice=mkl`
-To use the new IntelLLVM compiler you can 
+and select the MKL backend by setting `FFT_Choice=mkl`.
+
+To use the new IntelLLVM compiler specify it as the Fortran compiler using
 ```
 export export FC="mpiifort -fc=ifx"
 ```
-and in case of compilatio with ADIOS2 also the `C` and `CXX` compilers needs to be set-up
+and when building with ADIOS2 support you must also specify the `C` and `CXX` compilers
 ```
 export CXX="mpiicpc -cxx=icpx"
 export CC="mpiicc -cc=icx"

From 89f196d2b5a179f277c2d98f37aeada6ad0965c5 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 23 Jun 2023 16:04:44 +0100
Subject: [PATCH 279/436] Try using a different NVHPC version

---
 .github/workflows/Build_all.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 06af7445..238a6713 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -106,7 +106,8 @@ jobs:
     - name: install
       run: |
         sudo apt-get update  -y
-        sudo apt-get install -y nvhpc-22-7
+        sudo apt-get install -y nvhpc-23-latest
+###        sudo apt-get install -y nvhpc-22-7
 ###        sudo apt-get install -y nvhpc-22-7-cuda-multi
 
     # Runs a set of commands using the runners shell

From 1a83f9046c4a5b7a5f1eaaf1c7745c8b617a614a Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 23 Jun 2023 16:11:20 +0100
Subject: [PATCH 280/436] Try a different value of nvhpc

---
 .github/workflows/Build_all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 238a6713..8443b8ff 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -106,7 +106,7 @@ jobs:
     - name: install
       run: |
         sudo apt-get update  -y
-        sudo apt-get install -y nvhpc-23-latest
+        sudo apt-get install -y nvhpc
 ###        sudo apt-get install -y nvhpc-22-7
 ###        sudo apt-get install -y nvhpc-22-7-cuda-multi
 

From bdc13fbce6ce6328a83d3d5922a64e38130c7e51 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Fri, 23 Jun 2023 16:34:32 +0100
Subject: [PATCH 281/436] Update Build_all.yml

Try to pin NVHPC on 23.5 (seems to be what apt-get install nvhpc uses)
---
 .github/workflows/Build_all.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 8443b8ff..41446f06 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -91,7 +91,8 @@ jobs:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
     env:
-      TMP: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/mpi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin
+      TMP: /opt/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/mpi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.5/compilers/bin
+###      TMP: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/mpi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -106,7 +107,7 @@ jobs:
     - name: install
       run: |
         sudo apt-get update  -y
-        sudo apt-get install -y nvhpc
+        sudo apt-get install -y nvhpc-23-5
 ###        sudo apt-get install -y nvhpc-22-7
 ###        sudo apt-get install -y nvhpc-22-7-cuda-multi
 

From d827f8e1fb16071c71d11d9a81a7d50f5c28bda2 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Sat, 24 Jun 2023 10:55:15 +0100
Subject: [PATCH 282/436] Fix issue 214 adding the implicit procedure/interface
 only with generic FFT

---
 CMakeLists.txt                      | 7 ++++---
 cmake/compilers/D2D_flags_gnu.cmake | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ec6f4e5b..ac6ef502 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -70,9 +70,6 @@ endif (BUILD_TARGET MATCHES "gpu")
 
 include(D2D_MPI)
 
-include(D2D_Compilers)
-include(D2D_Profilers)
-
 # FFT options
 if (BUILD_TARGET MATCHES "gpu")
   set(FFT_Choice "cufft" CACHE STRING "FFT for XCompact3d project (with GPU cufft is the default)")
@@ -80,6 +77,10 @@ else ()
   set(FFT_Choice "generic" CACHE STRING "FFT for XCompact3d project (generic is the default)")
 endif (BUILD_TARGET MATCHES "gpu")
 set_property(CACHE FFT_Choice PROPERTY STRINGS generic fftw fftw_f03 mkl cufft)
+
+# Set now compilers flags
+include(D2D_Compilers)
+include(D2D_Profilers)
 include(fft)
 
 # Create a static library for the fft
diff --git a/cmake/compilers/D2D_flags_gnu.cmake b/cmake/compilers/D2D_flags_gnu.cmake
index 81b453ca..9e0ba0f2 100644
--- a/cmake/compilers/D2D_flags_gnu.cmake
+++ b/cmake/compilers/D2D_flags_gnu.cmake
@@ -14,6 +14,6 @@ if (((${D2D_MPI_FAMILY} STREQUAL "MPICH") OR (${D2D_MPI_FAMILY} STREQUAL "Unknow
 else()
   set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG} -Wall -Wpedantic -Wno-unused-function -Werror -Wno-integer-division")
 endif()
-if (${D2D_MPI_FAMILY} STREQUAL "OMPI")
+if ((${D2D_MPI_FAMILY} STREQUAL "OMPI") AND (${FFT_Choice} MATCHES "generic"))
   set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEV} -Wimplicit-procedure -Wimplicit-interface")
 endif()

From ddb19674ab9928bd4abd22d9a1cc2c6ca3caba16 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 26 Jun 2023 11:59:36 +0000
Subject: [PATCH 283/436] First Test for GitLab CI/CD

---
 .gitlab-ci.yml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 .gitlab-ci.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 00000000..6385f651
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,23 @@
+# Defines stages which are to be executed
+stages:
+  - API_Build
+
+# Stage "build"
+build:GNU:API:
+  stage: API_Build
+  tags:
+    - Test_CCP
+  script:
+    - module list
+    - echo $CI_PROJECT_DIR
+      #- module use /opt/nvidia/hpc_sdk/modulefiles
+      #- module load nvhpc/22.11
+      #- mkdir build
+      #- cd build
+      #- cmake ../  -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL
+      #- cmake --build . -j 4
+      #- cmake --install .
+  artifacts:
+    expire_in: 1 week
+    paths:
+    - $CI_PROJECT_DIR

From 58826dc410102ada52c0684442a52f928038771f Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 3 Jul 2023 11:10:31 +0000
Subject: [PATCH 284/436] Remove the GitLab YML from main

---
 .gitlab-ci.yml | 23 -----------------------
 1 file changed, 23 deletions(-)
 delete mode 100644 .gitlab-ci.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
deleted file mode 100644
index 6385f651..00000000
--- a/.gitlab-ci.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-# Defines stages which are to be executed
-stages:
-  - API_Build
-
-# Stage "build"
-build:GNU:API:
-  stage: API_Build
-  tags:
-    - Test_CCP
-  script:
-    - module list
-    - echo $CI_PROJECT_DIR
-      #- module use /opt/nvidia/hpc_sdk/modulefiles
-      #- module load nvhpc/22.11
-      #- mkdir build
-      #- cd build
-      #- cmake ../  -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL
-      #- cmake --build . -j 4
-      #- cmake --install .
-  artifacts:
-    expire_in: 1 week
-    paths:
-    - $CI_PROJECT_DIR

From 4e8e6763bbc676086988beccb7e70dd048ff71ef Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 06:05:06 +0000
Subject: [PATCH 285/436] Add YML file for local runner

---
 .github/workflows/Build_Runner.yml | 37 ++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 .github/workflows/Build_Runner.yml

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
new file mode 100644
index 00000000..924d7efd
--- /dev/null
+++ b/.github/workflows/Build_Runner.yml
@@ -0,0 +1,37 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Runner_test
+
+# Controls when the action will run. Triggers the workflow on pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ main ]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a job called "bld_gnu_ompi"
+  bld_nvhpc_ompi:
+    # The type of runner that the job will run on
+    runs-on: self-hosted
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - uses: actions/checkout@v3
+
+    # Runs a single command using the runners shell
+    - name: Install requirements
+      run: |
+        module avail 
+        module load nvhpc
+        module list
+
+    # Runs a set of commands using the runners shell
+    - name: Compile
+      run: |
+        export FC=mpif90
+        cmake -S . -B build -DBUILD_TESTING=ON
+        cmake --build build --target decomp2d
+        cmake --install build 
+

From bd8511aae3afa69a9c5c2a36c8180040ac98286d Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 06:12:06 +0000
Subject: [PATCH 286/436] Update Runnel YML file

---
 .github/workflows/Build_Runner.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 924d7efd..2e163a17 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -4,9 +4,7 @@ name: Runner_test
 
 # Controls when the action will run. Triggers the workflow on pull request
 # events but only for the master branch
-on:
-  push:
-    branches: [ main ]
+on: [push, fork]
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:

From 62ff812af9f61df58b2eb3dab7c5be70781803c9 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 09:12:14 +0000
Subject: [PATCH 287/436] Test Build_Runner.yml

---
 .github/workflows/Build_Runner.yml | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 2e163a17..5f672f81 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -21,15 +21,17 @@ jobs:
     # Runs a single command using the runners shell
     - name: Install requirements
       run: |
-        module avail 
-        module load nvhpc
-        module list
+        env
+        ls -ltr ~/.bashrc
+        #module avail 
+        #module load nvhpc
+        #module list
 
-    # Runs a set of commands using the runners shell
-    - name: Compile
-      run: |
-        export FC=mpif90
-        cmake -S . -B build -DBUILD_TESTING=ON
-        cmake --build build --target decomp2d
-        cmake --install build 
+        ## Runs a set of commands using the runners shell
+        #- name: Compile
+        #  run: |
+        #    export FC=mpif90
+        #    cmake -S . -B build -DBUILD_TESTING=ON
+        #    cmake --build build --target decomp2d
+        #    cmake --install build 
 

From 37a8d6387bc9d4dd4be602b8b781e4059a8ff350 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 09:16:16 +0000
Subject: [PATCH 288/436] Add sourcing of local .bashrc

---
 .github/workflows/Build_Runner.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 5f672f81..0c4b0663 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -23,7 +23,8 @@ jobs:
       run: |
         env
         ls -ltr ~/.bashrc
-        #module avail 
+        source ~/.bashrc
+        module avail 
         #module load nvhpc
         #module list
 

From dfe681a8df114e3dafa7b880ff6f745d2a5dd4b8 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 09:19:19 +0000
Subject: [PATCH 289/436] Add sourcing of local .bashrc

---
 .github/workflows/Build_Runner.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 0c4b0663..579b5f4d 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -24,7 +24,10 @@ jobs:
         env
         ls -ltr ~/.bashrc
         source ~/.bashrc
-        module avail 
+        cd ~
+        ls *
+        . ./.bashrc
+        #module avail 
         #module load nvhpc
         #module list
 

From e9db7d4bb9a1ecca8d7d75f3650ebd2d0141730d Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 09:20:26 +0000
Subject: [PATCH 290/436] Add sourcing of local .bashrc test 2

---
 .github/workflows/Build_Runner.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 579b5f4d..a6da53a1 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -27,6 +27,7 @@ jobs:
         cd ~
         ls *
         . ./.bashrc
+        cd -
         #module avail 
         #module load nvhpc
         #module list

From 7a96601788d06eedf4d4d6a4e24e4c170f55e119 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 09:44:24 +0000
Subject: [PATCH 291/436] Add sourcing of local .bashrc test 3

---
 .github/workflows/Build_Runner.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index a6da53a1..3d0ba799 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -23,7 +23,6 @@ jobs:
       run: |
         env
         ls -ltr ~/.bashrc
-        source ~/.bashrc
         cd ~
         ls *
         . ./.bashrc

From 1f07b9583eafcec89aee6be0560055ad11960d26 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 10:10:44 +0000
Subject: [PATCH 292/436] Add sourcing of local .bashrc test 4

---
 .github/workflows/Build_Runner.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 3d0ba799..9b318566 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -2,6 +2,10 @@
 
 name: Runner_test
 
+defaults:
+  run:
+    shell: bash -ieo pipefail {0}
+
 # Controls when the action will run. Triggers the workflow on pull request
 # events but only for the master branch
 on: [push, fork]

From 058ad1809ecd720da44463316b34e5452611ec0e Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 10:23:28 +0000
Subject: [PATCH 293/436] Add sourcing of local .bashrc test 5

---
 .github/workflows/Build_Runner.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 9b318566..8c6e4a4f 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -4,7 +4,7 @@ name: Runner_test
 
 defaults:
   run:
-    shell: bash -ieo pipefail {0}
+    shell: bash 
 
 # Controls when the action will run. Triggers the workflow on pull request
 # events but only for the master branch

From fe5f1f9124b97dce287151cea05ee1b6810f1fb6 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 10:31:20 +0000
Subject: [PATCH 294/436] Move the YML file to use git actions

---
 .github/workflows/Build_Runner.yml | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 8c6e4a4f..68c5c537 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -25,21 +25,23 @@ jobs:
     # Runs a single command using the runners shell
     - name: Install requirements
       run: |
-        env
-        ls -ltr ~/.bashrc
-        cd ~
-        ls *
-        . ./.bashrc
-        cd -
+        sudo apt-get update
+        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev cmake
+        #env
+        #ls -ltr ~/.bashrc
+        #cd ~
+        #ls *
+        #. ./.bashrc
+        #cd -
         #module avail 
         #module load nvhpc
         #module list
 
-        ## Runs a set of commands using the runners shell
-        #- name: Compile
-        #  run: |
-        #    export FC=mpif90
-        #    cmake -S . -B build -DBUILD_TESTING=ON
-        #    cmake --build build --target decomp2d
-        #    cmake --install build 
+        # Runs a set of commands using the runners shell
+        - name: Compile
+          run: |
+            export FC=mpif90
+            cmake -S . -B build -DBUILD_TESTING=ON
+            cmake --build build --target decomp2d
+            cmake --install build 
 

From 55ee28bf9564f3f7e409b2b2618e7ebdb85b4b23 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 11:07:12 +0000
Subject: [PATCH 295/436] Move the YML file to use git actions test 2

---
 .github/workflows/Build_Runner.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 68c5c537..88c5132b 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -26,7 +26,10 @@ jobs:
     - name: Install requirements
       run: |
         sudo apt-get update
-        sudo apt-get install -y gfortran openmpi-bin libopenmpi-dev cmake
+        sudo apt-get install -y gfortran 
+        sudo apt-get install -y openmpi-bin 
+        sudo apt-get install -y libopenmpi-dev 
+        sudo apt-get install -y cmake
         #env
         #ls -ltr ~/.bashrc
         #cd ~

From 2b5dbf22bfad15f57cc01c9b2d613401bd99fc39 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 12:48:38 +0000
Subject: [PATCH 296/436] Move the YML file to use git actions test 6

---
 .github/workflows/Build_Runner.yml | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 88c5132b..c8c1e456 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -15,13 +15,15 @@ jobs:
   # This workflow contains a job called "bld_gnu_ompi"
   bld_nvhpc_ompi:
     # The type of runner that the job will run on
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, x64]
+
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v3
 
+
     # Runs a single command using the runners shell
     - name: Install requirements
       run: |
@@ -40,11 +42,12 @@ jobs:
         #module load nvhpc
         #module list
 
-        # Runs a set of commands using the runners shell
-        - name: Compile
-          run: |
-            export FC=mpif90
-            cmake -S . -B build -DBUILD_TESTING=ON
-            cmake --build build --target decomp2d
-            cmake --install build 
+    # Runs a set of commands using the runners shell
+    - name: Compile
+      run: |
+        which cmake
+        export FC=mpif90
+        cmake -S . -B build -DBUILD_TESTING=ON
+        cmake --build build --target decomp2d
+        cmake --install build 
 

From 026751a53fa07419fb5d5115137c857ab29bfa59 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 12:54:34 +0000
Subject: [PATCH 297/436] Move the YML file to use git actions test 7

---
 .github/workflows/Build_Runner.yml | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index c8c1e456..4e315680 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -27,13 +27,15 @@ jobs:
     # Runs a single command using the runners shell
     - name: Install requirements
       run: |
-        sudo apt-get update
-        sudo apt-get install -y gfortran 
-        sudo apt-get install -y openmpi-bin 
-        sudo apt-get install -y libopenmpi-dev 
-        sudo apt-get install -y cmake
+              #sudo apt-get update
+              #sudo apt-get install -y gfortran 
+              #sudo apt-get install -y openmpi-bin 
+              #sudo apt-get install -y libopenmpi-dev 
+              #sudo apt-get install -y cmake
         #env
-        #ls -ltr ~/.bashrc
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        module avail 
         #cd ~
         #ls *
         #. ./.bashrc

From 04cf22e41b0f50f613be15648a7ba8e33b45ff51 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 12:56:28 +0000
Subject: [PATCH 298/436] Bug correction

---
 .github/workflows/Build_Runner.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 4e315680..e85b5dea 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -27,11 +27,6 @@ jobs:
     # Runs a single command using the runners shell
     - name: Install requirements
       run: |
-              #sudo apt-get update
-              #sudo apt-get install -y gfortran 
-              #sudo apt-get install -y openmpi-bin 
-              #sudo apt-get install -y libopenmpi-dev 
-              #sudo apt-get install -y cmake
         #env
         ls -ltr ~/.bashrc
         source ~/.bashrc
@@ -53,3 +48,8 @@ jobs:
         cmake --build build --target decomp2d
         cmake --install build 
 
+        #sudo apt-get update
+        #sudo apt-get install -y gfortran 
+        #sudo apt-get install -y openmpi-bin 
+        #sudo apt-get install -y libopenmpi-dev 
+        #sudo apt-get install -y cmake

From 3d34a1dcc725a57534976ad976ae106e6d595795 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 13:16:01 +0000
Subject: [PATCH 299/436] Test the global bash

---
 .github/workflows/Build_Runner.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index e85b5dea..b583136a 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -29,6 +29,7 @@ jobs:
       run: |
         #env
         ls -ltr ~/.bashrc
+        source /etc/bash.bashrc 
         source ~/.bashrc
         module avail 
         #cd ~

From 8ca5c031036aa7b0593361006bcc7b48e795a4d7 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 13:22:11 +0000
Subject: [PATCH 300/436] Source profile instead of bashrc

---
 .github/workflows/Build_Runner.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index b583136a..f31807d8 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -29,8 +29,8 @@ jobs:
       run: |
         #env
         ls -ltr ~/.bashrc
-        source /etc/bash.bashrc 
-        source ~/.bashrc
+        source /etc/profile 
+        source ~/.profile
         module avail 
         #cd ~
         #ls *

From 333c58fdf1e1e27f0010cb8116f7b66f120e00d1 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 13:26:11 +0000
Subject: [PATCH 301/436] Add messages to source profile

---
 .github/workflows/Build_Runner.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index f31807d8..744451b1 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -29,9 +29,12 @@ jobs:
       run: |
         #env
         ls -ltr ~/.bashrc
-        source /etc/profile 
+        echo "Start source" 
+        source /etc/profile
+        echo "Ok source profile" 
         source ~/.profile
-        module avail 
+        echo "Ok source local .profile" 
+        #module avail 
         #cd ~
         #ls *
         #. ./.bashrc

From 955102f89767da381370e6a28fe4f31a8657edfb Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 13:32:08 +0000
Subject: [PATCH 302/436] Add LMOD command

---
 .github/workflows/Build_Runner.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 744451b1..8b458bb5 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -29,6 +29,7 @@ jobs:
       run: |
         #env
         ls -ltr ~/.bashrc
+        echo $LMOD_CMD
         echo "Start source" 
         source /etc/profile
         echo "Ok source profile" 

From acf14d26ae665c1c5b9fb03a2fb56fbae24fcbe0 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 13:43:41 +0000
Subject: [PATCH 303/436] Add direct invoke of LMOD

---
 .github/workflows/Build_Runner.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 8b458bb5..98ce152f 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -29,12 +29,12 @@ jobs:
       run: |
         #env
         ls -ltr ~/.bashrc
-        echo $LMOD_CMD
         echo "Start source" 
-        source /etc/profile
+        source ~/.bashrc
         echo "Ok source profile" 
-        source ~/.profile
+        source ~/.profile_github
         echo "Ok source local .profile" 
+        echo $LMOD_CMD
         #module avail 
         #cd ~
         #ls *

From 65a4019c0ae0ef8da6eabc2d1e25d93815e1e013 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 13:50:21 +0000
Subject: [PATCH 304/436] Try with new local github profile

---
 .github/workflows/Build_Runner.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 98ce152f..00857d49 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -32,6 +32,7 @@ jobs:
         echo "Start source" 
         source ~/.bashrc
         echo "Ok source profile" 
+        echo "Ok source profile" 
         source ~/.profile_github
         echo "Ok source local .profile" 
         echo $LMOD_CMD

From 0f1e292b75f8576c99b347281a0ce425cdc45c0d Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 14:40:44 +0000
Subject: [PATCH 305/436] Update the module

---
 .github/workflows/Build_Runner.yml | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 00857d49..a17f8690 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -29,21 +29,13 @@ jobs:
       run: |
         #env
         ls -ltr ~/.bashrc
-        echo "Start source" 
         source ~/.bashrc
-        echo "Ok source profile" 
-        echo "Ok source profile" 
         source ~/.profile_github
-        echo "Ok source local .profile" 
         echo $LMOD_CMD
-        #module avail 
-        #cd ~
-        #ls *
-        #. ./.bashrc
-        #cd -
-        #module avail 
-        #module load nvhpc
-        #module list
+        module avail 
+        module avail 
+        module load nvhpc
+        module list
 
     # Runs a set of commands using the runners shell
     - name: Compile

From 828a813f83fe6b287e898aaa4f873c6a8d0a813c Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 14:47:02 +0000
Subject: [PATCH 306/436] Load NVHPC within runner

---
 .github/workflows/Build_Runner.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index a17f8690..008d7257 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -33,14 +33,14 @@ jobs:
         source ~/.profile_github
         echo $LMOD_CMD
         module avail 
-        module avail 
         module load nvhpc
         module list
 
     # Runs a set of commands using the runners shell
     - name: Compile
       run: |
-        which cmake
+        module avail 
+        module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
         cmake --build build --target decomp2d

From dc6e51f39b99eff5c7882195662c61953d087792 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 14:54:41 +0000
Subject: [PATCH 307/436] Add all testing in single run

---
 .github/workflows/Build_Runner.yml | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 008d7257..adcc2b66 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -25,29 +25,26 @@ jobs:
 
 
     # Runs a single command using the runners shell
-    - name: Install requirements
+    - name: Compile NHVHPC
       run: |
         #env
         ls -ltr ~/.bashrc
         source ~/.bashrc
         source ~/.profile_github
         echo $LMOD_CMD
-        module avail 
-        module load nvhpc
-        module list
-
-    # Runs a set of commands using the runners shell
-    - name: Compile
-      run: |
         module avail 
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
         cmake --build build --target decomp2d
-        cmake --install build 
+        ctest --test-dir build 
+
+        ## Runs a set of commands using the runners shell
+        #- name: Compile
+        #  run: |
 
-        #sudo apt-get update
-        #sudo apt-get install -y gfortran 
-        #sudo apt-get install -y openmpi-bin 
-        #sudo apt-get install -y libopenmpi-dev 
-        #sudo apt-get install -y cmake
+        #    #sudo apt-get update
+        #    #sudo apt-get install -y gfortran 
+        #    #sudo apt-get install -y openmpi-bin 
+        #    #sudo apt-get install -y libopenmpi-dev 
+        #    #sudo apt-get install -y cmake

From 090c3b16fa61486b45b191e0c5771c700473a90f Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 16:18:00 +0000
Subject: [PATCH 308/436] Use the correct CMake in CI and update policy to have
 at least CMake 3.20

---
 .github/workflows/Build_Runner.yml | 3 ++-
 CMakeLists.txt                     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index adcc2b66..511979e8 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -32,7 +32,8 @@ jobs:
         source ~/.bashrc
         source ~/.profile_github
         echo $LMOD_CMD
-        module avail 
+        module avail
+        module load CMake 
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ac6ef502..ddda700c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.12)                        
+cmake_minimum_required(VERSION 3.21)                        
 cmake_policy(SET CMP0074 NEW)
 
 project(decomp2d

From 21a52274f65de64bdbfc3f5d77702a56acf1a0a4 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 16:21:55 +0000
Subject: [PATCH 309/436] Update CMake building

---
 .github/workflows/Build_Runner.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 511979e8..93a1d83f 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -37,7 +37,8 @@ jobs:
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
-        cmake --build build --target decomp2d
+        cmake --build build -j 4
+        cmake --install build 
         ctest --test-dir build 
 
         ## Runs a set of commands using the runners shell

From 5a104437395266cf3b735c190cbee403f509571c Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 17:20:44 +0000
Subject: [PATCH 310/436] Add parallel test on larger mesh

---
 .github/workflows/Build_Runner.yml | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 93a1d83f..6af75634 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -1,5 +1,4 @@
 # This is a basic workflow to help you get started with Actions
-
 name: Runner_test
 
 defaults:
@@ -17,17 +16,14 @@ jobs:
     # The type of runner that the job will run on
     runs-on: [self-hosted, linux, x64]
 
-
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v3
 
-
-    # Runs a single command using the runners shell
-    - name: Compile NHVHPC
+    # Run and Compile NHHPC for CPU
+    - name: Compile NHVHPC CPU
       run: |
-        #env
         ls -ltr ~/.bashrc
         source ~/.bashrc
         source ~/.profile_github
@@ -40,13 +36,9 @@ jobs:
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
 
-        ## Runs a set of commands using the runners shell
-        #- name: Compile
-        #  run: |
 
-        #    #sudo apt-get update
-        #    #sudo apt-get install -y gfortran 
-        #    #sudo apt-get install -y openmpi-bin 
-        #    #sudo apt-get install -y libopenmpi-dev 
-        #    #sudo apt-get install -y cmake

From ad80fea0db052689c4f647e9c9c0959ddeee7fd7 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 4 Jul 2023 17:30:05 +0000
Subject: [PATCH 311/436] Add GPU build for NVHPC

---
 .github/workflows/Build_Runner.yml | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 6af75634..4f893340 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -20,6 +20,8 @@ jobs:
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v3
+      with: 
+        clean: true
 
     # Run and Compile NHHPC for CPU
     - name: Compile NHVHPC CPU
@@ -27,8 +29,6 @@ jobs:
         ls -ltr ~/.bashrc
         source ~/.bashrc
         source ~/.profile_github
-        echo $LMOD_CMD
-        module avail
         module load CMake 
         module load nvhpc
         export FC=mpif90
@@ -39,6 +39,26 @@ jobs:
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
+    
+    # Run and Compile NHHPC for GPU & NCCL
+    - name: Compile NHVHPC GPU NCCL
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load nvhpc
+        export FC=mpif90
+        cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes
+        cmake --build build -j 4
+        cmake --install build 
         ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
 
 

From 73a802ef294c9d926db02a5a506dc8918473113d Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Fri, 7 Jul 2023 11:15:12 +0200
Subject: [PATCH 312/436] Use pointers to share real and complex CPU transpose
 buffers (#218)

* Use pointers to share real and complex transpose buffers

* Nullify pointers and deallocate arrays only if needed

* Limit the extension of the real transpose buffer for CPUs
---
 src/decomp_2d.f90          | 41 ++++++++++++++++++--------------------
 src/decomp_2d_init_fin.f90 |  7 ++++++-
 2 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 7bf849f7..60aa72d9 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -123,8 +123,11 @@ module decomp_2d
 
    ! These are the buffers used by MPI_ALLTOALL(V) calls
    integer, save :: decomp_buf_size = 0
-   real(mytype), allocatable, dimension(:) :: work1_r, work2_r
-   complex(mytype), allocatable, dimension(:) :: work1_c, work2_c
+   ! Shared real/complex buffers
+   real(mytype), target, allocatable, dimension(:) :: work1, work2
+   ! Real/complex pointers to buffers
+   real(mytype), pointer, contiguous, dimension(:) :: work1_r, work2_r
+   complex(mytype), pointer, contiguous, dimension(:) :: work1_c, work2_c
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    ! To define smaller arrays using every several mesh points
@@ -357,6 +360,8 @@ end function get_decomp_dims
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    subroutine decomp_info_init(nx, ny, nz, decomp)
 
+      use, intrinsic:: iso_c_binding, only: c_f_pointer, c_loc
+
       implicit none
 
       integer, intent(IN) :: nx, ny, nz
@@ -414,43 +419,35 @@ subroutine decomp_info_init(nx, ny, nz, decomp)
 #endif
 
       ! check if additional memory is required
-      ! *** TODO: consider how to share the real/complex buffers
       if (buf_size > decomp_buf_size) then
          decomp_buf_size = buf_size
 #if defined(_GPU)
          call decomp_2d_cumpi_init(buf_size)
 #endif
-         if (allocated(work1_r)) deallocate (work1_r)
-         if (allocated(work2_r)) deallocate (work2_r)
-         if (allocated(work1_c)) deallocate (work1_c)
-         if (allocated(work2_c)) deallocate (work2_c)
-         allocate (work1_r(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work2_r(buf_size), STAT=status)
+         if (associated(work1_r)) nullify (work1_r)
+         if (associated(work2_r)) nullify (work2_r)
+         if (associated(work1_c)) nullify (work1_c)
+         if (associated(work2_c)) nullify (work2_c)
+         if (allocated(work1)) deallocate (work1)
+         if (allocated(work2)) deallocate (work2)
+         allocate (work1(2 * buf_size), STAT=status)
          if (status /= 0) then
             errorcode = 2
             call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
                                  'Out of memory when allocating 2DECOMP workspace')
          end if
-         allocate (work1_c(buf_size), STAT=status)
-         if (status /= 0) then
-            errorcode = 2
-            call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
-                                 'Out of memory when allocating 2DECOMP workspace')
-         end if
-         allocate (work2_c(buf_size), STAT=status)
+         allocate (work2(2 * buf_size), STAT=status)
          if (status /= 0) then
             errorcode = 2
             call decomp_2d_abort(__FILE__, __LINE__, errorcode, &
                                  'Out of memory when allocating 2DECOMP workspace')
          end if
+         call c_f_pointer(c_loc(work1), work1_r, [buf_size])
+         call c_f_pointer(c_loc(work2), work2_r, [buf_size])
+         call c_f_pointer(c_loc(work1), work1_c, [buf_size])
+         call c_f_pointer(c_loc(work2), work2_c, [buf_size])
       end if
 
-      return
    end subroutine decomp_info_init
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index dd220a95..1294d473 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -192,7 +192,12 @@ subroutine decomp_2d_finalize_ref
      call decomp_info_finalize(decomp_main)
 
      decomp_buf_size = 0
-     deallocate (work1_r, work2_r, work1_c, work2_c)
+     if (associated(work1_r)) nullify (work1_r)
+     if (associated(work2_r)) nullify (work2_r)
+     if (associated(work1_c)) nullify (work1_c)
+     if (associated(work2_c)) nullify (work2_c)
+     if (allocated(work1)) deallocate (work1)
+     if (allocated(work2)) deallocate (work2)
 #if defined(_GPU)
      call decomp_2d_cumpi_fin()
 #if defined(_NCCL)

From b6b2c701ecd1aae7a90c2c3fd5594bd85813221f Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 10 Jul 2023 11:17:46 +0000
Subject: [PATCH 313/436] CICD for NVHPC covering all options

---
 .github/workflows/Build_Runner.yml | 61 ++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 4f893340..a69f40eb 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -42,6 +42,48 @@ jobs:
         ctest --test-dir build
         rm -rf build  
     
+    # Run and Compile NHHPC for CPU with ADIOS2
+    # Need to fix issue with latest NVHPC and ADIOS
+    # Need to revert to 23.1
+    - name: Compile NHVHPC CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load nvhpc/23.1
+        export FC=mpif90
+        export CC=mpicc
+        export CXX=mpixcc
+        cmake -S . -B build -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_nvhpc_cpu/opt/lib/cmake/adios2
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
+    
+    # Run and Compile NHHPC for GPU & cuMPI
+    - name: Compile NHVHPC GPU cuMPI
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load nvhpc
+        export FC=mpif90
+        cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=no
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
+    
     # Run and Compile NHHPC for GPU & NCCL
     - name: Compile NHVHPC GPU NCCL
       run: |
@@ -60,5 +102,24 @@ jobs:
         cmake --install build 
         ctest --test-dir build
         rm -rf build  
+    
+    # Run and Compile NHHPC for GPU & NCCL & Memory Managment
+    - name: Compile NHVHPC GPU NCCL MemManag
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load nvhpc
+        export FC=mpif90
+        cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes -DENABLE_MANAGED=yes
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
 
 

From e200f61f7e0031c9fa0d3c92873b1d5cf8e85e68 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 10 Jul 2023 11:21:29 +0000
Subject: [PATCH 314/436] CICD for NVHPC: bug correction on ADIOS2 build

---
 .github/workflows/Build_Runner.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index a69f40eb..3397becd 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -45,7 +45,7 @@ jobs:
     # Run and Compile NHHPC for CPU with ADIOS2
     # Need to fix issue with latest NVHPC and ADIOS
     # Need to revert to 23.1
-    - name: Compile NHVHPC CPU
+    - name: Compile NHVHPC CPU & ADIOS2
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
@@ -54,7 +54,7 @@ jobs:
         module load nvhpc/23.1
         export FC=mpif90
         export CC=mpicc
-        export CXX=mpixcc
+        export CXX=mpicxx
         cmake -S . -B build -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_nvhpc_cpu/opt/lib/cmake/adios2
         cmake --build build -j 4
         cmake --install build 

From aa9b0412d4acc9f52a8180acd1d9c8e67f7d0852 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 10 Jul 2023 11:45:30 +0000
Subject: [PATCH 315/436] CICD for GNU with FFTW

---
 .github/workflows/Build_Runner.yml | 55 ++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 3397becd..f88d789a 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -122,4 +122,59 @@ jobs:
         ctest --test-dir build
         rm -rf build  
 
+    # Run and Compile with GCC openMPI and FFTW
+    - name: Compile GNU openMPI FFTW CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load foss
+        export FC=mpif90
+        cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
 
+    # Run and Compile with GCC openMPI and FFTW
+    - name: Compile GNU openMPI FFTW_F03 CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load foss
+        export FC=mpif90
+        cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
+    
+    # Run and Compile with GCC openMPI and FFTW
+    - name: Compile GNU_Dev openMPI FFTW_F03 CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load foss
+        export FC=mpif90
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  

From b67281ed1e0298524e1ca6a73175e5312ae8a609 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 10 Jul 2023 12:03:33 +0000
Subject: [PATCH 316/436] CICD for GNU with FFTW: add ADIOS2

---
 .github/workflows/Build_Runner.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index f88d789a..f68fc4ed 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -12,7 +12,7 @@ on: [push, fork]
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   # This workflow contains a job called "bld_gnu_ompi"
-  bld_nvhpc_ompi:
+  Full_CI_workflow:
     # The type of runner that the job will run on
     runs-on: [self-hosted, linux, x64]
 
@@ -161,7 +161,7 @@ jobs:
         rm -rf build  
     
     # Run and Compile with GCC openMPI and FFTW
-    - name: Compile GNU_Dev openMPI FFTW_F03 CPU
+    - name: Compile GNU_Dev openMPI FFTW_F03 ADIOS2 CPU
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
@@ -169,7 +169,9 @@ jobs:
         module load CMake 
         module load foss
         export FC=mpif90
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
+        export CC=mpicc
+        export CXX=mpicxx
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_gcc12_cpu/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build 

From e24ed9be6a1d571ddcefc23ccd4a42dcf9c6bd21 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 10 Jul 2023 15:12:34 +0000
Subject: [PATCH 317/436] CICD for GNU with FFTW: add Caliper

---
 .github/workflows/Build_Runner.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index f68fc4ed..469c97bf 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -158,6 +158,28 @@ jobs:
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build
+        rm -rf build
+
+    # Run and Compile with GCC openMPI and FFTW
+    - name: Compile GNU openMPI FFTW_F03 Caliper CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load foss
+        export FC=mpif90
+        export CC=mpicc
+        export CXX=mpicxx
+        export caliper_DIR=~/GIT_Work/caliper_github/build_gnu/opt/share/cmake/caliper/
+        cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03 -DENABLE_PROFILER=caliper
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
         rm -rf build  
     
     # Run and Compile with GCC openMPI and FFTW

From dcf4220695c53293a0334603aa4c214a4e36a2a2 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 10 Jul 2023 15:32:00 +0000
Subject: [PATCH 318/436] CICD for Intel and MKL

---
 .github/workflows/Build_Runner.yml | 40 ++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index 469c97bf..a2c9843e 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -202,3 +202,43 @@ jobs:
         cmake --install build 
         ctest --test-dir build
         rm -rf build  
+    
+    # Run and Compile with GCC openMPI and FFTW
+    - name: Compile Intel MKL CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load intel
+        export FC=mpiifort
+        export MKL_DIR=${MKLROOT}/lib/cmake/mkl
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
+    
+    # Run and Compile with GCC openMPI and FFTW
+    - name: Compile InteilLLVM MKL CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load intel
+        export export FC="mpiifort -fc=ifx"
+        export MKL_DIR=${MKLROOT}/lib/cmake/mkl
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  

From fdb8ee76f411e4d4af7acaa4572e2c083b267249 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 13 Jul 2023 10:40:24 +0000
Subject: [PATCH 319/436] Add ADIOS2 with IntelLLVM

---
 .github/workflows/Build_Runner.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index a2c9843e..c01d2701 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -23,7 +23,7 @@ jobs:
       with: 
         clean: true
 
-    # Run and Compile NHHPC for CPU
+    # Configure-Build-Run-Run on 4 cores 
     - name: Compile NHVHPC CPU
       run: |
         ls -ltr ~/.bashrc
@@ -65,7 +65,7 @@ jobs:
         ctest --test-dir build
         rm -rf build  
     
-    # Run and Compile NHHPC for GPU & cuMPI
+    # Configure-Build-Run-Run on 4 cores 
     - name: Compile NHVHPC GPU cuMPI
       run: |
         ls -ltr ~/.bashrc
@@ -84,7 +84,7 @@ jobs:
         ctest --test-dir build
         rm -rf build  
     
-    # Run and Compile NHHPC for GPU & NCCL
+    # Configure-Build-Run 
     - name: Compile NHVHPC GPU NCCL
       run: |
         ls -ltr ~/.bashrc
@@ -103,7 +103,7 @@ jobs:
         ctest --test-dir build
         rm -rf build  
     
-    # Run and Compile NHHPC for GPU & NCCL & Memory Managment
+    # Configure-Build-Run 
     - name: Compile NHVHPC GPU NCCL MemManag
       run: |
         ls -ltr ~/.bashrc
@@ -122,7 +122,7 @@ jobs:
         ctest --test-dir build
         rm -rf build  
 
-    # Run and Compile with GCC openMPI and FFTW
+    # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU openMPI FFTW CPU
       run: |
         ls -ltr ~/.bashrc
@@ -141,7 +141,7 @@ jobs:
         ctest --test-dir build
         rm -rf build  
 
-    # Run and Compile with GCC openMPI and FFTW
+    # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU openMPI FFTW_F03 CPU
       run: |
         ls -ltr ~/.bashrc
@@ -160,7 +160,7 @@ jobs:
         ctest --test-dir build
         rm -rf build
 
-    # Run and Compile with GCC openMPI and FFTW
+    # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU openMPI FFTW_F03 Caliper CPU
       run: |
         ls -ltr ~/.bashrc
@@ -182,7 +182,7 @@ jobs:
         ctest --test-dir build
         rm -rf build  
     
-    # Run and Compile with GCC openMPI and FFTW
+    # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU_Dev openMPI FFTW_F03 ADIOS2 CPU
       run: |
         ls -ltr ~/.bashrc
@@ -203,7 +203,7 @@ jobs:
         ctest --test-dir build
         rm -rf build  
     
-    # Run and Compile with GCC openMPI and FFTW
+    # Configure-Build-Run-Run on 4 cores 
     - name: Compile Intel MKL CPU
       run: |
         ls -ltr ~/.bashrc
@@ -223,7 +223,7 @@ jobs:
         ctest --test-dir build
         rm -rf build  
     
-    # Run and Compile with GCC openMPI and FFTW
+    # Configure-Build-Run-Run on 4 cores 
     - name: Compile InteilLLVM MKL CPU
       run: |
         ls -ltr ~/.bashrc

From f1dfe1590b2dfb25d9a9d270bcf3b1b45107dab6 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 13 Jul 2023 10:54:52 +0000
Subject: [PATCH 320/436] Add ADIOS2 with IntelLLVM: correct addition of the
 full YML

---
 .github/workflows/Build_Runner.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/Build_Runner.yml
index c01d2701..7f50282d 100644
--- a/.github/workflows/Build_Runner.yml
+++ b/.github/workflows/Build_Runner.yml
@@ -241,4 +241,26 @@ jobs:
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build
+        rm -rf build 
+
+    # Configure-Build-Run-Run on 4 cores 
+    - name: Compile InteilLLVM MKL ADIOS2 CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load intel
+        export export FC="mpiifort -fc=ifx"
+        export CXX="mpiicpc -cxx=icpx"
+        export CC="mpiicc -cc=icx"
+        export MKL_DIR=${MKLROOT}/lib/cmake/mkl
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_ifx_cpu/opt/lib/cmake/adios2 -DFFT_Choice=mkl
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
         rm -rf build  

From 32863210e014b6cb0729e58a1a58cadf69cb34b1 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 13 Jul 2023 11:13:28 +0000
Subject: [PATCH 321/436] Rename of the Runner YML file

---
 .github/workflows/{Build_Runner.yml => RunnerFullCI.yml} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .github/workflows/{Build_Runner.yml => RunnerFullCI.yml} (100%)

diff --git a/.github/workflows/Build_Runner.yml b/.github/workflows/RunnerFullCI.yml
similarity index 100%
rename from .github/workflows/Build_Runner.yml
rename to .github/workflows/RunnerFullCI.yml

From fc9c194b3db3e4b11dbd528c3c9071266bb2b1dd Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 13 Jul 2023 11:17:44 +0000
Subject: [PATCH 322/436] Perform the full CI only on pull request

---
 .github/workflows/RunnerFullCI.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index 7f50282d..f1703e75 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -7,7 +7,7 @@ defaults:
 
 # Controls when the action will run. Triggers the workflow on pull request
 # events but only for the master branch
-on: [push, fork]
+on: [pull_request, fork]
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:

From 6785acc77d0d6216e3d704be407e91666d67ed31 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 13 Jul 2023 11:24:20 +0000
Subject: [PATCH 323/436] Add fast CI for Configure/Build/Test on PR

---
 .github/workflows/RunnerFastCI.yml | 104 +++++++++++++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100644 .github/workflows/RunnerFastCI.yml

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
new file mode 100644
index 00000000..620ed577
--- /dev/null
+++ b/.github/workflows/RunnerFastCI.yml
@@ -0,0 +1,104 @@
+# This is a basic workflow to help you get started with Actions
+name: Runner_test
+
+defaults:
+  run:
+    shell: bash 
+
+# Controls when the action will run. Triggers the workflow on pull request
+# events but only for the master branch
+on: [push, fork]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a job called "bld_gnu_ompi"
+  Full_CI_workflow:
+    # The type of runner that the job will run on
+    runs-on: [self-hosted, linux, x64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - uses: actions/checkout@v3
+      with: 
+        clean: true
+
+    # Configure-Build-Run-Run on 4 cores 
+    - name: Compile NHVHPC CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load nvhpc
+        export FC=mpif90
+        cmake -S . -B build -DBUILD_TESTING=ON
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
+    
+    # Configure-Build-Run 
+    - name: Compile NHVHPC GPU NCCL
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load nvhpc
+        export FC=mpif90
+        cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
+    
+    # Configure-Build-Run-Run on 4 cores 
+    - name: Compile GNU_Dev openMPI FFTW_F03 ADIOS2 CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load foss
+        export FC=mpif90
+        export CC=mpicc
+        export CXX=mpicxx
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_gcc12_cpu/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build  
+    
+    # Configure-Build-Run-Run on 4 cores 
+    - name: Compile InteilLLVM MKL CPU
+      run: |
+        ls -ltr ~/.bashrc
+        source ~/.bashrc
+        source ~/.profile_github
+        module load CMake 
+        module load intel
+        export export FC="mpiifort -fc=ifx"
+        export MKL_DIR=${MKLROOT}/lib/cmake/mkl
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build 
+        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake --build build -j 4
+        cmake --install build 
+        ctest --test-dir build
+        rm -rf build 
+

From 00c74ca1e487205e0220d2854db1c6242ddf838a Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Thu, 13 Jul 2023 11:26:50 +0000
Subject: [PATCH 324/436] Give different names to the workflows running on
 Runner

---
 .github/workflows/RunnerFastCI.yml | 2 +-
 .github/workflows/RunnerFullCI.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index 620ed577..3193553c 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -1,5 +1,5 @@
 # This is a basic workflow to help you get started with Actions
-name: Runner_test
+name: FastCI4Push
 
 defaults:
   run:
diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index f1703e75..b989bfd7 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -1,5 +1,5 @@
 # This is a basic workflow to help you get started with Actions
-name: Runner_test
+name: FullCI4PR
 
 defaults:
   run:

From c2b51412dfb96879c400ef40c80eb46efbbd2036 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 17 Jul 2023 09:39:01 +0100
Subject: [PATCH 325/436] Set D2D_RELEASE to .true.

---
 src/decomp_2d_constants.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 6944878c..2cf74a9d 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -115,7 +115,7 @@ module decomp_2d_constants
    !
    integer, parameter :: D2D_MAJOR = 2
    integer, parameter :: D2D_MINOR = 0
-   logical, parameter :: D2D_RELEASE = .false.
+   logical, parameter :: D2D_RELEASE = .true.
 
 end module decomp_2d_constants
 

From a9572a0f831295331ea47f966b6c8bcae1c48b81 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 22 Jul 2023 17:38:54 +0100
Subject: [PATCH 326/436] Move checking to subroutine in io_test

---
 examples/io_test/io_test.f90 | 121 +++++++++++++++++++++--------------
 1 file changed, 73 insertions(+), 48 deletions(-)

diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index e1810cb0..eeddd3d8 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -183,59 +183,84 @@ program io_test
 #endif
 
    ! compare
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) stop 1
-         end do
-      end do
-   end do
+   call check("write solution")
+   
+   deallocate (u1, u2, u3)
+   deallocate (u1b, u2b, u3b)
+   deallocate (data1)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
 
-   do k = ystart(3), yend(3)
-      do j = ystart(2), yend(2)
-         do i = ystart(1), yend(1)
-            if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 2
-         end do
-      end do
-   end do
+ contains
 
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            if (abs((u3(i, j, k) - u3b(i, j, k))) > eps) stop 3
-         end do
-      end do
-   end do
+   subroutine check(stage)
 
-   ! Also check against the global data array
-   do k = xstart(3), xend(3)
-      do j = xstart(2), xend(2)
-         do i = xstart(1), xend(1)
-            if (abs(data1(i, j, k) - u1b(i, j, k)) > eps) stop 4
-         end do
-      end do
-   end do
+     character(len=*), intent(in) :: stage
+     
+     integer :: ierr
+     
+     if (nrank == 0) then
+        print *, "Checking "//stage
+     end if
+     call MPI_Barrier(MPI_COMM_WORLD, ierr)
 
-   do k = ystart(3), yend(3)
-      do j = ystart(2), yend(2)
-         do i = ystart(1), yend(1)
-            if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
-         end do
-      end do
-   end do
+     do k = xstart(3), xend(3)
+        do j = xstart(2), xend(2)
+           do i = xstart(1), xend(1)
+              if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) then
+                 print *, u1(i, j, k), u1b(i, j, k)
+                 stop 1
+              end if
+           end do
+        end do
+     end do
 
-   do k = zstart(3), zend(3)
-      do j = zstart(2), zend(2)
-         do i = zstart(1), zend(1)
-            if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
-         end do
-      end do
-   end do
+     do k = ystart(3), yend(3)
+        do j = ystart(2), yend(2)
+           do i = ystart(1), yend(1)
+              if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 2
+           end do
+        end do
+     end do
+     
+     do k = zstart(3), zend(3)
+        do j = zstart(2), zend(2)
+           do i = zstart(1), zend(1)
+              if (abs((u3(i, j, k) - u3b(i, j, k))) > eps) stop 3
+           end do
+        end do
+     end do
 
-   deallocate (u1, u2, u3)
-   deallocate (u1b, u2b, u3b)
-   deallocate (data1)
-   call decomp_2d_finalize
-   call MPI_FINALIZE(ierror)
+     ! Also check against the global data array
+     do k = xstart(3), xend(3)
+        do j = xstart(2), xend(2)
+           do i = xstart(1), xend(1)
+              if (abs(data1(i, j, k) - u1b(i, j, k)) > eps) stop 4
+           end do
+        end do
+     end do
+     
+     do k = ystart(3), yend(3)
+        do j = ystart(2), yend(2)
+           do i = ystart(1), yend(1)
+              if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
+           end do
+        end do
+     end do
+
+     do k = zstart(3), zend(3)
+        do j = zstart(2), zend(2)
+           do i = zstart(1), zend(1)
+              if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
+           end do
+        end do
+     end do
 
+     call MPI_Barrier(MPI_COMM_WORLD, ierr)
+     if (nrank == 0) then
+        print *, "Checking "//stage//" pass!"
+     end if
+     
+   end subroutine check
+   
 end program io_test

From 355a25855d7193fd3d7d90cdeb7bf9f585f28fe0 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 22 Jul 2023 17:55:26 +0100
Subject: [PATCH 327/436] Add multifield read/write test (for checkpointing)

---
 examples/io_test/io_test.f90 | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index eeddd3d8..46b59efa 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -20,6 +20,8 @@ program io_test
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
+   integer :: ierr
+   
 #ifdef COMPLEX_TEST
    complex(mytype), allocatable, dimension(:, :, :) :: data1
 
@@ -157,6 +159,7 @@ program io_test
    end if
 #endif
 
+   ! Standard I/O pattern - file per field
 #ifdef ADIOS2
    call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
    call decomp_2d_start_io(io_name, "out")
@@ -183,7 +186,35 @@ program io_test
 #endif
 
    ! compare
-   call check("write solution")
+   call check("file per field")
+
+   ! Checkpoint I/O pattern - multiple fields per file
+   call decomp_2d_open_io(io_name, "checkpoint", decomp_2d_write_mode)
+   call decomp_2d_start_io(io_name, "checkpoint")
+   call decomp_2d_write_one(1, u1, 'checkpoint', 'u1.dat', 0, io_name)
+   call decomp_2d_write_one(2, u2, 'checkpoint', 'u2.dat', 0, io_name)
+   call decomp_2d_write_one(3, u3, 'checkpoint', 'u3.dat', 0, io_name)
+   call decomp_2d_end_io(io_name, "checkpoint")
+   call decomp_2d_close_io(io_name, "checkpoint")
+
+   print *, "Write complete"
+   call MPI_Barrier(MPI_COMM_WORLD, ierr)
+
+   ! read back to different arrays
+   u1b = 0; u2b = 0; u3b = 0
+   call decomp_2d_open_io(io_name, "checkpoint", decomp_2d_read_mode)
+   call decomp_2d_start_io(io_name, "checkpoint")
+   call decomp_2d_read_one(1, u1b, 'checkpoint', 'u1.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(2, u2b, 'checkpoint', 'u2.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(3, u3b, 'checkpoint', 'u3.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_end_io(io_name, "checkpoint")
+   call decomp_2d_close_io(io_name, "checkpoint")
+
+   print *, "Read complete"
+   call MPI_Barrier(MPI_COMM_WORLD, ierr)
+
+   ! compare
+   call check("one file, multiple fields")
    
    deallocate (u1, u2, u3)
    deallocate (u1b, u2b, u3b)

From 2fe0b86798433cec0a19bc3ab470d35853de2976 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 22 Jul 2023 18:13:22 +0100
Subject: [PATCH 328/436] Update the mutifield per file (checkpoint) test to
 run under ADIOS2

Note that we need to switch our io handle to reset the steps when
reading back in
---
 examples/io_test/adios2_config.xml |  8 ++++++++
 examples/io_test/io_test.f90       | 23 +++++++++++++----------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/examples/io_test/adios2_config.xml b/examples/io_test/adios2_config.xml
index 756f2ce2..92ece049 100644
--- a/examples/io_test/adios2_config.xml
+++ b/examples/io_test/adios2_config.xml
@@ -8,4 +8,12 @@
       <parameter key="ProfileUnits" value="Milliseconds"/>
     </transport>
   </io>
+  <io name="restart-io">
+    <engine type="BP4">
+    </engine>
+    <transport type="File">
+      <parameter key="Library" value="fstream"/>
+      <parameter key="ProfileUnits" value="Milliseconds"/>
+    </transport>
+  </io>
 </adios-config>
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 46b59efa..9558125e 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -37,6 +37,7 @@ program io_test
    real(mytype), parameter :: eps = 1.0E-7_mytype
 
    character(len=*), parameter :: io_name = "test-io"
+   character(len=*), parameter :: io_restart = "restart-io"
 
    integer :: i, j, k, m, ierror
    integer :: xst1, xst2, xst3
@@ -101,6 +102,10 @@ program io_test
    call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
    call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
    call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
+   call decomp_2d_init_io(io_restart)
+   call decomp_2d_register_variable(io_restart, "u1.dat", 1, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_restart, "u2.dat", 2, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_restart, "u3.dat", 3, 0, output2D, mytype)
 
    ! ***** global data *****
    allocate (data1(nx, ny, nz))
@@ -197,22 +202,20 @@ program io_test
    call decomp_2d_end_io(io_name, "checkpoint")
    call decomp_2d_close_io(io_name, "checkpoint")
 
-   print *, "Write complete"
    call MPI_Barrier(MPI_COMM_WORLD, ierr)
 
    ! read back to different arrays
    u1b = 0; u2b = 0; u3b = 0
-   call decomp_2d_open_io(io_name, "checkpoint", decomp_2d_read_mode)
-   call decomp_2d_start_io(io_name, "checkpoint")
-   call decomp_2d_read_one(1, u1b, 'checkpoint', 'u1.dat', io_name, reduce_prec=.false.)
-   call decomp_2d_read_one(2, u2b, 'checkpoint', 'u2.dat', io_name, reduce_prec=.false.)
-   call decomp_2d_read_one(3, u3b, 'checkpoint', 'u3.dat', io_name, reduce_prec=.false.)
-   call decomp_2d_end_io(io_name, "checkpoint")
-   call decomp_2d_close_io(io_name, "checkpoint")
+   call decomp_2d_open_io(io_restart, "checkpoint", decomp_2d_read_mode)
+   call decomp_2d_start_io(io_restart, "checkpoint")
+   call decomp_2d_read_one(1, u1b, 'checkpoint', 'u1.dat', io_restart, reduce_prec=.false.)
+   call decomp_2d_read_one(2, u2b, 'checkpoint', 'u2.dat', io_restart, reduce_prec=.false.)
+   call decomp_2d_read_one(3, u3b, 'checkpoint', 'u3.dat', io_restart, reduce_prec=.false.)
+   call decomp_2d_end_io(io_restart, "checkpoint")
+   call decomp_2d_close_io(io_restart, "checkpoint")
 
-   print *, "Read complete"
    call MPI_Barrier(MPI_COMM_WORLD, ierr)
-
+   
    ! compare
    call check("one file, multiple fields")
    

From 54e73de9a186afaadc67cb1c5b785c50b86f056d Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Sat, 22 Jul 2023 18:16:36 +0100
Subject: [PATCH 329/436] Remove debugging print statements

---
 src/io.f90 | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/io.f90 b/src/io.f90
index 9096ca72..837ad639 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -449,21 +449,21 @@ subroutine adios2_read_one_real(var, engine_name, varname, io_name)
 
       call adios2_variable_steps(nsteps, var_handle, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_variable_steps")
-      print *, "AVAILABLE steps for ", nsteps
+      ! print *, "AVAILABLE steps for ", nsteps
 
-      print *, "IO_NAME: ", io_name
-      print *, "ENGINE_NAME: ", engine_name
-      print *, "VAR_NAME: ", varname
+      ! print *, "IO_NAME: ", io_name
+      ! print *, "ENGINE_NAME: ", engine_name
+      ! print *, "VAR_NAME: ", varname
       idx = get_io_idx(io_name, engine_name)
-      print *, idx
+      ! print *, idx
       call adios2_get(engine_registry(idx), var_handle, var, adios2_mode_deferred, ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_get")
 
-      print *, "MAX: ", maxval(var)
+      ! print *, "MAX: ", maxval(var)
 
       call adios2_current_step(curstep, engine_registry(idx), ierror)
       if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "adios2_current_step")
-      print *, "Current step: ", curstep
+      ! print *, "Current step: ", curstep
 
 #ifdef PROFILER
       if (decomp_profiler_io) call decomp_profiler_end("adios2_read_one_real")

From 7f6ed40f1188710f949a1f721175ae86b55d6e0d Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 24 Jul 2023 07:42:38 +0100
Subject: [PATCH 330/436] Remove spelling mistake in CI

---
 .github/workflows/RunnerFastCI.yml | 2 +-
 .github/workflows/RunnerFullCI.yml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index 3193553c..a2361d8f 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -83,7 +83,7 @@ jobs:
         rm -rf build  
     
     # Configure-Build-Run-Run on 4 cores 
-    - name: Compile InteilLLVM MKL CPU
+    - name: Compile IntelLLVM MKL CPU
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index b989bfd7..265a1431 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -224,7 +224,7 @@ jobs:
         rm -rf build  
     
     # Configure-Build-Run-Run on 4 cores 
-    - name: Compile InteilLLVM MKL CPU
+    - name: Compile IntelLLVM MKL CPU
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
@@ -244,7 +244,7 @@ jobs:
         rm -rf build 
 
     # Configure-Build-Run-Run on 4 cores 
-    - name: Compile InteilLLVM MKL ADIOS2 CPU
+    - name: Compile IntelLLVM MKL ADIOS2 CPU
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc

From f0f504661c345d275fa1bb5f6cee91ee84d76e47 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 26 Jul 2023 11:25:18 +0100
Subject: [PATCH 331/436] Update documentation to be consistent with website

---
 INSTALL.md | 75 +++++++++++++++++++++++++++++++++++++++++-------------
 README.md  | 54 ++++++++++++++++++++++++++-------------
 2 files changed, 93 insertions(+), 36 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index dc9a13db..ff595176 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -1,4 +1,4 @@
-# Bulding and installing 2decomp-fft
+# Bulding and installing 2DECOMP&FFT
 
 The library 2decomp is a Fortran library compatible with the Fortran 2008 standard.
 It requires a MPI library compatible with MPI-2.0 with extended Fortran support.
@@ -12,7 +12,9 @@ The following [optional libraries](#optional-dependencies) can be used :
 
 ## Building
 
-The build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 
+The build system is driven by `cmake`. It is good practice to directly point to the 
+MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. 
+This can be done by setting the default Fortran environmental variable 
 ```
 export FC=my_mpif90
 ```
@@ -20,7 +22,8 @@ To generate the build system run
 ```
 cmake -S $path_to_sources -B $path_to_build_directory -DOPTION1 -DOPTION2 ...
 ```
-If the directory does not exist it will be generated and it will contain the configuration files. The configuration can be further
+If the directory does not exist it will be generated and it will contain the configuration files. 
+The configuration can be further
 edited by using the `ccmake` utility as
 ```
 ccmake $path_to_build_directory
@@ -33,9 +36,11 @@ The behaviour of debug and development versions of the library can be changed be
 initialization using the variable ``decomp_debug`` or the environment variable ``DECOMP_2D_DEBUG``.
 The value provided with the environment variable must be a positive integer below 9999.
 
-Two `BUILD_TARGETS` are available namely `mpi` and `gpu`.  For the `mpi` target no additional options should be required. whereas for `gpu` extra options are necessary at the configure stage. Please see section [GPU Compilation](#gpu-compilation)
+Two `BUILD_TARGETS` are available namely `mpi` and `gpu`.  For the `mpi` target no additional 
+options should be required. whereas for `gpu` extra options are necessary at the configure stage. 
+Please see section [GPU Compilation](#gpu-compilation)
 
-Once the build system has been configured, you can build `2decomp&fft` by running
+Once the build system has been configured, you can build 2DECOMP&FFT` by running
 ```
 cmake --build $path_to_build_directory -j <nproc>
 ```
@@ -47,10 +52,14 @@ Options can be added to change the level of verbosity. Finally, the build librar
 ```
 cmake --install $path_to_build_directory
 ```
-The default location for `libdecomp2d.a` is `$path_to_build_directory/opt/lib`or  `$path_to_build_directory/opt/lib64` unless the variable `CMAKE_INSTALL_PREFIX` is modified.
-The module files generated by the build process will similarly be installed to `$path_to_build_directory/opt/install`, users of the library should add this to the include paths for their program.
+The default location for `libdecomp2d.a` is `$path_to_build_directory/opt/lib` 
+or `$path_to_build_directory/opt/lib64` unless the variable `CMAKE_INSTALL_PREFIX` is modified.
+The module files generated by the build process will similarly be installed to 
+`$path_to_build_directory/opt/install`, 
+users of the library should add this to the include paths for their program.
 
-As indicated above, by default a static `libdecomp2d.a` will be compiled, if desired a shared library can be built by setting `BUILD_SHARED_LIBS=ON` either on the command line:
+As indicated above, by default a static `libdecomp2d.a` will be compiled, 
+if desired a shared library can be built by setting `BUILD_SHARED_LIBS=ON` either on the command line:
 ```
 cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_SHARED_LIBS=ON
 ```
@@ -93,7 +102,7 @@ It is possible that your default C compiler is too recent and not supported by `
 
 ### Codes using Makefiles
 
-When building a code that links 2decomp-fft using a Makefile you will need to add the include and link paths as appropriate (`inlude/` and `lib/` under the installation directory, respectively).
+When building a code that links 2DECOMP&FFT using a Makefile you will need to add the include and link paths as appropriate (`inlude/` and `lib/` under the installation directory, respectively).
 ```
 DECOMP_ROOT = /path/to/2decomp-fft
 DECOMP_BUILD_DIR = $(DECOMP_ROOT)/build
@@ -110,13 +119,14 @@ my_exec: $(OBJ)
 	$(F90) -o $@ $(OBJ) $(LIBS)
 
 ```
-In case 2decomp-fft has been compiled with an external FFT, such as FFTW3, `LIBS` should also contain the following 
+In case 2DECOMP&FFT has been compiled with an external FFT, such as FFTW3, `LIBS` 
+should also contain the following 
 ```
 FFTW3_PATH=/my_path_to_FFTW/lib
 LIBFFT=-L$(FFTW3_PATH) -lfftw3 -lfftw3f
 LIBS += $(LIBFFT)
 ```
-In case of 2decomp-fft compiled for GPU with NVHPC, linking against cuFFT is mandatory 
+In case of 2DECOMP&FFT compiled for GPU with NVHPC, linking against cuFFT is mandatory 
 ```
 LIBS += -cudalib=cufft
 ```
@@ -152,7 +162,8 @@ clean-decomp:
 
 ## Profiling
 
-Profiling can be activated via `cmake` configuration, the recommended approach is to run the initial configuration as follows:
+Profiling can be activated via `cmake` configuration, 
+the recommended approach is to run the initial configuration as follows:
 ```
 export caliper_DIR=/path/to/caliper/install/share/cmake/caliper
 export CXX=mpicxx
@@ -239,7 +250,11 @@ make -j test
 make -j install
 ```
 
-To build `2decomp&fft` with the adios2 IO backend, one can provide the package configuration for adios2 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/adios2/install/lib/cmake/adios2`. One can also provide the option `-Dadios2_DIR=/path/to/adios2/install/lib/cmake/adios2`. Then either specify on the command line when configuring the build
+To build 2DECOMP&FFT with the adios2 IO backend, one can provide the package configuration for adios2 
+in the `PKG_CONFIG_PATH` environment variable, 
+this should be found under `/path/to/adios2/install/lib/cmake/adios2`. 
+One can also provide the option `-Dadios2_DIR=/path/to/adios2/install/lib/cmake/adios2`. 
+Then either specify on the command line when configuring the build
 ```
 cmake -S . -B ./build -DIO_BACKEND=adios2 -Dadios2_DIR=/path/to/adios2/install/lib/cmake/adios2
 ```
@@ -247,7 +262,13 @@ or modify the build configuration using `ccmake`. Please note that the support f
 
 ### FFTW
 
-The library [fftw](http://www.fftw.org/index.html) can be used as a backend for the FFT engine. The version 3.3.10 was tested, is supported and can be downloaded [here](http://www.fftw.org/download.html). Please note that one should build fftw and decomp2d against the same compilers. For build instructions, please check [here](http://www.fftw.org/fftw3_doc/Installation-on-Unix.html). Below is a suggestion for the compilation of the library in double precision (add `--enable-single` for a single precision build):
+The library [fftw](http://www.fftw.org/index.html) can be used as a backend for the FFT engine. 
+The version 3.3.10 was tested, is supported and can be downloaded 
+[here](http://www.fftw.org/download.html). 
+Please note that one should build fftw and decomp2d against the same compilers. 
+For build instructions, please check [here](http://www.fftw.org/fftw3_doc/Installation-on-Unix.html). 
+Below is a suggestion for the compilation of the library in double precision 
+(add `--enable-single` for a single precision build):
 
 ```
 wget http://www.fftw.org/fftw-3.3.10.tar.gz
@@ -258,22 +279,40 @@ make -j
 make -j check
 make install
 ```
-Please note that the resulting build is not compatible with CMake (https://github.com/FFTW/fftw3/issues/130). As a workaround, one can open the file `/path/to/fftw3/install/lib/cmake/fftw3/FFTW3Config.cmake` and comment the line
+Please note that the resulting build is not compatible with `cmake` 
+(https://github.com/FFTW/fftw3/issues/130). 
+As a workaround, one can open the file 
+`/path/to/fftw3/install/lib/cmake/fftw3/FFTW3Config.cmake` 
+and comment the line
 ```
 include ("${CMAKE_CURRENT_LIST_DIR}/FFTW3LibraryDepends.cmake")
 ```
 
-To build `2decomp&fft` against fftw3, one can provide the package configuration for fftw3 in the `PKG_CONFIG_PATH` environment variable, this should be found under `/path/to/fftw3/install/lib/pkgconfig`. One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. Then either specify on the command line when configuring the build
+To build `2decomp&fft` against fftw3, one can provide the package configuration for fftw3 
+in the `PKG_CONFIG_PATH` environment variable, 
+this should be found under `/path/to/fftw3/install/lib/pkgconfig`. 
+One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. 
+Then either specify on the command line when configuring the build
 ```
 cmake -S . -B build -DFFT_Choice=<fftw|fftw_f03> -DFFTW_ROOT=/path/to/fftw3/install
 ```
 or modify the build configuration using `ccmake`.
 
-Note the legacy `fftw` interface lacks interface definitions and will fail when stricter compilation flags are used (e.g. when `-DCMAKE_BUILD_TYPE=Dev`) for this it is recommended to use `fftw_f03` which provides proper interfaces.
+Note the legacy `fftw` interface lacks interface definitions 
+and will fail when stricter compilation flags are used (e.g. when `-DCMAKE_BUILD_TYPE=Dev`) 
+for this it is recommended to use `fftw_f03` which provides proper interfaces.
 
 ### Caliper
 
-The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. The version 2.9.1 was tested and is supported, version 2.8.0 has also been tested and is still expected to work. Please note that one must build caliper and decomp2d against the same C/C++/Fortran compilers and MPI libray. For build instructions, please check [here](https://github.com/LLNL/Caliper#building-and-installing) and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). Below is a suggestion for the compilation of the library using the GNU compilers:
+The library [caliper](https://github.com/LLNL/Caliper) can be used to profile the execution of the code. 
+The version 2.9.1 was tested and is supported, version 2.8.0 has also been tested 
+and is still expected to work. 
+Please note that one must build caliper and decomp2d against the same C/C++/Fortran 
+compilers and MPI libray. 
+For build instructions, please check 
+[here](https://github.com/LLNL/Caliper#building-and-installing) 
+and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-install). 
+Below is a suggestion for the compilation of the library using the GNU compilers:
 
 ```
 git clone https://github.com/LLNL/Caliper.git caliper_github
diff --git a/README.md b/README.md
index 552c8c28..999b3784 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
-# 2decomp-fft
+# 2DECOMP&FFT
 
-This README contains basic instructions for building and installing the `2decomp&fft` library, more
+This README contains basic instructions for building and installing the 2DECOMP&FFT library, more
 detailed instructions can be found in [INSTALL.md](INSTALL.md).
 
 ## Building
@@ -17,11 +17,12 @@ for many users a configuration line
 ```
 cmake -S . -B build
 ```
-run from the `2decomp&fft` root directory will be sufficient.
+run from the 2DECOMP&FFT root directory will be sufficient.
 If the build directory does not exist it will be generated and it will contain the configuration files.
-By default a `RELEASE` build will built for CPU using MPI and the `generic` FFT backend included with `2decomp&fft`, please see [INSTALL.md](INSTALL.md) for instructions on changing the build, including debugging builds, building for GPUs and selecting external FFT libraries.
+By default a ``RELEASE`` build will built for CPU using MPI and the ``generic`` FFT backend included 
+with 2DECOMP&FFT, please see [INSTALL.md](INSTALL.md) for instructions on changing the build, including debugging builds, building for GPUs and selecting external FFT libraries.
 
-Once the build system has been configured, you can build `2decomp&fft` by running
+Once the build system has been configured, you can build 2DECOMP&FFT by running
 ```
 cmake --build $path_to_build_directory -j <nproc>
 ```
@@ -47,27 +48,33 @@ The library can perform multi GPU offoloading using the NVHPC compiler suite for
 The implementation is based on CUDA-aware MPI and NVIDIA Collective Communication Library (NCCL).
 The FFT is based on cuFFT.
 
-For details of how to configure `2decomp&fft` for GPU offload, see the GPU compilation section in
+For details of how to configure 2DECOMP&FFT for GPU offload, see the GPU compilation section in
 [INSTALL.md](INSTALL.md).
 
 ## Testing and examples
 
-By default building of the tests is deactivated. To activate the testing the option `-DBUILD_TESTING=ON` can be added or 
+By default building of the tests is deactivated. 
+To activate the testing the option `-DBUILD_TESTING=ON` can be added or 
 alternativey the option can be activated in the GUI interface `ccmake`.
 After building the library can be tested by running
 ```
 ctest --test-dir $path_to_build_directory
 ```
-which uses the `ctest` utility. By default tests are performed in serial, but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS` under `ccmake` utility.
-It is also possible to specify the decomposition by setting `PROW` and `PCOL` parameters at the configure stage or using `ccmake`. 
-During the configure stage users should ensure that the number of MPI tasks `MPIEXEC_MAX_NUMPROCS` is equal to the product of PROW times PCOL. 
+which uses the `ctest` utility. By default tests are performed in serial, 
+but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS` under `ccmake` utility.
+It is also possible to specify the decomposition by setting 
+`PROW` and `PCOL` parameters at the configure stage or using `ccmake`. 
+During the configure stage users should ensure that the number of MPI tasks `MPIEXEC_MAX_NUMPROCS` 
+is equal to the product of PROW times PCOL. 
 Mesh resolution can also be imposed using the parameters `NX`, `NY` and `NZ`. 
 
-For the GPU implementation please be aware that it is based on a single MPI rank per GPU. Therefore, to test multiple GPUs, use the maximum number of available GPUs on the system/node and not the maximum number of MPI tasks. 
+For the GPU implementation please be aware that it is based on a single MPI rank per GPU. 
+Therefore, to test multiple GPUs, use the maximum number of available GPUs 
+on the system/node and not the maximum number of MPI tasks. 
 
 ## Profiling
 
-The `2decomp&fft` library has integrated profiling support via external libraries, see the Profiling
+The 2DECOMP&FFT library has integrated profiling support via external libraries, see the Profiling
 section of [INSTALL.md](INSTALL.md) for instructions on configuring a profiling build.
 Currently, support for profiling is provided by the `caliper` library.
 
@@ -83,7 +90,10 @@ Each input allow activation / deactivation of the profiling as follows :
 
 ## FFT backends
 
-The library provides a built-in FFT engine and supports various FFT backends : FFTW, Intel oneMKL, Nvidia cuFFT. The FFT engine selected during compilation is available through the variable `D2D_FFT_BACKEND` defined in the module `decomp_2d_fft`. The expected value is defined by the integer constants
+The library provides a built-in FFT engine and supports various FFT backends : 
+FFTW, Intel oneMKL, Nvidia cuFFT. 
+The FFT engine selected during compilation is available through the variable `D2D_FFT_BACKEND` 
+defined in the module `decomp_2d_fft`. The expected value is defined by the integer constants
 ```
 integer, parameter, public :: D2D_FFT_BACKEND_GENERIC = 0   ! Built-in engine
 integer, parameter, public :: D2D_FFT_BACKEND_FFTW3 = 1     ! FFTW
@@ -105,7 +115,8 @@ The external code can use the named variables to check the FFT backend used in a
 
 ### Print the log to a file or to stdout
 
-Before calling `decomp_2d_init`, the external code can modify the variable `decomp_log` to change the output for the log. The expected value is defined by the integer constants
+Before calling `decomp_2d_init`, the external code can modify the variable `decomp_log` 
+to change the output for the log. The expected value is defined by the integer constants
 ```
 integer, parameter, public :: D2D_LOG_QUIET = 0       ! No logging output
 integer, parameter, public :: D2D_LOG_STDOUT = 1      ! Root rank logs output to stdout
@@ -118,17 +129,24 @@ The default value used is `D2D_LOG_TOFILE` for the default build and `D2D_LOG_TO
 
 ### Change the debug level for debug builds
 
-Before calling `decomp_2d_init`, the external code can modify the variable `decomp_debug` to change the debug level. The user can also modify this variable using the environment variable `DECOMP_2D_DEBUG`. Please note that the environment variable is processed only for debug builds. The expected value for the variable `decomp_debug` is some integer between 0 and 6, bounds included.
+Before calling `decomp_2d_init`, the external code can modify the variable `decomp_debug` 
+to change the debug level. The user can also modify this variable using the environment 
+variable `DECOMP_2D_DEBUG`. Please note that the environment variable is processed only for debug builds.
+The expected value for the variable `decomp_debug` is some integer between 0 and 6, bounds included.
 
 ### Code formatting
 
-The code is formatted using the `fprettify` program (available via `pip`), to ensure consistency of use there is a script file `scripts/format.sh` which will run `fprettify` across the 2decomp&fft source, you can also use the `format` build target to run the script.
+The code is formatted using the `fprettify` program (available via `pip`), 
+to ensure consistency of use there is a script file `scripts/format.sh` 
+which will run `fprettify` across the 2decomp&fft source, 
+you can also use the `format` build target to run the script.
 It is recommended that you should format the code before making a pull request.
 
 ### Versioning
 
-The development of `2decomp&fft` occurs on Github, with release versions on the `main` branch.
-New features will be implemented on the `develop` branch and merged into `main` once a new release
+The development of 2DECOMP&FFT occurs on Github, with release versions on the `main` branch.
+New features will be implemented on the `dev` branch 
+and merged into `main` once a new release
 is ready.
 For example, starting from `v2.0.0` the `main` branch will only be updated to receive fixes giving
 `v2.0.1`, etc. until the next release (either `v2.1.0` or `v3.0.0` depending on the magnitude of the

From 10e6e49a08df56a3838c355e36461b7a68e39b96 Mon Sep 17 00:00:00 2001
From: rfj82982 <35581400+rfj82982@users.noreply.github.com>
Date: Wed, 26 Jul 2023 12:08:02 +0100
Subject: [PATCH 332/436] Update Contribute.md

---
 Contribute.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Contribute.md b/Contribute.md
index 38b2327f..e189fa6e 100644
--- a/Contribute.md
+++ b/Contribute.md
@@ -7,12 +7,12 @@
 ## Get started
 
 The recommended strategy to contribute is to start with a [discussion](https://github.com/2decomp-fft/2decomp-fft/discussions) or to pick an existing issue.
-To modify or experiment with the code, fork the 2decomp github repository and commit changes in a dedicated branch of your fork.
+To modify or experiment with the code, fork the 2DECOMP&FFT github repository and commit changes in a dedicated branch of your fork.
 When the modification is ready for review, one can open a pull request as described in the [advanced](#advanced-contribution) section below.
 
 ## Bug
 
-It appears that you have identified a bug in the 2decomp library.
+It appears that you have identified a bug in the 2DECOMP&FFT library.
 If you are not sure this is really a bug in the library, you should go to the [discussions](https://github.com/2decomp-fft/2decomp-fft/discussions) section and open a new discussion.
 Otherwise, follow the steps below.
 
@@ -23,11 +23,11 @@ Please note that for a debug build, the log contains all the environment variabl
 Use it to hunt the bug but think twice before sharing it as it can expose sensitive and personal information.
 At least, please try to reproduce the bug on another machine with another compiler.
 
-Secondly, if you have modified the source code of the 2decomp library, you must reproduce the bug without the modifications in 2decomp.
+Secondly, if you have modified the source code of the 2DECOMP&FFT library, you must reproduce the bug without the modifications in 2DECOMP&FFT.
 The development team can only provide support for sections of code available in the present repository.
 
 Thirdly, you must provide a minimal working example.
-The program using 2decomp and exposing the bug should be relatively small.
+The program using 2DECOMP&FFT and exposing the bug should be relatively small.
 The development team will not provide support if the program exposing the bug is very long.
 The programs available in the examples section are a good starting point for a minimal working example, ideally you could contribute the minimal working example to the existing examples.
 

From 08e0c3bc9e71b3605f426ea7d5b5ca759da5f40e Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 26 Jul 2023 16:06:05 +0100
Subject: [PATCH 333/436] Update the licence statement in all files. Use SPDX

---
 examples/fft_physical_x/fft_c2c_x.f90  |  1 +
 examples/fft_physical_x/fft_grid_x.f90 |  1 +
 examples/fft_physical_x/fft_r2c_x.f90  |  1 +
 examples/fft_physical_z/fft_c2c_z.f90  |  1 +
 examples/fft_physical_z/fft_r2c_z.f90  |  1 +
 examples/halo_test/halo_test.f90       |  1 +
 examples/init_test/init_test.f90       |  2 ++
 examples/io_test/io_bench.f90          |  1 +
 examples/io_test/io_plane_test.f90     |  1 +
 examples/io_test/io_test.f90           |  1 +
 examples/io_test/io_var_test.f90       |  1 +
 examples/test2d/test2d.f90             |  1 +
 examples/test2d/timing2d_complex.f90   |  1 +
 examples/test2d/timing2d_real.f90      |  1 +
 src/alloc.f90                          | 12 +-----------
 src/decomp_2d.f90                      | 12 +-----------
 src/decomp_2d_constants.f90            | 12 +-----------
 src/decomp_2d_cumpi.f90                | 12 +-----------
 src/decomp_2d_init_fin.f90             | 12 +-----------
 src/decomp_2d_mpi.f90                  | 12 +-----------
 src/decomp_2d_nccl.f90                 | 12 +-----------
 src/factor.f90                         | 12 +-----------
 src/fft_common.f90                     | 11 +----------
 src/fft_common_3d.f90                  | 11 +----------
 src/fft_cufft.f90                      | 11 +----------
 src/fft_fftw3.f90                      | 11 +----------
 src/fft_fftw3_f03.f90                  | 11 +----------
 src/fft_generic.f90                    | 11 +----------
 src/fft_log.f90                        | 12 +-----------
 src/fft_mkl.f90                        | 11 +----------
 src/glassman.f90                       | 11 +----------
 src/halo.f90                           | 11 +----------
 src/halo_common.f90                    | 11 +----------
 src/io.f90                             | 12 +-----------
 src/io_read_inflow.f90                 | 11 +----------
 src/io_read_one.inc                    | 11 +----------
 src/io_read_var.inc                    | 11 +----------
 src/io_write_every.inc                 | 11 +----------
 src/io_write_one.inc                   | 12 +-----------
 src/io_write_outflow.f90               | 11 +----------
 src/io_write_plane.inc                 | 11 +----------
 src/io_write_var.inc                   | 11 +----------
 src/log.f90                            | 12 +-----------
 src/profiler_caliper.f90               | 12 +-----------
 src/transpose_x_to_y.f90               | 11 +----------
 src/transpose_y_to_x.f90               | 11 +----------
 src/transpose_y_to_z.f90               | 11 +----------
 src/transpose_z_to_y.f90               | 11 +----------
 48 files changed, 49 insertions(+), 353 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index d1edd4d0..8b9c7b01 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program fft_c2c_x
 
    use decomp_2d
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index eca12d4c..ba7db600 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program fft_physical_x
 
    use decomp_2d
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index a887ef0e..9eff2c1c 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program fft_r2c_x
 
    use decomp_2d
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 3b56a414..5ef70720 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program fft_c2c_z
 
    use decomp_2d
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 2798a63a..21b7bf4f 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program fft_r2c_z
 
    use decomp_2d
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index d45eff16..35e79b9b 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 ! This example calculates the divergence of a random field using
 !   (1) global transposition
diff --git a/examples/init_test/init_test.f90 b/examples/init_test/init_test.f90
index 7f932266..9f620015 100644
--- a/examples/init_test/init_test.f90
+++ b/examples/init_test/init_test.f90
@@ -1,3 +1,5 @@
+!! SPDX-License-Identifier: BSD-3-Clause
+!!!=====================================================
 !!!! init_test.f90
 !!!  Tests initialising the 2decomp&fft library.
 !!!=====================================================
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index 34b2d29f..f8d17ab4 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program io_bench
 
    use decomp_2d_constants
diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index b0009400..8a04e21d 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 !!
 !! FIXME The issue below is specific to GPU and should be discussed in a dedicated github issue
 !!
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 9558125e..920e5001 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program io_test
 
    use mpi
diff --git a/examples/io_test/io_var_test.f90 b/examples/io_test/io_var_test.f90
index 1b253c9f..23007191 100644
--- a/examples/io_test/io_var_test.f90
+++ b/examples/io_test/io_var_test.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 ! Sample application to test the read/write_var sets of routines
 ! in the IO library
 
diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 1535e944..9a2019e7 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program test2d
 
    use mpi
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 6735e2f9..7e0e5a3b 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program timing2d_complex
 
    use mpi
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 1f9e216b..9d4850e5 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -1,3 +1,4 @@
+!! SPDX-License-Identifier: BSD-3-Clause
 program timing2d_real
 
    use mpi
diff --git a/src/alloc.f90 b/src/alloc.f90
index 51c893ef..68197d6d 100644
--- a/src/alloc.f90
+++ b/src/alloc.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2021 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
-
+!! SPDX-License-Identifier: BSD-3-Clause
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   ! Utility routine to help allocate 3D arrays
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 60aa72d9..d89ebe53 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2021               the University of Edinburgh (UoE)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This is the main 2D pencil decomposition module
 
diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index 2cf74a9d..c3cd274e 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2021               the University of Edinburgh (UoE)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! Constants for the 2decomp&fft library
 
diff --git a/src/decomp_2d_cumpi.f90 b/src/decomp_2d_cumpi.f90
index 4e4b54af..45917cca 100644
--- a/src/decomp_2d_cumpi.f90
+++ b/src/decomp_2d_cumpi.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2022               Science and Technology Facilities Council(STFC)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! Module for the cuda aware MPI
 
diff --git a/src/decomp_2d_init_fin.f90 b/src/decomp_2d_init_fin.f90
index 1294d473..71ace9e8 100644
--- a/src/decomp_2d_init_fin.f90
+++ b/src/decomp_2d_init_fin.f90
@@ -1,14 +1,4 @@
-!!!=======================================================================
-!!! This is part of the 2DECOMP&FFT library
-!!!
-!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-!!! decomposition. It also implements a highly scalable distributed
-!!! three-dimensional Fast Fourier Transform (FFT).
-!!!
-!!! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-!!! Copyright (C) 2022-              the Xcompact3d developers
-!!!
-!!!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
   !======================================================================
   ! Routine to be called by applications to initialise this library
diff --git a/src/decomp_2d_mpi.f90 b/src/decomp_2d_mpi.f90
index 0e478625..df4531c9 100644
--- a/src/decomp_2d_mpi.f90
+++ b/src/decomp_2d_mpi.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2021               the University of Edinburgh (UoE)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! MPI module for 2decomp&fft library
 
diff --git a/src/decomp_2d_nccl.f90 b/src/decomp_2d_nccl.f90
index 65065d23..01c64bc6 100644
--- a/src/decomp_2d_nccl.f90
+++ b/src/decomp_2d_nccl.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2022               Science and Technology Facilities Council(STFC)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! Module for the cuda aware MPI
 
diff --git a/src/factor.f90 b/src/factor.f90
index 8f4060ae..cc7030fe 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -1,14 +1,4 @@
-!!!=======================================================================
-!!! This is part of the 2DECOMP&FFT library
-!!!
-!!! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-!!! decomposition. It also implements a highly scalable distributed
-!!! three-dimensional Fast Fourier Transform (FFT).
-!!!
-!!! Copyright (C) 2009-2021 Ning Li, the Numerical Algorithms Group (NAG)
-!!! Copyright (C) 2022-              the Xcompact3d developers
-!!!
-!!!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 !!! A few utility routines to find factors of integer numbers
 module factor
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 34632a40..ace10103 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contains common code shared by all FFT engines
 
diff --git a/src/fft_common_3d.f90 b/src/fft_common_3d.f90
index fdee1b43..28c4ec0d 100644
--- a/src/fft_common_3d.f90
+++ b/src/fft_common_3d.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contains 3D c2c/r2c/c2r transform subroutines which are
 ! identical for several FFT engines
diff --git a/src/fft_cufft.f90 b/src/fft_cufft.f90
index 125854d4..8a2d84eb 100644
--- a/src/fft_cufft.f90
+++ b/src/fft_cufft.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This is the FFTW (version 3.x) implementation of the FFT library
 
diff --git a/src/fft_fftw3.f90 b/src/fft_fftw3.f90
index ced99dbd..9c1284ef 100644
--- a/src/fft_fftw3.f90
+++ b/src/fft_fftw3.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This is the FFTW (version 3.x) implementation of the FFT library
 
diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index 4f0ceb46..c1704ec0 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This is the FFTW implementation of the FFT library using
 ! the Fortran 2003 interface introduced in FFTW 3.3-beta1
diff --git a/src/fft_generic.f90 b/src/fft_generic.f90
index c07c52e2..6d9e18cb 100644
--- a/src/fft_generic.f90
+++ b/src/fft_generic.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This is the 'generic' implementation of the FFT library
 
diff --git a/src/fft_log.f90 b/src/fft_log.f90
index 45349373..7dfbb45f 100644
--- a/src/fft_log.f90
+++ b/src/fft_log.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2021               the University of Edinburgh (UoE)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 submodule(decomp_2d_fft) d2d_fft_log
 
diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index cf63924d..0d58aaa3 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This is the Intel MKL implementation of the FFT library
 
diff --git a/src/glassman.f90 b/src/glassman.f90
index bd0e9ae6..8fca08fe 100644
--- a/src/glassman.f90
+++ b/src/glassman.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This module contains a few 'generic' FFT routines, making the
 ! 2DECOMP&FFT library not dependent on any external libraries
diff --git a/src/halo.f90 b/src/halo.f90
index 16bad6e6..dfe43fda 100644
--- a/src/halo.f90
+++ b/src/halo.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2021 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   ! Halo cell support for neighbouring pencils to exchange data
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index 2148b9f0..b8dedca6 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2021 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contain common code to be included by subroutines
 ! 'update_halo_...' in halo.f90
diff --git a/src/io.f90 b/src/io.f90
index 837ad639..23994017 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2013 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2021               the University of Edinburgh (UoE)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This module provides parallel IO facilities for applications based on
 ! 2D decomposition.
diff --git a/src/io_read_inflow.f90 b/src/io_read_inflow.f90
index 75fa9488..19e3a05c 100644
--- a/src/io_read_inflow.f90
+++ b/src/io_read_inflow.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contain common code to be included by subroutines
 ! 'read_var_...' in io.f90
diff --git a/src/io_read_one.inc b/src/io_read_one.inc
index 1167e872..0063f377 100644
--- a/src/io_read_one.inc
+++ b/src/io_read_one.inc
@@ -1,14 +1,5 @@
 !                          -*- mode: f90 -*-
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contain common code to be included by subroutines
 ! 'mpiio_read_one_...' in io.f90
diff --git a/src/io_read_var.inc b/src/io_read_var.inc
index b3fa33f9..c72d09a7 100644
--- a/src/io_read_var.inc
+++ b/src/io_read_var.inc
@@ -1,14 +1,5 @@
 !                          -*- mode: f90 -*-
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contain common code to be included by subroutines
 ! 'read_var_...' in io.f90
diff --git a/src/io_write_every.inc b/src/io_write_every.inc
index f11af06a..4fa4b61a 100644
--- a/src/io_write_every.inc
+++ b/src/io_write_every.inc
@@ -1,14 +1,5 @@
 !                          -*- mode: f90 -*-
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contain common code to be included by subroutines
 ! 'write_every_...' in io.f90
diff --git a/src/io_write_one.inc b/src/io_write_one.inc
index 93a89317..85cb8399 100644
--- a/src/io_write_one.inc
+++ b/src/io_write_one.inc
@@ -1,15 +1,5 @@
 !                          -*- mode: f90 -*-
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2021               the University of Edinburgh (UoE)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contain common code to be included by subroutines
 ! 'mpiio_write_one_...' in io.f90
diff --git a/src/io_write_outflow.f90 b/src/io_write_outflow.f90
index 9e9e4f98..7b680170 100644
--- a/src/io_write_outflow.f90
+++ b/src/io_write_outflow.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contain common code to be included by subroutines
 ! 'write_var_...' in io.f90
diff --git a/src/io_write_plane.inc b/src/io_write_plane.inc
index 5db42b72..69b51bd7 100644
--- a/src/io_write_plane.inc
+++ b/src/io_write_plane.inc
@@ -1,14 +1,5 @@
 !                          -*- mode: f90 -*-
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contain common code to be included by subroutines
 ! 'mpiio_write_plane_3d_...' in io.f90
diff --git a/src/io_write_var.inc b/src/io_write_var.inc
index f5b34b82..d6eef576 100644
--- a/src/io_write_var.inc
+++ b/src/io_write_var.inc
@@ -1,14 +1,5 @@
 !                          -*- mode: f90 -*-
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2011 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contain common code to be included by subroutines
 ! 'write_var_...' in io.f90
diff --git a/src/log.f90 b/src/log.f90
index 55b0694f..ae121eca 100644
--- a/src/log.f90
+++ b/src/log.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2021               the University of Edinburgh (UoE)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 submodule(decomp_2d) d2d_log
 
diff --git a/src/profiler_caliper.f90 b/src/profiler_caliper.f90
index 5c5a1fa2..944129fb 100644
--- a/src/profiler_caliper.f90
+++ b/src/profiler_caliper.f90
@@ -1,14 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2012 Ning Li, the Numerical Algorithms Group (NAG)
-! Copyright (C) 2021               the University of Edinburgh (UoE)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 !
 ! Submodule for the caliper profiler
diff --git a/src/transpose_x_to_y.f90 b/src/transpose_x_to_y.f90
index ea8f4a74..efc59538 100644
--- a/src/transpose_x_to_y.f90
+++ b/src/transpose_x_to_y.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2021 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contains the routines that transpose data from X to Y pencil
 
diff --git a/src/transpose_y_to_x.f90 b/src/transpose_y_to_x.f90
index 2b1a1efd..e5fa2a16 100644
--- a/src/transpose_y_to_x.f90
+++ b/src/transpose_y_to_x.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2021 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contains the routines that transpose data from Y to X pencil
 
diff --git a/src/transpose_y_to_z.f90 b/src/transpose_y_to_z.f90
index cca04207..4d212f4f 100644
--- a/src/transpose_y_to_z.f90
+++ b/src/transpose_y_to_z.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2021 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contains the routines that transpose data from Y to Z pencil
 
diff --git a/src/transpose_z_to_y.f90 b/src/transpose_z_to_y.f90
index 35d6714e..6bfb36b2 100644
--- a/src/transpose_z_to_y.f90
+++ b/src/transpose_z_to_y.f90
@@ -1,13 +1,4 @@
-!=======================================================================
-! This is part of the 2DECOMP&FFT library
-!
-! 2DECOMP&FFT is a software framework for general-purpose 2D (pencil)
-! decomposition. It also implements a highly scalable distributed
-! three-dimensional Fast Fourier Transform (FFT).
-!
-! Copyright (C) 2009-2021 Ning Li, the Numerical Algorithms Group (NAG)
-!
-!=======================================================================
+!! SPDX-License-Identifier: BSD-3-Clause
 
 ! This file contains the routines that transpose data from Z to Y pencil
 

From e49351c32050a552ec8d757be5bca371282a15a6 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 27 Jul 2023 14:05:46 +0100
Subject: [PATCH 334/436] Update LICENSE

---
 LICENSE | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/LICENSE b/LICENSE
index 0a58604f..17de2d2a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2011-2021, The Numerical Algorithms Group (NAG)
+Copyright (c) 2011-2021, Ning Li, The Numerical Algorithms Group (NAG)
 Copyright (c) 2022-    , Xcompact3d
 All rights reserved.
 

From 1dee8c1c10ef8e2d06b5b90d34ae8972add47fbd Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sun, 6 Aug 2023 17:55:03 +0200
Subject: [PATCH 335/436] fprettify

---
 examples/halo_test/halo_test.f90 |   4 +-
 examples/io_test/io_test.f90     | 142 +++++++++++++++----------------
 src/factor.f90                   |   4 +-
 3 files changed, 75 insertions(+), 75 deletions(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 35e79b9b..78cd3c07 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -381,7 +381,7 @@ subroutine test_div_haloY()
       integer :: kfirst, klast ! K loop start/end
 
       call alloc_x(div2, global)
-      
+
       ! Expected sizes
       nx_expected = ysize(1) + 2
       ny_expected = ny
@@ -439,7 +439,7 @@ subroutine test_div_haloZ()
       implicit none
 
       real(mytype), allocatable, dimension(:, :, :) :: div3
-      
+
       real(mytype), allocatable, dimension(:, :, :) :: uh, vh
 #if defined(_GPU)
       attributes(device) :: vh, uh
diff --git a/examples/io_test/io_test.f90 b/examples/io_test/io_test.f90
index 920e5001..cc6fa142 100644
--- a/examples/io_test/io_test.f90
+++ b/examples/io_test/io_test.f90
@@ -22,7 +22,7 @@ program io_test
    character(len=80) :: InputFN
 
    integer :: ierr
-   
+
 #ifdef COMPLEX_TEST
    complex(mytype), allocatable, dimension(:, :, :) :: data1
 
@@ -216,86 +216,86 @@ program io_test
    call decomp_2d_close_io(io_restart, "checkpoint")
 
    call MPI_Barrier(MPI_COMM_WORLD, ierr)
-   
+
    ! compare
    call check("one file, multiple fields")
-   
+
    deallocate (u1, u2, u3)
    deallocate (u1b, u2b, u3b)
    deallocate (data1)
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)
 
- contains
+contains
 
    subroutine check(stage)
 
-     character(len=*), intent(in) :: stage
-     
-     integer :: ierr
-     
-     if (nrank == 0) then
-        print *, "Checking "//stage
-     end if
-     call MPI_Barrier(MPI_COMM_WORLD, ierr)
-
-     do k = xstart(3), xend(3)
-        do j = xstart(2), xend(2)
-           do i = xstart(1), xend(1)
-              if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) then
-                 print *, u1(i, j, k), u1b(i, j, k)
-                 stop 1
-              end if
-           end do
-        end do
-     end do
-
-     do k = ystart(3), yend(3)
-        do j = ystart(2), yend(2)
-           do i = ystart(1), yend(1)
-              if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 2
-           end do
-        end do
-     end do
-     
-     do k = zstart(3), zend(3)
-        do j = zstart(2), zend(2)
-           do i = zstart(1), zend(1)
-              if (abs((u3(i, j, k) - u3b(i, j, k))) > eps) stop 3
-           end do
-        end do
-     end do
-
-     ! Also check against the global data array
-     do k = xstart(3), xend(3)
-        do j = xstart(2), xend(2)
-           do i = xstart(1), xend(1)
-              if (abs(data1(i, j, k) - u1b(i, j, k)) > eps) stop 4
-           end do
-        end do
-     end do
-     
-     do k = ystart(3), yend(3)
-        do j = ystart(2), yend(2)
-           do i = ystart(1), yend(1)
-              if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
-           end do
-        end do
-     end do
-
-     do k = zstart(3), zend(3)
-        do j = zstart(2), zend(2)
-           do i = zstart(1), zend(1)
-              if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
-           end do
-        end do
-     end do
-
-     call MPI_Barrier(MPI_COMM_WORLD, ierr)
-     if (nrank == 0) then
-        print *, "Checking "//stage//" pass!"
-     end if
-     
+      character(len=*), intent(in) :: stage
+
+      integer :: ierr
+
+      if (nrank == 0) then
+         print *, "Checking "//stage
+      end if
+      call MPI_Barrier(MPI_COMM_WORLD, ierr)
+
+      do k = xstart(3), xend(3)
+         do j = xstart(2), xend(2)
+            do i = xstart(1), xend(1)
+               if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) then
+                  print *, u1(i, j, k), u1b(i, j, k)
+                  stop 1
+               end if
+            end do
+         end do
+      end do
+
+      do k = ystart(3), yend(3)
+         do j = ystart(2), yend(2)
+            do i = ystart(1), yend(1)
+               if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 2
+            end do
+         end do
+      end do
+
+      do k = zstart(3), zend(3)
+         do j = zstart(2), zend(2)
+            do i = zstart(1), zend(1)
+               if (abs((u3(i, j, k) - u3b(i, j, k))) > eps) stop 3
+            end do
+         end do
+      end do
+
+      ! Also check against the global data array
+      do k = xstart(3), xend(3)
+         do j = xstart(2), xend(2)
+            do i = xstart(1), xend(1)
+               if (abs(data1(i, j, k) - u1b(i, j, k)) > eps) stop 4
+            end do
+         end do
+      end do
+
+      do k = ystart(3), yend(3)
+         do j = ystart(2), yend(2)
+            do i = ystart(1), yend(1)
+               if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
+            end do
+         end do
+      end do
+
+      do k = zstart(3), zend(3)
+         do j = zstart(2), zend(2)
+            do i = zstart(1), zend(1)
+               if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
+            end do
+         end do
+      end do
+
+      call MPI_Barrier(MPI_COMM_WORLD, ierr)
+      if (nrank == 0) then
+         print *, "Checking "//stage//" pass!"
+      end if
+
    end subroutine check
-   
+
 end program io_test
diff --git a/src/factor.f90 b/src/factor.f90
index cc7030fe..7c68fae9 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -23,8 +23,8 @@ subroutine findfactor(num, factors, nfact)
 
       ! find the factors <= sqrt(num)
       ! Cast the int as double to make sure of the correct result of sqrt
-      ! IntelLLVM got an issue with 1.0 but not with 1.d0 
-      m = int(sqrt(real(num,8)))
+      ! IntelLLVM got an issue with 1.0 but not with 1.d0
+      m = int(sqrt(real(num, 8)))
       nfact = 1
       do i = 1, m
          if (num / i * i == num) then

From 5034f3a2461375c51ee7857d84f2a862b15705a1 Mon Sep 17 00:00:00 2001
From: rfj82982 <35581400+rfj82982@users.noreply.github.com>
Date: Wed, 23 Aug 2023 10:06:22 +0200
Subject: [PATCH 336/436] Timing examples (#244)

* Update timing_complex and fft_c2c_x

* Remove unnecessary print from halo example

* Update timing for real

* Update timing for fft examples

* Halo Test: remove unnecessary print when checking the halo size

* Add timing for iteration 0 on complex test2d

* Add the timing of the 0 iteration for all timing tests

* fprettify the code

* Minor update in the listing of examples

* Update factor.f90

Update to be portable

---------

Co-authored-by: cflag <cedric.flageul@univ-poitiers.fr>
Co-authored-by: CFLAG <mathrack@users.noreply.github.com>
Co-authored-by: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
---
 examples/fft_physical_x/fft_c2c_x.f90  | 50 ++++++++++++++++++++------
 examples/fft_physical_x/fft_grid_x.f90 | 50 ++++++++++++++++++++------
 examples/fft_physical_x/fft_r2c_x.f90  | 45 ++++++++++++++++++-----
 examples/fft_physical_z/fft_c2c_z.f90  | 49 +++++++++++++++++++------
 examples/fft_physical_z/fft_r2c_z.f90  | 49 +++++++++++++++++++------
 examples/halo_test/halo_test.f90       |  2 --
 examples/io_test/io_bench.f90          |  7 +++-
 examples/test2d/timing2d_complex.f90   | 39 ++++++++++++++------
 examples/test2d/timing2d_real.f90      | 38 ++++++++++++++------
 src/factor.f90                         |  2 +-
 src/io.f90                             |  3 +-
 11 files changed, 255 insertions(+), 79 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 8b9c7b01..84a4d327 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -22,7 +22,7 @@ program fft_c2c_x
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
-   integer, parameter :: ntest = 10  ! repeat test this times
+   integer :: ntest = 10  ! repeat test this times
 
    type(decomp_info), pointer :: ph => null()
    complex(mytype), allocatable, dimension(:, :, :) :: in, out
@@ -44,7 +44,7 @@ program fft_c2c_x
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
-   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5) .or. (nargin == 6)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
          read (InputFN, *, iostat=status) DecInd
@@ -58,6 +58,8 @@ program fft_c2c_x
             ny = DecInd
          elseif (arg == 5) then
             nz = DecInd
+         elseif (arg == 6) then
+            ntest = DecInd
          end if
       end do
    else
@@ -68,12 +70,13 @@ program fft_c2c_x
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
-         print *, "or 5 inputs as"
+         print *, "or 5-6 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
          print *, "  3) nx "
          print *, "  4) ny "
          print *, "  5) nz "
+         print *, "  6) n iterations (optional)"
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
       end if
@@ -106,9 +109,32 @@ program fft_c2c_x
       end do
    end do
 
+   !$acc data copyin(in) copy(out)
+   ! First iterations out of the counting loop
+   t1 = MPI_WTIME()
+   call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+   t2 = MPI_WTIME() - t1
+   t3 = MPI_WTIME()
+   call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+   t4 = MPI_WTIME() - t3
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1 / dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3 / dble(nproc)
+   if (nrank == 0) then
+      write (*, *) '===== c2c interface ====='
+      write (*, *) 'First iteration with dedicated timer'
+      write (*, *) '     time (sec): ', t1, t3
+      write (*, *) ''
+   end if
+   ! Init the time
    t2 = 0.d0
    t4 = 0.d0
-   !$acc data copyin(in) copy(out)
+   !$acc kernels
+   in = in / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+   !$acc end kernels
    do m = 1, ntest
 
       ! forward FFT
@@ -123,7 +149,7 @@ program fft_c2c_x
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in = in / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
+      in = in / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
       !$acc end kernels
 
    end do
@@ -133,10 +159,10 @@ program fft_c2c_x
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1 / dble(nproc)
+   t1 = t1 / dble(nproc) / dble(ntest)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3 / dble(nproc)
+   t3 = t3 / dble(nproc) / dble(ntest)
 
    ! checking accuracy
    error = 0._mytype
@@ -155,20 +181,22 @@ program fft_c2c_x
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
+   err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
    if (nrank == 0) then
-      write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
-      write (*, *) 'time (sec): ', t1, t3
+      write (*, *) 'Avg time (sec): ', t1, t3
       n1 = real(nx) * real(ny) * real(nz)
       n1 = n1**(1.d0 / 3.d0)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
       flops = 5.d0 * n1 * log(n1) / log(2.d0)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
       flops = flops * 3.d0 * n1**2
-      flops = 2.d0 * flops / ((t1 + t3) / real(NTEST))
+      flops = 2.d0 * flops / (t1 + t3)
       write (*, *) 'GFLOPS : ', flops / 1000.d0**3
+      write (*, *) '   '
+      write (*, *) 'fft_c2c_x completed '
+      write (*, *) '   '
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index ba7db600..0afe318f 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -22,7 +22,7 @@ program fft_physical_x
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
-   integer, parameter :: ntest = 10  ! repeat test this times
+   integer :: ntest = 10  ! repeat test this times
 
    type(decomp_info), pointer :: ph => null()
    complex(mytype), allocatable, dimension(:, :, :) :: in, out
@@ -44,7 +44,7 @@ program fft_physical_x
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
-   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5) .or. (nargin == 6)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
          read (InputFN, *, iostat=status) DecInd
@@ -58,6 +58,8 @@ program fft_physical_x
             ny = DecInd
          elseif (arg == 5) then
             nz = DecInd
+         elseif (arg == 6) then
+            ntest = DecInd
          end if
       end do
    else
@@ -68,12 +70,13 @@ program fft_physical_x
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
-         print *, "or 5 inputs as"
+         print *, "or 5-6 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
          print *, "  3) nx "
          print *, "  4) ny "
          print *, "  5) nz "
+         print *, "  6) n iterations (optional)"
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
       end if
@@ -107,9 +110,32 @@ program fft_physical_x
       end do
    end do
 
+   !$acc data copyin(in) copy(out)
+   ! First iterations out of the counting loop
+   t1 = MPI_WTIME()
+   call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+   t2 = MPI_WTIME() - t1
+   t3 = MPI_WTIME()
+   call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+   t4 = MPI_WTIME() - t3
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1 / dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3 / dble(nproc)
+   if (nrank == 0) then
+      write (*, *) '===== c2c interface ====='
+      write (*, *) 'First iteration with dedicated timer'
+      write (*, *) '     time (sec): ', t1, t3
+      write (*, *) ''
+   end if
+   ! Init the time
    t2 = 0.d0
    t4 = 0.d0
-   !$acc data copyin(in) copy(out)
+   !$acc kernels
+   in = in / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+   !$acc end kernels
    do m = 1, ntest
 
       ! forward FFT
@@ -124,7 +150,7 @@ program fft_physical_x
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in = in / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
+      in = in / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
       !$acc end kernels
 
    end do
@@ -134,10 +160,10 @@ program fft_physical_x
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1 / dble(nproc)
+   t1 = t1 / dble(nproc) / dble(ntest)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3 / dble(nproc)
+   t3 = t3 / dble(nproc) / dble(ntest)
 
    ! checking accuracy
    error = 0._mytype
@@ -156,20 +182,22 @@ program fft_physical_x
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
+   err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
    if (nrank == 0) then
-      write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
-      write (*, *) 'time (sec): ', t1, t3
+      write (*, *) 'Avg time (sec): ', t1, t3
       n1 = real(nx) * real(ny) * real(nz)
       n1 = n1**(1.d0 / 3.d0)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
       flops = 5.d0 * n1 * log(n1) / log(2.d0)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
       flops = flops * 3.d0 * n1**2
-      flops = 2.d0 * flops / ((t1 + t3) / real(NTEST))
+      flops = 2.d0 * flops / (t1 + t3)
       write (*, *) 'GFLOPS : ', flops / 1000.d0**3
+      write (*, *) '   '
+      write (*, *) 'fft_physical_x completed '
+      write (*, *) '   '
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 9eff2c1c..771d5158 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -22,7 +22,7 @@ program fft_r2c_x
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
-   integer, parameter :: ntest = 10  ! repeat test this times
+   integer :: ntest = 10  ! repeat test this times
 
    type(decomp_info), pointer :: ph => null(), sp => null()
    complex(mytype), allocatable, dimension(:, :, :) :: out
@@ -45,7 +45,7 @@ program fft_r2c_x
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
-   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5) .or. (nargin == 6)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
          read (InputFN, *, iostat=status) DecInd
@@ -59,6 +59,8 @@ program fft_r2c_x
             ny = DecInd
          elseif (arg == 5) then
             nz = DecInd
+         elseif (arg == 6) then
+            ntest = DecInd
          end if
       end do
    else
@@ -69,12 +71,13 @@ program fft_r2c_x
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
-         print *, "or 5 inputs as"
+         print *, "or 5-6 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
          print *, "  3) nx "
          print *, "  4) ny "
          print *, "  5) nz "
+         print *, "  6) n iterations (optional)"
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
       end if
@@ -106,9 +109,32 @@ program fft_r2c_x
       end do
    end do
 
+   !$acc data copyin(in_r) copy(out)
+   ! First iterations out of the counting loop
+   t1 = MPI_WTIME()
+   call decomp_2d_fft_3d(in_r, out)
+   t2 = MPI_WTIME() - t1
+   t3 = MPI_WTIME()
+   call decomp_2d_fft_3d(out, in_r)
+   t4 = MPI_WTIME() - t3
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1 / dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3 / dble(nproc)
+   if (nrank == 0) then
+      write (*, *) '===== r2c/c2r interface ====='
+      write (*, *) 'First iteration with dedicated timer'
+      write (*, *) '     time (sec): ', t1, t3
+      write (*, *) ''
+   end if
+   ! Init the time
    t2 = 0.d0
    t4 = 0.d0
-   !$acc data copyin(in_r) copy(out)
+   !$acc kernels
+   in_r = in_r / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+   !$acc end kernels
    do m = 1, ntest
 
       ! 3D r2c FFT
@@ -133,10 +159,10 @@ program fft_r2c_x
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1 / dble(nproc)
+   t1 = t1 / dble(nproc) / dble(ntest)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3 / dble(nproc)
+   t3 = t3 / dble(nproc) / dble(ntest)
 
    ! checking accuracy
    error = 0._mytype
@@ -151,13 +177,16 @@ program fft_r2c_x
       end do
    end do
    !$acc end loop
+
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
+   err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
    if (nrank == 0) then
-      write (*, *) '===== r2c/c2r interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
+      write (*, *) '   '
+      write (*, *) 'fft_r2c_x completed '
+      write (*, *) '   '
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 5ef70720..c0a6612f 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -22,7 +22,7 @@ program fft_c2c_z
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
-   integer, parameter :: ntest = 10  ! repeat test this times
+   integer :: ntest = 10  ! repeat test this times
 
    type(decomp_info), pointer :: ph => null()
    complex(mytype), allocatable, dimension(:, :, :) :: in, out
@@ -44,7 +44,7 @@ program fft_c2c_z
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
-   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5) .or. (nargin == 6)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
          read (InputFN, *, iostat=status) DecInd
@@ -58,6 +58,8 @@ program fft_c2c_z
             ny = DecInd
          elseif (arg == 5) then
             nz = DecInd
+         elseif (arg == 6) then
+            ntest = DecInd
          end if
       end do
    else
@@ -68,12 +70,13 @@ program fft_c2c_z
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
-         print *, "or 5 inputs as"
+         print *, "or 5-6 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
          print *, "  3) nx "
          print *, "  4) ny "
          print *, "  5) nz "
+         print *, "  6) n iterations (optional)"
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
       end if
@@ -107,9 +110,32 @@ program fft_c2c_z
       end do
    end do
 
+   !$acc data copyin(in) copy(out)
+   ! First iterations out of the counting loop
+   t1 = MPI_WTIME()
+   call decomp_2d_fft_3d(in, out, DECOMP_2D_FFT_FORWARD)
+   t2 = MPI_WTIME() - t1
+   t3 = MPI_WTIME()
+   call decomp_2d_fft_3d(out, in, DECOMP_2D_FFT_BACKWARD)
+   t4 = MPI_WTIME() - t3
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1 / dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3 / dble(nproc)
+   if (nrank == 0) then
+      write (*, *) '===== c2c interface ====='
+      write (*, *) 'First iteration with dedicated timer'
+      write (*, *) '     time (sec): ', t1, t3
+      write (*, *) ''
+   end if
+   ! Init the time
    t2 = 0.d0
    t4 = 0.d0
-   !$acc data copyin(in) copy(out)
+   !$acc kernels
+   in = in / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+   !$acc end kernels
    do m = 1, ntest
 
       ! forward FFT
@@ -124,7 +150,7 @@ program fft_c2c_z
 
       ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in = in / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
+      in = in / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
       !$acc end kernels
 
    end do
@@ -134,10 +160,10 @@ program fft_c2c_z
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1 / dble(nproc)
+   t1 = t1 / dble(nproc) / dble(ntest)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3 / dble(nproc)
+   t3 = t3 / dble(nproc) / dble(ntest)
 
    ! checking accuracy
    error = 0._mytype
@@ -156,20 +182,21 @@ program fft_c2c_z
    end do
    !$acc end loop
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
+   err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
    if (nrank == 0) then
-      write (*, *) '===== c2c interface ====='
       write (*, *) 'error / mesh point: ', err_all
-      write (*, *) 'time (sec): ', t1, t3
+      write (*, *) 'Avg time (sec): ', t1, t3
       n1 = real(nx) * real(ny) * real(nz)
       n1 = n1**(1.d0 / 3.d0)
       ! 5n*log(n) flops per 1D FFT of size n using Cooley-Tukey algorithm
       flops = 5.d0 * n1 * log(n1) / log(2.d0)
       ! 3 sets of 1D FFTs for 3 directions, each having n^2 1D FFTs
       flops = flops * 3.d0 * n1**2
-      flops = 2.d0 * flops / ((t1 + t3) / real(NTEST))
+      flops = 2.d0 * flops / (t1 + t3)
       write (*, *) 'GFLOPS : ', flops / 1000.d0**3
+      write (*, *) '   '
+      write (*, *) 'fft_c2c_z completed '
    end if
    !$acc end data
 
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 21b7bf4f..d9cae9d4 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -22,7 +22,7 @@ program fft_r2c_z
    integer :: nargin, arg, FNLength, status, DecInd
    character(len=80) :: InputFN
 
-   integer, parameter :: ntest = 10  ! repeat test this times
+   integer :: ntest = 10  ! repeat test this times
 
    type(decomp_info), pointer :: ph => null(), sp => null()
    complex(mytype), allocatable, dimension(:, :, :) :: out
@@ -45,7 +45,7 @@ program fft_r2c_z
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
-   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5) .or. (nargin == 6)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
          read (InputFN, *, iostat=status) DecInd
@@ -59,6 +59,8 @@ program fft_r2c_z
             ny = DecInd
          elseif (arg == 5) then
             nz = DecInd
+         elseif (arg == 6) then
+            ntest = DecInd
          end if
       end do
    else
@@ -69,12 +71,13 @@ program fft_r2c_z
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
-         print *, "or 5 inputs as"
+         print *, "or 5-6 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
          print *, "  3) nx "
          print *, "  4) ny "
          print *, "  5) nz "
+         print *, "  6) n iterations (optional)"
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
       end if
@@ -107,9 +110,32 @@ program fft_r2c_z
       end do
    end do
 
+   !$acc data copyin(in_r) copy(out)
+   ! First iterations out of the counting loop
+   t1 = MPI_WTIME()
+   call decomp_2d_fft_3d(in_r, out)
+   t2 = MPI_WTIME() - t1
+   t3 = MPI_WTIME()
+   call decomp_2d_fft_3d(out, in_r)
+   t4 = MPI_WTIME() - t3
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1 / dble(nproc)
+   call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t3 = t3 / dble(nproc)
+   if (nrank == 0) then
+      write (*, *) '===== r2c/c2r interface ====='
+      write (*, *) 'First iteration with dedicated timer'
+      write (*, *) '     time (sec): ', t1, t3
+      write (*, *) ''
+   end if
+   ! Init the time
    t2 = 0.d0
    t4 = 0.d0
-   !$acc data copyin(in_r) copy(out)
+   !$acc kernels
+   in_r = in_r / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+   !$acc end kernels
    do m = 1, ntest
 
       ! 3D r2c FFT
@@ -122,8 +148,9 @@ program fft_r2c_z
       call decomp_2d_fft_3d(out, in_r)
       t4 = t4 + MPI_WTIME() - t3
 
+      ! normalisation - note 2DECOMP&FFT doesn't normalise
       !$acc kernels
-      in_r = in_r / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
+      in_r = in_r / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
       !$acc end kernels
 
    end do
@@ -133,10 +160,10 @@ program fft_r2c_z
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1 / dble(nproc)
+   t1 = t1 / dble(nproc) / dble(ntest)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3 / dble(nproc)
+   t3 = t3 / dble(nproc) / dble(ntest)
 
    ! checking accuracy
    error = 0._mytype
@@ -147,20 +174,20 @@ program fft_r2c_z
             dr = real(i, mytype) / real(nx, mytype) * real(j, mytype) &
                  / real(ny, mytype) * real(k, mytype) / real(nz, mytype)
             error = error + abs(in_r(i, j, k) - dr)
-            !write(*,10) nrank,k,j,i,dr,in_r(i,j,k)
          end do
       end do
    end do
    !$acc end loop
-!10 format('in_r final ', I2,1x,I2,1x,I2,1x,I2,1x,F12.6,1x,F12.6)
 
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-   err_all = err_all / real(nx, mytype) / real(ny, mytype) / real(nz, mytype)
+   err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
    if (nrank == 0) then
-      write (*, *) '===== r2c/c2r interface ====='
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
+      write (*, *) '   '
+      write (*, *) 'fft_r2c_z completed '
+      write (*, *) '   '
    end if
    !$acc end data
 
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 78cd3c07..8211173c 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -617,8 +617,6 @@ subroutine test_halo_size(arrh, nx_expected, ny_expected, nz_expected, tag)
          write (*, *) trim(rank_lbl), " ", "+ Got:      ", nx, " ", ny, " ", nz, " "
 
          all_pass = .false.
-      else
-         write (*, *) trim(rank_lbl), " ", tag, ":PASS"
       end if
 
    end subroutine test_halo_size
diff --git a/examples/io_test/io_bench.f90 b/examples/io_test/io_bench.f90
index f8d17ab4..e60f4d62 100644
--- a/examples/io_test/io_bench.f90
+++ b/examples/io_test/io_bench.f90
@@ -81,7 +81,12 @@ program io_bench
    call decomp_2d_write_one(1, u1, 'io.dat')
    t2 = MPI_WTIME()
 
-   if (nrank == 0) write (*, *) 'I/O time: ', t2 - t1
+   if (nrank == 0) then
+      write (*, *) 'I/O time: ', t2 - t1
+      write (*, *) '   '
+      write (*, *) 'IO_bench completed '
+      write (*, *) '   '
+   end if
 
    deallocate (u1)
    call decomp_2d_finalize
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 7e0e5a3b..62537883 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -50,7 +50,7 @@ program timing2d_complex
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
-   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5) .or. (nargin == 6)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
          read (InputFN, *, iostat=status) DecInd
@@ -64,6 +64,8 @@ program timing2d_complex
             ny = DecInd
          elseif (arg == 5) then
             nz = DecInd
+         elseif (arg == 6) then
+            niter = DecInd
          end if
       end do
    else
@@ -74,12 +76,13 @@ program timing2d_complex
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
-         print *, "or 5 inputs as"
+         print *, "or 5-6 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
          print *, "  3) nx "
          print *, "  4) ny "
          print *, "  5) nz "
+         print *, "  6) n iterations (optional)"
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
       end if
@@ -144,10 +147,23 @@ program timing2d_complex
 
    ! call decomp_2d_write_one(1,u1,'u1.dat')
 
+   t1 = MPI_WTIME()
+   call transpose_x_to_y(u1, u2)
+   call transpose_y_to_z(u2, u3)
+   call transpose_z_to_y(u3, u2)
+   call transpose_y_to_x(u2, u1)
+   t2 = MPI_WTIME() - t1
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1 / dble(nproc)
+   ! Init the total times
    t2 = 0.d0
    t4 = 0.d0
    t6 = 0.d0
    t8 = 0.d0
+   if (nrank == 0) then
+      write (*, *) 'Tot time it 0 ', t1
+   end if
    do iter = 1, niter
       !!!!!!!!!!!!!!!!!!!!!!!
       ! x-pensil ==> y-pensil
@@ -259,27 +275,28 @@ program timing2d_complex
       call MPI_ALLREDUCE(MPI_IN_PLACE, error_flag, 1, MPI_LOGICAL, MPI_LOR, MPI_COMM_WORLD, ierror)
       if (ierror /= 0) call decomp_2d_abort(ierror, "MPI_ALLREDUCE")
       if (error_flag) call decomp_2d_abort(4, "error swaping y->x")
+
    end do
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1 / dble(nproc)
+   t1 = t1 / dble(nproc) / dble(niter)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3 / dble(nproc)
+   t3 = t3 / dble(nproc) / dble(niter)
    call MPI_ALLREDUCE(t6, t5, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t5 / dble(nproc)
+   t5 = t5 / dble(nproc) / dble(niter)
    call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t7 / dble(nproc)
+   t7 = t7 / dble(nproc) / dble(niter)
    t8 = t1 + t3 + t5 + t7
    if (nrank == 0) then
-      write (*, *) 'Time X->Y ', t1
-      write (*, *) 'Time Y->Z ', t3
-      write (*, *) 'Time Z->Y ', t5
-      write (*, *) 'Time Y->X ', t7
-      write (*, *) 'Time TOT  ', t8
+      write (*, *) 'Avg Time X->Y ', t1
+      write (*, *) 'Avg Time Y->Z ', t3
+      write (*, *) 'Avg Time Z->Y ', t5
+      write (*, *) 'Avg Time Y->X ', t7
+      write (*, *) 'Avg Time TOT  ', t8
    end if
 
    if (nrank == 0) then
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 9d4850e5..257d6fef 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -49,7 +49,7 @@ program timing2d_real
    ! Now we can check if user put some inputs
    ! Handle input file like a boss -- GD
    nargin = command_argument_count()
-   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5) .or. (nargin == 6)) then
       do arg = 1, nargin
          call get_command_argument(arg, InputFN, FNLength, status)
          read (InputFN, *, iostat=status) DecInd
@@ -63,6 +63,8 @@ program timing2d_real
             ny = DecInd
          elseif (arg == 5) then
             nz = DecInd
+         elseif (arg == 6) then
+            niter = DecInd
          end if
       end do
    else
@@ -73,12 +75,13 @@ program timing2d_real
          print *, "This Test takes no inputs or 2 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
-         print *, "or 5 inputs as"
+         print *, "or 5-6 inputs as"
          print *, "  1) p_row (default=0)"
          print *, "  2) p_col (default=0)"
          print *, "  3) nx "
          print *, "  4) ny "
          print *, "  5) nz "
+         print *, "  6) n iterations (optional)"
          print *, "Number of inputs is not correct and the defult settings"
          print *, "will be used"
       end if
@@ -143,10 +146,23 @@ program timing2d_real
 
    ! call decomp_2d_write_one(1,u1,'u1.dat')
 
+   t1 = MPI_WTIME()
+   call transpose_x_to_y(u1, u2)
+   call transpose_y_to_z(u2, u3)
+   call transpose_z_to_y(u3, u2)
+   call transpose_y_to_x(u2, u1)
+   t2 = MPI_WTIME() - t1
+   call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
+                      MPI_COMM_WORLD, ierror)
+   t1 = t1 / dble(nproc)
+   ! Init the total times
    t2 = 0.d0
    t4 = 0.d0
    t6 = 0.d0
    t8 = 0.d0
+   if (nrank == 0) then
+      write (*, *) 'Tot time it 0 ', t1
+   end if
    do iter = 1, niter
       !!!!!!!!!!!!!!!!!!!!!!!
       ! x-pensil ==> y-pensil
@@ -258,23 +274,23 @@ program timing2d_real
 
    call MPI_ALLREDUCE(t2, t1, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t1 = t1 / dble(nproc)
+   t1 = t1 / dble(nproc) / dble(niter)
    call MPI_ALLREDUCE(t4, t3, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t3 = t3 / dble(nproc)
+   t3 = t3 / dble(nproc) / dble(niter)
    call MPI_ALLREDUCE(t6, t5, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t5 = t5 / dble(nproc)
+   t5 = t5 / dble(nproc) / dble(niter)
    call MPI_ALLREDUCE(t8, t7, 1, MPI_DOUBLE_PRECISION, MPI_SUM, &
                       MPI_COMM_WORLD, ierror)
-   t7 = t7 / dble(nproc)
+   t7 = t7 / dble(nproc) / dble(niter)
    t8 = t1 + t3 + t5 + t7
    if (nrank == 0) then
-      write (*, *) 'Time X->Y ', t1
-      write (*, *) 'Time Y->Z ', t3
-      write (*, *) 'Time Z->Y ', t5
-      write (*, *) 'Time Y->X ', t7
-      write (*, *) 'Time TOT  ', t8
+      write (*, *) 'Avg Time X->Y ', t1
+      write (*, *) 'Avg Time Y->Z ', t3
+      write (*, *) 'Avg Time Z->Y ', t5
+      write (*, *) 'Avg Time Y->X ', t7
+      write (*, *) 'Avg Time TOT  ', t8
    end if
 
    if (nrank == 0) then
diff --git a/src/factor.f90 b/src/factor.f90
index 7c68fae9..68d666e5 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -24,7 +24,7 @@ subroutine findfactor(num, factors, nfact)
       ! find the factors <= sqrt(num)
       ! Cast the int as double to make sure of the correct result of sqrt
       ! IntelLLVM got an issue with 1.0 but not with 1.d0
-      m = int(sqrt(real(num, 8)))
+      m = int(sqrt(num*1.d0))
       nfact = 1
       do i = 1, m
          if (num / i * i == num) then
diff --git a/src/io.f90 b/src/io.f90
index 23994017..7ea699c3 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -1194,7 +1194,8 @@ subroutine coarse_extents(ipencil, icoarse, sizes, subsizes, starts, opt_decomp)
 
    end subroutine coarse_extents
 
-  subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_name, opt_decomp, reduce_prec, opt_deferred_writes)
+   subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_name, &
+                                      opt_decomp, reduce_prec, opt_deferred_writes)
 
       ! USE param
       ! USE variables

From 74744be6a833d72cffbd5f86e77eef5d372b59c8 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 30 Aug 2023 12:29:59 +0200
Subject: [PATCH 337/436] Update the README for the examples

---
 examples/README.md             | 20 +++++++-------------
 examples/fft_physical_x/README | 18 +++++++++++++++---
 examples/fft_physical_z/README | 17 ++++++++++++++---
 examples/halo_test/README      |  2 --
 examples/init_test/README      | 17 +++++++++++++++++
 examples/io_test/README        | 25 +++++++++++++++++++++++++
 examples/test2d/README         | 26 ++++++++++++++++++--------
 7 files changed, 96 insertions(+), 29 deletions(-)
 create mode 100644 examples/init_test/README
 create mode 100644 examples/io_test/README

diff --git a/examples/README.md b/examples/README.md
index 48ebdea2..c55f5307 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -1,23 +1,17 @@
 Examples
 ========
 
-* test2d       - to test the base 2D pencil decomposition module 
+* init_test      - to test the initialisation of the DECOMP2D&FFT library
 
-* fft_test_c2c - to test the complex-to-complex FFTs 
+* test2d         - various tests for the 2D pencil decomposition module and timing 
 
-* fft_test_r2c - to test the real-to-complex/complex-to-real FFTs
+* fft_physical_x - various tests for the FFT starting from the ``X`` direction 
 
-* timing       - to benchmark the FFT library
+* fft_physical_z - various tests for the FFT starting from the ``Z`` direction 
 
-* halo_test    - to test the halo-cell exchange code
+* halo_test      - to test the halo-cell exchange code
 
-* io_test      - to test various IO functions
+* io_test        - various tests for the IO module
 
-* p3dfft       - to crosscheck the library against P3DFFT
 
-* non_blocking - to test the idea of overlap communication and computation
-
-* tecplot_view - to generate Tecplot visualisation of the decomposition
-
-
-Some examples may require external libraries to be built first. Refer to the README files for each example for details.
+Refer to the README files for each example for details.
diff --git a/examples/fft_physical_x/README b/examples/fft_physical_x/README
index 01fa4dc7..769d2165 100644
--- a/examples/fft_physical_x/README
+++ b/examples/fft_physical_x/README
@@ -1,13 +1,25 @@
-fft_physical_x
+fft_c2c_x, fft_r2c_x, fft_grid_x
 ------
 
-This program can be used to test the fft trasform using X as starting physical dimension. 
-Both c2c and r2c/c2r transforms are testes.
+These programs can be used to test the fft trasform using X as starting physical dimension. 
+Both c2c (fft_c2c_x) and r2c/c2r (fft_r2c_x) transforms are tested.
+The case with the FFT grid different from the grid used for initialization is also tested (fft_grid) 
 The results should recover the input data up to machine accuracy
 after a forward and a backward transform and proper normalisation.
 The test automatically resize the problem depending on the number of MPI processes in use
 
+What to input: The program takes max 6 inputs as : 
+               1) p_row [optional]
+	       2) p_col [optional] 
+	       3) nx    [optional]
+	       4) ny    [optional]
+	       5) nz    [optional]
+	       6) nt    [optional]
+	       In case the decomposition is imposed both (1) and (2) are necessary. 
+	       If also the resolution is imposed (1-5) are necessary
+
 What to expect:
 - The timing results 
 - The error reported should be around machine accuracy (~ 10^-6 for single
   precision and 10^-15 for double)
+- In case of the GENERIC FFT expect an increase in the order of the error
diff --git a/examples/fft_physical_z/README b/examples/fft_physical_z/README
index e726d909..9c8842d6 100644
--- a/examples/fft_physical_z/README
+++ b/examples/fft_physical_z/README
@@ -1,13 +1,24 @@
-fft_physical_z
+fft_c2c_z, fft_r2c_z
 ------
 
-This program can be used to test the fft trasform using Z as starting physical dimension. 
-Both c2c and r2c/c2r transforms are testes.
+These programs can be used to test the fft trasform using Z as starting physical dimension. 
+Both c2c (fft_c2c_z) and r2c/c2r (fft_r2c_z) transforms are tested.
 The results should recover the input data up to machine accuracy
 after a forward and a backward transform and proper normalisation.
 The test automatically resize the problem depending on the number of MPI processes in use
 
+What to input: The program takes max 6 inputs as : 
+               1) p_row [optional]
+	       2) p_col [optional] 
+	       3) nx    [optional]
+	       4) ny    [optional]
+	       5) nz    [optional]
+	       6) nt    [optional]
+	       In case the decomposition is imposed both (1) and (2) are necessary. 
+	       If also the resolution is imposed (1-5) are necessary
+
 What to expect:
 - The timing results 
 - The error reported should be around machine accuracy (~ 10^-6 for single
   precision and 10^-15 for double)
+- In case of the GENERIC FFT expect an increase in the order of the error
diff --git a/examples/halo_test/README b/examples/halo_test/README
index 8fb08e25..623587f4 100644
--- a/examples/halo_test/README
+++ b/examples/halo_test/README
@@ -9,7 +9,5 @@ results are to be expected regardless of the communication algorithm. The
 computation is based on an explicit finite difference method so clearly using 
 the halo-cell support API is more efficient.
 
-To run: use 12 MPI processes.
-
 What to expect: the output using different communication algorithms should be 
 exactly the same.
diff --git a/examples/init_test/README b/examples/init_test/README
new file mode 100644
index 00000000..b683b7c7
--- /dev/null
+++ b/examples/init_test/README
@@ -0,0 +1,17 @@
+init_test
+---------
+
+This example demonstrates the initialisation of DECOMP2D&FFT library and tests 
+that the size of the mesh in every direction is the expected
+
+What to input: The program takes max 5 inputs as : 
+               1) p_row [optional]
+	       2) p_col [optional] 
+	       3) nx    [optional]
+	       4) ny    [optional]
+	       5) nz    [optional]
+	       In case the decomposition is imposed both (1) and (2) are necessary. 
+	       If also the resolution is imposed (1-5) are necessary
+
+What to expect: Success/error message if initialization process has been complted or 
+                errors have been encountered.
diff --git a/examples/io_test/README b/examples/io_test/README
new file mode 100644
index 00000000..9c64bc85
--- /dev/null
+++ b/examples/io_test/README
@@ -0,0 +1,25 @@
+io_bench, io_plane_test, io_read, io_test, io_var_test
+---------
+
+These examples demostrate the use of the different I/O features of DECOMP2D&FFT
+using both MPI I/O and ADIOS2. The test performed are the following: 
+- io_test: write of the files u1.dat, u2.dat and u3.dat allocated in the X, Y and Z pencil decomposition respectively            and the write of all 3 files in a combined single file (used for checkpointing)
+           The program checks that the written files are the excpeted.
+- io_read: read the files u1.dat, u2.dat and u3.dat written by io_test and check that they are the expected one. 
+- io_var_test: test the writing of writing real and complex data for scalar and 3D arrays 
+               and for different resolutions 
+- io_plane_test: test the writing of 2D planes in different direction  
+- io_bench: timing the writing on disk of a 3D array
+
+What to input: The program takes max 6 inputs as : 
+               1) p_row [optional]
+	       2) p_col [optional] 
+	       3) nx    [optional]
+	       4) ny    [optional]
+	       5) nz    [optional]
+	       In case the decomposition is imposed both (1) and (2) are necessary. 
+	       If also the resolution is imposed (1-5) are necessary
+
+What to expect: 
+- All programs print out a success message otherwise an error message with location of the error is reported
+- io_bench gives also the timing of the writing on disk
diff --git a/examples/test2d/README b/examples/test2d/README
index 1bf62819..ff6a8967 100644
--- a/examples/test2d/README
+++ b/examples/test2d/README
@@ -1,13 +1,23 @@
-test2d
+test2d, timing2d_real, timing2d_complex 
 ------
 
-This program is to validate the 2D pencil decomposition library. It transposes
+The program test2d is to validate the 2D pencil decomposition library. It transposes
 a set of data into different storage formats so that all communication routines
-are tested. The test code also demonstrates to use of the I/O library.
+are tested.
+The programs timing2d_real and timing2d_complex perform the same tests several times 
+and are used to report timing and performances for real and complex transforms. 
 
-To run: use 12 MPI processes.
+What to input: The program takes max 6 inputs as : 
+               1) p_row [optional]
+	       2) p_col [optional] 
+	       3) nx    [optional]
+	       4) ny    [optional]
+	       5) nz    [optional]
+	       6) nt    [optional]
+	       In case the decomposition is imposed both (1) and (2) are necessary. 
+	       If also the resolution is imposed (1-5) are necessary
 
-What to expect: many files are written to disk. Regardless of the pencil-
-orientation of the distributed data, files written collectively by all MPI
-processes should contain identical information. For example, u1.dat, u2.dat,
-u3.dat, u2b.dat and u1b.dat should be all identical.
+What to expect: 
+- For test2d the output is e success message or an error message with the direction of error in swapping. 
+- For the timing, beside the error/success message timing for all transpostions
+  (X->Y, Y->Z, Z->Y, Y->X) together with the sum are reported. 

From d7ec60306d12559a160d9de4a1c3125d98e0e694 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Sun, 3 Sep 2023 19:38:26 +0100
Subject: [PATCH 338/436] Create visualisation example

---
 examples/io_test/CMakeLists.txt |   4 +
 examples/io_test/visu.f90       | 258 ++++++++++++++++++++++++++++++++
 2 files changed, 262 insertions(+)
 create mode 100644 examples/io_test/visu.f90

diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 2f5b5ac6..13369f5d 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -3,6 +3,7 @@ file(GLOB files_read io_read.f90)
 file(GLOB files_var_test io_var_test.f90)
 file(GLOB files_plane_test io_plane_test.f90)
 file(GLOB files_bench io_bench.f90)
+file(GLOB files_visu visu.f90)
 
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
@@ -11,14 +12,17 @@ add_executable(io_read ${files_read})
 add_executable(io_var_test ${files_var_test})
 add_executable(io_plane_test ${files_plane_test})
 add_executable(io_bench ${files_bench})
+add_executable(visu ${files_visu})
 
 target_link_libraries(io_test PRIVATE decomp2d)
 target_link_libraries(io_read PRIVATE decomp2d)
 target_link_libraries(io_var_test PRIVATE decomp2d)
 target_link_libraries(io_plane_test PRIVATE decomp2d)
 target_link_libraries(io_bench PRIVATE decomp2d)
+target_link_libraries(visu PRIVATE decomp2d)
 
 # Run the test(s)
+# Note visu is not a test - it is an example to show/create visu files
 set(run_dir "${test_dir}/io_test")
 file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
 message(STATUS "Example dir ${run_dir}")
diff --git a/examples/io_test/visu.f90 b/examples/io_test/visu.f90
new file mode 100644
index 00000000..2ac04fec
--- /dev/null
+++ b/examples/io_test/visu.f90
@@ -0,0 +1,258 @@
+!!! SPDX-License-Identifier: BSD-3-Clause
+!!!
+!!! Example code to demonstrate visualisation of fields.
+
+program visu
+
+  use mpi
+
+  use decomp_2d
+  use decomp_2d_mpi
+  use decomp_2d_constants
+  use decomp_2d_io
+
+  implicit none
+
+  integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+  integer :: nx, ny, nz
+  integer :: p_row = 0, p_col = 0
+
+#ifdef COMPLEX_TEST
+  complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+#else
+  real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+#endif
+
+  integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
+   character(len=*), parameter :: io_name = "visu-io"
+#ifndef ADIOS2
+  logical :: dir_exists
+#endif
+
+  integer :: nargin, arg, FNLength, status, DecInd
+
+  integer :: i, j, k
+
+  integer :: ierr
+
+  call init_example()
+  call init_data()
+
+  call write_data()
+  call write_visu()
+  
+  call fin()
+  
+contains
+
+  subroutine init_example()
+
+    integer :: resize_domain
+    integer :: nranks_tot
+    character(len=80) :: InputFN
+
+    call MPI_INIT(ierr)
+    ! To resize the domain we need to know global number of ranks
+    ! This operation is also done as part of decomp_2d_init
+    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierr)
+    resize_domain = int(nranks_tot / 4) + 1
+    nx = nx_base * resize_domain
+    ny = ny_base * resize_domain
+    nz = nz_base * resize_domain
+    ! Now we can check if user put some inputs
+    ! Handle input file like a boss -- GD
+    nargin = command_argument_count()
+    if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+       do arg = 1, nargin
+          call get_command_argument(arg, InputFN, FNLength, status)
+          read (InputFN, *, iostat=status) DecInd
+          if (arg == 1) then
+             p_row = DecInd
+          elseif (arg == 2) then
+             p_col = DecInd
+          elseif (arg == 3) then
+             nx = DecInd
+          elseif (arg == 4) then
+             ny = DecInd
+          elseif (arg == 5) then
+             nz = DecInd
+          end if
+       end do
+    else
+       ! nrank not yet computed we need to avoid write
+       ! for every rank
+       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierr)
+       if (nrank == 0) then
+          print *, "This Test takes no inputs or 2 inputs as"
+          print *, "  1) p_row (default=0)"
+          print *, "  2) p_col (default=0)"
+          print *, "or 5 inputs as"
+          print *, "  1) p_row (default=0)"
+          print *, "  2) p_col (default=0)"
+          print *, "  3) nx "
+          print *, "  4) ny "
+          print *, "  5) nz "
+          print *, "Number of inputs is not correct and the defult settings"
+          print *, "will be used"
+       end if
+    end if
+
+    call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+    call decomp_2d_io_init()
+    call decomp_2d_init_io(io_name)
+    call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
+    call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
+    call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
+    
+  end subroutine init_example
+
+  subroutine init_data()
+
+    call alloc_x(u1)
+    call alloc_y(u2)
+    call alloc_z(u3)
+    
+    do k = 1, xsize(3)
+       do j = 1, xsize(2)
+          do i = 1, xsize(1)
+#ifndef COMPLEX_TEST
+             u1(i, j, k) = real(nrank, mytype)
+#else
+             u1(i, j, k) = cmplx(real(nrank, mytype), -real(nrank, mytype))
+#endif
+          end do
+       end do
+    end do
+    call transpose_x_to_y(u1, u2)
+    call transpose_y_to_z(u2, u3)
+    
+  end subroutine init_data
+
+  subroutine write_data()
+    
+    !! Write arrays + visu data from orientations 1, 2 and 3
+#ifndef ADIOS2
+    if (nrank == 0) then
+       inquire (file="out", exist=dir_exists)
+       if (.not. dir_exists) then
+          call execute_command_line("mkdir out 2> /dev/null")
+       end if
+    end if
+#endif
+    
+    ! Standard I/O pattern - file per field
+#ifdef ADIOS2
+    call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
+    call decomp_2d_start_io(io_name, "out")
+#endif
+    call decomp_2d_write_one(1, u1, 'out', 'u1.dat', 0, io_name)
+    call decomp_2d_write_one(2, u2, 'out', 'u2.dat', 0, io_name)
+    call decomp_2d_write_one(3, u3, 'out', 'u3.dat', 0, io_name)
+#ifdef ADIOS2
+    call decomp_2d_end_io(io_name, "out")
+    call decomp_2d_close_io(io_name, "out")
+#endif
+
+  end subroutine write_data
+
+  subroutine write_visu()
+    ! This subroutine is based on the xdmf writers in Xcompact3d.
+    ! Copyright (c) 2012-2022, Xcompact3d
+    ! SPDX-License-Identifier: BSD 3-Clause
+
+    integer :: ioxdmf
+
+    character(len=:), allocatable :: fmt
+
+    integer :: precision
+
+    integer :: varctr
+    character(len=16) :: filename
+    
+    if (nrank == 0) then
+       OPEN(newunit=ioxdmf, file="./out.xdmf")
+
+       write(ioxdmf,'(A22)')'<?xml version="1.0" ?>'
+       write(ioxdmf,*)'<!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>'
+       write(ioxdmf,*)'<Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">'
+       write(ioxdmf,*)'<Domain>'
+
+       write(ioxdmf,'(A)')'    <Topology name="topo" TopologyType="3DCoRectMesh"'
+       fmt = "(A, I0, A, I0, A, I0, A)"
+       write(ioxdmf,fmt)'        Dimensions="', nz, " ", ny, " ", nx, '">'
+       write(ioxdmf,'(A)')'    </Topology>'
+
+       write(ioxdmf,*)'    <Geometry name="geo" Type="ORIGIN_DXDYDZ">'
+       write(ioxdmf,*)'        <!-- Origin -->'
+       write(ioxdmf,*)'        <DataItem Format="XML" Dimensions="3">'
+       write(ioxdmf,*)'          0.0 0.0 0.0'
+       write(ioxdmf,*)'        </DataItem>'
+       write(ioxdmf,*)'        <!-- DxDyDz -->'
+       write(ioxdmf,*)'        <DataItem Format="XML" Dimensions="3">'
+       if (mytype == kind(0.0d0)) then
+          fmt = "(A, E24.17, A, E24.17, A, E24.17)"
+       else
+          fmt = "(A, E16.9, A, E16.9, A, E16.9)"
+       end if
+       write(ioxdmf,fmt)'        ', 1.0_mytype, " ", 1.0_mytype, " ", 1.0_mytype
+       write(ioxdmf,*)'        </DataItem>'
+       write(ioxdmf,*)'    </Geometry>'
+
+        write(ioxdmf, *)'    <Grid Name="main" GridType="Uniform">'
+       write(ioxdmf, *)'        <Topology Reference="/Xdmf/Domain/Topology[1]"/>'
+       write(ioxdmf, *)'        <Geometry Reference="/Xdmf/Domain/Geometry[1]"/>'
+
+       do varctr = 1, 3
+          write(filename, '(A, I0, A)') "./out/u", varctr, ".dat"
+          write(ioxdmf,*)'        <Attribute Name="'//trim(varname)//'" Center="Node">'
+#ifndef ADIOS2
+          write(ioxdmf,*)'           <DataItem Format="Binary"'
+#else
+          write(ioxdmf,*)'           <DataItem Format="HDF"'
+#endif
+
+#ifdef DOUBLE_PREC
+#ifdef SAVE_SINGLE
+          if (output2D.eq.0) then
+             precision = 4
+          else
+             precision = 8
+          endif
+#else
+          precision = 8
+#endif
+#else
+          precision = 4
+#endif
+          write(ioxdmf,"(A,I0,A)")'            DataType="Float" Precision="', precision, '" Endian="little" Seek="0"'
+
+          fmt = "(A, I0, A, I0, A, I0, A)"
+          write(ioxdmf,fmt)'            Dimensions="', nz, " ", ny, " ", nx, '">'
+
+          write(ioxdmf,*)'              '//trim(filename)
+
+          write(ioxdmf,*)'           </DataItem>'
+          write(ioxdmf,*)'        </Attribute>'
+       end do
+       
+       write(ioxdmf,'(/)')
+       write(ioxdmf,*)'    </Grid>'
+       write(ioxdmf,*)'</Domain>'
+       write(ioxdmf,'(A7)')'</Xdmf>'
+       close(ioxdmf)
+    endif
+    
+  end subroutine write_visu
+  
+  subroutine fin()
+
+    integer :: ierr
+    
+    call decomp_2d_finalize
+    call MPI_FINALIZE(ierr)
+
+ end subroutine fin
+  
+end program visu
+

From cbf782a61204b376b22ffec251975781af8ea3d1 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 5 Sep 2023 09:21:08 +0100
Subject: [PATCH 339/436] Fix broken commit

---
 examples/io_test/visu.f90 | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/examples/io_test/visu.f90 b/examples/io_test/visu.f90
index 2ac04fec..aae27955 100644
--- a/examples/io_test/visu.f90
+++ b/examples/io_test/visu.f90
@@ -24,7 +24,7 @@ program visu
 #endif
 
   integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
-   character(len=*), parameter :: io_name = "visu-io"
+  character(len=*), parameter :: io_name = "visu-io"
 #ifndef ADIOS2
   logical :: dir_exists
 #endif
@@ -169,6 +169,7 @@ subroutine write_visu()
 
     integer :: varctr
     character(len=16) :: filename
+    character(len=2) :: varname
     
     if (nrank == 0) then
        OPEN(newunit=ioxdmf, file="./out.xdmf")
@@ -199,20 +200,22 @@ subroutine write_visu()
        write(ioxdmf,*)'        </DataItem>'
        write(ioxdmf,*)'    </Geometry>'
 
-        write(ioxdmf, *)'    <Grid Name="main" GridType="Uniform">'
-       write(ioxdmf, *)'        <Topology Reference="/Xdmf/Domain/Topology[1]"/>'
-       write(ioxdmf, *)'        <Geometry Reference="/Xdmf/Domain/Geometry[1]"/>'
+       write(ioxdmf, *)'   <Grid Name="1" GridType="Uniform">'
+       write(ioxdmf, *)'       <Topology Reference="/Xdmf/Domain/Topology[1]"/>'
+       write(ioxdmf, *)'       <Geometry Reference="/Xdmf/Domain/Geometry[1]"/>'
 
        do varctr = 1, 3
+          write(varname, "(A, I0)") "u", varctr
           write(filename, '(A, I0, A)') "./out/u", varctr, ".dat"
-          write(ioxdmf,*)'        <Attribute Name="'//trim(varname)//'" Center="Node">'
+          write(ioxdmf,*)'       <Attribute Name="'//trim(varname)//'" Center="Node">'
 #ifndef ADIOS2
-          write(ioxdmf,*)'           <DataItem Format="Binary"'
+          write(ioxdmf,*)'          <DataItem Format="Binary"'
 #else
-          write(ioxdmf,*)'           <DataItem Format="HDF"'
+          write(ioxdmf,*)'          <DataItem Format="HDF"'
 #endif
 
 #ifdef DOUBLE_PREC
+          print *, "Double precision build"
 #ifdef SAVE_SINGLE
           if (output2D.eq.0) then
              precision = 4

From 6bc5c5a02208d78dce2dd5058fadc926fdce7cef Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 5 Sep 2023 10:16:19 +0100
Subject: [PATCH 340/436] visu test output real/imaginary fields for complex
 tests

---
 examples/CMakeLists.txt   |  3 +++
 examples/io_test/visu.f90 | 53 ++++++++++++++++++++++++++++++++++++---
 2 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index a2da4c8e..93991071 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -12,3 +12,6 @@ add_subdirectory(fft_physical_x)
 add_subdirectory(fft_physical_z)
 add_subdirectory(halo_test)
 add_subdirectory(io_test)
+
+# Set real/complex tests
+set(COMPLEX_TESTS "OFF" CACHE STRING "Enables complex numbers for tests that support it")
diff --git a/examples/io_test/visu.f90 b/examples/io_test/visu.f90
index aae27955..28f28592 100644
--- a/examples/io_test/visu.f90
+++ b/examples/io_test/visu.f90
@@ -101,10 +101,19 @@ subroutine init_example()
 
     call decomp_2d_io_init()
     call decomp_2d_init_io(io_name)
+#ifndef COMPLEX_TEST
     call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
     call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
     call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
-    
+#else
+    call decomp_2d_register_variable(io_name, "u1_re.dat", 1, 0, output2D, mytype)
+    call decomp_2d_register_variable(io_name, "u2_re.dat", 2, 0, output2D, mytype)
+    call decomp_2d_register_variable(io_name, "u3_re.dat", 3, 0, output2D, mytype)
+    call decomp_2d_register_variable(io_name, "u1_im.dat", 1, 0, output2D, mytype)
+    call decomp_2d_register_variable(io_name, "u2_im.dat", 2, 0, output2D, mytype)
+    call decomp_2d_register_variable(io_name, "u3_im.dat", 3, 0, output2D, mytype)
+#endif
+
   end subroutine init_example
 
   subroutine init_data()
@@ -124,8 +133,28 @@ subroutine init_data()
           end do
        end do
     end do
-    call transpose_x_to_y(u1, u2)
-    call transpose_y_to_z(u2, u3)
+    do k = 1, ysize(3)
+       do j = 1, ysize(2)
+          do i = 1, ysize(1)
+#ifndef COMPLEX_TEST
+             u2(i, j, k) = real(nrank, mytype)
+#else
+             u2(i, j, k) = cmplx(real(nrank, mytype), -real(nrank, mytype))
+#endif
+          end do
+       end do
+    end do
+    do k = 1, zsize(3)
+       do j = 1, zsize(2)
+          do i = 1, zsize(1)
+#ifndef COMPLEX_TEST
+             u3(i, j, k) = real(nrank, mytype)
+#else
+             u3(i, j, k) = cmplx(real(nrank, mytype), -real(nrank, mytype))
+#endif
+          end do
+       end do
+    end do
     
   end subroutine init_data
 
@@ -146,9 +175,20 @@ subroutine write_data()
     call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
     call decomp_2d_start_io(io_name, "out")
 #endif
+
+#ifndef COMPLEX_TEST
     call decomp_2d_write_one(1, u1, 'out', 'u1.dat', 0, io_name)
     call decomp_2d_write_one(2, u2, 'out', 'u2.dat', 0, io_name)
     call decomp_2d_write_one(3, u3, 'out', 'u3.dat', 0, io_name)
+#else
+    call decomp_2d_write_one(1, real(u1, mytype), 'out', 'u1_re.dat', 0, io_name)
+    call decomp_2d_write_one(2, real(u1, mytype), 'out', 'u2_re.dat', 0, io_name)
+    call decomp_2d_write_one(3, real(u1, mytype), 'out', 'u3_re.dat', 0, io_name)
+    call decomp_2d_write_one(1, aimag(u1, mytype), 'out', 'u1_im.dat', 0, io_name)
+    call decomp_2d_write_one(2, aimag(u2, mytype), 'out', 'u2_im.dat', 0, io_name)
+    call decomp_2d_write_one(3, aimag(u3, mytype), 'out', 'u3_im.dat', 0, io_name)
+#endif
+
 #ifdef ADIOS2
     call decomp_2d_end_io(io_name, "out")
     call decomp_2d_close_io(io_name, "out")
@@ -205,8 +245,15 @@ subroutine write_visu()
        write(ioxdmf, *)'       <Geometry Reference="/Xdmf/Domain/Geometry[1]"/>'
 
        do varctr = 1, 3
+#ifndef COMPLEX_TEST
           write(varname, "(A, I0)") "u", varctr
           write(filename, '(A, I0, A)') "./out/u", varctr, ".dat"
+#else
+          write(varname, "(A, I0, A)") "u", varctr, "_re"
+          write(filename, '(A, I0, A)') "./out/u", varctr, "_re.dat"
+          write(varname, "(A, I0, A)") "u", varctr, "_im"
+          write(filename, '(A, I0, A)') "./out/u", varctr, "_im.dat"
+#endif
           write(ioxdmf,*)'       <Attribute Name="'//trim(varname)//'" Center="Node">'
 #ifndef ADIOS2
           write(ioxdmf,*)'          <DataItem Format="Binary"'

From 9061647068b155cc83dfd03e0bda304160f36c07 Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 12 Sep 2023 10:50:09 +0100
Subject: [PATCH 341/436] Correct halo test - use local, not global variable

---
 examples/halo_test/halo_test.f90 | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index 8211173c..cb6ab3ad 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -37,7 +37,6 @@ program halo_test
 
    integer, allocatable, dimension(:) :: seed
 
-   real(mytype) :: err
    integer :: xlast, ylast, zlast
 
    integer :: nx_expected, ny_expected, nz_expected
@@ -534,7 +533,7 @@ subroutine check_err(divh, divref, pencil)
       !$acc end kernels
       divmag = mag(tmp)
 
-      if (err < epsilon(divmag) * divmag) then
+      if (error < epsilon(divmag) * divmag) then
          passing = .true.
       else
          passing = .false.
@@ -547,7 +546,7 @@ subroutine check_err(divh, divref, pencil)
 #ifdef DEBUG
          write (*, *) (divh(i, i, i), i=2, 13)
 #endif
-         write (*, *) 'Error: ', err, '; Relative: ', err / divmag
+         write (*, *) 'Error: ', error, '; Relative: ', error / divmag
          write (*, *) 'Pass: ', passing
       end if
       deallocate (tmp)

From 4fc5d8cdacd005ad2d497abd88486e984093b9ae Mon Sep 17 00:00:00 2001
From: Paul <ptb0890@gmail.com>
Date: Tue, 12 Sep 2023 16:06:46 +0100
Subject: [PATCH 342/436] Correcting halo test

---
 examples/halo_test/halo_test.f90 | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index e33be609..63545010 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -36,7 +36,6 @@ program halo_test
 
    integer, allocatable, dimension(:) :: seed
 
-   real(mytype) :: err
    integer :: xlast, ylast, zlast
 
    integer :: nx_expected, ny_expected, nz_expected
@@ -509,15 +508,15 @@ subroutine check_err(divh, pencil)
       allocate (tmp(size(divh, 1), size(divh, 2), size(divh, 3)))
 
       !$acc kernels default(present)
-      tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - div1(2:xlast, 2:ylast, 2:zlast)
+      tmp(2:xlast, 2:ylast, 2:zlast) = divh(2:xlast, 2:ylast, 2:zlast) - div(2:xlast, 2:ylast, 2:zlast)
       !$acc end kernels
       error = mag(tmp)
       !$acc kernels default(present)
-      tmp(2:xlast, 2:ylast, 2:zlast) = div1(2:xlast, 2:ylast, 2:zlast)
+      tmp(2:xlast, 2:ylast, 2:zlast) = div(2:xlast, 2:ylast, 2:zlast)
       !$acc end kernels
       divmag = mag(tmp)
 
-      if (err < epsilon(divmag) * divmag) then
+      if (error < epsilon(divmag) * divmag) then
          passing = .true.
       else
          passing = .false.
@@ -530,7 +529,8 @@ subroutine check_err(divh, pencil)
 #ifdef DEBUG
          write (*, *) (divh(i, i, i), i=2, 13)
 #endif
-         write (*, *) 'Error: ', err, '; Relative: ', err / divmag
+         print *, error, divmag
+         write (*, *) 'Error: ', error, '; Relative: ', error / divmag
          write (*, *) 'Pass: ', passing
       end if
       deallocate (tmp)

From 3ce989dbedfa950347ff95e27ebdf2804cffe185 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 16 Oct 2023 14:17:04 +0100
Subject: [PATCH 343/436] Update the visu example to support only real and use
 GPU

---
 examples/CMakeLists.txt                    |  2 +-
 examples/io_test/CMakeLists.txt            | 17 ++++--
 examples/io_test/{visu.f90 => io_visu.f90} | 60 ++++++----------------
 3 files changed, 31 insertions(+), 48 deletions(-)
 rename examples/io_test/{visu.f90 => io_visu.f90} (79%)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 93991071..c2fa43d0 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -14,4 +14,4 @@ add_subdirectory(halo_test)
 add_subdirectory(io_test)
 
 # Set real/complex tests
-set(COMPLEX_TESTS "OFF" CACHE STRING "Enables complex numbers for tests that support it")
+#set(COMPLEX_TESTS "OFF" CACHE STRING "Enables complex numbers for tests that support it")
diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 13369f5d..65329b2c 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -3,7 +3,7 @@ file(GLOB files_read io_read.f90)
 file(GLOB files_var_test io_var_test.f90)
 file(GLOB files_plane_test io_plane_test.f90)
 file(GLOB files_bench io_bench.f90)
-file(GLOB files_visu visu.f90)
+file(GLOB files_visu io_visu.f90)
 
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
@@ -12,14 +12,14 @@ add_executable(io_read ${files_read})
 add_executable(io_var_test ${files_var_test})
 add_executable(io_plane_test ${files_plane_test})
 add_executable(io_bench ${files_bench})
-add_executable(visu ${files_visu})
+add_executable(io_visu ${files_visu})
 
 target_link_libraries(io_test PRIVATE decomp2d)
 target_link_libraries(io_read PRIVATE decomp2d)
 target_link_libraries(io_var_test PRIVATE decomp2d)
 target_link_libraries(io_plane_test PRIVATE decomp2d)
 target_link_libraries(io_bench PRIVATE decomp2d)
-target_link_libraries(visu PRIVATE decomp2d)
+target_link_libraries(io_visu PRIVATE decomp2d)
 
 # Run the test(s)
 # Note visu is not a test - it is an example to show/create visu files
@@ -68,3 +68,14 @@ if (BUILD_TARGET MATCHES "gpu")
 else ()
   add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_bench> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
+# Run the test(s)
+set(run_dir "${test_dir}/io_visu")
+file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME io_visu COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:i0_visu> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+else ()
+  add_test(NAME io_visu COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_visu> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+endif ()
diff --git a/examples/io_test/visu.f90 b/examples/io_test/io_visu.f90
similarity index 79%
rename from examples/io_test/visu.f90
rename to examples/io_test/io_visu.f90
index 28f28592..3a369ac1 100644
--- a/examples/io_test/visu.f90
+++ b/examples/io_test/io_visu.f90
@@ -17,11 +17,7 @@ program visu
   integer :: nx, ny, nz
   integer :: p_row = 0, p_col = 0
 
-#ifdef COMPLEX_TEST
-  complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
-#else
   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
-#endif
 
   integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
   character(len=*), parameter :: io_name = "visu-io"
@@ -101,18 +97,9 @@ subroutine init_example()
 
     call decomp_2d_io_init()
     call decomp_2d_init_io(io_name)
-#ifndef COMPLEX_TEST
     call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
     call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
     call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
-#else
-    call decomp_2d_register_variable(io_name, "u1_re.dat", 1, 0, output2D, mytype)
-    call decomp_2d_register_variable(io_name, "u2_re.dat", 2, 0, output2D, mytype)
-    call decomp_2d_register_variable(io_name, "u3_re.dat", 3, 0, output2D, mytype)
-    call decomp_2d_register_variable(io_name, "u1_im.dat", 1, 0, output2D, mytype)
-    call decomp_2d_register_variable(io_name, "u2_im.dat", 2, 0, output2D, mytype)
-    call decomp_2d_register_variable(io_name, "u3_im.dat", 3, 0, output2D, mytype)
-#endif
 
   end subroutine init_example
 
@@ -122,39 +109,40 @@ subroutine init_data()
     call alloc_y(u2)
     call alloc_z(u3)
     
+    ! distribute the data
+    !$acc data copy(u1,u2,u3)
+
+    !$acc parallel loop default(present)
     do k = 1, xsize(3)
        do j = 1, xsize(2)
           do i = 1, xsize(1)
-#ifndef COMPLEX_TEST
-             u1(i, j, k) = real(nrank, mytype)
-#else
-             u1(i, j, k) = cmplx(real(nrank, mytype), -real(nrank, mytype))
-#endif
+             u1(i, j, k) = real(100+nrank, mytype)
           end do
        end do
     end do
+    !$acc end loop
+    !$acc parallel loop default(present)
     do k = 1, ysize(3)
        do j = 1, ysize(2)
           do i = 1, ysize(1)
-#ifndef COMPLEX_TEST
-             u2(i, j, k) = real(nrank, mytype)
-#else
-             u2(i, j, k) = cmplx(real(nrank, mytype), -real(nrank, mytype))
-#endif
+             u2(i, j, k) = real(100+nrank, mytype)
           end do
        end do
     end do
+    !$acc end loop
+    !$acc parallel loop default(present)
     do k = 1, zsize(3)
        do j = 1, zsize(2)
           do i = 1, zsize(1)
-#ifndef COMPLEX_TEST
-             u3(i, j, k) = real(nrank, mytype)
-#else
-             u3(i, j, k) = cmplx(real(nrank, mytype), -real(nrank, mytype))
-#endif
+             u3(i, j, k) = real(100+nrank, mytype)
           end do
        end do
     end do
+    !$acc end loop
+    !$acc update self (u1)
+    !$acc update self (u2)
+    !$acc update self (u3)
+    !$acc end data
     
   end subroutine init_data
 
@@ -176,18 +164,9 @@ subroutine write_data()
     call decomp_2d_start_io(io_name, "out")
 #endif
 
-#ifndef COMPLEX_TEST
     call decomp_2d_write_one(1, u1, 'out', 'u1.dat', 0, io_name)
     call decomp_2d_write_one(2, u2, 'out', 'u2.dat', 0, io_name)
     call decomp_2d_write_one(3, u3, 'out', 'u3.dat', 0, io_name)
-#else
-    call decomp_2d_write_one(1, real(u1, mytype), 'out', 'u1_re.dat', 0, io_name)
-    call decomp_2d_write_one(2, real(u1, mytype), 'out', 'u2_re.dat', 0, io_name)
-    call decomp_2d_write_one(3, real(u1, mytype), 'out', 'u3_re.dat', 0, io_name)
-    call decomp_2d_write_one(1, aimag(u1, mytype), 'out', 'u1_im.dat', 0, io_name)
-    call decomp_2d_write_one(2, aimag(u2, mytype), 'out', 'u2_im.dat', 0, io_name)
-    call decomp_2d_write_one(3, aimag(u3, mytype), 'out', 'u3_im.dat', 0, io_name)
-#endif
 
 #ifdef ADIOS2
     call decomp_2d_end_io(io_name, "out")
@@ -245,15 +224,8 @@ subroutine write_visu()
        write(ioxdmf, *)'       <Geometry Reference="/Xdmf/Domain/Geometry[1]"/>'
 
        do varctr = 1, 3
-#ifndef COMPLEX_TEST
           write(varname, "(A, I0)") "u", varctr
           write(filename, '(A, I0, A)') "./out/u", varctr, ".dat"
-#else
-          write(varname, "(A, I0, A)") "u", varctr, "_re"
-          write(filename, '(A, I0, A)') "./out/u", varctr, "_re.dat"
-          write(varname, "(A, I0, A)") "u", varctr, "_im"
-          write(filename, '(A, I0, A)') "./out/u", varctr, "_im.dat"
-#endif
           write(ioxdmf,*)'       <Attribute Name="'//trim(varname)//'" Center="Node">'
 #ifndef ADIOS2
           write(ioxdmf,*)'          <DataItem Format="Binary"'

From fc4ef2a6c0c8815e8d764f8c1466533fb5236dc4 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 16 Oct 2023 14:18:38 +0100
Subject: [PATCH 344/436] fprettify the commit

---
 examples/io_test/io_visu.f90 | 476 +++++++++++++++++------------------
 src/factor.f90               |   2 +-
 2 files changed, 239 insertions(+), 239 deletions(-)

diff --git a/examples/io_test/io_visu.f90 b/examples/io_test/io_visu.f90
index 3a369ac1..26efc597 100644
--- a/examples/io_test/io_visu.f90
+++ b/examples/io_test/io_visu.f90
@@ -4,277 +4,277 @@
 
 program visu
 
-  use mpi
+   use mpi
 
-  use decomp_2d
-  use decomp_2d_mpi
-  use decomp_2d_constants
-  use decomp_2d_io
+   use decomp_2d
+   use decomp_2d_mpi
+   use decomp_2d_constants
+   use decomp_2d_io
 
-  implicit none
+   implicit none
 
-  integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
-  integer :: nx, ny, nz
-  integer :: p_row = 0, p_col = 0
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
 
-  real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
 
-  integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
-  character(len=*), parameter :: io_name = "visu-io"
+   integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
+   character(len=*), parameter :: io_name = "visu-io"
 #ifndef ADIOS2
-  logical :: dir_exists
+   logical :: dir_exists
 #endif
 
-  integer :: nargin, arg, FNLength, status, DecInd
+   integer :: nargin, arg, FNLength, status, DecInd
 
-  integer :: i, j, k
+   integer :: i, j, k
 
-  integer :: ierr
+   integer :: ierr
 
-  call init_example()
-  call init_data()
+   call init_example()
+   call init_data()
+
+   call write_data()
+   call write_visu()
+
+   call fin()
 
-  call write_data()
-  call write_visu()
-  
-  call fin()
-  
 contains
 
-  subroutine init_example()
-
-    integer :: resize_domain
-    integer :: nranks_tot
-    character(len=80) :: InputFN
-
-    call MPI_INIT(ierr)
-    ! To resize the domain we need to know global number of ranks
-    ! This operation is also done as part of decomp_2d_init
-    call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierr)
-    resize_domain = int(nranks_tot / 4) + 1
-    nx = nx_base * resize_domain
-    ny = ny_base * resize_domain
-    nz = nz_base * resize_domain
-    ! Now we can check if user put some inputs
-    ! Handle input file like a boss -- GD
-    nargin = command_argument_count()
-    if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
-       do arg = 1, nargin
-          call get_command_argument(arg, InputFN, FNLength, status)
-          read (InputFN, *, iostat=status) DecInd
-          if (arg == 1) then
-             p_row = DecInd
-          elseif (arg == 2) then
-             p_col = DecInd
-          elseif (arg == 3) then
-             nx = DecInd
-          elseif (arg == 4) then
-             ny = DecInd
-          elseif (arg == 5) then
-             nz = DecInd
-          end if
-       end do
-    else
-       ! nrank not yet computed we need to avoid write
-       ! for every rank
-       call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierr)
-       if (nrank == 0) then
-          print *, "This Test takes no inputs or 2 inputs as"
-          print *, "  1) p_row (default=0)"
-          print *, "  2) p_col (default=0)"
-          print *, "or 5 inputs as"
-          print *, "  1) p_row (default=0)"
-          print *, "  2) p_col (default=0)"
-          print *, "  3) nx "
-          print *, "  4) ny "
-          print *, "  5) nz "
-          print *, "Number of inputs is not correct and the defult settings"
-          print *, "will be used"
-       end if
-    end if
-
-    call decomp_2d_init(nx, ny, nz, p_row, p_col)
-
-    call decomp_2d_io_init()
-    call decomp_2d_init_io(io_name)
-    call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
-    call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
-    call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
-
-  end subroutine init_example
-
-  subroutine init_data()
-
-    call alloc_x(u1)
-    call alloc_y(u2)
-    call alloc_z(u3)
-    
-    ! distribute the data
-    !$acc data copy(u1,u2,u3)
-
-    !$acc parallel loop default(present)
-    do k = 1, xsize(3)
-       do j = 1, xsize(2)
-          do i = 1, xsize(1)
-             u1(i, j, k) = real(100+nrank, mytype)
-          end do
-       end do
-    end do
-    !$acc end loop
-    !$acc parallel loop default(present)
-    do k = 1, ysize(3)
-       do j = 1, ysize(2)
-          do i = 1, ysize(1)
-             u2(i, j, k) = real(100+nrank, mytype)
-          end do
-       end do
-    end do
-    !$acc end loop
-    !$acc parallel loop default(present)
-    do k = 1, zsize(3)
-       do j = 1, zsize(2)
-          do i = 1, zsize(1)
-             u3(i, j, k) = real(100+nrank, mytype)
-          end do
-       end do
-    end do
-    !$acc end loop
-    !$acc update self (u1)
-    !$acc update self (u2)
-    !$acc update self (u3)
-    !$acc end data
-    
-  end subroutine init_data
-
-  subroutine write_data()
-    
+   subroutine init_example()
+
+      integer :: resize_domain
+      integer :: nranks_tot
+      character(len=80) :: InputFN
+
+      call MPI_INIT(ierr)
+      ! To resize the domain we need to know global number of ranks
+      ! This operation is also done as part of decomp_2d_init
+      call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierr)
+      resize_domain = int(nranks_tot / 4) + 1
+      nx = nx_base * resize_domain
+      ny = ny_base * resize_domain
+      nz = nz_base * resize_domain
+      ! Now we can check if user put some inputs
+      ! Handle input file like a boss -- GD
+      nargin = command_argument_count()
+      if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+         do arg = 1, nargin
+            call get_command_argument(arg, InputFN, FNLength, status)
+            read (InputFN, *, iostat=status) DecInd
+            if (arg == 1) then
+               p_row = DecInd
+            elseif (arg == 2) then
+               p_col = DecInd
+            elseif (arg == 3) then
+               nx = DecInd
+            elseif (arg == 4) then
+               ny = DecInd
+            elseif (arg == 5) then
+               nz = DecInd
+            end if
+         end do
+      else
+         ! nrank not yet computed we need to avoid write
+         ! for every rank
+         call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierr)
+         if (nrank == 0) then
+            print *, "This Test takes no inputs or 2 inputs as"
+            print *, "  1) p_row (default=0)"
+            print *, "  2) p_col (default=0)"
+            print *, "or 5 inputs as"
+            print *, "  1) p_row (default=0)"
+            print *, "  2) p_col (default=0)"
+            print *, "  3) nx "
+            print *, "  4) ny "
+            print *, "  5) nz "
+            print *, "Number of inputs is not correct and the defult settings"
+            print *, "will be used"
+         end if
+      end if
+
+      call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+      call decomp_2d_io_init()
+      call decomp_2d_init_io(io_name)
+      call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
+      call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
+      call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
+
+   end subroutine init_example
+
+   subroutine init_data()
+
+      call alloc_x(u1)
+      call alloc_y(u2)
+      call alloc_z(u3)
+
+      ! distribute the data
+      !$acc data copy(u1,u2,u3)
+
+      !$acc parallel loop default(present)
+      do k = 1, xsize(3)
+         do j = 1, xsize(2)
+            do i = 1, xsize(1)
+               u1(i, j, k) = real(100 + nrank, mytype)
+            end do
+         end do
+      end do
+      !$acc end loop
+      !$acc parallel loop default(present)
+      do k = 1, ysize(3)
+         do j = 1, ysize(2)
+            do i = 1, ysize(1)
+               u2(i, j, k) = real(100 + nrank, mytype)
+            end do
+         end do
+      end do
+      !$acc end loop
+      !$acc parallel loop default(present)
+      do k = 1, zsize(3)
+         do j = 1, zsize(2)
+            do i = 1, zsize(1)
+               u3(i, j, k) = real(100 + nrank, mytype)
+            end do
+         end do
+      end do
+      !$acc end loop
+      !$acc update self (u1)
+      !$acc update self (u2)
+      !$acc update self (u3)
+      !$acc end data
+
+   end subroutine init_data
+
+   subroutine write_data()
+
     !! Write arrays + visu data from orientations 1, 2 and 3
 #ifndef ADIOS2
-    if (nrank == 0) then
-       inquire (file="out", exist=dir_exists)
-       if (.not. dir_exists) then
-          call execute_command_line("mkdir out 2> /dev/null")
-       end if
-    end if
+      if (nrank == 0) then
+         inquire (file="out", exist=dir_exists)
+         if (.not. dir_exists) then
+            call execute_command_line("mkdir out 2> /dev/null")
+         end if
+      end if
 #endif
-    
-    ! Standard I/O pattern - file per field
+
+      ! Standard I/O pattern - file per field
 #ifdef ADIOS2
-    call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
-    call decomp_2d_start_io(io_name, "out")
+      call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
+      call decomp_2d_start_io(io_name, "out")
 #endif
 
-    call decomp_2d_write_one(1, u1, 'out', 'u1.dat', 0, io_name)
-    call decomp_2d_write_one(2, u2, 'out', 'u2.dat', 0, io_name)
-    call decomp_2d_write_one(3, u3, 'out', 'u3.dat', 0, io_name)
+      call decomp_2d_write_one(1, u1, 'out', 'u1.dat', 0, io_name)
+      call decomp_2d_write_one(2, u2, 'out', 'u2.dat', 0, io_name)
+      call decomp_2d_write_one(3, u3, 'out', 'u3.dat', 0, io_name)
 
 #ifdef ADIOS2
-    call decomp_2d_end_io(io_name, "out")
-    call decomp_2d_close_io(io_name, "out")
+      call decomp_2d_end_io(io_name, "out")
+      call decomp_2d_close_io(io_name, "out")
 #endif
 
-  end subroutine write_data
-
-  subroutine write_visu()
-    ! This subroutine is based on the xdmf writers in Xcompact3d.
-    ! Copyright (c) 2012-2022, Xcompact3d
-    ! SPDX-License-Identifier: BSD 3-Clause
-
-    integer :: ioxdmf
-
-    character(len=:), allocatable :: fmt
-
-    integer :: precision
-
-    integer :: varctr
-    character(len=16) :: filename
-    character(len=2) :: varname
-    
-    if (nrank == 0) then
-       OPEN(newunit=ioxdmf, file="./out.xdmf")
-
-       write(ioxdmf,'(A22)')'<?xml version="1.0" ?>'
-       write(ioxdmf,*)'<!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>'
-       write(ioxdmf,*)'<Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">'
-       write(ioxdmf,*)'<Domain>'
-
-       write(ioxdmf,'(A)')'    <Topology name="topo" TopologyType="3DCoRectMesh"'
-       fmt = "(A, I0, A, I0, A, I0, A)"
-       write(ioxdmf,fmt)'        Dimensions="', nz, " ", ny, " ", nx, '">'
-       write(ioxdmf,'(A)')'    </Topology>'
-
-       write(ioxdmf,*)'    <Geometry name="geo" Type="ORIGIN_DXDYDZ">'
-       write(ioxdmf,*)'        <!-- Origin -->'
-       write(ioxdmf,*)'        <DataItem Format="XML" Dimensions="3">'
-       write(ioxdmf,*)'          0.0 0.0 0.0'
-       write(ioxdmf,*)'        </DataItem>'
-       write(ioxdmf,*)'        <!-- DxDyDz -->'
-       write(ioxdmf,*)'        <DataItem Format="XML" Dimensions="3">'
-       if (mytype == kind(0.0d0)) then
-          fmt = "(A, E24.17, A, E24.17, A, E24.17)"
-       else
-          fmt = "(A, E16.9, A, E16.9, A, E16.9)"
-       end if
-       write(ioxdmf,fmt)'        ', 1.0_mytype, " ", 1.0_mytype, " ", 1.0_mytype
-       write(ioxdmf,*)'        </DataItem>'
-       write(ioxdmf,*)'    </Geometry>'
-
-       write(ioxdmf, *)'   <Grid Name="1" GridType="Uniform">'
-       write(ioxdmf, *)'       <Topology Reference="/Xdmf/Domain/Topology[1]"/>'
-       write(ioxdmf, *)'       <Geometry Reference="/Xdmf/Domain/Geometry[1]"/>'
-
-       do varctr = 1, 3
-          write(varname, "(A, I0)") "u", varctr
-          write(filename, '(A, I0, A)') "./out/u", varctr, ".dat"
-          write(ioxdmf,*)'       <Attribute Name="'//trim(varname)//'" Center="Node">'
+   end subroutine write_data
+
+   subroutine write_visu()
+      ! This subroutine is based on the xdmf writers in Xcompact3d.
+      ! Copyright (c) 2012-2022, Xcompact3d
+      ! SPDX-License-Identifier: BSD 3-Clause
+
+      integer :: ioxdmf
+
+      character(len=:), allocatable :: fmt
+
+      integer :: precision
+
+      integer :: varctr
+      character(len=16) :: filename
+      character(len=2) :: varname
+
+      if (nrank == 0) then
+         OPEN (newunit=ioxdmf, file="./out.xdmf")
+
+         write (ioxdmf, '(A22)') '<?xml version="1.0" ?>'
+         write (ioxdmf, *) '<!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>'
+         write (ioxdmf, *) '<Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">'
+         write (ioxdmf, *) '<Domain>'
+
+         write (ioxdmf, '(A)') '    <Topology name="topo" TopologyType="3DCoRectMesh"'
+         fmt = "(A, I0, A, I0, A, I0, A)"
+         write (ioxdmf, fmt) '        Dimensions="', nz, " ", ny, " ", nx, '">'
+         write (ioxdmf, '(A)') '    </Topology>'
+
+         write (ioxdmf, *) '    <Geometry name="geo" Type="ORIGIN_DXDYDZ">'
+         write (ioxdmf, *) '        <!-- Origin -->'
+         write (ioxdmf, *) '        <DataItem Format="XML" Dimensions="3">'
+         write (ioxdmf, *) '          0.0 0.0 0.0'
+         write (ioxdmf, *) '        </DataItem>'
+         write (ioxdmf, *) '        <!-- DxDyDz -->'
+         write (ioxdmf, *) '        <DataItem Format="XML" Dimensions="3">'
+         if (mytype == kind(0.0d0)) then
+            fmt = "(A, E24.17, A, E24.17, A, E24.17)"
+         else
+            fmt = "(A, E16.9, A, E16.9, A, E16.9)"
+         end if
+         write (ioxdmf, fmt) '        ', 1.0_mytype, " ", 1.0_mytype, " ", 1.0_mytype
+         write (ioxdmf, *) '        </DataItem>'
+         write (ioxdmf, *) '    </Geometry>'
+
+         write (ioxdmf, *) '   <Grid Name="1" GridType="Uniform">'
+         write (ioxdmf, *) '       <Topology Reference="/Xdmf/Domain/Topology[1]"/>'
+         write (ioxdmf, *) '       <Geometry Reference="/Xdmf/Domain/Geometry[1]"/>'
+
+         do varctr = 1, 3
+            write (varname, "(A, I0)") "u", varctr
+            write (filename, '(A, I0, A)') "./out/u", varctr, ".dat"
+            write (ioxdmf, *) '       <Attribute Name="'//trim(varname)//'" Center="Node">'
 #ifndef ADIOS2
-          write(ioxdmf,*)'          <DataItem Format="Binary"'
+            write (ioxdmf, *) '          <DataItem Format="Binary"'
 #else
-          write(ioxdmf,*)'          <DataItem Format="HDF"'
+            write (ioxdmf, *) '          <DataItem Format="HDF"'
 #endif
 
 #ifdef DOUBLE_PREC
-          print *, "Double precision build"
+            print *, "Double precision build"
 #ifdef SAVE_SINGLE
-          if (output2D.eq.0) then
-             precision = 4
-          else
-             precision = 8
-          endif
+            if (output2D == 0) then
+               precision = 4
+            else
+               precision = 8
+            end if
 #else
-          precision = 8
+            precision = 8
 #endif
 #else
-          precision = 4
+            precision = 4
 #endif
-          write(ioxdmf,"(A,I0,A)")'            DataType="Float" Precision="', precision, '" Endian="little" Seek="0"'
-
-          fmt = "(A, I0, A, I0, A, I0, A)"
-          write(ioxdmf,fmt)'            Dimensions="', nz, " ", ny, " ", nx, '">'
-
-          write(ioxdmf,*)'              '//trim(filename)
-
-          write(ioxdmf,*)'           </DataItem>'
-          write(ioxdmf,*)'        </Attribute>'
-       end do
-       
-       write(ioxdmf,'(/)')
-       write(ioxdmf,*)'    </Grid>'
-       write(ioxdmf,*)'</Domain>'
-       write(ioxdmf,'(A7)')'</Xdmf>'
-       close(ioxdmf)
-    endif
-    
-  end subroutine write_visu
-  
-  subroutine fin()
-
-    integer :: ierr
-    
-    call decomp_2d_finalize
-    call MPI_FINALIZE(ierr)
-
- end subroutine fin
-  
+            write (ioxdmf, "(A,I0,A)") '            DataType="Float" Precision="', precision, '" Endian="little" Seek="0"'
+
+            fmt = "(A, I0, A, I0, A, I0, A)"
+            write (ioxdmf, fmt) '            Dimensions="', nz, " ", ny, " ", nx, '">'
+
+            write (ioxdmf, *) '              '//trim(filename)
+
+            write (ioxdmf, *) '           </DataItem>'
+            write (ioxdmf, *) '        </Attribute>'
+         end do
+
+         write (ioxdmf, '(/)')
+         write (ioxdmf, *) '    </Grid>'
+         write (ioxdmf, *) '</Domain>'
+         write (ioxdmf, '(A7)') '</Xdmf>'
+         close (ioxdmf)
+      end if
+
+   end subroutine write_visu
+
+   subroutine fin()
+
+      integer :: ierr
+
+      call decomp_2d_finalize
+      call MPI_FINALIZE(ierr)
+
+   end subroutine fin
+
 end program visu
 
diff --git a/src/factor.f90 b/src/factor.f90
index 68d666e5..e067c5a1 100644
--- a/src/factor.f90
+++ b/src/factor.f90
@@ -24,7 +24,7 @@ subroutine findfactor(num, factors, nfact)
       ! find the factors <= sqrt(num)
       ! Cast the int as double to make sure of the correct result of sqrt
       ! IntelLLVM got an issue with 1.0 but not with 1.d0
-      m = int(sqrt(num*1.d0))
+      m = int(sqrt(num * 1.d0))
       nfact = 1
       do i = 1, m
          if (num / i * i == num) then

From cef24799575f3a3eeee20c145564e7ab31bbcbcd Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 16 Oct 2023 14:22:17 +0100
Subject: [PATCH 345/436] Bug correction for folder name

---
 examples/io_test/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 65329b2c..89255067 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -75,7 +75,7 @@ message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
-  add_test(NAME io_visu COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:i0_visu> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+  add_test(NAME io_visu COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_visu> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 else ()
   add_test(NAME io_visu COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_visu> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()

From f395b8e037dcfec12c9e248bbd4c6628b806743c Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 16 Oct 2023 13:42:31 +0000
Subject: [PATCH 346/436] Bug correction for GPU: add scalar for end index
 loops

---
 examples/io_test/io_visu.f90 | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/examples/io_test/io_visu.f90 b/examples/io_test/io_visu.f90
index 26efc597..d88c3e9e 100644
--- a/examples/io_test/io_visu.f90
+++ b/examples/io_test/io_visu.f90
@@ -105,35 +105,48 @@ end subroutine init_example
 
    subroutine init_data()
 
+      integer :: xen1, xen2, xen3
+      integer :: yen1, yen2, yen3
+      integer :: zen1, zen2, zen3
+
       call alloc_x(u1)
       call alloc_y(u2)
       call alloc_z(u3)
+      xen1 = xsize(1)
+      xen2 = xsize(2)
+      xen3 = xsize(3)
+      yen1 = ysize(1)
+      yen2 = ysize(2)
+      yen3 = ysize(3)
+      zen1 = zsize(1)
+      zen2 = zsize(2)
+      zen3 = zsize(3)
 
       ! distribute the data
       !$acc data copy(u1,u2,u3)
 
       !$acc parallel loop default(present)
-      do k = 1, xsize(3)
-         do j = 1, xsize(2)
-            do i = 1, xsize(1)
+      do k = 1, xen3
+         do j = 1, xen2
+            do i = 1, xen1
                u1(i, j, k) = real(100 + nrank, mytype)
             end do
          end do
       end do
       !$acc end loop
       !$acc parallel loop default(present)
-      do k = 1, ysize(3)
-         do j = 1, ysize(2)
-            do i = 1, ysize(1)
+      do k = 1, yen3
+         do j = 1, yen2
+            do i = 1, yen1
                u2(i, j, k) = real(100 + nrank, mytype)
             end do
          end do
       end do
       !$acc end loop
       !$acc parallel loop default(present)
-      do k = 1, zsize(3)
-         do j = 1, zsize(2)
-            do i = 1, zsize(1)
+      do k = 1, zen3
+         do j = 1, zen2
+            do i = 1, zen1
                u3(i, j, k) = real(100 + nrank, mytype)
             end do
          end do

From 9b0c7a04aec2a5c5b1db4b58a3872e00a31823d6 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Mon, 16 Oct 2023 15:45:29 +0100
Subject: [PATCH 347/436] Update ADIOS2 config for visu example

---
 examples/io_test/adios2_config.xml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/examples/io_test/adios2_config.xml b/examples/io_test/adios2_config.xml
index 92ece049..de39b7f6 100644
--- a/examples/io_test/adios2_config.xml
+++ b/examples/io_test/adios2_config.xml
@@ -16,4 +16,12 @@
       <parameter key="ProfileUnits" value="Milliseconds"/>
     </transport>
   </io>
+  <io name="visu-io">
+    <engine type="HDF5">
+    </engine>
+    <transport type="File">
+      <parameter key="Library" value="fstream"/>
+      <parameter key="ProfileUnits" value="Milliseconds"/>
+    </transport>
+  </io>
 </adios-config>

From e34445e014329f612ec34207b8cede144eaa8bbe Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 23 Oct 2023 14:18:11 +0000
Subject: [PATCH 348/436] Use BP4 also for the io_visu test to avoid using HDF5

---
 examples/io_test/adios2_config.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/io_test/adios2_config.xml b/examples/io_test/adios2_config.xml
index de39b7f6..69867f4f 100644
--- a/examples/io_test/adios2_config.xml
+++ b/examples/io_test/adios2_config.xml
@@ -17,7 +17,7 @@
     </transport>
   </io>
   <io name="visu-io">
-    <engine type="HDF5">
+    <engine type="BP4">
     </engine>
     <transport type="File">
       <parameter key="Library" value="fstream"/>

From a8ebaa5d93c5c67e6ae703cf0a9efb8ff8e8b0fc Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 24 Oct 2023 11:55:27 +0100
Subject: [PATCH 349/436] Rename all README file for the examples as MarkDown
 files

---
 examples/fft_physical_x/{README => README.md} | 0
 examples/fft_physical_z/{README => README.md} | 0
 examples/halo_test/{README => README.md}      | 0
 examples/init_test/{README => README.md}      | 0
 examples/io_test/{README => README.md}        | 0
 examples/test2d/{README => README.md}         | 0
 6 files changed, 0 insertions(+), 0 deletions(-)
 rename examples/fft_physical_x/{README => README.md} (100%)
 rename examples/fft_physical_z/{README => README.md} (100%)
 rename examples/halo_test/{README => README.md} (100%)
 rename examples/init_test/{README => README.md} (100%)
 rename examples/io_test/{README => README.md} (100%)
 rename examples/test2d/{README => README.md} (100%)

diff --git a/examples/fft_physical_x/README b/examples/fft_physical_x/README.md
similarity index 100%
rename from examples/fft_physical_x/README
rename to examples/fft_physical_x/README.md
diff --git a/examples/fft_physical_z/README b/examples/fft_physical_z/README.md
similarity index 100%
rename from examples/fft_physical_z/README
rename to examples/fft_physical_z/README.md
diff --git a/examples/halo_test/README b/examples/halo_test/README.md
similarity index 100%
rename from examples/halo_test/README
rename to examples/halo_test/README.md
diff --git a/examples/init_test/README b/examples/init_test/README.md
similarity index 100%
rename from examples/init_test/README
rename to examples/init_test/README.md
diff --git a/examples/io_test/README b/examples/io_test/README.md
similarity index 100%
rename from examples/io_test/README
rename to examples/io_test/README.md
diff --git a/examples/test2d/README b/examples/test2d/README.md
similarity index 100%
rename from examples/test2d/README
rename to examples/test2d/README.md

From ce91332799aa31a966dfc52f7cd21e6ea12cadf1 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 24 Oct 2023 11:59:41 +0100
Subject: [PATCH 350/436] Update heading for README

---
 examples/README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/README.md b/examples/README.md
index c55f5307..f251bb55 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -1,5 +1,4 @@
-Examples
-========
+# Examples
 
 * init_test      - to test the initialisation of the DECOMP2D&FFT library
 

From 6fdcdef0a9bec986893a09bbfae21bd2241bbd65 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 24 Oct 2023 12:30:37 +0100
Subject: [PATCH 351/436] New version of the README for example FFT

---
 examples/README.md                | 13 ++++++-------
 examples/fft_physical_x/README.md | 28 +++++++++++++++++-----------
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/examples/README.md b/examples/README.md
index f251bb55..185266f9 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -1,16 +1,15 @@
 # Examples
 
-* init_test      - to test the initialisation of the DECOMP2D&FFT library
+* [Initialization test](init_test)         - to test the initialisation of the DECOMP2D&FFT library
 
-* test2d         - various tests for the 2D pencil decomposition module and timing 
+* [Test decomposition](test2d)             - various tests for the 2D pencil decomposition module and timing 
 
-* fft_physical_x - various tests for the FFT starting from the ``X`` direction 
+* [Test FFT from X pencil](fft_physical_x) - various tests for the FFT starting from the ``X`` direction 
 
-* fft_physical_z - various tests for the FFT starting from the ``Z`` direction 
+* [Test FFT from Z pencil](fft_physical_z) - various tests for the FFT starting from the ``Z`` direction 
 
-* halo_test      - to test the halo-cell exchange code
-
-* io_test        - various tests for the IO module
+* [Test HALO exchange](halo_test)          - to test the halo-cell exchange code
 
+* [Test IO features](io_test)              - various tests for the IO module
 
 Refer to the README files for each example for details.
diff --git a/examples/fft_physical_x/README.md b/examples/fft_physical_x/README.md
index 769d2165..06ac4493 100644
--- a/examples/fft_physical_x/README.md
+++ b/examples/fft_physical_x/README.md
@@ -1,22 +1,28 @@
-fft_c2c_x, fft_r2c_x, fft_grid_x
-------
+# Test FFT for X pencil decomposition 
+
+List of the tests:
+- [fft_c2c_x](fft_c2c_x.f90): Test Complex to Complex FFT transform; 
+- [fft_r2c_x](fft_r2c_x.f90): Test Real to Complex FFT transform;
+- [fft_grid_x](fft_grid_x.f90): Test Real to Complex transform of a different grid than the one used 
+                                for the initialization. 
+
 
 These programs can be used to test the fft trasform using X as starting physical dimension. 
 Both c2c (fft_c2c_x) and r2c/c2r (fft_r2c_x) transforms are tested.
-The case with the FFT grid different from the grid used for initialization is also tested (fft_grid) 
+The case fft_grod uses a different resolution from the one used for the initialization. 
 The results should recover the input data up to machine accuracy
 after a forward and a backward transform and proper normalisation.
 The test automatically resize the problem depending on the number of MPI processes in use
 
 What to input: The program takes max 6 inputs as : 
-               1) p_row [optional]
-	       2) p_col [optional] 
-	       3) nx    [optional]
-	       4) ny    [optional]
-	       5) nz    [optional]
-	       6) nt    [optional]
-	       In case the decomposition is imposed both (1) and (2) are necessary. 
-	       If also the resolution is imposed (1-5) are necessary
+1. p_row [optional]
+1. p_col [optional] 
+1. nx    [optional]
+1. ny    [optional]
+1. nz    [optional]
+1. nt    [optional]
+In case the decomposition is imposed both (1) and (2) are necessary. 
+If also the resolution is imposed (1-5) are necessary
 
 What to expect:
 - The timing results 

From 1320f7b84890f1f3035ca6649b6edce7b4f8144f Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 24 Oct 2023 14:14:10 +0100
Subject: [PATCH 352/436] Update all README for example folder

---
 examples/fft_physical_x/README.md |  2 ++
 examples/fft_physical_z/README.md | 26 +++++++++++++++----------
 examples/halo_test/README.md      | 20 +++++++++++++++----
 examples/init_test/README.md      | 23 +++++++++++++---------
 examples/io_test/README.md        | 32 ++++++++++++++++---------------
 examples/test2d/README.md         | 27 ++++++++++++++++----------
 6 files changed, 82 insertions(+), 48 deletions(-)

diff --git a/examples/fft_physical_x/README.md b/examples/fft_physical_x/README.md
index 06ac4493..6fc8d623 100644
--- a/examples/fft_physical_x/README.md
+++ b/examples/fft_physical_x/README.md
@@ -15,12 +15,14 @@ after a forward and a backward transform and proper normalisation.
 The test automatically resize the problem depending on the number of MPI processes in use
 
 What to input: The program takes max 6 inputs as : 
+
 1. p_row [optional]
 1. p_col [optional] 
 1. nx    [optional]
 1. ny    [optional]
 1. nz    [optional]
 1. nt    [optional]
+
 In case the decomposition is imposed both (1) and (2) are necessary. 
 If also the resolution is imposed (1-5) are necessary
 
diff --git a/examples/fft_physical_z/README.md b/examples/fft_physical_z/README.md
index 9c8842d6..a0dd9d4e 100644
--- a/examples/fft_physical_z/README.md
+++ b/examples/fft_physical_z/README.md
@@ -1,5 +1,9 @@
-fft_c2c_z, fft_r2c_z
-------
+# Test FFT for Z pencil decomposition 
+
+List of the tests:
+- [fft_c2c_z](fft_c2c_z.f90): Test Complex to Complex FFT transform; 
+- [fft_r2c_x](fft_r2c_z.f90): Test Real to Complex FFT transform;
+
 
 These programs can be used to test the fft trasform using Z as starting physical dimension. 
 Both c2c (fft_c2c_z) and r2c/c2r (fft_r2c_z) transforms are tested.
@@ -8,14 +12,16 @@ after a forward and a backward transform and proper normalisation.
 The test automatically resize the problem depending on the number of MPI processes in use
 
 What to input: The program takes max 6 inputs as : 
-               1) p_row [optional]
-	       2) p_col [optional] 
-	       3) nx    [optional]
-	       4) ny    [optional]
-	       5) nz    [optional]
-	       6) nt    [optional]
-	       In case the decomposition is imposed both (1) and (2) are necessary. 
-	       If also the resolution is imposed (1-5) are necessary
+
+1. p_row [optional]
+1. p_col [optional] 
+1. nx    [optional]
+1. ny    [optional]
+1. nz    [optional]
+1. nt    [optional]
+
+In case the decomposition is imposed both (1) and (2) are necessary. 
+If also the resolution is imposed (1-5) are necessary
 
 What to expect:
 - The timing results 
diff --git a/examples/halo_test/README.md b/examples/halo_test/README.md
index 623587f4..f3cc9a65 100644
--- a/examples/halo_test/README.md
+++ b/examples/halo_test/README.md
@@ -1,5 +1,7 @@
-halo_test
----------
+# Halo test
+
+List of the tests:
+- [HALO test](halo_test.f90): Test for the halo exchange; 
 
 This example demonstrates the use of the halo-cell support API. It calculates
 the divergency of an arbitrary field, which contains evaluation of spatial
@@ -9,5 +11,15 @@ results are to be expected regardless of the communication algorithm. The
 computation is based on an explicit finite difference method so clearly using 
 the halo-cell support API is more efficient.
 
-What to expect: the output using different communication algorithms should be 
-exactly the same.
+What to input: The program takes max 5 inputs as : 
+
+1. p_row [optional]
+1. p_col [optional] 
+1. nx    [optional]
+1. ny    [optional]
+1. nz    [optional]
+
+In case the decomposition is imposed both (1) and (2) are necessary. 
+If also the resolution is imposed (1-5) are necessary
+
+What to expect: the output using different communication algorithms should be the same up to machine precision. 
diff --git a/examples/init_test/README.md b/examples/init_test/README.md
index b683b7c7..73bc06ae 100644
--- a/examples/init_test/README.md
+++ b/examples/init_test/README.md
@@ -1,17 +1,22 @@
-init_test
----------
+# Initialization test
+
+List of the tests:
+- [Initialization test](init_test.f90): Test for initialization process; 
 
 This example demonstrates the initialisation of DECOMP2D&FFT library and tests 
 that the size of the mesh in every direction is the expected
 
+
 What to input: The program takes max 5 inputs as : 
-               1) p_row [optional]
-	       2) p_col [optional] 
-	       3) nx    [optional]
-	       4) ny    [optional]
-	       5) nz    [optional]
-	       In case the decomposition is imposed both (1) and (2) are necessary. 
-	       If also the resolution is imposed (1-5) are necessary
+
+1. p_row [optional]
+1. p_col [optional] 
+1. nx    [optional]
+1. ny    [optional]
+1. nz    [optional]
+
+In case the decomposition is imposed both (1) and (2) are necessary. 
+If also the resolution is imposed (1-5) are necessary
 
 What to expect: Success/error message if initialization process has been complted or 
                 errors have been encountered.
diff --git a/examples/io_test/README.md b/examples/io_test/README.md
index 9c64bc85..d11e2541 100644
--- a/examples/io_test/README.md
+++ b/examples/io_test/README.md
@@ -1,24 +1,26 @@
-io_bench, io_plane_test, io_read, io_test, io_var_test
----------
+# IO tests
 
 These examples demostrate the use of the different I/O features of DECOMP2D&FFT
 using both MPI I/O and ADIOS2. The test performed are the following: 
-- io_test: write of the files u1.dat, u2.dat and u3.dat allocated in the X, Y and Z pencil decomposition respectively            and the write of all 3 files in a combined single file (used for checkpointing)
+- [Write Test](io_test.f90): write of the files u1.dat, u2.dat and u3.dat allocated in the X, Y and Z pencil decomposition respectively            and the write of all 3 files in a combined single file (used for checkpointing)
            The program checks that the written files are the excpeted.
-- io_read: read the files u1.dat, u2.dat and u3.dat written by io_test and check that they are the expected one. 
-- io_var_test: test the writing of writing real and complex data for scalar and 3D arrays 
+- [Read Test](io_read.f90): read the files u1.dat, u2.dat and u3.dat written by io_test and check that they are the expected one. 
+- [Write Real/Complex variables](io_var_test.f90): test the writing of real and complex data for scalar and 3D arrays 
                and for different resolutions 
-- io_plane_test: test the writing of 2D planes in different direction  
-- io_bench: timing the writing on disk of a 3D array
+- [Write 2D planes files](io_plane_test.f90): test the writing of 2D planes in different direction  
+- [Timing IO](io_bench.f90): timing the writing on disk of a 3D array
 
-What to input: The program takes max 6 inputs as : 
-               1) p_row [optional]
-	       2) p_col [optional] 
-	       3) nx    [optional]
-	       4) ny    [optional]
-	       5) nz    [optional]
-	       In case the decomposition is imposed both (1) and (2) are necessary. 
-	       If also the resolution is imposed (1-5) are necessary
+
+What to input: The program takes max 5 inputs as : 
+
+1. p_row [optional]
+1. p_col [optional] 
+1. nx    [optional]
+1. ny    [optional]
+1. nz    [optional]
+
+In case the decomposition is imposed both (1) and (2) are necessary. 
+If also the resolution is imposed (1-5) are necessary
 
 What to expect: 
 - All programs print out a success message otherwise an error message with location of the error is reported
diff --git a/examples/test2d/README.md b/examples/test2d/README.md
index ff6a8967..75f99c33 100644
--- a/examples/test2d/README.md
+++ b/examples/test2d/README.md
@@ -1,5 +1,10 @@
-test2d, timing2d_real, timing2d_complex 
-------
+# Test transpose operations test2d, timing2d_real, timing2d_complex 
+
+
+List of the tests:
+- [Test transpose operations](test2d.f90): Test the transpose operation for a 3D real array; 
+- [Time transpose real](timing2d_real.f90): Time the transpose operation for a 3D real array; 
+- [Time transpose complex](timing2d_complex.f90): Time the transpose operation for a 3D complex array; 
 
 The program test2d is to validate the 2D pencil decomposition library. It transposes
 a set of data into different storage formats so that all communication routines
@@ -8,14 +13,16 @@ The programs timing2d_real and timing2d_complex perform the same tests several t
 and are used to report timing and performances for real and complex transforms. 
 
 What to input: The program takes max 6 inputs as : 
-               1) p_row [optional]
-	       2) p_col [optional] 
-	       3) nx    [optional]
-	       4) ny    [optional]
-	       5) nz    [optional]
-	       6) nt    [optional]
-	       In case the decomposition is imposed both (1) and (2) are necessary. 
-	       If also the resolution is imposed (1-5) are necessary
+
+1. p_row [optional]
+1. p_col [optional] 
+1. nx    [optional]
+1. ny    [optional]
+1. nz    [optional]
+1. nt    [optional]
+
+In case the decomposition is imposed both (1) and (2) are necessary. 
+If also the resolution is imposed (1-5) are necessary
 
 What to expect: 
 - For test2d the output is e success message or an error message with the direction of error in swapping. 

From d427fc4dfa41f33e57aa7a36e8feda22f2192113 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 24 Oct 2023 14:35:16 +0100
Subject: [PATCH 353/436] Add link into main README to the Contribution file

---
 HOWTO.md  | 9 +++++++++
 README.md | 5 +++++
 2 files changed, 14 insertions(+)
 create mode 100644 HOWTO.md

diff --git a/HOWTO.md b/HOWTO.md
new file mode 100644
index 00000000..81306a22
--- /dev/null
+++ b/HOWTO.md
@@ -0,0 +1,9 @@
+# How To use 2DECOMP&FFT
+
+The 2D Pencil Decomposition API is defined in three Fortran module which should be used by applications as:
+
+::
+
+  use decomp_2d_constants
+  use decomp_2d_mpi
+  use decomp_2d
diff --git a/README.md b/README.md
index 999b3784..5e6b58a3 100644
--- a/README.md
+++ b/README.md
@@ -151,3 +151,8 @@ is ready.
 For example, starting from `v2.0.0` the `main` branch will only be updated to receive fixes giving
 `v2.0.1`, etc. until the next release (either `v2.1.0` or `v3.0.0` depending on the magnitude of the
 change is ready).
+
+### Contributing
+
+If you would like to contribute to the development of the 2DECOMP&FFT library or just to report a bug please refer to 
+the [Contributing section](Contribute.md)

From f171d15c810b9d17a35cab9055e17c9a368e41a3 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 24 Oct 2023 17:06:56 +0100
Subject: [PATCH 354/436] Update the README and add the HOWTO use the library
 file

---
 HOWTO.md   | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 INSTALL.md |   2 +-
 README.md  |   5 ++-
 3 files changed, 114 insertions(+), 5 deletions(-)

diff --git a/HOWTO.md b/HOWTO.md
index 81306a22..8c5a47af 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -1,9 +1,115 @@
 # How To use 2DECOMP&FFT
+Full informations on how to use the 2DECOMP&FFT library together with some related theory are available [here](https://2decomp-fft.github.io/).
 
 The 2D Pencil Decomposition API is defined in three Fortran module which should be used by applications as:
-
-::
-
+```
   use decomp_2d_constants
   use decomp_2d_mpi
   use decomp_2d
+```
+where ``use decomp_2d_constants`` defines all the parameters, ``use decomp_2d_mpi`` introduces all the MPI 
+related interfaces and ``use decomp_2d`` cointains the main decomposition and transposition APIs. 
+```
+  call decomp_2d_init(nx, ny, nz, p_row, p_col)
+```
+where ``nx``, ``ny`` and ``nz`` are the size of 3D global data to be distributed over
+a 2D processor grid :math:`p_row \times p_col`.
+Note that none of the dimensions need to be divisible by ``p_row`` or ``p_col``, i.e. the library can handle non-evenly distributed data.
+In case of ``p_row=p_col=0`` an automatic decomposition is selected among all possible combination available.
+A key element of this library is a set of communication routines that actually perform the data transpositions.
+As mentioned, one needs to perform 4 global transpositions to go through all 3 pencil orientations.
+Correspondingly, the library provides 4 communication subroutines:
+```
+  call transpose_x_to_y(var_in,var_out)
+  call transpose_y_to_z(var_in,var_out)
+  call transpose_z_to_y(var_in,var_out)
+  call transpose_y_to_x(var_in,var_out)
+```
+The input array ``var_in`` and ``var_output`` array out should have been defined
+and contain distributed data for the correct pencil orientations.
+
+Note that the library is written using Fortran's generic interface so different data types are supported
+without user intervention. That means in and out above can be either real arrays or complex arrays,
+the latter being useful for FFT-type of applications.
+Finally, before exit, applications should clean up the memory by:
+```
+  call decomp_2d_finalize
+```
+Detailed information about the decomposition API are available [here](https://2decomp-fft.github.io/pages/api_domain.html) 
+### Use of the FFT module
+To use the FFT programming interface, first of all, one additional Fortran module has to be used:
+```
+  use decomp_2d_fft
+```
+Then one needs to initialise the FFT interface by:
+```
+  call decomp_2d_fft_init(pencil, n1, n2, n3)
+```
+where ``pencil=PHYSICAL_IN_X`` or ``PHYSICAL_IN_Z`` and ``n1, n2, n3`` is an arbitrary problem size
+that can be different from :math:`nx\times ny\times nz`.
+For complex-to-complex (c2c) FFTs, the user interface is:
+```
+  call decomp_2d_fft_3d(input, output, direction)
+```
+where ``direction`` can be either ``DECOMP_2D_FFT_FORWARD == -1`` for forward transforms, 
+or ``DECOMP_2D_FFT_BACKWARD == 1`` for backward transforms.
+The input array ``input`` and ``output`` array out are both complex
+and have to be either a X-pencil/Z-pencil combination or vice-versa.
+The interface for the the real-to-complex and complex-to-real transform is
+```
+  call decomp_2d_fft_3d(input, output)
+```
+If the ``input`` data are real type a forward transform is assumed obtaining a complex ``output``.
+Similarly a backward FFT is computed if ``input`` is a complex array and ``output`` a real array.
+Finally, to release the memory used by the FFT interface:
+```
+  call decomp_2d_fft_finalize
+```
+Detailed information about the FFT API are available [here](https://2decomp-fft.github.io/pages/api_fft.html) 
+### Use of the IO module
+All the I/O functions have been packed in a Fortran module:
+```
+  use decomp_2d_io
+```
+To write a single three-dimensional array to a file
+```
+  call decomp_2d_write_one(ipencil,var,directory,filename,icoarse,io_name)
+```
+where ``ipencil`` describes how the data is distributed (valid values are: 1 for X-pencil; 2 for
+Y-pencil and 3 for Z-pencil); ``var`` is the reference to the data array, which can be either real or
+complex; ``directory`` is the path to where I/O should be written; ``filename`` is the name of the
+file to be written; ``icoarse`` indicates whether the I/O should be coarsend (valid values are: 0
+for no; 1 for the ``nstat`` and 2 for the ``nvisu`` coarsenings); ``io_name`` is the name of the I/O
+group to be used. 
+
+There are two ways of writing multiple variables to a single file which may
+be used for check-pointing purposes, for example. The newer interface is described first and allows
+codes to use the ADIOS2 and MPI-IO backends, the older interface is supported for backwards
+compatibility.
+
+When ``decomp_2d_write_one`` is called, the ``directory`` and ``io_name`` are combined to check
+whether a particular output location is already opened, if not then a new file will be opened and
+written to - this is the "standard" use.  If, however, a file is opened first then the call to
+``decomp_2d_write_one`` will append to the current file, resulting in a single file with multiple
+fields.  Once the check-pointing is complete the file can then be closed.
+
+The original interface for writing multiple variables to a file and is only
+supported by the MPI-IO backend and takes the following form:
+```
+   call decomp_2d_write_var(fh,disp,ipencil,var)
+```
+where ``fh`` is a MPI-IO file handle provided by the application (file opened using ``MPI_FILE_OPEN``);
+``ipencil`` describes the distribution of the input data (valid values are: 1 for X-pencil; 2 for
+Y-pencil and 3 for Z-pencil); ``disp`` (meaning displacement) is a variable of ``kind MPI_OFFSET_KIND``
+and of ``intent INOUT``. 
+Detailed information about the IO API are available [here](https://2decomp-fft.github.io/pages/api_io.html) 
+### Use of the halo exchange
+The halo-cell support API provides data structures and nearest-neighbour communication routines 
+that support explicit message passing between neighbouring pencils. 
+```
+  call update_halo(var, var_halo, level)
+```
+Here the first parameter ``var``, a 3D input array, contains the normal pencil-distributed data as defined by the decomposition library. 
+After the subroutine call, the second parameter ``var_halo`` returns all original data plus halo data from the neighbouring processes.
+The third parameter level defines how many layers of overlapping are required. 
+Detailed information about the halo module are available [here](https://2decomp-fft.github.io/pages/api_halo.html) 
diff --git a/INSTALL.md b/INSTALL.md
index ff595176..ab1064c2 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -1,4 +1,4 @@
-# Bulding and installing 2DECOMP&FFT
+# Bulding installing and linking 2DECOMP&FFT
 
 The library 2decomp is a Fortran library compatible with the Fortran 2008 standard.
 It requires a MPI library compatible with MPI-2.0 with extended Fortran support.
diff --git a/README.md b/README.md
index 5e6b58a3..948e47e1 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,10 @@
 # 2DECOMP&FFT
 
 This README contains basic instructions for building and installing the 2DECOMP&FFT library, more
-detailed instructions can be found in [INSTALL.md](INSTALL.md).
+detailed instructions about installation and linking to the library within an external project 
+can be found in the [install section](INSTALL.md).
+For how to use the library within your projec please refers to [HOWTO.md](HOWTO.md) and at the [examples](examples/README.md) 
+provided. 
 
 ## Building
 

From 7dfd3aef5f11286b2f7c0065b7b7a5ad880c9a00 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 24 Oct 2023 18:57:15 +0200
Subject: [PATCH 355/436] Minor update in HOWTO.md

---
 HOWTO.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/HOWTO.md b/HOWTO.md
index 8c5a47af..b8622d59 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -1,14 +1,14 @@
 # How To use 2DECOMP&FFT
 Full informations on how to use the 2DECOMP&FFT library together with some related theory are available [here](https://2decomp-fft.github.io/).
 
-The 2D Pencil Decomposition API is defined in three Fortran module which should be used by applications as:
+The 2D Pencil Decomposition API is defined in three Fortran modules which should be used by applications as:
 ```
   use decomp_2d_constants
   use decomp_2d_mpi
   use decomp_2d
 ```
 where ``use decomp_2d_constants`` defines all the parameters, ``use decomp_2d_mpi`` introduces all the MPI 
-related interfaces and ``use decomp_2d`` cointains the main decomposition and transposition APIs. 
+related interfaces and ``use decomp_2d`` cointains the main decomposition and transposition APIs. The library is initialized using
 ```
   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 ```
@@ -25,23 +25,23 @@ Correspondingly, the library provides 4 communication subroutines:
   call transpose_z_to_y(var_in,var_out)
   call transpose_y_to_x(var_in,var_out)
 ```
-The input array ``var_in`` and ``var_output`` array out should have been defined
+The input array ``var_in`` and output array ``var_out`` are defined by the code using the library
 and contain distributed data for the correct pencil orientations.
 
 Note that the library is written using Fortran's generic interface so different data types are supported
-without user intervention. That means in and out above can be either real arrays or complex arrays,
+without user intervention. Arrays above can be either real arrays or complex ones,
 the latter being useful for FFT-type of applications.
-Finally, before exit, applications should clean up the memory by:
+Finally, before exit, applications should clean up the memory using
 ```
   call decomp_2d_finalize
 ```
 Detailed information about the decomposition API are available [here](https://2decomp-fft.github.io/pages/api_domain.html) 
 ### Use of the FFT module
-To use the FFT programming interface, first of all, one additional Fortran module has to be used:
+To use the FFT programming interface, first of all, one additional Fortran module is needed:
 ```
   use decomp_2d_fft
 ```
-Then one needs to initialise the FFT interface by:
+Then one needs to initialise the FFT interface using
 ```
   call decomp_2d_fft_init(pencil, n1, n2, n3)
 ```

From aa4e467c0c0afdcbf3bc96ddf59fa833e8c3b87b Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 24 Oct 2023 18:57:53 +0200
Subject: [PATCH 356/436] Typo in INSTALL.md

---
 INSTALL.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/INSTALL.md b/INSTALL.md
index ab1064c2..e5591f76 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -1,4 +1,4 @@
-# Bulding installing and linking 2DECOMP&FFT
+# Building, installing and linking 2DECOMP&FFT
 
 The library 2decomp is a Fortran library compatible with the Fortran 2008 standard.
 It requires a MPI library compatible with MPI-2.0 with extended Fortran support.

From 54753532088e14ec363e9bea2638879f07ff33af Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 24 Oct 2023 18:58:46 +0200
Subject: [PATCH 357/436] Typo README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 948e47e1..429b9fbd 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
 This README contains basic instructions for building and installing the 2DECOMP&FFT library, more
 detailed instructions about installation and linking to the library within an external project 
 can be found in the [install section](INSTALL.md).
-For how to use the library within your projec please refers to [HOWTO.md](HOWTO.md) and at the [examples](examples/README.md) 
+For how to use the library within your project please refers to [HOWTO.md](HOWTO.md) and at the [examples](examples/README.md) 
 provided. 
 
 ## Building

From d99fd521060b21dbe0a561e573cf0bb8a75a9a01 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 24 Oct 2023 19:00:07 +0200
Subject: [PATCH 358/436] Minor update in README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 429b9fbd..9eff1ad7 100644
--- a/README.md
+++ b/README.md
@@ -157,5 +157,5 @@ change is ready).
 
 ### Contributing
 
-If you would like to contribute to the development of the 2DECOMP&FFT library or just to report a bug please refer to 
+If you would like to contribute to the development of the 2DECOMP&FFT library or report a bug please refer to 
 the [Contributing section](Contribute.md)

From 756ccaff11e1caba7d438b46b5a99690e12d22f8 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Wed, 25 Oct 2023 10:09:13 +0200
Subject: [PATCH 359/436] Minor update in HOWTO.md

---
 HOWTO.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/HOWTO.md b/HOWTO.md
index b8622d59..47d350d0 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -53,7 +53,7 @@ For complex-to-complex (c2c) FFTs, the user interface is:
 ```
 where ``direction`` can be either ``DECOMP_2D_FFT_FORWARD == -1`` for forward transforms, 
 or ``DECOMP_2D_FFT_BACKWARD == 1`` for backward transforms.
-The input array ``input`` and ``output`` array out are both complex
+The input array (``input``) and the output one (``output``) are both complex
 and have to be either a X-pencil/Z-pencil combination or vice-versa.
 The interface for the the real-to-complex and complex-to-real transform is
 ```

From b1f9778b9e780669fa26466f92ad918c0d906b30 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 25 Oct 2023 14:00:44 +0100
Subject: [PATCH 360/436] Further spell corrections

---
 HOWTO.md                          | 23 ++++++++++++-----------
 README.md                         |  3 +--
 examples/README.md                | 17 ++++++-----------
 examples/fft_physical_x/README.md |  8 ++++----
 examples/fft_physical_z/README.md |  8 ++++----
 examples/halo_test/README.md      |  8 ++++----
 examples/init_test/README.md      | 10 +++++-----
 examples/io_test/README.md        | 23 ++++++++++++-----------
 examples/test2d/README.md         | 13 ++++++-------
 9 files changed, 54 insertions(+), 59 deletions(-)

diff --git a/HOWTO.md b/HOWTO.md
index 8c5a47af..1351e84e 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -1,7 +1,7 @@
 # How To use 2DECOMP&FFT
-Full informations on how to use the 2DECOMP&FFT library together with some related theory are available [here](https://2decomp-fft.github.io/).
+Full informations on how to use the 2DECOMP&FFT library together with some related information [here](https://2decomp-fft.github.io/).
 
-The 2D Pencil Decomposition API is defined in three Fortran module which should be used by applications as:
+The 2D Pencil Decomposition API is defined with three Fortran module which should be used by applications as:
 ```
   use decomp_2d_constants
   use decomp_2d_mpi
@@ -12,12 +12,13 @@ related interfaces and ``use decomp_2d`` cointains the main decomposition and tr
 ```
   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 ```
-where ``nx``, ``ny`` and ``nz`` are the size of 3D global data to be distributed over
+where ``nx``, ``ny`` and ``nz`` are the spatial dimensions of the problem, to be distributed over
 a 2D processor grid :math:`p_row \times p_col`.
-Note that none of the dimensions need to be divisible by ``p_row`` or ``p_col``, i.e. the library can handle non-evenly distributed data.
+Note that none of the dimensions need to be divisible by ``p_row`` or ``p_col``.
 In case of ``p_row=p_col=0`` an automatic decomposition is selected among all possible combination available.
 A key element of this library is a set of communication routines that actually perform the data transpositions.
-As mentioned, one needs to perform 4 global transpositions to go through all 3 pencil orientations.
+As mentioned, one needs to perform 4 global transpositions to go through all 3 pencil orientations
+(i.e. one has to go from x-pencils to y-pencils to z-pencils to y-pencils to x-pencils)
 Correspondingly, the library provides 4 communication subroutines:
 ```
   call transpose_x_to_y(var_in,var_out)
@@ -29,8 +30,8 @@ The input array ``var_in`` and ``var_output`` array out should have been defined
 and contain distributed data for the correct pencil orientations.
 
 Note that the library is written using Fortran's generic interface so different data types are supported
-without user intervention. That means in and out above can be either real arrays or complex arrays,
-the latter being useful for FFT-type of applications.
+without user input. That means in and out above can be either real arrays or complex arrays,
+the latter being useful for applications involving 3D Fast Fourier Transforms.
 Finally, before exit, applications should clean up the memory by:
 ```
   call decomp_2d_finalize
@@ -55,7 +56,7 @@ where ``direction`` can be either ``DECOMP_2D_FFT_FORWARD == -1`` for forward tr
 or ``DECOMP_2D_FFT_BACKWARD == 1`` for backward transforms.
 The input array ``input`` and ``output`` array out are both complex
 and have to be either a X-pencil/Z-pencil combination or vice-versa.
-The interface for the the real-to-complex and complex-to-real transform is
+The interface foe real-to-complex and complex-to-real transform is
 ```
   call decomp_2d_fft_3d(input, output)
 ```
@@ -84,7 +85,7 @@ group to be used.
 
 There are two ways of writing multiple variables to a single file which may
 be used for check-pointing purposes, for example. The newer interface is described first and allows
-codes to use the ADIOS2 and MPI-IO backends, the older interface is supported for backwards
+codes to use the ADIOS2 and MPI-IO backends, the original interface is supported for backwards
 compatibility.
 
 When ``decomp_2d_write_one`` is called, the ``directory`` and ``io_name`` are combined to check
@@ -93,8 +94,8 @@ written to - this is the "standard" use.  If, however, a file is opened first th
 ``decomp_2d_write_one`` will append to the current file, resulting in a single file with multiple
 fields.  Once the check-pointing is complete the file can then be closed.
 
-The original interface for writing multiple variables to a file and is only
-supported by the MPI-IO backend and takes the following form:
+The original interface for writing multiple variables to a file, only
+supported by the MPI-IO backend, takes the following form:
 ```
    call decomp_2d_write_var(fh,disp,ipencil,var)
 ```
diff --git a/README.md b/README.md
index 948e47e1..7a2773c7 100644
--- a/README.md
+++ b/README.md
@@ -3,8 +3,7 @@
 This README contains basic instructions for building and installing the 2DECOMP&FFT library, more
 detailed instructions about installation and linking to the library within an external project 
 can be found in the [install section](INSTALL.md).
-For how to use the library within your projec please refers to [HOWTO.md](HOWTO.md) and at the [examples](examples/README.md) 
-provided. 
+Please have a look at [HOWTO.md](HOWTO.md) and at the examples [examples](examples/README.md) for how to use the library with your application
 
 ## Building
 
diff --git a/examples/README.md b/examples/README.md
index 185266f9..9849cf25 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -1,15 +1,10 @@
 # Examples
 
-* [Initialization test](init_test)         - to test the initialisation of the DECOMP2D&FFT library
-
-* [Test decomposition](test2d)             - various tests for the 2D pencil decomposition module and timing 
-
-* [Test FFT from X pencil](fft_physical_x) - various tests for the FFT starting from the ``X`` direction 
-
-* [Test FFT from Z pencil](fft_physical_z) - various tests for the FFT starting from the ``Z`` direction 
-
-* [Test HALO exchange](halo_test)          - to test the halo-cell exchange code
-
-* [Test IO features](io_test)              - various tests for the IO module
+- [Initialization test](init_test)         - to test the initialisation of the DECOMP2D&FFT library
+- [Test decomposition](test2d)             - various tests for the 2D pencil decomposition module and timing 
+- [Test FFT from X pencil](fft_physical_x) - various tests for the FFT starting from the ``X`` direction 
+- [Test FFT from Z pencil](fft_physical_z) - various tests for the FFT starting from the ``Z`` direction 
+- [Test HALO exchange](halo_test)          - to test the halo-cell exchange capability
+- [Test IO features](io_test)              - various tests for the IO module
 
 Refer to the README files for each example for details.
diff --git a/examples/fft_physical_x/README.md b/examples/fft_physical_x/README.md
index 6fc8d623..732ea5d3 100644
--- a/examples/fft_physical_x/README.md
+++ b/examples/fft_physical_x/README.md
@@ -7,11 +7,11 @@ List of the tests:
                                 for the initialization. 
 
 
-These programs can be used to test the fft trasform using X as starting physical dimension. 
+These programs can be used to test the FFT transform using X-pencils as starting domain decomposition. 
 Both c2c (fft_c2c_x) and r2c/c2r (fft_r2c_x) transforms are tested.
 The case fft_grod uses a different resolution from the one used for the initialization. 
 The results should recover the input data up to machine accuracy
-after a forward and a backward transform and proper normalisation.
+after a forward and a backward transform and appropriate normalisation.
 The test automatically resize the problem depending on the number of MPI processes in use
 
 What to input: The program takes max 6 inputs as : 
@@ -23,8 +23,8 @@ What to input: The program takes max 6 inputs as :
 1. nz    [optional]
 1. nt    [optional]
 
-In case the decomposition is imposed both (1) and (2) are necessary. 
-If also the resolution is imposed (1-5) are necessary
+If the decomposition is imposed both (1) and (2) are necessary. 
+If the resolution is imposed (1-5) are necessary
 
 What to expect:
 - The timing results 
diff --git a/examples/fft_physical_z/README.md b/examples/fft_physical_z/README.md
index a0dd9d4e..ec569db3 100644
--- a/examples/fft_physical_z/README.md
+++ b/examples/fft_physical_z/README.md
@@ -5,10 +5,10 @@ List of the tests:
 - [fft_r2c_x](fft_r2c_z.f90): Test Real to Complex FFT transform;
 
 
-These programs can be used to test the fft trasform using Z as starting physical dimension. 
+These programs can be used to test the FFT transform using Z-pencils as starting domain decomposition. 
 Both c2c (fft_c2c_z) and r2c/c2r (fft_r2c_z) transforms are tested.
 The results should recover the input data up to machine accuracy
-after a forward and a backward transform and proper normalisation.
+after a forward and a backward transform and appropriate normalisation.
 The test automatically resize the problem depending on the number of MPI processes in use
 
 What to input: The program takes max 6 inputs as : 
@@ -20,8 +20,8 @@ What to input: The program takes max 6 inputs as :
 1. nz    [optional]
 1. nt    [optional]
 
-In case the decomposition is imposed both (1) and (2) are necessary. 
-If also the resolution is imposed (1-5) are necessary
+If the decomposition is imposed both (1) and (2) are necessary. 
+If the resolution is imposed (1-5) are necessary
 
 What to expect:
 - The timing results 
diff --git a/examples/halo_test/README.md b/examples/halo_test/README.md
index f3cc9a65..e99b02ac 100644
--- a/examples/halo_test/README.md
+++ b/examples/halo_test/README.md
@@ -1,7 +1,7 @@
 # Halo test
 
 List of the tests:
-- [HALO test](halo_test.f90): Test for the halo exchange; 
+- [HALO test](halo_test.f90): Test for the halo exchange capability of the library. 
 
 This example demonstrates the use of the halo-cell support API. It calculates
 the divergency of an arbitrary field, which contains evaluation of spatial
@@ -11,7 +11,7 @@ results are to be expected regardless of the communication algorithm. The
 computation is based on an explicit finite difference method so clearly using 
 the halo-cell support API is more efficient.
 
-What to input: The program takes max 5 inputs as : 
+What to input: The program takes max 5 inputs as: 
 
 1. p_row [optional]
 1. p_col [optional] 
@@ -19,7 +19,7 @@ What to input: The program takes max 5 inputs as :
 1. ny    [optional]
 1. nz    [optional]
 
-In case the decomposition is imposed both (1) and (2) are necessary. 
-If also the resolution is imposed (1-5) are necessary
+If the decomposition is imposed both (1) and (2) are necessary. 
+If the resolution is imposed (1-5) are necessary.
 
 What to expect: the output using different communication algorithms should be the same up to machine precision. 
diff --git a/examples/init_test/README.md b/examples/init_test/README.md
index 73bc06ae..466c9ed1 100644
--- a/examples/init_test/README.md
+++ b/examples/init_test/README.md
@@ -1,13 +1,13 @@
 # Initialization test
 
 List of the tests:
-- [Initialization test](init_test.f90): Test for initialization process; 
+- [Initialization test](init_test.f90): Test for initialization process. 
 
 This example demonstrates the initialisation of DECOMP2D&FFT library and tests 
-that the size of the mesh in every direction is the expected
+that the size of the mesh in every direction is as expected
 
 
-What to input: The program takes max 5 inputs as : 
+What to input: The program takes max 5 inputs as: 
 
 1. p_row [optional]
 1. p_col [optional] 
@@ -15,8 +15,8 @@ What to input: The program takes max 5 inputs as :
 1. ny    [optional]
 1. nz    [optional]
 
-In case the decomposition is imposed both (1) and (2) are necessary. 
-If also the resolution is imposed (1-5) are necessary
+If the decomposition is imposed both (1) and (2) are necessary. 
+If the resolution is imposed (1-5) are necessary
 
 What to expect: Success/error message if initialization process has been complted or 
                 errors have been encountered.
diff --git a/examples/io_test/README.md b/examples/io_test/README.md
index d11e2541..637e5064 100644
--- a/examples/io_test/README.md
+++ b/examples/io_test/README.md
@@ -1,14 +1,15 @@
 # IO tests
 
 These examples demostrate the use of the different I/O features of DECOMP2D&FFT
-using both MPI I/O and ADIOS2. The test performed are the following: 
-- [Write Test](io_test.f90): write of the files u1.dat, u2.dat and u3.dat allocated in the X, Y and Z pencil decomposition respectively            and the write of all 3 files in a combined single file (used for checkpointing)
-           The program checks that the written files are the excpeted.
-- [Read Test](io_read.f90): read the files u1.dat, u2.dat and u3.dat written by io_test and check that they are the expected one. 
+using both MPI I/O and ADIOS2. The tests performed are the following: 
+- [Write Test](io_test.f90): write the files u1.dat, u2.dat and u3.dat defined in X, Y and Z pencils respectively
+                             and write these files in a combined single file (used for checkpointing)
+                             The program also checks that the written files are correct;
+- [Read Test](io_read.f90): read files u1.dat, u2.dat and u3.dat which were written by io_test and check that they are the expected ones;
 - [Write Real/Complex variables](io_var_test.f90): test the writing of real and complex data for scalar and 3D arrays 
-               and for different resolutions 
-- [Write 2D planes files](io_plane_test.f90): test the writing of 2D planes in different direction  
-- [Timing IO](io_bench.f90): timing the writing on disk of a 3D array
+                                                   and for different resolutions;
+- [Write 2D planes files](io_plane_test.f90): test the writing of 2D planes in different direction;  
+- [Timing IO](io_bench.f90): timing the writing on disk of a 3D array.
 
 
 What to input: The program takes max 5 inputs as : 
@@ -19,9 +20,9 @@ What to input: The program takes max 5 inputs as :
 1. ny    [optional]
 1. nz    [optional]
 
-In case the decomposition is imposed both (1) and (2) are necessary. 
-If also the resolution is imposed (1-5) are necessary
+If the decomposition is imposed both (1) and (2) are necessary. 
+If the resolution is imposed (1-5) are necessary.
 
 What to expect: 
-- All programs print out a success message otherwise an error message with location of the error is reported
-- io_bench gives also the timing of the writing on disk
+- All programs print out a success message otherwise an error message with location of the error;
+- io_bench gives also the timing of the writing on disk.
diff --git a/examples/test2d/README.md b/examples/test2d/README.md
index 75f99c33..41021240 100644
--- a/examples/test2d/README.md
+++ b/examples/test2d/README.md
@@ -1,10 +1,9 @@
-# Test transpose operations test2d, timing2d_real, timing2d_complex 
-
+# Test and timing of the transpose operations 
 
 List of the tests:
 - [Test transpose operations](test2d.f90): Test the transpose operation for a 3D real array; 
-- [Time transpose real](timing2d_real.f90): Time the transpose operation for a 3D real array; 
-- [Time transpose complex](timing2d_complex.f90): Time the transpose operation for a 3D complex array; 
+- [Time transpose real](timing2d_real.f90): Time to solution for the transpose operation for a 3D real array; 
+- [Time transpose complex](timing2d_complex.f90): Time to solution for the transpose operation for a 3D complex array.
 
 The program test2d is to validate the 2D pencil decomposition library. It transposes
 a set of data into different storage formats so that all communication routines
@@ -21,10 +20,10 @@ What to input: The program takes max 6 inputs as :
 1. nz    [optional]
 1. nt    [optional]
 
-In case the decomposition is imposed both (1) and (2) are necessary. 
-If also the resolution is imposed (1-5) are necessary
+If the decomposition is imposed both (1) and (2) are necessary. 
+If the resolution is imposed (1-5) are necessary
 
 What to expect: 
-- For test2d the output is e success message or an error message with the direction of error in swapping. 
+- For test2d the output is e success message or an error message with the direction of error in swapping;
 - For the timing, beside the error/success message timing for all transpostions
   (X->Y, Y->Z, Z->Y, Y->X) together with the sum are reported. 

From b09256235222cfcaa1e4a8e4dc15ab3e1cec3e13 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 25 Oct 2023 19:23:30 +0100
Subject: [PATCH 361/436] Update Intro of HOWTO

---
 HOWTO.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/HOWTO.md b/HOWTO.md
index 57393456..29997f1c 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -1,5 +1,7 @@
 # How To use 2DECOMP&FFT
-Full informations on how to use the 2DECOMP&FFT library together with some related information [here](https://2decomp-fft.github.io/).
+This document presents the basic usage of 2DECOMP&FFT. 
+Full informations on how to use the 2DECOMP&FFT library can be found
+[here](https://2decomp-fft.github.io/).
 
 The 2D Pencil Decomposition API is defined with three Fortran module which should be used by applications as:
 ```

From c61dc4b2d8f96d1bdb1ee875660b93729f30a570 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Wed, 25 Oct 2023 20:48:47 +0100
Subject: [PATCH 362/436] Minor edits in HOWTO.md

Minor rewording, and highlighting that load imbalance will occur for arbitrary grid/decomposition combinations
---
 HOWTO.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/HOWTO.md b/HOWTO.md
index 29997f1c..ad756f4e 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -1,6 +1,6 @@
 # How To use 2DECOMP&FFT
 This document presents the basic usage of 2DECOMP&FFT. 
-Full informations on how to use the 2DECOMP&FFT library can be found
+Detailed instructions on how to use the 2DECOMP&FFT library can be found
 [here](https://2decomp-fft.github.io/).
 
 The 2D Pencil Decomposition API is defined with three Fortran module which should be used by applications as:
@@ -10,13 +10,13 @@ The 2D Pencil Decomposition API is defined with three Fortran module which shoul
   use decomp_2d
 ```
 where ``use decomp_2d_constants`` defines all the parameters, ``use decomp_2d_mpi`` introduces all the MPI 
-related interfaces and ``use decomp_2d`` cointains the main decomposition and transposition APIs. The library is initialized using
+related interfaces and ``use decomp_2d`` contains the main decomposition and transposition APIs. The library is initialized using
 ```
   call decomp_2d_init(nx, ny, nz, p_row, p_col)
 ```
 where ``nx``, ``ny`` and ``nz`` are the spatial dimensions of the problem, to be distributed over
 a 2D processor grid :math:`p_row \times p_col`.
-Note that none of the dimensions need to be divisible by ``p_row`` or ``p_col``.
+Note that none of the dimensions need to be divisible by ``p_row`` or ``p_col`` however a load imbalance will occur if not.
 In case of ``p_row=p_col=0`` an automatic decomposition is selected among all possible combination available.
 A key element of this library is a set of communication routines that actually perform the data transpositions.
 As mentioned, one needs to perform 4 global transpositions to go through all 3 pencil orientations

From aef63b5141e4dbfbd0e5e87fc6a64a68f7c959ac Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Wed, 25 Oct 2023 20:51:21 +0100
Subject: [PATCH 363/436] Update HOWTO.md

Users shouldn't use +/-1 directly in calls to FFT direction
---
 HOWTO.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/HOWTO.md b/HOWTO.md
index ad756f4e..259ff006 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -56,6 +56,8 @@ For complex-to-complex (c2c) FFTs, the user interface is:
 ```
 where ``direction`` can be either ``DECOMP_2D_FFT_FORWARD == -1`` for forward transforms, 
 or ``DECOMP_2D_FFT_BACKWARD == 1`` for backward transforms.
+We recommend using the ``DECOMP_2D_FFT_XXX`` variables, rather than literal ``1`` or ``-1``,
+to avoid potential issues if these values change in future versions.
 The input array (``input``) and the output one (``output``) are both complex
 and have to be either a X-pencil/Z-pencil combination or vice-versa.
 The interface foe real-to-complex and complex-to-real transform is

From 7518ff4ea6a216c28cfc5f5bea6a7f8cc2b2b153 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Wed, 25 Oct 2023 20:53:19 +0100
Subject: [PATCH 364/436] Correct typo

---
 HOWTO.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/HOWTO.md b/HOWTO.md
index 259ff006..d0f753e9 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -60,7 +60,7 @@ We recommend using the ``DECOMP_2D_FFT_XXX`` variables, rather than literal ``1`
 to avoid potential issues if these values change in future versions.
 The input array (``input``) and the output one (``output``) are both complex
 and have to be either a X-pencil/Z-pencil combination or vice-versa.
-The interface foe real-to-complex and complex-to-real transform is
+The interface for real-to-complex and complex-to-real transform is
 ```
   call decomp_2d_fft_3d(input, output)
 ```

From 5a68edca99a8ef6fb8b3d12a98829af1b4984654 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 26 Oct 2023 10:21:17 +0100
Subject: [PATCH 365/436] Minor typo

---
 examples/halo_test/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/halo_test/README.md b/examples/halo_test/README.md
index e99b02ac..27c910b3 100644
--- a/examples/halo_test/README.md
+++ b/examples/halo_test/README.md
@@ -4,7 +4,7 @@ List of the tests:
 - [HALO test](halo_test.f90): Test for the halo exchange capability of the library. 
 
 This example demonstrates the use of the halo-cell support API. It calculates
-the divergency of an arbitrary field, which contains evaluation of spatial
+the divergence of an arbitrary field, which contains evaluation of spatial
 derivatives in all three dimensions. The calculation was first implemented via
 the global transposition routines, then via halo-cell exchanges. Identical
 results are to be expected regardless of the communication algorithm. The 

From eca2a2bf9b75b51a874465517ed03af71485bde0 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Sun, 5 Nov 2023 17:10:40 +0000
Subject: [PATCH 366/436] Add test for deferred writes

Runs both file per field and multiple field per file checks
---
 examples/io_test/CMakeLists.txt  |  14 ++
 examples/io_test/io_tmp_test.f90 | 388 +++++++++++++++++++++++++++++++
 2 files changed, 402 insertions(+)
 create mode 100644 examples/io_test/io_tmp_test.f90

diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt
index 2f5b5ac6..9e67543b 100644
--- a/examples/io_test/CMakeLists.txt
+++ b/examples/io_test/CMakeLists.txt
@@ -1,5 +1,6 @@
 file(GLOB files_test io_test.f90)
 file(GLOB files_read io_read.f90)
+file(GLOB files_tmp_test io_tmp_test.f90)
 file(GLOB files_var_test io_var_test.f90)
 file(GLOB files_plane_test io_plane_test.f90)
 file(GLOB files_bench io_bench.f90)
@@ -7,12 +8,14 @@ file(GLOB files_bench io_bench.f90)
 include_directories(${CMAKE_SOURCE_DIR}/src)
 
 add_executable(io_test ${files_test})
+add_executable(io_tmp_test ${files_tmp_test})
 add_executable(io_read ${files_read})
 add_executable(io_var_test ${files_var_test})
 add_executable(io_plane_test ${files_plane_test})
 add_executable(io_bench ${files_bench})
 
 target_link_libraries(io_test PRIVATE decomp2d)
+target_link_libraries(io_tmp_test PRIVATE decomp2d)
 target_link_libraries(io_read PRIVATE decomp2d)
 target_link_libraries(io_var_test PRIVATE decomp2d)
 target_link_libraries(io_plane_test PRIVATE decomp2d)
@@ -32,6 +35,17 @@ else ()
   add_test(NAME io_read COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_read> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
 endif ()
 # Run the test(s)
+set(run_dir "${test_dir}/io_tmp_test")
+file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME io_tmp_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:io_tmp_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+else ()
+  add_test(NAME io_tmp_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:io_tmp_test> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+endif ()
+# Run the test(s)
 set(run_dir "${test_dir}/io_var_test")
 file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
 message(STATUS "Example dir ${run_dir}")
diff --git a/examples/io_test/io_tmp_test.f90 b/examples/io_test/io_tmp_test.f90
new file mode 100644
index 00000000..8f046140
--- /dev/null
+++ b/examples/io_test/io_tmp_test.f90
@@ -0,0 +1,388 @@
+!!! io_tmp_test.f90
+!!
+!! Tests writing from temporary arrays using the flush functionality, only relevant for ADIOS2.
+!!
+!! SPDX-License-Identifier: BSD-3-Clause
+program io_test
+
+   use mpi
+   use decomp_2d_constants
+   use decomp_2d_mpi
+   use decomp_2d
+   use decomp_2d_io
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
+
+   implicit none
+
+   integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
+
+   integer :: ierr
+
+#ifdef COMPLEX_TEST
+   complex(mytype), allocatable, dimension(:, :, :) :: data1
+
+   complex(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   complex(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
+   complex(mytype), allocatable, dimension(:, :, :) :: v1, v2, v3
+   complex(mytype), allocatable, dimension(:, :, :) :: v1b, v2b, v3b
+
+   complex(mytype), allocatable, dimension(:, :, :) :: tmp1, tmp2, tmp3
+#else
+   real(mytype), allocatable, dimension(:, :, :) :: data1
+
+   real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3
+   real(mytype), allocatable, dimension(:, :, :) :: u1b, u2b, u3b
+   real(mytype), allocatable, dimension(:, :, :) :: v1, v2, v3
+   real(mytype), allocatable, dimension(:, :, :) :: v1b, v2b, v3b
+
+   real(mytype), allocatable, dimension(:, :, :) :: tmp1, tmp2, tmp3
+#endif
+
+   real(mytype), parameter :: eps = 1.0E-7_mytype
+
+   character(len=*), parameter :: io_name = "test-io"
+   character(len=*), parameter :: io_restart = "restart-io"
+
+   integer :: i, j, k, m, ierror
+   integer :: xst1, xst2, xst3
+   integer :: xen1, xen2, xen3
+
+#ifndef ADIOS2
+   logical :: dir_exists
+#endif
+
+   integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
+            p_row = DecInd
+         elseif (arg == 2) then
+            p_col = DecInd
+         elseif (arg == 3) then
+            nx = DecInd
+         elseif (arg == 4) then
+            ny = DecInd
+         elseif (arg == 5) then
+            nz = DecInd
+         end if
+      end do
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank == 0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      end if
+   end if
+
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   call decomp_2d_io_init()
+   call decomp_2d_init_io(io_name)
+   call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_name, "v1.dat", 1, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_name, "v2.dat", 2, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_name, "v3.dat", 3, 0, output2D, mytype)
+   call decomp_2d_init_io(io_restart)
+   call decomp_2d_register_variable(io_restart, "u1.dat", 1, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_restart, "u2.dat", 2, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_restart, "u3.dat", 3, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_restart, "v1.dat", 1, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_restart, "v2.dat", 2, 0, output2D, mytype)
+   call decomp_2d_register_variable(io_restart, "v3.dat", 3, 0, output2D, mytype)
+
+   ! ***** global data *****
+   allocate (data1(nx, ny, nz))
+   m = 1
+   do k = 1, nz
+      do j = 1, ny
+         do i = 1, nx
+#ifdef COMPLEX_TEST
+            data1(i, j, k) = cmplx(real(m, mytype), real(nx * ny * nz - m, mytype))
+#else
+            data1(i, j, k) = real(m, mytype)
+#endif
+            m = m + 1
+         end do
+      end do
+   end do
+
+   call alloc_x(u1, .true.)
+   call alloc_y(u2, .true.)
+   call alloc_z(u3, .true.)
+
+   call alloc_x(u1b, .true.)
+   call alloc_y(u2b, .true.)
+   call alloc_z(u3b, .true.)
+
+   call alloc_x(v1, .true.)
+   call alloc_y(v2, .true.)
+   call alloc_z(v3, .true.)
+
+   call alloc_x(v1b, .true.)
+   call alloc_y(v2b, .true.)
+   call alloc_z(v3b, .true.)
+
+   call alloc_x(tmp1, .true.)
+   call alloc_y(tmp2, .true.)
+   call alloc_z(tmp3, .true.)
+
+   xst1 = xstart(1); xen1 = xend(1)
+   xst2 = xstart(2); xen2 = xend(2)
+   xst3 = xstart(3); xen3 = xend(3)
+   ! original x-pencil based data
+   !$acc data copyin(data1) copy(u1,u2,u3,v1,v2,v3)
+   !$acc parallel loop default(present)
+   do k = xst3, xen3
+      do j = xst2, xen2
+         do i = xst1, xen1
+            u1(i, j, k) = data1(i, j, k)
+            v1(i, j, k) = 10 * data1(i, j, k)
+         end do
+      end do
+   end do
+   !$acc end loop
+
+   ! transpose
+   call transpose_x_to_y(u1, u2)
+   call transpose_y_to_z(u2, u3)
+   call transpose_x_to_y(v1, v2)
+   call transpose_y_to_z(v2, v3)
+   !$acc update self(u1)
+   !$acc update self(u2)
+   !$acc update self(u3)
+   !$acc update self(v1)
+   !$acc update self(v2)
+   !$acc update self(v3)
+   !$acc end data
+
+   ! write to disk
+#ifndef ADIOS2
+   if (nrank == 0) then
+      inquire (file="out", exist=dir_exists)
+      if (.not. dir_exists) then
+         call execute_command_line("mkdir out 2> /dev/null")
+      end if
+   end if
+#endif
+
+   ! Standard I/O pattern - file per field
+#ifdef ADIOS2
+   call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
+   call decomp_2d_start_io(io_name, "out")
+#endif
+   ! Copy data to temporary memory and write from the temporary memory
+
+   tmp1(:, :, :) = u1(:, :, :)
+   call decomp_2d_write_one(1, tmp1, 'out', 'u1.dat', 0, io_name, opt_deferred_writes=.false.)
+   tmp1(:, :, :) = v1(:, :, :)
+   call decomp_2d_write_one(1, tmp1, 'out', 'v1.dat', 0, io_name, opt_deferred_writes=.false.)
+
+   tmp2(:, :, :) = u2(:, :, :)
+   call decomp_2d_write_one(2, tmp2, 'out', 'u2.dat', 0, io_name, opt_deferred_writes=.false.)
+   tmp2(:, :, :) = v2(:, :, :)
+   call decomp_2d_write_one(2, tmp2, 'out', 'v2.dat', 0, io_name, opt_deferred_writes=.false.)
+
+   tmp3(:, :, :) = u3(:, :, :)
+   call decomp_2d_write_one(3, tmp3, 'out', 'u3.dat', 0, io_name, opt_deferred_writes=.false.)
+   tmp3(:, :, :) = v3(:, :, :)
+   call decomp_2d_write_one(3, tmp3, 'out', 'v3.dat', 0, io_name, opt_deferred_writes=.false.)
+#ifdef ADIOS2
+   call decomp_2d_end_io(io_name, "out")
+   call decomp_2d_close_io(io_name, "out")
+#endif
+
+   ! read back to different arrays
+#ifdef ADIOS2
+   call decomp_2d_open_io(io_name, "out", decomp_2d_read_mode)
+   call decomp_2d_start_io(io_name, "out")
+#endif
+   call decomp_2d_read_one(1, u1b, 'out', 'u1.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(2, u2b, 'out', 'u2.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(3, u3b, 'out', 'u3.dat', io_name, reduce_prec=.false.)
+
+   call decomp_2d_read_one(1, v1b, 'out', 'v1.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(2, v2b, 'out', 'v2.dat', io_name, reduce_prec=.false.)
+   call decomp_2d_read_one(3, v3b, 'out', 'v3.dat', io_name, reduce_prec=.false.)
+#ifdef ADIOS2
+   call decomp_2d_end_io(io_name, "out")
+   call decomp_2d_close_io(io_name, "out")
+#endif
+
+   ! compare
+   call check("file per field")
+
+   ! Checkpoint I/O pattern - multiple fields per file
+   call decomp_2d_open_io(io_name, "checkpoint", decomp_2d_write_mode)
+   call decomp_2d_start_io(io_name, "checkpoint")
+
+   tmp1(:, :, :) = u1(:, :, :)
+   call decomp_2d_write_one(1, tmp1, 'checkpoint', 'u1.dat', 0, io_name, opt_deferred_writes=.false.)
+   tmp1(:, :, :) = v1(:, :, :)
+   call decomp_2d_write_one(1, tmp1, 'checkpoint', 'v1.dat', 0, io_name, opt_deferred_writes=.false.)
+
+   tmp2(:, :, :) = u2(:, :, :)
+   call decomp_2d_write_one(2, tmp2, 'checkpoint', 'u2.dat', 0, io_name, opt_deferred_writes=.false.)
+   tmp2(:, :, :) = v2(:, :, :)
+   call decomp_2d_write_one(2, tmp2, 'checkpoint', 'v2.dat', 0, io_name, opt_deferred_writes=.false.)
+
+   tmp3(:, :, :) = u3(:, :, :)
+   call decomp_2d_write_one(3, tmp3, 'checkpoint', 'u3.dat', 0, io_name, opt_deferred_writes=.false.)
+   tmp3(:, :, :) = v3(:, :, :)
+   call decomp_2d_write_one(3, tmp3, 'checkpoint', 'v3.dat', 0, io_name, opt_deferred_writes=.false.)
+
+   call decomp_2d_end_io(io_name, "checkpoint")
+   call decomp_2d_close_io(io_name, "checkpoint")
+
+   call MPI_Barrier(MPI_COMM_WORLD, ierr)
+
+   ! read back to different arrays
+   ! XXX: For the MPI-IO backend the order of reading must match the order of writing!
+   u1b = 0; u2b = 0; u3b = 0
+   v1b = 0; v2b = 0; v3b = 0
+   call decomp_2d_open_io(io_restart, "checkpoint", decomp_2d_read_mode)
+   call decomp_2d_start_io(io_restart, "checkpoint")
+
+   call decomp_2d_read_one(1, u1b, 'checkpoint', 'u1.dat', io_restart, reduce_prec=.false.)
+   call decomp_2d_read_one(1, v1b, 'checkpoint', 'v1.dat', io_restart, reduce_prec=.false.)
+
+   call decomp_2d_read_one(2, u2b, 'checkpoint', 'u2.dat', io_restart, reduce_prec=.false.)
+   call decomp_2d_read_one(2, v2b, 'checkpoint', 'v2.dat', io_restart, reduce_prec=.false.)
+
+   call decomp_2d_read_one(3, u3b, 'checkpoint', 'u3.dat', io_restart, reduce_prec=.false.)
+   call decomp_2d_read_one(3, v3b, 'checkpoint', 'v3.dat', io_restart, reduce_prec=.false.)
+
+   call decomp_2d_end_io(io_restart, "checkpoint")
+   call decomp_2d_close_io(io_restart, "checkpoint")
+
+   call MPI_Barrier(MPI_COMM_WORLD, ierr)
+
+   ! compare
+   call check("one file, multiple fields")
+
+   deallocate (u1, u2, u3)
+   deallocate (u1b, u2b, u3b)
+   deallocate (data1)
+   call decomp_2d_finalize
+   call MPI_FINALIZE(ierror)
+
+contains
+
+   subroutine check(stage)
+
+      character(len=*), intent(in) :: stage
+
+      integer :: ierr
+
+      if (nrank == 0) then
+         print *, "Checking "//stage
+      end if
+      call MPI_Barrier(MPI_COMM_WORLD, ierr)
+
+      do k = xstart(3), xend(3)
+         do j = xstart(2), xend(2)
+            do i = xstart(1), xend(1)
+               if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) then
+                  print *, u1(i, j, k), u1b(i, j, k)
+                  stop 1
+               end if
+               if (abs((v1(i, j, k) - v1b(i, j, k))) > eps) then
+                  print *, v1(i, j, k), v1b(i, j, k)
+                  stop 11
+               end if
+            end do
+         end do
+      end do
+
+      do k = ystart(3), yend(3)
+         do j = ystart(2), yend(2)
+            do i = ystart(1), yend(1)
+               if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) then
+                  print *, u2(i, j, k), u2b(i, j, k)
+                  stop 2
+               end if
+               if (abs((v2(i, j, k) - v2b(i, j, k))) > eps) stop 22
+            end do
+         end do
+      end do
+
+      do k = zstart(3), zend(3)
+         do j = zstart(2), zend(2)
+            do i = zstart(1), zend(1)
+               if (abs((u3(i, j, k) - u3b(i, j, k))) > eps) stop 3
+               if (abs((v3(i, j, k) - v3b(i, j, k))) > eps) stop 33
+            end do
+         end do
+      end do
+
+      ! Also check against the global data array
+      do k = xstart(3), xend(3)
+         do j = xstart(2), xend(2)
+            do i = xstart(1), xend(1)
+               if (abs(data1(i, j, k) - u1b(i, j, k)) > eps) stop 4
+               if (abs(10 * data1(i, j, k) - v1b(i, j, k)) > eps) stop 44
+            end do
+         end do
+      end do
+
+      do k = ystart(3), yend(3)
+         do j = ystart(2), yend(2)
+            do i = ystart(1), yend(1)
+               if (abs((data1(i, j, k) - u2b(i, j, k))) > eps) stop 5
+               if (abs((10 * data1(i, j, k) - v2b(i, j, k))) > eps) stop 55
+            end do
+         end do
+      end do
+
+      do k = zstart(3), zend(3)
+         do j = zstart(2), zend(2)
+            do i = zstart(1), zend(1)
+               if (abs((data1(i, j, k) - u3b(i, j, k))) > eps) stop 6
+               if (abs((10 * data1(i, j, k) - v3b(i, j, k))) > eps) stop 66
+            end do
+         end do
+      end do
+
+      call MPI_Barrier(MPI_COMM_WORLD, ierr)
+      if (nrank == 0) then
+         print *, "Checking "//stage//" pass!"
+      end if
+
+   end subroutine check
+
+end program io_test

From bac7c994493f902b943fd14cd3b3bdcf3bffb40b Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Sun, 5 Nov 2023 17:39:12 +0000
Subject: [PATCH 367/436] Add missing deallocations in io_tmp_test

---
 examples/io_test/io_tmp_test.f90 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/io_test/io_tmp_test.f90 b/examples/io_test/io_tmp_test.f90
index 8f046140..7a295430 100644
--- a/examples/io_test/io_tmp_test.f90
+++ b/examples/io_test/io_tmp_test.f90
@@ -297,6 +297,8 @@ program io_test
 
    deallocate (u1, u2, u3)
    deallocate (u1b, u2b, u3b)
+   deallocate (v1, v2, v3)
+   deallocate (v1b, v2b, v3b)
    deallocate (data1)
    call decomp_2d_finalize
    call MPI_FINALIZE(ierror)

From 9e685197240868c517d0229a1f8e1de192b78c5c Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Sun, 5 Nov 2023 20:13:55 +0000
Subject: [PATCH 368/436] Add description of deferred writes operation

---
 HOWTO.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/HOWTO.md b/HOWTO.md
index d0f753e9..23ccab60 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -85,7 +85,11 @@ Y-pencil and 3 for Z-pencil); ``var`` is the reference to the data array, which
 complex; ``directory`` is the path to where I/O should be written; ``filename`` is the name of the
 file to be written; ``icoarse`` indicates whether the I/O should be coarsend (valid values are: 0
 for no; 1 for the ``nstat`` and 2 for the ``nvisu`` coarsenings); ``io_name`` is the name of the I/O
-group to be used. 
+group to be used. When using ADIOS2 write operations are deferred by default, this means that before the
+end of the step the data stored in ``var`` may not have been written. Overwriting ``var``, for example
+when used as a temporary variable, would cause the output data to become corrupted. In such situations
+``decomp_2d_write_one`` accepts an optional argument ``opt_deferred_writes`` (default ``.true.``) which
+when set to ``.false.`` causes the data to be flushed immediately.
 
 There are two ways of writing multiple variables to a single file which may
 be used for check-pointing purposes, for example. The newer interface is described first and allows

From 6f4c0725e76e630a6587d073859600cb5e494f78 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 6 Nov 2023 07:31:36 +0000
Subject: [PATCH 369/436] Update HOWTO.md for text corrections

---
 HOWTO.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/HOWTO.md b/HOWTO.md
index d0f753e9..908d9f79 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -1,5 +1,5 @@
-# How To use 2DECOMP&FFT
-This document presents the basic usage of 2DECOMP&FFT. 
+# How to use 2DECOMP&FFT
+This document presents the main features of 2DECOMP&FFT library. 
 Detailed instructions on how to use the 2DECOMP&FFT library can be found
 [here](https://2decomp-fft.github.io/).
 
@@ -76,14 +76,14 @@ All the I/O functions have been packed in a Fortran module:
 ```
   use decomp_2d_io
 ```
-To write a single three-dimensional array to a file
+To write a single three-dimensional array to a file, the following call to a subroutine can be used:
 ```
   call decomp_2d_write_one(ipencil,var,directory,filename,icoarse,io_name)
 ```
 where ``ipencil`` describes how the data is distributed (valid values are: 1 for X-pencil; 2 for
-Y-pencil and 3 for Z-pencil); ``var`` is the reference to the data array, which can be either real or
+Y-pencil and 3 for Z-pencil); ``var`` is the data array to be written on disk, which can be either real or
 complex; ``directory`` is the path to where I/O should be written; ``filename`` is the name of the
-file to be written; ``icoarse`` indicates whether the I/O should be coarsend (valid values are: 0
+file to be written; ``icoarse`` indicates whether the I/O should be coarsened (valid values are: 0
 for no; 1 for the ``nstat`` and 2 for the ``nvisu`` coarsenings); ``io_name`` is the name of the I/O
 group to be used. 
 

From b239fe2ca4b6c8fe437f88b65433e886d7f0d590 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 6 Nov 2023 10:08:19 +0000
Subject: [PATCH 370/436] Update READMEs for text corrections

---
 examples/README.md        | 2 +-
 examples/test2d/README.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/README.md b/examples/README.md
index 9849cf25..bb6ef31b 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -5,6 +5,6 @@
 - [Test FFT from X pencil](fft_physical_x) - various tests for the FFT starting from the ``X`` direction 
 - [Test FFT from Z pencil](fft_physical_z) - various tests for the FFT starting from the ``Z`` direction 
 - [Test HALO exchange](halo_test)          - to test the halo-cell exchange capability
-- [Test IO features](io_test)              - various tests for the IO module
+- [Test I/O features](io_test)             - various tests for the I/O module
 
 Refer to the README files for each example for details.
diff --git a/examples/test2d/README.md b/examples/test2d/README.md
index 41021240..6ce544f0 100644
--- a/examples/test2d/README.md
+++ b/examples/test2d/README.md
@@ -24,6 +24,6 @@ If the decomposition is imposed both (1) and (2) are necessary.
 If the resolution is imposed (1-5) are necessary
 
 What to expect: 
-- For test2d the output is e success message or an error message with the direction of error in swapping;
+- For test2d the output is a success message or an error message with the direction of swapping;
 - For the timing, beside the error/success message timing for all transpostions
   (X->Y, Y->Z, Z->Y, Y->X) together with the sum are reported. 

From 71f33a0acb6af4bd7d5449704f23b5ecb9c8ed38 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Mon, 6 Nov 2023 15:27:41 +0000
Subject: [PATCH 371/436] Add the 2d grad test

---
 examples/CMakeLists.txt           |   1 +
 examples/grad3d/CMakeLists.txt    |  16 ++
 examples/grad3d/README.md         |  24 ++
 examples/grad3d/adios2_config.xml |  19 ++
 examples/grad3d/bind.sh           |  10 +
 examples/grad3d/grad3d.f90        | 379 ++++++++++++++++++++++++++++++
 6 files changed, 449 insertions(+)
 create mode 100644 examples/grad3d/CMakeLists.txt
 create mode 100644 examples/grad3d/README.md
 create mode 100644 examples/grad3d/adios2_config.xml
 create mode 100755 examples/grad3d/bind.sh
 create mode 100644 examples/grad3d/grad3d.f90

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index a2da4c8e..e8901e9e 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -12,3 +12,4 @@ add_subdirectory(fft_physical_x)
 add_subdirectory(fft_physical_z)
 add_subdirectory(halo_test)
 add_subdirectory(io_test)
+add_subdirectory(grad3d)
diff --git a/examples/grad3d/CMakeLists.txt b/examples/grad3d/CMakeLists.txt
new file mode 100644
index 00000000..3cdd6a61
--- /dev/null
+++ b/examples/grad3d/CMakeLists.txt
@@ -0,0 +1,16 @@
+file(GLOB files_test grad3d.f90)
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+add_executable(grad3d ${files_test})
+target_link_libraries(grad3d PRIVATE decomp2d)
+
+# Run the test(s)
+set(run_dir "${test_dir}/grad3d")
+message(STATUS "Example dir ${run_dir}")
+file(MAKE_DIRECTORY ${run_dir})
+if (BUILD_TARGET MATCHES "gpu")
+  file(COPY bind.sh DESTINATION ${run_dir})
+  add_test(NAME grad3d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:grad3d> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+else ()  
+  add_test(NAME grad3d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $<TARGET_FILE:grad3d> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
+endif ()
diff --git a/examples/grad3d/README.md b/examples/grad3d/README.md
new file mode 100644
index 00000000..d7f89ac8
--- /dev/null
+++ b/examples/grad3d/README.md
@@ -0,0 +1,24 @@
+# Halo test
+
+List of the tests:
+- [grad3d](grad3d.f90): Example to compute the gradient of a field. 
+
+This example demonstrates the use 2DECOMP&FFT library to compute the gradient
+of a field using an explicit second order finite difference scheme. 
+The purpose is to show how to use the transpose operations to allow explicit calculation
+of the gradient in all 3 directions. The results are written on a file and the function 
+is periodic over the interval [0-1] 
+
+What to input: The program takes max 5 inputs as: 
+
+1. p_row [optional]
+1. p_col [optional] 
+1. nx    [optional]
+1. ny    [optional]
+1. nz    [optional]
+
+If the decomposition is imposed both (1) and (2) are necessary. 
+If the resolution is imposed (1-5) are necessary.
+
+What to expect: the output is the original function and the gradient in the 3 direction. 
+                The program will also give the total error in L2 norm comparing with the anytical solution. 
diff --git a/examples/grad3d/adios2_config.xml b/examples/grad3d/adios2_config.xml
new file mode 100644
index 00000000..92ece049
--- /dev/null
+++ b/examples/grad3d/adios2_config.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<adios-config>
+  <io name="test-io">
+    <engine type="BP4">
+    </engine>
+    <transport type="File">
+      <parameter key="Library" value="fstream"/>
+      <parameter key="ProfileUnits" value="Milliseconds"/>
+    </transport>
+  </io>
+  <io name="restart-io">
+    <engine type="BP4">
+    </engine>
+    <transport type="File">
+      <parameter key="Library" value="fstream"/>
+      <parameter key="ProfileUnits" value="Milliseconds"/>
+    </transport>
+  </io>
+</adios-config>
diff --git a/examples/grad3d/bind.sh b/examples/grad3d/bind.sh
new file mode 100755
index 00000000..38d4fedf
--- /dev/null
+++ b/examples/grad3d/bind.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export LOCAL_RANK=${OMPI_COMM_WORLD_LOCAL_RANK}
+export CUDA_VISIBLE_DEVICES=${LOCAL_RANK}
+
+echo "[LOG] local rank $LOCAL_RANK: bind to $CUDA_VISIBLE_DEVICES"
+echo ""
+
+$* 
+
diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
new file mode 100644
index 00000000..8843e43a
--- /dev/null
+++ b/examples/grad3d/grad3d.f90
@@ -0,0 +1,379 @@
+!! SPDX-License-Identifier: BSD-3-Clause
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+! This example calculates the gradient of a periodic field using global
+! transposition
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+program grad3d
+
+   use mpi
+
+   use decomp_2d
+   use decomp_2d_constants
+   use decomp_2d_mpi
+   use MPI
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
+
+   implicit none
+
+   integer, parameter :: nx_base = 64, ny_base = 64, nz_base = 64
+   integer :: nx, ny, nz
+   integer :: p_row = 0, p_col = 0
+   integer :: resize_domain
+   integer :: nranks_tot
+   integer :: nargin, arg, FNLength, status, DecInd
+   character(len=80) :: InputFN
+
+   real(mytype), parameter :: lx = 1.0_mytype
+   real(mytype), parameter :: ly = 1.0_mytype
+   real(mytype), parameter :: lz = 1.0_mytype
+
+   real(mytype) :: dx, dy, dz
+
+   real(mytype), allocatable, dimension(:, :, :) :: phi1, phi2, phi3
+   real(mytype), allocatable, dimension(:, :, :) :: dphiX, dphiY, dphiz
+   real(mytype), allocatable, dimension(:, :, :) :: wk2, wk3
+
+   integer :: ierror
+
+   integer :: xlast, ylast, zlast
+
+
+   logical :: passing, all_pass
+
+   call MPI_INIT(ierror)
+   ! To resize the domain we need to know global number of ranks
+   ! This operation is also done as part of decomp_2d_init
+   call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierror)
+   resize_domain = int(nranks_tot / 4) + 1
+   nx = nx_base * resize_domain
+   ny = ny_base * resize_domain
+   nz = nz_base * resize_domain
+   ! Now we can check if user put some inputs
+   ! Handle input file like a boss -- GD
+   nargin = command_argument_count()
+   if ((nargin == 0) .or. (nargin == 2) .or. (nargin == 5)) then
+      do arg = 1, nargin
+         call get_command_argument(arg, InputFN, FNLength, status)
+         read (InputFN, *, iostat=status) DecInd
+         if (arg == 1) then
+            p_row = DecInd
+         elseif (arg == 2) then
+            p_col = DecInd
+         elseif (arg == 3) then
+            nx = DecInd
+         elseif (arg == 4) then
+            ny = DecInd
+         elseif (arg == 5) then
+            nz = DecInd
+         end if
+      end do
+   else
+      ! nrank not yet computed we need to avoid write
+      ! for every rank
+      call MPI_COMM_RANK(MPI_COMM_WORLD, nrank, ierror)
+      if (nrank == 0) then
+         print *, "This Test takes no inputs or 2 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "or 5 inputs as"
+         print *, "  1) p_row (default=0)"
+         print *, "  2) p_col (default=0)"
+         print *, "  3) nx "
+         print *, "  4) ny "
+         print *, "  5) nz "
+         print *, "Number of inputs is not correct and the defult settings"
+         print *, "will be used"
+      end if
+   end if
+   call decomp_2d_init(nx, ny, nz, p_row, p_col)
+
+   dx = lx/real(nx,mytype)
+   dy = ly/real(ny,mytype)
+   dz = lz/real(nz,mytype)
+
+   call allocate_var()
+   !$acc data create(phi2,phi3,wk2,wk3) copy(phi1,dphiX, dphiY, dphiZ)
+   call init_phi()
+   call compute_grad()
+   call test_derX(dphiX)
+   call test_derY()
+   call test_derZ()
+   !$acc end data
+
+   if (nrank == 0) then
+      write (*, *) '-----------------------------------------------'
+      write (*, *) "End GRAD calculation check all pass: ", all_pass
+      write (*, *) '==============================================='
+   end if
+
+   call finalize()
+
+   call decomp_2d_finalize
+
+   if (.not. all_pass) call decomp_2d_abort(1, "Error in grad3d")
+
+   call MPI_FINALIZE(ierror)
+
+contains
+
+   !=====================================================================
+   ! Initialize
+   !=====================================================================
+   subroutine allocate_var()
+
+      use decomp_2d
+
+      implicit none
+
+      logical, parameter :: glob = .false.
+
+      ! initialise u,v,w with random numbers in X-pencil
+      call alloc_x(phi1, glob)
+      call alloc_x(dphiX, glob)
+      call alloc_x(dphiY, glob)
+      call alloc_x(dphiZ, glob)
+
+      ! Working array used more than once
+      call alloc_y(phi2, glob)
+      call alloc_y(wk2, glob)
+
+      call alloc_z(phi3, glob)
+      call alloc_z(wk3, glob)
+
+      all_pass = .true.
+
+   end subroutine allocate_var
+
+   !=====================================================================
+   ! Initialize the scalar field
+   !=====================================================================
+   subroutine init_phi()
+
+      implicit none
+
+      integer :: i, j, k
+      real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
+      real(mytype) :: x, y, z
+      integer :: xe1, xe2, xe3
+      integer :: xs1, xs2, xs3
+
+      xe1 = xsize(1)
+      xe2 = xsize(2)
+      xe3 = xsize(3)
+      xs1 = xstart(1)
+      xs2 = xstart(2)
+      xs3 = xstart(3)
+
+      ! Initial velocity
+      !$acc kernels default(present)
+      do concurrent (k=1:xe3, j=1:xe2, i=1:xe1)
+         z = (k + xs3 - 2) * dz
+         y = (j + xs2 - 2) * dy
+         x = (i + xs1 - 2) * dx
+         phi1(i, j, k) = -2._mytype * cos(twopi * (x / lx)) * cos(twopi * (y / ly)) * sin(twopi * (z / lz))
+      enddo
+      !$acc end kernels
+
+   end subroutine init_phi
+   !=====================================================================
+   ! Finalize with deallocation of arrays
+   !=====================================================================
+   subroutine finalize()
+
+      implicit none
+
+      deallocate (phi1, phi2, phi3)
+      deallocate (dphiX, dphiY, dphiZ)
+      deallocate (wk2, wk3)
+
+   end subroutine finalize
+   !=====================================================================
+   ! Calculate gradient using global transposition
+   !=====================================================================
+   subroutine compute_grad()
+
+      implicit none
+
+      ! Compute X derivative
+      call derx(dphiX,phi1,dx,xsize(1),xsize(2),xsize(3))
+
+      ! Compute Y derivative
+      call transpose_x_to_y(phi1,phi2)
+      call dery(wk2,phi2,dy,ysize(1),ysize(2),ysize(3))
+      call transpose_y_to_x(wk2,dphiY)
+
+      ! Compute Z derivative
+      call transpose_y_to_z(phi2,phi3)
+      call derx(wk3,phi3,dz,zsize(1),zsize(2),zsize(3))
+      call transpose_z_to_y(wk3,wk2)
+      call transpose_y_to_x(wk2,dphiZ)
+
+   end subroutine compute_grad
+   !=====================================================================
+   ! Calculate gradient in X (data in X-pencil)
+   !=====================================================================
+   subroutine derx(df,ff,delta,nx,ny,nz)
+
+      implicit none
+
+      ! Arguments
+      integer, intent(in) :: nx, ny, nz
+      real(mytype), intent(in) :: delta
+      real(mytype), intent(out), dimension(nx,ny,nz) :: df
+      real(mytype), intent(in), dimension(nx,ny,nz) :: ff
+
+      ! Local variables
+      integer :: i, j, k
+      real(mytype) :: coeff = 0.5_mytype
+
+      coeff = coeff/delta
+
+      !$acc kernels default(present)
+      do concurrent (k=1:nz, j=1:ny)
+         df(1,j,k) = coeff*(ff(2,j,k)-ff(nx,j,k)) 
+         do concurrent (i=2:nx-1)
+            df(i,j,k) = coeff*(ff(i+1,j,k)-ff(i-1,j,k)) 
+         enddo
+         df(nx,j,k) = coeff*(ff(1,j,k)-ff(nx-1,j,k))
+      enddo
+      !$acc end kernels
+
+   end subroutine derx 
+   !=====================================================================
+   ! Calculate gradient in Y (data in Y-pencil)
+   !=====================================================================
+   subroutine dery(df,ff,delta,nx,ny,nz)
+
+      implicit none
+
+      ! Arguments
+      integer, intent(in) :: nx, ny, nz
+      real(mytype), intent(in) :: delta
+      real(mytype), intent(out), dimension(nx,ny,nz) :: df
+      real(mytype), intent(in), dimension(nx,ny,nz) :: ff
+
+      ! Local variables
+      integer :: i, j, k
+      real(mytype) :: coeff = 0.5_mytype
+
+      coeff = coeff/delta
+
+      !$acc kernels default(present)
+      do concurrent (k=1:nz)
+        do concurrent (i=1:nx)
+           df(i,1,k) = coeff*(ff(i,2,k)-ff(i,ny,k))
+        enddo
+        do concurrent (j=2:ny-1, i=1:nx)
+           df(i,j,k) = coeff*(ff(i,j+1,k)-ff(i,j-1,k))
+        enddo
+        do concurrent (i=1:nx)
+           df(i,ny,k) = coeff*(ff(i,1,k)-ff(i,ny-1,k)) 
+        enddo
+      enddo
+      !$acc end kernels
+
+   end subroutine dery 
+   !=====================================================================
+   ! Calculate gradient in Z (data in Z-pencil)
+   !=====================================================================
+   subroutine derz(df,ff,delta,nx,ny,nz)
+
+      implicit none
+
+      ! Arguments
+      integer, intent(in) :: nx, ny, nz
+      real(mytype), intent(in) :: delta
+      real(mytype), intent(out), dimension(nx,ny,nz) :: df
+      real(mytype), intent(in), dimension(nx,ny,nz) :: ff
+
+      ! Local variables
+      integer :: i, j, k
+      real(mytype) :: coeff = 0.5_mytype
+      
+      coeff = coeff/delta
+
+      !$acc kernels default(present)
+      do concurrent (j=1:ny, i=1:nx)
+         df(i,j,1) = coeff*(ff(i,j,2)-ff(i,j,nz  ))
+      enddo
+      do concurrent (k=2:nz-1, j=1:ny, i=1:nx)
+         df(i,j,k) = coeff*(ff(i,j,k+1)-ff(i,j,k-1))
+      enddo
+      do concurrent (j=1:ny, i=1:nx)
+         df(i,j,nz) = coeff*(ff(i,j,1)-ff(i,j,nz-1))
+      enddo
+      !$acc end kernels
+
+   end subroutine derz 
+   !=====================================================================
+   ! Calculate divergence using halo-cell exchange (data in X-pencil)
+   !=====================================================================
+   subroutine test_derX(df)
+
+      implicit none
+      ! Arguments
+      real(mytype), intent(out), dimension(xsize(1),xsize(2),xsize(3)) :: df
+
+      integer :: i, j, k
+      real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
+      real(mytype) :: x, y, z
+      real(mytype) :: dphi, dphi_num
+      real(mytype) :: error = 0._mytype
+      real(mytype) :: err_all = 0._mytype
+      integer :: xe1, xe2, xe3
+      integer :: xs1, xs2, xs3
+
+      xe1 = xsize(1)
+      xe2 = xsize(2)
+      xe3 = xsize(3)
+      xs1 = xstart(1)
+      xs2 = xstart(2)
+      xs3 = xstart(3)
+
+      ! Initial velocity
+      !$acc parallel loop default(present) reduction(+:error)
+      do k=1,xe3
+        do j=1,xe2
+          do i=1,xe1
+             z = (k + xs3 - 2) * dz
+             y = (j + xs2 - 2) * dy
+             x = (i + xs1 - 2) * dx
+             dphi = -2._mytype * twopi * sin (twopi * (x / lx)) * cos(twopi * (y / ly)) * sin(twopi * (z / lz))
+             dphi_num = df(i,j,k)
+             error = error + (dphi-dphi_num)*(dphi-dphi_num)
+          enddo
+        enddo
+      enddo
+      !$acc end loop
+      call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+      err_all = sqrt(err_all) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+
+      if (nrank == 0) then
+         write (*, *) 'DX error / mesh point: ', err_all
+      endif
+
+   end subroutine test_derX
+
+   !=====================================================================
+   ! Calculate divergence using halo-cell exchange (data in Y-pencil)
+   !=====================================================================
+   subroutine test_derY()
+
+      implicit none
+
+
+   end subroutine test_derY
+
+   !=====================================================================
+   ! Calculate divergence using halo-cell exchange (data in Z-pencil)
+   !=====================================================================
+   subroutine test_derZ()
+
+      implicit none
+
+   end subroutine test_derZ
+
+end program grad3d

From 14f8a88b5550a68d40136c1cdc66e99a783342f4 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 7 Nov 2023 08:02:16 +0000
Subject: [PATCH 372/436] Add visualisation to grad3d example

---
 examples/grad3d/grad3d.f90 | 274 +++++++++++++++++++++++++++++++++++--
 1 file changed, 260 insertions(+), 14 deletions(-)

diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
index 8843e43a..a65b6693 100644
--- a/examples/grad3d/grad3d.f90
+++ b/examples/grad3d/grad3d.f90
@@ -24,6 +24,8 @@ program grad3d
    integer :: resize_domain
    integer :: nranks_tot
    integer :: nargin, arg, FNLength, status, DecInd
+   integer :: ierror
+   logical :: passing, all_pass
    character(len=80) :: InputFN
 
    real(mytype), parameter :: lx = 1.0_mytype
@@ -31,17 +33,12 @@ program grad3d
    real(mytype), parameter :: lz = 1.0_mytype
 
    real(mytype) :: dx, dy, dz
+   real(mytype) :: error_ref 
 
    real(mytype), allocatable, dimension(:, :, :) :: phi1, phi2, phi3
    real(mytype), allocatable, dimension(:, :, :) :: dphiX, dphiY, dphiz
    real(mytype), allocatable, dimension(:, :, :) :: wk2, wk3
 
-   integer :: ierror
-
-   integer :: xlast, ylast, zlast
-
-
-   logical :: passing, all_pass
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -93,14 +90,21 @@ program grad3d
    dx = lx/real(nx,mytype)
    dy = ly/real(ny,mytype)
    dz = lz/real(nz,mytype)
+   
+   if (nrank == 0) then
+      write (*, *) '-----------------------------------------------'
+      write (*, *) "Mesh Resolution ", nx , ny, nz 
+      write (*, *) '-----------------------------------------------'
+   end if
 
    call allocate_var()
    !$acc data create(phi2,phi3,wk2,wk3) copy(phi1,dphiX, dphiY, dphiZ)
    call init_phi()
    call compute_grad()
    call test_derX(dphiX)
-   call test_derY()
-   call test_derZ()
+   call test_derY(dphiY)
+   call test_derZ(dphiZ)
+   call write_data()
    !$acc end data
 
    if (nrank == 0) then
@@ -207,7 +211,7 @@ subroutine compute_grad()
 
       ! Compute Z derivative
       call transpose_y_to_z(phi2,phi3)
-      call derx(wk3,phi3,dz,zsize(1),zsize(2),zsize(3))
+      call derz(wk3,phi3,dz,zsize(1),zsize(2),zsize(3))
       call transpose_z_to_y(wk3,wk2)
       call transpose_y_to_x(wk2,dphiZ)
 
@@ -323,6 +327,8 @@ subroutine test_derX(df)
       real(mytype) :: dphi, dphi_num
       real(mytype) :: error = 0._mytype
       real(mytype) :: err_all = 0._mytype
+      real(mytype) :: dphi2 = 0._mytype
+      real(mytype) :: sum_dphi2 = 0._mytype
       integer :: xe1, xe2, xe3
       integer :: xs1, xs2, xs3
 
@@ -341,7 +347,8 @@ subroutine test_derX(df)
              z = (k + xs3 - 2) * dz
              y = (j + xs2 - 2) * dy
              x = (i + xs1 - 2) * dx
-             dphi = -2._mytype * twopi * sin (twopi * (x / lx)) * cos(twopi * (y / ly)) * sin(twopi * (z / lz))
+             dphi = -2._mytype * twopi * sin(twopi * (x / lx)) * cos(twopi * (y / ly)) * sin(twopi * (z / lz))
+             dphi2 = dphi2 + dphi * dphi
              dphi_num = df(i,j,k)
              error = error + (dphi-dphi_num)*(dphi-dphi_num)
           enddo
@@ -349,8 +356,10 @@ subroutine test_derX(df)
       enddo
       !$acc end loop
       call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-      err_all = sqrt(err_all) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
-
+      call MPI_ALLREDUCE(dphi2, sum_dphi2, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+      err_all = sqrt(err_all/sum_dphi2) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+      error_ref = err_all
+      
       if (nrank == 0) then
          write (*, *) 'DX error / mesh point: ', err_all
       endif
@@ -360,20 +369,257 @@ end subroutine test_derX
    !=====================================================================
    ! Calculate divergence using halo-cell exchange (data in Y-pencil)
    !=====================================================================
-   subroutine test_derY()
+   subroutine test_derY(df)
 
       implicit none
+      ! Arguments
+      real(mytype), intent(out), dimension(xsize(1),xsize(2),xsize(3)) :: df
 
+      integer :: i, j, k
+      real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
+      real(mytype) :: x, y, z
+      real(mytype) :: dphi, dphi_num
+      real(mytype) :: error = 0._mytype
+      real(mytype) :: err_all = 0._mytype
+      real(mytype) :: dphi2 = 0._mytype
+      real(mytype) :: sum_dphi2 = 0._mytype
+      integer :: xe1, xe2, xe3
+      integer :: xs1, xs2, xs3
+
+      xe1 = xsize(1)
+      xe2 = xsize(2)
+      xe3 = xsize(3)
+      xs1 = xstart(1)
+      xs2 = xstart(2)
+      xs3 = xstart(3)
+
+      ! Initial velocity
+      !$acc parallel loop default(present) reduction(+:error)
+      do k=1,xe3
+        do j=1,xe2
+          do i=1,xe1
+             z = (k + xs3 - 2) * dz
+             y = (j + xs2 - 2) * dy
+             x = (i + xs1 - 2) * dx
+             dphi = -2._mytype * twopi * cos(twopi * (x / lx)) * sin(twopi * (y / ly)) * sin(twopi * (z / lz))
+             dphi2 = dphi2 + dphi * dphi
+             dphi_num = df(i,j,k)
+             error = error + (dphi-dphi_num)*(dphi-dphi_num)
+          enddo
+        enddo
+      enddo
+      !$acc end loop
+      call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+      call MPI_ALLREDUCE(dphi2, sum_dphi2, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+      err_all = sqrt(err_all/sum_dphi2) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+
+      if (nrank == 0) then
+         write (*, *) 'DY error / mesh point: ', err_all
+      endif
+
+      if (abs(err_all-error_ref) > 1.0e-5_mytype) all_pass = .false.
 
    end subroutine test_derY
 
    !=====================================================================
    ! Calculate divergence using halo-cell exchange (data in Z-pencil)
    !=====================================================================
-   subroutine test_derZ()
+   subroutine test_derZ(df)
 
       implicit none
+      ! Arguments
+      real(mytype), intent(out), dimension(xsize(1),xsize(2),xsize(3)) :: df
+
+      integer :: i, j, k
+      real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
+      real(mytype) :: x, y, z
+      real(mytype) :: dphi, dphi_num
+      real(mytype) :: error = 0._mytype
+      real(mytype) :: err_all = 0._mytype
+      real(mytype) :: dphi2 = 0._mytype
+      real(mytype) :: sum_dphi2 = 0._mytype
+      integer :: xe1, xe2, xe3
+      integer :: xs1, xs2, xs3
+
+      xe1 = xsize(1)
+      xe2 = xsize(2)
+      xe3 = xsize(3)
+      xs1 = xstart(1)
+      xs2 = xstart(2)
+      xs3 = xstart(3)
+
+      ! Initial velocity
+      !$acc parallel loop default(present) reduction(+:error)
+      do k=1,xe3
+        do j=1,xe2
+          do i=1,xe1
+             z = (k + xs3 - 2) * dz
+             y = (j + xs2 - 2) * dy
+             x = (i + xs1 - 2) * dx
+             dphi = 2._mytype * twopi * cos(twopi * (x / lx)) * cos(twopi * (y / ly)) * cos(twopi * (z / lz))
+             dphi2 = dphi2 + dphi * dphi
+             dphi_num = df(i,j,k)
+             error = error + (dphi-dphi_num)*(dphi-dphi_num)
+          enddo
+        enddo
+      enddo
+      !$acc end loop
+      call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+      call MPI_ALLREDUCE(dphi2, sum_dphi2, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
+      err_all = sqrt(err_all/sum_dphi2) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+
+      if (nrank == 0) then
+         write (*, *) 'DZ error / mesh point: ', err_all
+      endif
+
+      if (abs(err_all-error_ref) > 1.0e-5_mytype) all_pass = .false.
 
    end subroutine test_derZ
+   !=====================================================================
+   ! Write of the results (all data are in X-pencil)
+   !=====================================================================
+   subroutine write_data()
+    
+      use decomp_2d_io
+
+      implicit none
+    
+      character(len=*), parameter :: io_name = "grad-io"
+#ifndef ADIOS2
+      logical :: dir_exists
+#endif
+      
+      !$acc update self (phi1)
+      !$acc update self (dphiX)
+      !$acc update self (dphiY)
+      !$acc update self (dphiZ)
+#ifndef ADIOS2
+      if (nrank == 0) then
+         inquire (file="out", exist=dir_exists)
+         if (.not. dir_exists) then
+            call execute_command_line("mkdir out 2> /dev/null")
+         end if
+      end if
+#endif
+
+      ! Standard I/O pattern - file per field
+#ifdef ADIOS2
+      call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
+      call decomp_2d_start_io(io_name, "out")
+#endif
+      call decomp_2d_write_one(1, phi1, 'out', 'phi1.dat', 0, io_name)
+      call decomp_2d_write_one(1, dphiX, 'out', 'dphiX.dat', 0, io_name)
+      call decomp_2d_write_one(1, dphiY, 'out', 'dphiY.dat', 0, io_name)
+      call decomp_2d_write_one(1, dphiZ, 'out', 'dphiZ.dat', 0, io_name)
+#ifdef ADIOS2
+      call decomp_2d_end_io(io_name, "out")
+      call decomp_2d_close_io(io_name, "out")
+#else
+      call write_xdmf()
+#endif
+
+   end subroutine write_data
+   !=====================================================================
+   ! Write of the xdmf file to visualise in paraview
+   !=====================================================================
+   subroutine write_xdmf()
+      ! This subroutine is based on the xdmf writers in Xcompact3d.
+      ! Copyright (c) 2012-2022, Xcompact3d
+      ! SPDX-License-Identifier: BSD 3-Clause
+
+      integer :: ioxdmf
+
+      character(len=:), allocatable :: fmt
+
+      integer :: precision
+
+      integer :: varctr
+      character(len=16) :: filename
+      character(len=5) :: varname
+      if (nrank == 0) then
+         OPEN (newunit=ioxdmf, file="./out.xdmf")
+
+         write (ioxdmf, '(A22)') '<?xml version="1.0" ?>'
+         write (ioxdmf, *) '<!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>'
+         write (ioxdmf, *) '<Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">'
+         write (ioxdmf, *) '<Domain>'
+
+         write (ioxdmf, '(A)') '    <Topology name="topo" TopologyType="3DCoRectMesh"'
+         fmt = "(A, I0, A, I0, A, I0, A)"
+         write (ioxdmf, fmt) '        Dimensions="', nz, " ", ny, " ", nx, '">'
+         write (ioxdmf, '(A)') '    </Topology>'
+
+         write (ioxdmf, *) '    <Geometry name="geo" Type="ORIGIN_DXDYDZ">'
+         write (ioxdmf, *) '        <!-- Origin -->'
+         write (ioxdmf, *) '        <DataItem Format="XML" Dimensions="3">'
+         write (ioxdmf, *) '          0.0 0.0 0.0'
+         write (ioxdmf, *) '        </DataItem>'
+         write (ioxdmf, *) '        <!-- DxDyDz -->'
+         write (ioxdmf, *) '        <DataItem Format="XML" Dimensions="3">'
+         if (mytype == kind(0.0d0)) then
+            fmt = "(A, E24.17, A, E24.17, A, E24.17)"
+         else
+            fmt = "(A, E16.9, A, E16.9, A, E16.9)"
+         end if
+         write (ioxdmf, fmt) '        ', 1.0_mytype, " ", 1.0_mytype, " ", 1.0_mytype
+         write (ioxdmf, *) '        </DataItem>'
+         write (ioxdmf, *) '    </Geometry>'
+
+         write (ioxdmf, *) '   <Grid Name="1" GridType="Uniform">'
+         write (ioxdmf, *) '       <Topology Reference="/Xdmf/Domain/Topology[1]"/>'
+         write (ioxdmf, *) '       <Geometry Reference="/Xdmf/Domain/Geometry[1]"/>'
+         do varctr = 1, 4
+            select case (varctr)
+              case (1)
+                write (varname, "(A)") "phi1"
+                write (filename, '(A)') "./out/phi1.dat"
+              case (2)
+                write (varname, "(A)") "dphiX"
+                write (filename, '(A)') "./out/dphiX.dat"
+              case (3)
+                write (varname, "(A)") "dphiY"
+                write (filename, '(A)') "./out/dphiY.dat"
+              case (4)
+                write (varname, "(A)") "dphiZ"
+                write (filename, '(A)') "./out/dphiZ.dat"
+            end select
+            write (ioxdmf, *) '       <Attribute Name="'//trim(varname)//'" Center="Node">'
+#ifndef ADIOS2
+            write (ioxdmf, *) '          <DataItem Format="Binary"'
+#else
+            write (ioxdmf, *) '          <DataItem Format="HDF"'
+#endif
+
+#ifdef DOUBLE_PREC
+            print *, "Double precision build"
+#ifdef SAVE_SINGLE
+            if (output2D == 0) then
+               precision = 4
+            else
+               precision = 8
+            end if
+#else
+            precision = 8
+#endif
+#else
+            precision = 4
+#endif
+            write (ioxdmf, "(A,I0,A)") '            DataType="Float" Precision="', precision, '" Endian="little" Seek="0"'
+
+            fmt = "(A, I0, A, I0, A, I0, A)"
+            write (ioxdmf, fmt) '            Dimensions="', nz, " ", ny, " ", nx, '">'
+
+            write (ioxdmf, *) '              '//trim(filename)
+
+            write (ioxdmf, *) '           </DataItem>'
+            write (ioxdmf, *) '        </Attribute>'
+         end do
+         write (ioxdmf, '(/)')
+         write (ioxdmf, *) '    </Grid>'
+         write (ioxdmf, *) '</Domain>'
+         write (ioxdmf, '(A7)') '</Xdmf>'
+         close (ioxdmf)
+      end if
 
+   end subroutine write_xdmf
 end program grad3d

From ee4c04369f1dd5e7ed6386ded722b39991783459 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 7 Nov 2023 08:14:54 +0000
Subject: [PATCH 373/436] Remove unused variable

---
 examples/grad3d/grad3d.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
index a65b6693..8e4a9607 100644
--- a/examples/grad3d/grad3d.f90
+++ b/examples/grad3d/grad3d.f90
@@ -25,7 +25,7 @@ program grad3d
    integer :: nranks_tot
    integer :: nargin, arg, FNLength, status, DecInd
    integer :: ierror
-   logical :: passing, all_pass
+   logical :: all_pass
    character(len=80) :: InputFN
 
    real(mytype), parameter :: lx = 1.0_mytype

From cbd856c6728b0cded5bccaad2f53f9bfa69ef658 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 7 Nov 2023 09:09:53 +0000
Subject: [PATCH 374/436] Typos correction for grad3d example

---
 examples/grad3d/README.md  |  2 +-
 examples/grad3d/grad3d.f90 | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/examples/grad3d/README.md b/examples/grad3d/README.md
index d7f89ac8..ebfbbb39 100644
--- a/examples/grad3d/README.md
+++ b/examples/grad3d/README.md
@@ -1,4 +1,4 @@
-# Halo test
+# Gradient example
 
 List of the tests:
 - [grad3d](grad3d.f90): Example to compute the gradient of a field. 
diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
index 8e4a9607..be7ad168 100644
--- a/examples/grad3d/grad3d.f90
+++ b/examples/grad3d/grad3d.f90
@@ -134,7 +134,7 @@ subroutine allocate_var()
 
       logical, parameter :: glob = .false.
 
-      ! initialise u,v,w with random numbers in X-pencil
+      ! Allocate main variables in X-pencil
       call alloc_x(phi1, glob)
       call alloc_x(dphiX, glob)
       call alloc_x(dphiY, glob)
@@ -313,7 +313,7 @@ subroutine derz(df,ff,delta,nx,ny,nz)
 
    end subroutine derz 
    !=====================================================================
-   ! Calculate divergence using halo-cell exchange (data in X-pencil)
+   ! Test derivatives against analytical solution (data in X-pencil)
    !=====================================================================
    subroutine test_derX(df)
 
@@ -339,7 +339,7 @@ subroutine test_derX(df)
       xs2 = xstart(2)
       xs3 = xstart(3)
 
-      ! Initial velocity
+      ! Compute the error against analytical solution
       !$acc parallel loop default(present) reduction(+:error)
       do k=1,xe3
         do j=1,xe2
@@ -367,7 +367,7 @@ subroutine test_derX(df)
    end subroutine test_derX
 
    !=====================================================================
-   ! Calculate divergence using halo-cell exchange (data in Y-pencil)
+   ! Test derivatives against analytical solution (data in Y-pencil)
    !=====================================================================
    subroutine test_derY(df)
 
@@ -393,7 +393,7 @@ subroutine test_derY(df)
       xs2 = xstart(2)
       xs3 = xstart(3)
 
-      ! Initial velocity
+      ! Compute the error against analytical solution
       !$acc parallel loop default(present) reduction(+:error)
       do k=1,xe3
         do j=1,xe2
@@ -422,7 +422,7 @@ subroutine test_derY(df)
    end subroutine test_derY
 
    !=====================================================================
-   ! Calculate divergence using halo-cell exchange (data in Z-pencil)
+   ! Test derivatives against analytical solution (data in Z-pencil)
    !=====================================================================
    subroutine test_derZ(df)
 
@@ -448,7 +448,7 @@ subroutine test_derZ(df)
       xs2 = xstart(2)
       xs3 = xstart(3)
 
-      ! Initial velocity
+      ! Compute the error against analytical solution
       !$acc parallel loop default(present) reduction(+:error)
       do k=1,xe3
         do j=1,xe2

From cf09575d37052128af845525f9a87ef55ed83636 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 7 Nov 2023 09:16:41 +0000
Subject: [PATCH 375/436] Correct typo

---
 examples/grad3d/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/grad3d/README.md b/examples/grad3d/README.md
index ebfbbb39..470c3057 100644
--- a/examples/grad3d/README.md
+++ b/examples/grad3d/README.md
@@ -6,7 +6,7 @@ List of the tests:
 This example demonstrates the use 2DECOMP&FFT library to compute the gradient
 of a field using an explicit second order finite difference scheme. 
 The purpose is to show how to use the transpose operations to allow explicit calculation
-of the gradient in all 3 directions. The results are written on a file and the function 
+of the gradient in all 3 directions. The results are written to a file and the function 
 is periodic over the interval [0-1] 
 
 What to input: The program takes max 5 inputs as: 

From 6ee1748cef4bfdc74ddda2ac5baa43cedfa25f34 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 7 Nov 2023 09:42:23 +0000
Subject: [PATCH 376/436] Correct analytical solution adding domain lenght

---
 examples/grad3d/grad3d.f90 | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
index be7ad168..f9d060f1 100644
--- a/examples/grad3d/grad3d.f90
+++ b/examples/grad3d/grad3d.f90
@@ -347,7 +347,8 @@ subroutine test_derX(df)
              z = (k + xs3 - 2) * dz
              y = (j + xs2 - 2) * dy
              x = (i + xs1 - 2) * dx
-             dphi = -2._mytype * twopi * sin(twopi * (x / lx)) * cos(twopi * (y / ly)) * sin(twopi * (z / lz))
+             dphi = -2._mytype * (twopi / lx) &
+                     * sin(twopi * (x / lx)) * cos(twopi * (y / ly)) * sin(twopi * (z / lz))
              dphi2 = dphi2 + dphi * dphi
              dphi_num = df(i,j,k)
              error = error + (dphi-dphi_num)*(dphi-dphi_num)
@@ -401,7 +402,8 @@ subroutine test_derY(df)
              z = (k + xs3 - 2) * dz
              y = (j + xs2 - 2) * dy
              x = (i + xs1 - 2) * dx
-             dphi = -2._mytype * twopi * cos(twopi * (x / lx)) * sin(twopi * (y / ly)) * sin(twopi * (z / lz))
+             dphi = -2._mytype * (twopi / ly) &
+                     * cos(twopi * (x / lx)) * sin(twopi * (y / ly)) * sin(twopi * (z / lz))
              dphi2 = dphi2 + dphi * dphi
              dphi_num = df(i,j,k)
              error = error + (dphi-dphi_num)*(dphi-dphi_num)
@@ -456,7 +458,8 @@ subroutine test_derZ(df)
              z = (k + xs3 - 2) * dz
              y = (j + xs2 - 2) * dy
              x = (i + xs1 - 2) * dx
-             dphi = 2._mytype * twopi * cos(twopi * (x / lx)) * cos(twopi * (y / ly)) * cos(twopi * (z / lz))
+             dphi = 2._mytype * (twopi / lz) &
+                    * cos(twopi * (x / lx)) * cos(twopi * (y / ly)) * cos(twopi * (z / lz))
              dphi2 = dphi2 + dphi * dphi
              dphi_num = df(i,j,k)
              error = error + (dphi-dphi_num)*(dphi-dphi_num)

From b38f9f1ea8d59402e2735c28c09c4b843f50f190 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 7 Nov 2023 09:18:40 +0000
Subject: [PATCH 377/436] Update adios2_config.xml for grad3d test

Remove unused io handles and match io handle name with code
---
 examples/grad3d/adios2_config.xml | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/examples/grad3d/adios2_config.xml b/examples/grad3d/adios2_config.xml
index 92ece049..ffbcb2db 100644
--- a/examples/grad3d/adios2_config.xml
+++ b/examples/grad3d/adios2_config.xml
@@ -1,14 +1,6 @@
 <?xml version="1.0"?>
 <adios-config>
-  <io name="test-io">
-    <engine type="BP4">
-    </engine>
-    <transport type="File">
-      <parameter key="Library" value="fstream"/>
-      <parameter key="ProfileUnits" value="Milliseconds"/>
-    </transport>
-  </io>
-  <io name="restart-io">
+  <io name="grad-io">
     <engine type="BP4">
     </engine>
     <transport type="File">

From 02f2caa869851b4a54a20604ad1be401a7822154 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 7 Nov 2023 09:46:00 +0000
Subject: [PATCH 378/436] Add io initialisation to grad3d

---
 examples/grad3d/CMakeLists.txt | 1 +
 examples/grad3d/grad3d.f90     | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/examples/grad3d/CMakeLists.txt b/examples/grad3d/CMakeLists.txt
index 3cdd6a61..f3cefeac 100644
--- a/examples/grad3d/CMakeLists.txt
+++ b/examples/grad3d/CMakeLists.txt
@@ -8,6 +8,7 @@ target_link_libraries(grad3d PRIVATE decomp2d)
 set(run_dir "${test_dir}/grad3d")
 message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
+file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
   add_test(NAME grad3d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:grad3d> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})
diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
index f9d060f1..548c2593 100644
--- a/examples/grad3d/grad3d.f90
+++ b/examples/grad3d/grad3d.f90
@@ -505,6 +505,14 @@ subroutine write_data()
       end if
 #endif
 
+      call decomp_2d_io_init()
+      call decomp_2d_init_io(io_name)
+
+      call decomp_2d_register_variable(io_name, "phi1.dat", 1, 0, 0, mytype)
+      call decomp_2d_register_variable(io_name, "dphiX.dat", 1, 0, 0, mytype)
+      call decomp_2d_register_variable(io_name, "dphiY.dat", 1, 0, 0, mytype)
+      call decomp_2d_register_variable(io_name, "dphiZ.dat", 1, 0, 0, mytype)
+      
       ! Standard I/O pattern - file per field
 #ifdef ADIOS2
       call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)

From f7fde73716ae629e897da8a603aa6fea833d3df1 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 7 Nov 2023 09:53:21 +0000
Subject: [PATCH 379/436] Correct typo

---
 examples/grad3d/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/grad3d/CMakeLists.txt b/examples/grad3d/CMakeLists.txt
index f3cefeac..98a09de5 100644
--- a/examples/grad3d/CMakeLists.txt
+++ b/examples/grad3d/CMakeLists.txt
@@ -6,9 +6,9 @@ target_link_libraries(grad3d PRIVATE decomp2d)
 
 # Run the test(s)
 set(run_dir "${test_dir}/grad3d")
+file (COPY "${CMAKE_SOURCE_DIR}/examples/grad3d/adios2_config.xml" DESTINATION ${run_dir})
 message(STATUS "Example dir ${run_dir}")
 file(MAKE_DIRECTORY ${run_dir})
-file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir})
 if (BUILD_TARGET MATCHES "gpu")
   file(COPY bind.sh DESTINATION ${run_dir})
   add_test(NAME grad3d COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $<TARGET_FILE:grad3d> ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir})

From 549761c8994a818db7dfb2af90acf18c6fff2072 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 7 Nov 2023 10:01:00 +0000
Subject: [PATCH 380/436] Update HOWTO.md

Add documentation on I/O initialisation
---
 HOWTO.md | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/HOWTO.md b/HOWTO.md
index 99eff2c6..2f188e05 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -90,6 +90,9 @@ end of the step the data stored in ``var`` may not have been written. Overwritin
 when used as a temporary variable, would cause the output data to become corrupted. In such situations
 ``decomp_2d_write_one`` accepts an optional argument ``opt_deferred_writes`` (default ``.true.``) which
 when set to ``.false.`` causes the data to be flushed immediately.
+The last argument ``io_name`` is a string used to group I/O operations together, and for the ADIOS2 backend
+allows for the runtime control of I/O through the file ``adios2_config.xml``, there must be a matching IO
+handle to specify the I/O engine - see the examples under ``examples/io_test/`` for how this works.
 
 There are two ways of writing multiple variables to a single file which may
 be used for check-pointing purposes, for example. The newer interface is described first and allows
@@ -112,6 +115,26 @@ where ``fh`` is a MPI-IO file handle provided by the application (file opened us
 Y-pencil and 3 for Z-pencil); ``disp`` (meaning displacement) is a variable of ``kind MPI_OFFSET_KIND``
 and of ``intent INOUT``. 
 Detailed information about the IO API are available [here](https://2decomp-fft.github.io/pages/api_io.html) 
+#### Initialising the IO module
+Before you can perform I/O you must initialise the IO module by calling
+```
+call decomp_2d_io_init()
+```
+somewhere near the start of the program (and before beginning any I/O operations).
+The IO module supports multiple handles (see above discussion of ``io_name``), these are initialised by calling
+```
+call decomp_2d_init_io(io_name)
+...
+```
+#### Registering variables for ADIOS2
+The ADIOS2 backend needs information about the variables it is going to write, during initialisation of I/O operations
+each variable must be registered by calling
+```
+call decomp_2d_register_variable(io_name,filename,ipencil,icoarse,iplanes,kind) 
+```
+where the arguments are described as above.
+The argument ``iplanes`` is used to declare writing planes from a field - pass ``0`` for 3-D field output and the ``kind``
+argument is the argument used when declaring the variable, e.g. ``real(kind)``.
 ### Use of the halo exchange
 The halo-cell support API provides data structures and nearest-neighbour communication routines 
 that support explicit message passing between neighbouring pencils. 

From 413f4143454763a08af3f097171430ba3b58b871 Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:24:36 +0000
Subject: [PATCH 381/436] Fix Comment

---
 examples/grad3d/grad3d.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
index 548c2593..8b5ad774 100644
--- a/examples/grad3d/grad3d.f90
+++ b/examples/grad3d/grad3d.f90
@@ -171,7 +171,7 @@ subroutine init_phi()
       xs2 = xstart(2)
       xs3 = xstart(3)
 
-      ! Initial velocity
+      ! Scalar field
       !$acc kernels default(present)
       do concurrent (k=1:xe3, j=1:xe2, i=1:xe1)
          z = (k + xs3 - 2) * dz

From b699dc861b3903079eb9749620f7ad7bb38ef1a3 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Tue, 7 Nov 2023 20:26:27 +0000
Subject: [PATCH 382/436] Add output2D definition for xdmf output in grad3d
 example

---
 examples/grad3d/grad3d.f90 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
index 8b5ad774..23c44f6e 100644
--- a/examples/grad3d/grad3d.f90
+++ b/examples/grad3d/grad3d.f90
@@ -543,6 +543,7 @@ subroutine write_xdmf()
       character(len=:), allocatable :: fmt
 
       integer :: precision
+      integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D)
 
       integer :: varctr
       character(len=16) :: filename

From e6a75085c283e46f1ed7f77d853735f61a551131 Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 8 Nov 2023 14:23:01 +0000
Subject: [PATCH 383/436] Change intent for gradient in test routines  under
 the grad3d example

---
 examples/grad3d/grad3d.f90 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
index 23c44f6e..109ea7ad 100644
--- a/examples/grad3d/grad3d.f90
+++ b/examples/grad3d/grad3d.f90
@@ -319,7 +319,7 @@ subroutine test_derX(df)
 
       implicit none
       ! Arguments
-      real(mytype), intent(out), dimension(xsize(1),xsize(2),xsize(3)) :: df
+      real(mytype), intent(in), dimension(xsize(1),xsize(2),xsize(3)) :: df
 
       integer :: i, j, k
       real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
@@ -374,7 +374,7 @@ subroutine test_derY(df)
 
       implicit none
       ! Arguments
-      real(mytype), intent(out), dimension(xsize(1),xsize(2),xsize(3)) :: df
+      real(mytype), intent(in), dimension(xsize(1),xsize(2),xsize(3)) :: df
 
       integer :: i, j, k
       real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
@@ -430,7 +430,7 @@ subroutine test_derZ(df)
 
       implicit none
       ! Arguments
-      real(mytype), intent(out), dimension(xsize(1),xsize(2),xsize(3)) :: df
+      real(mytype), intent(in), dimension(xsize(1),xsize(2),xsize(3)) :: df
 
       integer :: i, j, k
       real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)

From a4a416821e74f18776e56d4ea85d77c01515d38c Mon Sep 17 00:00:00 2001
From: Stefano Rolfo <stefano.rolfo@stfc.ac.uk>
Date: Wed, 8 Nov 2023 14:55:23 +0000
Subject: [PATCH 384/436] Add Grad3d test to list of example and perform
 fprettify

---
 examples/README.md         |   1 +
 examples/grad3d/grad3d.f90 | 267 ++++++++++++++++++-------------------
 2 files changed, 134 insertions(+), 134 deletions(-)

diff --git a/examples/README.md b/examples/README.md
index bb6ef31b..b9ae57ce 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -6,5 +6,6 @@
 - [Test FFT from Z pencil](fft_physical_z) - various tests for the FFT starting from the ``Z`` direction 
 - [Test HALO exchange](halo_test)          - to test the halo-cell exchange capability
 - [Test I/O features](io_test)             - various tests for the I/O module
+- [Example gradient of a scalar](grad3d)   - example of how to compute the gradient of a scalar field
 
 Refer to the README files for each example for details.
diff --git a/examples/grad3d/grad3d.f90 b/examples/grad3d/grad3d.f90
index 109ea7ad..49dcf655 100644
--- a/examples/grad3d/grad3d.f90
+++ b/examples/grad3d/grad3d.f90
@@ -33,13 +33,12 @@ program grad3d
    real(mytype), parameter :: lz = 1.0_mytype
 
    real(mytype) :: dx, dy, dz
-   real(mytype) :: error_ref 
+   real(mytype) :: error_ref
 
    real(mytype), allocatable, dimension(:, :, :) :: phi1, phi2, phi3
    real(mytype), allocatable, dimension(:, :, :) :: dphiX, dphiY, dphiz
    real(mytype), allocatable, dimension(:, :, :) :: wk2, wk3
 
-
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
    ! This operation is also done as part of decomp_2d_init
@@ -87,13 +86,13 @@ program grad3d
    end if
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-   dx = lx/real(nx,mytype)
-   dy = ly/real(ny,mytype)
-   dz = lz/real(nz,mytype)
-   
+   dx = lx / real(nx, mytype)
+   dy = ly / real(ny, mytype)
+   dz = lz / real(nz, mytype)
+
    if (nrank == 0) then
       write (*, *) '-----------------------------------------------'
-      write (*, *) "Mesh Resolution ", nx , ny, nz 
+      write (*, *) "Mesh Resolution ", nx, ny, nz
       write (*, *) '-----------------------------------------------'
    end if
 
@@ -159,7 +158,7 @@ subroutine init_phi()
       implicit none
 
       integer :: i, j, k
-      real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
+      real(mytype), parameter :: twopi = 2._mytype * acos(-1._mytype)
       real(mytype) :: x, y, z
       integer :: xe1, xe2, xe3
       integer :: xs1, xs2, xs3
@@ -173,12 +172,12 @@ subroutine init_phi()
 
       ! Scalar field
       !$acc kernels default(present)
-      do concurrent (k=1:xe3, j=1:xe2, i=1:xe1)
+      do concurrent(k=1:xe3, j=1:xe2, i=1:xe1)
          z = (k + xs3 - 2) * dz
          y = (j + xs2 - 2) * dy
          x = (i + xs1 - 2) * dx
          phi1(i, j, k) = -2._mytype * cos(twopi * (x / lx)) * cos(twopi * (y / ly)) * sin(twopi * (z / lz))
-      enddo
+      end do
       !$acc end kernels
 
    end subroutine init_phi
@@ -202,116 +201,116 @@ subroutine compute_grad()
       implicit none
 
       ! Compute X derivative
-      call derx(dphiX,phi1,dx,xsize(1),xsize(2),xsize(3))
+      call derx(dphiX, phi1, dx, xsize(1), xsize(2), xsize(3))
 
       ! Compute Y derivative
-      call transpose_x_to_y(phi1,phi2)
-      call dery(wk2,phi2,dy,ysize(1),ysize(2),ysize(3))
-      call transpose_y_to_x(wk2,dphiY)
+      call transpose_x_to_y(phi1, phi2)
+      call dery(wk2, phi2, dy, ysize(1), ysize(2), ysize(3))
+      call transpose_y_to_x(wk2, dphiY)
 
       ! Compute Z derivative
-      call transpose_y_to_z(phi2,phi3)
-      call derz(wk3,phi3,dz,zsize(1),zsize(2),zsize(3))
-      call transpose_z_to_y(wk3,wk2)
-      call transpose_y_to_x(wk2,dphiZ)
+      call transpose_y_to_z(phi2, phi3)
+      call derz(wk3, phi3, dz, zsize(1), zsize(2), zsize(3))
+      call transpose_z_to_y(wk3, wk2)
+      call transpose_y_to_x(wk2, dphiZ)
 
    end subroutine compute_grad
    !=====================================================================
    ! Calculate gradient in X (data in X-pencil)
    !=====================================================================
-   subroutine derx(df,ff,delta,nx,ny,nz)
+   subroutine derx(df, ff, delta, nx, ny, nz)
 
       implicit none
 
       ! Arguments
       integer, intent(in) :: nx, ny, nz
       real(mytype), intent(in) :: delta
-      real(mytype), intent(out), dimension(nx,ny,nz) :: df
-      real(mytype), intent(in), dimension(nx,ny,nz) :: ff
+      real(mytype), intent(out), dimension(nx, ny, nz) :: df
+      real(mytype), intent(in), dimension(nx, ny, nz) :: ff
 
       ! Local variables
       integer :: i, j, k
       real(mytype) :: coeff = 0.5_mytype
 
-      coeff = coeff/delta
+      coeff = coeff / delta
 
       !$acc kernels default(present)
-      do concurrent (k=1:nz, j=1:ny)
-         df(1,j,k) = coeff*(ff(2,j,k)-ff(nx,j,k)) 
-         do concurrent (i=2:nx-1)
-            df(i,j,k) = coeff*(ff(i+1,j,k)-ff(i-1,j,k)) 
-         enddo
-         df(nx,j,k) = coeff*(ff(1,j,k)-ff(nx-1,j,k))
-      enddo
+      do concurrent(k=1:nz, j=1:ny)
+         df(1, j, k) = coeff * (ff(2, j, k) - ff(nx, j, k))
+         do concurrent(i=2:nx - 1)
+            df(i, j, k) = coeff * (ff(i + 1, j, k) - ff(i - 1, j, k))
+         end do
+         df(nx, j, k) = coeff * (ff(1, j, k) - ff(nx - 1, j, k))
+      end do
       !$acc end kernels
 
-   end subroutine derx 
+   end subroutine derx
    !=====================================================================
    ! Calculate gradient in Y (data in Y-pencil)
    !=====================================================================
-   subroutine dery(df,ff,delta,nx,ny,nz)
+   subroutine dery(df, ff, delta, nx, ny, nz)
 
       implicit none
 
       ! Arguments
       integer, intent(in) :: nx, ny, nz
       real(mytype), intent(in) :: delta
-      real(mytype), intent(out), dimension(nx,ny,nz) :: df
-      real(mytype), intent(in), dimension(nx,ny,nz) :: ff
+      real(mytype), intent(out), dimension(nx, ny, nz) :: df
+      real(mytype), intent(in), dimension(nx, ny, nz) :: ff
 
       ! Local variables
       integer :: i, j, k
       real(mytype) :: coeff = 0.5_mytype
 
-      coeff = coeff/delta
+      coeff = coeff / delta
 
       !$acc kernels default(present)
-      do concurrent (k=1:nz)
-        do concurrent (i=1:nx)
-           df(i,1,k) = coeff*(ff(i,2,k)-ff(i,ny,k))
-        enddo
-        do concurrent (j=2:ny-1, i=1:nx)
-           df(i,j,k) = coeff*(ff(i,j+1,k)-ff(i,j-1,k))
-        enddo
-        do concurrent (i=1:nx)
-           df(i,ny,k) = coeff*(ff(i,1,k)-ff(i,ny-1,k)) 
-        enddo
-      enddo
+      do concurrent(k=1:nz)
+         do concurrent(i=1:nx)
+            df(i, 1, k) = coeff * (ff(i, 2, k) - ff(i, ny, k))
+         end do
+         do concurrent(j=2:ny - 1, i=1:nx)
+            df(i, j, k) = coeff * (ff(i, j + 1, k) - ff(i, j - 1, k))
+         end do
+         do concurrent(i=1:nx)
+            df(i, ny, k) = coeff * (ff(i, 1, k) - ff(i, ny - 1, k))
+         end do
+      end do
       !$acc end kernels
 
-   end subroutine dery 
+   end subroutine dery
    !=====================================================================
    ! Calculate gradient in Z (data in Z-pencil)
    !=====================================================================
-   subroutine derz(df,ff,delta,nx,ny,nz)
+   subroutine derz(df, ff, delta, nx, ny, nz)
 
       implicit none
 
       ! Arguments
       integer, intent(in) :: nx, ny, nz
       real(mytype), intent(in) :: delta
-      real(mytype), intent(out), dimension(nx,ny,nz) :: df
-      real(mytype), intent(in), dimension(nx,ny,nz) :: ff
+      real(mytype), intent(out), dimension(nx, ny, nz) :: df
+      real(mytype), intent(in), dimension(nx, ny, nz) :: ff
 
       ! Local variables
       integer :: i, j, k
       real(mytype) :: coeff = 0.5_mytype
-      
-      coeff = coeff/delta
+
+      coeff = coeff / delta
 
       !$acc kernels default(present)
-      do concurrent (j=1:ny, i=1:nx)
-         df(i,j,1) = coeff*(ff(i,j,2)-ff(i,j,nz  ))
-      enddo
-      do concurrent (k=2:nz-1, j=1:ny, i=1:nx)
-         df(i,j,k) = coeff*(ff(i,j,k+1)-ff(i,j,k-1))
-      enddo
-      do concurrent (j=1:ny, i=1:nx)
-         df(i,j,nz) = coeff*(ff(i,j,1)-ff(i,j,nz-1))
-      enddo
+      do concurrent(j=1:ny, i=1:nx)
+         df(i, j, 1) = coeff * (ff(i, j, 2) - ff(i, j, nz))
+      end do
+      do concurrent(k=2:nz - 1, j=1:ny, i=1:nx)
+         df(i, j, k) = coeff * (ff(i, j, k + 1) - ff(i, j, k - 1))
+      end do
+      do concurrent(j=1:ny, i=1:nx)
+         df(i, j, nz) = coeff * (ff(i, j, 1) - ff(i, j, nz - 1))
+      end do
       !$acc end kernels
 
-   end subroutine derz 
+   end subroutine derz
    !=====================================================================
    ! Test derivatives against analytical solution (data in X-pencil)
    !=====================================================================
@@ -319,10 +318,10 @@ subroutine test_derX(df)
 
       implicit none
       ! Arguments
-      real(mytype), intent(in), dimension(xsize(1),xsize(2),xsize(3)) :: df
+      real(mytype), intent(in), dimension(xsize(1), xsize(2), xsize(3)) :: df
 
       integer :: i, j, k
-      real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
+      real(mytype), parameter :: twopi = 2._mytype * acos(-1._mytype)
       real(mytype) :: x, y, z
       real(mytype) :: dphi, dphi_num
       real(mytype) :: error = 0._mytype
@@ -341,29 +340,29 @@ subroutine test_derX(df)
 
       ! Compute the error against analytical solution
       !$acc parallel loop default(present) reduction(+:error)
-      do k=1,xe3
-        do j=1,xe2
-          do i=1,xe1
-             z = (k + xs3 - 2) * dz
-             y = (j + xs2 - 2) * dy
-             x = (i + xs1 - 2) * dx
-             dphi = -2._mytype * (twopi / lx) &
-                     * sin(twopi * (x / lx)) * cos(twopi * (y / ly)) * sin(twopi * (z / lz))
-             dphi2 = dphi2 + dphi * dphi
-             dphi_num = df(i,j,k)
-             error = error + (dphi-dphi_num)*(dphi-dphi_num)
-          enddo
-        enddo
-      enddo
+      do k = 1, xe3
+         do j = 1, xe2
+            do i = 1, xe1
+               z = (k + xs3 - 2) * dz
+               y = (j + xs2 - 2) * dy
+               x = (i + xs1 - 2) * dx
+               dphi = -2._mytype * (twopi / lx) &
+                      * sin(twopi * (x / lx)) * cos(twopi * (y / ly)) * sin(twopi * (z / lz))
+               dphi2 = dphi2 + dphi * dphi
+               dphi_num = df(i, j, k)
+               error = error + (dphi - dphi_num) * (dphi - dphi_num)
+            end do
+         end do
+      end do
       !$acc end loop
       call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
       call MPI_ALLREDUCE(dphi2, sum_dphi2, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-      err_all = sqrt(err_all/sum_dphi2) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+      err_all = sqrt(err_all / sum_dphi2) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
       error_ref = err_all
-      
+
       if (nrank == 0) then
          write (*, *) 'DX error / mesh point: ', err_all
-      endif
+      end if
 
    end subroutine test_derX
 
@@ -374,10 +373,10 @@ subroutine test_derY(df)
 
       implicit none
       ! Arguments
-      real(mytype), intent(in), dimension(xsize(1),xsize(2),xsize(3)) :: df
+      real(mytype), intent(in), dimension(xsize(1), xsize(2), xsize(3)) :: df
 
       integer :: i, j, k
-      real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
+      real(mytype), parameter :: twopi = 2._mytype * acos(-1._mytype)
       real(mytype) :: x, y, z
       real(mytype) :: dphi, dphi_num
       real(mytype) :: error = 0._mytype
@@ -396,30 +395,30 @@ subroutine test_derY(df)
 
       ! Compute the error against analytical solution
       !$acc parallel loop default(present) reduction(+:error)
-      do k=1,xe3
-        do j=1,xe2
-          do i=1,xe1
-             z = (k + xs3 - 2) * dz
-             y = (j + xs2 - 2) * dy
-             x = (i + xs1 - 2) * dx
-             dphi = -2._mytype * (twopi / ly) &
-                     * cos(twopi * (x / lx)) * sin(twopi * (y / ly)) * sin(twopi * (z / lz))
-             dphi2 = dphi2 + dphi * dphi
-             dphi_num = df(i,j,k)
-             error = error + (dphi-dphi_num)*(dphi-dphi_num)
-          enddo
-        enddo
-      enddo
+      do k = 1, xe3
+         do j = 1, xe2
+            do i = 1, xe1
+               z = (k + xs3 - 2) * dz
+               y = (j + xs2 - 2) * dy
+               x = (i + xs1 - 2) * dx
+               dphi = -2._mytype * (twopi / ly) &
+                      * cos(twopi * (x / lx)) * sin(twopi * (y / ly)) * sin(twopi * (z / lz))
+               dphi2 = dphi2 + dphi * dphi
+               dphi_num = df(i, j, k)
+               error = error + (dphi - dphi_num) * (dphi - dphi_num)
+            end do
+         end do
+      end do
       !$acc end loop
       call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
       call MPI_ALLREDUCE(dphi2, sum_dphi2, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-      err_all = sqrt(err_all/sum_dphi2) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+      err_all = sqrt(err_all / sum_dphi2) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
       if (nrank == 0) then
          write (*, *) 'DY error / mesh point: ', err_all
-      endif
+      end if
 
-      if (abs(err_all-error_ref) > 1.0e-5_mytype) all_pass = .false.
+      if (abs(err_all - error_ref) > 1.0e-5_mytype) all_pass = .false.
 
    end subroutine test_derY
 
@@ -430,10 +429,10 @@ subroutine test_derZ(df)
 
       implicit none
       ! Arguments
-      real(mytype), intent(in), dimension(xsize(1),xsize(2),xsize(3)) :: df
+      real(mytype), intent(in), dimension(xsize(1), xsize(2), xsize(3)) :: df
 
       integer :: i, j, k
-      real(mytype), parameter :: twopi=2._mytype*acos(-1._mytype)
+      real(mytype), parameter :: twopi = 2._mytype * acos(-1._mytype)
       real(mytype) :: x, y, z
       real(mytype) :: dphi, dphi_num
       real(mytype) :: error = 0._mytype
@@ -452,46 +451,46 @@ subroutine test_derZ(df)
 
       ! Compute the error against analytical solution
       !$acc parallel loop default(present) reduction(+:error)
-      do k=1,xe3
-        do j=1,xe2
-          do i=1,xe1
-             z = (k + xs3 - 2) * dz
-             y = (j + xs2 - 2) * dy
-             x = (i + xs1 - 2) * dx
-             dphi = 2._mytype * (twopi / lz) &
-                    * cos(twopi * (x / lx)) * cos(twopi * (y / ly)) * cos(twopi * (z / lz))
-             dphi2 = dphi2 + dphi * dphi
-             dphi_num = df(i,j,k)
-             error = error + (dphi-dphi_num)*(dphi-dphi_num)
-          enddo
-        enddo
-      enddo
+      do k = 1, xe3
+         do j = 1, xe2
+            do i = 1, xe1
+               z = (k + xs3 - 2) * dz
+               y = (j + xs2 - 2) * dy
+               x = (i + xs1 - 2) * dx
+               dphi = 2._mytype * (twopi / lz) &
+                      * cos(twopi * (x / lx)) * cos(twopi * (y / ly)) * cos(twopi * (z / lz))
+               dphi2 = dphi2 + dphi * dphi
+               dphi_num = df(i, j, k)
+               error = error + (dphi - dphi_num) * (dphi - dphi_num)
+            end do
+         end do
+      end do
       !$acc end loop
       call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
       call MPI_ALLREDUCE(dphi2, sum_dphi2, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
-      err_all = sqrt(err_all/sum_dphi2) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
+      err_all = sqrt(err_all / sum_dphi2) / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
       if (nrank == 0) then
          write (*, *) 'DZ error / mesh point: ', err_all
-      endif
+      end if
 
-      if (abs(err_all-error_ref) > 1.0e-5_mytype) all_pass = .false.
+      if (abs(err_all - error_ref) > 1.0e-5_mytype) all_pass = .false.
 
    end subroutine test_derZ
    !=====================================================================
    ! Write of the results (all data are in X-pencil)
    !=====================================================================
    subroutine write_data()
-    
+
       use decomp_2d_io
 
       implicit none
-    
+
       character(len=*), parameter :: io_name = "grad-io"
 #ifndef ADIOS2
       logical :: dir_exists
 #endif
-      
+
       !$acc update self (phi1)
       !$acc update self (dphiX)
       !$acc update self (dphiY)
@@ -512,7 +511,7 @@ subroutine write_data()
       call decomp_2d_register_variable(io_name, "dphiX.dat", 1, 0, 0, mytype)
       call decomp_2d_register_variable(io_name, "dphiY.dat", 1, 0, 0, mytype)
       call decomp_2d_register_variable(io_name, "dphiZ.dat", 1, 0, 0, mytype)
-      
+
       ! Standard I/O pattern - file per field
 #ifdef ADIOS2
       call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode)
@@ -582,18 +581,18 @@ subroutine write_xdmf()
          write (ioxdmf, *) '       <Geometry Reference="/Xdmf/Domain/Geometry[1]"/>'
          do varctr = 1, 4
             select case (varctr)
-              case (1)
-                write (varname, "(A)") "phi1"
-                write (filename, '(A)') "./out/phi1.dat"
-              case (2)
-                write (varname, "(A)") "dphiX"
-                write (filename, '(A)') "./out/dphiX.dat"
-              case (3)
-                write (varname, "(A)") "dphiY"
-                write (filename, '(A)') "./out/dphiY.dat"
-              case (4)
-                write (varname, "(A)") "dphiZ"
-                write (filename, '(A)') "./out/dphiZ.dat"
+            case (1)
+               write (varname, "(A)") "phi1"
+               write (filename, '(A)') "./out/phi1.dat"
+            case (2)
+               write (varname, "(A)") "dphiX"
+               write (filename, '(A)') "./out/dphiX.dat"
+            case (3)
+               write (varname, "(A)") "dphiY"
+               write (filename, '(A)') "./out/dphiY.dat"
+            case (4)
+               write (varname, "(A)") "dphiZ"
+               write (filename, '(A)') "./out/dphiZ.dat"
             end select
             write (ioxdmf, *) '       <Attribute Name="'//trim(varname)//'" Center="Node">'
 #ifndef ADIOS2

From 1778965cf8460c8fa69ccfa101b0a51e8f4a2b5b Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Sun, 10 Dec 2023 12:38:26 +0000
Subject: [PATCH 385/436] Add reduction to openACC check tests

---
 examples/test2d/test2d.f90           | 22 +++++-----------------
 examples/test2d/timing2d_complex.f90 | 16 ++++------------
 examples/test2d/timing2d_real.f90    | 18 ++++--------------
 3 files changed, 13 insertions(+), 43 deletions(-)

diff --git a/examples/test2d/test2d.f90 b/examples/test2d/test2d.f90
index 9a2019e7..948330cc 100644
--- a/examples/test2d/test2d.f90
+++ b/examples/test2d/test2d.f90
@@ -82,18 +82,6 @@ program test2d
    end if
    call decomp_2d_init(nx, ny, nz, p_row, p_col)
 
-   !! ***** global data *****
-   !allocate(data1(nx,ny,nz))
-   !m = 1
-   !do k = 1, nz
-   !   do j = 1, ny
-   !      do i = 1, nx
-   !         data1(i, j, k) = float(m)
-   !         m = m + 1
-   !      end do
-   !   end do
-   !end do
-
    ! Fill the local index
    xst1 = xstart(1); xen1 = xend(1)
    xst2 = xstart(2); xen2 = xend(2)
@@ -115,7 +103,7 @@ program test2d
 
    !$acc data copy(u1,u2,u3)
    ! original x-pensil based data
-   !$acc parallel loop default(present) collapse(3)
+   !$acc parallel loop default(present) collapse(3) private(m)
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
@@ -157,7 +145,7 @@ program test2d
    ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
 
    ! also check the transposition this way
-   !$acc parallel loop default(present) collapse(3)
+   !$acc parallel loop default(present) collapse(3) private(m) reduction(.or.:error_flag)
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
@@ -187,7 +175,7 @@ program test2d
    ! call decomp_2d_write_one(3,u3,'u3.dat')
    ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
 
-   !$acc parallel loop default(present) collapse(3)
+   !$acc parallel loop default(present) collapse(3) private(m) reduction(.or.:error_flag)
    do k = zst3, zen3
       do j = zst2, zen2
          do i = zst1, zen1
@@ -206,7 +194,7 @@ program test2d
    call transpose_z_to_y(u3, u2)
    ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
-   !$acc parallel loop default(present) collapse(3)
+   !$acc parallel loop default(present) collapse(3) private(m) reduction(.or.:error_flag)
    do k = yst3, yen3
       do j = yst2, yen2
          do i = yst1, yen1
@@ -225,7 +213,7 @@ program test2d
    call transpose_y_to_x(u2, u1)
    ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
-   !$acc parallel loop default(present) collapse(3)
+   !$acc parallel loop default(present) collapse(3) private(m) reduction(.or.:error_flag)
    do k = xst3, xen3
       do j = xst2, xen2
          do i = xst1, xen1
diff --git a/examples/test2d/timing2d_complex.f90 b/examples/test2d/timing2d_complex.f90
index 62537883..4916fbcb 100644
--- a/examples/test2d/timing2d_complex.f90
+++ b/examples/test2d/timing2d_complex.f90
@@ -180,11 +180,8 @@ program timing2d_complex
       end if
 #endif
 
-      ! call decomp_2d_write_one(2,u2,'u2.dat')
-      ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
-
       ! also check the transposition this way
-      !$acc parallel loop default(present) private(m,cm)
+      !$acc parallel loop default(present) collapse(3) private(m,cm) reduction(.or.:error_flag)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
@@ -214,10 +211,7 @@ program timing2d_complex
       end if
 #endif
 
-      ! call decomp_2d_write_one(3,u3,'u3.dat')
-      ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
-
-      !$acc parallel loop default(present) private(m,cm)
+      !$acc parallel loop default(present) collapse(3) private(m,cm) reduction(.or.:error_flag)
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
@@ -237,9 +231,8 @@ program timing2d_complex
       t5 = MPI_WTIME()
       call transpose_z_to_y(u3, u2)
       t6 = t6 + MPI_WTIME() - t5
-      ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
-      !$acc parallel loop default(present) private(m,cm)
+      !$acc parallel loop default(present) collapse(3) private(m,cm) reduction(.or.:error_flag)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
@@ -259,9 +252,8 @@ program timing2d_complex
       t7 = MPI_WTIME()
       call transpose_y_to_x(u2, u1)
       t8 = t8 + MPI_WTIME() - t7
-      ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
-      !$acc parallel loop default(present) private(m,cm)
+      !$acc parallel loop default(present) collapse(3) private(m,cm) reduction(.or.:error_flag)
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1
diff --git a/examples/test2d/timing2d_real.f90 b/examples/test2d/timing2d_real.f90
index 257d6fef..adb8f1e4 100644
--- a/examples/test2d/timing2d_real.f90
+++ b/examples/test2d/timing2d_real.f90
@@ -144,8 +144,6 @@ program timing2d_real
    end if
 #endif
 
-   ! call decomp_2d_write_one(1,u1,'u1.dat')
-
    t1 = MPI_WTIME()
    call transpose_x_to_y(u1, u2)
    call transpose_y_to_z(u2, u3)
@@ -179,11 +177,8 @@ program timing2d_real
       end if
 #endif
 
-      ! call decomp_2d_write_one(2,u2,'u2.dat')
-      ! 'u1.dat' and 'u2.dat' should be identical byte-by-byte
-
       ! also check the transposition this way
-      !$acc parallel loop default(present) private(m)
+      !$acc parallel loop default(present) collapse(3) private(m) reduction(.or.:error_flag)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
@@ -212,10 +207,7 @@ program timing2d_real
       end if
 #endif
 
-      ! call decomp_2d_write_one(3,u3,'u3.dat')
-      ! 'u1.dat','u2.dat' and 'u3.dat' should be identical
-
-      !$acc parallel loop default(present) private(m)
+      !$acc parallel loop default(present) collapse(3) private(m) reduction(.or.:error_flag)
       do k = zst3, zen3
          do j = zst2, zen2
             do i = zst1, zen1
@@ -234,9 +226,8 @@ program timing2d_real
       t5 = MPI_WTIME()
       call transpose_z_to_y(u3, u2)
       t6 = t6 + MPI_WTIME() - t5
-      ! call decomp_2d_write_one(2,u2,'u2b.dat')
 
-      !$acc parallel loop default(present) private(m)
+      !$acc parallel loop default(present) collapse(3) private(m) reduction(.or.:error_flag)
       do k = yst3, yen3
          do j = yst2, yen2
             do i = yst1, yen1
@@ -255,9 +246,8 @@ program timing2d_real
       t7 = MPI_WTIME()
       call transpose_y_to_x(u2, u1)
       t8 = t8 + MPI_WTIME() - t7
-      ! call decomp_2d_write_one(1,u1,'u1b.dat')
 
-      !$acc parallel loop default(present) private(m)
+      !$acc parallel loop default(present) collapse(3) private(m) reduction(.or.:error_flag)
       do k = xst3, xen3
          do j = xst2, xen2
             do i = xst1, xen1

From 8840ea37ac3e3db3eab902905a0d787cebdce2a0 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Sun, 10 Dec 2023 12:45:26 +0000
Subject: [PATCH 386/436] Add cudaStreamSyncronize for NCCL transpose

---
 src/decomp_2d_nccl.f90 | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/decomp_2d_nccl.f90 b/src/decomp_2d_nccl.f90
index 01c64bc6..f63f0cba 100644
--- a/src/decomp_2d_nccl.f90
+++ b/src/decomp_2d_nccl.f90
@@ -98,10 +98,14 @@ subroutine decomp_2d_nccl_fin()
 
       implicit none
 
+      integer :: cuda_stat
       type(ncclResult) :: nccl_stat
 
       nccl_stat = ncclCommDestroy(nccl_comm_2decomp)
       if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclCommDestroy")
+      cuda_stat = cudaStreamDestroy(cuda_stream_2decomp)
+      if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamDestroy")
+      
 
    end subroutine decomp_2d_nccl_fin
    !
@@ -138,6 +142,8 @@ subroutine decomp_2d_nccl_send_recv_real_col(dst_d, &
       end do
       nccl_stat = ncclGroupEnd()
       if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+      cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)     
+      if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 
    end subroutine decomp_2d_nccl_send_recv_real_col
    !
@@ -233,6 +239,8 @@ subroutine decomp_2d_nccl_send_recv_real_row(dst_d, &
       end do
       nccl_stat = ncclGroupEnd()
       if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
+      cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)     
+      if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 
    end subroutine decomp_2d_nccl_send_recv_real_row
    !

From c76fc25b493003ee543f45e4ef45c94e418371e5 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 11 Dec 2023 16:32:30 +0000
Subject: [PATCH 387/436] Add test on precision for FFT examples

---
 examples/fft_physical_x/fft_c2c_x.f90  | 11 +++++++++++
 examples/fft_physical_x/fft_grid_x.f90 | 11 +++++++++++
 examples/fft_physical_x/fft_r2c_x.f90  | 13 ++++++++++++-
 examples/fft_physical_z/fft_c2c_z.f90  | 11 +++++++++++
 examples/fft_physical_z/fft_r2c_z.f90  | 11 +++++++++++
 5 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index 84a4d327..ef6f1b19 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -32,6 +32,12 @@ program fft_c2c_x
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
    double precision :: n1, flops, t1, t2, t3, t4
+#ifdef DOUBLE_PREC
+   real(mytype), parameter :: error_precision = 1.e-14_mytype
+#else
+   real(mytype), parameter :: error_precision = 1.e-6_mytype
+#endif
+   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -183,6 +189,11 @@ program fft_c2c_x
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
+   if (err_all > error_precision ) then
+      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
+      call decomp_2d_abort(1, "error for the FFT too large") 
+   endif 
+
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'Avg time (sec): ', t1, t3
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 0afe318f..cc3fcd03 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -32,6 +32,12 @@ program fft_physical_x
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
    double precision :: n1, flops, t1, t2, t3, t4
+#ifdef DOUBLE_PREC
+   real(mytype), parameter :: error_precision = 1.e-14_mytype
+#else
+   real(mytype), parameter :: error_precision = 1.e-6_mytype
+#endif
+   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -184,6 +190,11 @@ program fft_physical_x
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
+   if (err_all > error_precision ) then
+      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
+      call decomp_2d_abort(1, "error for the FFT too large") 
+   endif 
+
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'Avg time (sec): ', t1, t3
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 771d5158..b10efd87 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -32,7 +32,13 @@ program fft_r2c_x
    integer :: ierror, i, j, k, m
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
-   double precision :: t1, t2, t3, t4
+   double precision :: n1, flops, t1, t2, t3, t4
+#ifdef DOUBLE_PREC
+   real(mytype), parameter :: error_precision = 1.e-14_mytype
+#else
+   real(mytype), parameter :: error_precision = 1.e-6_mytype
+#endif
+   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -181,6 +187,11 @@ program fft_r2c_x
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
+   if (err_all > error_precision ) then
+      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
+      call decomp_2d_abort(1, "error for the FFT too large") 
+   endif 
+
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index c0a6612f..73efcdf7 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -32,6 +32,12 @@ program fft_c2c_z
    integer :: zst1, zst2, zst3
    integer :: zen1, zen2, zen3
    double precision :: n1, flops, t1, t2, t3, t4
+#ifdef DOUBLE_PREC
+   real(mytype), parameter :: error_precision = 1.e-14_mytype
+#else
+   real(mytype), parameter :: error_precision = 1.e-6_mytype
+#endif
+   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -184,6 +190,11 @@ program fft_c2c_z
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
+   if (err_all > error_precision ) then
+      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
+      call decomp_2d_abort(1, "error for the FFT too large") 
+   endif 
+
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'Avg time (sec): ', t1, t3
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index d9cae9d4..e704bdb4 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -33,6 +33,12 @@ program fft_r2c_z
    integer :: zst1, zst2, zst3
    integer :: zen1, zen2, zen3
    double precision :: t1, t2, t3, t4
+#ifdef DOUBLE_PREC
+   real(mytype), parameter :: error_precision = 1.e-14_mytype
+#else
+   real(mytype), parameter :: error_precision = 1.e-6_mytype
+#endif
+   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -182,6 +188,11 @@ program fft_r2c_z
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
+   if (err_all > error_precision ) then
+      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
+      call decomp_2d_abort(1, "error for the FFT too large") 
+   endif 
+
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
       write (*, *) 'time (sec): ', t1, t3

From 01c6862ebbb2be85ced17db35df34f2e76f59932 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 11 Dec 2023 16:47:01 +0000
Subject: [PATCH 388/436] Change precision to 1e-12 for FFT tests

---
 examples/fft_physical_x/fft_c2c_x.f90  | 2 +-
 examples/fft_physical_x/fft_grid_x.f90 | 2 +-
 examples/fft_physical_x/fft_r2c_x.f90  | 2 +-
 examples/fft_physical_z/fft_c2c_z.f90  | 2 +-
 examples/fft_physical_z/fft_r2c_z.f90  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index ef6f1b19..e4338c2a 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -33,7 +33,7 @@ program fft_c2c_x
    integer :: xen1, xen2, xen3
    double precision :: n1, flops, t1, t2, t3, t4
 #ifdef DOUBLE_PREC
-   real(mytype), parameter :: error_precision = 1.e-14_mytype
+   real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
    real(mytype), parameter :: error_precision = 1.e-6_mytype
 #endif
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index cc3fcd03..3a6f037d 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -33,7 +33,7 @@ program fft_physical_x
    integer :: xen1, xen2, xen3
    double precision :: n1, flops, t1, t2, t3, t4
 #ifdef DOUBLE_PREC
-   real(mytype), parameter :: error_precision = 1.e-14_mytype
+   real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
    real(mytype), parameter :: error_precision = 1.e-6_mytype
 #endif
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index b10efd87..f1227685 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -34,7 +34,7 @@ program fft_r2c_x
    integer :: xen1, xen2, xen3
    double precision :: n1, flops, t1, t2, t3, t4
 #ifdef DOUBLE_PREC
-   real(mytype), parameter :: error_precision = 1.e-14_mytype
+   real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
    real(mytype), parameter :: error_precision = 1.e-6_mytype
 #endif
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 73efcdf7..c86635ca 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -33,7 +33,7 @@ program fft_c2c_z
    integer :: zen1, zen2, zen3
    double precision :: n1, flops, t1, t2, t3, t4
 #ifdef DOUBLE_PREC
-   real(mytype), parameter :: error_precision = 1.e-14_mytype
+   real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
    real(mytype), parameter :: error_precision = 1.e-6_mytype
 #endif
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index e704bdb4..6156b8e1 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -34,7 +34,7 @@ program fft_r2c_z
    integer :: zen1, zen2, zen3
    double precision :: t1, t2, t3, t4
 #ifdef DOUBLE_PREC
-   real(mytype), parameter :: error_precision = 1.e-14_mytype
+   real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
    real(mytype), parameter :: error_precision = 1.e-6_mytype
 #endif

From f90949759a9467e154697466b087211072d36c94 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 11 Dec 2023 16:57:43 +0000
Subject: [PATCH 389/436] Remove unnessary variables added by mistake

---
 examples/fft_physical_x/fft_r2c_x.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index f1227685..e85a9452 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -32,7 +32,7 @@ program fft_r2c_x
    integer :: ierror, i, j, k, m
    integer :: xst1, xst2, xst3
    integer :: xen1, xen2, xen3
-   double precision :: n1, flops, t1, t2, t3, t4
+   double precision :: t1, t2, t3, t4
 #ifdef DOUBLE_PREC
    real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else

From f311c900e7afa4e88a9e77c3882596a12deed2eb Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Wed, 20 Dec 2023 12:50:10 +0000
Subject: [PATCH 390/436] Backport support for AOCC (#307)

---
 cmake/D2D_Compilers.cmake             | 3 +++
 cmake/compilers/D2D_flags_flang.cmake | 5 +++++
 src/fft_log.f90                       | 4 ++--
 3 files changed, 10 insertions(+), 2 deletions(-)
 create mode 100644 cmake/compilers/D2D_flags_flang.cmake

diff --git a/cmake/D2D_Compilers.cmake b/cmake/D2D_Compilers.cmake
index 42a68c30..101651fd 100644
--- a/cmake/D2D_Compilers.cmake
+++ b/cmake/D2D_Compilers.cmake
@@ -39,6 +39,9 @@ elseif (Fortran_COMPILER_NAME MATCHES "NVHPC")
 elseif (Fortran_COMPILER_NAME MATCHES "Fujitsu")
   message(STATUS "Setting Fujitsu fortran flags")
   include(D2D_flags_fujitsu)
+elseif (Fortran_COMPILER_NAME MATCHES "Flang")
+  message(STATUS "Setting Flang flags")
+  include(D2D_flags_flang)
 else (Fortran_COMPILER_NAME MATCHES "GNU")
   message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
   message ("Fortran compiler: " ${Fortran_COMPILER_NAME})
diff --git a/cmake/compilers/D2D_flags_flang.cmake b/cmake/compilers/D2D_flags_flang.cmake
new file mode 100644
index 00000000..e50be9b4
--- /dev/null
+++ b/cmake/compilers/D2D_flags_flang.cmake
@@ -0,0 +1,5 @@
+# Compilers Flags for AOCC
+set(D2D_FFLAGS "-cpp -g")
+set(D2D_FFLAGS_RELEASE "-O3")
+set(D2D_FFLAGS_DEBUG   "-g -O0 -DDEBUG")
+set(D2D_FFLAGS_DEV     "${D2D_FFLAGS_DEBUG}")
diff --git a/src/fft_log.f90 b/src/fft_log.f90
index 7dfbb45f..059bb258 100644
--- a/src/fft_log.f90
+++ b/src/fft_log.f90
@@ -40,8 +40,8 @@ module subroutine decomp_2d_fft_log(backend)
          if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
             write (io_unit, *) 'decomp_info object ph is a pointer to decomp_main'
          end if
-         call decomp_info_print(ph, io_unit, "ph")
-         call decomp_info_print(sp, io_unit, "sp")
+         call decomp_info_print(decomp_2d_fft_get_ph(), io_unit, "ph")
+         call decomp_info_print(decomp_2d_fft_get_sp(), io_unit, "sp")
          write (io_unit, *) ''
 #ifdef OVERWRITE
          if (D2D_FFT_BACKEND == D2D_FFT_BACKEND_GENERIC) then

From 5a38c00064880fd06c9c03983f99a985511a4190 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Thu, 21 Dec 2023 16:21:52 +0000
Subject: [PATCH 391/436] Backport replace profiler enum with integer
 parameters (#309)

---
 src/decomp_2d_constants.f90 | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/decomp_2d_constants.f90 b/src/decomp_2d_constants.f90
index c3cd274e..6023ee62 100644
--- a/src/decomp_2d_constants.f90
+++ b/src/decomp_2d_constants.f90
@@ -74,10 +74,8 @@ module decomp_2d_constants
    !    0 => no profiling, default
    !    1 => Caliper (https://github.com/LLNL/Caliper)
    !
-   enum, bind(c)
-      enumerator :: decomp_profiler_none = 0
-      enumerator :: decomp_profiler_caliper = 1
-   end enum
+   integer, parameter, public :: DECOMP_PROFILER_NONE = 0
+   integer, parameter, public :: DECOMP_PROFILER_CALIPER = 1
 
    !
    ! Supported FFT backends

From 417e4280268b9154d4ea5c7b4e28cddeba55b507 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 20 Feb 2024 15:04:18 +0000
Subject: [PATCH 392/436] Run fprettify

---
 src/alloc.f90       | 134 +++++++++++++--------------
 src/decomp_2d.f90   |   4 +-
 src/halo.f90        | 215 ++++++++++++++++++++++----------------------
 src/halo_comm_x.f90 | 124 ++++++++++++-------------
 src/halo_comm_y.f90 | 124 ++++++++++++-------------
 src/halo_comm_z.f90 | 132 +++++++++++++--------------
 src/halo_common.f90 |   6 +-
 7 files changed, 367 insertions(+), 372 deletions(-)

diff --git a/src/alloc.f90 b/src/alloc.f90
index b6ad9209..f48b94c2 100644
--- a/src/alloc.f90
+++ b/src/alloc.f90
@@ -18,7 +18,7 @@ subroutine alloc_x_real_short(var, opt_xlevel, opt_global)
 
      implicit none
 
-     real(mytype), allocatable, dimension(:,:,:) :: var
+     real(mytype), allocatable, dimension(:, :, :) :: var
      logical, intent(IN), optional :: opt_global
      integer, dimension(3), intent(IN), optional :: opt_xlevel
      integer, dimension(3) :: xlevel
@@ -60,15 +60,15 @@ subroutine alloc_x_real(var, decomp, opt_xlevel, opt_global)
      end if
 
      if (global) then
-        allocate(var(decomp%xst(1)-xlevel(1):decomp%xen(1)+xlevel(1), &
-             decomp%xst(2)-xlevel(2):decomp%xen(2)+xlevel(2), &
-             decomp%xst(3)-xlevel(3):decomp%xen(3)+xlevel(3)), &
-             stat=alloc_stat)
+        allocate (var(decomp%xst(1) - xlevel(1):decomp%xen(1) + xlevel(1), &
+                      decomp%xst(2) - xlevel(2):decomp%xen(2) + xlevel(2), &
+                      decomp%xst(3) - xlevel(3):decomp%xen(3) + xlevel(3)), &
+                  stat=alloc_stat)
      else
-        allocate(var(1-xlevel(1):decomp%xsz(1)+xlevel(1), &
-             1-xlevel(2):decomp%xsz(2)+xlevel(2), &
-             1-xlevel(3):decomp%xsz(3)+xlevel(3)), &
-             stat=alloc_stat)
+        allocate (var(1 - xlevel(1):decomp%xsz(1) + xlevel(1), &
+                      1 - xlevel(2):decomp%xsz(2) + xlevel(2), &
+                      1 - xlevel(3):decomp%xsz(3) + xlevel(3)), &
+                  stat=alloc_stat)
      end if
 
      if (alloc_stat /= 0) then
@@ -85,18 +85,18 @@ subroutine alloc_x_complex_short(var, opt_xlevel, opt_global)
 
      implicit none
 
-    complex(mytype), allocatable, dimension(:,:,:) :: var
-    logical, intent(IN), optional :: opt_global
-    integer, dimension(3), intent(IN), optional :: opt_xlevel
-    integer, dimension(3) :: xlevel
+     complex(mytype), allocatable, dimension(:, :, :) :: var
+     logical, intent(IN), optional :: opt_global
+     integer, dimension(3), intent(IN), optional :: opt_xlevel
+     integer, dimension(3) :: xlevel
 
-    if (present(opt_xlevel)) then
-       xlevel = opt_xlevel
-    else
-       xlevel = decomp_main%xlevel
-    end if
+     if (present(opt_xlevel)) then
+        xlevel = opt_xlevel
+     else
+        xlevel = decomp_main%xlevel
+     end if
 
-    call alloc_x(var, decomp_main, xlevel, opt_global)
+     call alloc_x(var, decomp_main, xlevel, opt_global)
 
   end subroutine alloc_x_complex_short
 
@@ -126,15 +126,15 @@ subroutine alloc_x_complex(var, decomp, opt_xlevel, opt_global)
      end if
 
      if (global) then
-        allocate(var(decomp%xst(1)-xlevel(1):decomp%xen(1)+xlevel(1), &
-             decomp%xst(2)-xlevel(2):decomp%xen(2)+xlevel(2), &
-             decomp%xst(3)-xlevel(3):decomp%xen(3)+xlevel(3)), &
-             stat=alloc_stat)
+        allocate (var(decomp%xst(1) - xlevel(1):decomp%xen(1) + xlevel(1), &
+                      decomp%xst(2) - xlevel(2):decomp%xen(2) + xlevel(2), &
+                      decomp%xst(3) - xlevel(3):decomp%xen(3) + xlevel(3)), &
+                  stat=alloc_stat)
      else
-        allocate(var(1-xlevel(1):decomp%xsz(1)+xlevel(1), &
-             1-xlevel(2):decomp%xsz(2)+xlevel(2), &
-             1-xlevel(3):decomp%xsz(3)+xlevel(3)), &
-             stat=alloc_stat)
+        allocate (var(1 - xlevel(1):decomp%xsz(1) + xlevel(1), &
+                      1 - xlevel(2):decomp%xsz(2) + xlevel(2), &
+                      1 - xlevel(3):decomp%xsz(3) + xlevel(3)), &
+                  stat=alloc_stat)
      end if
 
      if (alloc_stat /= 0) then
@@ -151,7 +151,7 @@ subroutine alloc_y_real_short(var, opt_ylevel, opt_global)
 
      implicit none
 
-     real(mytype), allocatable, dimension(:,:,:) :: var
+     real(mytype), allocatable, dimension(:, :, :) :: var
      logical, intent(IN), optional :: opt_global
      integer, dimension(3), intent(IN), optional :: opt_ylevel
      integer, dimension(3) :: ylevel
@@ -192,15 +192,15 @@ subroutine alloc_y_real(var, decomp, opt_ylevel, opt_global)
      end if
 
      if (global) then
-        allocate(var(decomp%yst(1)-ylevel(1):decomp%yen(1)+ylevel(1), &
-             decomp%yst(2)-ylevel(2):decomp%yen(2)+ylevel(2), &
-             decomp%yst(3)-ylevel(3):decomp%yen(3)+ylevel(3)), &
-             stat=alloc_stat)
+        allocate (var(decomp%yst(1) - ylevel(1):decomp%yen(1) + ylevel(1), &
+                      decomp%yst(2) - ylevel(2):decomp%yen(2) + ylevel(2), &
+                      decomp%yst(3) - ylevel(3):decomp%yen(3) + ylevel(3)), &
+                  stat=alloc_stat)
      else
-        allocate(var(1-ylevel(1):decomp%ysz(1)+ylevel(1), &
-             1-ylevel(2):decomp%ysz(2)+ylevel(2), &
-             1-ylevel(3):decomp%ysz(3)+ylevel(3)), &
-             stat=alloc_stat)
+        allocate (var(1 - ylevel(1):decomp%ysz(1) + ylevel(1), &
+                      1 - ylevel(2):decomp%ysz(2) + ylevel(2), &
+                      1 - ylevel(3):decomp%ysz(3) + ylevel(3)), &
+                  stat=alloc_stat)
      end if
 
      if (alloc_stat /= 0) then
@@ -217,7 +217,7 @@ subroutine alloc_y_complex_short(var, opt_ylevel, opt_global)
 
      implicit none
 
-     complex(mytype), allocatable, dimension(:,:,:) :: var
+     complex(mytype), allocatable, dimension(:, :, :) :: var
      logical, intent(IN), optional :: opt_global
      integer, dimension(3), intent(IN), optional :: opt_ylevel
      integer, dimension(3) :: ylevel
@@ -258,15 +258,15 @@ subroutine alloc_y_complex(var, decomp, opt_ylevel, opt_global)
      end if
 
      if (global) then
-        allocate(var(decomp%yst(1)-ylevel(1):decomp%yen(1)+ylevel(1), &
-             decomp%yst(2)-ylevel(2):decomp%yen(2)+ylevel(2), &
-             decomp%yst(3)-ylevel(3):decomp%yen(3)+ylevel(3)), &
-             stat=alloc_stat)
+        allocate (var(decomp%yst(1) - ylevel(1):decomp%yen(1) + ylevel(1), &
+                      decomp%yst(2) - ylevel(2):decomp%yen(2) + ylevel(2), &
+                      decomp%yst(3) - ylevel(3):decomp%yen(3) + ylevel(3)), &
+                  stat=alloc_stat)
      else
-        allocate(var(1-ylevel(1):decomp%ysz(1)+ylevel(1), &
-             1-ylevel(2):decomp%ysz(2)+ylevel(2), &
-             1-ylevel(3):decomp%ysz(3)+ylevel(3)), &
-             stat=alloc_stat)
+        allocate (var(1 - ylevel(1):decomp%ysz(1) + ylevel(1), &
+                      1 - ylevel(2):decomp%ysz(2) + ylevel(2), &
+                      1 - ylevel(3):decomp%ysz(3) + ylevel(3)), &
+                  stat=alloc_stat)
      end if
 
      if (alloc_stat /= 0) then
@@ -283,7 +283,7 @@ subroutine alloc_z_real_short(var, opt_zlevel, opt_global)
 
      implicit none
 
-     real(mytype), allocatable, dimension(:,:,:) :: var
+     real(mytype), allocatable, dimension(:, :, :) :: var
      logical, intent(IN), optional :: opt_global
      integer, dimension(3), intent(IN), optional :: opt_zlevel
      integer, dimension(3) :: zlevel
@@ -324,21 +324,21 @@ subroutine alloc_z_real(var, decomp, opt_zlevel, opt_global)
      end if
 
      if (global) then
-        allocate(var(decomp%zst(1)-zlevel(1):decomp%zen(1)+zlevel(1), &
-             decomp%zst(2)-zlevel(2):decomp%zen(2)+zlevel(2), &
-             decomp%zst(3)-zlevel(3):decomp%zen(3)+zlevel(3)), &
-             stat=alloc_stat)
+        allocate (var(decomp%zst(1) - zlevel(1):decomp%zen(1) + zlevel(1), &
+                      decomp%zst(2) - zlevel(2):decomp%zen(2) + zlevel(2), &
+                      decomp%zst(3) - zlevel(3):decomp%zen(3) + zlevel(3)), &
+                  stat=alloc_stat)
      else
-        allocate(var(1-zlevel(1):decomp%zsz(1)+zlevel(1), &
-             1-zlevel(2):decomp%zsz(2)+zlevel(2), &
-             1-zlevel(3):decomp%zsz(3)+zlevel(3)), &
-             stat=alloc_stat)
+        allocate (var(1 - zlevel(1):decomp%zsz(1) + zlevel(1), &
+                      1 - zlevel(2):decomp%zsz(2) + zlevel(2), &
+                      1 - zlevel(3):decomp%zsz(3) + zlevel(3)), &
+                  stat=alloc_stat)
      end if
 
      if (alloc_stat /= 0) then
         errorcode = 8
         call decomp_2d_abort(errorcode, &
-             'Memory allocation failed when creating new arrays')
+                             'Memory allocation failed when creating new arrays')
      end if
 
      return
@@ -349,7 +349,7 @@ subroutine alloc_z_complex_short(var, opt_zlevel, opt_global)
 
      implicit none
 
-     complex(mytype), allocatable, dimension(:,:,:) :: var
+     complex(mytype), allocatable, dimension(:, :, :) :: var
      logical, intent(IN), optional :: opt_global
      integer, dimension(3), intent(IN), optional :: opt_zlevel
      integer, dimension(3) :: zlevel
@@ -362,9 +362,9 @@ subroutine alloc_z_complex_short(var, opt_zlevel, opt_global)
 
      call alloc_z(var, decomp_main, zlevel, opt_global)
 
-   end subroutine alloc_z_complex_short
+  end subroutine alloc_z_complex_short
 
-   subroutine alloc_z_complex(var, decomp, opt_zlevel, opt_global)
+  subroutine alloc_z_complex(var, decomp, opt_zlevel, opt_global)
 
      implicit none
 
@@ -390,21 +390,21 @@ subroutine alloc_z_complex(var, decomp, opt_zlevel, opt_global)
      end if
 
      if (global) then
-        allocate(var(decomp%zst(1)-zlevel(1):decomp%zen(1)+zlevel(1), &
-             decomp%zst(2)-zlevel(2):decomp%zen(2)+zlevel(2), &
-             decomp%zst(3)-zlevel(3):decomp%zen(3)+zlevel(3)), &
-             stat=alloc_stat)
+        allocate (var(decomp%zst(1) - zlevel(1):decomp%zen(1) + zlevel(1), &
+                      decomp%zst(2) - zlevel(2):decomp%zen(2) + zlevel(2), &
+                      decomp%zst(3) - zlevel(3):decomp%zen(3) + zlevel(3)), &
+                  stat=alloc_stat)
      else
-        allocate(var(1-zlevel(1):decomp%zsz(1)+zlevel(1), &
-             1-zlevel(2):decomp%zsz(2)+zlevel(2), &
-             1-zlevel(3):decomp%zsz(3)+zlevel(3)), &
-             stat=alloc_stat)
+        allocate (var(1 - zlevel(1):decomp%zsz(1) + zlevel(1), &
+                      1 - zlevel(2):decomp%zsz(2) + zlevel(2), &
+                      1 - zlevel(3):decomp%zsz(3) + zlevel(3)), &
+                  stat=alloc_stat)
      end if
 
      if (alloc_stat /= 0) then
         errorcode = 8
         call decomp_2d_abort(errorcode, &
-             'Memory allocation failed when creating new arrays')
+                             'Memory allocation failed when creating new arrays')
      end if
 
      return
diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index bc354b1b..57baf0d5 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -107,8 +107,8 @@ module decomp_2d
       logical :: even
 #endif
 
-     ! number of halo cells in each direction (index 1,2,3), for each pencil (x,y,z)
-     integer, dimension(3) :: xlevel, ylevel, zlevel
+      ! number of halo cells in each direction (index 1,2,3), for each pencil (x,y,z)
+      integer, dimension(3) :: xlevel, ylevel, zlevel
 
    END TYPE DECOMP_INFO
 
diff --git a/src/halo.f90 b/src/halo.f90
index cf29a2ed..03f84275 100644
--- a/src/halo.f90
+++ b/src/halo.f90
@@ -34,8 +34,8 @@ subroutine update_halo_real(in, out, level, decomp, opt_global, opt_pencil)
      implicit none
 
      integer, intent(IN) :: level      ! levels of halo cells required
-     real(mytype), dimension(:,:,:), intent(IN) :: in
-     real(mytype), allocatable, dimension(:,:,:), intent(OUT) :: out
+     real(mytype), dimension(:, :, :), intent(IN) :: in
+     real(mytype), allocatable, dimension(:, :, :), intent(OUT) :: out
 #if defined(_GPU)
      attributes(device) :: out
 #endif
@@ -114,161 +114,156 @@ subroutine update_halo_complex(in, out, level, decomp, opt_global, opt_pencil)
      return
   end subroutine update_halo_complex
 
-
   subroutine exchange_halo_x_real(inout, opt_decomp, opt_xlevel)
 
-    implicit none
-
-    real(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    TYPE(DECOMP_INFO), optional :: opt_decomp
-    integer, dimension(3), optional :: opt_xlevel
+     implicit none
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: level_x, level_y, level_z
-    integer :: ierror
-    integer :: icount, ilength, ijump
-    integer :: halo12
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_n, tag_s, tag_t, tag_b
-    integer :: data_type
-    integer :: xs, ys, zs, ye, ze, s1, s2, s3
+     real(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), optional :: opt_decomp
+     integer, dimension(3), optional :: opt_xlevel
+
+     TYPE(DECOMP_INFO) :: decomp
+     integer :: level_x, level_y, level_z
+     integer :: ierror
+     integer :: icount, ilength, ijump
+     integer :: halo12
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_n, tag_s, tag_t, tag_b
+     integer :: data_type
+     integer :: xs, ys, zs, ye, ze, s1, s2, s3
 
-    data_type = real_type
+     data_type = real_type
 
 #include "halo_comm_x.f90"
 
-    return
+     return
   end subroutine exchange_halo_x_real
 
-
   subroutine exchange_halo_x_complex(inout, opt_decomp, opt_xlevel)
 
-    implicit none
-
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    TYPE(DECOMP_INFO), optional :: opt_decomp
-    integer, dimension(3), optional :: opt_xlevel
+     implicit none
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: level_x, level_y, level_z
-    integer :: ierror
-    integer :: icount, ilength, ijump
-    integer :: halo12
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_n, tag_s, tag_t, tag_b
-    integer :: data_type
-    integer :: xs, ys, zs, ye, ze, s1, s2, s3
+     complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), optional :: opt_decomp
+     integer, dimension(3), optional :: opt_xlevel
+
+     TYPE(DECOMP_INFO) :: decomp
+     integer :: level_x, level_y, level_z
+     integer :: ierror
+     integer :: icount, ilength, ijump
+     integer :: halo12
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_n, tag_s, tag_t, tag_b
+     integer :: data_type
+     integer :: xs, ys, zs, ye, ze, s1, s2, s3
 
-    data_type = complex_type
+     data_type = complex_type
 
 #include "halo_comm_x.f90"
 
-    return
+     return
   end subroutine exchange_halo_x_complex
 
-
   subroutine exchange_halo_y_real(inout, opt_decomp, opt_ylevel)
 
-    implicit none
-
-    real(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    TYPE(DECOMP_INFO), optional :: opt_decomp
-    integer, dimension(3), optional :: opt_ylevel
+     implicit none
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: level_x, level_y, level_z
-    integer :: ierror
-    integer :: icount, ilength, ijump
-    integer :: halo21
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_e, tag_w, tag_t, tag_b
-    integer :: data_type
-    integer :: xs, ys, zs, xe, ze, s1, s2, s3
+     real(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), optional :: opt_decomp
+     integer, dimension(3), optional :: opt_ylevel
+
+     TYPE(DECOMP_INFO) :: decomp
+     integer :: level_x, level_y, level_z
+     integer :: ierror
+     integer :: icount, ilength, ijump
+     integer :: halo21
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_e, tag_w, tag_t, tag_b
+     integer :: data_type
+     integer :: xs, ys, zs, xe, ze, s1, s2, s3
 
-    data_type = real_type
+     data_type = real_type
 
 #include "halo_comm_y.f90"
 
-    return
+     return
   end subroutine exchange_halo_y_real
 
-
   subroutine exchange_halo_y_complex(inout, opt_decomp, opt_ylevel)
 
-    implicit none
-
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    TYPE(DECOMP_INFO), optional :: opt_decomp
-    integer, dimension(3), optional :: opt_ylevel
+     implicit none
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: level_x, level_y, level_z
-    integer :: ierror
-    integer :: icount, ilength, ijump
-    integer :: halo21
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_e, tag_w, tag_t, tag_b
-    integer :: data_type
-    integer :: xs, ys, zs, xe, ze, s1, s2, s3
+     complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), optional :: opt_decomp
+     integer, dimension(3), optional :: opt_ylevel
+
+     TYPE(DECOMP_INFO) :: decomp
+     integer :: level_x, level_y, level_z
+     integer :: ierror
+     integer :: icount, ilength, ijump
+     integer :: halo21
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_e, tag_w, tag_t, tag_b
+     integer :: data_type
+     integer :: xs, ys, zs, xe, ze, s1, s2, s3
 
-    data_type = complex_type
+     data_type = complex_type
 
 #include "halo_comm_y.f90"
 
-    return
+     return
   end subroutine exchange_halo_y_complex
 
-
   subroutine exchange_halo_z_real(inout, opt_decomp, opt_zlevel)
-    implicit none
-    real(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    TYPE(DECOMP_INFO), optional :: opt_decomp
-    integer, dimension(3), optional :: opt_zlevel
-
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: level_x, level_y, level_z
-    integer :: ierror
-    integer :: icount, ilength, ijump
-    integer :: halo31, halo32
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_e, tag_w, tag_n, tag_s
-    integer :: data_type
-    integer :: xs, ys, zs, xe, ye, s1, s2, s3
-
-    data_type = real_type
+     implicit none
+     real(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), optional :: opt_decomp
+     integer, dimension(3), optional :: opt_zlevel
+
+     TYPE(DECOMP_INFO) :: decomp
+     integer :: level_x, level_y, level_z
+     integer :: ierror
+     integer :: icount, ilength, ijump
+     integer :: halo31, halo32
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_e, tag_w, tag_n, tag_s
+     integer :: data_type
+     integer :: xs, ys, zs, xe, ye, s1, s2, s3
+
+     data_type = real_type
 
 #include "halo_comm_z.f90"
 
-    return
+     return
   end subroutine exchange_halo_z_real
 
   subroutine exchange_halo_z_complex(inout, opt_decomp, opt_zlevel)
 
-    implicit none
-
-    complex(mytype), dimension(:,:,:), intent(INOUT) :: inout
-    TYPE(DECOMP_INFO), optional :: opt_decomp
-    integer, dimension(3), optional :: opt_zlevel
+     implicit none
 
-    TYPE(DECOMP_INFO) :: decomp
-    integer :: level_x, level_y, level_z
-    integer :: ierror
-    integer :: icount, ilength, ijump
-    integer :: halo31, halo32
-    integer, dimension(4) :: requests
-    integer, dimension(MPI_STATUS_SIZE,4) :: status
-    integer :: tag_e, tag_w, tag_n, tag_s
-    integer :: data_type
-    integer :: xs, ys, zs, xe, ye, s1, s2, s3
+     complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), optional :: opt_decomp
+     integer, dimension(3), optional :: opt_zlevel
+
+     TYPE(DECOMP_INFO) :: decomp
+     integer :: level_x, level_y, level_z
+     integer :: ierror
+     integer :: icount, ilength, ijump
+     integer :: halo31, halo32
+     integer, dimension(4) :: requests
+     integer, dimension(MPI_STATUS_SIZE, 4) :: status
+     integer :: tag_e, tag_w, tag_n, tag_s
+     integer :: data_type
+     integer :: xs, ys, zs, xe, ye, s1, s2, s3
 
-    data_type = complex_type
+     data_type = complex_type
 
 #include "halo_comm_z.f90"
 
-    return
+     return
   end subroutine exchange_halo_z_complex
diff --git a/src/halo_comm_x.f90 b/src/halo_comm_x.f90
index b2d5d568..6d0b380a 100644
--- a/src/halo_comm_x.f90
+++ b/src/halo_comm_x.f90
@@ -13,19 +13,19 @@
 ! 'exchange_halo_x_...' in halo.f90
 
     if (present(opt_decomp)) then
-      decomp = opt_decomp
+       decomp = opt_decomp
     else
-      decomp = decomp_main
+       decomp = decomp_main
     end if
 
     if (present(opt_xlevel)) then
-      level_x = opt_xlevel(1)
-      level_y = opt_xlevel(2)
-      level_z = opt_xlevel(3)
+       level_x = opt_xlevel(1)
+       level_y = opt_xlevel(2)
+       level_z = opt_xlevel(3)
     else
-      level_x = decomp%xlevel(1)
-      level_y = decomp%xlevel(2)
-      level_z = decomp%xlevel(3)
+       level_x = decomp%xlevel(1)
+       level_y = decomp%xlevel(2)
+       level_z = decomp%xlevel(3)
     end if
 
     s1 = decomp%xsz(1)
@@ -34,9 +34,9 @@
 
     xs = 1 + level_x
     ys = 1
-    ye = s2 + 2*level_y
+    ye = s2 + 2 * level_y
     zs = 1
-    ze = s3 + 2*level_z
+    ze = s3 + 2 * level_z
 
     ! if (decomp%halos_for_pencil) then
     !   ! don't communicate lower halo (west boundary)
@@ -47,14 +47,14 @@
     ! end if
 
 #ifdef HALO_DEBUG
-    if (nrank==4) then
-      write(*,*) 'X-pencil input'
-      write(*,*) '=============='
-      write(*,*) 'Data on a y-z plane is shown'
-      write(*,*) 'Before halo exchange'
-      do j=ye,ys,-1
-        write(*,'(10F4.0)') (inout(1,j,k),k=zs,ze)
-      end do
+    if (nrank == 4) then
+       write (*, *) 'X-pencil input'
+       write (*, *) '=============='
+       write (*, *) 'Data on a y-z plane is shown'
+       write (*, *) 'Before halo exchange'
+       do j = ye, ys, -1
+          write (*, '(10F4.0)') (inout(1, j, k), k=zs, ze)
+       end do
     end if
 #endif
 
@@ -63,49 +63,49 @@
 
     ! *** north/south ***
     tag_s = coord(1)
-    if (coord(1)==dims(1)-1 .AND. periodic_y) then
-      tag_n = 0
+    if (coord(1) == dims(1) - 1 .AND. periodic_y) then
+       tag_n = 0
     else
-      tag_n = coord(1) + 1
+       tag_n = coord(1) + 1
     end if
-    icount = s3 + 2*level_z
+    icount = s3 + 2 * level_z
     ilength = level_y * s1
-    ijump = s1 * (s2 + 2*level_y)
+    ijump = s1 * (s2 + 2 * level_y)
     call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-      data_type, halo12, ierror)
+                         data_type, halo12, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
     call MPI_TYPE_COMMIT(halo12, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
     ! receive from south
-    call MPI_IRECV(inout(xs,ys,zs), 1, halo12, &
-      neighbour(1,4), tag_s, DECOMP_2D_COMM_CART_X, &
-      requests(1), ierror)
+    call MPI_IRECV(inout(xs, ys, zs), 1, halo12, &
+                   neighbour(1, 4), tag_s, DECOMP_2D_COMM_CART_X, &
+                   requests(1), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! receive from north
-    call MPI_IRECV(inout(xs,ye-level_y+1,zs), 1, halo12, &
-      neighbour(1,3), tag_n, DECOMP_2D_COMM_CART_X, &
-      requests(2), ierror)
+    call MPI_IRECV(inout(xs, ye - level_y + 1, zs), 1, halo12, &
+                   neighbour(1, 3), tag_n, DECOMP_2D_COMM_CART_X, &
+                   requests(2), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! send to south
-    call MPI_ISSEND(inout(xs,ys+level_y,zs), 1, halo12, &
-      neighbour(1,4), tag_s, DECOMP_2D_COMM_CART_X, &
-      requests(3), ierror)
+    call MPI_ISSEND(inout(xs, ys + level_y, zs), 1, halo12, &
+                    neighbour(1, 4), tag_s, DECOMP_2D_COMM_CART_X, &
+                    requests(3), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     ! send to north
-    call MPI_ISSEND(inout(xs,ye-level_y-level_y+1,zs), 1, halo12, &
-      neighbour(1,3), tag_n, DECOMP_2D_COMM_CART_X, &
-      requests(4), ierror)
+    call MPI_ISSEND(inout(xs, ye - level_y - level_y + 1, zs), 1, halo12, &
+                    neighbour(1, 3), tag_n, DECOMP_2D_COMM_CART_X, &
+                    requests(4), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     call MPI_WAITALL(4, requests, status, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
     call MPI_TYPE_FREE(halo12, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-    if (nrank==4) then
-      write(*,*) 'After exchange in Y'
-      do j=ye,ys,-1
-        write(*,'(10F4.0)') (inout(1,j,k),k=zs,ze)
-      end do
+    if (nrank == 4) then
+       write (*, *) 'After exchange in Y'
+       do j = ye, ys, -1
+          write (*, '(10F4.0)') (inout(1, j, k), k=zs, ze)
+       end do
     end if
 #endif
 
@@ -114,40 +114,40 @@
     ! all contiguous in memory, which can be sent/received using
     ! MPI directly
     tag_b = coord(2)
-    if (coord(2)==dims(2)-1 .AND. periodic_z) then
-      tag_t = 0
+    if (coord(2) == dims(2) - 1 .AND. periodic_z) then
+       tag_t = 0
     else
-      tag_t = coord(2) + 1
+       tag_t = coord(2) + 1
     end if
-    icount = (s1 * (s2 + 2*level_y)) * level_z
+    icount = (s1 * (s2 + 2 * level_y)) * level_z
     ! receive from bottom
-    call MPI_IRECV(inout(xs,ys,zs), icount, data_type, &
-      neighbour(1,6), tag_b, DECOMP_2D_COMM_CART_X, &
-      requests(1), ierror)
+    call MPI_IRECV(inout(xs, ys, zs), icount, data_type, &
+                   neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
+                   requests(1), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! receive from top
-    call MPI_IRECV(inout(xs,ys,ze-level_z+1), icount, data_type, &
-      neighbour(1,5), tag_t, DECOMP_2D_COMM_CART_X, &
-      requests(2), ierror)
+    call MPI_IRECV(inout(xs, ys, ze - level_z + 1), icount, data_type, &
+                   neighbour(1, 5), tag_t, DECOMP_2D_COMM_CART_X, &
+                   requests(2), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! send to bottom
-    call MPI_ISSEND(inout(xs,ys,zs+level_z), icount, data_type, &
-      neighbour(1,6), tag_b, DECOMP_2D_COMM_CART_X, &
-      requests(3), ierror)
+    call MPI_ISSEND(inout(xs, ys, zs + level_z), icount, data_type, &
+                    neighbour(1, 6), tag_b, DECOMP_2D_COMM_CART_X, &
+                    requests(3), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     ! send to top
-    call MPI_ISSEND(inout(xs,ys,ze-level_z-level_z+1), icount, data_type, &
-      neighbour(1,5), tag_t, DECOMP_2D_COMM_CART_X, &
-      requests(4), ierror)
+    call MPI_ISSEND(inout(xs, ys, ze - level_z - level_z + 1), icount, data_type, &
+                    neighbour(1, 5), tag_t, DECOMP_2D_COMM_CART_X, &
+                    requests(4), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     call MPI_WAITALL(4, requests, status, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
 
 #ifdef HALO_DEBUG
-    if (nrank==4) then
-      write(*,*) 'After exchange in Z'
-      do j=ye,ys,-1
-        write(*,'(10F4.0)') (inout(1,j,k),k=zs,ze)
-      end do
+    if (nrank == 4) then
+       write (*, *) 'After exchange in Z'
+       do j = ye, ys, -1
+          write (*, '(10F4.0)') (inout(1, j, k), k=zs, ze)
+       end do
     end if
 #endif
diff --git a/src/halo_comm_y.f90 b/src/halo_comm_y.f90
index 38ab6b8a..3776d24f 100644
--- a/src/halo_comm_y.f90
+++ b/src/halo_comm_y.f90
@@ -13,19 +13,19 @@
 ! 'exchange_halo_y_...' in halo.f90
 
     if (present(opt_decomp)) then
-      decomp = opt_decomp
+       decomp = opt_decomp
     else
-      decomp = decomp_main
+       decomp = decomp_main
     end if
 
     if (present(opt_ylevel)) then
-      level_x = opt_ylevel(1)
-      level_y = opt_ylevel(2)
-      level_z = opt_ylevel(3)
+       level_x = opt_ylevel(1)
+       level_y = opt_ylevel(2)
+       level_z = opt_ylevel(3)
     else
-      level_x = decomp%ylevel(1)
-      level_y = decomp%ylevel(2)
-      level_z = decomp%ylevel(3)
+       level_x = decomp%ylevel(1)
+       level_y = decomp%ylevel(2)
+       level_z = decomp%ylevel(3)
     end if
 
     s1 = decomp%ysz(1)
@@ -33,10 +33,10 @@
     s3 = decomp%ysz(3)
 
     xs = 1
-    xe = s1 + 2*level_x
+    xe = s1 + 2 * level_x
     ys = 1 + level_y
     zs = 1
-    ze = s3 + 2*level_z
+    ze = s3 + 2 * level_z
 
     ! if (decomp%halos_for_pencil) then
     !   ! don't communicate lower halo (south boundary)
@@ -47,62 +47,62 @@
     ! end if
 
 #ifdef HALO_DEBUG
-    if (nrank==4) then
-      write(*,*) 'Y-pencil input'
-      write(*,*) '=============='
-      write(*,*) 'Data on a x-z plane is shown'
-      write(*,*) 'Before halo exchange'
-      do i=xe,xs,-1
-        write(*,'(10F4.0)') (inout(i,1,k),k=zs,ze)
-      end do
+    if (nrank == 4) then
+       write (*, *) 'Y-pencil input'
+       write (*, *) '=============='
+       write (*, *) 'Data on a x-z plane is shown'
+       write (*, *) 'Before halo exchange'
+       do i = xe, xs, -1
+          write (*, '(10F4.0)') (inout(i, 1, k), k=zs, ze)
+       end do
     end if
 #endif
 
     ! *** east/west ***
     tag_w = coord(1)
-    if (coord(1)==dims(1)-1 .AND. periodic_x) then
-      tag_e = 0
+    if (coord(1) == dims(1) - 1 .AND. periodic_x) then
+       tag_e = 0
     else
-      tag_e = coord(1) + 1
+       tag_e = coord(1) + 1
     end if
-    icount = s2 * (s3 + 2*level_z)
+    icount = s2 * (s3 + 2 * level_z)
     ilength = level_x
-    ijump = s1 + 2*level_x
+    ijump = s1 + 2 * level_x
     call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-      data_type, halo21, ierror)
+                         data_type, halo21, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
     call MPI_TYPE_COMMIT(halo21, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
     ! receive from west
-    call MPI_IRECV(inout(xs,ys,zs), 1, halo21, &
-      neighbour(2,2), tag_w, DECOMP_2D_COMM_CART_Y, &
-      requests(1), ierror)
+    call MPI_IRECV(inout(xs, ys, zs), 1, halo21, &
+                   neighbour(2, 2), tag_w, DECOMP_2D_COMM_CART_Y, &
+                   requests(1), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! receive from east
-    call MPI_IRECV(inout(xe-level_x+1,ys,zs), 1, halo21, &
-      neighbour(2,1), tag_e, DECOMP_2D_COMM_CART_Y, &
-      requests(2), ierror)
+    call MPI_IRECV(inout(xe - level_x + 1, ys, zs), 1, halo21, &
+                   neighbour(2, 1), tag_e, DECOMP_2D_COMM_CART_Y, &
+                   requests(2), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! send to west
-    call MPI_ISSEND(inout(xs+level_x,ys,zs), 1, halo21, &
-      neighbour(2,2), tag_w, DECOMP_2D_COMM_CART_Y, &
-      requests(3), ierror)
+    call MPI_ISSEND(inout(xs + level_x, ys, zs), 1, halo21, &
+                    neighbour(2, 2), tag_w, DECOMP_2D_COMM_CART_Y, &
+                    requests(3), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     ! send to east
-    call MPI_ISSEND(inout(xe-level_x-level_x+1,ys,zs), 1, halo21, &
-      neighbour(2,1), tag_e, DECOMP_2D_COMM_CART_Y, &
-      requests(4), ierror)
+    call MPI_ISSEND(inout(xe - level_x - level_x + 1, ys, zs), 1, halo21, &
+                    neighbour(2, 1), tag_e, DECOMP_2D_COMM_CART_Y, &
+                    requests(4), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     call MPI_WAITALL(4, requests, status, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
     call MPI_TYPE_FREE(halo21, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-    if (nrank==4) then
-    write(*,*) 'After exchange in X'
-    do i=xe,xs,-1
-      write(*,'(10F4.0)') (inout(i,1,k),k=zs,ze)
-    end do
+    if (nrank == 4) then
+       write (*, *) 'After exchange in X'
+       do i = xe, xs, -1
+          write (*, '(10F4.0)') (inout(i, 1, k), k=zs, ze)
+       end do
     end if
 #endif
 
@@ -114,39 +114,39 @@
     ! all contiguous in memory, which can be sent/received using
     ! MPI directly
     tag_b = coord(2)
-    if (coord(2)==dims(2)-1 .AND. periodic_z) then
-      tag_t = 0
+    if (coord(2) == dims(2) - 1 .AND. periodic_z) then
+       tag_t = 0
     else
-      tag_t = coord(2) + 1
+       tag_t = coord(2) + 1
     end if
-    icount = (s2 * (s1 + 2*level_x)) * level_z
+    icount = (s2 * (s1 + 2 * level_x)) * level_z
     ! receive from bottom
-    call MPI_IRECV(inout(xs,ys,zs), icount, data_type, &
-      neighbour(2,6), tag_b, DECOMP_2D_COMM_CART_Y, &
-      requests(1), ierror)
+    call MPI_IRECV(inout(xs, ys, zs), icount, data_type, &
+                   neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
+                   requests(1), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! receive from top
-    call MPI_IRECV(inout(xs,ys,ze-level_z+1), icount, data_type, &
-      neighbour(2,5), tag_t, DECOMP_2D_COMM_CART_Y, &
-      requests(2), ierror)
+    call MPI_IRECV(inout(xs, ys, ze - level_z + 1), icount, data_type, &
+                   neighbour(2, 5), tag_t, DECOMP_2D_COMM_CART_Y, &
+                   requests(2), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! send to bottom
-    call MPI_ISSEND(inout(xs,ys,zs+level_z), icount, data_type, &
-      neighbour(2,6), tag_b, DECOMP_2D_COMM_CART_Y, &
-      requests(3), ierror)
+    call MPI_ISSEND(inout(xs, ys, zs + level_z), icount, data_type, &
+                    neighbour(2, 6), tag_b, DECOMP_2D_COMM_CART_Y, &
+                    requests(3), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     ! send to top
-    call MPI_ISSEND(inout(xs,ys,ze-level_z-level_z+1), icount, data_type, &
-      neighbour(2,5), tag_t, DECOMP_2D_COMM_CART_Y, &
-      requests(4), ierror)
+    call MPI_ISSEND(inout(xs, ys, ze - level_z - level_z + 1), icount, data_type, &
+                    neighbour(2, 5), tag_t, DECOMP_2D_COMM_CART_Y, &
+                    requests(4), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     call MPI_WAITALL(4, requests, status, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
 #ifdef HALO_DEBUG
-    if (nrank==4) then
-      write(*,*) 'After exchange in Z'
-      do i=xe,xs,-1
-        write(*,'(10F4.0)') (inout(i,1,k),k=zs,ze)
-      end do
+    if (nrank == 4) then
+       write (*, *) 'After exchange in Z'
+       do i = xe, xs, -1
+          write (*, '(10F4.0)') (inout(i, 1, k), k=zs, ze)
+       end do
     end if
 #endif
diff --git a/src/halo_comm_z.f90 b/src/halo_comm_z.f90
index 78524f43..d9368c9a 100644
--- a/src/halo_comm_z.f90
+++ b/src/halo_comm_z.f90
@@ -13,20 +13,20 @@
 ! 'exchange_halo_z_...' in halo.f90
 
     if (present(opt_decomp)) then
-      decomp = opt_decomp
+       decomp = opt_decomp
     else
-      decomp = decomp_main
+       decomp = decomp_main
     end if
 
     if (present(opt_zlevel)) then ! assume same level for all directions
-      level_x = opt_zlevel(1)
-      level_y = opt_zlevel(2)
-      level_z = opt_zlevel(3)
+       level_x = opt_zlevel(1)
+       level_y = opt_zlevel(2)
+       level_z = opt_zlevel(3)
     else
-      level_x = decomp%zlevel(1)
-      level_y = decomp%zlevel(2)
-      level_z = decomp%zlevel(3)
-      ! add checks to make sure level_x and level_y are sensible values (positive integer)?
+       level_x = decomp%zlevel(1)
+       level_y = decomp%zlevel(2)
+       level_z = decomp%zlevel(3)
+       ! add checks to make sure level_x and level_y are sensible values (positive integer)?
     end if
 
     s1 = decomp%zsz(1)
@@ -34,9 +34,9 @@
     s3 = decomp%zsz(3)
 
     xs = 1
-    xe = s1 + 2*level_x
+    xe = s1 + 2 * level_x
     ys = 1
-    ye = s2 + 2*level_y
+    ye = s2 + 2 * level_y
     !zs = 1 + level_z
     zs = 1
 
@@ -49,51 +49,51 @@
     ! end if
 
 #ifdef HALO_DEBUG
-    if (nrank==4) then
-      write(*,*) 'Z-pencil input'
-      write(*,*) '=============='
-      write(*,*) 'Data on a x-y plane is shown'
-      write(*,*) 'Before halo exchange'
-      do i=xe,xs,-1
-        write(*,'(10F4.0)') (inout(i,j,1),j=ys,ye)
-      end do
+    if (nrank == 4) then
+       write (*, *) 'Z-pencil input'
+       write (*, *) '=============='
+       write (*, *) 'Data on a x-y plane is shown'
+       write (*, *) 'Before halo exchange'
+       do i = xe, xs, -1
+          write (*, '(10F4.0)') (inout(i, j, 1), j=ys, ye)
+       end do
     end if
 #endif
     ! *** east/west ***
     tag_w = coord(1)
-    if (coord(1)==dims(1)-1 .AND. periodic_x) then
-      tag_e = 0
+    if (coord(1) == dims(1) - 1 .AND. periodic_x) then
+       tag_e = 0
     else
-      tag_e = coord(1) + 1
+       tag_e = coord(1) + 1
     end if
     !icount = (s2 + 2*level_y) * s3
-    icount = (s2 + 2*level_y) * (s3 + 2*level_z)
+    icount = (s2 + 2 * level_y) * (s3 + 2 * level_z)
     ilength = level_x
-    ijump = s1 + 2*level_x
+    ijump = s1 + 2 * level_x
     call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-        data_type, halo31, ierror)
+                         data_type, halo31, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
     call MPI_TYPE_COMMIT(halo31, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
     ! receive from west
-    call MPI_IRECV(inout(xs,ys,zs), 1, halo31, &
-        neighbour(3,2), tag_w, DECOMP_2D_COMM_CART_Z, &
-        requests(1), ierror)
+    call MPI_IRECV(inout(xs, ys, zs), 1, halo31, &
+                   neighbour(3, 2), tag_w, DECOMP_2D_COMM_CART_Z, &
+                   requests(1), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! receive from east
-    call MPI_IRECV(inout(xe-level_x+1,ys,zs), 1, halo31, &
-        neighbour(3,1), tag_e, DECOMP_2D_COMM_CART_Z, &
-        requests(2), ierror)
+    call MPI_IRECV(inout(xe - level_x + 1, ys, zs), 1, halo31, &
+                   neighbour(3, 1), tag_e, DECOMP_2D_COMM_CART_Z, &
+                   requests(2), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! send to west
-    call MPI_ISSEND(inout(xs+level_x,ys,zs), 1, halo31, &
-        neighbour(3,2), tag_w, DECOMP_2D_COMM_CART_Z, &
-        requests(3), ierror)
+    call MPI_ISSEND(inout(xs + level_x, ys, zs), 1, halo31, &
+                    neighbour(3, 2), tag_w, DECOMP_2D_COMM_CART_Z, &
+                    requests(3), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     ! send to east
-    call MPI_ISSEND(inout(xe-level_x-level_x+1,ys,zs), 1, halo31, &
-        neighbour(3,1), tag_e, DECOMP_2D_COMM_CART_Z, &
-        requests(4), ierror)
+    call MPI_ISSEND(inout(xe - level_x - level_x + 1, ys, zs), 1, halo31, &
+                    neighbour(3, 1), tag_e, DECOMP_2D_COMM_CART_Z, &
+                    requests(4), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     call MPI_WAITALL(4, requests, status, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
@@ -101,60 +101,60 @@
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 
 #ifdef HALO_DEBUG
-    if (nrank==4) then
-      write(*,*) 'After exchange in X'
-      do i=xe,xs,-1
-        write(*,'(10F4.0)') (inout(i,j,1),j=ys,ye)
-      end do
+    if (nrank == 4) then
+       write (*, *) 'After exchange in X'
+       do i = xe, xs, -1
+          write (*, '(10F4.0)') (inout(i, j, 1), j=ys, ye)
+       end do
     end if
 #endif
 
     ! *** north/south ***
     tag_s = coord(2)
-    if (coord(2)==dims(2)-1 .AND. periodic_y) then
-      tag_n = 0
+    if (coord(2) == dims(2) - 1 .AND. periodic_y) then
+       tag_n = 0
     else
-      tag_n = coord(2) + 1
+       tag_n = coord(2) + 1
     end if
     !icount = s3
-    icount = s3 + (2*level_z)
-    ilength = level_y * (s1 + 2*level_x)
-    ijump = (s1 + 2*level_x) * (s2 + 2*level_y)
+    icount = s3 + (2 * level_z)
+    ilength = level_y * (s1 + 2 * level_x)
+    ijump = (s1 + 2 * level_x) * (s2 + 2 * level_y)
     call MPI_TYPE_VECTOR(icount, ilength, ijump, &
-        data_type, halo32, ierror)
+                         data_type, halo32, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_VECTOR")
     call MPI_TYPE_COMMIT(halo32, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_COMMIT")
     ! receive from south
-    call MPI_IRECV(inout(xs,ys,zs), 1, halo32, &
-        neighbour(3,4), tag_s, DECOMP_2D_COMM_CART_Z, &
-        requests(1), ierror)
+    call MPI_IRECV(inout(xs, ys, zs), 1, halo32, &
+                   neighbour(3, 4), tag_s, DECOMP_2D_COMM_CART_Z, &
+                   requests(1), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! receive from north
-    call MPI_IRECV(inout(xs,ye-level_y+1,zs), 1, halo32, &
-        neighbour(3,3), tag_n, DECOMP_2D_COMM_CART_Z, &
-        requests(2), ierror)
+    call MPI_IRECV(inout(xs, ye - level_y + 1, zs), 1, halo32, &
+                   neighbour(3, 3), tag_n, DECOMP_2D_COMM_CART_Z, &
+                   requests(2), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_IRECV")
     ! send to south
-    call MPI_ISSEND(inout(xs,ys+level_y,zs), 1, halo32, &
-        neighbour(3,4), tag_s, DECOMP_2D_COMM_CART_Z, &
-        requests(3), ierror)
+    call MPI_ISSEND(inout(xs, ys + level_y, zs), 1, halo32, &
+                    neighbour(3, 4), tag_s, DECOMP_2D_COMM_CART_Z, &
+                    requests(3), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     ! send to north
-    call MPI_ISSEND(inout(xs,ye-level_y-level_y+1,zs), 1, halo32, &
-        neighbour(3,3), tag_n, DECOMP_2D_COMM_CART_Z, &
-        requests(4), ierror)
+    call MPI_ISSEND(inout(xs, ye - level_y - level_y + 1, zs), 1, halo32, &
+                    neighbour(3, 3), tag_n, DECOMP_2D_COMM_CART_Z, &
+                    requests(4), ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_ISSEND")
     call MPI_WAITALL(4, requests, status, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_WAITALL")
     call MPI_TYPE_FREE(halo32, ierror)
     if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_TYPE_FREE")
 #ifdef HALO_DEBUG
-    if (nrank==4) then
-      write(*,*) 'After exchange in Y'
-      do i=xe,xs,-1
-        write(*,'(10F4.0)') (inout(i,j,1),j=ys,ye)
-      end do
+    if (nrank == 4) then
+       write (*, *) 'After exchange in Y'
+       do i = xe, xs, -1
+          write (*, '(10F4.0)') (inout(i, j, 1), j=ys, ye)
+       end do
     end if
 #endif
 
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index f72fd5af..0e1bc4f0 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -175,9 +175,9 @@
     end if
 
     if (ipencil == 1) then
-      call exchange_halo_x(out,opt_xlevel=(/0,level,level/))
+       call exchange_halo_x(out, opt_xlevel=(/0, level, level/))
     else if (ipencil == 2) then
-      call exchange_halo_y(out,opt_ylevel=(/level,0,level/))
+       call exchange_halo_y(out, opt_ylevel=(/level, 0, level/))
     else if (ipencil == 3) then
-      call exchange_halo_z(out,opt_zlevel=(/level,level,0/))
+       call exchange_halo_z(out, opt_zlevel=(/level, level, 0/))
     end if

From 25c946d6c824219f7d69c019e02f5d9fbac73fb0 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 20 Feb 2024 15:08:19 +0000
Subject: [PATCH 393/436] Raise error if pencil passed to halo is out of range

---
 src/halo_common.f90 | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/halo_common.f90 b/src/halo_common.f90
index 0e1bc4f0..2ca75d2f 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -12,6 +12,11 @@
 ! This file contain common code to be included by subroutines
 ! 'update_halo_...' in halo.f90
 
+    ! Check for invalid argument
+    if ((ipencil < 1) .or. (ipencil > 3)) then
+       call decomp_2d_abort(__FILE__, __LINE__, ipencil, "Invalid pencil for halo exchange, should be in range 1<=pencil<=3")
+    end if
+
     if (present(opt_global)) then
        global = opt_global
     else

From 7f27993e9fe2f6b033b97eaf72516d0738d8430b Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 20 Feb 2024 15:31:11 +0000
Subject: [PATCH 394/436] Add short versions of halo exchanges - eliminate
 decomp info copies

---
 src/decomp_2d.f90   |  6 +++
 src/halo.f90        | 92 ++++++++++++++++++++++++++++++++++++---------
 src/halo_comm_x.f90 |  6 ---
 src/halo_comm_y.f90 |  6 ---
 src/halo_comm_z.f90 |  6 ---
 5 files changed, 80 insertions(+), 36 deletions(-)

diff --git a/src/decomp_2d.f90 b/src/decomp_2d.f90
index 57baf0d5..6c3575b8 100644
--- a/src/decomp_2d.f90
+++ b/src/decomp_2d.f90
@@ -227,16 +227,22 @@ module decomp_2d
    interface exchange_halo_x
       module procedure exchange_halo_x_real
       module procedure exchange_halo_x_complex
+      module procedure exchange_halo_x_real_short
+      module procedure exchange_halo_x_complex_short
    end interface exchange_halo_x
 
    interface exchange_halo_y
       module procedure exchange_halo_y_real
       module procedure exchange_halo_y_complex
+      module procedure exchange_halo_y_real_short
+      module procedure exchange_halo_y_complex_short
    end interface exchange_halo_y
 
    interface exchange_halo_z
       module procedure exchange_halo_z_real
       module procedure exchange_halo_z_complex
+      module procedure exchange_halo_z_real_short
+      module procedure exchange_halo_z_complex_short
    end interface exchange_halo_z
 
    interface update_halo
diff --git a/src/halo.f90 b/src/halo.f90
index 03f84275..23c9c8a6 100644
--- a/src/halo.f90
+++ b/src/halo.f90
@@ -114,15 +114,25 @@ subroutine update_halo_complex(in, out, level, decomp, opt_global, opt_pencil)
      return
   end subroutine update_halo_complex
 
-  subroutine exchange_halo_x_real(inout, opt_decomp, opt_xlevel)
+  subroutine exchange_halo_x_real_short(inout, opt_xlevel)
 
      implicit none
 
      real(mytype), dimension(:, :, :), intent(INOUT) :: inout
-     TYPE(DECOMP_INFO), optional :: opt_decomp
      integer, dimension(3), optional :: opt_xlevel
 
-     TYPE(DECOMP_INFO) :: decomp
+     call exchange_halo_x_real(inout, decomp_main, opt_xlevel)
+
+  end subroutine exchange_halo_x_real_short
+
+  subroutine exchange_halo_x_real(inout, decomp, opt_xlevel)
+
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), intent(in) :: decomp
+     integer, dimension(3), optional :: opt_xlevel
+
      integer :: level_x, level_y, level_z
      integer :: ierror
      integer :: icount, ilength, ijump
@@ -140,15 +150,25 @@ subroutine exchange_halo_x_real(inout, opt_decomp, opt_xlevel)
      return
   end subroutine exchange_halo_x_real
 
-  subroutine exchange_halo_x_complex(inout, opt_decomp, opt_xlevel)
+  subroutine exchange_halo_x_complex_short(inout, opt_xlevel)
 
      implicit none
 
      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-     TYPE(DECOMP_INFO), optional :: opt_decomp
      integer, dimension(3), optional :: opt_xlevel
 
-     TYPE(DECOMP_INFO) :: decomp
+     call exchange_halo_x_complex(inout, decomp_main, opt_xlevel)
+
+  end subroutine exchange_halo_x_complex_short
+
+  subroutine exchange_halo_x_complex(inout, decomp, opt_xlevel)
+
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), intent(in) :: decomp
+     integer, dimension(3), optional :: opt_xlevel
+
      integer :: level_x, level_y, level_z
      integer :: ierror
      integer :: icount, ilength, ijump
@@ -166,15 +186,25 @@ subroutine exchange_halo_x_complex(inout, opt_decomp, opt_xlevel)
      return
   end subroutine exchange_halo_x_complex
 
-  subroutine exchange_halo_y_real(inout, opt_decomp, opt_ylevel)
+  subroutine exchange_halo_y_real_short(inout, opt_ylevel)
+
+     implicit none
+
+     real(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     integer, dimension(3), optional :: opt_ylevel
+
+     call exchange_halo_y_real(inout, decomp_main, opt_ylevel)
+
+  end subroutine exchange_halo_y_real_short
+
+  subroutine exchange_halo_y_real(inout, decomp, opt_ylevel)
 
      implicit none
 
      real(mytype), dimension(:, :, :), intent(INOUT) :: inout
-     TYPE(DECOMP_INFO), optional :: opt_decomp
+     TYPE(DECOMP_INFO), intent(in) :: decomp
      integer, dimension(3), optional :: opt_ylevel
 
-     TYPE(DECOMP_INFO) :: decomp
      integer :: level_x, level_y, level_z
      integer :: ierror
      integer :: icount, ilength, ijump
@@ -192,15 +222,25 @@ subroutine exchange_halo_y_real(inout, opt_decomp, opt_ylevel)
      return
   end subroutine exchange_halo_y_real
 
-  subroutine exchange_halo_y_complex(inout, opt_decomp, opt_ylevel)
+  subroutine exchange_halo_y_complex_short(inout, opt_ylevel)
 
      implicit none
 
      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-     TYPE(DECOMP_INFO), optional :: opt_decomp
      integer, dimension(3), optional :: opt_ylevel
 
-     TYPE(DECOMP_INFO) :: decomp
+     call exchange_halo_y_complex(inout, decomp_main, opt_ylevel)
+
+  end subroutine exchange_halo_y_complex_short
+
+  subroutine exchange_halo_y_complex(inout, decomp, opt_ylevel)
+
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), intent(in) :: decomp
+     integer, dimension(3), optional :: opt_ylevel
+
      integer :: level_x, level_y, level_z
      integer :: ierror
      integer :: icount, ilength, ijump
@@ -218,13 +258,20 @@ subroutine exchange_halo_y_complex(inout, opt_decomp, opt_ylevel)
      return
   end subroutine exchange_halo_y_complex
 
-  subroutine exchange_halo_z_real(inout, opt_decomp, opt_zlevel)
+  subroutine exchange_halo_z_real_short(inout, opt_zlevel)
      implicit none
      real(mytype), dimension(:, :, :), intent(INOUT) :: inout
-     TYPE(DECOMP_INFO), optional :: opt_decomp
      integer, dimension(3), optional :: opt_zlevel
 
-     TYPE(DECOMP_INFO) :: decomp
+     call exchange_halo_z_real(inout, decomp_main, opt_zlevel)
+  end subroutine exchange_halo_z_real_short
+
+  subroutine exchange_halo_z_real(inout, decomp, opt_zlevel)
+     implicit none
+     real(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     TYPE(DECOMP_INFO), intent(in) :: decomp
+     integer, dimension(3), optional :: opt_zlevel
+
      integer :: level_x, level_y, level_z
      integer :: ierror
      integer :: icount, ilength, ijump
@@ -242,15 +289,24 @@ subroutine exchange_halo_z_real(inout, opt_decomp, opt_zlevel)
      return
   end subroutine exchange_halo_z_real
 
-  subroutine exchange_halo_z_complex(inout, opt_decomp, opt_zlevel)
+  subroutine exchange_halo_z_complex_short(inout, opt_zlevel)
+
+     implicit none
+
+     complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
+     integer, dimension(3), optional :: opt_zlevel
+
+     call exchange_halo_z_complex(inout, decomp_main, opt_zlevel)
+  end subroutine exchange_halo_z_complex_short
+
+  subroutine exchange_halo_z_complex(inout, decomp, opt_zlevel)
 
      implicit none
 
      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
-     TYPE(DECOMP_INFO), optional :: opt_decomp
+     TYPE(DECOMP_INFO), intent(in) :: decomp
      integer, dimension(3), optional :: opt_zlevel
 
-     TYPE(DECOMP_INFO) :: decomp
      integer :: level_x, level_y, level_z
      integer :: ierror
      integer :: icount, ilength, ijump
diff --git a/src/halo_comm_x.f90 b/src/halo_comm_x.f90
index 6d0b380a..0ba475f1 100644
--- a/src/halo_comm_x.f90
+++ b/src/halo_comm_x.f90
@@ -12,12 +12,6 @@
 ! This file contain common code to be included by subroutines
 ! 'exchange_halo_x_...' in halo.f90
 
-    if (present(opt_decomp)) then
-       decomp = opt_decomp
-    else
-       decomp = decomp_main
-    end if
-
     if (present(opt_xlevel)) then
        level_x = opt_xlevel(1)
        level_y = opt_xlevel(2)
diff --git a/src/halo_comm_y.f90 b/src/halo_comm_y.f90
index 3776d24f..7a6c2ac2 100644
--- a/src/halo_comm_y.f90
+++ b/src/halo_comm_y.f90
@@ -12,12 +12,6 @@
 ! This file contain common code to be included by subroutines
 ! 'exchange_halo_y_...' in halo.f90
 
-    if (present(opt_decomp)) then
-       decomp = opt_decomp
-    else
-       decomp = decomp_main
-    end if
-
     if (present(opt_ylevel)) then
        level_x = opt_ylevel(1)
        level_y = opt_ylevel(2)
diff --git a/src/halo_comm_z.f90 b/src/halo_comm_z.f90
index d9368c9a..78f3d1d2 100644
--- a/src/halo_comm_z.f90
+++ b/src/halo_comm_z.f90
@@ -12,12 +12,6 @@
 ! This file contain common code to be included by subroutines
 ! 'exchange_halo_z_...' in halo.f90
 
-    if (present(opt_decomp)) then
-       decomp = opt_decomp
-    else
-       decomp = decomp_main
-    end if
-
     if (present(opt_zlevel)) then ! assume same level for all directions
        level_x = opt_zlevel(1)
        level_y = opt_zlevel(2)

From f7c636e3ec6d8a997b4b0b736ea8c99c447b9c31 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Tue, 20 Feb 2024 15:40:42 +0000
Subject: [PATCH 395/436] Remove duplicate CMake code

---
 CMakeLists.txt | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ff578e53..721143f2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,16 +6,6 @@ project(decomp2d
 set(version 2.0.0)
 enable_testing()
 
-option(DOUBLE_PRECISION "Build 2decomp_fft with double precision" ON)
-if (DOUBLE_PRECISION)
-  add_definitions("-DDOUBLE_PREC")
-endif()
-
-option(SINGLE_PRECISION_OUTPUT "Build 2decomp_fft with output in single precision" OFF)
-if (SINGLE_PRECISION_OUTPUT)
-  add_definitions("-DSAVE_SINGLE")
-endif()
-
 set(BUILD_TARGET "mpi" CACHE STRING "Target for acceleration (mpi (default) or gpu)")
 set_property(CACHE BUILD_TARGET PROPERTY STRINGS mpi gpu)
 

From 232de050eda533214471f827a2597698ae2eda18 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Mon, 11 Mar 2024 14:32:42 +0000
Subject: [PATCH 396/436] Update Intel flags and correct bugs related to: halo,
 check on single precision. FFT ph deallocation is only an issue with Intel

---
 cmake/D2D_MPI.cmake                    | 1 +
 cmake/compilers/D2D_flags_intel.cmake  | 6 ++++--
 examples/fft_physical_x/fft_c2c_x.f90  | 2 +-
 examples/fft_physical_x/fft_grid_x.f90 | 2 +-
 examples/fft_physical_x/fft_r2c_x.f90  | 2 +-
 examples/fft_physical_z/fft_c2c_z.f90  | 2 +-
 examples/fft_physical_z/fft_r2c_z.f90  | 2 +-
 examples/halo_test/halo_test.f90       | 2 +-
 src/fft_common.f90                     | 2 +-
 src/halo_common.f90                    | 2 +-
 10 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/cmake/D2D_MPI.cmake b/cmake/D2D_MPI.cmake
index ca932b70..d2ac8060 100644
--- a/cmake/D2D_MPI.cmake
+++ b/cmake/D2D_MPI.cmake
@@ -58,6 +58,7 @@ if (MPI_FOUND)
     # Force the mpirun to be coherent with the mpifortran
     string(REGEX REPLACE "mpif90" "mpirun" PATH_TO_MPIRUN "${MPI_Fortran_COMPILER}")
     string(REPLACE "mpiifort" "mpirun" PATH_TO_MPIRUN "${PATH_TO_MPIRUN}")
+    string(REPLACE "mpiifx" "mpirun" PATH_TO_MPIRUN "${PATH_TO_MPIRUN}")
     message(STATUS "Path to mpirun ${PATH_TO_MPIRUN}")
     set(MPIEXEC_EXECUTABLE "${PATH_TO_MPIRUN}" CACHE STRING
         "Force MPIRUN to be consistent with MPI_Fortran_COMPILER" FORCE)
diff --git a/cmake/compilers/D2D_flags_intel.cmake b/cmake/compilers/D2D_flags_intel.cmake
index f502d3af..598a7a86 100644
--- a/cmake/compilers/D2D_flags_intel.cmake
+++ b/cmake/compilers/D2D_flags_intel.cmake
@@ -1,8 +1,10 @@
 # Compilers Flags for Intel
 # Check is the compiler is the new ifx based on LLVM or the old ifort
 if (Fortran_COMPILER_NAME MATCHES "IntelLLVM")
-  set(D2D_FFLAGS "-fpp -std08")
-  set(D2D_FFLAGS_RELEASE "-O2")
+  set(D2D_FFLAGS "-fpp -std08 -safe-cray-ptr -g -traceback")
+  set(D2D_FFLAGS_RELEASE "-O3")
+  #set(D2D_FFLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback")
+  #set(D2D_FFLAGS_RELEASE "-O3 -ipo")
 else (Fortran_COMPILER_NAME MATCHES "IntelLLVM")
   #set(CMAKE_Fortran_FLAGS "-cpp xSSE4.2 -axAVX,CORE-AVX-I,CORE-AVX2 -ipo -fp-model fast=2 -mcmodel=large -safe-cray-ptr")
   set(D2D_FFLAGS "-fpp -std08 -xHost -heaparrays -safe-cray-ptr -g -traceback")
diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index e4338c2a..e38d1d8a 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -35,7 +35,7 @@ program fft_c2c_x
 #ifdef DOUBLE_PREC
    real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
-   real(mytype), parameter :: error_precision = 1.e-6_mytype
+   real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
    
 
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index 3a6f037d..d399d492 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -35,7 +35,7 @@ program fft_physical_x
 #ifdef DOUBLE_PREC
    real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
-   real(mytype), parameter :: error_precision = 1.e-6_mytype
+   real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
    
 
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index e85a9452..92ddb81b 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -36,7 +36,7 @@ program fft_r2c_x
 #ifdef DOUBLE_PREC
    real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
-   real(mytype), parameter :: error_precision = 1.e-6_mytype
+   real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
    
 
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index c86635ca..5442d6fe 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -35,7 +35,7 @@ program fft_c2c_z
 #ifdef DOUBLE_PREC
    real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
-   real(mytype), parameter :: error_precision = 1.e-6_mytype
+   real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
    
 
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index 6156b8e1..eeedbab3 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -36,7 +36,7 @@ program fft_r2c_z
 #ifdef DOUBLE_PREC
    real(mytype), parameter :: error_precision = 1.e-12_mytype
 #else
-   real(mytype), parameter :: error_precision = 1.e-6_mytype
+   real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
    
 
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index cb6ab3ad..edaa16cb 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -533,7 +533,7 @@ subroutine check_err(divh, divref, pencil)
       !$acc end kernels
       divmag = mag(tmp)
 
-      if (error < epsilon(divmag) * divmag) then
+      if (error < real(2.0,mytype) * epsilon(divmag) * divmag) then
          passing = .true.
       else
          passing = .false.
diff --git a/src/fft_common.f90 b/src/fft_common.f90
index ace10103..438fa025 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -186,7 +186,7 @@ subroutine decomp_2d_fft_finalize
 
    if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
       call decomp_info_finalize(ph)
-      deallocate (ph)
+      !deallocate (ph)
    end if
    nullify (ph)
    call decomp_info_finalize(sp)
diff --git a/src/halo_common.f90 b/src/halo_common.f90
index b8dedca6..50061dd3 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -140,7 +140,7 @@
           !$acc end kernels
        else if (ipencil == 3) then
           jst = decomp%zst(2); jen = decomp%zen(2)
-          ist = decomp%xst(1); ien = decomp%xen(1)
+          ist = decomp%zst(1); ien = decomp%zen(1)
           !$acc kernels default(present)
           do k = 1, s3  ! z all local
              do j = jst, jen

From f6fed8760a23e047307a268dfc0900d62ccfbed6 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 12 Mar 2024 10:19:20 +0100
Subject: [PATCH 397/436] Add description of HALO_GLOBAL in INSTALL.md

---
 INSTALL.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/INSTALL.md b/INSTALL.md
index e5591f76..1b0bbfc1 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -226,6 +226,10 @@ This variable leads to overwrite the input array when computing FFT. The support
 
 This variable is used to debug the halo operations. This preprocessor variable is driven by the CMake on/off variable `HALO_DEBUG`.
 
+#### HALO_GLOBAL
+
+This variable is used in the example used to test the halo operations. If the variable is defined, the test are using arrays defined with the key `opt_global` set to `.true.`. Otherwise, the arrays are defined with the key set to `.false.`.
+
 #### _GPU
 
 This variable is automatically added in GPU builds.

From 78e6f93b69ce5f35cece3a5ff7cbaa5d03b6652d Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 10:12:22 +0000
Subject: [PATCH 398/436] Update Fast Runner for main

---
 .github/workflows/RunnerFastCI.yml | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index a2361d8f..57ab83fe 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -28,8 +28,7 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
+        source ~/.profile_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
@@ -47,8 +46,7 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
+        source ~/.profile_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes
@@ -66,13 +64,14 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
+        source ~/.profile_aliases
+        source ~/.bash_aliases
         module load foss
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_gcc12_cpu/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
+        #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_gcc12_cpu/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build 
@@ -87,10 +86,9 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
-        module load intel
-        export export FC="mpiifort -fc=ifx"
+        source ~/.bash_aliases
+        load_intel
+        export FC=mpiifx
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl
         cmake --build build -j 4

From 467cca2ed0ef748b0a0e386e08a48a14ea6c31c9 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 10:19:01 +0000
Subject: [PATCH 399/436] Update Fast Runner for main

---
 .github/workflows/RunnerFastCI.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index 57ab83fe..e3dcd303 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -28,7 +28,7 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
-        source ~/.profile_aliases
+        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
@@ -46,7 +46,7 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
-        source ~/.profile_aliases
+        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes
@@ -64,7 +64,6 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source ~/.bashrc
-        source ~/.profile_aliases
         source ~/.bash_aliases
         module load foss
         export FC=mpif90

From d95ddaff99a493be741ee0676206623116f1df8e Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 10:36:25 +0000
Subject: [PATCH 400/436] Update Fast Runner for main

---
 .github/workflows/RunnerFastCI.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index e3dcd303..93628618 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -27,8 +27,8 @@ jobs:
     - name: Compile NHVHPC CPU
       run: |
         ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
         source ~/.bashrc
-        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
@@ -45,8 +45,8 @@ jobs:
     - name: Compile NHVHPC GPU NCCL
       run: |
         ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
         source ~/.bashrc
-        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes
@@ -63,8 +63,8 @@ jobs:
     - name: Compile GNU_Dev openMPI FFTW_F03 ADIOS2 CPU
       run: |
         ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
         source ~/.bashrc
-        source ~/.bash_aliases
         module load foss
         export FC=mpif90
         export CC=mpicc
@@ -84,8 +84,8 @@ jobs:
     - name: Compile IntelLLVM MKL CPU
       run: |
         ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
         source ~/.bashrc
-        source ~/.bash_aliases
         load_intel
         export FC=mpiifx
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl

From 066fe8933609ce6957918983caeb521f28d014c6 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 10:51:23 +0000
Subject: [PATCH 401/436] Update Fast Runner for main

---
 .github/workflows/RunnerFastCI.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index 93628618..f2f6589a 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -28,6 +28,7 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
+        echo $MODULESHOME
         source ~/.bashrc
         module load nvhpc
         export FC=mpif90

From 22bf243107a776006dabb219e1070271ce200a51 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 10:53:54 +0000
Subject: [PATCH 402/436] Update Fast Runner for main

---
 .github/workflows/RunnerFastCI.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index f2f6589a..467953b9 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -30,6 +30,7 @@ jobs:
         source /etc/profile.d/lmod.sh
         echo $MODULESHOME
         source ~/.bashrc
+        source ~/.bashrc_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON

From 49b6f44f072b31117ef9a952f041d0ec3fdde7a6 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 10:56:30 +0000
Subject: [PATCH 403/436] Update Fast Runner for main

---
 .github/workflows/RunnerFastCI.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index 467953b9..aa9e9a05 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -30,7 +30,7 @@ jobs:
         source /etc/profile.d/lmod.sh
         echo $MODULESHOME
         source ~/.bashrc
-        source ~/.bashrc_aliases
+        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON

From 25ce356b420270822b4fbdd9fbeb438f8c82c4f9 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 11:01:41 +0000
Subject: [PATCH 404/436] Update Fast Runner for main

---
 .github/workflows/RunnerFastCI.yml | 27 ++++++---------------------
 1 file changed, 6 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index aa9e9a05..ee7a829b 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -28,16 +28,11 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
-        echo $MODULESHOME
-        source ~/.bashrc
         source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
-        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake -S . -B build -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build
@@ -48,17 +43,13 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
-        source ~/.bashrc
+        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build 
-        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build
         rm -rf build  
     
     # Configure-Build-Run-Run on 4 cores 
@@ -66,17 +57,14 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
-        source ~/.bashrc
+        source ~/.bash_aliases
         module load foss
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
         #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_gcc12_cpu/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
-        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake -S . -B build -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build
@@ -87,15 +75,12 @@ jobs:
       run: |
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
-        source ~/.bashrc
+        source ~/.bash_aliases
         load_intel
         export FC=mpiifx
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
-        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
+        cmake -S . -B build -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build

From 383d4c0dc66fa1ad67f4efedcdbcbb1eb8633570 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 11:06:42 +0000
Subject: [PATCH 405/436] Update Fast Runner for main

---
 .github/workflows/RunnerFastCI.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index ee7a829b..71eab721 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -76,7 +76,12 @@ jobs:
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
         source ~/.bash_aliases
-        load_intel
+        ml tbb
+        ml compiler-rt
+        ml oclfpga
+        ml compiler
+        ml mpi
+        ml mkl
         export FC=mpiifx
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl

From b3039fdc6655d85a92266d9f5e67677966a4a2c4 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 11:50:37 +0000
Subject: [PATCH 406/436] Add comments for PH deallocation with Intel compiler

---
 src/fft_common.f90 | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 438fa025..9ab69e0e 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -186,6 +186,10 @@ subroutine decomp_2d_fft_finalize
 
    if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
       call decomp_info_finalize(ph)
+      ! Intel compiler does not like a deallocated on a pointer
+      ! We expect the nullify below to free the memory
+      ! Indeed, this is the end of the FFT and memory should be 
+      ! cleared in any case
       !deallocate (ph)
    end if
    nullify (ph)

From 0ffb896749fe68cf5b42baa89de68ca44ff80cc7 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 12:08:30 +0000
Subject: [PATCH 407/436] Remove unnecessary deallocate(ph) statement

---
 src/fft_common.f90 | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 9ab69e0e..87c612f0 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -186,11 +186,6 @@ subroutine decomp_2d_fft_finalize
 
    if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
       call decomp_info_finalize(ph)
-      ! Intel compiler does not like a deallocated on a pointer
-      ! We expect the nullify below to free the memory
-      ! Indeed, this is the end of the FFT and memory should be 
-      ! cleared in any case
-      !deallocate (ph)
    end if
    nullify (ph)
    call decomp_info_finalize(sp)

From eb7e7c04067103d21a5fb86ba6ce62979eec294d Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 16:01:27 +0000
Subject: [PATCH 408/436] Update the Runner with the full CI

---
 .github/workflows/RunnerFullCI.yml | 105 ++++-------------------------
 1 file changed, 12 insertions(+), 93 deletions(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index 265a1431..bc6276ec 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -26,16 +26,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile NHVHPC CPU
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -47,18 +40,12 @@ jobs:
     # Need to revert to 23.1
     - name: Compile NHVHPC CPU & ADIOS2
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load nvhpc/23.1
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
-        cmake -S . -B build -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_nvhpc_cpu/opt/lib/cmake/adios2
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
+        #cmake -S . -B build -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_nvhpc_cpu/opt/lib/cmake/adios2
+        cmake -S . -B build -DBUILD_TESTING=ON 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -68,16 +55,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile NHVHPC GPU cuMPI
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=no
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128
         cmake --build build -j 4
         cmake --install build 
@@ -87,16 +67,9 @@ jobs:
     # Configure-Build-Run 
     - name: Compile NHVHPC GPU NCCL
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128
         cmake --build build -j 4
         cmake --install build 
@@ -106,16 +79,9 @@ jobs:
     # Configure-Build-Run 
     - name: Compile NHVHPC GPU NCCL MemManag
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes -DENABLE_MANAGED=yes
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128
         cmake --build build -j 4
         cmake --install build 
@@ -125,16 +91,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU openMPI FFTW CPU
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load foss
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -144,16 +103,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU openMPI FFTW_F03 CPU
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load foss
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -163,19 +115,13 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU openMPI FFTW_F03 Caliper CPU
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load foss
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
         export caliper_DIR=~/GIT_Work/caliper_github/build_gnu/opt/share/cmake/caliper/
-        cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03 -DENABLE_PROFILER=caliper
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
+        #cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03 -DENABLE_PROFILER=caliper
+        cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -185,18 +131,12 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU_Dev openMPI FFTW_F03 ADIOS2 CPU
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load foss
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_gcc12_cpu/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
+        #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_gcc12_cpu/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -206,17 +146,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile Intel MKL CPU
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
-        module load intel
         export FC=mpiifort
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -226,17 +158,10 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile IntelLLVM MKL CPU
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load intel
-        export export FC="mpiifort -fc=ifx"
+        export export FC=mpiifx
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -246,19 +171,13 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile IntelLLVM MKL ADIOS2 CPU
       run: |
-        ls -ltr ~/.bashrc
-        source ~/.bashrc
-        source ~/.profile_github
-        module load CMake 
         module load intel
-        export export FC="mpiifort -fc=ifx"
-        export CXX="mpiicpc -cxx=icpx"
-        export CC="mpiicc -cc=icx"
+        export FC=mpiifx
+        export CXX=mpiicpx
+        export CC=mpiicx
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_ifx_cpu/opt/lib/cmake/adios2 -DFFT_Choice=mkl
-        cmake --build build -j 4
-        cmake --install build 
-        ctest --test-dir build 
+        #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_ifx_cpu/opt/lib/cmake/adios2 -DFFT_Choice=mkl
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=mkl
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 

From 0443799f809195f08d60414d015560644df40023 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 16:15:07 +0000
Subject: [PATCH 409/436] Update the Runner with the full CI

---
 .github/workflows/RunnerFullCI.yml | 56 ++++++++++++++++++++++++++++--
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index bc6276ec..a1c4db59 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -26,6 +26,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile NHVHPC CPU
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON
@@ -40,6 +43,9 @@ jobs:
     # Need to revert to 23.1
     - name: Compile NHVHPC CPU & ADIOS2
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
         module load nvhpc/23.1
         export FC=mpif90
         export CC=mpicc
@@ -55,6 +61,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile NHVHPC GPU cuMPI
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=no
@@ -67,6 +76,9 @@ jobs:
     # Configure-Build-Run 
     - name: Compile NHVHPC GPU NCCL
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes
@@ -79,6 +91,9 @@ jobs:
     # Configure-Build-Run 
     - name: Compile NHVHPC GPU NCCL MemManag
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
         module load nvhpc
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=yes -DENABLE_MANAGED=yes
@@ -91,6 +106,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU openMPI FFTW CPU
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
         module load foss
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw
@@ -103,6 +121,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU openMPI FFTW_F03 CPU
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
         module load foss
         export FC=mpif90
         cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
@@ -115,6 +136,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU openMPI FFTW_F03 Caliper CPU
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
         module load foss
         export FC=mpif90
         export CC=mpicc
@@ -131,6 +155,9 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile GNU_Dev openMPI FFTW_F03 ADIOS2 CPU
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
         module load foss
         export FC=mpif90
         export CC=mpicc
@@ -146,6 +173,15 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile Intel MKL CPU
       run: |
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
+        ml tbb
+        ml compiler-rt
+        ml oclfpga
+        ml compiler
+        ml mpi
+        ml mkl
         export FC=mpiifort
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl
@@ -158,7 +194,15 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile IntelLLVM MKL CPU
       run: |
-        module load intel
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
+        ml tbb
+        ml compiler-rt
+        ml oclfpga
+        ml compiler
+        ml mpi
+        ml mkl
         export export FC=mpiifx
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl
@@ -171,7 +215,15 @@ jobs:
     # Configure-Build-Run-Run on 4 cores 
     - name: Compile IntelLLVM MKL ADIOS2 CPU
       run: |
-        module load intel
+        ls -ltr ~/.bashrc
+        source /etc/profile.d/lmod.sh
+        source ~/.bash_aliases
+        ml tbb
+        ml compiler-rt
+        ml oclfpga
+        ml compiler
+        ml mpi
+        ml mkl
         export FC=mpiifx
         export CXX=mpiicpx
         export CC=mpiicx

From 771d1aa79c6ffaa6489f1a99aaeb10a29a2ec1dd Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Tue, 12 Mar 2024 16:20:00 +0000
Subject: [PATCH 410/436] Update the Runner with the full CI

---
 .github/workflows/RunnerFullCI.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index a1c4db59..23a4cc63 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -46,7 +46,8 @@ jobs:
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
         source ~/.bash_aliases
-        module load nvhpc/23.1
+        #module load nvhpc/23.1
+        module load nvhpc
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx

From b646aaf8e3eefa5c540b5c0f49b1ef8067c0f1f2 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Wed, 13 Mar 2024 08:00:57 +0000
Subject: [PATCH 411/436] Add additional libraries to Fast Runner

---
 .github/workflows/RunnerFastCI.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index 71eab721..8cbcba54 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -62,8 +62,7 @@ jobs:
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
-        #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_gcc12_cpu/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_foss2023b/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
         cmake -S . -B build -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 

From 43f117bb9824c59d8aadf99d7775e7b9d52def31 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Fri, 15 Mar 2024 09:12:48 +0000
Subject: [PATCH 412/436] Add additional libraries to Full Runner

---
 .github/workflows/RunnerFullCI.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index 23a4cc63..27c5a20e 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -46,13 +46,11 @@ jobs:
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
         source ~/.bash_aliases
-        #module load nvhpc/23.1
         module load nvhpc
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
-        #cmake -S . -B build -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_nvhpc_cpu/opt/lib/cmake/adios2
-        cmake -S . -B build -DBUILD_TESTING=ON 
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_nvhpc241/opt/lib/cmake/adios2
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -163,8 +161,7 @@ jobs:
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
-        #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_gcc12_cpu/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_foss2023b/opt/lib/cmake/adios2 -DFFT_Choice=fftw_f03
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
@@ -229,9 +226,8 @@ jobs:
         export CXX=mpiicpx
         export CC=mpiicx
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
-        #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT_Work/ADIOS2/build_install_ifx_cpu/opt/lib/cmake/adios2 -DFFT_Choice=mkl
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_intelLLVM/opt/lib/cmake/adios2 -DFFT_Choice=mkl
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=mkl
-        cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 
         ctest --test-dir build

From 49239d99fc07413ddb991aa409fcffd673c9cee7 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Fri, 15 Mar 2024 13:18:43 +0000
Subject: [PATCH 413/436] Add additional libraries to Full Runner

---
 .github/workflows/RunnerFullCI.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index 27c5a20e..760f0bd2 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -50,7 +50,8 @@ jobs:
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_nvhpc241/opt/lib/cmake/adios2
+        #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_nvhpc241/opt/lib/cmake/adios2
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 

From 3faf24b818c48baf7c385d077976caa3dfb7e3e1 Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Fri, 15 Mar 2024 13:30:17 +0000
Subject: [PATCH 414/436] Add additional libraries to Full Runner

---
 .github/workflows/RunnerFullCI.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index 760f0bd2..7dfb3a7f 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -51,7 +51,7 @@ jobs:
         export CC=mpicc
         export CXX=mpicxx
         #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_nvhpc241/opt/lib/cmake/adios2
-        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON 
+        cmake -S . -B build -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_nvhpc241/opt/lib/cmake/adios2
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 

From 16ab7dba88c70453feea2afe01901a622503fa3c Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Fri, 15 Mar 2024 13:47:06 +0000
Subject: [PATCH 415/436] Add additional libraries to Full Runner

---
 .github/workflows/RunnerFullCI.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index 7dfb3a7f..e033cf54 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -143,7 +143,7 @@ jobs:
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
-        export caliper_DIR=~/GIT_Work/caliper_github/build_gnu/opt/share/cmake/caliper/
+        export caliper_DIR=~/GIT/caliper_github/build_gnu/opt/share/cmake/caliper/
         #cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03 -DENABLE_PROFILER=caliper
         cmake -S . -B build -DBUILD_TESTING=ON -DFFT_Choice=fftw_f03 
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4

From 00544a3e302e8fe2ae7ccbd4168bc50930304edf Mon Sep 17 00:00:00 2001
From: rfj82982 <stefano.rolfo@stfc.ac.uk>
Date: Fri, 15 Mar 2024 14:24:44 +0000
Subject: [PATCH 416/436] Add additional libraries to Full Runner. For NVHPC
 Dev revert to 23.9

---
 .github/workflows/RunnerFullCI.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index e033cf54..451772c4 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -39,19 +39,18 @@ jobs:
         rm -rf build  
     
     # Run and Compile NHHPC for CPU with ADIOS2
-    # Need to fix issue with latest NVHPC and ADIOS
-    # Need to revert to 23.1
+    # Need to fix issue with DEV build and latest NVHPC
+    # Need to revert to 23.9
     - name: Compile NHVHPC CPU & ADIOS2
       run: |
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
         source ~/.bash_aliases
-        module load nvhpc
+        module load nvhpc/23.9
         export FC=mpif90
         export CC=mpicc
         export CXX=mpicxx
-        #cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_nvhpc241/opt/lib/cmake/adios2
-        cmake -S . -B build -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_nvhpc241/opt/lib/cmake/adios2
+        cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DIO_BACKEND=adios2 -Dadios2_DIR=~/GIT/ADIOS2/build/build_nvhpc241/opt/lib/cmake/adios2
         cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 -DMPIEXEC_MAX_NUMPROCS=4
         cmake --build build -j 4
         cmake --install build 

From 8c9e50549efc53e379d2f08b8c10963224cfbfef Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 16 Mar 2024 08:25:43 +0100
Subject: [PATCH 417/436] Fix memory leak introduced in
 https://github.com/2decomp-fft/2decomp-fft/pull/321

---
 src/fft_common.f90 | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/fft_common.f90 b/src/fft_common.f90
index 87c612f0..12c6650d 100644
--- a/src/fft_common.f90
+++ b/src/fft_common.f90
@@ -16,6 +16,7 @@
 
 ! Decomposition objects
 TYPE(DECOMP_INFO), pointer, save :: ph => null()  ! physical space
+TYPE(DECOMP_INFO), target, save :: ph_target  ! ph => ph_target or ph => decomp_main
 TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
 
 ! Workspace to store the intermediate Y-pencil data
@@ -115,8 +116,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
    if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
       ph => decomp_main
    else
-      if (.not. associated(ph)) allocate (ph)
-      call decomp_info_init(nx, ny, nz, ph)
+      call decomp_info_init(nx, ny, nz, ph_target)
+      ph => ph_target
    end if
    if (format == PHYSICAL_IN_X) then
       call decomp_info_init(nx / 2 + 1, ny, nz, sp)
@@ -185,7 +186,7 @@ subroutine decomp_2d_fft_finalize
 #endif
 
    if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
-      call decomp_info_finalize(ph)
+      call decomp_info_finalize(ph_target)
    end if
    nullify (ph)
    call decomp_info_finalize(sp)

From 3d3de19147b84f9622259b941dc119c571670b08 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 16 Mar 2024 08:26:08 +0100
Subject: [PATCH 418/436] Apply the fix to fft_fftw3_f03

---
 src/fft_fftw3_f03.f90 | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/fft_fftw3_f03.f90 b/src/fft_fftw3_f03.f90
index c1704ec0..5057cc0d 100644
--- a/src/fft_fftw3_f03.f90
+++ b/src/fft_fftw3_f03.f90
@@ -45,6 +45,7 @@ module decomp_2d_fft
 
    ! Decomposition objects
    TYPE(DECOMP_INFO), pointer, save :: ph => null()  ! physical space
+   TYPE(DECOMP_INFO), target, save :: ph_target ! ph => ph_target or ph => decomp_main
    TYPE(DECOMP_INFO), target, save :: sp  ! spectral space
 
    ! Workspace to store the intermediate Y-pencil data
@@ -143,8 +144,8 @@ subroutine fft_init_general(pencil, nx, ny, nz)
       if (nx_fft == nx_global .and. ny_fft == ny_global .and. nz_fft == nz_global) then
          ph => decomp_main
       else
-         if (.not. associated(ph)) allocate (ph)
-         call decomp_info_init(nx, ny, nz, ph)
+         call decomp_info_init(nx, ny, nz, ph_target)
+         ph => ph_target
       end if
       if (format == PHYSICAL_IN_X) then
          call decomp_info_init(nx / 2 + 1, ny, nz, sp)
@@ -209,8 +210,7 @@ subroutine decomp_2d_fft_finalize
 #endif
 
       if (nx_fft /= nx_global .or. ny_fft /= ny_global .or. nz_fft /= nz_global) then
-         call decomp_info_finalize(ph)
-         deallocate (ph)
+         call decomp_info_finalize(ph_target)
       end if
       nullify (ph)
       call decomp_info_finalize(sp)

From 17d0bc3a5d43ec386c63f80019d9329239a62ec1 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 16 Mar 2024 08:26:18 +0100
Subject: [PATCH 419/436] Avoid unused variables

---
 examples/io_test/io_plane_test.f90 | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/examples/io_test/io_plane_test.f90 b/examples/io_test/io_plane_test.f90
index 8a04e21d..ab296313 100644
--- a/examples/io_test/io_plane_test.f90
+++ b/examples/io_test/io_plane_test.f90
@@ -1,4 +1,7 @@
 !! SPDX-License-Identifier: BSD-3-Clause
+
+#define unused(x) associate(tmp => x); end associate
+
 !!
 !! FIXME The issue below is specific to GPU and should be discussed in a dedicated github issue
 !!
@@ -231,6 +234,11 @@ program io_plane_test
       end if
 
    end if
+#else
+   ! Avoid unused variables
+   unused(found)
+   unused(work)
+   unused(iol)
 #endif
 
    deallocate (u1, u2, u3)

From e0d56aae8916efe95f5af1b574e0eecfa828a942 Mon Sep 17 00:00:00 2001
From: cflag <cedric.flageul@univ-poitiers.fr>
Date: Sat, 16 Mar 2024 08:28:58 +0100
Subject: [PATCH 420/436] fprettify

---
 examples/fft_physical_x/fft_c2c_x.f90  | 9 ++++-----
 examples/fft_physical_x/fft_grid_x.f90 | 9 ++++-----
 examples/fft_physical_x/fft_r2c_x.f90  | 9 ++++-----
 examples/fft_physical_z/fft_c2c_z.f90  | 9 ++++-----
 examples/fft_physical_z/fft_r2c_z.f90  | 9 ++++-----
 examples/halo_test/halo_test.f90       | 2 +-
 src/decomp_2d_nccl.f90                 | 5 ++---
 7 files changed, 23 insertions(+), 29 deletions(-)

diff --git a/examples/fft_physical_x/fft_c2c_x.f90 b/examples/fft_physical_x/fft_c2c_x.f90
index e38d1d8a..9767e548 100644
--- a/examples/fft_physical_x/fft_c2c_x.f90
+++ b/examples/fft_physical_x/fft_c2c_x.f90
@@ -37,7 +37,6 @@ program fft_c2c_x
 #else
    real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
-   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -189,10 +188,10 @@ program fft_c2c_x
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
-   if (err_all > error_precision ) then
-      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
-      call decomp_2d_abort(1, "error for the FFT too large") 
-   endif 
+   if (err_all > error_precision) then
+      if (nrank == 0) write (*, *) 'error / mesh point: ', err_all
+      call decomp_2d_abort(1, "error for the FFT too large")
+   end if
 
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
diff --git a/examples/fft_physical_x/fft_grid_x.f90 b/examples/fft_physical_x/fft_grid_x.f90
index d399d492..2e6ff013 100644
--- a/examples/fft_physical_x/fft_grid_x.f90
+++ b/examples/fft_physical_x/fft_grid_x.f90
@@ -37,7 +37,6 @@ program fft_physical_x
 #else
    real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
-   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -190,10 +189,10 @@ program fft_physical_x
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
-   if (err_all > error_precision ) then
-      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
-      call decomp_2d_abort(1, "error for the FFT too large") 
-   endif 
+   if (err_all > error_precision) then
+      if (nrank == 0) write (*, *) 'error / mesh point: ', err_all
+      call decomp_2d_abort(1, "error for the FFT too large")
+   end if
 
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
diff --git a/examples/fft_physical_x/fft_r2c_x.f90 b/examples/fft_physical_x/fft_r2c_x.f90
index 92ddb81b..173c6582 100644
--- a/examples/fft_physical_x/fft_r2c_x.f90
+++ b/examples/fft_physical_x/fft_r2c_x.f90
@@ -38,7 +38,6 @@ program fft_r2c_x
 #else
    real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
-   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -187,10 +186,10 @@ program fft_r2c_x
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
-   if (err_all > error_precision ) then
-      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
-      call decomp_2d_abort(1, "error for the FFT too large") 
-   endif 
+   if (err_all > error_precision) then
+      if (nrank == 0) write (*, *) 'error / mesh point: ', err_all
+      call decomp_2d_abort(1, "error for the FFT too large")
+   end if
 
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
diff --git a/examples/fft_physical_z/fft_c2c_z.f90 b/examples/fft_physical_z/fft_c2c_z.f90
index 5442d6fe..4c8b7181 100644
--- a/examples/fft_physical_z/fft_c2c_z.f90
+++ b/examples/fft_physical_z/fft_c2c_z.f90
@@ -37,7 +37,6 @@ program fft_c2c_z
 #else
    real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
-   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -190,10 +189,10 @@ program fft_c2c_z
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
-   if (err_all > error_precision ) then
-      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
-      call decomp_2d_abort(1, "error for the FFT too large") 
-   endif 
+   if (err_all > error_precision) then
+      if (nrank == 0) write (*, *) 'error / mesh point: ', err_all
+      call decomp_2d_abort(1, "error for the FFT too large")
+   end if
 
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
diff --git a/examples/fft_physical_z/fft_r2c_z.f90 b/examples/fft_physical_z/fft_r2c_z.f90
index eeedbab3..f1e46057 100644
--- a/examples/fft_physical_z/fft_r2c_z.f90
+++ b/examples/fft_physical_z/fft_r2c_z.f90
@@ -38,7 +38,6 @@ program fft_r2c_z
 #else
    real(mytype), parameter :: error_precision = 5.e-6_mytype
 #endif
-   
 
    call MPI_INIT(ierror)
    ! To resize the domain we need to know global number of ranks
@@ -188,10 +187,10 @@ program fft_r2c_z
    call MPI_ALLREDUCE(error, err_all, 1, real_type, MPI_SUM, MPI_COMM_WORLD, ierror)
    err_all = err_all / (real(nx, mytype) * real(ny, mytype) * real(nz, mytype))
 
-   if (err_all > error_precision ) then
-      if (nrank == 0 ) write (*, *) 'error / mesh point: ', err_all 
-      call decomp_2d_abort(1, "error for the FFT too large") 
-   endif 
+   if (err_all > error_precision) then
+      if (nrank == 0) write (*, *) 'error / mesh point: ', err_all
+      call decomp_2d_abort(1, "error for the FFT too large")
+   end if
 
    if (nrank == 0) then
       write (*, *) 'error / mesh point: ', err_all
diff --git a/examples/halo_test/halo_test.f90 b/examples/halo_test/halo_test.f90
index edaa16cb..fa9cdd6a 100644
--- a/examples/halo_test/halo_test.f90
+++ b/examples/halo_test/halo_test.f90
@@ -533,7 +533,7 @@ subroutine check_err(divh, divref, pencil)
       !$acc end kernels
       divmag = mag(tmp)
 
-      if (error < real(2.0,mytype) * epsilon(divmag) * divmag) then
+      if (error < real(2.0, mytype) * epsilon(divmag) * divmag) then
          passing = .true.
       else
          passing = .false.
diff --git a/src/decomp_2d_nccl.f90 b/src/decomp_2d_nccl.f90
index f63f0cba..db948302 100644
--- a/src/decomp_2d_nccl.f90
+++ b/src/decomp_2d_nccl.f90
@@ -105,7 +105,6 @@ subroutine decomp_2d_nccl_fin()
       if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclCommDestroy")
       cuda_stat = cudaStreamDestroy(cuda_stream_2decomp)
       if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamDestroy")
-      
 
    end subroutine decomp_2d_nccl_fin
    !
@@ -142,7 +141,7 @@ subroutine decomp_2d_nccl_send_recv_real_col(dst_d, &
       end do
       nccl_stat = ncclGroupEnd()
       if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-      cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)     
+      cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
       if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 
    end subroutine decomp_2d_nccl_send_recv_real_col
@@ -239,7 +238,7 @@ subroutine decomp_2d_nccl_send_recv_real_row(dst_d, &
       end do
       nccl_stat = ncclGroupEnd()
       if (nccl_stat /= ncclSuccess) call decomp_2d_abort(__FILE__, __LINE__, nccl_stat, "ncclGroupEnd")
-      cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)     
+      cuda_stat = cudaStreamSynchronize(cuda_stream_2decomp)
       if (cuda_stat /= 0) call decomp_2d_abort(__FILE__, __LINE__, cuda_stat, "cudaStreamSynchronize")
 
    end subroutine decomp_2d_nccl_send_recv_real_row

From 74ca344a0b6936a48aa2d7bfc56cd97066ba9d3a Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Fri, 22 Mar 2024 16:06:13 +0000
Subject: [PATCH 421/436] Update README.md

---
 README.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index e44c5c67..5e1210c4 100644
--- a/README.md
+++ b/README.md
@@ -9,15 +9,15 @@ Please have a look at [HOWTO.md](HOWTO.md) and at the examples [examples](exampl
 
 The build system is driven by `cmake`. It is good practice to directly point to the MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. This can be done by setting the default Fortran environmental variable 
 ```
-export FC=my_mpif90
+$ export FC=my_mpif90
 ```
 The build system can then be generated by running
 ```
-cmake -S $path_to_sources -B $path_to_build_directory -DOPTION1 -DOPTION2 ...
+$ cmake -S $path_to_sources -B $path_to_build_directory -DOPTION1 -DOPTION2 ...
 ```
 for many users a configuration line
 ```
-cmake -S . -B build
+$ cmake -S . -B build
 ```
 run from the 2DECOMP&FFT root directory will be sufficient.
 If the build directory does not exist it will be generated and it will contain the configuration files.
@@ -26,7 +26,7 @@ with 2DECOMP&FFT, please see [INSTALL.md](INSTALL.md) for instructions on changi
 
 Once the build system has been configured, you can build 2DECOMP&FFT by running
 ```
-cmake --build $path_to_build_directory -j <nproc>
+$ cmake --build $path_to_build_directory -j <nproc>
 ```
 appending `-v` will display additional information about the build, such as compiler flags.
 
@@ -34,14 +34,14 @@ After building the library can be tested. Please see section [Testing and exampl
 
 Finally, the build library can be installed by running 
 ```
-cmake --install $path_to_build_directory
+$ cmake --install $path_to_build_directory
 ```
 The default location for `libdecomp2d.a` is `$path_to_build_directory/opt/lib`or  `$path_to_build_directory/opt/lib64` unless the variable `CMAKE_INSTALL_PREFIX` is modified.
 The module files generated by the build process will similarly be installed to `$path_to_build_directory/opt/install`, users of the library should add this to the include paths for their program.
 
 Occasionally a clean build is required, this can be performed by running
 ```
-cmake --build $path_to_build_directory --target clean
+$ cmake --build $path_to_build_directory --target clean
 ```
 
 ### GPU compilation
@@ -60,7 +60,7 @@ To activate the testing the option `-DBUILD_TESTING=ON` can be added or
 alternativey the option can be activated in the GUI interface `ccmake`.
 After building the library can be tested by running
 ```
-ctest --test-dir $path_to_build_directory
+$ ctest --test-dir $path_to_build_directory
 ```
 which uses the `ctest` utility. By default tests are performed in serial, 
 but more than 1 rank can be used by setting `MPIEXEC_MAX_NUMPROCS` under `ccmake` utility.

From 86168ef3e2c0e9d88920386374d5a62f7e2f6624 Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Fri, 22 Mar 2024 16:13:20 +0000
Subject: [PATCH 422/436] Update INSTALL.md

---
 INSTALL.md | 77 +++++++++++++++++++++++++++---------------------------
 1 file changed, 39 insertions(+), 38 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 1b0bbfc1..70af86f5 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -16,17 +16,17 @@ The build system is driven by `cmake`. It is good practice to directly point to
 MPI Fortran wrapper that you would like to use to guarantee consistency between Fortran compiler and MPI. 
 This can be done by setting the default Fortran environmental variable 
 ```
-export FC=my_mpif90
+$ export FC=my_mpif90
 ```
 To generate the build system run
 ```
-cmake -S $path_to_sources -B $path_to_build_directory -DOPTION1 -DOPTION2 ...
+$ cmake -S $path_to_sources -B $path_to_build_directory -DOPTION1 -DOPTION2 ...
 ```
 If the directory does not exist it will be generated and it will contain the configuration files. 
 The configuration can be further
 edited by using the `ccmake` utility as
 ```
-ccmake $path_to_build_directory
+$ ccmake $path_to_build_directory
 ```
 and editing as desired, variables that are likely of interest are: `CMAKE_BUILD_TYPE` and `FFT_Choice`;
 additional variables can be shown by entering "advanced mode" by pressing `t`.
@@ -42,7 +42,7 @@ Please see section [GPU Compilation](#gpu-compilation)
 
 Once the build system has been configured, you can build 2DECOMP&FFT` by running
 ```
-cmake --build $path_to_build_directory -j <nproc>
+$ cmake --build $path_to_build_directory -j <nproc>
 ```
 appending `-v` will display additional information about the build, such as compiler flags.
 
@@ -50,7 +50,7 @@ After building the library can be tested. Please see section [Testing and exampl
 
 Options can be added to change the level of verbosity. Finally, the build library can be installed by running 
 ```
-cmake --install $path_to_build_directory
+$ cmake --install $path_to_build_directory
 ```
 The default location for `libdecomp2d.a` is `$path_to_build_directory/opt/lib` 
 or `$path_to_build_directory/opt/lib64` unless the variable `CMAKE_INSTALL_PREFIX` is modified.
@@ -61,14 +61,14 @@ users of the library should add this to the include paths for their program.
 As indicated above, by default a static `libdecomp2d.a` will be compiled, 
 if desired a shared library can be built by setting `BUILD_SHARED_LIBS=ON` either on the command line:
 ```
-cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_SHARED_LIBS=ON
+$ cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_SHARED_LIBS=ON
 ```
 or by editing the configuration using `ccmake`.
 This might be useful for a centralised install supporting multiple users that is upgraded over time.
 
 Occasionally a clean build is required, this can be performed by running
 ```
-cmake --build $path_to_build_directory --target clean
+$ cmake --build $path_to_build_directory --target clean
 ```
 
 ## GPU compilation
@@ -79,7 +79,7 @@ The FFT is based on cuFFT.
 
 To properly configure for GPU build the following needs to be used 
 ```
-cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_TARGET=gpu
+$ cmake -S $path_to_sources -B $path_to_build_directory -DBUILD_TARGET=gpu
 ```
 Note, further configuration can be performed using `ccmake`, however the initial configuration of GPU builds must include the `-DBUILD_TARGET=gpu` flag as shown above.
 
@@ -165,9 +165,9 @@ clean-decomp:
 Profiling can be activated via `cmake` configuration, 
 the recommended approach is to run the initial configuration as follows:
 ```
-export caliper_DIR=/path/to/caliper/install/share/cmake/caliper
-export CXX=mpicxx
-cmake -S $path_to_sources -B $path_to_build_directory -DENABLE_PROFILER=caliper
+$ export caliper_DIR=/path/to/caliper/install/share/cmake/caliper
+$ export CXX=mpicxx
+$ cmake -S $path_to_sources -B $path_to_build_directory -DENABLE_PROFILER=caliper
 ```
 where `ENABLE_PROFILER` is set to the profiling tool desired, currently supported values are: `caliper`.
 Note that when using `caliper` a C++ compiler is required as indicated in the above command line.
@@ -178,11 +178,11 @@ Note that when using `caliper` a C++ compiler is required as indicated in the ab
 
 In order to compile with the MKL libraries the following environmental variable needs to be set up
 ```
-export MKL_DIR=${MKLROOT}/lib/cmake/mkl
+$ export MKL_DIR=${MKLROOT}/lib/cmake/mkl
 ```
 and select the MKL backend by setting `FFT_Choice=mkl`.
 
-To use the new IntelLLVM compiler specify it as the Fortran compiler using
+To use the new IntelLLVM compiler, up until the 2023 version, specify it as the Fortran compiler using
 ```
 export export FC="mpiifort -fc=ifx"
 ```
@@ -191,6 +191,7 @@ and when building with ADIOS2 support you must also specify the `C` and `CXX` co
 export CXX="mpiicpc -cxx=icpx"
 export CC="mpiicc -cc=icx"
 ```
+From the 2024 version new MPI wrapper are available as `mpiifx`, `mpiicx` and `mpiicpx`. 
 
 ### List of preprocessor variables
 
@@ -245,13 +246,13 @@ This variable is valid only for GPU builds. The NVIDIA Collective Communication
 The library [adios2](https://adios2.readthedocs.io/en/latest/) can be used as a backend for IO. The version 2.9.0 was tested, is supported and can be downloaded [here](https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.9.0.tar.gz). Below are build instructions for the library. However, it is recommended to use the one provided by the administrators of the computing centre if available.
 
 ```
-wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.9.0.tar.gz
-tar xzf v2.9.0.tar.gz
-mkdir 2.9.0_tmp && cd 2.9.0_tmp
-CC=mpicc CXX=mpicxx FC=mpif90 cmake -S ../ADIOS2-2.9.0 -DCMAKE_INSTALL_PREFIX=../2.9.0_bld
-make -j
-make -j test
-make -j install
+$ wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.9.0.tar.gz
+$ tar xzf v2.9.0.tar.gz
+$ mkdir 2.9.0_tmp && cd 2.9.0_tmp
+$ CC=mpicc CXX=mpicxx FC=mpif90 cmake -S ../ADIOS2-2.9.0 -DCMAKE_INSTALL_PREFIX=../2.9.0_bld
+$ make -j
+$ make -j test
+$ make -j install
 ```
 
 To build 2DECOMP&FFT with the adios2 IO backend, one can provide the package configuration for adios2 
@@ -260,7 +261,7 @@ this should be found under `/path/to/adios2/install/lib/cmake/adios2`.
 One can also provide the option `-Dadios2_DIR=/path/to/adios2/install/lib/cmake/adios2`. 
 Then either specify on the command line when configuring the build
 ```
-cmake -S . -B ./build -DIO_BACKEND=adios2 -Dadios2_DIR=/path/to/adios2/install/lib/cmake/adios2
+$ cmake -S . -B ./build -DIO_BACKEND=adios2 -Dadios2_DIR=/path/to/adios2/install/lib/cmake/adios2
 ```
 or modify the build configuration using `ccmake`. Please note that the support for ADIOS2 is not complete. Currently, for a given IO operation, when the ADIOS2 backend is not supported, the MPI backend is used.
 
@@ -275,13 +276,13 @@ Below is a suggestion for the compilation of the library in double precision
 (add `--enable-single` for a single precision build):
 
 ```
-wget http://www.fftw.org/fftw-3.3.10.tar.gz
-tar xzf fftw-3.3.10.tar.gz
-mkdir fftw-3.3.10_tmp && cd fftw-3.3.10_tmp
-../fftw-3.3.10/configure --prefix=xxxxxxx/fftw3/fftw-3.3.10_bld --enable-shared
-make -j
-make -j check
-make install
+$ wget http://www.fftw.org/fftw-3.3.10.tar.gz
+$ tar xzf fftw-3.3.10.tar.gz
+$ mkdir fftw-3.3.10_tmp && cd fftw-3.3.10_tmp
+$ ../fftw-3.3.10/configure --prefix=xxxxxxx/fftw3/fftw-3.3.10_bld --enable-shared
+$ make -j
+$ make -j check
+$ make install
 ```
 Please note that the resulting build is not compatible with `cmake` 
 (https://github.com/FFTW/fftw3/issues/130). 
@@ -298,7 +299,7 @@ this should be found under `/path/to/fftw3/install/lib/pkgconfig`.
 One can also provide the option `-DFFTW_ROOT=/path/to/fftw3/install`. 
 Then either specify on the command line when configuring the build
 ```
-cmake -S . -B build -DFFT_Choice=<fftw|fftw_f03> -DFFTW_ROOT=/path/to/fftw3/install
+$ cmake -S . -B build -DFFT_Choice=<fftw|fftw_f03> -DFFTW_ROOT=/path/to/fftw3/install
 ```
 or modify the build configuration using `ccmake`.
 
@@ -319,19 +320,19 @@ and [here](https://software.llnl.gov/Caliper/CaliperBasics.html#build-and-instal
 Below is a suggestion for the compilation of the library using the GNU compilers:
 
 ```
-git clone https://github.com/LLNL/Caliper.git caliper_github
-cd caliper_github
-git checkout v2.9.1
-mkdir build && cd build
-cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../caliper_build_2.9.1 -DWITH_FORTRAN=yes -DWITH_MPI=yes -DBUILD_TESTING=yes ../
-make -j
-make test
-make install
+$ git clone https://github.com/LLNL/Caliper.git caliper_github
+$ cd caliper_github
+$ git checkout v2.9.1
+$ mkdir build && cd build
+$ cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=../../caliper_build_2.9.1 -DWITH_FORTRAN=yes -DWITH_MPI=yes -DBUILD_TESTING=yes ../
+$ make -j
+$ make test
+$ make install
 ```
 
 After installing Caliper ensure to set `caliper_DIR=/path/to/caliper/install/share/cmake/caliper`.
 Following this the `2decomp-fft` build can be configured to use Caliper profiling as
 ```
-cmake -S . -B -DENABLE_PROFILER=caliper
+$ cmake -S . -B -DENABLE_PROFILER=caliper
 ```
 or by modifying the configuration to set `ENABLE_PROFILER=caliper` via `ccmake`.

From ded2345a1152c08d23a3337bcf0e1214bcca5659 Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Fri, 22 Mar 2024 16:13:48 +0000
Subject: [PATCH 423/436] Update CMakeLists.txt

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ddda700c..5cf2b10b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,7 +3,7 @@ cmake_policy(SET CMP0074 NEW)
 
 project(decomp2d
   LANGUAGES Fortran)
-set(version 2.0.0)
+set(version 2.0.3)
 enable_testing()
 if (IO_BACKEND MATCHES "adios2")
   # Can be useful to also activate CXX, sometimes is needed by packages

From 81dffa507629302828c228b90d4d13e1aacf6ab9 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Thu, 4 Apr 2024 11:22:03 +0200
Subject: [PATCH 424/436] cherry-pick PR #325 (#326)

* Update github action

* Update Build_all.yml
---
 .github/workflows/Build.yml     |  2 +-
 .github/workflows/Build_all.yml | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml
index 29210335..c9eb720c 100644
--- a/.github/workflows/Build.yml
+++ b/.github/workflows/Build.yml
@@ -18,7 +18,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Runs a single command using the runners shell
     - name: Install requirements
diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 41446f06..f9a66fb2 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -18,7 +18,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Runs a single command using the runners shell
     - name: Install requirements
@@ -72,7 +72,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Runs a single command using the runners shell
     - name: Install requirements
@@ -97,7 +97,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Add Nvidia HPC SDK
     - name: setup repo
@@ -129,7 +129,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Add Intel OneApi
     - name: setup repo
@@ -166,7 +166,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Runs a single command using the runners shell
     - name: Install requirements
@@ -195,7 +195,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Runs a single command using the runners shell
     - name: Install requirements

From 7bd46f4c61299e8c55ce1a5cb64091591bf27ebe Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Mon, 22 Apr 2024 20:23:00 +0200
Subject: [PATCH 425/436] Fix 3D c2c FFT for the MKL backend

---
 src/fft_mkl.f90 | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/fft_mkl.f90 b/src/fft_mkl.f90
index 0d58aaa3..da064e46 100644
--- a/src/fft_mkl.f90
+++ b/src/fft_mkl.f90
@@ -373,13 +373,13 @@ subroutine fft_3d_c2c(in, out, isign)
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
-         do k = 1, ph%xsz(3)
+         do k = 1, ph%ysz(3)
             status = wrapper_c2c_inplace(c2c_y, wk2_c2c(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
-         do k = 1, ph%xsz(3) ! one Z-plane at a time
+         do k = 1, ph%ysz(3) ! one Z-plane at a time
             !          if (isign==DECOMP_2D_FFT_FORWARD) then
             !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
             !          else if (isign==DECOMP_2D_FFT_BACKWARD) then
@@ -438,13 +438,13 @@ subroutine fft_3d_c2c(in, out, isign)
 
          ! ===== 1D FFTs in Y =====
 #ifdef OVERWRITE
-         do k = 1, ph%xsz(3)
+         do k = 1, ph%ysz(3)
             status = wrapper_c2c_inplace(c2c_y, wk2_c2c(1, 1, k), isign)
             if (status /= 0) call decomp_2d_abort(__FILE__, __LINE__, status, "wrapper_c2c")
          end do
 #else
          allocate (wk2b(ph%ysz(1), ph%ysz(2), ph%ysz(3)))
-         do k = 1, ph%xsz(3) ! one Z-plane at a time
+         do k = 1, ph%ysz(3) ! one Z-plane at a time
             !          if (isign==DECOMP_2D_FFT_FORWARD) then
             !             status = DftiComputeForward(c2c_y, wk2(:,1,k), wk2b(:,1,k))
             !          else if (isign==DECOMP_2D_FFT_BACKWARD) then

From 5428973d902cc67f4ecf4b7c50b8d8a670f62832 Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <ptb0890@gmail.com>
Date: Tue, 8 Oct 2024 20:40:34 +0100
Subject: [PATCH 426/436] The exchange routines need the device attribute

---
 src/halo.f90 | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/src/halo.f90 b/src/halo.f90
index fb018fdf..ccb98594 100644
--- a/src/halo.f90
+++ b/src/halo.f90
@@ -111,6 +111,9 @@ subroutine exchange_halo_x_real_short(inout, opt_xlevel)
 
      real(mytype), dimension(:, :, :), intent(INOUT) :: inout
      integer, dimension(3), optional :: opt_xlevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      call exchange_halo_x_real(inout, decomp_main, opt_xlevel)
 
@@ -123,6 +126,9 @@ subroutine exchange_halo_x_real(inout, decomp, opt_xlevel)
      real(mytype), dimension(:, :, :), intent(INOUT) :: inout
      TYPE(DECOMP_INFO), intent(in) :: decomp
      integer, dimension(3), optional :: opt_xlevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      integer :: level_x, level_y, level_z
      integer :: ierror
@@ -147,6 +153,9 @@ subroutine exchange_halo_x_complex_short(inout, opt_xlevel)
 
      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
      integer, dimension(3), optional :: opt_xlevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      call exchange_halo_x_complex(inout, decomp_main, opt_xlevel)
 
@@ -159,6 +168,9 @@ subroutine exchange_halo_x_complex(inout, decomp, opt_xlevel)
      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
      TYPE(DECOMP_INFO), intent(in) :: decomp
      integer, dimension(3), optional :: opt_xlevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      integer :: level_x, level_y, level_z
      integer :: ierror
@@ -183,6 +195,9 @@ subroutine exchange_halo_y_real_short(inout, opt_ylevel)
 
      real(mytype), dimension(:, :, :), intent(INOUT) :: inout
      integer, dimension(3), optional :: opt_ylevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      call exchange_halo_y_real(inout, decomp_main, opt_ylevel)
 
@@ -195,6 +210,9 @@ subroutine exchange_halo_y_real(inout, decomp, opt_ylevel)
      real(mytype), dimension(:, :, :), intent(INOUT) :: inout
      TYPE(DECOMP_INFO), intent(in) :: decomp
      integer, dimension(3), optional :: opt_ylevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      integer :: level_x, level_y, level_z
      integer :: ierror
@@ -219,6 +237,9 @@ subroutine exchange_halo_y_complex_short(inout, opt_ylevel)
 
      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
      integer, dimension(3), optional :: opt_ylevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      call exchange_halo_y_complex(inout, decomp_main, opt_ylevel)
 
@@ -231,6 +252,9 @@ subroutine exchange_halo_y_complex(inout, decomp, opt_ylevel)
      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
      TYPE(DECOMP_INFO), intent(in) :: decomp
      integer, dimension(3), optional :: opt_ylevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      integer :: level_x, level_y, level_z
      integer :: ierror
@@ -253,6 +277,9 @@ subroutine exchange_halo_z_real_short(inout, opt_zlevel)
      implicit none
      real(mytype), dimension(:, :, :), intent(INOUT) :: inout
      integer, dimension(3), optional :: opt_zlevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      call exchange_halo_z_real(inout, decomp_main, opt_zlevel)
   end subroutine exchange_halo_z_real_short
@@ -262,6 +289,9 @@ subroutine exchange_halo_z_real(inout, decomp, opt_zlevel)
      real(mytype), dimension(:, :, :), intent(INOUT) :: inout
      TYPE(DECOMP_INFO), intent(in) :: decomp
      integer, dimension(3), optional :: opt_zlevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      integer :: level_x, level_y, level_z
      integer :: ierror
@@ -286,6 +316,9 @@ subroutine exchange_halo_z_complex_short(inout, opt_zlevel)
 
      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
      integer, dimension(3), optional :: opt_zlevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      call exchange_halo_z_complex(inout, decomp_main, opt_zlevel)
   end subroutine exchange_halo_z_complex_short
@@ -297,6 +330,9 @@ subroutine exchange_halo_z_complex(inout, decomp, opt_zlevel)
      complex(mytype), dimension(:, :, :), intent(INOUT) :: inout
      TYPE(DECOMP_INFO), intent(in) :: decomp
      integer, dimension(3), optional :: opt_zlevel
+#if defined(_GPU)
+     attributes(device) :: inout
+#endif
 
      integer :: level_x, level_y, level_z
      integer :: ierror

From d55b66677d8ca52993228d67d9ecd152c2013fe3 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Fri, 8 Nov 2024 13:42:30 +0100
Subject: [PATCH 427/436] Fix IO for reduced precision and 3D array in y / z
 pencil

---
 src/io.f90 | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/io.f90 b/src/io.f90
index 7ea699c3..a8152449 100644
--- a/src/io.f90
+++ b/src/io.f90
@@ -339,7 +339,9 @@ subroutine read_one_real(ipencil, var, dirname, varname, io_name, opt_decomp, re
                                 newtype, 'native', MPI_INFO_NULL, ierror)
          if (ierror /= 0) call decomp_2d_abort(__FILE__, __LINE__, ierror, "MPI_FILE_SET_VIEW")
          if (read_reduce_prec) then
-            allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
+            allocate (varsingle(starts(1):(starts(1)+subsizes(1)), &
+                                starts(2):(starts(2)+subsizes(2)), &
+                                starts(3):(starts(3)+subsizes(3))))
             call MPI_FILE_READ_ALL(fh, varsingle, &
                                    subsizes(1) * subsizes(2) * subsizes(3), &
                                    data_type, MPI_STATUS_IGNORE, ierror)
@@ -1263,7 +1265,9 @@ subroutine mpiio_write_real_coarse(ipencil, var, dirname, varname, icoarse, io_n
          call coarse_extents(ipencil, icoarse, sizes, subsizes, starts)
       end if
       if (write_reduce_prec) then
-         allocate (varsingle(xstV(1):xenV(1), xstV(2):xenV(2), xstV(3):xenV(3)))
+         allocate (varsingle(starts(1):(starts(1)+subsizes(1)), &
+                             starts(2):(starts(2)+subsizes(2)), &
+                             starts(3):(starts(3)+subsizes(3))))
          varsingle = real(var, mytype_single)
       end if
 

From 7d4b986485ce082ce4f4b04aeac826dc6025b18d Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Fri, 8 Nov 2024 13:48:11 +0000
Subject: [PATCH 428/436] Update RunnerFullCI.yml

---
 .github/workflows/RunnerFullCI.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index 451772c4..d1140161 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -174,6 +174,7 @@ jobs:
         ls -ltr ~/.bashrc
         source /etc/profile.d/lmod.sh
         source ~/.bash_aliases
+        module avail 
         ml tbb
         ml compiler-rt
         ml oclfpga

From 057cf9405c2592821cbeeaa18d68ea8780548831 Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Fri, 8 Nov 2024 14:31:55 +0000
Subject: [PATCH 429/436] Update RunnerFastCI.yml

---
 .github/workflows/RunnerFastCI.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml
index 8cbcba54..80bce909 100644
--- a/.github/workflows/RunnerFastCI.yml
+++ b/.github/workflows/RunnerFastCI.yml
@@ -77,7 +77,7 @@ jobs:
         source ~/.bash_aliases
         ml tbb
         ml compiler-rt
-        ml oclfpga
+        ml umf
         ml compiler
         ml mpi
         ml mkl

From de950849671859e18fc30c684a180247a1d4c641 Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Fri, 8 Nov 2024 14:33:19 +0000
Subject: [PATCH 430/436] Update RunnerFullCI.yml

---
 .github/workflows/RunnerFullCI.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index d1140161..1d5c637d 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -177,7 +177,7 @@ jobs:
         module avail 
         ml tbb
         ml compiler-rt
-        ml oclfpga
+        ml umf
         ml compiler
         ml mpi
         ml mkl
@@ -198,7 +198,7 @@ jobs:
         source ~/.bash_aliases
         ml tbb
         ml compiler-rt
-        ml oclfpga
+        ml umf
         ml compiler
         ml mpi
         ml mkl
@@ -219,7 +219,7 @@ jobs:
         source ~/.bash_aliases
         ml tbb
         ml compiler-rt
-        ml oclfpga
+        ml umf
         ml compiler
         ml mpi
         ml mkl

From 7c43c8658991d2e8e2a3e05cf3c2533dfbb1f949 Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Fri, 8 Nov 2024 15:20:38 +0000
Subject: [PATCH 431/436] Update RunnerFullCI.yml

---
 .github/workflows/RunnerFullCI.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml
index 1d5c637d..2ae07096 100644
--- a/.github/workflows/RunnerFullCI.yml
+++ b/.github/workflows/RunnerFullCI.yml
@@ -181,6 +181,7 @@ jobs:
         ml compiler
         ml mpi
         ml mkl
+        ml ifort
         export FC=mpiifort
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON -DFFT_Choice=mkl

From 116967448f1eadd5aa9d4a600da0e3e35879df73 Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Mon, 13 Jan 2025 22:23:32 +0000
Subject: [PATCH 432/436] Fix main (#359)

* Fix CMake version to 3.20

* Add codafor module to fix cuda functions used by NCCL

* Update modules load for Intel Compiler
---
 CMakeLists.txt         | 2 +-
 src/decomp_2d_nccl.f90 | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5cf2b10b..b80fbd77 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.21)                        
+cmake_minimum_required(VERSION 3.20)                        
 cmake_policy(SET CMP0074 NEW)
 
 project(decomp2d
diff --git a/src/decomp_2d_nccl.f90 b/src/decomp_2d_nccl.f90
index db948302..620e83ef 100644
--- a/src/decomp_2d_nccl.f90
+++ b/src/decomp_2d_nccl.f90
@@ -8,6 +8,7 @@ module decomp_2d_nccl
    use decomp_2d_constants
    use decomp_2d_mpi
    use decomp_2d_cumpi
+   use cudafor 
    use nccl
 
    implicit none

From 80ccc0dde3deca1f3625fffb35ea2f5d38db62ea Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Tue, 14 Jan 2025 09:58:06 +0000
Subject: [PATCH 433/436] Fix main (#360)

* Fix CMake version to 3.20

* Add codafor module to fix cuda functions used by NCCL

* Update modules load for Intel Compiler

* Update CI for NVHPC and Intel
---
 .github/workflows/Build_all.yml | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index f9a66fb2..4bc350ae 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -91,9 +91,7 @@ jobs:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
     env:
-      TMP: /opt/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/mpi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/23.5/compilers/bin
-###      TMP: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/mpi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin
-
+      TMP: /opt/nvidia/hpc_sdk/Linux_x86_64/24.11/comm_libs/mpi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/24.11/compilers/bin
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -107,10 +105,7 @@ jobs:
     - name: install
       run: |
         sudo apt-get update  -y
-        sudo apt-get install -y nvhpc-23-5
-###        sudo apt-get install -y nvhpc-22-7
-###        sudo apt-get install -y nvhpc-22-7-cuda-multi
-
+        sudo apt-get install -y nvhpc-24-11
     # Runs a set of commands using the runners shell
     - name: CPU version
       run: |
@@ -145,7 +140,7 @@ jobs:
     - name: Compile
       run: |
         source /opt/intel/oneapi/setvars.sh
-        export FC=mpiifort
+        export FC=mpiifx
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DBUILD_TESTING=ON
         cmake --build build --verbose
 
@@ -153,7 +148,7 @@ jobs:
     - name: Compile
       run: |
         source /opt/intel/oneapi/setvars.sh
-        export FC=mpiifort
+        export FC=mpiifx
         export MKL_DIR=${MKLROOT}/lib/cmake/mkl
         rm -rf ./build
         cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DFFT_Choice=mkl -DBUILD_TESTING=ON

From 01d268a2a2b370d8782e05373fd157144afa5c88 Mon Sep 17 00:00:00 2001
From: CFLAG <mathrack@users.noreply.github.com>
Date: Tue, 14 Jan 2025 11:08:27 +0100
Subject: [PATCH 434/436] Fix caliper CI

---
 .github/workflows/Build_all.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/Build_all.yml b/.github/workflows/Build_all.yml
index 4bc350ae..d4057275 100644
--- a/.github/workflows/Build_all.yml
+++ b/.github/workflows/Build_all.yml
@@ -201,16 +201,16 @@ jobs:
     # Runs a set of commands using the runners shell
     - name: Compile caliper
       run: |
-        wget --no-check-certificate https://github.com/LLNL/Caliper/archive/refs/tags/v2.8.0.tar.gz
-        tar xzf v2.8.0.tar.gz && cd Caliper-2.8.0
+        wget --no-check-certificate https://github.com/LLNL/Caliper/archive/refs/tags/v2.12.1.tar.gz
+        tar xzf v2.12.1.tar.gz && cd Caliper-2.12.1
         mkdir build && cd build
         echo ${PWD}
-        cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=${HOME}/Caliper-2.8.0_bld -DWITH_FORTRAN=yes -DWITH_MPI=yes ../
+        cmake -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_INSTALL_PREFIX=${HOME}/Caliper-2.12.1_bld -DWITH_FORTRAN=yes -DWITH_MPI=yes ../
         make -j && make install
         cd ../../
 
     - name: Compile
       run: |
-        export caliper_DIR=${HOME}/Caliper-2.8.0_bld/share/cmake/caliper
+        export caliper_DIR=${HOME}/Caliper-2.12.1_bld/share/cmake/caliper
         FC=mpif90 cmake -S . -B build -DCMAKE_BUILD_TYPE=Dev -DENABLE_PROFILER=caliper -DBUILD_TESTING=ON
         cmake --build build --verbose

From 959c2ff706336d0f1793eb843bb2cc85d8636ac9 Mon Sep 17 00:00:00 2001
From: Stefano  Rolfo <35581400+rfj82982@users.noreply.github.com>
Date: Tue, 14 Jan 2025 14:53:31 +0000
Subject: [PATCH 435/436] Update version number in CMake file (#363)

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b80fbd77..8f2b359a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,7 +3,7 @@ cmake_policy(SET CMP0074 NEW)
 
 project(decomp2d
   LANGUAGES Fortran)
-set(version 2.0.3)
+set(version 2.0.4)
 enable_testing()
 if (IO_BACKEND MATCHES "adios2")
   # Can be useful to also activate CXX, sometimes is needed by packages

From 24f3a4d17b546bf4a76aa6b7036f9c0d503828ff Mon Sep 17 00:00:00 2001
From: Paul Bartholomew <p.bartholomew@epcc.ed.ac.uk>
Date: Wed, 5 Feb 2025 08:18:34 +0000
Subject: [PATCH 436/436] Correct logic error in checking for invalid ipencil
 input for halos

---
 src/halo_common.f90 | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/halo_common.f90 b/src/halo_common.f90
index 67692923..9053b958 100644
--- a/src/halo_common.f90
+++ b/src/halo_common.f90
@@ -3,11 +3,6 @@
 ! This file contain common code to be included by subroutines
 ! 'update_halo_...' in halo.f90
 
-    ! Check for invalid argument
-    if ((ipencil < 1) .or. (ipencil > 3)) then
-       call decomp_2d_abort(__FILE__, __LINE__, ipencil, "Invalid pencil for halo exchange, should be in range 1<=pencil<=3")
-    end if
-
     if (present(opt_global)) then
        global = opt_global
     else
@@ -20,6 +15,10 @@
 
     if (present(opt_pencil)) then
        ipencil = opt_pencil
+       ! Check for invalid argument
+       if ((ipencil < 1) .or. (ipencil > 3)) then
+          call decomp_2d_abort(__FILE__, __LINE__, ipencil, "Invalid pencil for halo exchange, should be in range 1<=pencil<=3")
+       end if
     else
        ! Historic/default behaviour
        if (s1 == decomp%xsz(1)) then