Updates examples for non managed case with exception of Halo

examples/Makefile

@@ -1,7 +1,7 @@
-.PHONY: init_test test2d fft_physical_x fft_physical_z 
+.PHONY: init_test test2d fft_physical_x fft_physical_z io_test 
 
 # Just build the examples
-examples: init_test test2d fft_physical_x fft_physical_z 
+examples: init_test test2d fft_physical_x fft_physical_z io_test
 	@echo "Built the examples"
 
 init_test:
@@ -12,17 +12,21 @@ fft_physical_x:
 	$(MAKE) -C $@ all
 fft_physical_z:
 	$(MAKE) -C $@ all
+io_test:
+	$(MAKE) -C $@ all
 
 check:
 	cd init_test; $(MAKE) $@
 	cd test2d; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
+	cd io_test; $(MAKE) $@
 
 clean:
 	cd init_test; $(MAKE) $@
 	cd test2d; $(MAKE) $@
 	cd fft_physical_x; $(MAKE) $@
 	cd fft_physical_z; $(MAKE) $@
+	cd io_test; $(MAKE) $@
 
 export

examples/io_test/io_plane_test.f90

@@ -1,9 +1,15 @@
+!! NB in case of GPU only the writing in the aligned pencil (i.e. X for a 1 array) is performed. 
+!! IO subrotines needs update for non managed GPU case
 program io_plane_test
 
    use mpi
 
    use decomp_2d
    use decomp_2d_io
+#if defined(_GPU)
+   use cudafor
+   use openacc
+#endif
 
    implicit none
 
@@ -37,30 +43,47 @@ program io_plane_test
    call alloc_y(u2, .true.)
    call alloc_z(u3, .true.)
 
+   ! For GPU we port the global data create the different pencil arrays 
+   ! Move back to host the arrays for writing on disk
+
+   !$acc data copyin(data1,xstart,xend) copy(u1,u2,u3) 
    ! original X-pensil based data
+   !$acc parallel loop default(present) 
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
             u1(i, j, k) = data1(i, j, k)
          end do
       end do
    end do
+   !$acc end loop
+   write(*,*) 'End Loop'
+   call transpose_x_to_y(u1, u2)
+   call transpose_y_to_z(u2, u3)
+   !$acc update self(u1)  
+   !$acc update self(u2)  
+   !$acc update self(u3)  
+   !$acc end data
+   call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
+
    call decomp_2d_write_plane(1, u1, 1, nx/2, '.', 'x_pencil-x_plane.dat', 'test')
+#if !defined(_GPU)
    call decomp_2d_write_plane(1, u1, 2, ny/2, '.', 'x_pencil-y_plane.dat', 'test')
    call decomp_2d_write_plane(1, u1, 3, nz/2, '.', 'x_pencil-z_plane.dat', 'test')
-
+#endif
    ! Y-pencil data
-   call transpose_x_to_y(u1, u2)
-   call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(2, u2, 2, ny/2, '.', 'y_pencil-y_plane.dat', 'test')
+#if !defined(_GPU)
+   call decomp_2d_write_plane(2, u2, 1, nx/2, '.', 'y_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(2, u2, 3, nz/2, '.', 'y_pencil-z_plane.dat', 'test')
+#endif
 
    ! Z-pencil data
-   call transpose_y_to_z(u2, u3)
+#if !defined(_GPU)
    call decomp_2d_write_plane(3, u3, 1, nx/2, '.', 'z_pencil-x_plane.dat', 'test')
    call decomp_2d_write_plane(3, u3, 2, ny/2, '.', 'z_pencil-y_plane.dat', 'test')
+#endif
    call decomp_2d_write_plane(3, u3, 3, nz/2, '.', 'z_pencil-z_plane.dat', 'test')
-
    ! Attemp to read the files
    if (nrank == 0) then
       inquire (iolength=iol) data1(1, 1, 1)
@@ -84,6 +107,7 @@ program io_plane_test
       write (*, *) 'passed self test x-plane'
 
       ! Y-plane
+#if !defined(_GPU)
       allocate (work(nx, 1, nz))
       open (10, FILE='x_pencil-y_plane.dat', FORM='unformatted', &
             ACCESS='DIRECT', RECL=iol)
@@ -118,6 +142,7 @@ program io_plane_test
       deallocate (work)
 
       write (*, *) 'passed self test z-plane'
+#endif
 
    end if
 

examples/io_test/io_test.f90

@@ -53,17 +53,24 @@ program io_test
    call alloc_z(u3b, .true.)
 
    ! original x-pencil based data
+   !$acc data copyin(data1,xstart,xend) copy(u1,u2,u3) 
+   !$acc parallel loop default(present)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
             u1(i, j, k) = data1(i, j, k)
          end do
       end do
    end do
+   !$acc end loop
 
    ! transpose
    call transpose_x_to_y(u1, u2)
    call transpose_y_to_z(u2, u3)
+   !$acc update self(u1)  
+   !$acc update self(u2)  
+   !$acc update self(u3)
+   !$acc end data  
 
    ! write to disk
    call decomp_2d_write_one(1, u1, '.', 'u1.dat', 0, 'test')
@@ -79,7 +86,7 @@ program io_test
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
-            if (abs((u1(i, j, k) - u1b(i, j, k))) > eps) stop 1
+            if (abs((u2(i, j, k) - u2b(i, j, k))) > eps) stop 1
          end do
       end do
    end do

examples/io_test/io_var_test.f90

@@ -112,6 +112,8 @@ program io_var_test
    call alloc_z(u3l_b, large, .true.)
 
    ! distribute the data
+   !$acc data copyin(data1,cdata1,data1_large,xstart,xend,large%xst,large%xen) copy(u1,u2,u3,u1l,u2l,u3l,cu1,cu2,cu3) 
+   !$acc parallel loop default(present)
    do k = xstart(3), xend(3)
       do j = xstart(2), xend(2)
          do i = xstart(1), xend(1)
@@ -120,13 +122,16 @@ program io_var_test
          end do
       end do
    end do
+   !$acc end loop
+   !$acc parallel loop default(present)
    do k = large%xst(3), large%xen(3)
       do j = large%xst(2), large%xen(2)
          do i = large%xst(1), large%xen(1)
             u1l(i, j, k) = data1_large(i, j, k)
          end do
       end do
    end do
+   !$acc end loop
 
    ! transpose
    call transpose_x_to_y(u1, u2)
@@ -135,6 +140,16 @@ program io_var_test
    call transpose_y_to_z(u2l, u3l, large)
    call transpose_x_to_y(cu1, cu2)
    call transpose_y_to_z(cu2, cu3)
+   !$acc update self (u1)
+   !$acc update self (u2)
+   !$acc update self (u3)
+   !$acc update self (u1l)
+   !$acc update self (u2l)
+   !$acc update self (u3l)
+   !$acc update self (cu1)
+   !$acc update self (cu2)
+   !$acc update self (cu3)
+   !$acc end data
 
    ! open file for IO
    write (filename, '(A,I3.3)') 'io_var_data.', nproc

examples/test2d/test2d.f90

@@ -61,6 +61,7 @@ program test2d
 
 #ifdef DEBUG
    if (nrank == 0) then
+      !$acc update self(u1)  
       write (*, *) 'Numbers held on Rank 0'
       write (*, *) ' '
       write (*, *) 'X-pencil'