From fa5d477dc5371d2a7c1709a249c8df8d237a9eae Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 13:34:50 +0300 Subject: [PATCH 01/38] Added workflow for sync to lumi and a first version of a gitlab CI workflow --- .github/workflows/lumi-sync.yaml | 26 ++++++++++++++++++++++++++ .gitlab-ci.yml | 13 +++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 .github/workflows/lumi-sync.yaml create mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/lumi-sync.yaml b/.github/workflows/lumi-sync.yaml new file mode 100644 index 000000000..3121338ac --- /dev/null +++ b/.github/workflows/lumi-sync.yaml @@ -0,0 +1,26 @@ +name: Run CICD on Lumi (using .gitlab-ci.yml) + +on: push + branches: + - refactoring + +jobs: + gitlabsync: + runs-on: ubuntu-latest + steps: + - name: "Check out code" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Gitlab mirror and CI" + uses: "tiggi/gitlab-mirror-and-ci-action@tiggi/fixes" + with: + args: "https://gitlab.com/lumi-cicd/fesom2.git" + env: + FOLLOW_TAGS: "true" + FORCE_PUSH: "true" + GITLAB_HOSTNAME: "gitlab.com" + GITLAB_USERNAME: "tiggi" + GITLAB_PASSWORD: ${{ secrets.GITLAB_PASSWORD }} + GITLAB_PROJECT_ID: "51374059" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..bdd7339e1 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,13 @@ +fesom-job: + stage: build + variables: + SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=16G -t 00:30:00 -A project_462000376" + tags: + - lumi + artifacts: + paths: + - fesom_build.log + script: + - echo "building fesom branch" + - bash -l configure.sh lumi + - touch fesom_build.log From 33155cc84bfe2bb50fe3e36854997f9ff4735e31 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 13:45:55 +0300 Subject: [PATCH 02/38] Added lumi_gpu_evatali branch to workflow --- .github/workflows/lumi-sync.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lumi-sync.yaml b/.github/workflows/lumi-sync.yaml index 3121338ac..859c0a2eb 100644 --- a/.github/workflows/lumi-sync.yaml +++ b/.github/workflows/lumi-sync.yaml @@ -3,6 +3,7 @@ name: Run CICD on Lumi (using .gitlab-ci.yml) on: push branches: - refactoring + - lumi_gpu_evatali jobs: gitlabsync: From 4a3ea6f97dac6d83003dc2f7dc0b16361aad5517 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:03:06 +0300 Subject: [PATCH 03/38] Fixed bad YAML in Lumi CICD workflow --- .github/workflows/lumi-sync.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lumi-sync.yaml b/.github/workflows/lumi-sync.yaml index 859c0a2eb..11ca2531c 100644 --- a/.github/workflows/lumi-sync.yaml +++ b/.github/workflows/lumi-sync.yaml @@ -1,9 +1,10 @@ name: Run CICD on Lumi (using .gitlab-ci.yml) -on: push - branches: - - refactoring - - lumi_gpu_evatali +on: + push: + branches: + - refactoring + - lumi_gpu_evatali jobs: gitlabsync: From 17b396e3019a4ad48b9e75f2333866146fab6675 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:16:25 +0300 Subject: [PATCH 04/38] Add a partition for the Lumi CICD job --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdd7339e1..78c5607bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ fesom-job: stage: build variables: - SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=16G -t 00:30:00 -A project_462000376" + SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=32G -p dev-g -t 00:30:00 -A project_462000376" tags: - lumi artifacts: From 62098b4235e049f9a123e8d9b6ab9c7fb7ba20d5 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:24:28 +0300 Subject: [PATCH 05/38] Remove the artifact from gitlab-ci, it's better to have it printed out so that github also gets it --- .gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78c5607bf..d2dd6cf2c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,10 +4,9 @@ fesom-job: SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=32G -p dev-g -t 00:30:00 -A project_462000376" tags: - lumi - artifacts: - paths: - - fesom_build.log + # artifacts: + #paths: + # - fesom_build.log script: - echo "building fesom branch" - bash -l configure.sh lumi - - touch fesom_build.log From 473de2a93baf73fb4684861253341d72d21d2f64 Mon Sep 17 00:00:00 2001 From: Ulf 'Tiggi' Tigerstedt Date: Thu, 19 Oct 2023 14:27:11 +0300 Subject: [PATCH 06/38] Remove the artifact from gitlab-ci, it's better to have it printed out so that github also gets it #2 --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2dd6cf2c..9822c32bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,9 +4,9 @@ fesom-job: SCHEDULER_PARAMETERS: "-N 1 -n 56 --mem=32G -p dev-g -t 00:30:00 -A project_462000376" tags: - lumi - # artifacts: - #paths: - # - fesom_build.log +# artifacts: +# paths: +# - fesom_build.log script: - echo "building fesom branch" - bash -l configure.sh lumi From 62d3e6a1213e188b682ddb267963a5b94d7bc52d Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Thu, 26 Oct 2023 14:36:53 +0200 Subject: [PATCH 07/38] an OMP bux fix in tracer FCT --- src/oce_adv_tra_fct.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oce_adv_tra_fct.F90 b/src/oce_adv_tra_fct.F90 index 013a64aff..24a43bef2 100644 --- a/src/oce_adv_tra_fct.F90 +++ b/src/oce_adv_tra_fct.F90 @@ -104,7 +104,7 @@ subroutine oce_tra_adv_fct(dt, ttf, lo, adf_h, adf_v, fct_ttf_min, fct_ttf_max, #include "associate_mesh_ass.h" !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(n, nz, k, elem, enodes, num, el, nl1, nl2, nu1, nu2, nl12, nu12, edge, & -!$OMP flux, ae,tvert_max, tvert_min) +!$OMP flux, ae) ! -------------------------------------------------------------------------- ! ttf is the tracer field on step n ! del_ttf is the increment From ef9e91f5d8ddfff34cdfb2e380a1ac169435ffc0 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 22 Aug 2023 10:22:37 +0200 Subject: [PATCH 08/38] Add forward declarations in parms for standard compliance --- lib/parms/src/parms_pc_schurras.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/parms/src/parms_pc_schurras.c b/lib/parms/src/parms_pc_schurras.c index 979acdfe9..505dfc1c2 100755 --- a/lib/parms/src/parms_pc_schurras.c +++ b/lib/parms/src/parms_pc_schurras.c @@ -10,6 +10,11 @@ #endif #endif +/* Forward declarations */ +int parms_OperatorGetU(parms_Operator self, void **mat); +int parms_MatGetOffDiag(parms_Mat self, void **mat); +int parms_CommGetOdvlist(parms_Comm self, int **odvlist); + typedef struct schurras_data { parms_Operator op_out,op_in; From 6298b59867ca8caa1c7808610acc874d56ae78f5 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 12:19:49 +0200 Subject: [PATCH 09/38] Fix warning: format string is not a string literal (potentially insecure) --- lib/parms/src/DDPQ/sets.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/parms/src/DDPQ/sets.c b/lib/parms/src/DDPQ/sets.c index 1cfb68b68..8e3db4e56 100755 --- a/lib/parms/src/DDPQ/sets.c +++ b/lib/parms/src/DDPQ/sets.c @@ -19,7 +19,7 @@ void parms_errexit( char *f_str, ... ) sprintf(out2, "Error! %s\n", out1); - fprintf(stdout, out2); + fprintf(stdout, "%s", out2); fflush(stdout); exit( -1 ); From 66eaaafa8a8d80170de11c328e0a902da5222e31 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 12:24:12 +0200 Subject: [PATCH 10/38] Fix warning: add explicit braces to avoid dangling else --- lib/parms/src/parms_ilu_vcsr.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/parms/src/parms_ilu_vcsr.c b/lib/parms/src/parms_ilu_vcsr.c index 116aa51ba..17169e210 100755 --- a/lib/parms/src/parms_ilu_vcsr.c +++ b/lib/parms/src/parms_ilu_vcsr.c @@ -1683,14 +1683,15 @@ int parms_ilu_update(parms_Mat self, parms_FactParam param, void *mat, if(ABS_VALUE(t1) < DBL_EPSILON) continue; if( ii+start < schur_start ){ - for(jj = 1; jj < nnz; jj++) { - iw = jw[rowjj[jj]]; - if(iw != -1) - if(iw < ii+start) - rowm[iw] -= t1*rowmm[jj]; - else - data->U->pa[ii+start][iw-ii-start] -= t1*rowmm[jj]; - } + for(jj = 1; jj < nnz; jj++) { + iw = jw[rowjj[jj]]; + if(iw != -1) { + if(iw < ii+start) + rowm[iw] -= t1*rowmm[jj]; + else + data->U->pa[ii+start][iw-ii-start] -= t1*rowmm[jj]; + } + } } else { for(jj = 1; jj < nnz; jj++){ From 74f796e555bc21516d6574a495669a68ed6d1342 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 12:43:16 +0200 Subject: [PATCH 11/38] Fix warning: incompatible function pointer types initializing... TODO: Assign 'getu' (#512) --- lib/parms/src/DDPQ/arms2.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/parms/src/DDPQ/arms2.c b/lib/parms/src/DDPQ/arms2.c index d7a4ce88d..5dad20138 100755 --- a/lib/parms/src/DDPQ/arms2.c +++ b/lib/parms/src/DDPQ/arms2.c @@ -227,14 +227,15 @@ static int parms_arms_getssize_vcsr(parms_Operator self) } static struct parms_Operator_ops parms_arms_sol_vptr = { - parms_arms_sol_vcsr, - parms_arms_lsol_vcsr, - parms_arms_invs_vcsr, - parms_arms_ascend_vcsr, - parms_arms_getssize_vcsr, - parms_arms_nnz, - arms_free_vcsr, - arms_view_vcsr + parms_arms_sol_vcsr, /* apply */ + parms_arms_lsol_vcsr, /* lsol */ + parms_arms_invs_vcsr, /* invs */ + NULL, /* getu !!! WARNING, UNASSIGNED !!! */ + parms_arms_ascend_vcsr, /* ascend */ + parms_arms_getssize_vcsr, /* getssize */ + parms_arms_nnz, /* getnnz */ + arms_free_vcsr, /* operator_free */ + arms_view_vcsr /* operator_view */ }; From e5509536c182374ff9a40420c9f9864859990501 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 20:07:46 +0200 Subject: [PATCH 12/38] Fix warning: '%s' directive writing up to 255 bytes into a region of size 249 --- lib/parms/src/DDPQ/sets.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/parms/src/DDPQ/sets.c b/lib/parms/src/DDPQ/sets.c index 8e3db4e56..1ccf3adb3 100755 --- a/lib/parms/src/DDPQ/sets.c +++ b/lib/parms/src/DDPQ/sets.c @@ -11,7 +11,7 @@ void parms_errexit( char *f_str, ... ) { va_list argp; - char out1[256], out2[256]; + char out1[256], out2[512]; va_start(argp, f_str); vsprintf(out1, f_str, argp); From 3cdb8d0657430383c307f8328a820ce479306ba6 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 20:41:02 +0200 Subject: [PATCH 13/38] Fix warning: extra tokens at end of #endif directive --- src/gen_modules_partitioning.F90 | 15 ++++++---- src/ice_fct.F90 | 50 ++++++++++++++++---------------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/gen_modules_partitioning.F90 b/src/gen_modules_partitioning.F90 index edf441e73..9c506dac9 100644 --- a/src/gen_modules_partitioning.F90 +++ b/src/gen_modules_partitioning.F90 @@ -93,8 +93,10 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI #else !For ECHAM coupled runs we use the old OASIS nameing scheme (prism / prism_proto) use mod_prism -#endif ! oifs/echam -#endif ! oasis +#endif + ! oifs/echam +#endif + ! oasis implicit none integer, intent(in) :: COMM @@ -112,7 +114,8 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI call MPI_Barrier(COMM, error) call MPI_Finalize(error) endif -#else ! standalone +#else + ! standalone ! From here on the two coupled options !------------------------------------- @@ -136,8 +139,10 @@ subroutine par_ex(COMM, mype, abort) ! finalizes MPI if (mype==0) print *, 'FESOM calls MPI_Finalize' call MPI_Finalize(error) -#endif ! oifs/echam -#endif ! oasis +#endif + ! oifs/echam +#endif + ! oasis ! Regardless of standalone, OpenIFS oder ECHAM coupling, if we reach to this point ! we should be fine shutting the whole model down diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index a2ee681ed..8c42cc997 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -136,7 +136,7 @@ subroutine ice_TG_rhs(ice, partit, mesh) rhs_ms(row)=0._WP #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=0._WP -#endif /* (__oifs) */ +#endif END DO !$OMP END DO ! Velocities at nodes @@ -174,7 +174,7 @@ subroutine ice_TG_rhs(ice, partit, mesh) rhs_ms(row)=rhs_ms(row)+sum(entries*m_snow(elnodes)) #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=rhs_temp(row)+sum(entries*ice_temp(elnodes)) -#endif /* (__oifs) */ +#endif END DO end do !$OMP END DO @@ -210,7 +210,7 @@ subroutine ice_fct_solve(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) call ice_fem_fct(4, ice, partit, mesh) ! ice_temp -#endif /* (__oifs) */ +#endif end subroutine ice_fct_solve ! @@ -297,7 +297,7 @@ subroutine ice_solve_low_order(ice, partit, mesh) m_templ(row)=(rhs_temp(row)+gamma*sum(mass_matrix(clo:clo2)* & ice_temp(location(1:cn))))/area(1,row) + & (1.0_WP-gamma)*ice_temp(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END PARALLEL DO @@ -305,7 +305,7 @@ subroutine ice_solve_low_order(ice, partit, mesh) call exchange_nod(m_icel,a_icel,m_snowl, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(m_templ, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end subroutine ice_solve_low_order @@ -372,7 +372,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) dm_snow(row)=rhs_ms(row)/area(1,row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=rhs_temp(row)/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -380,7 +380,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) call exchange_nod(dm_ice, da_ice, dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER !___________________________________________________________________________ !iterate @@ -407,7 +407,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) rhs_new = rhs_temp(row) - sum(mass_matrix(clo:clo2)*dm_temp(location(1:cn))) m_templ(row)= dm_temp(row)+rhs_new/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -424,7 +424,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) dm_snow(row)=m_snowl(row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=m_templ(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -434,7 +434,7 @@ subroutine ice_solve_high_order(ice, partit, mesh) call exchange_nod(dm_ice, da_ice, dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end do end subroutine ice_solve_high_order @@ -569,7 +569,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) dm_temp(elnodes)))*(vol/area(1,elnodes(q)))/12.0_WP end do end if -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END DO @@ -646,7 +646,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) !$ACC END PARALLEL LOOP !$OMP END DO end if -#endif /* (__oifs) */ +#endif !___________________________________________________________________________ ! Sums of positive/negative fluxes to node row @@ -946,12 +946,12 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) #endif !$OMP END DO end if -#endif /* (__oifs) */ || defined (__ifsinterface) +#endif !$OMP END PARALLEL call exchange_nod(m_ice, a_ice, m_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(ice_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$ACC END DATA @@ -1126,13 +1126,13 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) rhs_ms(row)=0.0_WP #if defined (__oifs) || defined (__ifsinterface) rhs_temp(row)=0.0_WP -#endif /* (__oifs) */ +#endif rhs_mdiv(row)=0.0_WP rhs_adiv(row)=0.0_WP rhs_msdiv(row)=0.0_WP #if defined (__oifs) || defined (__ifsinterface) rhs_tempdiv(row)=0.0_WP -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1180,7 +1180,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) cx3=vol*ice%ice_dt*c4*(sum(m_snow(elnodes))+m_snow(elnodes(n))+sum(entries2*m_snow(elnodes)))/12.0_WP #if defined (__oifs) || defined (__ifsinterface) cx4=vol*ice%ice_dt*c4*(sum(ice_temp(elnodes))+ice_temp(elnodes(n))+sum(entries2*ice_temp(elnodes)))/12.0_WP -#endif /* (__oifs) */ +#endif !___________________________________________________________________ #if defined(_OPENMP) && !defined(__openmp_reproducible) @@ -1212,7 +1212,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) !$ACC ATOMIC UPDATE #endif rhs_temp(row)=rhs_temp(row)+tmp_sum+cx4 -#endif /* (__oifs) */ +#endif !___________________________________________________________________ #if !defined(DISABLE_OPENACC_ATOMICS) @@ -1232,7 +1232,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) !$ACC ATOMIC UPDATE #endif rhs_tempdiv(row)=rhs_tempdiv(row)-cx4 -#endif /* (__oifs) */ +#endif #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(row)) #else @@ -1316,7 +1316,7 @@ subroutine ice_update_for_div(ice, partit, mesh) dm_snow(row)=rhs_msdiv(row)/area(1,row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row)=rhs_tempdiv(row)/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1326,7 +1326,7 @@ subroutine ice_update_for_div(ice, partit, mesh) call exchange_nod(dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER !___________________________________________________________________________ !iterate @@ -1356,7 +1356,7 @@ subroutine ice_update_for_div(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) rhs_new = rhs_tempdiv(row) - sum(mass_matrix(clo:clo2)*dm_temp(location(1:cn))) m_templ(row)= dm_temp(row)+rhs_new/area(1,row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1372,7 +1372,7 @@ subroutine ice_update_for_div(ice, partit, mesh) dm_snow(row) = m_snowl(row) #if defined (__oifs) || defined (__ifsinterface) dm_temp(row) = m_templ(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP @@ -1383,7 +1383,7 @@ subroutine ice_update_for_div(ice, partit, mesh) call exchange_nod(dm_snow, partit, luse_g2g = .true.) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) -#endif /* (__oifs) */ +#endif !$OMP BARRIER end do @@ -1396,7 +1396,7 @@ subroutine ice_update_for_div(ice, partit, mesh) m_snow(row) = m_snow(row)+dm_snow(row) #if defined (__oifs) || defined (__ifsinterface) ice_temp(row)= ice_temp(row)+dm_temp(row) -#endif /* (__oifs) */ +#endif end do !$ACC END PARALLEL LOOP !$OMP END PARALLEL DO From 4ae7703dc810a6299f5841c11667b3b92c110a03 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 23:48:09 +0200 Subject: [PATCH 14/38] Improve FindNETCDF with extra search hints and handling of COMPONENTS --- cmake/FindNETCDF.cmake | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cmake/FindNETCDF.cmake b/cmake/FindNETCDF.cmake index 03b985d67..cbd823e8e 100644 --- a/cmake/FindNETCDF.cmake +++ b/cmake/FindNETCDF.cmake @@ -13,9 +13,13 @@ if(CMAKE_Fortran_COMPILER_LOADED) if(HAVE_Fortran_NETCDF) set(NETCDF_Fortran_INCLUDE_DIRECTORIES "") set(NETCDF_Fortran_LIBRARIES "") + set(NETCDF_Fortran_FOUND 1) else() - find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) + find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) find_library(NETCDF_Fortran_LIBRARIES netcdff HINTS ${NETCDF_Fortran_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_Fortran_INCLUDE_DIRECTORIES AND NETCDF_Fortran_LIBRARIES ) + set(NETCDF_Fortran_FOUND 1) + endif() endif() endif() @@ -27,14 +31,18 @@ if(CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED) if(HAVE_C_NETCDF) set(NETCDF_C_INCLUDE_DIRECTORIES "") set(NETCDF_C_LIBRARIES "") + set(NETCDF_C_FOUND 1) else() - find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) + find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) find_library(NETCDF_C_LIBRARIES netcdf HINTS ${NETCDF_C_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_C_INCLUDE_DIRECTORIES AND NETCDF_C_LIBRARIES ) + set(NETCDF_C_FOUND 1) + endif() endif() endif() if(CMAKE_CXX_COMPILER_LOADED) - find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_CXX_INCLUDE_DIRECTORIES) + find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_ROOT}/include $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR} ENV NETCDF_CXX_INCLUDE_DIRECTORIES) # the cray toolchain (e.g. hlrn) disables dynamic linking by default. to enable it at build time do e.g. "CRAYPE_LINK_TYPE=dynamic make". find_library(NETCDF_CXX_LIBRARIES NAMES netcdf_c++4 netcdf-cxx4 HINTS ${NETCDF_CXX_INCLUDE_DIRECTORIES}/../lib) if(NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_C_INCLUDE_DIRECTORIES) @@ -43,4 +51,21 @@ if(CMAKE_CXX_COMPILER_LOADED) if(NETCDF_CXX_LIBRARIES AND NETCDF_C_LIBRARIES) list(APPEND NETCDF_CXX_LIBRARIES ${NETCDF_C_LIBRARIES}) endif() -endif() \ No newline at end of file + if( NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_CXX_LIBRARIES ) + set(NETCDF_CXX_FOUND 1) + endif() +endif() + +if(NOT ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + set(${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS C) +endif() + +unset({CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) +foreach(COMP C CXX Fortran) + if("${COMP}" IN_LIST ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + list(APPEND ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS NETCDF_${COMP}_FOUND) + endif() +endforeach() +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} HANDLE_COMPONENTS REQUIRED_VARS ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) + From d526ae43632798c227c1539957b2a319d8b7981b Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 11 Oct 2023 02:52:06 +0300 Subject: [PATCH 15/38] Update lumi environment, showing that FindNETCDF works without intervention --- env/lumi/shell | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/env/lumi/shell b/env/lumi/shell index 40a2cb047..aaf0f589c 100644 --- a/env/lumi/shell +++ b/env/lumi/shell @@ -11,10 +11,10 @@ module load cray-netcdf/4.8.1.5 export FC=ftn export CC=cc export CXX=cc -export NETCDF_Fortran_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include -export NETCDF_C_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include -export NETCDF_C_LIBRARIES=$CRAY_NETCDF_DIR/lib -export NETCDF_Fortran_LIBRARIES=$CRAY_NETCDF_DIR/lib +#export NETCDF_Fortran_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include +#export NETCDF_C_INCLUDE_DIRECTORIES=$CRAY_NETCDF_DIR/include +#export NETCDF_C_LIBRARIES=$CRAY_NETCDF_DIR/lib +#export NETCDF_Fortran_LIBRARIES=$CRAY_NETCDF_DIR/lib $CC -v $FC -V $CXX -v From ae6958ef8a429f896802d936cd28af197accc81a Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 23:08:13 +0000 Subject: [PATCH 16/38] Update atosecmwf environment, showing mpi compiler wrappers are no longer needed and FindNETCDF also works --- env/atosecmwf/shell | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/env/atosecmwf/shell b/env/atosecmwf/shell index 90f118607..97cd921a7 100644 --- a/env/atosecmwf/shell +++ b/env/atosecmwf/shell @@ -33,7 +33,5 @@ module load cmake/3.20.2 module load ninja/1.10.0 module load fcm/2019.05.0 -export NETCDF_DIR=$NETCDF4_DIR +export FC=ifort CC=icc CXX=icpc -export FC=mpif90 CC=mpicc CXX=mpicxx # MPI wrappers for Fortran, cc and CC similarly -#export FC=mpif90 CC=gcc CXX=mpicxx # MPI wrappers for Fortran, cc and CC similarly From 89fbf651ab6da5edf4bee5bbfed50a689cda0674 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Fri, 27 Oct 2023 10:38:55 +0200 Subject: [PATCH 17/38] addes missing OMP clauses --- src/ice_fct.F90 | 13 ++++++++----- src/oce_ale.F90 | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index a2ee681ed..129f0a5a4 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -504,7 +504,8 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) ! it takes memory and time. For every element ! we need its antidiffusive contribution to ! each of its 3 nodes - +!$OMP PARALLEL +!$OMP DO !$ACC DATA CREATE(icoef, elnodes) !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) @@ -513,20 +514,21 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) tmin(n) = 0.0_WP end do !$ACC END PARALLEL LOOP - +!$OMP END DO ! Auxiliary elemental operator (mass matrix- lumped mass matrix) !$ACC KERNELS icoef = 1 !$ACC END KERNELS - +!$OMP DO !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) do n=1,3 ! three upper nodes ! Cycle over rows row=elnodes(n) icoef(n,n)=-2 end do !$ACC END PARALLEL LOOP - +!$OMP END DO +!$OMP END PARALLEL !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(n, q, elem, elnodes, row, vol, flux, ae) !$OMP DO @@ -1117,7 +1119,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) ! Computes the rhs in a Taylor-Galerkin way (with upwind type of ! correction for the advection operator) ! In this version I tr to split divergent term off, so that FCT works without it. - +!$OMP PARALLEL DO !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) do row=1, myDim_nod2D !! row=myList_nod2D(m) @@ -1135,6 +1137,7 @@ subroutine ice_TG_rhs_div(ice, partit, mesh) #endif /* (__oifs) */ end do !$ACC END PARALLEL LOOP +!$OMP END PARALLEL DO !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(diff, entries, um, vm, vol, dx, dy, n, q, row, elem, elnodes, c1, c2, c3, c4, cx1, cx2, cx3, cx4, entries2) !$OMP DO diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 3c25bb38c..973f81161 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -2495,7 +2495,7 @@ subroutine vert_vel_ale(dynamics, partit, mesh) end do !$OMP END PARALLEL DO -!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax) +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax, c1, c2) do n=1, myDim_nod2D+eDim_nod2D nzmin = ulevels_nod2D(n) nzmax = nlevels_nod2D(n)-1 From efbb3ed1412288fe2dbe4dcc14463e9b9e956cc0 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Fri, 27 Oct 2023 10:58:33 +0200 Subject: [PATCH 18/38] removed just introduced OMP statement im ice_fct (was not needed) --- src/ice_fct.F90 | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index 129f0a5a4..07f57531f 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -504,8 +504,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) ! it takes memory and time. For every element ! we need its antidiffusive contribution to ! each of its 3 nodes -!$OMP PARALLEL -!$OMP DO +!$OMP PARALLEL DO !$ACC DATA CREATE(icoef, elnodes) !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) @@ -514,21 +513,18 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) tmin(n) = 0.0_WP end do !$ACC END PARALLEL LOOP -!$OMP END DO +!$OMP END PARALLEL DO ! Auxiliary elemental operator (mass matrix- lumped mass matrix) !$ACC KERNELS icoef = 1 !$ACC END KERNELS -!$OMP DO !$ACC PARALLEL LOOP GANG VECTOR DEFAULT(PRESENT) do n=1,3 ! three upper nodes ! Cycle over rows row=elnodes(n) icoef(n,n)=-2 end do !$ACC END PARALLEL LOOP -!$OMP END DO -!$OMP END PARALLEL !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(n, q, elem, elnodes, row, vol, flux, ae) !$OMP DO From c52360af56e31192919330b29e0d2f1b31943486 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorenko Date: Fri, 6 Oct 2023 11:21:25 +0200 Subject: [PATCH 19/38] CMakefiles from Razvan. This wont work for standalone runs! --- lib/parms/CMakeLists.txt | 45 ++++++++++------------------ src/CMakeLists.txt | 42 +++++++++++--------------- src/async_threads_cpp/CMakeLists.txt | 18 +++++------ 3 files changed, 41 insertions(+), 64 deletions(-) diff --git a/lib/parms/CMakeLists.txt b/lib/parms/CMakeLists.txt index 7f85d5eef..62ba062fa 100644 --- a/lib/parms/CMakeLists.txt +++ b/lib/parms/CMakeLists.txt @@ -6,40 +6,27 @@ project(parms C) set(src_home ${CMAKE_CURRENT_LIST_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt file(GLOB all_sources ${src_home}/src/*.c ${src_home}/src/DDPQ/*.c) -find_package(BLAS) # standard way to find blas -if( NOT BLAS_FOUND) # try older way to find blas - include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/FindBLAS.cmake") -endif() +include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/FindBLAS.cmake") # create our library (set its name to name of this project) -if(${BUILD_FESOM_AS_LIBRARY}) - add_library(${PROJECT_NAME} ${all_sources}) -else() - add_library(${PROJECT_NAME} ${all_sources}) -endif() +add_library(${PROJECT_NAME} SHARED ${all_sources}) -if(${BLAS_FOUND}) - message("BLAS FOUND ---------------------------") - target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 HAS_BLAS) - target_link_libraries(${PROJECT_NAME} INTERFACE ${BLAS_C_LIBRARIES} $ENV{UBUNTU_BLAS_LIBRARY}) -else() - message("BLAS NOT FOUND ***********************") - target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8) - target_link_libraries(${PROJECT_NAME} INTERFACE $ENV{UBUNTU_BLAS_LIBRARY}) -endif() +target_compile_definitions(${PROJECT_NAME} + PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 HAS_BLAS) target_include_directories(${PROJECT_NAME} - PRIVATE ${src_home}/src/../include ${src_home}/src/include - INTERFACE ${src_home}/src/../include + PUBLIC $ + PRIVATE ${src_home}/src/include ) + +target_link_libraries(${PROJECT_NAME} PRIVATE ${BLAS_C_LIBRARIES} $ENV{UBUNTU_BLAS_LIBRARY}) + if(${CMAKE_C_COMPILER_ID} STREQUAL "Intel") - target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-div -no-prec-sqrt -fast-transcendentals -fp-model precise) - - if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) - target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) - endif() -endif() -if(${BUILD_FESOM_AS_LIBRARY}) - target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) - install(TARGETS ${PROJECT_NAME} DESTINATION "${FESOM_INSTALL_PREFIX}/lib") + target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-div -no-prec-sqrt -fast-transcendentals -fp-model precise) + if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) + target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) + endif() endif() + +target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) +install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 79307ff0b..7d7c318ee 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -46,9 +46,7 @@ else() endif() option(ENABLE_OPENACC "compile with OpenACC support" OFF) -option(DISABLE_OPENACC_ATOMICS "disable kernels using atomic statement for reproducible results" ON) -set(GPU_COMPUTE_CAPABILITY "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)") -set(GPU_FLAGS "cuda11.7,${GPU_COMPUTE_CAPABILITY}" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)") +set(NV_GPU_ARCH "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)") option(ENABLE_OPENMP "build FESOM with OpenMP" OFF) if(${ENABLE_OPENMP}) @@ -60,6 +58,8 @@ endif() # two targets would allow e.g. setting different compiler options or preprocessor definition, which would be error prone. option(BUILD_FESOM_AS_LIBRARY "Build a library instead of an executable" OFF) +set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/../.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") + # get our source files set(src_home ${CMAKE_CURRENT_LIST_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt if(${USE_ICEPACK}) @@ -120,7 +120,7 @@ if(${BUILD_FESOM_AS_LIBRARY}) else() add_executable(${PROJECT_NAME} ${sources_Fortran} ${src_home}/fesom_main.F90) endif() -target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS -DMETIS_VERSION=5 -DPART_WEIGHTED -DMETISRANDOMSEED=35243) +target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS -DMETIS_VERSION=5 -DPART_WEIGHTED -DMETISRANDOMSEED=35243 __MULTIO) if(${DISABLE_MULTITHREADING}) target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_MULTITHREADING) endif() @@ -146,9 +146,9 @@ endif() # CMAKE_Fortran_COMPILER_ID will also work if a wrapper is being used (e.g. mpif90 wraps ifort -> compiler id is Intel) if(${CMAKE_Fortran_COMPILER_ID} STREQUAL Intel ) if(${BUILD_FESOM_AS_LIBRARY}) - target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -xHost -ip -init=zero -no-wrap-margin -fpe0) # add -fpe0 for RAPS environment + target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin -fpe0 -g -traceback) # add -fpe0 for RAPS environment else() - target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin) + target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin -g -traceback) endif() if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) @@ -175,21 +175,16 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray ) endif() elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC ) target_compile_definitions(${PROJECT_NAME} PRIVATE ENABLE_NVHPC_WORKAROUNDS) - #target_compile_options(${PROJECT_NAME} PRIVATE -fast -fastsse -O3 -Mallocatable=95 -Mr8 -pgf90libs) - target_compile_options(${PROJECT_NAME} PRIVATE -Mnofma -Mallocatable=95 -Mr8 -pgf90libs) + target_compile_options(${PROJECT_NAME} PRIVATE -fast -fastsse -O3 -Mallocatable=95 -Mr8 -pgf90libs) if(${ENABLE_OPENACC}) # additional compiler settings - target_compile_options(${PROJECT_NAME} PRIVATE -acc -gpu=${GPU_FLAGS} -Minfo=accel) - set(CMAKE_EXE_LINKER_FLAGS "-acc -gpu=${GPU_FLAGS}") - if(${DISABLE_OPENACC_ATOMICS}) - target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_OPENACC_ATOMICS) - endif() + target_compile_options(${PROJECT_NAME} PRIVATE -acc -ta=tesla:${NV_GPU_ARCH} -Minfo=accel) + set(CMAKE_EXE_LINKER_FLAGS "-acc -ta=tesla:${NV_GPU_ARCH}") endif() if(${ENABLE_OPENMP}) target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast) - #else() - ## Mipa enables -O2, so it is disabled during gpu development for testing - #target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline) + else() + target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline) endif() endif() if(${BUILD_FESOM_AS_LIBRARY}) @@ -201,21 +196,20 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${MCT_Fortran_INCLUDE_DIRECTO target_include_directories(${PROJECT_NAME} PRIVATE ${SCRIP_Fortran_INCLUDE_DIRECTORIES}) target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) -target_link_libraries(${PROJECT_NAME} async_threads_cpp) +target_link_libraries(${PROJECT_NAME} async_threads_cpp multio-fapi) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) if(${ENABLE_OPENMP} AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) target_compile_options(${PROJECT_NAME} PRIVATE ${OpenMP_Fortran_FLAGS}) # currently we only have OpenMP in the Fortran part target_link_libraries(${PROJECT_NAME} OpenMP::OpenMP_Fortran) endif() - -set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}/.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") +#set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}/.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") if(${BUILD_FESOM_AS_LIBRARY}) - install(TARGETS ${PROJECT_NAME} DESTINATION "${FESOM_INSTALL_PREFIX}/lib") - install(TARGETS ${PROJECT_NAME}_C DESTINATION "${FESOM_INSTALL_PREFIX}/lib") + install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") + install(TARGETS ${PROJECT_NAME}_C EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") else() set(FESOM_INSTALL_FILEPATH "${FESOM_INSTALL_PREFIX}/bin/fesom.x") -get_filename_component(FESOM_INSTALL_PATH ${FESOM_INSTALL_FILEPATH} DIRECTORY) -get_filename_component(FESOM_INSTALL_NAME ${FESOM_INSTALL_FILEPATH} NAME) -install(PROGRAMS ${PROJECT_BINARY_DIR}/${PROJECT_NAME} DESTINATION ${FESOM_INSTALL_PATH} RENAME ${FESOM_INSTALL_NAME}) + get_filename_component(FESOM_INSTALL_PATH ${FESOM_INSTALL_FILEPATH} DIRECTORY) + get_filename_component(FESOM_INSTALL_NAME ${FESOM_INSTALL_FILEPATH} NAME) + install(PROGRAMS ${PROJECT_BINARY_DIR}/${PROJECT_NAME} DESTINATION ${FESOM_INSTALL_PATH} RENAME ${FESOM_INSTALL_NAME}) endif() diff --git a/src/async_threads_cpp/CMakeLists.txt b/src/async_threads_cpp/CMakeLists.txt index d72d7ce7d..f54186d8f 100644 --- a/src/async_threads_cpp/CMakeLists.txt +++ b/src/async_threads_cpp/CMakeLists.txt @@ -8,21 +8,17 @@ file(GLOB sources_CXX ${CMAKE_CURRENT_LIST_DIR}/*.cpp) include(FortranCInterface) FortranCInterface_HEADER(ThreadsManagerFCMacros.h MACRO_NAMESPACE "ThreadsManagerFCMacros_" SYMBOLS init_ccall begin_ccall end_ccall) -if(${BUILD_FESOM_AS_LIBRARY}) - add_library(${PROJECT_NAME} ${sources_CXX}) -else() - add_library(${PROJECT_NAME} ${sources_CXX}) -endif() +add_library(${PROJECT_NAME} SHARED ${sources_CXX}) + target_include_directories(${PROJECT_NAME} - INTERFACE ${CMAKE_CURRENT_LIST_DIR} - PUBLIC ${CMAKE_CURRENT_BINARY_DIR} + PRIVATE ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) + if(${CMAKE_CXX_COMPILER_ID} STREQUAL Cray ) target_compile_options(${PROJECT_NAME} PRIVATE -hstd=c++11) else() target_compile_options(${PROJECT_NAME} PRIVATE -std=c++11) endif() -if(${BUILD_FESOM_AS_LIBRARY}) - target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) - install(TARGETS ${PROJECT_NAME} DESTINATION "${FESOM_INSTALL_PREFIX}/lib") -endif() + +target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) +install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") From 52cde6216968016eb08f770afc423d0570c4ea26 Mon Sep 17 00:00:00 2001 From: Razvan Aguridan Date: Mon, 2 Oct 2023 14:26:00 +0000 Subject: [PATCH 20/38] Fix compilation errors after multio API update --- src/ifs_interface/iom.F90 | 79 +++++++++++++++------------------------ 1 file changed, 30 insertions(+), 49 deletions(-) diff --git a/src/ifs_interface/iom.F90 b/src/ifs_interface/iom.F90 index f4d5bbb50..a7778834a 100644 --- a/src/ifs_interface/iom.F90 +++ b/src/ifs_interface/iom.F90 @@ -6,7 +6,7 @@ !----------------------------------------------------- MODULE iom -#if defined(__MULTIO) +#if defined(__MULTIO) USE multio_api USE, INTRINSIC :: iso_fortran_env, only: real64 @@ -39,27 +39,27 @@ MODULE iom CONTAINS - SUBROUTINE multio_custom_error_handler(context, err) + SUBROUTINE multio_custom_error_handler(context, err, info) USE mpi IMPLICIT NONE INTEGER(8), INTENT(INOUT) :: context ! Use mpi communicator as context INTEGER, INTENT(IN) :: err + CLASS(multio_failure_info), INTENT(in) :: info INTEGER :: mpierr - + IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop( 'MULTIO ERROR: ', multio_error_string(err)) + CALL ctl_stop( 'MULTIO ERROR: ', multio_error_string(err, info)) IF (context /= MPI_UNDEFINED) THEN CALL mpi_abort(int(context), MPI_ERR_OTHER, mpierr) context = MPI_UNDEFINED ENDIF ENDIF END SUBROUTINE - - + SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) USE mpi - + IMPLICIT NONE CHARACTER(LEN=*), INTENT(IN) :: client_id INTEGER,INTENT(IN), OPTIONAL :: local_comm @@ -103,16 +103,17 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) CALL ctl_stop('Creating multio configuration context failed: ', multio_error_string(err)) END IF + ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface + err = conf_ctx%set_failure_handler(multio_custom_error_handler, mio_parent_comm) + if (err /= MULTIO_SUCCESS) then + CALL ctl_stop( 'setting multio failure handler failed: ', multio_error_string(err)) + end if + err = conf_ctx%mpi_allow_world_default_comm(.FALSE._1) IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('conf_ctx%mpi_allow_world_default_comm(.FALSE._1) failed: ', multio_error_string(err)) END IF - err = conf_ctx%mpi_client_id(client_id) - IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('conf_ctx%mpi_client_id(', TRIM(client_id),') failed: ', multio_error_string(err)) - END IF - err = conf_ctx%mpi_return_client_comm(return_comm) IF (err /= MULTIO_SUCCESS) THEN WRITE (err_str, "(I10)") return_comm @@ -135,19 +136,12 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) CALL ctl_stop('conf_ctx%delete() failed: ', multio_error_string(err)) END IF - ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface -#if defined __ifsinterface - err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) - IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) - END IF -#endif err = mio_handle%open_connections(); IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('mio_handle%open_connections failed: ', multio_error_string(err)) END IF END SUBROUTINE iom_initialize - + SUBROUTINE iom_finalize() IMPLICIT NONE INTEGER :: err @@ -162,14 +156,14 @@ SUBROUTINE iom_finalize() CALL ctl_stop('mio_handle%delete failed: ', multio_error_string(err)) END IF END SUBROUTINE iom_finalize - + SUBROUTINE iom_init_server(server_comm) IMPLICIT NONE INTEGER, INTENT(IN) :: server_comm type(multio_configuration) :: conf_ctx INTEGER :: err CHARACTER(len=16) :: err_str - + mio_parent_comm = server_comm err = multio_initialise() @@ -200,6 +194,13 @@ SUBROUTINE iom_init_server(server_comm) CALL ctl_stop('Creating multio server configuration context failed: ', multio_error_string(err)) END IF + ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface + ! Set handler before invoking blocking start server call + err = conf_ctx%set_failure_handler(multio_custom_error_handler, mio_parent_comm) + IF (err /= MULTIO_SUCCESS) THEN + CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) + END IF + err = conf_ctx%mpi_allow_world_default_comm(.FALSE._1) IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('conf_ctx%mpi_allow_world_default_comm(.FALSE._1) failed: ', multio_error_string(err)) @@ -211,14 +212,6 @@ SUBROUTINE iom_init_server(server_comm) CALL ctl_stop('conf_ctx%mpi_parent_comm(', err_str,') failed: ', multio_error_string(err)) END IF - ! Setting a failure handler that reacts on interface problems or exceptions that are not handled within the interface - ! Set handler before invoking blocking start server call -#if defined __ifsinterface - err = multio_set_failure_handler(multio_custom_error_handler, mio_parent_comm) - IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) - END IF -#endif ! Blocking call err = multio_start_server(conf_ctx) IF (err /= MULTIO_SUCCESS) THEN @@ -236,24 +229,20 @@ SUBROUTINE iom_send_fesom_domains(partit, mesh) USE MOD_PARTIT IMPLICIT NONE - + TYPE(multio_metadata) :: md INTEGER :: cerr INTEGER :: elem, elnodes(3), aux TYPE(t_partit), INTENT(IN), TARGET :: partit TYPE(t_mesh), intent(in), TARGET :: mesh INTEGER, DIMENSION(:), POINTER :: temp - + #include "../associate_part_def.h" #include "../associate_mesh_def.h" #include "../associate_part_ass.h" #include "../associate_mesh_ass.h" -#if defined __ifsinterface - cerr = md%new() -#else cerr = md%new(mio_handle) -#endif IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_domains: ngrid, md%new() failed: ', multio_error_string(cerr)) END IF @@ -295,11 +284,7 @@ SUBROUTINE iom_send_fesom_domains(partit, mesh) END IF !declare grid at elements -#if defined __ifsinterface - cerr = md%new() -#else cerr = md%new(mio_handle) -#endif IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_domains: egrid, md%new() failed: ', multio_error_string(cerr)) END IF @@ -343,16 +328,12 @@ END SUBROUTINE iom_send_fesom_domains SUBROUTINE iom_send_fesom_data(data) USE g_clock IMPLICIT NONE - + TYPE(iom_field_request), INTENT(INOUT) :: data INTEGER :: cerr TYPE(multio_metadata) :: md - -#if defined __ifsinterface - cerr = md%new() -#else + cerr = md%new(mio_handle) -#endif IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: md%new() failed: ', multio_error_string(cerr)) END IF @@ -415,7 +396,7 @@ SUBROUTINE iom_send_fesom_data(data) cerr = md%set_int("timeSpanInHours", 24) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: md%set_int(timeSpanInHours) failed: ', multio_error_string(cerr)) - END IF + END IF cerr = md%set_int("currentDate", yearnew * 10000 + month * 100 + day_in_month) cerr = md%set_int("currentTime", INT(INT(timenew / 3600) * 10000 + (INT(timenew / 60) - INT(timenew / 3600) * 60) * 100 + (timenew-INT(timenew / 60) * 60))) @@ -424,7 +405,7 @@ SUBROUTINE iom_send_fesom_data(data) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: md%set_int(date) failed: ', multio_error_string(cerr)) END IF - + cerr = mio_handle%write_field(md, data%values) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: mio_handle%write_field failed: ', multio_error_string(cerr)) @@ -452,5 +433,5 @@ SUBROUTINE ctl_stop(m1, m2, m3, m4) END SUBROUTINE ctl_stop !!====================================================================== -#endif +#endif END MODULE iom From a50696549c40e98bd41f00af87c55f641888cd3b Mon Sep 17 00:00:00 2001 From: Razvan Aguridan Date: Tue, 29 Aug 2023 09:47:40 +0000 Subject: [PATCH 21/38] Find MPI on atos --- src/CMakeLists.txt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7d7c318ee..ffd62c5a3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.9) project(fesom C Fortran) +find_package(MPI REQUIRED COMPONENTS C CXX Fortran) + if(DEFINED ENV{FESOM_PLATFORM_STRATEGY}) set(FESOM_PLATFORM_STRATEGY $ENV{FESOM_PLATFORM_STRATEGY} CACHE STRING "switch to platform specific compile settings, this is usually determined via the env.sh script") else() @@ -109,7 +111,7 @@ else() add_library(${PROJECT_NAME}_C ${sources_C}) endif() target_compile_definitions(${PROJECT_NAME}_C PRIVATE PARMS USE_MPI REAL=double DBL HAS_BLAS FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 SGI LINUX UNDER_ MPI2) -target_link_libraries(${PROJECT_NAME}_C parms) #metis +target_link_libraries(${PROJECT_NAME}_C PUBLIC parms) #metis # create our binary or library (set its name to name of this project) @@ -194,13 +196,16 @@ endif() target_include_directories(${PROJECT_NAME} PRIVATE ${NETCDF_Fortran_INCLUDE_DIRECTORIES} ${OASIS_Fortran_INCLUDE_DIRECTORIES}) target_include_directories(${PROJECT_NAME} PRIVATE ${MCT_Fortran_INCLUDE_DIRECTORIES} ${MPEU_Fortran_INCLUDE_DIRECTORIES}) target_include_directories(${PROJECT_NAME} PRIVATE ${SCRIP_Fortran_INCLUDE_DIRECTORIES}) -target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) -target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) -target_link_libraries(${PROJECT_NAME} async_threads_cpp multio-fapi) + +target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) +target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_C ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) +target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_Fortran) +target_link_libraries(${PROJECT_NAME} PRIVATE async_threads_cpp multio-fapi) + set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) if(${ENABLE_OPENMP} AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) target_compile_options(${PROJECT_NAME} PRIVATE ${OpenMP_Fortran_FLAGS}) # currently we only have OpenMP in the Fortran part - target_link_libraries(${PROJECT_NAME} OpenMP::OpenMP_Fortran) + target_link_libraries(${PROJECT_NAME} PRIVATE OpenMP::OpenMP_Fortran) endif() #set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}/.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") From e648a8d3f5ee90fd1cbd46f9f83dbaf7118bb4d5 Mon Sep 17 00:00:00 2001 From: suvarchal Date: Mon, 9 Oct 2023 14:58:09 +0200 Subject: [PATCH 22/38] use assumed shapes instead of legacy assumed sizes, Merged io_netcdf_module.F90 io_netcdf_file_module.F90 from razvan_nextgems_multio --- src/io_netcdf_file_module.F90 | 12 ++++++------ src/io_netcdf_module.F90 | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/io_netcdf_file_module.F90 b/src/io_netcdf_file_module.F90 index c786ff5dd..82a211b0b 100644 --- a/src/io_netcdf_file_module.F90 +++ b/src/io_netcdf_file_module.F90 @@ -304,7 +304,7 @@ subroutine read_var_r8(this, varindex, starts, sizes, values) class(netcdf_file_type), intent(in) :: this integer, intent(in) :: varindex integer, dimension(:) :: starts, sizes - real(8), intent(inout), target :: values(..) ! must be inout or the allocation might be screwed + real(8), intent(inout), target :: values(:) ! must be inout or the allocation might be screwed ! EO parameters include "netcdf.inc" real(8), pointer :: values_ptr(:) @@ -324,7 +324,7 @@ subroutine read_var_r4(this, varindex, starts, sizes, values) class(netcdf_file_type), intent(in) :: this integer, intent(in) :: varindex integer, dimension(:) :: starts, sizes - real(4), intent(inout), target :: values(..) ! must be inout or the allocation might be screwed + real(4), intent(inout), target :: values(:) ! must be inout or the allocation might be screwed ! EO parameters include "netcdf.inc" real(4), pointer :: values_ptr(:) @@ -344,7 +344,7 @@ subroutine read_var_integer(this, varindex, starts, sizes, values) class(netcdf_file_type), intent(in) :: this integer, intent(in) :: varindex integer, dimension(:) :: starts, sizes - integer, intent(inout), target :: values(..) ! must be inout or the allocation might be screwed + integer, intent(inout), target :: values(:) ! must be inout or the allocation might be screwed ! EO parameters include "netcdf.inc" integer, pointer :: values_ptr(:) @@ -474,7 +474,7 @@ subroutine write_var_r8(this, varindex, starts, sizes, values) class(netcdf_file_type), intent(in) :: this integer, intent(in) :: varindex integer, dimension(:) :: starts, sizes - real(8), intent(in), target :: values(..) ! must be inout or the allocation might be screwed + real(8), intent(in), target :: values(:) ! must be inout or the allocation might be screwed ! EO parameters include "netcdf.inc" real(8), pointer :: values_ptr(:) @@ -493,7 +493,7 @@ subroutine write_var_r4(this, varindex, starts, sizes, values) class(netcdf_file_type), intent(in) :: this integer, intent(in) :: varindex integer, dimension(:) :: starts, sizes - real(4), intent(in), target :: values(..) ! must be inout or the allocation might be screwed + real(4), intent(in), target :: values(:) ! must be inout or the allocation might be screwed ! EO parameters include "netcdf.inc" real(4), pointer :: values_ptr(:) @@ -512,7 +512,7 @@ subroutine write_var_integer(this, varindex, starts, sizes, values) class(netcdf_file_type), intent(in) :: this integer, intent(in) :: varindex integer, dimension(:) :: starts, sizes - integer, intent(in), target :: values(..) ! must be inout or the allocation might be screwed + integer, intent(in), target :: values(:) ! must be inout or the allocation might be screwed ! EO parameters include "netcdf.inc" integer, pointer :: values_ptr(:) diff --git a/src/io_netcdf_module.F90 b/src/io_netcdf_module.F90 index 1fe6f7cd9..134b3ed88 100644 --- a/src/io_netcdf_module.F90 +++ b/src/io_netcdf_module.F90 @@ -106,7 +106,7 @@ subroutine read_values_r8(this, timeindex, values) use, intrinsic :: ISO_C_BINDING class(netcdf_variable_handle), intent(in) :: this integer, intent(in) :: timeindex - real(8), intent(inout), target :: values(..) ! must be inout or the allocation might be screwed + real(8), intent(inout), target :: values(:) ! must be inout or the allocation might be screwed ! EO args real(8), pointer :: values_ptr(:) integer, allocatable, dimension(:) :: starts, sizes @@ -125,7 +125,7 @@ subroutine read_values_r4(this, timeindex, values) use, intrinsic :: ISO_C_BINDING class(netcdf_variable_handle), intent(in) :: this integer, intent(in) :: timeindex - real(4), intent(inout), target :: values(..) ! must be inout or the allocation might be screwed + real(4), intent(inout), target :: values(:) ! must be inout or the allocation might be screwed ! EO args real(4), pointer :: values_ptr(:) integer, allocatable, dimension(:) :: starts, sizes From 7335b134ed5931d608f53e03ccb610c530319d21 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 21:34:31 +0200 Subject: [PATCH 23/38] Fix warning: Use of the NUMERIC_STORAGE_SIZE named constant from intrinsic module ISO_FORTRAN_ENV is incompatible with option -fdefault-real-8 --- src/MOD_DYN.F90 | 2 +- src/MOD_ICE.F90 | 2 +- src/MOD_MESH.F90 | 2 +- src/MOD_PARTIT.F90 | 2 +- src/MOD_TRACER.F90 | 2 +- src/fesom_module.F90 | 2 ++ src/gen_halo_exchange.F90 | 2 +- src/gen_modules_read_NetCDF.F90 | 5 +++-- 8 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/MOD_DYN.F90 b/src/MOD_DYN.F90 index ded902231..542992876 100644 --- a/src/MOD_DYN.F90 +++ b/src/MOD_DYN.F90 @@ -1,7 +1,7 @@ !========================================================== MODULE MOD_DYN USE O_PARAM -USE, intrinsic :: ISO_FORTRAN_ENV +USE, intrinsic :: ISO_FORTRAN_ENV, only : int32 USE MOD_WRITE_BINARY_ARRAYS USE MOD_READ_BINARY_ARRAYS IMPLICIT NONE diff --git a/src/MOD_ICE.F90 b/src/MOD_ICE.F90 index b2a91acd8..d2a29f7ef 100644 --- a/src/MOD_ICE.F90 +++ b/src/MOD_ICE.F90 @@ -1,6 +1,6 @@ MODULE MOD_ICE USE o_PARAM, only: WP -USE, intrinsic :: ISO_FORTRAN_ENV +USE, intrinsic :: ISO_FORTRAN_ENV, only : int32 USE MOD_WRITE_BINARY_ARRAYS USE MOD_READ_BINARY_ARRAYS IMPLICIT NONE diff --git a/src/MOD_MESH.F90 b/src/MOD_MESH.F90 index 30c2b928f..27b02fb3e 100644 --- a/src/MOD_MESH.F90 +++ b/src/MOD_MESH.F90 @@ -3,7 +3,7 @@ MODULE MOD_MESH USE O_PARAM USE MOD_WRITE_BINARY_ARRAYS USE MOD_READ_BINARY_ARRAYS -USE, intrinsic :: ISO_FORTRAN_ENV +USE, intrinsic :: ISO_FORTRAN_ENV, only : int32 IMPLICIT NONE SAVE integer, parameter :: MAX_ADJACENT=32 ! Max allowed number of adjacent nodes diff --git a/src/MOD_PARTIT.F90 b/src/MOD_PARTIT.F90 index fb4a88542..d64183086 100644 --- a/src/MOD_PARTIT.F90 +++ b/src/MOD_PARTIT.F90 @@ -2,7 +2,7 @@ ! Variables to organize parallel work module MOD_PARTIT USE O_PARAM -USE, intrinsic :: ISO_FORTRAN_ENV +USE, intrinsic :: ISO_FORTRAN_ENV, only : int32 USE MOD_WRITE_BINARY_ARRAYS USE MOD_READ_BINARY_ARRAYS #if defined(_OPENMP) diff --git a/src/MOD_TRACER.F90 b/src/MOD_TRACER.F90 index 0e66e96aa..8082a9b33 100644 --- a/src/MOD_TRACER.F90 +++ b/src/MOD_TRACER.F90 @@ -1,7 +1,7 @@ !========================================================== MODULE MOD_TRACER USE O_PARAM -USE, intrinsic :: ISO_FORTRAN_ENV +USE, intrinsic :: ISO_FORTRAN_ENV, only : int32 USE MOD_WRITE_BINARY_ARRAYS USE MOD_READ_BINARY_ARRAYS IMPLICIT NONE diff --git a/src/fesom_module.F90 b/src/fesom_module.F90 index fea998944..7cab119cf 100755 --- a/src/fesom_module.F90 +++ b/src/fesom_module.F90 @@ -29,6 +29,8 @@ module fesom_main_storage_module use read_mesh_interface use fesom_version_info_module use command_line_options_module + use, intrinsic :: iso_fortran_env, only : real32 + ! Define icepack module #if defined (__icepack) use icedrv_main, only: set_icepack, init_icepack, alloc_icepack diff --git a/src/gen_halo_exchange.F90 b/src/gen_halo_exchange.F90 index ca6f87cb4..bdc94359c 100755 --- a/src/gen_halo_exchange.F90 +++ b/src/gen_halo_exchange.F90 @@ -16,7 +16,7 @@ module g_comm - use, intrinsic :: ISO_FORTRAN_ENV + use, intrinsic :: ISO_FORTRAN_ENV, only: int16, int32, real32, real64 implicit none diff --git a/src/gen_modules_read_NetCDF.F90 b/src/gen_modules_read_NetCDF.F90 index 8118fa78a..581e518d2 100755 --- a/src/gen_modules_read_NetCDF.F90 +++ b/src/gen_modules_read_NetCDF.F90 @@ -12,7 +12,7 @@ subroutine read_other_NetCDF(file, vari, itime, model_2Darray, check_dummy, do_o ! if check_dummy=.true., missing value is replaced with a meaningful value nearby ! if check_dummy=.false., missing value is replaced with 0.0 - use, intrinsic :: ISO_FORTRAN_ENV + use, intrinsic :: ISO_FORTRAN_ENV, only: real64 use g_config use o_param USE MOD_MESH @@ -198,6 +198,7 @@ subroutine read_surf_hydrography_NetCDF(file, vari, itime, model_2Darray, partit USE MOD_PARTIT USE MOD_PARSUP use g_rotate_grid + use, intrinsic :: ISO_FORTRAN_ENV, only: real64 implicit none #include "netcdf.inc" type(t_mesh), intent(in), target :: mesh @@ -312,7 +313,7 @@ end subroutine read_surf_hydrography_NetCDF ! subroutine read_2ddata_on_grid_NetCDF(file, vari, itime, model_2Darray, partit, mesh) - use, intrinsic :: ISO_FORTRAN_ENV + use, intrinsic :: ISO_FORTRAN_ENV, only: real64 use g_config use o_param From 534462518c861b697ad50d2e52f87a5b916a996d Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 11 Oct 2023 00:33:13 +0200 Subject: [PATCH 24/38] Deprecate cmake option BUILD_FESOM_AS_LIBRARY, use ENABLE_IFS_INTERFACE instead --- src/CMakeLists.txt | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ffd62c5a3..538f8b1a2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,7 +58,22 @@ endif() # option to trigger building a library version of FESOM # we do not always build the library along with the executable to avoid having two targets here in the CMakeLists.txt # two targets would allow e.g. setting different compiler options or preprocessor definition, which would be error prone. -option(BUILD_FESOM_AS_LIBRARY "Build a library instead of an executable" OFF) +set( ENABLE_IFS_INTERFACE_DEFAULT OFF ) +if(DEFINED BUILD_FESOM_AS_LIBRARY AND NOT DEFINED ENABLE_IFS_INTERFACE) + message(DEPRECATION "Use ENABLE_IFS_INTERFACE instead of BUILD_FESOM_AS_LIBRARY") + set(ENABLE_IFS_INTERFACE_DEFAULT ${BUILD_FESOM_AS_LIBRARY}) +endif() +option(ENABLE_IFS_INTERFACE "Enable IFS interface" ${ENABLE_IFS_INTERFACE_DEFAULT}) +if(DEFINED FESOM_ENABLE_IFS_INTERFACE) + set(ENABLE_IFS_INTERFACE ${FESOM_ENABLE_IFS_INTERFACE}) # To distinguish option in a nested cmake project (bundle) +endif() +message(STATUS "ENABLE_IFS_INTERFACE: ${ENABLE_IFS_INTERFACE}") + +if(ENABLE_IFS_INTERFACE) + if(USE_ICEPACK) + message(FATAL_ERROR "Could not enable IFS interface as it is incompatible with USE_ICEPACK") + endif() +endif() set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/../.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") @@ -68,7 +83,7 @@ if(${USE_ICEPACK}) file(GLOB sources_Fortran ${src_home}/*.F90 ${src_home}/icepack_drivers/*.F90 ${src_home}/icepack_drivers/Icepack/columnphysics/*.F90) -elseif(${BUILD_FESOM_AS_LIBRARY}) +elseif(ENABLE_IFS_INTERFACE) file(GLOB sources_Fortran ${src_home}/*.F90 ${src_home}/ifs_interface/*.F90) # ICEPACK + LIBRARY NOT SUPPORTED (YET) else() @@ -136,7 +151,8 @@ endif() if(${USE_ICEPACK}) target_compile_definitions(${PROJECT_NAME} PRIVATE __icepack) endif() -if(${BUILD_FESOM_AS_LIBRARY}) + +if(ENABLE_IFS_INTERFACE) target_compile_definitions(${PROJECT_NAME} PRIVATE __ifsinterface) endif() if(${VERBOSE}) From 3bb3b02afe6fa1cc7352170f827435b6bcf4b626 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 10 Oct 2023 11:31:01 +0200 Subject: [PATCH 25/38] Fix cmake configuration for compilation of parms --- cmake/FindBLAS.cmake | 61 ---------------------------------------- lib/parms/CMakeLists.txt | 47 +++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 76 deletions(-) delete mode 100644 cmake/FindBLAS.cmake diff --git a/cmake/FindBLAS.cmake b/cmake/FindBLAS.cmake deleted file mode 100644 index 27c4a76d5..000000000 --- a/cmake/FindBLAS.cmake +++ /dev/null @@ -1,61 +0,0 @@ - - - -# BLAS__FOUND -# BLAS__LIBRARIES -# BLAS__COMPILER_WRAPPER - - -if(CMAKE_C_COMPILER_LOADED) - #check_function_exists(cblas_sgemm SGEMM_C_FOUND) # this is also found if it is in the standard library paths, so we can not be sure we already have it in the wrapper - - include(CheckCSourceCompiles) - set(CMAKE_REQUIRED_FLAGS -c) - check_c_source_compiles([[ - #include - typedef void (*foo)(const enum CBLAS_ORDER, const enum CBLAS_TRANSPOSE, - const enum CBLAS_TRANSPOSE, const int, const int, - const int, const float, const float *, - const int, const float *, const int, - const float, float *, const int); - foo func = &cblas_sgemm; -]] BLAS_C_COMPILER_WRAPPER) - - if(NOT BLAS_C_COMPILER_WRAPPER) - # try to locate BLAS via the find module shipped with cmake - # the module does not search for C based blas if the Fortran language is enabled - include(${CMAKE_ROOT}/Modules/FindBLAS.cmake) - set(BLAS_C_LIBRARIES ${BLAS_LIBRARIES}) - # unset the results from the original FindBlas.cmake - unset(BLAS_FOUND) - unset(BLAS_LIBRARIES) - else() - set(BLAS_C_LIBRARIES "") # we do not want to explicitly add libraries as the wrapper will do it - set(BLAS_C_FOUND TRUE) - endif() - - if(BLAS_C_LIBRARIES) - set(BLAS_C_FOUND TRUE) - endif() -endif() - - -# include(${CMAKE_ROOT}/Modules/FindBLAS.cmake) -# This module sets the following variables: -# -# :: -# -# BLAS_FOUND - set to true if a library implementing the BLAS interface -# is found -# BLAS_LINKER_FLAGS - uncached list of required linker flags (excluding -l -# and -L). -# BLAS_LIBRARIES - uncached list of libraries (using full path name) to -# link against to use BLAS -# BLAS95_LIBRARIES - uncached list of libraries (using full path name) -# to link against to use BLAS95 interface -# BLAS95_FOUND - set to true if a library implementing the BLAS f95 interface -# is found -# BLA_STATIC if set on this determines what kind of linkage we do (static) -# BLA_VENDOR if set checks only the specified vendor, if not set checks -# all the possibilities -# BLA_F95 if set on tries to find the f95 interfaces for BLAS/LAPACK diff --git a/lib/parms/CMakeLists.txt b/lib/parms/CMakeLists.txt index 62ba062fa..2fa851a43 100644 --- a/lib/parms/CMakeLists.txt +++ b/lib/parms/CMakeLists.txt @@ -1,32 +1,49 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.9) project(parms C) +# Set location to look for find_package modules +set( CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../cmake ) + +# Set -fPIC flag etc. +set( CMAKE_POSITION_INDEPENDENT_CODE ON ) + # get our source files set(src_home ${CMAKE_CURRENT_LIST_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt file(GLOB all_sources ${src_home}/src/*.c ${src_home}/src/DDPQ/*.c) -include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/FindBLAS.cmake") +if( NOT LAPACK_LIBRARIES ) + # First check if we have the Cray libraries + set( _cray_libsci_loaded $ENV{CRAY_LIBSCI_DIR} ) + if( _cray_libsci_loaded ) + # set( _CRAY_PRGENV $ENV{PE_ENV} ) + # string( TOLOWER "${_CRAY_PRGENV}" _cray_prgenv ) + # set( LAPACK_LIBRARIES sci_${_cray_prgenv} ) + else() + find_package(LAPACK REQUIRED) # cannot compile without! + endif() +endif() +if( NOT TARGET MPI::MPI_C ) + find_package(MPI REQUIRED COMPONENTS C) +endif() # create our library (set its name to name of this project) -add_library(${PROJECT_NAME} SHARED ${all_sources}) +add_library(${PROJECT_NAME} ${all_sources}) -target_compile_definitions(${PROJECT_NAME} - PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 HAS_BLAS) +target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8) target_include_directories(${PROJECT_NAME} - PUBLIC $ - PRIVATE ${src_home}/src/include + PRIVATE ${src_home}/src/include + PUBLIC $ ) -target_link_libraries(${PROJECT_NAME} PRIVATE ${BLAS_C_LIBRARIES} $ENV{UBUNTU_BLAS_LIBRARY}) +target_link_libraries(${PROJECT_NAME} PRIVATE ${LAPACK_LIBRARIES}) +target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_C) if(${CMAKE_C_COMPILER_ID} STREQUAL "Intel") - target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-div -no-prec-sqrt -fast-transcendentals -fp-model precise) - if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) - target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) - endif() + target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-div -no-prec-sqrt -fast-transcendentals -fp-model precise) + + if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) + target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) + endif() endif() - -target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) -install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") From 27aea3a53586d9caabb0e966cb7a14f9bd5cb880 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 11 Oct 2023 00:44:49 +0200 Subject: [PATCH 26/38] CMake installation and cleanup --- CMakeLists.txt | 18 ++- cmake/fesom_export.cmake | 59 +++++++ cmake/fesom_setup.cmake | 145 ++++++++++++++++++ lib/parms/CMakeLists.txt | 2 +- src/CMakeLists.txt | 137 +++++++++-------- src/async_threads_cpp/CMakeLists.txt | 24 ++- .../async_threads_module.F90 | 0 src/fesom-config.cmake.in | 36 +++++ 8 files changed, 333 insertions(+), 88 deletions(-) create mode 100644 cmake/fesom_export.cmake create mode 100644 cmake/fesom_setup.cmake rename src/{ => async_threads_cpp}/async_threads_module.F90 (100%) create mode 100644 src/fesom-config.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 60a65d66f..b256b38b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.9) +cmake_minimum_required(VERSION 3.16) # set default build type cache entry (do so before project(...) is called, which would create this cache entry on its own) if(NOT CMAKE_BUILD_TYPE) @@ -8,9 +8,7 @@ endif() project(FESOM2.0) -set(CMAKE_VERBOSE_MAKEFILE ON) - -option(BUILD_SHARED_LIBS "Build using shared libraries" OFF) # cmake-internal switch to toggle if library targets are being build as STATIC or SHARED, see https://cmake.org/cmake/help/latest/guide/tutorial/Selecting%20Static%20or%20Shared%20Libraries.html +set(BUILD_SHARED_LIBS ON CACHE BOOL "Default to using shared libs") set(TOPLEVEL_DIR ${CMAKE_CURRENT_LIST_DIR}) set(FESOM_COUPLED OFF CACHE BOOL "compile fesom standalone or with oasis support (i.e. coupled)") set(OIFS_COUPLED OFF CACHE BOOL "compile fesom coupled to OpenIFS. (Also needs FESOM_COUPLED to work)") @@ -21,3 +19,15 @@ set(OPENMP_REPRODUCIBLE OFF CACHE BOOL "serialize OpenMP loops that are critical #set(VERBOSE OFF CACHE BOOL "toggle debug output") #add_subdirectory(oasis3-mct/lib/psmile) add_subdirectory(src) + +foreach( _file fesom-config.cmake fesom-config-version.cmake fesom-targets.cmake ) + execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ${PROJECT_BINARY_DIR}/src/${_file} ${PROJECT_BINARY_DIR}/${_file} ) +endforeach() + +# Define ${PROJECT_NAME}_DIR in PARENT_SCOPE so that a `find_package( )` in a bundle +# will easily find the project without requiring a `HINT _BINARY_DIR` argument +if( NOT CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR ) + # Guard needed because PARENT_SCOPE cannot be used in top-level CMake project + + set( fesom_DIR ${fesom_DIR} PARENT_SCOPE ) +endif() diff --git a/cmake/fesom_export.cmake b/cmake/fesom_export.cmake new file mode 100644 index 000000000..7a57f34b7 --- /dev/null +++ b/cmake/fesom_export.cmake @@ -0,0 +1,59 @@ + +macro( fesom_export ) + +set( options ) +set( single_value_args ) +set( multi_value_args TARGETS ) + +cmake_parse_arguments( _PAR "${options}" "${single_value_args}" "${multi_value_args}" ${_FIRST_ARG} ${ARGN} ) + +set( PROJECT_TARGETS_FILE "${PROJECT_BINARY_DIR}/${PROJECT_NAME}-targets.cmake" ) +file( REMOVE ${PROJECT_TARGETS_FILE} ) + + +foreach( tgt ${_PAR_TARGETS} ) + install( TARGETS ${tgt} + EXPORT ${PROJECT_NAME}-targets + RUNTIME DESTINATION ${INSTALL_BIN_DIR} + LIBRARY DESTINATION ${INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${INSTALL_LIB_DIR} ) + set_target_properties( ${tgt} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib ) + export( TARGETS ${tgt} APPEND FILE "${PROJECT_TARGETS_FILE}" ) +endforeach() + +include(CMakePackageConfigHelpers) +write_basic_package_version_file( + "${PROJECT_NAME}-config-version.cmake" + VERSION ${${PROJECT_NAME}_VERSION} + COMPATIBILITY AnyNewerVersion) + +if( EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}-config.cmake.in ) + configure_package_config_file(${PROJECT_NAME}-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake + INSTALL_DESTINATION ${INSTALL_CMAKE_DIR}) +else() + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake.in "include(${CMAKE_CURRENT_LIST_DIR}/${PROJECT_NAME}-targets.cmake)") + configure_package_config_file(${PROJECT_NAME}-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake + INSTALL_DESTINATION ${INSTALL_CMAKE_DIR}) +endif() + +install(EXPORT ${PROJECT_NAME}-targets DESTINATION "${INSTALL_CMAKE_DIR}") +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake + DESTINATION "${INSTALL_CMAKE_DIR}") + +install( DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/${CMAKE_CFG_INTDIR} + DESTINATION module/${PROJECT_NAME} + COMPONENT modules ) + +# Define ${PROJECT_NAME}_DIR in PARENT_SCOPE so that a `find_package( )` in a bundle +# will easily find the project without requiring a `HINT _BINARY_DIR` argument [ECBUILD-460] +if( NOT CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR ) + # Guard needed because PARENT_SCOPE cannot be used in top-level CMake project + + set( ${PROJECT_NAME}_DIR ${PROJECT_BINARY_DIR} PARENT_SCOPE ) +endif() + +endmacro() diff --git a/cmake/fesom_setup.cmake b/cmake/fesom_setup.cmake new file mode 100644 index 000000000..e916074a5 --- /dev/null +++ b/cmake/fesom_setup.cmake @@ -0,0 +1,145 @@ +# To a large degree this follows the ecbuild setup +message("[${PROJECT_NAME}]") + +### Setup project + +if(CMAKE_VERSION VERSION_LESS 3.21) + get_property(not_top DIRECTORY PROPERTY PARENT_DIRECTORY) + if(NOT not_top) + set(PROJECT_IS_TOP_LEVEL true) + endif() +endif() + +set(BUILD_SHARED_LIBS ON CACHE BOOL "Default to using shared libs") +set(CMAKE_LINK_DEPENDS_NO_SHARED ON) # relink of downstream libraries not required when shared library is rebuilt + +# Default build type: Release +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.") + +# Set location to look for find_package modules +set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake ${CMAKE_MODULE_PATH}) + +# Set Fortran module directory +set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${PROJECT_NAME} ) + +# Build-dir destinations +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# Set install directories +include(GNUInstallDirs) +if( NOT INSTALL_BIN_DIR ) + set( INSTALL_BIN_DIR ${CMAKE_INSTALL_BINDIR} ) +endif() +if( NOT INSTALL_LIB_DIR ) + set( INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR} ) +endif() +if( NOT INSTALL_INCLUDE_DIR ) + set( INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR} ) +endif() +set( INSTALL_CMAKE_DIR ${INSTALL_LIB_DIR}/cmake/${PROJECT_NAME} ) + +mark_as_advanced( INSTALL_BIN_DIR ) +mark_as_advanced( INSTALL_LIB_DIR ) +mark_as_advanced( INSTALL_INCLUDE_DIR ) +mark_as_advanced( INSTALL_CMAKE_DIR ) + +# make sure nothing breaks if INSTALL_LIB_DIR is not lib +if( NOT INSTALL_LIB_DIR STREQUAL "lib" AND NOT EXISTS ${CMAKE_BINARY_DIR}/${INSTALL_LIB_DIR} ) + execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink lib ${CMAKE_BINARY_DIR}/${INSTALL_LIB_DIR} ) +endif() + +# for macosx use @rpath in a target’s install name (CMP0042) +set( CMAKE_MACOSX_RPATH ON ) + +# add the automatic parts to RPATH which point to dirs outside build tree +set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE ) + +# use RPATHs for the build tree +set( CMAKE_SKIP_BUILD_RPATH FALSE ) + +# build with *relative* rpaths by default +option( ENABLE_RPATHS "when installing insert RPATHS into binaries" ON ) +option( ENABLE_RELATIVE_RPATHS "try to use relative RPATHS, including build dir" ON ) +if( ENABLE_RELATIVE_RPATHS ) + set( CMAKE_BUILD_WITH_INSTALL_RPATH TRUE ) +else() + # in case the RPATH is absolute, the install RPATH cannot be set + # at build-time since it breaks the build tree dynamic links + set( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE ) +endif() + + +foreach( p LIB BIN INCLUDE DATA CMAKE ) + set( var INSTALL_${p}_DIR ) + set( ${PROJECT_NAME}_FULL_INSTALL_${p}_DIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_${p}_DIR}" + CACHE INTERNAL "${PROJECT_NAME} ${p} full install path" ) +endforeach() + + +function( _path_append var path ) + list( FIND ${var} ${path} _found ) + if( _found EQUAL "-1" ) + list( APPEND ${var} ${path}) + endif() + set( ${var} "${${var}}" PARENT_SCOPE ) # +endfunction() + +function( _make_relative_rpath_entry entry var ) + + if( CMAKE_SYSTEM_NAME MATCHES "Darwin" ) + set( ${var} "@loader_path/${entry}" PARENT_SCOPE ) + + elseif( CMAKE_SYSTEM_NAME MATCHES "FreeBSD|Linux|SunOS" ) + set( ${var} "$ORIGIN/${entry}" PARENT_SCOPE ) + + elseif( CMAKE_SYSTEM_NAME MATCHES "AIX" ) # always relative to executable path + set( ${var} "${entry}" PARENT_SCOPE ) + + else() + set( ${var} "${CMAKE_INSTALL_PREFIX}/${entry}" PARENT_SCOPE ) + + endif() +endfunction() + +macro( append_to_rpath RPATH_DIRS ) + + foreach( RPATH_DIR ${RPATH_DIRS} ) + + if( NOT ${RPATH_DIR} STREQUAL "" ) + + file( TO_CMAKE_PATH ${RPATH_DIR} RPATH_DIR ) # sanitize the path + + if( IS_ABSOLUTE ${RPATH_DIR} ) + _path_append( CMAKE_INSTALL_RPATH "${RPATH_DIR}" ) + else() + _make_relative_rpath_entry( "${RPATH_DIR}" rpath_dir_rel ) + _path_append( CMAKE_INSTALL_RPATH ${rpath_dir_rel} ) + endif() + + endif() + + endforeach() + +endmacro( append_to_rpath ) + + +if( ENABLE_RPATHS ) + if( ENABLE_RELATIVE_RPATHS ) + file( RELATIVE_PATH relative_rpath ${CMAKE_INSTALL_PREFIX}/${INSTALL_BIN_DIR} ${CMAKE_INSTALL_PREFIX}/${INSTALL_LIB_DIR} ) + append_to_rpath( ${relative_rpath} ) + else() # make rpaths absolute + append_to_rpath( "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIB_DIR}" ) + endif() +endif() + +# put the include dirs which are in the source or build tree +# before all other include dirs, so the headers in the sources +# are prefered over the already installed ones (since cmake 2.4.1) +set( CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE ON ) + +# Set -fPIC flag etc. +set( CMAKE_POSITION_INDEPENDENT_CODE ON ) + +include(fesom_export) diff --git a/lib/parms/CMakeLists.txt b/lib/parms/CMakeLists.txt index 2fa851a43..f81cc26fb 100644 --- a/lib/parms/CMakeLists.txt +++ b/lib/parms/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.9) +cmake_minimum_required(VERSION 3.16) project(parms C) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 538f8b1a2..db9fcfc35 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,8 +1,10 @@ -cmake_minimum_required(VERSION 3.9) +cmake_minimum_required(VERSION 3.16) -project(fesom C Fortran) +project(fesom LANGUAGES C CXX Fortran VERSION 2.0.0) -find_package(MPI REQUIRED COMPONENTS C CXX Fortran) +set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake ${CMAKE_MODULE_PATH}) + +include(fesom_setup) if(DEFINED ENV{FESOM_PLATFORM_STRATEGY}) set(FESOM_PLATFORM_STRATEGY $ENV{FESOM_PLATFORM_STRATEGY} CACHE STRING "switch to platform specific compile settings, this is usually determined via the env.sh script") @@ -48,13 +50,19 @@ else() endif() option(ENABLE_OPENACC "compile with OpenACC support" OFF) +message(STATUS "ENABLE_OPENACC: ${ENABLE_OPENACC}") + set(NV_GPU_ARCH "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)") option(ENABLE_OPENMP "build FESOM with OpenMP" OFF) -if(${ENABLE_OPENMP}) - find_package(OpenMP REQUIRED) +message(STATUS "ENABLE_OPENMP: ${ENABLE_OPENMP}") +if(ENABLE_OPENMP) + find_package(OpenMP REQUIRED COMPONENTS Fortran) endif() +option(USE_ICEPACK "Use ICEPACK" OFF) +message(STATUS "USE_ICEPACK: ${USE_ICEPACK}") + # option to trigger building a library version of FESOM # we do not always build the library along with the executable to avoid having two targets here in the CMakeLists.txt # two targets would allow e.g. setting different compiler options or preprocessor definition, which would be error prone. @@ -78,8 +86,8 @@ endif() set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/../.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") # get our source files -set(src_home ${CMAKE_CURRENT_LIST_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt -if(${USE_ICEPACK}) +set(src_home ${CMAKE_CURRENT_SOURCE_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt +if(USE_ICEPACK) file(GLOB sources_Fortran ${src_home}/*.F90 ${src_home}/icepack_drivers/*.F90 ${src_home}/icepack_drivers/Icepack/columnphysics/*.F90) @@ -89,9 +97,7 @@ elseif(ENABLE_IFS_INTERFACE) else() file(GLOB sources_Fortran ${src_home}/*.F90) endif() -#list(REMOVE_ITEM sources_Fortran ${src_home}/fesom_partition_init.F90) file(GLOB sources_C ${src_home}/*.c) -list(REMOVE_ITEM sources_C ${src_home}/psolve_feom.c) # does the file still exist? # generate a custom file from fesom_version_info.F90 which includes the current git SHA set(FESOM_ORIGINAL_VERSION_FILE ${src_home}/fesom_version_info.F90) @@ -103,13 +109,16 @@ add_custom_command(OUTPUT 5303B6F4_E4F4_45B2_A6E5_8E2B9FB5CDC4 ${FESOM_GENERATED WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMENT "determining ${PROJECT_NAME} git SHA ...") -#if(${FESOM_STANDALONE}) +#if(FESOM_STANDALONE) # list(REMOVE_ITEM sources_Fortran ${src_home}/cpl_driver.F90) #endif() list(REMOVE_ITEM sources_Fortran ${src_home}/fvom_init.F90 ${src_home}/oce_local.F90 ${src_home}/gen_comm.F90) list(REMOVE_ITEM sources_C ${src_home}/fort_part.c) list(REMOVE_ITEM sources_Fortran ${src_home}/fesom_main.F90) +find_package( NETCDF REQUIRED COMPONENTS C Fortran ) +find_package( MPI REQUIRED COMPONENTS C Fortran ) + # depends on the metis library #add_subdirectory(../lib/metis-5.1.0 ${PROJECT_BINARY_DIR}/metis) #include_directories(../lib/metis-5.1.0/include) @@ -118,56 +127,66 @@ add_subdirectory(../lib/parms ${PROJECT_BINARY_DIR}/parms) add_subdirectory(async_threads_cpp) -include(${CMAKE_CURRENT_LIST_DIR}/../cmake/FindNETCDF.cmake) - -if(${BUILD_FESOM_AS_LIBRARY}) - add_library(${PROJECT_NAME}_C ${sources_C}) -else() add_library(${PROJECT_NAME}_C ${sources_C}) -endif() target_compile_definitions(${PROJECT_NAME}_C PRIVATE PARMS USE_MPI REAL=double DBL HAS_BLAS FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 SGI LINUX UNDER_ MPI2) -target_link_libraries(${PROJECT_NAME}_C PUBLIC parms) #metis +target_link_libraries(${PROJECT_NAME}_C PRIVATE parms) #metis +target_link_libraries(${PROJECT_NAME}_C PRIVATE MPI::MPI_C) -# create our binary or library (set its name to name of this project) -# we do not always build the library along with the executable to avoid having two targets here in the CMakeLists.txt -# two targets would allow e.g. setting different compiler options or preprocessor definition, which would be error prone. -if(${BUILD_FESOM_AS_LIBRARY}) - add_library(${PROJECT_NAME} ${sources_Fortran}) -else() - add_executable(${PROJECT_NAME} ${sources_Fortran} ${src_home}/fesom_main.F90) -endif() + +# fesom library +add_library(${PROJECT_NAME} ${sources_Fortran}) target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS -DMETIS_VERSION=5 -DPART_WEIGHTED -DMETISRANDOMSEED=35243 __MULTIO) -if(${DISABLE_MULTITHREADING}) + +target_include_directories(${PROJECT_NAME} PUBLIC $) +target_include_directories(${PROJECT_NAME} PUBLIC $) +target_include_directories(${PROJECT_NAME} PRIVATE ${NETCDF_Fortran_INCLUDE_DIRECTORIES} ${OASIS_Fortran_INCLUDE_DIRECTORIES}) +target_include_directories(${PROJECT_NAME} PRIVATE ${MCT_Fortran_INCLUDE_DIRECTORIES} ${MPEU_Fortran_INCLUDE_DIRECTORIES}) +target_include_directories(${PROJECT_NAME} PRIVATE ${SCRIP_Fortran_INCLUDE_DIRECTORIES}) +target_link_libraries(${PROJECT_NAME} PRIVATE parms) #metis +target_link_libraries(${PROJECT_NAME} PRIVATE async_threads) +target_link_libraries(${PROJECT_NAME} PRIVATE multio-fapi) +target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_Fortran) +target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) +target_link_libraries(${PROJECT_NAME} PRIVATE ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) +set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) +if(ENABLE_OPENMP AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) + target_link_libraries(${PROJECT_NAME} PRIVATE OpenMP::OpenMP_Fortran) +endif() + + +if(DISABLE_MULTITHREADING) target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_MULTITHREADING) endif() -if(${FESOM_COUPLED}) - include(${CMAKE_CURRENT_LIST_DIR}/../cmake/FindOASIS.cmake) + +if(FESOM_COUPLED) + find_package(OASIS REQUIRED) target_compile_definitions(${PROJECT_NAME} PRIVATE __oasis) endif() -if(${OIFS_COUPLED}) + +if(OIFS_COUPLED) target_compile_definitions(${PROJECT_NAME} PRIVATE __oifs) endif() -if(${USE_ICEPACK}) + +if(USE_ICEPACK) target_compile_definitions(${PROJECT_NAME} PRIVATE __icepack) endif() if(ENABLE_IFS_INTERFACE) target_compile_definitions(${PROJECT_NAME} PRIVATE __ifsinterface) endif() -if(${VERBOSE}) + +if(VERBOSE) target_compile_definitions(${PROJECT_NAME} PRIVATE VERBOSE) endif() -if(${OPENMP_REPRODUCIBLE}) + +if(OPENMP_REPRODUCIBLE) target_compile_definitions(${PROJECT_NAME} PRIVATE __openmp_reproducible) endif() + # CMAKE_Fortran_COMPILER_ID will also work if a wrapper is being used (e.g. mpif90 wraps ifort -> compiler id is Intel) if(${CMAKE_Fortran_COMPILER_ID} STREQUAL Intel ) - if(${BUILD_FESOM_AS_LIBRARY}) - target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin -fpe0 -g -traceback) # add -fpe0 for RAPS environment - else() - target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin -g -traceback) - endif() + target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -xHost -ip -init=zero -no-wrap-margin -fpe0) # add -fpe0 for RAPS environment if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL albedo) @@ -186,51 +205,33 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL GNU ) target_compile_options(${PROJECT_NAME} PRIVATE -fallow-argument-mismatch) # gfortran v10 is strict about erroneous API calls: "Rank mismatch between actual argument at (1) and actual argument at (2) (scalar and rank-1)" endif() elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray ) - if(${ENABLE_OPENMP}) - target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -homp) + target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023) + if(ENABLE_OPENMP) + target_compile_options(${PROJECT_NAME} PRIVATE -homp) else() - target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -hnoomp) + target_compile_options(${PROJECT_NAME} PRIVATE -hnoomp) endif() elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC ) target_compile_definitions(${PROJECT_NAME} PRIVATE ENABLE_NVHPC_WORKAROUNDS) target_compile_options(${PROJECT_NAME} PRIVATE -fast -fastsse -O3 -Mallocatable=95 -Mr8 -pgf90libs) - if(${ENABLE_OPENACC}) + if(ENABLE_OPENACC) # additional compiler settings target_compile_options(${PROJECT_NAME} PRIVATE -acc -ta=tesla:${NV_GPU_ARCH} -Minfo=accel) set(CMAKE_EXE_LINKER_FLAGS "-acc -ta=tesla:${NV_GPU_ARCH}") endif() - if(${ENABLE_OPENMP}) + if(ENABLE_OPENMP) target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast) else() target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline) endif() endif() -if(${BUILD_FESOM_AS_LIBRARY}) - target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) - target_compile_options(${PROJECT_NAME}_C PRIVATE -fPIC) -endif() -target_include_directories(${PROJECT_NAME} PRIVATE ${NETCDF_Fortran_INCLUDE_DIRECTORIES} ${OASIS_Fortran_INCLUDE_DIRECTORIES}) -target_include_directories(${PROJECT_NAME} PRIVATE ${MCT_Fortran_INCLUDE_DIRECTORIES} ${MPEU_Fortran_INCLUDE_DIRECTORIES}) -target_include_directories(${PROJECT_NAME} PRIVATE ${SCRIP_Fortran_INCLUDE_DIRECTORIES}) -target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) -target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_C ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) -target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_Fortran) -target_link_libraries(${PROJECT_NAME} PRIVATE async_threads_cpp multio-fapi) -set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) -if(${ENABLE_OPENMP} AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) - target_compile_options(${PROJECT_NAME} PRIVATE ${OpenMP_Fortran_FLAGS}) # currently we only have OpenMP in the Fortran part - target_link_libraries(${PROJECT_NAME} PRIVATE OpenMP::OpenMP_Fortran) -endif() +# fesom.x executable +add_executable(${PROJECT_NAME}.x ${src_home}/fesom_main.F90) +target_link_libraries(${PROJECT_NAME}.x PUBLIC ${PROJECT_NAME}) -#set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}/.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") -if(${BUILD_FESOM_AS_LIBRARY}) - install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") - install(TARGETS ${PROJECT_NAME}_C EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") -else() - set(FESOM_INSTALL_FILEPATH "${FESOM_INSTALL_PREFIX}/bin/fesom.x") - get_filename_component(FESOM_INSTALL_PATH ${FESOM_INSTALL_FILEPATH} DIRECTORY) - get_filename_component(FESOM_INSTALL_NAME ${FESOM_INSTALL_FILEPATH} NAME) - install(PROGRAMS ${PROJECT_BINARY_DIR}/${PROJECT_NAME} DESTINATION ${FESOM_INSTALL_PATH} RENAME ${FESOM_INSTALL_NAME}) -endif() + +### Export and installation + +fesom_export(TARGETS ${PROJECT_NAME}_C ${PROJECT_NAME} parms async_threads fesom.x) diff --git a/src/async_threads_cpp/CMakeLists.txt b/src/async_threads_cpp/CMakeLists.txt index f54186d8f..619f0492f 100644 --- a/src/async_threads_cpp/CMakeLists.txt +++ b/src/async_threads_cpp/CMakeLists.txt @@ -1,24 +1,18 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.16) -project(async_threads_cpp CXX C Fortran) # the FortranCInterface requires the C language to be enabled +project(async_threads CXX C Fortran) # the FortranCInterface requires the C language to be enabled + +set (CMAKE_CXX_STANDARD 11) # get our source files -file(GLOB sources_CXX ${CMAKE_CURRENT_LIST_DIR}/*.cpp) +file(GLOB sources ${CMAKE_CURRENT_LIST_DIR}/*.cpp) include(FortranCInterface) FortranCInterface_HEADER(ThreadsManagerFCMacros.h MACRO_NAMESPACE "ThreadsManagerFCMacros_" SYMBOLS init_ccall begin_ccall end_ccall) -add_library(${PROJECT_NAME} SHARED ${sources_CXX}) - +add_library(${PROJECT_NAME} ${sources} async_threads_module.F90) target_include_directories(${PROJECT_NAME} - PRIVATE ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_BINARY_DIR} + PRIVATE $ + PRIVATE $ + PUBLIC $ ) - -if(${CMAKE_CXX_COMPILER_ID} STREQUAL Cray ) - target_compile_options(${PROJECT_NAME} PRIVATE -hstd=c++11) -else() - target_compile_options(${PROJECT_NAME} PRIVATE -std=c++11) -endif() - -target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) -install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") diff --git a/src/async_threads_module.F90 b/src/async_threads_cpp/async_threads_module.F90 similarity index 100% rename from src/async_threads_module.F90 rename to src/async_threads_cpp/async_threads_module.F90 diff --git a/src/fesom-config.cmake.in b/src/fesom-config.cmake.in new file mode 100644 index 000000000..c0735281f --- /dev/null +++ b/src/fesom-config.cmake.in @@ -0,0 +1,36 @@ +# This file will get included upon `find_package(fesom ... )` + +@PACKAGE_INIT@ + +### insert definitions for IMPORTED targets +if(NOT @PROJECT_NAME@_BINARY_DIR) + find_file(@PROJECT_NAME@_TARGETS_FILE + NAMES @PROJECT_NAME@-targets.cmake + HINTS ${CMAKE_CURRENT_LIST_DIR} @PACKAGE_TARGETS_DIRS@ + NO_DEFAULT_PATH) + if(@PROJECT_NAME@_TARGETS_FILE) + include(${@PROJECT_NAME@_TARGETS_FILE}) + endif() +endif() + +include(CMakeFindDependencyMacro) +if( NOT (TARGET MPI::MPI_C AND TARGET MPI::MPI_Fortran) ) + enable_language(C) + enable_language(Fortran) + find_dependency(MPI COMPONENTS C Fortran) +endif() +if( @ENABLE_OPENMP@ ) + if( NOT TARGET OpenMP::OpenMP_Fortran ) + enable_language(Fortran) + find_dependency(OpenMP COMPONENTS Fortran) + endif() + set( fesom_HAVE_OPENMP 1 ) +else() + set( fesom_HAVE_OPENMP 0 ) +endif() + +if( @ENABLE_IFS_INTERFACE@ ) + set( fesom_HAVE_IFS_INTERFACE 1 ) +else() + set( fesom_HAVE_IFS_INTERFACE 0 ) +endif() From ceec5082957d2e04f5f029a91549713d563dabcf Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Thu, 24 Aug 2023 10:46:41 +0200 Subject: [PATCH 27/38] Avoid recompilation of fesom_version_info-generate.F90 if git sha did not change --- src/CMakeLists.txt | 11 +++++++---- src/GitRepositoryInfo.cmake | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index db9fcfc35..e3208b4ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -104,10 +104,12 @@ set(FESOM_ORIGINAL_VERSION_FILE ${src_home}/fesom_version_info.F90) set(FESOM_GENERATED_VERSION_FILE ${CMAKE_CURRENT_BINARY_DIR}/fesom_version_info-generated.F90) list(REMOVE_ITEM sources_Fortran ${FESOM_ORIGINAL_VERSION_FILE}) # we want to compile the generated file instead list(APPEND sources_Fortran ${FESOM_GENERATED_VERSION_FILE}) -add_custom_command(OUTPUT 5303B6F4_E4F4_45B2_A6E5_8E2B9FB5CDC4 ${FESOM_GENERATED_VERSION_FILE} # the first arg to OUTPUT is a name for a file we never create to make sure this command will run on every re-build (let our file be the second arg, as the first file is inadvertently removed by make) - COMMAND ${CMAKE_COMMAND} -DFESOM_ORIGINAL_VERSION_FILE=${FESOM_ORIGINAL_VERSION_FILE} -DFESOM_GENERATED_VERSION_FILE=${FESOM_GENERATED_VERSION_FILE} -P GitRepositoryInfo.cmake - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - COMMENT "determining ${PROJECT_NAME} git SHA ...") +set_source_files_properties(${FESOM_GENERATED_VERSION_FILE} PROPERTIES GENERATED TRUE) + +add_custom_target(fesom_version_info-generated.F90 ALL + COMMENT "Determining ${PROJECT_NAME} git SHA ..." + COMMAND ${CMAKE_COMMAND} -DFESOM_ORIGINAL_VERSION_FILE=${FESOM_ORIGINAL_VERSION_FILE} -DFESOM_GENERATED_VERSION_FILE=${FESOM_GENERATED_VERSION_FILE} -P GitRepositoryInfo.cmake + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) #if(FESOM_STANDALONE) # list(REMOVE_ITEM sources_Fortran ${src_home}/cpl_driver.F90) @@ -136,6 +138,7 @@ target_link_libraries(${PROJECT_NAME}_C PRIVATE MPI::MPI_C) # fesom library add_library(${PROJECT_NAME} ${sources_Fortran}) +add_dependencies(${PROJECT_NAME} fesom_version_info-generated.F90) target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS -DMETIS_VERSION=5 -DPART_WEIGHTED -DMETISRANDOMSEED=35243 __MULTIO) target_include_directories(${PROJECT_NAME} PUBLIC $) diff --git a/src/GitRepositoryInfo.cmake b/src/GitRepositoryInfo.cmake index e5eb37c26..4f6a8172e 100644 --- a/src/GitRepositoryInfo.cmake +++ b/src/GitRepositoryInfo.cmake @@ -9,4 +9,6 @@ else() message("git not found, setting FESOM_GIT_SHA to: ${FESOM_GIT_SHA}") endif() -configure_file(${FESOM_ORIGINAL_VERSION_FILE} ${FESOM_GENERATED_VERSION_FILE} @ONLY) +configure_file(${FESOM_ORIGINAL_VERSION_FILE} ${FESOM_GENERATED_VERSION_FILE}.tmp @ONLY) +execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FESOM_GENERATED_VERSION_FILE}.tmp ${FESOM_GENERATED_VERSION_FILE}) +execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${FESOM_GENERATED_VERSION_FILE}.tmp) From ec82fb1f24c9338eb2122562b4f8db48f2c974b0 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Wed, 11 Oct 2023 00:03:53 +0200 Subject: [PATCH 28/38] Add test_downstream --- test_downstream/.gitignore | 1 + test_downstream/CMakeLists.txt | 37 ++++++++++++++++++++++++++++++++++ test_downstream/README | 9 +++++++++ test_downstream/main.F90 | 5 +++++ 4 files changed, 52 insertions(+) create mode 100644 test_downstream/.gitignore create mode 100644 test_downstream/CMakeLists.txt create mode 100644 test_downstream/README create mode 100644 test_downstream/main.F90 diff --git a/test_downstream/.gitignore b/test_downstream/.gitignore new file mode 100644 index 000000000..378eac25d --- /dev/null +++ b/test_downstream/.gitignore @@ -0,0 +1 @@ +build diff --git a/test_downstream/CMakeLists.txt b/test_downstream/CMakeLists.txt new file mode 100644 index 000000000..390aea563 --- /dev/null +++ b/test_downstream/CMakeLists.txt @@ -0,0 +1,37 @@ +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) + +project(test_downstream LANGUAGES Fortran) + +find_package(fesom REQUIRED) + +message("fesom found: ${fesom_DIR}") +message("fesom_VERSION: ${fesom_VERSION}") +message("fesom_HAVE_OPENMP: ${fesom_HAVE_OPENMP}") +message("fesom_HAVE_IFS_INTERFACE: ${fesom_HAVE_IFS_INTERFACE}") + +if(NOT TARGET fesom) + message(FATAL_ERROR "fesom was installed wrongly: fesom target not found") +endif() + +if (ASSERT_FESOM_DIR) + if (NOT fesom_DIR MATCHES "${ASSERT_FESOM_DIR}") + message(FATAL_ERROR "fesom was not found in expected location ${ASSERT_FESOM_DIR}") + endif() +endif() + +if (DEFINED ASSERT_HAVE_IFS_INTERFACE) + if (ASSERT_HAVE_IFS_INTERFACE AND NOT fesom_HAVE_IFS_INTERFACE) + message(FATAL_ERROR "fesom was installed wrongly: fesom_HAVE_IFS_INTERFACE=${fesom_HAVE_IFS_INTERFACE}") + elseif (NOT ASSERT_HAVE_IFS_INTERFACE AND fesom_HAVE_IFS_INTERFACE) + message(FATAL_ERROR "fesom was installed wrongly: fesom_HAVE_IFS_INTERFACE=${fesom_HAVE_IFS_INTERFACE}") + endif() +endif() + +get_target_property(target_type fesom TYPE) +if( NOT target_type MATCHES "STATIC_LIBRARY|SHARED_LIBRARY") + message(FATAL_ERROR "fesom target's TYPE is expected to be STATIC_LIBRARY or SHARED_LIBRARY. Actual TYPE: ${target_type}") +endif () + +add_executable( main main.F90 ) +target_link_libraries( main fesom ) + diff --git a/test_downstream/README b/test_downstream/README new file mode 100644 index 000000000..e46358cd9 --- /dev/null +++ b/test_downstream/README @@ -0,0 +1,9 @@ +This dummy project can be used to test if fesom was installed and can be found correctly in downstream CMake projects + +Example use: + + export fesom_ROOT= # can be install or build dir!!! + cmake -S . -B build + cmake --build build + build/main + diff --git a/test_downstream/main.F90 b/test_downstream/main.F90 new file mode 100644 index 000000000..3fe20d678 --- /dev/null +++ b/test_downstream/main.F90 @@ -0,0 +1,5 @@ +program main +use mod_mesh + + +end program From dbbd86d9d984b810ba64d7d3d7affc1c377b6cc6 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 30 Oct 2023 16:19:00 +0100 Subject: [PATCH 29/38] Fix compilation in iom related to incompatible logical type --- src/ifs_interface/iom.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ifs_interface/iom.F90 b/src/ifs_interface/iom.F90 index a7778834a..4548dca03 100644 --- a/src/ifs_interface/iom.F90 +++ b/src/ifs_interface/iom.F90 @@ -109,9 +109,9 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) CALL ctl_stop( 'setting multio failure handler failed: ', multio_error_string(err)) end if - err = conf_ctx%mpi_allow_world_default_comm(.FALSE._1) + err = conf_ctx%mpi_allow_world_default_comm(.FALSE.) IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('conf_ctx%mpi_allow_world_default_comm(.FALSE._1) failed: ', multio_error_string(err)) + CALL ctl_stop('conf_ctx%mpi_allow_world_default_comm(.FALSE.) failed: ', multio_error_string(err)) END IF err = conf_ctx%mpi_return_client_comm(return_comm) @@ -201,9 +201,9 @@ SUBROUTINE iom_init_server(server_comm) CALL ctl_stop('setting multio failure handler failed: ', multio_error_string(err)) END IF - err = conf_ctx%mpi_allow_world_default_comm(.FALSE._1) + err = conf_ctx%mpi_allow_world_default_comm(.FALSE.) IF (err /= MULTIO_SUCCESS) THEN - CALL ctl_stop('conf_ctx%mpi_allow_world_default_comm(.FALSE._1) failed: ', multio_error_string(err)) + CALL ctl_stop('conf_ctx%mpi_allow_world_default_comm(.FALSE.) failed: ', multio_error_string(err)) END IF err = conf_ctx%mpi_parent_comm(int(mio_parent_comm)) From fb4421f0e3911096cda19ec7c9d96656f3d1baae Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 30 Oct 2023 16:18:38 +0100 Subject: [PATCH 30/38] Fix usage of preprocessor definition __MULTIO --- src/ifs_interface/ifs_interface.F90 | 4 +--- src/ifs_interface/mpp_io.F90 | 9 +++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ifs_interface/ifs_interface.F90 b/src/ifs_interface/ifs_interface.F90 index 847fa60ae..4f0f659a9 100644 --- a/src/ifs_interface/ifs_interface.F90 +++ b/src/ifs_interface/ifs_interface.F90 @@ -9,7 +9,6 @@ MODULE nemogcmcoup_steps INTEGER :: substeps !per IFS timestep END MODULE nemogcmcoup_steps -#if defined(__MULTIO) SUBROUTINE nemogcmcoup_init_ioserver( icomm, lnemoioserver, irequired, iprovided, lmpi1) ! Initialize the NEMO mppio server @@ -46,8 +45,7 @@ SUBROUTINE nemogcmcoup_end_ioserver IMPLICIT NONE CALL mpp_stop() - END SUBROUTINE nemogcmcoup_end_ioserver -#endif +END SUBROUTINE nemogcmcoup_end_ioserver SUBROUTINE nemogcmcoup_init( mype, icomm, inidate, initime, itini, itend, zstp, & & lwaveonly, iatmunit, lwrite ) diff --git a/src/ifs_interface/mpp_io.F90 b/src/ifs_interface/mpp_io.F90 index eda8feae7..e285b111a 100644 --- a/src/ifs_interface/mpp_io.F90 +++ b/src/ifs_interface/mpp_io.F90 @@ -7,7 +7,8 @@ MODULE mpp_io #if defined(__MULTIO) - USE iom + USE iom, only : iom_initialize, iom_init_server, iom_finalize +#endif IMPLICIT NONE PRIVATE @@ -157,12 +158,14 @@ SUBROUTINE mpp_io_init_2( iicomm ) CALL mpi_comm_rank( iicommo, mppcomprank, ierr ) CALL mpi_comm_size( iicommo, mppcompsize, ierr ) +#if defined(__MULTIO) IF (.NOT.lioserver) THEN CALL iom_initialize( "for_xios_mpi_id", return_comm=iicommm, global_comm = pcommworldmultio ) ! nemo local communicator given by xios ELSE ! For io-server tasks start an run the right server CALL iom_init_server( server_comm = pcommworldmultio ) ENDIF +#endif ! Return to the model with iicomm being compute only tasks iicomm = iicommo @@ -172,11 +175,13 @@ END SUBROUTINE mpp_io_init_2 SUBROUTINE mpp_stop INTEGER :: ierr +#if defined(__MULTIO) IF (.NOT.lioserver) THEN call iom_finalize() ENDIF +#endif CALL mpi_finalize( ierr ) END SUBROUTINE mpp_stop -#endif + END MODULE mpp_io From c39f0d02262327d326779ce96357ed6371dae060 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 22 Aug 2023 16:35:37 +0200 Subject: [PATCH 31/38] Add option ENABLE_MULTIO (default ON, only when ENABLE_IFS_INTERFACE and multio_FOUND) --- src/CMakeLists.txt | 53 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e3208b4ec..6c9c8d50f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -77,26 +77,56 @@ if(DEFINED FESOM_ENABLE_IFS_INTERFACE) endif() message(STATUS "ENABLE_IFS_INTERFACE: ${ENABLE_IFS_INTERFACE}") +set(ENABLE_MULTIO_DEFAULT OFF) if(ENABLE_IFS_INTERFACE) if(USE_ICEPACK) message(FATAL_ERROR "Could not enable IFS interface as it is incompatible with USE_ICEPACK") endif() + if( NOT DEFINED ENABLE_MULTIO ) + find_package(multio QUIET) + if( multio_FOUND ) + set(ENABLE_MULTIO_DEFAULT ON) + endif() + endif() +endif() + +option(ENABLE_MULTIO "Enable MultIO" ${ENABLE_MULTIO_DEFAULT}) +if(DEFINED FESOM_ENABLE_MULTIO) + set(ENABLE_MULTIO ${FESOM_ENABLE_MULTIO}) # To distinguish option in a nested cmake project (bundle) endif() +message(STATUS "ENABLE_MULTIO: ${ENABLE_MULTIO}") +if(ENABLE_MULTIO) + find_package(multio) + if(multio_FOUND) + message(STATUS "Found multio: ${multio_DIR}") + else() + message(FATAL_ERROR "Could not enable multio as multio could not be found") + endif() + if(NOT TARGET multio-fapi) + message(FATAL_ERROR "Could not enable multio as multio was not compiled with 'multio-fapi' target") + endif() +endif() + set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/../.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") # get our source files set(src_home ${CMAKE_CURRENT_SOURCE_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt +file(GLOB sources_Fortran ${src_home}/*.F90) if(USE_ICEPACK) - file(GLOB sources_Fortran ${src_home}/*.F90 - ${src_home}/icepack_drivers/*.F90 - ${src_home}/icepack_drivers/Icepack/columnphysics/*.F90) -elseif(ENABLE_IFS_INTERFACE) - file(GLOB sources_Fortran ${src_home}/*.F90 - ${src_home}/ifs_interface/*.F90) # ICEPACK + LIBRARY NOT SUPPORTED (YET) -else() - file(GLOB sources_Fortran ${src_home}/*.F90) + file(GLOB sources_ice_pack ${src_home}/icepack_drivers/*.F90 + ${src_home}/icepack_drivers/Icepack/columnphysics/*.F90) + list(APPEND sources_Fortran ${sources_ice_pack}) +elseif(ENABLE_IFS_INTERFACE) # ICEPACK + IFS_INTERFACE NOT SUPPORTED (YET) + list(APPEND sources_Fortran ${src_home}/ifs_interface/ifs_interface.F90 + ${src_home}/ifs_interface/ifs_modules.F90 + ${src_home}/ifs_interface/ifs_notused.F90 + ${src_home}/ifs_interface/mpp_io.F90) +endif() +if(ENABLE_MULTIO) + list(APPEND sources_Fortran ${src_home}/ifs_interface/iom.F90) endif() + file(GLOB sources_C ${src_home}/*.c) # generate a custom file from fesom_version_info.F90 which includes the current git SHA @@ -139,7 +169,7 @@ target_link_libraries(${PROJECT_NAME}_C PRIVATE MPI::MPI_C) # fesom library add_library(${PROJECT_NAME} ${sources_Fortran}) add_dependencies(${PROJECT_NAME} fesom_version_info-generated.F90) -target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS -DMETIS_VERSION=5 -DPART_WEIGHTED -DMETISRANDOMSEED=35243 __MULTIO) +target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS -DMETIS_VERSION=5 -DPART_WEIGHTED -DMETISRANDOMSEED=35243) target_include_directories(${PROJECT_NAME} PUBLIC $) target_include_directories(${PROJECT_NAME} PUBLIC $) @@ -148,7 +178,6 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${MCT_Fortran_INCLUDE_DIRECTO target_include_directories(${PROJECT_NAME} PRIVATE ${SCRIP_Fortran_INCLUDE_DIRECTORIES}) target_link_libraries(${PROJECT_NAME} PRIVATE parms) #metis target_link_libraries(${PROJECT_NAME} PRIVATE async_threads) -target_link_libraries(${PROJECT_NAME} PRIVATE multio-fapi) target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_Fortran) target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) target_link_libraries(${PROJECT_NAME} PRIVATE ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) @@ -157,6 +186,10 @@ if(ENABLE_OPENMP AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) target_link_libraries(${PROJECT_NAME} PRIVATE OpenMP::OpenMP_Fortran) endif() +if(ENABLE_MULTIO) + target_compile_definitions(${PROJECT_NAME} PRIVATE __MULTIO) + target_link_libraries(${PROJECT_NAME} PRIVATE multio-fapi) +endif() if(DISABLE_MULTITHREADING) target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_MULTITHREADING) From 9ad264c379b30d9e7e2b6a74c8402ba7fa3d9642 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 30 Oct 2023 09:48:42 +0100 Subject: [PATCH 32/38] Discover LAPACK_LIBRARIES on ubuntu container --- configure.sh | 8 +++++--- env/ubuntu/shell | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/configure.sh b/configure.sh index 86d7eb02a..aeeb32994 100755 --- a/configure.sh +++ b/configure.sh @@ -5,7 +5,9 @@ set -e source env.sh # source this from your run script too mkdir build || true # make sure not to commit this to svn or git cd build -cmake .. $@ -DCMAKE_BUILD_TYPE=Debug # not required when re-compiling - # additional cmake arguments can be passed to configure.sh - # this also includes fesom specific options in CMakeLists, can be used as -DFESOM_COUPLED=ON +cmake .. -DCMAKE_BUILD_TYPE=Debug ${CMAKE_ARGS} $@ + # not required when re-compiling + # additional cmake arguments can be passed to configure.sh + # this also includes fesom specific options in CMakeLists, can be used as -DFESOM_COUPLED=ON make install -j`nproc --all` + diff --git a/env/ubuntu/shell b/env/ubuntu/shell index 29fc59dbc..c391d6e04 100644 --- a/env/ubuntu/shell +++ b/env/ubuntu/shell @@ -1,5 +1,6 @@ export FC=mpifort CC=mpicc CXX=mpicxx -export BLAS_LIBRARIES=/usr/lib/x86_64-linux-gnu/blas/ -export UBUNTU_BLAS_LIBRARY="libblas.a" - +if [[ -f "/usr/lib/x86_64-linux-gnu/liblapack.so.3" ]]; then + export LAPACK_LIBRARIES=/usr/lib/x86_64-linux-gnu/liblapack.so.3 + export CMAKE_ARGS=-DLAPACK_LIBRARIES=${LAPACK_LIBRARIES} +fi From 00bb8692a79f311b2fda365ea92d94d0607cb4b7 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 30 Oct 2023 14:14:24 +0100 Subject: [PATCH 33/38] Add env "local" --- env.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/env.sh b/env.sh index a073ad208..3e12d53f7 100755 --- a/env.sh +++ b/env.sh @@ -37,7 +37,11 @@ else fi -if [[ $LOGINHOST =~ ^m[A-Za-z0-9]+\.hpc\.dkrz\.de$ ]]; then +if [[ "$LOGINHOST" == "local" ]]; then + echo "Using local environment $BEING_EXECUTED" + [ $BEING_EXECUTED = true ] && exit + return 0 # if we are being sourced, return from this script here +elif [[ $LOGINHOST =~ ^m[A-Za-z0-9]+\.hpc\.dkrz\.de$ ]]; then STRATEGY="mistral.dkrz.de" elif [[ $LOGINHOST =~ ^levante ]] || [[ $LOGINHOST =~ ^l[:alnum:]+\.lvt\.dkrz\.de$ ]]; then STRATEGY="levante.dkrz.de" From 51bc1f58c33a5c8e342003ad4cb1587a5fef8f34 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 30 Oct 2023 10:05:13 +0100 Subject: [PATCH 34/38] Github Actions: remove sed and use CMake option --- .github/inactive_workflows/fesom2_icepack.yml | 3 +-- .github/workflows/fesom2_openmp.yml | 8 +------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/inactive_workflows/fesom2_icepack.yml b/.github/inactive_workflows/fesom2_icepack.yml index 9057057b1..9e295d101 100644 --- a/.github/inactive_workflows/fesom2_icepack.yml +++ b/.github/inactive_workflows/fesom2_icepack.yml @@ -21,14 +21,13 @@ jobs: - name: prepeare FESOM2_icepack run: | - sed -i 's/USE_ICEPACK OFF/USE_ICEPACK ON/g' CMakeLists.txt cd ./src/icepack_drivers/ ./download_icepack.sh cd ../../ - name: Compile model run: | - bash -l configure.sh ubuntu + bash -l configure.sh ubuntu -DUSE_ICEPACK=ON - name: Create global test run with ICEPACK run: | diff --git a/.github/workflows/fesom2_openmp.yml b/.github/workflows/fesom2_openmp.yml index fd1e8cafa..a30076757 100644 --- a/.github/workflows/fesom2_openmp.yml +++ b/.github/workflows/fesom2_openmp.yml @@ -19,15 +19,9 @@ jobs: # NK: this changes working directory to fesom2 - uses: actions/checkout@v2 - - name: switch OpenMP ON - run: | - cd ./src/ - sed -i 's/with OpenMP\" OFF/with OpenMP\" ON/g' CMakeLists.txt - cd ../ - - name: Compile model run: | - bash -l configure.sh ubuntu + bash -l configure.sh ubuntu -DENABLE_OPENMP=ON - name: Create global test run with 4 OpenMP threads run: | From 707e8d398ea684e20ae5d7319ae7803e7fb1fb4d Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 30 Oct 2023 11:43:02 +0100 Subject: [PATCH 35/38] Github Actions: git add safe directory --- .github/workflows/fesom2.1.yml | 6 +++++- .github/workflows/fesom2_openmp.yml | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fesom2.1.yml b/.github/workflows/fesom2.1.yml index 069a5a669..088d40d05 100644 --- a/.github/workflows/fesom2.1.yml +++ b/.github/workflows/fesom2.1.yml @@ -18,7 +18,11 @@ jobs: steps: # NK: this changes working directory to fesom2 - uses: actions/checkout@v2 - + + - name: Git safe directory + run: | + git config --global --add safe.directory /__w/fesom2/fesom2 + - name: Compile model (binary) run: | bash -l configure.sh ubuntu diff --git a/.github/workflows/fesom2_openmp.yml b/.github/workflows/fesom2_openmp.yml index a30076757..64c5a492b 100644 --- a/.github/workflows/fesom2_openmp.yml +++ b/.github/workflows/fesom2_openmp.yml @@ -19,6 +19,10 @@ jobs: # NK: this changes working directory to fesom2 - uses: actions/checkout@v2 + - name: Git safe directory + run: | + git config --global --add safe.directory /__w/fesom2/fesom2 + - name: Compile model run: | bash -l configure.sh ubuntu -DENABLE_OPENMP=ON From dc220f90810b59b8d6295a96e231f71a501c904b Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 30 Oct 2023 10:03:12 +0100 Subject: [PATCH 36/38] Github Actions: test compilation of IFS interface --- .github/workflows/fesom2.1.yml | 11 ++++------- configure.sh | 9 ++++++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/fesom2.1.yml b/.github/workflows/fesom2.1.yml index 088d40d05..d34d5a789 100644 --- a/.github/workflows/fesom2.1.yml +++ b/.github/workflows/fesom2.1.yml @@ -25,15 +25,12 @@ jobs: - name: Compile model (binary) run: | - bash -l configure.sh ubuntu - - - name: Compile model (library) - run: | - bash ./test/ifs_interface/configure_lib.sh -l + ./configure.sh ubuntu - - name: Library exists + - name: Compile model (ifs_interface) run: | - bash ./test/ifs_interface/check_exist.sh + export BUILD_DIR=$PWD/build.ifs_interface + ./configure.sh ubuntu -DENABLE_IFS_INTERFACE=ON -DCMAKE_INSTALL_PREFIX=$PWD/install.ifs_interface - name: Create global test run run: | diff --git a/configure.sh b/configure.sh index aeeb32994..78a064628 100755 --- a/configure.sh +++ b/configure.sh @@ -2,10 +2,13 @@ set -e +SOURCE_DIR="$( cd $( dirname "${BASH_SOURCE[0]}" ) && pwd -P )" +BUILD_DIR=${BUILD_DIR:-build} + source env.sh # source this from your run script too -mkdir build || true # make sure not to commit this to svn or git -cd build -cmake .. -DCMAKE_BUILD_TYPE=Debug ${CMAKE_ARGS} $@ +mkdir -p ${BUILD_DIR} # make sure not to commit this to svn or git +cd ${BUILD_DIR} +cmake ${SOURCE_DIR} -DCMAKE_BUILD_TYPE=Debug ${CMAKE_ARGS} $@ # not required when re-compiling # additional cmake arguments can be passed to configure.sh # this also includes fesom specific options in CMakeLists, can be used as -DFESOM_COUPLED=ON From 8f7d4edb2dcf87663ac59e78aebab2d4355a2e09 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Mon, 30 Oct 2023 14:19:39 +0100 Subject: [PATCH 37/38] Github Actions: Add test downstream --- .github/workflows/fesom2.1.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/fesom2.1.yml b/.github/workflows/fesom2.1.yml index d34d5a789..0b54ee9e8 100644 --- a/.github/workflows/fesom2.1.yml +++ b/.github/workflows/fesom2.1.yml @@ -32,6 +32,30 @@ jobs: export BUILD_DIR=$PWD/build.ifs_interface ./configure.sh ubuntu -DENABLE_IFS_INTERFACE=ON -DCMAKE_INSTALL_PREFIX=$PWD/install.ifs_interface + - name: Test downstream from /usr/local install-dir + run: | + rm -rf test_downstream/build + cmake -S test_downstream -B test_downstream/build -DASSERT_HAVE_IFS_INTERFACE=OFF -DASSERT_FESOM_DIR=/usr/local + cmake --build test_downstream/build + test_downstream/build/main + + - name: Test downstream from ifs_interface build-dir + run: | + cmake --version + export fesom_ROOT=$PWD/build.ifs_interface + rm -rf test_downstream/build + cmake -S test_downstream -B test_downstream/build -DASSERT_HAVE_IFS_INTERFACE=ON -DASSERT_FESOM_DIR=${fesom_ROOT} + cmake --build test_downstream/build + test_downstream/build/main + + - name: Test downstream from ifs_interface install-dir + run: | + export fesom_ROOT=$PWD/install.ifs_interface + rm -rf test_downstream/build + cmake -S test_downstream -B test_downstream/build -DASSERT_HAVE_IFS_INTERFACE=ON -DASSERT_FESOM_DIR=${fesom_ROOT} + cmake --build test_downstream/build + test_downstream/build/main + - name: Create global test run run: | mkrun pi test_pi -m docker From f0b842f0d283b88a596a2c137002b700bd432131 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Tue, 31 Oct 2023 13:47:10 +0100 Subject: [PATCH 38/38] Fix possible too long line length --- src/async_threads_cpp/async_threads_module.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/async_threads_cpp/async_threads_module.F90 b/src/async_threads_cpp/async_threads_module.F90 index 345f14eb5..198eb98dd 100644 --- a/src/async_threads_cpp/async_threads_module.F90 +++ b/src/async_threads_cpp/async_threads_module.F90 @@ -1,3 +1,4 @@ +#define __FILENAME__ "async_threads_module.F90" module async_threads_module implicit none public thread_type @@ -110,7 +111,7 @@ subroutine assert(val, line) integer, intent(in) :: line ! EO args if(.NOT. val) then - print *, "error in line ",line, __FILE__ + print *, "error in line ",line, __FILENAME__ stop 1 end if end subroutine