From 16412dca96d7213482819cde7083a1e748cc413d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 22 Dec 2023 11:24:31 -0700 Subject: [PATCH 01/14] Initial import of test_chunked_data (a recursive copy of test_blocked_data with the word 'blocked' replaced by 'chunked' everywhere, no other changes) --- .../test_chunked_data/CMakeLists.txt | 90 +++++++++++ test_prebuild/test_chunked_data/README.md | 13 ++ .../test_chunked_data/ccpp_prebuild_config.py | 81 ++++++++++ .../test_chunked_data/chunked_data_scheme.F90 | 118 ++++++++++++++ .../chunked_data_scheme.meta | 147 ++++++++++++++++++ test_prebuild/test_chunked_data/data.F90 | 41 +++++ test_prebuild/test_chunked_data/data.meta | 69 ++++++++ test_prebuild/test_chunked_data/main.F90 | 107 +++++++++++++ .../suite_chunked_data_suite.xml | 9 ++ 9 files changed, 675 insertions(+) create mode 100644 test_prebuild/test_chunked_data/CMakeLists.txt create mode 100644 test_prebuild/test_chunked_data/README.md create mode 100644 test_prebuild/test_chunked_data/ccpp_prebuild_config.py create mode 100644 test_prebuild/test_chunked_data/chunked_data_scheme.F90 create mode 100644 test_prebuild/test_chunked_data/chunked_data_scheme.meta create mode 100644 test_prebuild/test_chunked_data/data.F90 create mode 100644 test_prebuild/test_chunked_data/data.meta create mode 100644 test_prebuild/test_chunked_data/main.F90 create mode 100644 test_prebuild/test_chunked_data/suite_chunked_data_suite.xml diff --git a/test_prebuild/test_chunked_data/CMakeLists.txt b/test_prebuild/test_chunked_data/CMakeLists.txt new file mode 100644 index 00000000..0d055768 --- /dev/null +++ b/test_prebuild/test_chunked_data/CMakeLists.txt @@ -0,0 +1,90 @@ +#------------------------------------------------------------------------------ +cmake_minimum_required(VERSION 3.0) + +project(ccpp_chunked_data + VERSION 1.0.0 + LANGUAGES Fortran) + +#------------------------------------------------------------------------------ +# Request a static build +option(BUILD_SHARED_LIBS "Build a shared library" OFF) + +#------------------------------------------------------------------------------ +# Set the sources: physics type definitions +set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) +if(TYPEDEFS) + message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") +else(TYPEDEFS) + include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) + message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") +endif(TYPEDEFS) + +# Generate list of Fortran modules from the CCPP type +# definitions that need need to be installed +foreach(typedef_module ${TYPEDEFS}) + list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) +endforeach() + +#------------------------------------------------------------------------------ +# Set the sources: physics schemes +set(SCHEMES $ENV{CCPP_SCHEMES}) +if(SCHEMES) + message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") +else(SCHEMES) + include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) + message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") +endif(SCHEMES) + +# Set the sources: physics scheme caps +set(CAPS $ENV{CCPP_CAPS}) +if(CAPS) + message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") +else(CAPS) + include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) + message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") +endif(CAPS) + +# Set the sources: physics scheme caps +set(API $ENV{CCPP_API}) +if(API) + message(STATUS "Got CCPP API from environment variable: ${API}") +else(API) + include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) + message(STATUS "Got CCPP API from cmakefile include file: ${API}") +endif(API) + +set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") + +#------------------------------------------------------------------------------ +add_library(ccpp_chunked_data STATIC ${SCHEMES} ${CAPS} ${API}) +# Generate list of Fortran modules from defined sources +foreach(source_f90 ${CAPS} ${API}) + get_filename_component(tmp_source_f90 ${source_f90} NAME) + string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) + string(TOLOWER ${tmp_module_f90} module_f90) + list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) +endforeach() + +set_target_properties(ccpp_chunked_data PROPERTIES VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR}) + +add_executable(test_chunked_data.x main.F90) +add_dependencies(test_chunked_data.x ccpp_chunked_data) +target_link_libraries(test_chunked_data.x ccpp_chunked_data) +set_target_properties(test_chunked_data.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Define where to install the library +install(TARGETS ccpp_chunked_data + EXPORT ccpp_chunked_data-targets + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + RUNTIME DESTINATION lib +) +# Export our configuration +install(EXPORT ccpp_chunked_data-targets + FILE ccpp_chunked_data-config.cmake + DESTINATION lib/cmake +) +# Define where to install the C headers and Fortran modules +#install(FILES ${HEADERS_C} DESTINATION include) +install(FILES ${MODULES_F90} DESTINATION include) diff --git a/test_prebuild/test_chunked_data/README.md b/test_prebuild/test_chunked_data/README.md new file mode 100644 index 00000000..40812515 --- /dev/null +++ b/test_prebuild/test_chunked_data/README.md @@ -0,0 +1,13 @@ +# How to build the chunked data test + +1. Set compiler environment as appropriate for your system +2. Run the following commands: +``` +cd test_prebuild/test_chunked_data/ +rm -fr build +mkdir build +../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build +cd build +cmake .. 2>&1 | tee log.cmake +make 2>&1 | tee log.make +``` diff --git a/test_prebuild/test_chunked_data/ccpp_prebuild_config.py b/test_prebuild/test_chunked_data/ccpp_prebuild_config.py new file mode 100644 index 00000000..345a3cf8 --- /dev/null +++ b/test_prebuild/test_chunked_data/ccpp_prebuild_config.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) + +############################################################################### +# Definitions # +############################################################################### + +HOST_MODEL_IDENTIFIER = "FV3" + +# Add all files with metadata tables on the host model side and in CCPP, +# relative to basedir = top-level directory of host model. This includes +# kind and type definitions used in CCPP physics. Also add any internal +# dependencies of these files to the list. +VARIABLE_DEFINITION_FILES = [ + # actual variable definition files + '../../src/ccpp_types.F90', + 'data.F90', + ] + +TYPEDEFS_NEW_METADATA = { + 'ccpp_types' : { + 'ccpp_t' : 'cdata', + 'ccpp_types' : '', + }, + 'data' : { + 'chunked_data_type' : 'chunked_data_instance(cdata%blk_no)', + 'data' : '', + }, + } + +# Add all physics scheme files relative to basedir +SCHEME_FILES = [ + 'chunked_data_scheme.F90', + ] + +# Default build dir, relative to current working directory, +# if not specified as command-line argument +DEFAULT_BUILD_DIR = 'build' + +# Auto-generated makefile/cmakefile snippets that contain all type definitions +TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' +TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' +TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' + +# Auto-generated makefile/cmakefile snippets that contain all schemes +SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' +SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' +SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' + +# Auto-generated makefile/cmakefile snippets that contain all caps +CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' +CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' +CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' + +# Directory where to put all auto-generated physics caps +CAPS_DIR = '{build_dir}' + +# Directory where the suite definition files are stored +SUITES_DIR = '.' + +# Optional arguments - only required for schemes that use +# optional arguments. ccpp_prebuild.py will throw an exception +# if it encounters a scheme subroutine with optional arguments +# if no entry is made here. Possible values are: 'all', 'none', +# or a list of standard_names: [ 'var1', 'var3' ]. +OPTIONAL_ARGUMENTS = {} + +# Directory where to write static API to +STATIC_API_DIR = '{build_dir}' +STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' +STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' + +# Directory for writing HTML pages generated from metadata files +METADATA_HTML_OUTPUT_DIR = '{build_dir}' + +# HTML document containing the model-defined CCPP variables +HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_CHUNKED_DATA.html' + +# LaTeX document containing the provided vs requested CCPP variables +LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_CHUNKED_DATA.tex' diff --git a/test_prebuild/test_chunked_data/chunked_data_scheme.F90 b/test_prebuild/test_chunked_data/chunked_data_scheme.F90 new file mode 100644 index 00000000..69c77edd --- /dev/null +++ b/test_prebuild/test_chunked_data/chunked_data_scheme.F90 @@ -0,0 +1,118 @@ +!>\file chunked_data_scheme.F90 +!! This file contains a chunked_data_scheme CCPP scheme that does nothing +!! except requesting the minimum, mandatory variables. + +module chunked_data_scheme + + use, intrinsic :: iso_fortran_env, only: error_unit + implicit none + + private + public :: chunked_data_scheme_init, & + chunked_data_scheme_timestep_init, & + chunked_data_scheme_run, & + chunked_data_scheme_timestep_finalize, & + chunked_data_scheme_finalize + + ! This is for unit testing only + integer, parameter, dimension(4) :: data_array_sizes = (/6,6,6,3/) + + contains + +!! \section arg_table_chunked_data_scheme_init Argument Table +!! \htmlinclude chunked_data_scheme_init.html +!! + subroutine chunked_data_scheme_init(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit,'(a,i3)') 'In chunked_data_scheme_init: checking size of data array to be', sum(data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_init + +!! \section arg_table_chunked_data_scheme_timestep_init Argument Table +!! \htmlinclude chunked_data_scheme_timestep_init.html +!! + subroutine chunked_data_scheme_timestep_init(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit,'(a,i3)') 'In chunked_data_scheme_timestep_init: checking size of data array to be', sum(data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), " but got ", size(data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_timestep_init + +!! \section arg_table_chunked_data_scheme_run Argument Table +!! \htmlinclude chunked_data_scheme_run.html +!! + subroutine chunked_data_scheme_run(nb, data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: nb + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit,'(2(a,i3))') 'In chunked_data_scheme_run: checking size of data array for block', nb, ' to be', data_array_sizes(nb) + if (size(data_array)/=data_array_sizes(nb)) then + write(errmsg,'(a,i4)') "Error in chunked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_run + + !! \section arg_table_chunked_data_scheme_timestep_finalize Argument Table + !! \htmlinclude chunked_data_scheme_timestep_finalize.html + !! + subroutine chunked_data_scheme_timestep_finalize(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit,'(a,i3)') 'In chunked_data_scheme_timestep_finalize: checking size of data array to be', sum(data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_timestep_finalize + +!! \section arg_table_chunked_data_scheme_finalize Argument Table +!! \htmlinclude chunked_data_scheme_finalize.html +!! + subroutine chunked_data_scheme_finalize(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit,'(a,i3)') 'In chunked_data_scheme_finalize: checking size of data array to be', sum(data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_finalize + +end module chunked_data_scheme diff --git a/test_prebuild/test_chunked_data/chunked_data_scheme.meta b/test_prebuild/test_chunked_data/chunked_data_scheme.meta new file mode 100644 index 00000000..8e8806fc --- /dev/null +++ b/test_prebuild/test_chunked_data/chunked_data_scheme.meta @@ -0,0 +1,147 @@ +[ccpp-table-properties] + name = chunked_data_scheme + type = scheme + dependencies = + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_init + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_timestep_init + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_run + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[nb] + standard_name = ccpp_block_number + long_name = number of block for explicit data blocking in CCPP + units = index + dimensions = () + type = integer + intent = in +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_loop_extent) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_timestep_finalize + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_finalize + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + diff --git a/test_prebuild/test_chunked_data/data.F90 b/test_prebuild/test_chunked_data/data.F90 new file mode 100644 index 00000000..f7cf1108 --- /dev/null +++ b/test_prebuild/test_chunked_data/data.F90 @@ -0,0 +1,41 @@ +module data + +!! \section arg_table_data Argument Table +!! \htmlinclude data.html +!! + use ccpp_types, only: ccpp_t + + implicit none + + private + + public nblks, blksz, ncols + public ccpp_data_domain, ccpp_data_blocks, chunked_data_type, chunked_data_instance + + integer, parameter :: nblks = 4 + type(ccpp_t), target :: ccpp_data_domain + type(ccpp_t), dimension(nblks), target :: ccpp_data_blocks + + integer, parameter, dimension(nblks) :: blksz = (/6,6,6,3/) + integer, parameter :: ncols = sum(blksz) + +!! \section arg_table_chunked_data_type +!! \htmlinclude chunked_data_type.html +!! + type chunked_data_type + integer, dimension(:), allocatable :: array_data + contains + procedure :: create => chunked_data_create + end type chunked_data_type + + type(chunked_data_type), dimension(nblks) :: chunked_data_instance + +contains + + subroutine chunked_data_create(chunked_data_instance, ncol) + class(chunked_data_type), intent(inout) :: chunked_data_instance + integer, intent(in) :: ncol + allocate(chunked_data_instance%array_data(ncol)) + end subroutine chunked_data_create + +end module data diff --git a/test_prebuild/test_chunked_data/data.meta b/test_prebuild/test_chunked_data/data.meta new file mode 100644 index 00000000..127261cd --- /dev/null +++ b/test_prebuild/test_chunked_data/data.meta @@ -0,0 +1,69 @@ +[ccpp-table-properties] + name = chunked_data_type + type = ddt + dependencies = +[ccpp-arg-table] + name = chunked_data_type + type = ddt +[array_data] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_loop_extent) + type = integer + +[ccpp-table-properties] + name = data + type = module + dependencies = +[ccpp-arg-table] + name = data + type = module +[cdata] + standard_name = ccpp_t_instance + long_name = instance of derived data type ccpp_t + units = DDT + dimensions = () + type = ccpp_t +[nblks] + standard_name = ccpp_block_count + long_name = for explicit data blocking: number of blocks + units = count + dimensions = () + type = integer +[blksz] + standard_name = ccpp_block_sizes + long_name = for explicit data blocking: block sizes of all blocks + units = count + dimensions = (ccpp_block_count) + type = integer +[blksz(ccpp_block_number)] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent + units = count + dimensions = () + type = integer +[ncols] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[chunked_data_type] + standard_name = chunked_data_type + long_name = definition of type chunked_data_type + units = DDT + dimensions = () + type = chunked_data_type +[chunked_data_instance(ccpp_block_number)] + standard_name = chunked_data_type_instance + long_name = instance of derived data type chunked_data_type + units = DDT + dimensions = () + type = chunked_data_type +[chunked_data_instance] + standard_name = chunked_data_type_instance_all_blocks + long_name = instance of derived data type chunked_data_type + units = DDT + dimensions = (ccpp_block_count) + type = chunked_data_type diff --git a/test_prebuild/test_chunked_data/main.F90 b/test_prebuild/test_chunked_data/main.F90 new file mode 100644 index 00000000..b9158e46 --- /dev/null +++ b/test_prebuild/test_chunked_data/main.F90 @@ -0,0 +1,107 @@ +program test_chunked_data + + use, intrinsic :: iso_fortran_env, only: error_unit + + use ccpp_types, only: ccpp_t + use data, only: nblks, blksz, ncols + use data, only: ccpp_data_domain, ccpp_data_blocks, & + chunked_data_type, chunked_data_instance + + use ccpp_static_api, only: ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_finalize, & + ccpp_physics_finalize + + implicit none + + character(len=*), parameter :: ccpp_suite = 'chunked_data_suite' + integer :: ib, ierr + type(ccpp_t), pointer :: cdata => null() + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + ! For physics running over the entire domain, block and thread + ! number are not used; set to safe values + ccpp_data_domain%blk_no = 1 + ccpp_data_domain%thrd_no = 1 + + ! Loop over all blocks and threads for ccpp_data_blocks + do ib=1,nblks + ! Assign the correct block numbers, only one thread + ccpp_data_blocks(ib)%blk_no = ib + ccpp_data_blocks(ib)%thrd_no = 1 + end do + + do ib=1,size(chunked_data_instance) + allocate(chunked_data_instance(ib)%array_data(blksz(ib))) + write(error_unit,'(2(a,i3))') "Allocated array_data for block", ib, " to size", size(chunked_data_instance(ib)%array_data) + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + cdata => ccpp_data_domain + call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(error_unit,'(a)') "An error occurred in ccpp_physics_init:" + write(error_unit,'(a)') trim(cdata%errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + cdata => ccpp_data_domain + call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(error_unit,'(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit,'(a)') trim(cdata%errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ib=1,nblks + cdata => ccpp_data_blocks(ib) + call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(error_unit,'(a,i3,a)') "An error occurred in ccpp_physics_run for block", ib, ":" + write(error_unit,'(a)') trim(cdata%errmsg) + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + cdata => ccpp_data_domain + call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(error_unit,'(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit,'(a)') trim(cdata%errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + cdata => ccpp_data_domain + call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(error_unit,'(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit,'(a)') trim(cdata%errmsg) + stop 1 + end if + +contains + +end program test_chunked_data \ No newline at end of file diff --git a/test_prebuild/test_chunked_data/suite_chunked_data_suite.xml b/test_prebuild/test_chunked_data/suite_chunked_data_suite.xml new file mode 100644 index 00000000..923e5fb7 --- /dev/null +++ b/test_prebuild/test_chunked_data/suite_chunked_data_suite.xml @@ -0,0 +1,9 @@ + + + + + + chunked_data_scheme + + + From 15fc38a3d7da0fad088a3617dc2618f86b86943a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 22 Dec 2023 12:03:41 -0700 Subject: [PATCH 02/14] Add member chunk_no to ccpp_t with standard name ccpp_chunk_number --- src/ccpp_types.F90 | 16 ++++++++++------ src/ccpp_types.meta | 6 ++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/ccpp_types.F90 b/src/ccpp_types.F90 index bebb53cb..ef08b6f2 100644 --- a/src/ccpp_types.F90 +++ b/src/ccpp_types.F90 @@ -35,8 +35,10 @@ module ccpp_types integer, parameter :: CCPP_DEFAULT_LOOP_CNT = -999 integer, parameter :: CCPP_DEFAULT_LOOP_MAX = -999 - !> @var The default values for block and thread numbers indicating invalid data - integer, parameter :: CCPP_DEFAULT_BLOCK_AND_THREAD_NUMBER = -999 + !> @var The default values for block, chunk and thread numbers indicating invalid data + integer, parameter :: CCPP_DEFAULT_BLOCK_NUMBER = -999 + integer, parameter :: CCPP_DEFAULT_CHUNK_NUMBER = -999 + integer, parameter :: CCPP_DEFAULT_THREAD_NUMBER = -999 !! \section arg_table_ccpp_t !! \htmlinclude ccpp_t.html @@ -56,8 +58,9 @@ module ccpp_types character(len=512) :: errmsg = '' integer :: loop_cnt = CCPP_DEFAULT_LOOP_CNT integer :: loop_max = CCPP_DEFAULT_LOOP_MAX - integer :: blk_no = CCPP_DEFAULT_BLOCK_AND_THREAD_NUMBER - integer :: thrd_no = CCPP_DEFAULT_BLOCK_AND_THREAD_NUMBER + integer :: blk_no = CCPP_DEFAULT_BLOCK_NUMBER + integer :: chunk_no = CCPP_DEFAULT_CHUNK_NUMBER + integer :: thrd_no = CCPP_DEFAULT_THREAD_NUMBER integer :: ccpp_instance = 1 contains @@ -74,8 +77,9 @@ function ccpp_t_initialized(ccpp_d) result(initialized) class(ccpp_t) :: ccpp_d logical :: initialized ! - initialized = (ccpp_d%blk_no /= CCPP_DEFAULT_BLOCK_AND_THREAD_NUMBER .and. & - ccpp_d%thrd_no /= CCPP_DEFAULT_BLOCK_AND_THREAD_NUMBER) + initialized = ccpp_d%thrd_no /= CCPP_DEFAULT_THREAD_NUMBER .and. & + (ccpp_d%blk_no /= CCPP_DEFAULT_BLOCK_NUMBER .and. & + ccpp_d%chunk_no /= CCPP_DEFAULT_CHUNK_NUMBER) end function ccpp_t_initialized end module ccpp_types diff --git a/src/ccpp_types.meta b/src/ccpp_types.meta index 6ad84522..41a01f52 100644 --- a/src/ccpp_types.meta +++ b/src/ccpp_types.meta @@ -38,6 +38,12 @@ units = index dimensions = () type = integer +[chunk_no] + standard_name = ccpp_chunk_number + long_name = number of chunk for using array chunks in CCPP + units = index + dimensions = () + type = integer [thrd_no] standard_name = ccpp_thread_number long_name = number of thread for threading in CCPP From 6f15dc11d2425bd50f2f32ba821f7d67861bf8d2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 22 Dec 2023 12:05:47 -0700 Subject: [PATCH 03/14] Update test_chunked_data. Now passing init, physics_init, timestep_init, and expected failing run --- .../test_chunked_data/ccpp_prebuild_config.py | 2 +- test_prebuild/test_chunked_data/data.F90 | 14 ++++---- test_prebuild/test_chunked_data/data.meta | 29 ++--------------- test_prebuild/test_chunked_data/main.F90 | 32 +++++++++---------- 4 files changed, 26 insertions(+), 51 deletions(-) mode change 100644 => 100755 test_prebuild/test_chunked_data/ccpp_prebuild_config.py diff --git a/test_prebuild/test_chunked_data/ccpp_prebuild_config.py b/test_prebuild/test_chunked_data/ccpp_prebuild_config.py old mode 100644 new mode 100755 index 345a3cf8..4e32d37d --- a/test_prebuild/test_chunked_data/ccpp_prebuild_config.py +++ b/test_prebuild/test_chunked_data/ccpp_prebuild_config.py @@ -24,7 +24,7 @@ 'ccpp_types' : '', }, 'data' : { - 'chunked_data_type' : 'chunked_data_instance(cdata%blk_no)', + 'chunked_data_type' : 'chunked_data_instance', 'data' : '', }, } diff --git a/test_prebuild/test_chunked_data/data.F90 b/test_prebuild/test_chunked_data/data.F90 index f7cf1108..0aa5bfb0 100644 --- a/test_prebuild/test_chunked_data/data.F90 +++ b/test_prebuild/test_chunked_data/data.F90 @@ -9,15 +9,15 @@ module data private - public nblks, blksz, ncols - public ccpp_data_domain, ccpp_data_blocks, chunked_data_type, chunked_data_instance + public nchunks, chunksize, ncols + public ccpp_data_domain, ccpp_data_chunks, chunked_data_type, chunked_data_instance - integer, parameter :: nblks = 4 + integer, parameter :: nchunks = 4 type(ccpp_t), target :: ccpp_data_domain - type(ccpp_t), dimension(nblks), target :: ccpp_data_blocks + type(ccpp_t), dimension(nchunks), target :: ccpp_data_chunks - integer, parameter, dimension(nblks) :: blksz = (/6,6,6,3/) - integer, parameter :: ncols = sum(blksz) + integer, parameter, dimension(nchunks) :: chunksize = (/6,6,6,3/) + integer, parameter :: ncols = sum(chunksize) !! \section arg_table_chunked_data_type !! \htmlinclude chunked_data_type.html @@ -28,7 +28,7 @@ module data procedure :: create => chunked_data_create end type chunked_data_type - type(chunked_data_type), dimension(nblks) :: chunked_data_instance + type(chunked_data_type) :: chunked_data_instance contains diff --git a/test_prebuild/test_chunked_data/data.meta b/test_prebuild/test_chunked_data/data.meta index 127261cd..ad3db025 100644 --- a/test_prebuild/test_chunked_data/data.meta +++ b/test_prebuild/test_chunked_data/data.meta @@ -9,8 +9,9 @@ standard_name = chunked_data_array long_name = chunked data array units = 1 - dimensions = (horizontal_loop_extent) + dimensions = (ccpp_constant_one:horizontal_dimension) type = integer +# Todo: define an additional array running from -1 to ncols-2 [ccpp-table-properties] name = data @@ -25,24 +26,6 @@ units = DDT dimensions = () type = ccpp_t -[nblks] - standard_name = ccpp_block_count - long_name = for explicit data blocking: number of blocks - units = count - dimensions = () - type = integer -[blksz] - standard_name = ccpp_block_sizes - long_name = for explicit data blocking: block sizes of all blocks - units = count - dimensions = (ccpp_block_count) - type = integer -[blksz(ccpp_block_number)] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer [ncols] standard_name = horizontal_dimension long_name = horizontal dimension @@ -55,15 +38,9 @@ units = DDT dimensions = () type = chunked_data_type -[chunked_data_instance(ccpp_block_number)] +[chunked_data_instance] standard_name = chunked_data_type_instance long_name = instance of derived data type chunked_data_type units = DDT dimensions = () type = chunked_data_type -[chunked_data_instance] - standard_name = chunked_data_type_instance_all_blocks - long_name = instance of derived data type chunked_data_type - units = DDT - dimensions = (ccpp_block_count) - type = chunked_data_type diff --git a/test_prebuild/test_chunked_data/main.F90 b/test_prebuild/test_chunked_data/main.F90 index b9158e46..0fb0c0fd 100644 --- a/test_prebuild/test_chunked_data/main.F90 +++ b/test_prebuild/test_chunked_data/main.F90 @@ -3,8 +3,8 @@ program test_chunked_data use, intrinsic :: iso_fortran_env, only: error_unit use ccpp_types, only: ccpp_t - use data, only: nblks, blksz, ncols - use data, only: ccpp_data_domain, ccpp_data_blocks, & + use data, only: nchunks, chunksize, ncols + use data, only: ccpp_data_domain, ccpp_data_chunks, & chunked_data_type, chunked_data_instance use ccpp_static_api, only: ccpp_physics_init, & @@ -16,29 +16,27 @@ program test_chunked_data implicit none character(len=*), parameter :: ccpp_suite = 'chunked_data_suite' - integer :: ib, ierr + integer :: ic, ierr type(ccpp_t), pointer :: cdata => null() !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! CCPP init step ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! For physics running over the entire domain, block and thread - ! number are not used; set to safe values - ccpp_data_domain%blk_no = 1 + ! For physics running over the entire domain, + ! ccpp_thread_number is not used; set to safe value ccpp_data_domain%thrd_no = 1 + ccpp_data_domain%chunk_no = 1 - ! Loop over all blocks and threads for ccpp_data_blocks - do ib=1,nblks + ! Loop over all blocks and threads for ccpp_data_chunks + do ic=1,nchunks ! Assign the correct block numbers, only one thread - ccpp_data_blocks(ib)%blk_no = ib - ccpp_data_blocks(ib)%thrd_no = 1 + ccpp_data_chunks(ic)%blk_no = ic + ccpp_data_chunks(ic)%thrd_no = 1 end do - do ib=1,size(chunked_data_instance) - allocate(chunked_data_instance(ib)%array_data(blksz(ib))) - write(error_unit,'(2(a,i3))') "Allocated array_data for block", ib, " to size", size(chunked_data_instance(ib)%array_data) - end do + call chunked_data_instance%create(ncols) + write(error_unit,'(2(a,i3))') "Chunked_data_instance%array_data to size", size(chunked_data_instance%array_data) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! CCPP physics init step ! @@ -68,11 +66,11 @@ program test_chunked_data ! CCPP physics run step ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - do ib=1,nblks - cdata => ccpp_data_blocks(ib) + do ic=1,nchunks + cdata => ccpp_data_chunks(ic) call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) if (ierr/=0) then - write(error_unit,'(a,i3,a)') "An error occurred in ccpp_physics_run for block", ib, ":" + write(error_unit,'(a,i3,a)') "An error occurred in ccpp_physics_run for block", ic, ":" write(error_unit,'(a)') trim(cdata%errmsg) stop 1 end if From 2d84692073678d731d7ea5d364a8e63d9bc1308b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Dec 2023 14:00:56 -0700 Subject: [PATCH 04/14] Update of test_chunked_data to make tests pass with next commit (framework updates) --- .../test_chunked_data/chunked_data_scheme.F90 | 8 ++--- .../chunked_data_scheme.meta | 6 ++-- test_prebuild/test_chunked_data/data.F90 | 4 ++- test_prebuild/test_chunked_data/data.meta | 30 +++++++++++++++++++ test_prebuild/test_chunked_data/main.F90 | 11 ++++--- 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/test_prebuild/test_chunked_data/chunked_data_scheme.F90 b/test_prebuild/test_chunked_data/chunked_data_scheme.F90 index 69c77edd..1bb2a266 100644 --- a/test_prebuild/test_chunked_data/chunked_data_scheme.F90 +++ b/test_prebuild/test_chunked_data/chunked_data_scheme.F90 @@ -60,17 +60,17 @@ end subroutine chunked_data_scheme_timestep_init !! \section arg_table_chunked_data_scheme_run Argument Table !! \htmlinclude chunked_data_scheme_run.html !! - subroutine chunked_data_scheme_run(nb, data_array, errmsg, errflg) + subroutine chunked_data_scheme_run(nchunk, data_array, errmsg, errflg) character(len=*), intent(out) :: errmsg integer, intent(out) :: errflg - integer, intent(in) :: nb + integer, intent(in) :: nchunk integer, intent(in) :: data_array(:) ! Initialize CCPP error handling variables errmsg = '' errflg = 0 ! Check size of data array - write(error_unit,'(2(a,i3))') 'In chunked_data_scheme_run: checking size of data array for block', nb, ' to be', data_array_sizes(nb) - if (size(data_array)/=data_array_sizes(nb)) then + write(error_unit,'(2(a,i3))') 'In chunked_data_scheme_run: checking size of data array for chunk', nchunk, ' to be', data_array_sizes(nchunk) + if (size(data_array)/=data_array_sizes(nchunk)) then write(errmsg,'(a,i4)') "Error in chunked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) errflg = 1 return diff --git a/test_prebuild/test_chunked_data/chunked_data_scheme.meta b/test_prebuild/test_chunked_data/chunked_data_scheme.meta index 8e8806fc..13830dbf 100644 --- a/test_prebuild/test_chunked_data/chunked_data_scheme.meta +++ b/test_prebuild/test_chunked_data/chunked_data_scheme.meta @@ -76,9 +76,9 @@ dimensions = () type = integer intent = out -[nb] - standard_name = ccpp_block_number - long_name = number of block for explicit data blocking in CCPP +[nchunk] + standard_name = ccpp_chunk_number + long_name = number of chunk for chunked arrays in CCPP units = index dimensions = () type = integer diff --git a/test_prebuild/test_chunked_data/data.F90 b/test_prebuild/test_chunked_data/data.F90 index 0aa5bfb0..25518bef 100644 --- a/test_prebuild/test_chunked_data/data.F90 +++ b/test_prebuild/test_chunked_data/data.F90 @@ -9,7 +9,7 @@ module data private - public nchunks, chunksize, ncols + public nchunks, chunksize, chunk_begin, chunk_end, ncols public ccpp_data_domain, ccpp_data_chunks, chunked_data_type, chunked_data_instance integer, parameter :: nchunks = 4 @@ -17,6 +17,8 @@ module data type(ccpp_t), dimension(nchunks), target :: ccpp_data_chunks integer, parameter, dimension(nchunks) :: chunksize = (/6,6,6,3/) + integer, parameter, dimension(nchunks) :: chunk_begin = (/1,7,13,19/) + integer, parameter, dimension(nchunks) :: chunk_end = (/6,12,18,21/) integer, parameter :: ncols = sum(chunksize) !! \section arg_table_chunked_data_type diff --git a/test_prebuild/test_chunked_data/data.meta b/test_prebuild/test_chunked_data/data.meta index ad3db025..c14217df 100644 --- a/test_prebuild/test_chunked_data/data.meta +++ b/test_prebuild/test_chunked_data/data.meta @@ -32,6 +32,36 @@ units = count dimensions = () type = integer +[nchunks] + standard_name = ccpp_chunk_extent + long_name = number of chunks of array data used in run phase + units = count + dimensions = () + type = integer +[chunk_begin] + standard_name = horizontal_loop_begin_all_chunks + long_name = first index for horizontal loop extent in run phase + units = index + dimensions = (ccpp_chunk_extent) + type = integer +[chunk_begin(ccpp_chunk_number)] + standard_name = horizontal_loop_begin + long_name = first index for horizontal loop extent in run phase + units = index + dimensions = () + type = integer +[chunk_end] + standard_name = horizontal_loop_end_all_chunks + long_name = last index for horizontal loop extent in run phase + units = index + dimensions = (ccpp_chunk_extent) + type = integer +[chunk_end(ccpp_chunk_number)] + standard_name = horizontal_loop_end + long_name = last index for horizontal loop extent in run phase + units = index + dimensions = () + type = integer [chunked_data_type] standard_name = chunked_data_type long_name = definition of type chunked_data_type diff --git a/test_prebuild/test_chunked_data/main.F90 b/test_prebuild/test_chunked_data/main.F90 index 0fb0c0fd..9822b27d 100644 --- a/test_prebuild/test_chunked_data/main.F90 +++ b/test_prebuild/test_chunked_data/main.F90 @@ -24,14 +24,17 @@ program test_chunked_data !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! For physics running over the entire domain, - ! ccpp_thread_number is not used; set to safe value - ccpp_data_domain%thrd_no = 1 - ccpp_data_domain%chunk_no = 1 + ! ccpp_thread_number and ccpp_chunk_number are + ! set to 0, indicating that arrays are to be sent + ! following their dimension specification in the + ! metadata (must match horizontal_dimension). + ccpp_data_domain%thrd_no = 0 + ccpp_data_domain%chunk_no = 0 ! Loop over all blocks and threads for ccpp_data_chunks do ic=1,nchunks ! Assign the correct block numbers, only one thread - ccpp_data_chunks(ic)%blk_no = ic + ccpp_data_chunks(ic)%chunk_no = ic ccpp_data_chunks(ic)%thrd_no = 1 end do From 75ea5d578b5966ca4da00a9c1f8b629292bbf137 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Dec 2023 14:03:58 -0700 Subject: [PATCH 05/14] Add capability to handle chunked arrays while maintaining backward compatibility with blocked data structures --- scripts/common.py | 4 + scripts/mkstatic.py | 202 ++++++++++++++++++++++++++++---------------- 2 files changed, 134 insertions(+), 72 deletions(-) diff --git a/scripts/common.py b/scripts/common.py index a4b43eb7..f115aa80 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -30,6 +30,10 @@ CCPP_BLOCK_SIZES = 'ccpp_block_sizes' CCPP_THREAD_NUMBER = 'ccpp_thread_number' +CCPP_CHUNK_EXTENT = 'ccpp_chunk_extent' +CCPP_HORIZONTAL_LOOP_BEGIN = 'horizontal_loop_begin' +CCPP_HORIZONTAL_LOOP_END = 'horizontal_loop_end' + CCPP_HORIZONTAL_LOOP_EXTENT = 'horizontal_loop_extent' CCPP_HORIZONTAL_DIMENSION = 'horizontal_dimension' diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index f86f738f..48cfa765 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -16,6 +16,7 @@ from common import CCPP_STAGES from common import CCPP_T_INSTANCE_VARIABLE, CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE, CCPP_LOOP_COUNTER, CCPP_LOOP_EXTENT from common import CCPP_BLOCK_NUMBER, CCPP_BLOCK_COUNT, CCPP_BLOCK_SIZES, CCPP_THREAD_NUMBER, CCPP_INTERNAL_VARIABLES +from common import CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, CCPP_CHUNK_EXTENT from common import CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_DIMENSION, CCPP_HORIZONTAL_LOOP_EXTENT, CCPP_NUM_INSTANCES from common import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX from common import CCPP_TYPE, STANDARD_VARIABLE_TYPES, STANDARD_CHARACTER_TYPE @@ -1053,6 +1054,19 @@ def write(self, metadata_request, metadata_define, arguments, debug): # and add them to the list of required variables for the cap additional_variables_required = [] # + if CCPP_HORIZONTAL_LOOP_EXTENT in metadata_define.keys(): + for add_var in [ CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_LOOP_EXTENT]: + if not add_var in local_vars.keys() \ + and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: + logging.debug("Adding variable {} for handling blocked data structures".format(add_var)) + additional_variables_required.append(add_var) + elif ccpp_stage == 'run': + for add_var in [ CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, CCPP_CHUNK_EXTENT]: + if not add_var in local_vars.keys() \ + and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: + logging.debug("Adding variable {} for handling chunked data arrays".format(add_var)) + additional_variables_required.append(add_var) + # for var_standard_name in arguments[scheme_name][subroutine_name]: if not var_standard_name in metadata_define.keys(): raise Exception('Variable {standard_name} not defined in host model metadata'.format( @@ -1076,18 +1090,11 @@ def write(self, metadata_request, metadata_define, arguments, debug): # If blocked data structures need to be converted, add necessary variables if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in var.local_name: - if not CCPP_BLOCK_COUNT in local_vars.keys() \ - and not CCPP_BLOCK_COUNT in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling blocked data structures".format(CCPP_BLOCK_COUNT)) - additional_variables_required.append(CCPP_BLOCK_COUNT) - if not CCPP_HORIZONTAL_LOOP_EXTENT in local_vars.keys() \ - and not CCPP_HORIZONTAL_LOOP_EXTENT in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling blocked data structures".format(CCPP_HORIZONTAL_LOOP_EXTENT)) - additional_variables_required.append(CCPP_HORIZONTAL_LOOP_EXTENT) - if not CCPP_HORIZONTAL_DIMENSION in local_vars.keys() \ - and not CCPP_HORIZONTAL_DIMENSION in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling blocked data structures".format(CCPP_HORIZONTAL_DIMENSION)) - additional_variables_required.append(CCPP_HORIZONTAL_DIMENSION) + for add_var in [ CCPP_BLOCK_COUNT, CCPP_HORIZONTAL_DIMENSION]: + if not add_var in local_vars.keys() \ + and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: + logging.debug("Adding variable {} for handling blocked data structures".format(add_var)) + additional_variables_required.append(add_var) # If the variable is only active/used under certain conditions, add necessary variables # also record the conditional for later use in unit conversions / blocked data conversions. @@ -1263,7 +1270,100 @@ def write(self, metadata_request, metadata_define, arguments, debug): if container == var.container: break + # Derive correct horizontal loop extent for this variable for the rest of this function + if var.rank: + array_size = [] + dim_substrings = [] + for dim in var.dimensions: + # This is not supported/implemented: tmpvar would have one dimension less + # than the original array, and the metadata requesting the variable would + # not pass the initial test that host model variables and scheme variables + # have the same rank. + if dim == CCPP_BLOCK_NUMBER: + raise Exception("{} cannot be part of the dimensions of variable {}".format( + CCPP_BLOCK_NUMBER, var_standard_name)) + else: + # Handle dimensions like "A:B", "A:3", "-1:Z" + if ':' in dim: + dims = [ x.lower() for x in dim.split(':')] + try: + dim0 = int(dims[0]) + dim0 = dims[0] + except ValueError: + if not dims[0].lower() in metadata_define.keys(): + raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( + dims[0].lower(), var_standard_name)) + dim0 = metadata_define[dims[0].lower()][0].local_name + try: + dim1 = int(dims[1]) + dim1 = dims[1] + except ValueError: + # Use correct horizontal variables in run phase + if ccpp_stage == 'run' and dims[1].lower() == CCPP_HORIZONTAL_LOOP_EXTENT: + # Provide backward compatibility with blocked data structures + if CCPP_BLOCK_NUMBER in additional_variables_required + arguments[scheme_name][subroutine_name]: + dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name + dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name + else: + dim0 = metadata_define[CCPP_HORIZONTAL_LOOP_BEGIN][0].local_name + dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_END][0].local_name + else: + if not dims[1].lower() in metadata_define.keys(): + raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( + dims[1].lower(), var_standard_name)) + dim1 = metadata_define[dims[1].lower()][0].local_name + # Single dimensions are indices + else: + try: + dim1 = int(dim) + dim1 = dim + except ValueError: + # This should not happen with capgen, because dimensions always have a lower and upper bound (n:m)? + if dim.lower() in [CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, \ + CCPP_HORIZONTAL_LOOP_EXTENT, CCPP_HORIZONTAL_DIMENSION]: + raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ + f"horizontal dimension for {var_standard_name} is {var.dimensions}") + if not dim.lower() in metadata_define.keys(): + raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( + dim.lower(), var_standard_name)) + dim1 = metadata_define[dim.lower()][0].local_name + dim0 = dim1 + + # Handle horizontal dimensions correctly + if ccpp_stage == 'run': + # This should not happen when parsing metadata with capgen's metadata parser, remove? + if dims[1] == CCPP_HORIZONTAL_LOOP_EXTENT and not dim0: + raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ + f"horizontal dimension for {var_standard_name} is {var.dimensions}") + # This should not happen when parsing metadata with capgen's metadata parser, remove? + elif CCPP_HORIZONTAL_LOOP_BEGIN in dims or CCPP_HORIZONTAL_LOOP_END in dims or \ + CCPP_HORIZONTAL_DIMENSION in dims: + raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ + f"horizontal dimension for {var_standard_name} is {var.dimensions}") + else: + # This should not happen when parsing metadata with capgen's metadata parser, remove? + if dims[1] == CCPP_HORIZONTAL_DIMENSION and not dim0: + raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ + f"horizontal dimension for {var_standard_name} is {var.dimensions}") + # This should not happen when parsing metadata with capgen's metadata parser, remove? + if CCPP_HORIZONTAL_LOOP_BEGIN in dims or CCPP_HORIZONTAL_LOOP_END in dims or \ + CCPP_LOOP_EXTENT in dims: + raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ + f"horizontal dimension for {var_standard_name} is {var.dimensions}") + if dim0 == dim1: + array_size.append('1') + dim_substrings.append(f'{dim1}') + else: + array_size.append(f'({dim1}-{dim0}+1)') + dim_substrings.append(f'{dim0}:{dim1}') + dim_string = '({})'.format(','.join(dim_substrings)) + var_size_expected = '({})'.format('*'.join(array_size)) + else: + dim_string = '' + var_size_expected = 1 # To assist debugging efforts, check if arrays have the correct size (ignore scalars for now) + # DH* 20231226 this doesn't make much sense anymore. let's rather check that the lower and upper bounds + # are correct. assign_test = '' if debug: if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ @@ -1274,56 +1374,14 @@ def write(self, metadata_request, metadata_define, arguments, debug): # deals with non-uniform block sizes etc. pass elif var.rank: - array_size = [] - for dim in var.dimensions: - # This is not supported/implemented: tmpvar would have one dimension less - # than the original array, and the metadata requesting the variable would - # not pass the initial test that host model variables and scheme variables - # have the same rank. - if dim == CCPP_BLOCK_NUMBER: - raise Exception("{} cannot be part of the dimensions of variable {}".format( - CCPP_BLOCK_NUMBER, var_standard_name)) - else: - # Handle dimensions like "A:B", "A:3", "-1:Z" - if ':' in dim: - dims = [ x.lower() for x in dim.split(':')] - try: - dim0 = int(dims[0]) - dim0 = dims[0] - except ValueError: - if not dims[0].lower() in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dims[0].lower(), var_standard_name)) - dim0 = metadata_define[dims[0].lower()][0].local_name - try: - dim1 = int(dims[1]) - dim1 = dims[1] - except ValueError: - if not dims[1].lower() in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dims[1].lower(), var_standard_name)) - dim1 = metadata_define[dims[1].lower()][0].local_name - # Single dimensions - else: - dim0 = 1 - try: - dim1 = int(dim) - dim1 = dim - except ValueError: - if not dim.lower() in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dim.lower(), var_standard_name)) - dim1 = metadata_define[dim.lower()][0].local_name - array_size.append('({last}-{first}+1)'.format(last=dim1, first=dim0)) - var_size_expected = '({})'.format('*'.join(array_size)) assign_test = ''' ! Check if variable {var_name} is associated/allocated and has the correct size - if (size({var_name})/={var_size_expected}) then - write({ccpp_errmsg}, '(2(a,i8))') 'Detected size mismatch for variable {var_name} in group {group_name} before {subroutine_name}, expected ', & - {var_size_expected}, ' but got ', size({var_name}) + if (size({var_name}{dim_string})/={var_size_expected}) then + write({ccpp_errmsg}, '(2(a,i8))') 'Detected size mismatch for variable {var_name}{dim_string} in group {group_name} before {subroutine_name}, expected ', & + {var_size_expected}, ' but got ', size({var_name}{dim_string}) ierr = 1 return end if -'''.format(var_name=local_vars[var_standard_name]['name'], var_size_expected=var_size_expected, +'''.format(var_name=local_vars[var_standard_name]['name'], dim_string=dim_string, var_size_expected=var_size_expected, ccpp_errmsg=CCPP_INTERNAL_VARIABLES[CCPP_ERROR_MSG_VARIABLE], group_name = self.name, subroutine_name=subroutine_name) # end if debug @@ -1492,21 +1550,20 @@ def write(self, metadata_request, metadata_define, arguments, debug): tmpvar.local_name = 'tmpvar{0}'.format(tmpvar_cnt) tmpvars[local_vars[var_standard_name]['name']] = tmpvar if tmpvar.rank: - # Add allocate statement if the variable has a rank > 0 - # According to https://fortran-lang.discourse.group/t/allocated-array-bounds-with-mold-and-source-on-expressions/3032, - # allocating with 'source=...' sets the correct lower and upper bounds for the newly created array - actions_in += ' allocate({t}, source={v})\n'.format(t=tmpvar.local_name, - v=tmpvar.target) + # Add allocate statement if the variable has a rank > 0 using the dimstring derived above + actions_in += f' allocate({tmpvar.local_name}{dim_string})' if var.actions['in']: # Add unit conversion before entering the subroutine - actions_in += ' {t} = {c}\n'.format(t=tmpvar.local_name, - c=var.actions['in'].format(var=tmpvar.target, - kind=kind_string)) + actions_in += ' {t} = {c}{d}\n'.format(t=tmpvar.local_name, + c=var.actions['in'].format(var=tmpvar.target, + kind=kind_string), + d=dim_string) if var.actions['out']: # Add unit conversion after returning from the subroutine - actions_out += ' {v} = {c}\n'.format(v=tmpvar.target, - c=var.actions['out'].format(var=tmpvar.local_name, - kind=kind_string)) + actions_out += ' {v}{d} = {c}\n'.format(v=tmpvar.target, + d=dim_string, + c=var.actions['out'].format(var=tmpvar.local_name, + kind=kind_string)) if tmpvar.rank: # Add deallocate statement if the variable has a rank > 0 actions_out += ' deallocate({t})\n'.format(t=tmpvar.local_name) @@ -1527,7 +1584,8 @@ def write(self, metadata_request, metadata_define, arguments, debug): actions_out=actions_out.rstrip('\n')) # Add to argument list - arg = '{local_name}={var_name},'.format(local_name=var.local_name, var_name=tmpvar.local_name) + arg = '{local_name}={var_name}{dim_string},'.format(local_name=var.local_name, + var_name=tmpvar.local_name, dim_string=dim_string) # Ordinary variables, no blocked data or unit conversions elif var_standard_name in arguments[scheme_name][subroutine_name]: @@ -1542,8 +1600,8 @@ def write(self, metadata_request, metadata_define, arguments, debug): actions_in=actions_in.rstrip('\n')) # Add to argument list - arg = '{local_name}={var_name},'.format(local_name=var.local_name, - var_name=local_vars[var_standard_name]['name']) + arg = '{local_name}={var_name}{dim_string},'.format(local_name=var.local_name, + var_name=local_vars[var_standard_name]['name'], dim_string=dim_string) else: arg = '' args += arg From d094d3f0ab48dbaa69128d4a9484152c863a272d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 26 Dec 2023 20:27:03 -0700 Subject: [PATCH 06/14] Save progress on mkstatic.py --- scripts/mkstatic.py | 136 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 114 insertions(+), 22 deletions(-) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 48cfa765..0261fbb4 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -101,6 +101,49 @@ def extract_parents_and_indices_from_local_name(local_name): indices = [ x.strip() for x in indices ] return (parent, indices) +def extract_dimensions_from_local_name(local_name): + """Extract the dimensions from a local_name. + Throw away any parent information.""" + # First, find delimiter '%' between parent(s) and child '%' + parent_delimiter_index = -1 + if '%' in local_name: + i = len(local_name)-1 + opened = 0 + while i >= 0: + if local_name[i] == ')': + opened += 1 + elif local_name[i] == '(': + opened -= 1 + elif local_name[i] == '%' and opened == 0: + parent_delimiter_index = i + break + i -= 1 + print(f"{local_name[parent_delimiter_index+1:]}") + if '(' in local_name[parent_delimiter_index+1:]: + dim_string_start = local_name[parent_delimiter_index+1:].find('(') + dim_string_end = local_name[parent_delimiter_index+1:].rfind(')') + dim_string = local_name[parent_delimiter_index+1:][dim_string_start:dim_string_end+1] + logging.info(f"found dim_string '{dim_string}' in '{local_name}'") + # Now that we have a dim_string, find all dimensions in this string; + # ignore outermost opening and closing parentheses. + opened = 0 + dim = '' + i = 1 + dimensions = [] + while i <= len(dim_string)-1: + if dim_string[i] == ',' and opened == 0: + dimensions.append(dim) + dim = '' + elif i == len(dim_string)-1 and dim: + dimensions.append(dim) + else: + dim += dim_string[i] + i += 1 + else: + dimensions = [] + dim_string = '' + return (dimensions, dim_string) + def create_argument_list_wrapped(arguments): """Create a wrapped argument list, remove trailing ',' """ argument_list = '' @@ -1060,7 +1103,10 @@ def write(self, metadata_request, metadata_define, arguments, debug): and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: logging.debug("Adding variable {} for handling blocked data structures".format(add_var)) additional_variables_required.append(add_var) - elif ccpp_stage == 'run': + elif ccpp_stage == 'run' and \ + CCPP_HORIZONTAL_LOOP_BEGIN in metadata_define.keys() and \ + CCPP_HORIZONTAL_LOOP_END in metadata_define.keys() and \ + CCPP_CHUNK_EXTENT in metadata_define.keys(): for add_var in [ CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, CCPP_CHUNK_EXTENT]: if not add_var in local_vars.keys() \ and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: @@ -1270,6 +1316,9 @@ def write(self, metadata_request, metadata_define, arguments, debug): if container == var.container: break + + (dimensions_target_name, dim_string_target_name) = extract_dimensions_from_local_name(var.target) + # Derive correct horizontal loop extent for this variable for the rest of this function if var.rank: array_size = [] @@ -1301,12 +1350,21 @@ def write(self, metadata_request, metadata_define, arguments, debug): # Use correct horizontal variables in run phase if ccpp_stage == 'run' and dims[1].lower() == CCPP_HORIZONTAL_LOOP_EXTENT: # Provide backward compatibility with blocked data structures - if CCPP_BLOCK_NUMBER in additional_variables_required + arguments[scheme_name][subroutine_name]: + if CCPP_BLOCK_COUNT in metadata_define.keys(): dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name else: + ## Again provide backward compatibility with blocked data structures. + ## The following is a limitation of ccpp_prebuild, but I don't know any case + ## where this would be used in the currently supported models. We + ## can't have both CCPP_HORIZONTAL_LOOP_BEGIN+CCPP_HORIZONTAL_LOOP_END + ## and CCPP_HORIZONTAL_LOOP_EXTENT coming from the host model. + #if CCPP_HORIZONTAL_LOOP_BEGIN in additional_variables_required + arguments[scheme_name][subroutine_name]: dim0 = metadata_define[CCPP_HORIZONTAL_LOOP_BEGIN][0].local_name dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_END][0].local_name + #else: + # dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name + # dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name else: if not dims[1].lower() in metadata_define.keys(): raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( @@ -1314,20 +1372,22 @@ def write(self, metadata_request, metadata_define, arguments, debug): dim1 = metadata_define[dims[1].lower()][0].local_name # Single dimensions are indices else: - try: - dim1 = int(dim) - dim1 = dim - except ValueError: - # This should not happen with capgen, because dimensions always have a lower and upper bound (n:m)? - if dim.lower() in [CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, \ - CCPP_HORIZONTAL_LOOP_EXTENT, CCPP_HORIZONTAL_DIMENSION]: - raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - f"horizontal dimension for {var_standard_name} is {var.dimensions}") - if not dim.lower() in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dim.lower(), var_standard_name)) - dim1 = metadata_define[dim.lower()][0].local_name - dim0 = dim1 + raise Exception("THIS SHOULD NOT HAPPEN WITH CAPGEN'S METADATA PARSER") + #try: + # dim1 = int(dim) + # dim1 = dim + #except ValueError: + # # This should not happen with capgen, because dimensions always have a lower and upper bound (n:m)? + # if dim.lower() in [CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, \ + # CCPP_HORIZONTAL_LOOP_EXTENT, CCPP_HORIZONTAL_DIMENSION]: + # raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ + # f"horizontal dimension for {var_standard_name} is {var.dimensions}") + # # + # if not dim.lower() in metadata_define.keys(): + # raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( + # dim.lower(), var_standard_name)) + # dim1 = metadata_define[dim.lower()][0].local_name + #dim0 = dim1 # Handle horizontal dimensions correctly if ccpp_stage == 'run': @@ -1356,11 +1416,35 @@ def write(self, metadata_request, metadata_define, arguments, debug): else: array_size.append(f'({dim1}-{dim0}+1)') dim_substrings.append(f'{dim0}:{dim1}') - dim_string = '({})'.format(','.join(dim_substrings)) + + # Now we need to compare dim_substringa with a possible dim_string_target_name and merge them + if dimensions_target_name: + if len(dimensions_target_name) < len(dim_substrings): + raise Exception("THIS SHOULD NOT HAPPEN") + dim_counter = 0 + dim_string = '(' + for dim in dimensions_target_name: + if ':' in dim: + dim_string += dim_substrings[dim_counter] + ',' + dim_counter += 1 + else: + dim_string += dim + ',' + dim_string = dim_string.rstrip(',') + ')' + # Consistency check to make sure all dimensions from metadata are 'used' + if dim_counter < len(dim_substrings): + raise Exception(f"Mismatch of derived dimensions from metadata {dim_substrings} " + \ + f"vs target local name {dimensions_target_name} for {var_standard_name} and " + \ + f"scheme {scheme_name} / phase {ccpp_stage}") + else: + dim_string = '({})'.format(','.join(dim_substrings)) var_size_expected = '({})'.format('*'.join(array_size)) else: - dim_string = '' + if dimensions_target_name: + dim_string = dim_string_target_name + else: + dim_string = '' var_size_expected = 1 + # To assist debugging efforts, check if arrays have the correct size (ignore scalars for now) # DH* 20231226 this doesn't make much sense anymore. let's rather check that the lower and upper bounds # are correct. @@ -1393,6 +1477,8 @@ def write(self, metadata_request, metadata_define, arguments, debug): if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name'] and \ '{}:{}'.format(CCPP_CONSTANT_ONE,CCPP_HORIZONTAL_DIMENSION) in var.dimensions: + ## Need to reset dim_string since this is handled separately for blocked data + #dim_string = '' # Reuse existing temporary variable, if possible if local_vars[var_standard_name]['name'] in tmpvars.keys(): # If the variable already has a local variable (tmpvar), reuse it @@ -1446,6 +1532,10 @@ def write(self, metadata_request, metadata_define, arguments, debug): var_defs_manual.append('integer :: ib, nb') # Define actions before. Always copy data in, independent of intent. + # We intentionally omit the dim string for the assignment on the right hand side, + # since it worked without until now, since coding this up together with chunked array + # logic is tricky, and since all this logic will go away after the models transitioned + # to chunked arrays. actions_in = ''' ! Allocate local variable to copy blocked data {var} into a contiguous array allocate({tmpvar}({dims})) ib = 1 @@ -1555,12 +1645,12 @@ def write(self, metadata_request, metadata_define, arguments, debug): if var.actions['in']: # Add unit conversion before entering the subroutine actions_in += ' {t} = {c}{d}\n'.format(t=tmpvar.local_name, - c=var.actions['in'].format(var=tmpvar.target, + c=var.actions['in'].format(var=tmpvar.target.replace(dim_string_target_name, ''), kind=kind_string), d=dim_string) if var.actions['out']: # Add unit conversion after returning from the subroutine - actions_out += ' {v}{d} = {c}\n'.format(v=tmpvar.target, + actions_out += ' {v}{d} = {c}\n'.format(v=tmpvar.target.replace(dim_string_target_name, ''), d=dim_string, c=var.actions['out'].format(var=tmpvar.local_name, kind=kind_string)) @@ -1585,7 +1675,7 @@ def write(self, metadata_request, metadata_define, arguments, debug): # Add to argument list arg = '{local_name}={var_name}{dim_string},'.format(local_name=var.local_name, - var_name=tmpvar.local_name, dim_string=dim_string) + var_name=tmpvar.local_name.replace(dim_string_target_name, ''), dim_string=dim_string) # Ordinary variables, no blocked data or unit conversions elif var_standard_name in arguments[scheme_name][subroutine_name]: @@ -1600,8 +1690,10 @@ def write(self, metadata_request, metadata_define, arguments, debug): actions_in=actions_in.rstrip('\n')) # Add to argument list + #if var.local_name == 'qgraupel': #dim_string_target_name and dim_string and var.local_name == 'qgraupel': + # raise Exception(f"{var.local_name}: {dim_string_target_name}, {dim_string}; replace? {local_vars[var_standard_name]['name']} --> ") arg = '{local_name}={var_name}{dim_string},'.format(local_name=var.local_name, - var_name=local_vars[var_standard_name]['name'], dim_string=dim_string) + var_name=local_vars[var_standard_name]['name'].replace(dim_string_target_name, ''), dim_string=dim_string) else: arg = '' args += arg From b81082ecb726377ade143e8b11e4b918ef918513 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Dec 2023 09:54:28 -0700 Subject: [PATCH 07/14] First round of cleanup in scripts/mkstatic.py --- scripts/mkstatic.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 0261fbb4..7de84716 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -118,11 +118,11 @@ def extract_dimensions_from_local_name(local_name): parent_delimiter_index = i break i -= 1 - print(f"{local_name[parent_delimiter_index+1:]}") if '(' in local_name[parent_delimiter_index+1:]: dim_string_start = local_name[parent_delimiter_index+1:].find('(') dim_string_end = local_name[parent_delimiter_index+1:].rfind(')') dim_string = local_name[parent_delimiter_index+1:][dim_string_start:dim_string_end+1] + # DH* TODO REMOVE DEBUG STATEMENT logging.info(f"found dim_string '{dim_string}' in '{local_name}'") # Now that we have a dim_string, find all dimensions in this string; # ignore outermost opening and closing parentheses. @@ -1354,17 +1354,8 @@ def write(self, metadata_request, metadata_define, arguments, debug): dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name else: - ## Again provide backward compatibility with blocked data structures. - ## The following is a limitation of ccpp_prebuild, but I don't know any case - ## where this would be used in the currently supported models. We - ## can't have both CCPP_HORIZONTAL_LOOP_BEGIN+CCPP_HORIZONTAL_LOOP_END - ## and CCPP_HORIZONTAL_LOOP_EXTENT coming from the host model. - #if CCPP_HORIZONTAL_LOOP_BEGIN in additional_variables_required + arguments[scheme_name][subroutine_name]: dim0 = metadata_define[CCPP_HORIZONTAL_LOOP_BEGIN][0].local_name dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_END][0].local_name - #else: - # dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name - # dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name else: if not dims[1].lower() in metadata_define.keys(): raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( @@ -1372,6 +1363,7 @@ def write(self, metadata_request, metadata_define, arguments, debug): dim1 = metadata_define[dims[1].lower()][0].local_name # Single dimensions are indices else: + # DH* TODO UPDATE EXCEPTION MESSAGE AND REMOVE COMMENTS BELOW raise Exception("THIS SHOULD NOT HAPPEN WITH CAPGEN'S METADATA PARSER") #try: # dim1 = int(dim) @@ -1389,6 +1381,7 @@ def write(self, metadata_request, metadata_define, arguments, debug): # dim1 = metadata_define[dim.lower()][0].local_name #dim0 = dim1 + # DH* TODO REMOVE THIS ENTIRE BLOCK # Handle horizontal dimensions correctly if ccpp_stage == 'run': # This should not happen when parsing metadata with capgen's metadata parser, remove? @@ -1410,6 +1403,8 @@ def write(self, metadata_request, metadata_define, arguments, debug): CCPP_LOOP_EXTENT in dims: raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ f"horizontal dimension for {var_standard_name} is {var.dimensions}") + # *DH + if dim0 == dim1: array_size.append('1') dim_substrings.append(f'{dim1}') @@ -1446,8 +1441,8 @@ def write(self, metadata_request, metadata_define, arguments, debug): var_size_expected = 1 # To assist debugging efforts, check if arrays have the correct size (ignore scalars for now) - # DH* 20231226 this doesn't make much sense anymore. let's rather check that the lower and upper bounds - # are correct. + # DH* 20231226 This doesn't make much sense anymore. In a future PR, let's rather check that + # the lower and upper bounds of the arrays are correct. assign_test = '' if debug: if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ @@ -1477,8 +1472,6 @@ def write(self, metadata_request, metadata_define, arguments, debug): if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name'] and \ '{}:{}'.format(CCPP_CONSTANT_ONE,CCPP_HORIZONTAL_DIMENSION) in var.dimensions: - ## Need to reset dim_string since this is handled separately for blocked data - #dim_string = '' # Reuse existing temporary variable, if possible if local_vars[var_standard_name]['name'] in tmpvars.keys(): # If the variable already has a local variable (tmpvar), reuse it @@ -1690,8 +1683,6 @@ def write(self, metadata_request, metadata_define, arguments, debug): actions_in=actions_in.rstrip('\n')) # Add to argument list - #if var.local_name == 'qgraupel': #dim_string_target_name and dim_string and var.local_name == 'qgraupel': - # raise Exception(f"{var.local_name}: {dim_string_target_name}, {dim_string}; replace? {local_vars[var_standard_name]['name']} --> ") arg = '{local_name}={var_name}{dim_string},'.format(local_name=var.local_name, var_name=local_vars[var_standard_name]['name'].replace(dim_string_target_name, ''), dim_string=dim_string) else: From 4b0bf8d3674e46980a4df24cbed387e45c6271ef Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Dec 2023 10:25:32 -0700 Subject: [PATCH 08/14] Bug fix in scripts/mkstatic.py: also strip out original target dim string for assign tests --- scripts/mkstatic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 7de84716..e462a77c 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -1460,7 +1460,7 @@ def write(self, metadata_request, metadata_define, arguments, debug): ierr = 1 return end if -'''.format(var_name=local_vars[var_standard_name]['name'], dim_string=dim_string, var_size_expected=var_size_expected, +'''.format(var_name=local_vars[var_standard_name]['name'].replace(dim_string_target_name, ''), dim_string=dim_string, var_size_expected=var_size_expected, ccpp_errmsg=CCPP_INTERNAL_VARIABLES[CCPP_ERROR_MSG_VARIABLE], group_name = self.name, subroutine_name=subroutine_name) # end if debug From ff187876be34943ce250a95b1f62a21708deb1f8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Dec 2023 12:40:25 -0700 Subject: [PATCH 09/14] Another fix of the logic to determine the correct horizontal dimension in the run phase in scripts/mkstatic.py --- scripts/mkstatic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index e462a77c..7bd3deb3 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -1350,12 +1350,12 @@ def write(self, metadata_request, metadata_define, arguments, debug): # Use correct horizontal variables in run phase if ccpp_stage == 'run' and dims[1].lower() == CCPP_HORIZONTAL_LOOP_EXTENT: # Provide backward compatibility with blocked data structures - if CCPP_BLOCK_COUNT in metadata_define.keys(): - dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name - dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name - else: + if CCPP_HORIZONTAL_LOOP_BEGIN in metadata_define.keys(): dim0 = metadata_define[CCPP_HORIZONTAL_LOOP_BEGIN][0].local_name dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_END][0].local_name + else: + dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name + dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name else: if not dims[1].lower() in metadata_define.keys(): raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( From bd30e3beb0edc2f93d65cd10de5c4c2242d2cded Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Dec 2023 12:40:43 -0700 Subject: [PATCH 10/14] Bug fix in src/ccpp_types.F90 to test if CCPP is initialized or not --- src/ccpp_types.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ccpp_types.F90 b/src/ccpp_types.F90 index ef08b6f2..dafa7c80 100644 --- a/src/ccpp_types.F90 +++ b/src/ccpp_types.F90 @@ -77,9 +77,9 @@ function ccpp_t_initialized(ccpp_d) result(initialized) class(ccpp_t) :: ccpp_d logical :: initialized ! - initialized = ccpp_d%thrd_no /= CCPP_DEFAULT_THREAD_NUMBER .and. & - (ccpp_d%blk_no /= CCPP_DEFAULT_BLOCK_NUMBER .and. & - ccpp_d%chunk_no /= CCPP_DEFAULT_CHUNK_NUMBER) + initialized = ccpp_d%thrd_no /= CCPP_DEFAULT_THREAD_NUMBER .or. & + ccpp_d%blk_no /= CCPP_DEFAULT_BLOCK_NUMBER .or. & + ccpp_d%chunk_no /= CCPP_DEFAULT_CHUNK_NUMBER end function ccpp_t_initialized end module ccpp_types From f3193e9506a1421154ff1128e409ef196e2c1c7f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 27 Dec 2023 21:00:03 -0700 Subject: [PATCH 11/14] Don't use explicit dimensions for dimensions other than the horizontal dimension --- scripts/mkstatic.py | 59 ++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 7bd3deb3..3eee0818 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -1093,10 +1093,11 @@ def write(self, metadata_request, metadata_define, arguments, debug): args = '' length = 0 - # First identify all dimensions needed to handle the arguments - # and add them to the list of required variables for the cap + # First, add a few mandatory variables to the list of required + # variables. This is mostly for handling horizontal dimensions + # correctly for the different CCPP phases and for cases when + # blocked data structures or chunked arrays are used. additional_variables_required = [] - # if CCPP_HORIZONTAL_LOOP_EXTENT in metadata_define.keys(): for add_var in [ CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_LOOP_EXTENT]: if not add_var in local_vars.keys() \ @@ -1112,7 +1113,8 @@ def write(self, metadata_request, metadata_define, arguments, debug): and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: logging.debug("Adding variable {} for handling chunked data arrays".format(add_var)) additional_variables_required.append(add_var) - # + # Next, identify all dimensions needed to handle the arguments + # and add them to the list of required variables for the cap for var_standard_name in arguments[scheme_name][subroutine_name]: if not var_standard_name in metadata_define.keys(): raise Exception('Variable {standard_name} not defined in host model metadata'.format( @@ -1361,25 +1363,9 @@ def write(self, metadata_request, metadata_define, arguments, debug): raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( dims[1].lower(), var_standard_name)) dim1 = metadata_define[dims[1].lower()][0].local_name - # Single dimensions are indices + # Single dimensions are indices and should not be recorded as a dimension! else: - # DH* TODO UPDATE EXCEPTION MESSAGE AND REMOVE COMMENTS BELOW raise Exception("THIS SHOULD NOT HAPPEN WITH CAPGEN'S METADATA PARSER") - #try: - # dim1 = int(dim) - # dim1 = dim - #except ValueError: - # # This should not happen with capgen, because dimensions always have a lower and upper bound (n:m)? - # if dim.lower() in [CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, \ - # CCPP_HORIZONTAL_LOOP_EXTENT, CCPP_HORIZONTAL_DIMENSION]: - # raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - # f"horizontal dimension for {var_standard_name} is {var.dimensions}") - # # - # if not dim.lower() in metadata_define.keys(): - # raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - # dim.lower(), var_standard_name)) - # dim1 = metadata_define[dim.lower()][0].local_name - #dim0 = dim1 # DH* TODO REMOVE THIS ENTIRE BLOCK # Handle horizontal dimensions correctly @@ -1404,13 +1390,32 @@ def write(self, metadata_request, metadata_define, arguments, debug): raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ f"horizontal dimension for {var_standard_name} is {var.dimensions}") # *DH - - if dim0 == dim1: - array_size.append('1') - dim_substrings.append(f'{dim1}') + # DH* TODO: move this to a function "is_horizontal_dimension" ? + # maybe combine with some of the above logic to get the correct dimensions + # depending on phase etc? + # We only want to be explicit about horizontal dimensions, for which we need + # to subset in certain cases (therefore do it always). Other dimensions are + # passed as ':'. + if dim in [ + f"{CCPP_CONSTANT_ONE}:{CCPP_HORIZONTAL_LOOP_EXTENT}", + f"{CCPP_CONSTANT_ONE}:{CCPP_HORIZONTAL_DIMENSION}", + # This should never be the case for schemes, only for hosts! + #f"{CCPP_HORIZONTAL_LOOP_BEGIN}:{CCPP_HORIZONTAL_LOOP_END}", + ]: + if dim0 == dim1: + array_size.append('1') + dim_substrings.append(f'{dim1}') + else: + array_size.append(f'({dim1}-{dim0}+1)') + dim_substrings.append(f'{dim0}:{dim1}') else: - array_size.append(f'({dim1}-{dim0}+1)') - dim_substrings.append(f'{dim0}:{dim1}') + if dim0 == dim1: + array_size.append('1') + # DH* TODO - is this correct? + dim_substrings.append(f':') + else: + array_size.append(f'({dim1}-{dim0}+1)') + dim_substrings.append(f':') # Now we need to compare dim_substringa with a possible dim_string_target_name and merge them if dimensions_target_name: From 3d69bb9eba43805ecd68e59cdec3345008e1cad5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Dec 2023 09:38:47 -0700 Subject: [PATCH 12/14] More fine tuning in scripts/mkstatic.py to avoid problems with how inactive arrays are passed currently in ccpp_prebuild --- scripts/mkstatic.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 3eee0818..c942564c 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -1326,6 +1326,14 @@ def write(self, metadata_request, metadata_define, arguments, debug): array_size = [] dim_substrings = [] for dim in var.dimensions: + # DH* TODO + # By default, don't pass explicit dimensions, only ':'. This is because + # we have not solved the problem of passing inactive arrays correctly + # (something that needs to be done before we can use chunked arrays in + # models like the ufs-weather-model). See also note further down where + # this variable gets set to True and then where it gets used. + use_explicit_dimension = False + # *DH # This is not supported/implemented: tmpvar would have one dimension less # than the original array, and the metadata requesting the variable would # not pass the initial test that host model variables and scheme variables @@ -1352,9 +1360,11 @@ def write(self, metadata_request, metadata_define, arguments, debug): # Use correct horizontal variables in run phase if ccpp_stage == 'run' and dims[1].lower() == CCPP_HORIZONTAL_LOOP_EXTENT: # Provide backward compatibility with blocked data structures + # and bypass the unresolved problems with inactive data if CCPP_HORIZONTAL_LOOP_BEGIN in metadata_define.keys(): dim0 = metadata_define[CCPP_HORIZONTAL_LOOP_BEGIN][0].local_name dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_END][0].local_name + use_explicit_dimension = True else: dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name @@ -1367,7 +1377,8 @@ def write(self, metadata_request, metadata_define, arguments, debug): else: raise Exception("THIS SHOULD NOT HAPPEN WITH CAPGEN'S METADATA PARSER") - # DH* TODO REMOVE THIS ENTIRE BLOCK + # DH* TODO REMOVE THIS ENTIRE BLOCK - create a test suite to make sure these things are caught + # by the metadata parser - do this on the feature/capgen branch! # Handle horizontal dimensions correctly if ccpp_stage == 'run': # This should not happen when parsing metadata with capgen's metadata parser, remove? @@ -1390,18 +1401,19 @@ def write(self, metadata_request, metadata_define, arguments, debug): raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ f"horizontal dimension for {var_standard_name} is {var.dimensions}") # *DH + # DH* TODO: move this to a function "is_horizontal_dimension" ? # maybe combine with some of the above logic to get the correct dimensions # depending on phase etc? # We only want to be explicit about horizontal dimensions, for which we need # to subset in certain cases (therefore do it always). Other dimensions are # passed as ':'. - if dim in [ - f"{CCPP_CONSTANT_ONE}:{CCPP_HORIZONTAL_LOOP_EXTENT}", - f"{CCPP_CONSTANT_ONE}:{CCPP_HORIZONTAL_DIMENSION}", - # This should never be the case for schemes, only for hosts! - #f"{CCPP_HORIZONTAL_LOOP_BEGIN}:{CCPP_HORIZONTAL_LOOP_END}", - ]: + # + # DH* TODO: WE CANNOT ACTIVATE USING EXPLICIT HORIZONTAL BLOCKS + # UNTIL WE HAVE SOLVED THE PROBLEM WITH INACTIVE (NON-ALLOCATED) + # ARRAYS. THIS MUST BE ADDRESSED BEFORE WE SWITCH TO CONTIGUOUS + # ARRAYS FOR MODELS LIKE THE UFS-WEATHER-MODEL! + if use_explicit_dimension: if dim0 == dim1: array_size.append('1') dim_substrings.append(f'{dim1}') @@ -1446,8 +1458,6 @@ def write(self, metadata_request, metadata_define, arguments, debug): var_size_expected = 1 # To assist debugging efforts, check if arrays have the correct size (ignore scalars for now) - # DH* 20231226 This doesn't make much sense anymore. In a future PR, let's rather check that - # the lower and upper bounds of the arrays are correct. assign_test = '' if debug: if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ From 2c89204f924166fe5fc33743f118054f8b9df209 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Dec 2023 15:29:13 -0700 Subject: [PATCH 13/14] Remove debugging code from scripts/mkstatic.py --- scripts/mkstatic.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index c942564c..09a7853c 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -122,8 +122,6 @@ def extract_dimensions_from_local_name(local_name): dim_string_start = local_name[parent_delimiter_index+1:].find('(') dim_string_end = local_name[parent_delimiter_index+1:].rfind(')') dim_string = local_name[parent_delimiter_index+1:][dim_string_start:dim_string_end+1] - # DH* TODO REMOVE DEBUG STATEMENT - logging.info(f"found dim_string '{dim_string}' in '{local_name}'") # Now that we have a dim_string, find all dimensions in this string; # ignore outermost opening and closing parentheses. opened = 0 @@ -1377,9 +1375,11 @@ def write(self, metadata_request, metadata_define, arguments, debug): else: raise Exception("THIS SHOULD NOT HAPPEN WITH CAPGEN'S METADATA PARSER") - # DH* TODO REMOVE THIS ENTIRE BLOCK - create a test suite to make sure these things are caught + # DH* TODO REMOVE THIS ENTIRE BLOCK in a future PR to feature/capgen + # This block should not be needed, the metadata parser should take care + # of flagging invalid dimensions for the host model or the physics. + # TODO: create a test suite to make sure these things are caught # by the metadata parser - do this on the feature/capgen branch! - # Handle horizontal dimensions correctly if ccpp_stage == 'run': # This should not happen when parsing metadata with capgen's metadata parser, remove? if dims[1] == CCPP_HORIZONTAL_LOOP_EXTENT and not dim0: @@ -1402,14 +1402,7 @@ def write(self, metadata_request, metadata_define, arguments, debug): f"horizontal dimension for {var_standard_name} is {var.dimensions}") # *DH - # DH* TODO: move this to a function "is_horizontal_dimension" ? - # maybe combine with some of the above logic to get the correct dimensions - # depending on phase etc? - # We only want to be explicit about horizontal dimensions, for which we need - # to subset in certain cases (therefore do it always). Other dimensions are - # passed as ':'. - # - # DH* TODO: WE CANNOT ACTIVATE USING EXPLICIT HORIZONTAL BLOCKS + # DH* TODO: WE CANNOT ACTIVATE USING EXPLICIT HORIZONTAL DIMENSIONS # UNTIL WE HAVE SOLVED THE PROBLEM WITH INACTIVE (NON-ALLOCATED) # ARRAYS. THIS MUST BE ADDRESSED BEFORE WE SWITCH TO CONTIGUOUS # ARRAYS FOR MODELS LIKE THE UFS-WEATHER-MODEL! @@ -1429,7 +1422,7 @@ def write(self, metadata_request, metadata_define, arguments, debug): array_size.append(f'({dim1}-{dim0}+1)') dim_substrings.append(f':') - # Now we need to compare dim_substringa with a possible dim_string_target_name and merge them + # Now we need to compare dim_substrings with a possible dim_string_target_name and merge them if dimensions_target_name: if len(dimensions_target_name) < len(dim_substrings): raise Exception("THIS SHOULD NOT HAPPEN") From f3153392917c54ffc5fa4e5edef3d9e060129c05 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 10 Jan 2024 15:11:55 -0700 Subject: [PATCH 14/14] Replace 'block' with 'chunk' in a few comments --- test_prebuild/test_chunked_data/main.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_prebuild/test_chunked_data/main.F90 b/test_prebuild/test_chunked_data/main.F90 index 9822b27d..cad6f2b6 100644 --- a/test_prebuild/test_chunked_data/main.F90 +++ b/test_prebuild/test_chunked_data/main.F90 @@ -31,9 +31,9 @@ program test_chunked_data ccpp_data_domain%thrd_no = 0 ccpp_data_domain%chunk_no = 0 - ! Loop over all blocks and threads for ccpp_data_chunks + ! Loop over all chunks and threads for ccpp_data_chunks do ic=1,nchunks - ! Assign the correct block numbers, only one thread + ! Assign the correct chunk numbers, only one thread ccpp_data_chunks(ic)%chunk_no = ic ccpp_data_chunks(ic)%thrd_no = 1 end do @@ -73,7 +73,7 @@ program test_chunked_data cdata => ccpp_data_chunks(ic) call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) if (ierr/=0) then - write(error_unit,'(a,i3,a)') "An error occurred in ccpp_physics_run for block", ic, ":" + write(error_unit,'(a,i3,a)') "An error occurred in ccpp_physics_run for chunk", ic, ":" write(error_unit,'(a)') trim(cdata%errmsg) stop 1 end if