diff --git a/backends/memcheck/ceed-memcheck-qfunction.c b/backends/memcheck/ceed-memcheck-qfunction.c index 7c66e3601a..1bb8158584 100644 --- a/backends/memcheck/ceed-memcheck-qfunction.c +++ b/backends/memcheck/ceed-memcheck-qfunction.c @@ -19,6 +19,7 @@ static int CeedQFunctionApply_Memcheck(CeedQFunction qf, CeedInt Q, CeedVector *U, CeedVector *V) { Ceed ceed; void *ctx_data = NULL; + int input_block_ids[CEED_FIELD_MAX], output_block_ids[CEED_FIELD_MAX]; CeedInt num_in, num_out; CeedQFunctionUser f = NULL; CeedQFunctionField *output_fields; @@ -29,12 +30,21 @@ static int CeedQFunctionApply_Memcheck(CeedQFunction qf, CeedInt Q, CeedVector * CeedCallBackend(CeedQFunctionGetContextData(qf, CEED_MEM_HOST, &ctx_data)); CeedCallBackend(CeedQFunctionGetUserFunction(qf, &f)); CeedCallBackend(CeedQFunctionGetNumArgs(qf, &num_in, &num_out)); - int mem_block_ids[num_out]; - // Get input/output arrays + // Get input arrays for (CeedInt i = 0; i < num_in; i++) { + CeedSize len; + char name[32] = ""; + CeedCallBackend(CeedVectorGetArrayRead(U[i], CEED_MEM_HOST, &impl->inputs[i])); + + CeedCallBackend(CeedVectorGetLength(U[i], &len)); + + snprintf(name, 32, "QFunction input %" CeedInt_FMT, i); + input_block_ids[i] = VALGRIND_CREATE_BLOCK(impl->inputs[i], len, name); } + + // Get output arrays for (CeedInt i = 0; i < num_out; i++) { CeedSize len; char name[32] = ""; @@ -44,8 +54,8 @@ static int CeedQFunctionApply_Memcheck(CeedQFunction qf, CeedInt Q, CeedVector * CeedCallBackend(CeedVectorGetLength(V[i], &len)); VALGRIND_MAKE_MEM_UNDEFINED(impl->outputs[i], len); - snprintf(name, 32, "'QFunction output %" CeedInt_FMT "'", i); - mem_block_ids[i] = VALGRIND_CREATE_BLOCK(impl->outputs[i], len, name); + snprintf(name, 32, "QFunction output %" CeedInt_FMT, i); + output_block_ids[i] = VALGRIND_CREATE_BLOCK(impl->outputs[i], len, name); } // Call user function @@ -54,8 +64,10 @@ static int CeedQFunctionApply_Memcheck(CeedQFunction qf, CeedInt Q, CeedVector * // Restore input arrays for (CeedInt i = 0; i < num_in; i++) { CeedCallBackend(CeedVectorRestoreArrayRead(U[i], &impl->inputs[i])); + VALGRIND_DISCARD(input_block_ids[i]); } - // Check for unset output values + + // Check for unset output values and restore arrays { const char *kernel_name, *kernel_path; @@ -63,17 +75,19 @@ static int CeedQFunctionApply_Memcheck(CeedQFunction qf, CeedInt Q, CeedVector * CeedCallBackend(CeedQFunctionGetKernelName(qf, &kernel_name)); CeedCallBackend(CeedQFunctionGetFields(qf, NULL, NULL, NULL, &output_fields)); for (CeedInt i = 0; i < num_out; i++) { - CeedInt field_size; + const char *field_name; + CeedInt field_size; // Note: need field size because vector may be longer than needed for output CeedCallBackend(CeedQFunctionFieldGetSize(output_fields[i], &field_size)); + CeedCallBackend(CeedQFunctionFieldGetName(output_fields[i], &field_name)); for (CeedSize j = 0; j < field_size * (CeedSize)Q; j++) { CeedCheck(!isnan(impl->outputs[i][j]), ceed, CEED_ERROR_BACKEND, - "QFunction output %" CeedInt_FMT " entry %" CeedSize_FMT " is NaN after restoring write-only access: %s:%s ", i, j, kernel_path, - kernel_name); + "QFunction output %" CeedInt_FMT " '%s' entry %" CeedSize_FMT " is NaN after restoring write-only access: %s:%s ", i, field_name, j, + kernel_path, kernel_name); } CeedCallBackend(CeedVectorRestoreArray(V[i], &impl->outputs[i])); - VALGRIND_DISCARD(mem_block_ids[i]); + VALGRIND_DISCARD(output_block_ids[i]); } } CeedCallBackend(CeedQFunctionRestoreContextData(qf, &ctx_data)); diff --git a/backends/memcheck/ceed-memcheck-qfunctioncontext.c b/backends/memcheck/ceed-memcheck-qfunctioncontext.c index 4da0d0ee68..57afe981af 100644 --- a/backends/memcheck/ceed-memcheck-qfunctioncontext.c +++ b/backends/memcheck/ceed-memcheck-qfunctioncontext.c @@ -20,7 +20,7 @@ static int CeedQFunctionContextHasValidData_Memcheck(CeedQFunctionContext ctx, b CeedQFunctionContext_Memcheck *impl; CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); - *has_valid_data = impl->data; + *has_valid_data = !!impl->data_allocated; return CEED_ERROR_SUCCESS; } @@ -30,9 +30,10 @@ static int CeedQFunctionContextHasValidData_Memcheck(CeedQFunctionContext ctx, b static int CeedQFunctionContextHasBorrowedDataOfType_Memcheck(CeedQFunctionContext ctx, CeedMemType mem_type, bool *has_borrowed_data_of_type) { CeedQFunctionContext_Memcheck *impl; - CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); CeedCheck(mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); - *has_borrowed_data_of_type = impl->data_borrowed; + + CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); + *has_borrowed_data_of_type = !!impl->data_borrowed; return CEED_ERROR_SUCCESS; } @@ -43,35 +44,69 @@ static int CeedQFunctionContextSetData_Memcheck(CeedQFunctionContext ctx, CeedMe size_t ctx_size; CeedQFunctionContext_Memcheck *impl; + CeedCheck(mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); + CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); CeedCallBackend(CeedQFunctionContextGetContextSize(ctx, &ctx_size)); - CeedCheck(mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); - + // Clear previous owned data buffers + if (impl->data_allocated) { + memset(impl->data_allocated, -42, ctx_size); + VALGRIND_DISCARD(impl->allocated_block_id); + } CeedCallBackend(CeedFree(&impl->data_allocated)); + if (impl->data_owned) { + memset(impl->data_owned, -42, ctx_size); + VALGRIND_DISCARD(impl->owned_block_id); + } CeedCallBackend(CeedFree(&impl->data_owned)); + + // Clear borrowed block id, if present + if (impl->data_borrowed) VALGRIND_DISCARD(impl->borrowed_block_id); + + // Set internal pointers to external buffers switch (copy_mode) { case CEED_COPY_VALUES: - CeedCallBackend(CeedMallocArray(1, ctx_size, &impl->data_owned)); + impl->data_owned = NULL; impl->data_borrowed = NULL; - impl->data = impl->data_owned; - memcpy(impl->data, data, ctx_size); break; case CEED_OWN_POINTER: - impl->data_owned = data; - impl->data_borrowed = NULL; - impl->data = data; + impl->data_owned = data; + impl->data_borrowed = NULL; + impl->owned_block_id = VALGRIND_CREATE_BLOCK(impl->data_owned, ctx_size, "Owned external data buffer"); break; case CEED_USE_POINTER: - impl->data_borrowed = data; - impl->data = data; + impl->data_owned = NULL; + impl->data_borrowed = data; + impl->owned_block_id = VALGRIND_CREATE_BLOCK(impl->data_borrowed, ctx_size, "Borrowed external data buffer"); } - // Copy data to check ctx_size bounds + + // Create internal data buffer CeedCallBackend(CeedMallocArray(1, ctx_size, &impl->data_allocated)); - memcpy(impl->data_allocated, impl->data, ctx_size); - impl->data = impl->data_allocated; - VALGRIND_DISCARD(impl->mem_block_id); - impl->mem_block_id = VALGRIND_CREATE_BLOCK(impl->data, ctx_size, "'QFunction backend context data copy'"); + impl->allocated_block_id = VALGRIND_CREATE_BLOCK(impl->data_allocated, ctx_size, "'Allocated internal context data buffer"); + memcpy(impl->data_allocated, data, ctx_size); + return CEED_ERROR_SUCCESS; +} + +//------------------------------------------------------------------------------ +// Sync data +//------------------------------------------------------------------------------ +static int CeedQFunctionContextSyncData_Memcheck(CeedQFunctionContext ctx, CeedMemType mem_type) { + size_t ctx_size; + CeedQFunctionContext_Memcheck *impl; + + CeedCheck(mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); + + CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); + CeedCallBackend(CeedQFunctionContextGetContextSize(ctx, &ctx_size)); + + // Copy internal buffer back to owned or borrowed data buffer + if (impl->data_owned) { + memcpy(impl->data_owned, impl->data_allocated, ctx_size); + } + if (impl->data_borrowed) { + memcpy(impl->data_borrowed, impl->data_allocated, ctx_size); + } return CEED_ERROR_SUCCESS; } @@ -79,16 +114,27 @@ static int CeedQFunctionContextSetData_Memcheck(CeedQFunctionContext ctx, CeedMe // QFunctionContext Take Data //------------------------------------------------------------------------------ static int CeedQFunctionContextTakeData_Memcheck(CeedQFunctionContext ctx, CeedMemType mem_type, void *data) { + size_t ctx_size; CeedQFunctionContext_Memcheck *impl; + CeedCheck(mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); + CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); + CeedCallBackend(CeedQFunctionContextGetContextSize(ctx, &ctx_size)); - CeedCheck(mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); + // Synchronize memory + CeedCallBackend(CeedQFunctionContextSyncData_Memcheck(ctx, CEED_MEM_HOST)); + // Return borrowed buffer *(void **)data = impl->data_borrowed; impl->data_borrowed = NULL; - impl->data = NULL; - VALGRIND_DISCARD(impl->mem_block_id); + VALGRIND_DISCARD(impl->borrowed_block_id); + + // De-allocate internal memory + if (impl->data_allocated) { + memset(impl->data_allocated, -42, ctx_size); + VALGRIND_DISCARD(impl->allocated_block_id); + } CeedCallBackend(CeedFree(&impl->data_allocated)); return CEED_ERROR_SUCCESS; } @@ -97,13 +143,19 @@ static int CeedQFunctionContextTakeData_Memcheck(CeedQFunctionContext ctx, CeedM // QFunctionContext Get Data //------------------------------------------------------------------------------ static int CeedQFunctionContextGetData_Memcheck(CeedQFunctionContext ctx, CeedMemType mem_type, void *data) { + size_t ctx_size; CeedQFunctionContext_Memcheck *impl; - CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); + CeedCheck(mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); - CeedCheck(mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); + CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); + CeedCallBackend(CeedQFunctionContextGetContextSize(ctx, &ctx_size)); - *(void **)data = impl->data; + // Create and return writable buffer + CeedCallBackend(CeedMallocArray(1, ctx_size, &impl->data_writable_copy)); + impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->data_writable_copy, ctx_size, "Allocated writeable data buffer copy"); + memcpy(impl->data_writable_copy, impl->data_allocated, ctx_size); + *(void **)data = impl->data_writable_copy; return CEED_ERROR_SUCCESS; } @@ -114,13 +166,18 @@ static int CeedQFunctionContextGetDataRead_Memcheck(CeedQFunctionContext ctx, Ce size_t ctx_size; CeedQFunctionContext_Memcheck *impl; + CeedCheck(mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); + CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); CeedCallBackend(CeedQFunctionContextGetContextSize(ctx, &ctx_size)); - CeedCallBackend(CeedQFunctionContextGetData_Memcheck(ctx, mem_type, data)); - // Make copy to verify no write occurred - CeedCallBackend(CeedMallocArray(1, ctx_size, &impl->data_read_only_copy)); - memcpy(impl->data_read_only_copy, *(void **)data, ctx_size); + // Create and return read-only buffer + if (!impl->data_read_only_copy) { + CeedCallBackend(CeedMallocArray(1, ctx_size, &impl->data_read_only_copy)); + impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->data_read_only_copy, ctx_size, "Allocated read-only data buffer copy"); + memcpy(impl->data_read_only_copy, impl->data_allocated, ctx_size); + } + *(void **)data = impl->data_read_only_copy; return CEED_ERROR_SUCCESS; } @@ -134,8 +191,14 @@ static int CeedQFunctionContextRestoreData_Memcheck(CeedQFunctionContext ctx) { CeedCallBackend(CeedQFunctionContextGetContextSize(ctx, &ctx_size)); CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); - if (impl->data_borrowed) memcpy(impl->data_borrowed, impl->data, ctx_size); - if (impl->data_owned) memcpy(impl->data_owned, impl->data, ctx_size); + // Copy back to internal buffer and sync + memcpy(impl->data_allocated, impl->data_writable_copy, ctx_size); + CeedCallBackend(CeedQFunctionContextSyncData_Memcheck(ctx, CEED_MEM_HOST)); + + // Invalidate writable buffer + memset(impl->data_writable_copy, -42, ctx_size); + CeedCallBackend(CeedFree(&impl->data_writable_copy)); + VALGRIND_DISCARD(impl->writable_block_id); return CEED_ERROR_SUCCESS; } @@ -143,16 +206,23 @@ static int CeedQFunctionContextRestoreData_Memcheck(CeedQFunctionContext ctx) { // QFunctionContext Restore Data Read-Only //------------------------------------------------------------------------------ static int CeedQFunctionContextRestoreDataRead_Memcheck(CeedQFunctionContext ctx) { + Ceed ceed; size_t ctx_size; CeedQFunctionContext_Memcheck *impl; + CeedCallBackend(CeedQFunctionContextGetCeed(ctx, &ceed)); CeedCallBackend(CeedQFunctionContextGetContextSize(ctx, &ctx_size)); CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); - CeedCheck(!memcmp(impl->data, impl->data_read_only_copy, ctx_size), CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, - "Context data changed while accessed in read-only mode"); + // Verify no changes made during read-only access + bool is_changed = memcmp(impl->data_allocated, impl->data_read_only_copy, ctx_size); + + CeedCheck(!is_changed, ceed, CEED_ERROR_BACKEND, "Context data changed while accessed in read-only mode"); + // Invalidate read-only buffer + memset(impl->data_read_only_copy, -42, ctx_size); CeedCallBackend(CeedFree(&impl->data_read_only_copy)); + VALGRIND_DISCARD(impl->read_only_block_id); return CEED_ERROR_SUCCESS; } @@ -160,20 +230,37 @@ static int CeedQFunctionContextRestoreDataRead_Memcheck(CeedQFunctionContext ctx // QFunctionContext destroy user data //------------------------------------------------------------------------------ static int CeedQFunctionContextDataDestroy_Memcheck(CeedQFunctionContext ctx) { + Ceed ceed; CeedMemType data_destroy_mem_type; CeedQFunctionContextDataDestroyUser data_destroy_function; CeedQFunctionContext_Memcheck *impl; + CeedCallBackend(CeedQFunctionContextGetCeed(ctx, &ceed)); CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); - CeedCallBackend(CeedQFunctionContextGetDataDestroy(ctx, &data_destroy_mem_type, &data_destroy_function)); - CeedCheck(data_destroy_mem_type == CEED_MEM_HOST, CeedQFunctionContextReturnCeed(ctx), CEED_ERROR_BACKEND, - "Can only destroy HOST memory for this backend"); + CeedCallBackend(CeedQFunctionContextGetDataDestroy(ctx, &data_destroy_mem_type, &data_destroy_function)); + CeedCheck(data_destroy_mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only destroy HOST memory for this backend"); + // Run user destroy routine if (data_destroy_function) { - CeedCallBackend(data_destroy_function(impl->data_borrowed ? impl->data_borrowed : impl->data_owned)); + bool is_borrowed = !!impl->data_borrowed; + + CeedCallBackend(data_destroy_function(is_borrowed ? impl->data_borrowed : impl->data_owned)); + if (is_borrowed) VALGRIND_DISCARD(impl->borrowed_block_id); + else VALGRIND_DISCARD(impl->owned_block_id); + } + // Free allocations and discard block ids + if (impl->data_allocated) { + CeedCallBackend(CeedFree(&impl->data_allocated)); + VALGRIND_DISCARD(impl->allocated_block_id); + } + if (impl->data_owned) { + CeedCallBackend(CeedFree(&impl->data_owned)); + VALGRIND_DISCARD(impl->owned_block_id); + } + if (impl->data_borrowed) { + VALGRIND_DISCARD(impl->borrowed_block_id); } - CeedCallBackend(CeedFree(&impl->data_allocated)); return CEED_ERROR_SUCCESS; } @@ -183,9 +270,19 @@ static int CeedQFunctionContextDataDestroy_Memcheck(CeedQFunctionContext ctx) { static int CeedQFunctionContextDestroy_Memcheck(CeedQFunctionContext ctx) { CeedQFunctionContext_Memcheck *impl; + // Free allocations and discard block ids CeedCallBackend(CeedQFunctionContextGetBackendData(ctx, &impl)); - CeedCallBackend(CeedFree(&impl->data_allocated)); - CeedCallBackend(CeedFree(&impl->data_owned)); + if (impl->data_allocated) { + CeedCallBackend(CeedFree(&impl->data_allocated)); + VALGRIND_DISCARD(impl->allocated_block_id); + } + if (impl->data_owned) { + CeedCallBackend(CeedFree(&impl->data_owned)); + VALGRIND_DISCARD(impl->owned_block_id); + } + if (impl->data_borrowed) { + VALGRIND_DISCARD(impl->borrowed_block_id); + } CeedCallBackend(CeedFree(&impl)); return CEED_ERROR_SUCCESS; } diff --git a/backends/memcheck/ceed-memcheck-vector.c b/backends/memcheck/ceed-memcheck-vector.c index 0435fe7e5a..ca86978165 100644 --- a/backends/memcheck/ceed-memcheck-vector.c +++ b/backends/memcheck/ceed-memcheck-vector.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -21,7 +22,7 @@ static int CeedVectorHasValidArray_Memcheck(CeedVector vec, bool *has_valid_arra CeedVector_Memcheck *impl; CeedCallBackend(CeedVectorGetData(vec, &impl)); - *has_valid_array = impl->array; + *has_valid_array = !!impl->array_allocated; return CEED_ERROR_SUCCESS; } @@ -31,9 +32,10 @@ static int CeedVectorHasValidArray_Memcheck(CeedVector vec, bool *has_valid_arra static inline int CeedVectorHasBorrowedArrayOfType_Memcheck(const CeedVector vec, CeedMemType mem_type, bool *has_borrowed_array_of_type) { CeedVector_Memcheck *impl; - CeedCallBackend(CeedVectorGetData(vec, &impl)); CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); - *has_borrowed_array_of_type = impl->array_borrowed; + + CeedCallBackend(CeedVectorGetData(vec, &impl)); + *has_borrowed_array_of_type = !!impl->array_borrowed; return CEED_ERROR_SUCCESS; } @@ -44,45 +46,106 @@ static int CeedVectorSetArray_Memcheck(CeedVector vec, CeedMemType mem_type, Cee CeedSize length; CeedVector_Memcheck *impl; + CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); + CeedCallBackend(CeedVectorGetData(vec, &impl)); CeedCallBackend(CeedVectorGetLength(vec, &length)); - CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); - + // Clear previous owned arrays if (impl->array_allocated) { for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; + VALGRIND_DISCARD(impl->allocated_block_id); } CeedCallBackend(CeedFree(&impl->array_allocated)); if (impl->array_owned) { for (CeedSize i = 0; i < length; i++) impl->array_owned[i] = NAN; + VALGRIND_DISCARD(impl->owned_block_id); } CeedCallBackend(CeedFree(&impl->array_owned)); + + // Clear borrowed block id, if present + if (impl->array_borrowed) VALGRIND_DISCARD(impl->borrowed_block_id); + + // Set internal pointers to external arrays switch (copy_mode) { case CEED_COPY_VALUES: - CeedCallBackend(CeedCalloc(length, &impl->array_owned)); + impl->array_owned = NULL; impl->array_borrowed = NULL; - impl->array = impl->array_owned; - if (array) { - memcpy(impl->array, array, length * sizeof(CeedScalar)); - } else { - for (CeedInt i = 0; i < length; i++) impl->array[i] = NAN; - } break; case CEED_OWN_POINTER: impl->array_owned = array; impl->array_borrowed = NULL; - impl->array = array; + impl->owned_block_id = VALGRIND_CREATE_BLOCK(impl->array_owned, length * sizeof(CeedScalar), "Owned external array buffer"); break; case CEED_USE_POINTER: - impl->array_borrowed = array; - impl->array = array; + impl->array_owned = NULL; + impl->array_borrowed = array; + impl->borrowed_block_id = VALGRIND_CREATE_BLOCK(impl->array_borrowed, length * sizeof(CeedScalar), "Borrowed external array buffer"); + break; } - // Copy data to check access + + // Create internal array data buffer CeedCallBackend(CeedCalloc(length, &impl->array_allocated)); - memcpy(impl->array_allocated, impl->array, length * sizeof(CeedScalar)); - impl->array = impl->array_allocated; - VALGRIND_DISCARD(impl->mem_block_id); - impl->mem_block_id = VALGRIND_CREATE_BLOCK(impl->array, length * sizeof(CeedScalar), "'Vector backend array data copy'"); + impl->allocated_block_id = VALGRIND_CREATE_BLOCK(impl->array_allocated, length * sizeof(CeedScalar), "Allocated internal array buffer"); + if (array) { + memcpy(impl->array_allocated, array, length * sizeof(CeedScalar)); + } else { + for (CeedInt i = 0; i < length; i++) impl->array_allocated[i] = NAN; + } + return CEED_ERROR_SUCCESS; +} + +//------------------------------------------------------------------------------ +// Set internal array to value +//------------------------------------------------------------------------------ +static int CeedVectorSetValue_Memcheck(CeedVector vec, CeedScalar value) { + CeedSize length; + CeedVector_Memcheck *impl; + + CeedCallBackend(CeedVectorGetData(vec, &impl)); + CeedCallBackend(CeedVectorGetLength(vec, &length)); + + if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); + assert(impl->array_allocated); + for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = value; + return CEED_ERROR_SUCCESS; +} + +//------------------------------------------------------------------------------ +// Set internal array to value strided +//------------------------------------------------------------------------------ +static int CeedVectorSetValueStrided_Memcheck(CeedVector vec, CeedSize start, CeedSize step, CeedScalar val) { + CeedSize length; + CeedVector_Memcheck *impl; + + CeedCallBackend(CeedVectorGetData(vec, &impl)); + CeedCallBackend(CeedVectorGetLength(vec, &length)); + + if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); + assert(impl->array_allocated); + for (CeedSize i = start; i < length; i += step) impl->array_allocated[i] = val; + return CEED_ERROR_SUCCESS; +} + +//------------------------------------------------------------------------------ +// Sync arrays +//------------------------------------------------------------------------------ +static int CeedVectorSyncArray_Memcheck(const CeedVector vec, CeedMemType mem_type) { + CeedSize length; + CeedVector_Memcheck *impl; + + CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); + + CeedCallBackend(CeedVectorGetData(vec, &impl)); + CeedCallBackend(CeedVectorGetLength(vec, &length)); + + // Copy internal buffer back to owned or borrowed array + if (impl->array_owned) { + memcpy(impl->array_owned, impl->array_allocated, length * sizeof(CeedScalar)); + } + if (impl->array_borrowed) { + memcpy(impl->array_borrowed, impl->array_allocated, length * sizeof(CeedScalar)); + } return CEED_ERROR_SUCCESS; } @@ -93,17 +156,23 @@ static int CeedVectorTakeArray_Memcheck(CeedVector vec, CeedMemType mem_type, Ce CeedSize length; CeedVector_Memcheck *impl; + CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); + CeedCallBackend(CeedVectorGetData(vec, &impl)); CeedCallBackend(CeedVectorGetLength(vec, &length)); - CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); + // Synchronize memory + CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); + // Return borrowed array (*array) = impl->array_borrowed; impl->array_borrowed = NULL; - impl->array = NULL; - VALGRIND_DISCARD(impl->mem_block_id); + VALGRIND_DISCARD(impl->borrowed_block_id); + + // De-allocate internal memory if (impl->array_allocated) { for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; + VALGRIND_DISCARD(impl->allocated_block_id); } CeedCallBackend(CeedFree(&impl->array_allocated)); return CEED_ERROR_SUCCESS; @@ -116,13 +185,15 @@ static int CeedVectorGetArray_Memcheck(CeedVector vec, CeedMemType mem_type, Cee CeedSize length; CeedVector_Memcheck *impl; + CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); + CeedCallBackend(CeedVectorGetData(vec, &impl)); CeedCallBackend(CeedVectorGetLength(vec, &length)); - CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); - + // Create and return writable buffer CeedCallBackend(CeedCalloc(length, &impl->array_writable_copy)); - memcpy(impl->array_writable_copy, impl->array, length * sizeof(CeedScalar)); + impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_writable_copy, length * sizeof(CeedScalar), "Allocated writeable array buffer copy"); + memcpy(impl->array_writable_copy, impl->array_allocated, length * sizeof(CeedScalar)); *array = impl->array_writable_copy; return CEED_ERROR_SUCCESS; } @@ -134,15 +205,16 @@ static int CeedVectorGetArrayRead_Memcheck(CeedVector vec, CeedMemType mem_type, CeedSize length; CeedVector_Memcheck *impl; + CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); + CeedCallBackend(CeedVectorGetData(vec, &impl)); CeedCallBackend(CeedVectorGetLength(vec, &length)); - CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); - - // Make copy to verify no write occurred + // Create and return read-only buffer if (!impl->array_read_only_copy) { CeedCallBackend(CeedCalloc(length, &impl->array_read_only_copy)); - memcpy(impl->array_read_only_copy, impl->array, length * sizeof(CeedScalar)); + impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_read_only_copy, length * sizeof(CeedScalar), "Allocated read-only array buffer copy"); + memcpy(impl->array_read_only_copy, impl->array_allocated, length * sizeof(CeedScalar)); } *array = impl->array_read_only_copy; return CEED_ERROR_SUCCESS; @@ -155,12 +227,18 @@ static int CeedVectorGetArrayWrite_Memcheck(CeedVector vec, CeedMemType mem_type CeedSize length; CeedVector_Memcheck *impl; + CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); + CeedCallBackend(CeedVectorGetData(vec, &impl)); CeedCallBackend(CeedVectorGetLength(vec, &length)); - // Invalidate data to make sure no read occurs - if (!impl->array) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, mem_type, CEED_COPY_VALUES, NULL)); + // Allocate buffer if necessary + if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, mem_type, CEED_COPY_VALUES, NULL)); + + // Get writable buffer CeedCallBackend(CeedVectorGetArray_Memcheck(vec, mem_type, array)); + + // Invalidate array data to prevent accidental reads for (CeedSize i = 0; i < length; i++) (*array)[i] = NAN; impl->is_write_only_access = true; return CEED_ERROR_SUCCESS; @@ -174,26 +252,27 @@ static int CeedVectorRestoreArray_Memcheck(CeedVector vec) { CeedSize length; CeedVector_Memcheck *impl; + CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); CeedCallBackend(CeedVectorGetData(vec, &impl)); CeedCallBackend(CeedVectorGetLength(vec, &length)); - CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); - memcpy(impl->array, impl->array_writable_copy, length * sizeof(CeedScalar)); - for (CeedSize i = 0; i < length; i++) impl->array_writable_copy[i] = NAN; - CeedCallBackend(CeedFree(&impl->array_writable_copy)); + // Check for unset entries after write-only access if (impl->is_write_only_access) { for (CeedSize i = 0; i < length; i++) { - if (isnan(impl->array[i])) + if (isnan(impl->array_writable_copy[i])) CeedDebug256(ceed, CEED_DEBUG_COLOR_WARNING, "WARNING: Vec entry %" CeedSize_FMT " is NaN after restoring write-only access", i); } impl->is_write_only_access = false; } - if (impl->array_borrowed) { - memcpy(impl->array_borrowed, impl->array, length * sizeof(CeedScalar)); - } - if (impl->array_owned) { - memcpy(impl->array_owned, impl->array, length * sizeof(CeedScalar)); - } + + // Copy back to internal buffer and sync + memcpy(impl->array_allocated, impl->array_writable_copy, length * sizeof(CeedScalar)); + CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); + + // Invalidate writable buffer + for (CeedSize i = 0; i < length; i++) impl->array_writable_copy[i] = NAN; + CeedCallBackend(CeedFree(&impl->array_writable_copy)); + VALGRIND_DISCARD(impl->writable_block_id); return CEED_ERROR_SUCCESS; } @@ -201,17 +280,101 @@ static int CeedVectorRestoreArray_Memcheck(CeedVector vec) { // Vector Restore Array Read-Only //------------------------------------------------------------------------------ static int CeedVectorRestoreArrayRead_Memcheck(CeedVector vec) { + Ceed ceed; CeedSize length; CeedVector_Memcheck *impl; + CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); CeedCallBackend(CeedVectorGetData(vec, &impl)); CeedCallBackend(CeedVectorGetLength(vec, &length)); - CeedCheck(!memcmp(impl->array, impl->array_read_only_copy, length * sizeof(CeedScalar)), CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, - "Array data changed while accessed in read-only mode"); + // Verify no changes made during read-only access + bool is_changed = memcmp(impl->array_allocated, impl->array_read_only_copy, length * sizeof(CeedScalar)); + CeedCheck(!is_changed, ceed, CEED_ERROR_BACKEND, "Array data changed while accessed in read-only mode"); + + // Invalidate read-only buffer for (CeedSize i = 0; i < length; i++) impl->array_read_only_copy[i] = NAN; CeedCallBackend(CeedFree(&impl->array_read_only_copy)); + VALGRIND_DISCARD(impl->read_only_block_id); + return CEED_ERROR_SUCCESS; +} + +//------------------------------------------------------------------------------ +// Take reciprocal of a vector +//------------------------------------------------------------------------------ +static int CeedVectorReciprocal_Memcheck(CeedVector vec) { + CeedSize length; + CeedVector_Memcheck *impl; + + CeedCallBackend(CeedVectorGetData(vec, &impl)); + CeedCallBackend(CeedVectorGetLength(vec, &length)); + + for (CeedSize i = 0; i < length; i++) { + if (fabs(impl->array_allocated[i]) > CEED_EPSILON) impl->array_allocated[i] = 1. / impl->array_allocated[i]; + } + return CEED_ERROR_SUCCESS; +} + +//------------------------------------------------------------------------------ +// Compute x = alpha x +//------------------------------------------------------------------------------ +static int CeedVectorScale_Memcheck(CeedVector x, CeedScalar alpha) { + CeedSize length; + CeedVector_Memcheck *impl; + + CeedCallBackend(CeedVectorGetData(x, &impl)); + CeedCallBackend(CeedVectorGetLength(x, &length)); + + for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] *= alpha; + return CEED_ERROR_SUCCESS; +} + +//------------------------------------------------------------------------------ +// Compute y = alpha x + y +//------------------------------------------------------------------------------ +static int CeedVectorAXPY_Memcheck(CeedVector y, CeedScalar alpha, CeedVector x) { + CeedSize length; + CeedVector_Memcheck *impl_x, *impl_y; + + CeedCallBackend(CeedVectorGetData(x, &impl_x)); + CeedCallBackend(CeedVectorGetData(y, &impl_y)); + CeedCallBackend(CeedVectorGetLength(y, &length)); + + for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] += alpha * impl_x->array_allocated[i]; + return CEED_ERROR_SUCCESS; +} + +//------------------------------------------------------------------------------ +// Compute y = alpha x + beta y +//------------------------------------------------------------------------------ +static int CeedVectorAXPBY_Memcheck(CeedVector y, CeedScalar alpha, CeedScalar beta, CeedVector x) { + CeedSize length; + CeedVector_Memcheck *impl_x, *impl_y; + + CeedCallBackend(CeedVectorGetData(x, &impl_x)); + CeedCallBackend(CeedVectorGetData(y, &impl_y)); + CeedCallBackend(CeedVectorGetLength(y, &length)); + + for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] = alpha * impl_x->array_allocated[i] + beta * impl_y->array_allocated[i]; + return CEED_ERROR_SUCCESS; +} + +//------------------------------------------------------------------------------ +// Compute the pointwise multiplication w = x .* y +//------------------------------------------------------------------------------ +static int CeedVectorPointwiseMult_Memcheck(CeedVector w, CeedVector x, CeedVector y) { + CeedSize length; + CeedVector_Memcheck *impl_x, *impl_y, *impl_w; + + CeedCallBackend(CeedVectorGetData(x, &impl_x)); + CeedCallBackend(CeedVectorGetData(y, &impl_y)); + CeedCallBackend(CeedVectorGetData(w, &impl_w)); + CeedCallBackend(CeedVectorGetLength(w, &length)); + + if (!impl_w->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(w, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); + assert(impl_w->array_allocated); + for (CeedSize i = 0; i < length; i++) impl_w->array_allocated[i] = impl_x->array_allocated[i] * impl_y->array_allocated[i]; return CEED_ERROR_SUCCESS; } @@ -221,10 +384,19 @@ static int CeedVectorRestoreArrayRead_Memcheck(CeedVector vec) { static int CeedVectorDestroy_Memcheck(CeedVector vec) { CeedVector_Memcheck *impl; + // Free allocations and discard block ids CeedCallBackend(CeedVectorGetData(vec, &impl)); - VALGRIND_DISCARD(impl->mem_block_id); - CeedCallBackend(CeedFree(&impl->array_allocated)); - CeedCallBackend(CeedFree(&impl->array_owned)); + if (impl->array_allocated) { + CeedCallBackend(CeedFree(&impl->array_allocated)); + VALGRIND_DISCARD(impl->allocated_block_id); + } + if (impl->array_owned) { + CeedCallBackend(CeedFree(&impl->array_owned)); + VALGRIND_DISCARD(impl->owned_block_id); + } + if (impl->array_borrowed) { + VALGRIND_DISCARD(impl->borrowed_block_id); + } CeedCallBackend(CeedFree(&impl)); return CEED_ERROR_SUCCESS; } @@ -243,12 +415,20 @@ int CeedVectorCreate_Memcheck(CeedSize n, CeedVector vec) { CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasValidArray", CeedVectorHasValidArray_Memcheck)); CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasBorrowedArrayOfType", CeedVectorHasBorrowedArrayOfType_Memcheck)); CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetArray", CeedVectorSetArray_Memcheck)); + CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValue", CeedVectorSetValue_Memcheck)); + CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValueStrided", CeedVectorSetValueStrided_Memcheck)); + CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SyncArray", CeedVectorSyncArray_Memcheck)); CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "TakeArray", CeedVectorTakeArray_Memcheck)); CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArray", CeedVectorGetArray_Memcheck)); CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayRead", CeedVectorGetArrayRead_Memcheck)); CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayWrite", CeedVectorGetArrayWrite_Memcheck)); CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArray", CeedVectorRestoreArray_Memcheck)); CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArrayRead", CeedVectorRestoreArrayRead_Memcheck)); + CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Reciprocal", CeedVectorReciprocal_Memcheck)); + CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Scale", CeedVectorScale_Memcheck)); + CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPY", CeedVectorAXPY_Memcheck)); + CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPBY", CeedVectorAXPBY_Memcheck)); + CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "PointwiseMult", CeedVectorPointwiseMult_Memcheck)); CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Destroy", CeedVectorDestroy_Memcheck)); return CEED_ERROR_SUCCESS; } diff --git a/backends/memcheck/ceed-memcheck.h b/backends/memcheck/ceed-memcheck.h index f4787cf1ce..45eb9c5ae3 100644 --- a/backends/memcheck/ceed-memcheck.h +++ b/backends/memcheck/ceed-memcheck.h @@ -10,13 +10,21 @@ #include typedef struct { - int mem_block_id; - bool is_write_only_access; - CeedScalar *array; + // Internal array buffer + int allocated_block_id; CeedScalar *array_allocated; + // Owned external array + int owned_block_id; CeedScalar *array_owned; + // Borrowed external array + int borrowed_block_id; CeedScalar *array_borrowed; + // Externally viewable read-only array + int read_only_block_id; CeedScalar *array_read_only_copy; + // Externally viewable writable array + bool is_write_only_access; + int writable_block_id; CeedScalar *array_writable_copy; } CeedVector_Memcheck; @@ -32,18 +40,27 @@ typedef struct { } CeedElemRestriction_Memcheck; typedef struct { + bool setup_done; const CeedScalar **inputs; CeedScalar **outputs; - bool setup_done; } CeedQFunction_Memcheck; typedef struct { - int mem_block_id; - void *data; + // Internal data buffer + int allocated_block_id; void *data_allocated; + // Owned external data + int owned_block_id; void *data_owned; + // Borrowed external data + int borrowed_block_id; void *data_borrowed; + // Externally viewable read-only data + int read_only_block_id; void *data_read_only_copy; + // Externally viewable writable data + int writable_block_id; + void *data_writable_copy; } CeedQFunctionContext_Memcheck; CEED_INTERN int CeedVectorCreate_Memcheck(CeedSize n, CeedVector vec);