diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7078d629f..50ce0f3f4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -46,6 +46,7 @@ set(UMF_SOURCES ${BA_SOURCES} libumf.c ipc.c + ipc_cache.c memory_pool.c memory_provider.c memory_provider_get_last_failed.c @@ -189,6 +190,7 @@ target_include_directories( $ $ $ + $ $) install(TARGETS umf EXPORT ${PROJECT_NAME}-targets) diff --git a/src/ipc.c b/src/ipc.c index f7497b340..45e098619 100644 --- a/src/ipc.c +++ b/src/ipc.c @@ -90,6 +90,8 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, return ret; } + // ipcData->handle_id is filled by tracking provider + ipcData->base = allocInfo.base; ipcData->pid = utils_getpid(); ipcData->baseSize = allocInfo.baseSize; ipcData->offset = (uintptr_t)ptr - (uintptr_t)allocInfo.base; diff --git a/src/ipc_cache.c b/src/ipc_cache.c new file mode 100644 index 000000000..bff251b2e --- /dev/null +++ b/src/ipc_cache.c @@ -0,0 +1,237 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include + +#include "base_alloc_global.h" +#include "ipc_cache.h" +#include "uthash.h" +#include "utils_common.h" +#include "utils_concurrency.h" +#include "utils_log.h" +#include "utlist.h" + +struct ipc_handle_cache_entry_t; + +typedef struct ipc_handle_cache_entry_t *hash_map_t; +typedef struct ipc_handle_cache_entry_t *lru_list_t; + +typedef struct ipc_handle_cache_entry_t { + UT_hash_handle hh; + struct ipc_handle_cache_entry_t *next, *prev; + ipc_mapped_handle_cache_key_t key; + uint64_t ref_count; + uint64_t handle_id; + hash_map_t + *hash_table; // pointer to the hash table to which the entry belongs + ipc_mapped_handle_cache_value_t value; +} ipc_handle_cache_entry_t; + +typedef struct ipc_mapped_handle_cache_global_t { + utils_mutex_t cache_lock; + umf_ba_pool_t *cache_allocator; + size_t max_size; + size_t cur_size; + lru_list_t lru_list; +} ipc_mapped_handle_cache_global_t; + +typedef struct ipc_mapped_handle_cache_t { + ipc_mapped_handle_cache_global_t *global; + hash_map_t hash_table; + ipc_mapped_handle_cache_eviction_cb_t eviction_cb; +} ipc_mapped_handle_cache_t; + +ipc_mapped_handle_cache_global_t *IPC_MAPPED_CACHE_GLOBAL = NULL; + +umf_result_t umfIpcCacheGlobalInit(void) { + umf_result_t ret = UMF_RESULT_SUCCESS; + ipc_mapped_handle_cache_global_t *cache_global = + umf_ba_global_alloc(sizeof(*cache_global)); + if (!cache_global) { + LOG_ERR("Failed to allocate memory for the IPC cache global data"); + ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + goto err_exit; + } + + if (NULL == utils_mutex_init(&(cache_global->cache_lock))) { + LOG_ERR("Failed to initialize mutex for the IPC global cache"); + ret = UMF_RESULT_ERROR_UNKNOWN; + goto err_cache_global_free; + } + + cache_global->cache_allocator = + umf_ba_create(sizeof(ipc_handle_cache_entry_t)); + if (!cache_global->cache_allocator) { + LOG_ERR("Failed to create IPC cache allocator"); + ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + goto err_mutex_destroy; + } + + // TODO: make max_size configurable via environment variable + cache_global->max_size = 0; + cache_global->cur_size = 0; + cache_global->lru_list = NULL; + + IPC_MAPPED_CACHE_GLOBAL = cache_global; + goto err_exit; + +err_mutex_destroy: + utils_mutex_destroy_not_free(&(cache_global->cache_lock)); +err_cache_global_free: + umf_ba_global_free(cache_global); +err_exit: + return ret; +} + +static size_t getGlobalLruListSize(lru_list_t lru_list) { + size_t size = 0; + ipc_handle_cache_entry_t *tmp; + DL_COUNT(lru_list, tmp, size); + return size; +} + +void umfIpcCacheGlobalTearDown(void) { + ipc_mapped_handle_cache_global_t *cache_global = IPC_MAPPED_CACHE_GLOBAL; + IPC_MAPPED_CACHE_GLOBAL = NULL; + + if (!cache_global) { + return; + } + + assert(cache_global->cur_size == 0); + assert(getGlobalLruListSize(cache_global->lru_list) == 0); + + umf_ba_destroy(cache_global->cache_allocator); + utils_mutex_destroy_not_free(&(cache_global->cache_lock)); + umf_ba_global_free(cache_global); +} + +ipc_mapped_handle_cache_handle_t umfIpcHandleMappedCacheCreate( + ipc_mapped_handle_cache_eviction_cb_t eviction_cb) { + if (eviction_cb == NULL) { + LOG_ERR("Eviction callback is NULL"); + return NULL; + } + + ipc_mapped_handle_cache_t *cache = umf_ba_global_alloc(sizeof(*cache)); + + if (!cache) { + LOG_ERR("Failed to allocate memory for the IPC cache"); + return NULL; + } + + assert(IPC_MAPPED_CACHE_GLOBAL != NULL); + + cache->global = IPC_MAPPED_CACHE_GLOBAL; + cache->hash_table = NULL; + cache->eviction_cb = eviction_cb; + + return cache; +} + +void umfIpcHandleMappedCacheDestroy(ipc_mapped_handle_cache_handle_t cache) { + ipc_handle_cache_entry_t *entry, *tmp; + HASH_ITER(hh, cache->hash_table, entry, tmp) { + DL_DELETE(cache->global->lru_list, entry); + HASH_DEL(cache->hash_table, entry); + cache->global->cur_size -= 1; + cache->eviction_cb(&entry->key, &entry->value); + utils_mutex_destroy_not_free(&(entry->value.mmap_lock)); + umf_ba_free(cache->global->cache_allocator, entry); + } + HASH_CLEAR(hh, cache->hash_table); + + umf_ba_global_free(cache); +} + +umf_result_t +umfIpcHandleMappedCacheGet(ipc_mapped_handle_cache_handle_t cache, + const ipc_mapped_handle_cache_key_t *key, + uint64_t handle_id, + ipc_mapped_handle_cache_value_t **retEntry) { + ipc_handle_cache_entry_t *entry = NULL; + umf_result_t ret = UMF_RESULT_SUCCESS; + bool evicted = false; + ipc_mapped_handle_cache_value_t evicted_value; + + if (!cache || !key || !retEntry) { + LOG_ERR("Some arguments are NULL, cache=%p, key=%p, retEntry=%p", + (void *)cache, (const void *)key, (void *)retEntry); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + assert(cache->global != NULL); + + utils_mutex_lock(&(cache->global->cache_lock)); + + HASH_FIND(hh, cache->hash_table, key, sizeof(*key), entry); + if (entry && entry->handle_id == handle_id) { // cache hit + // update frequency list + // remove the entry from the current position + DL_DELETE(cache->global->lru_list, entry); + // add the entry to the head of the list + DL_PREPEND(cache->global->lru_list, entry); + } else { //cache miss + // Look for eviction candidate + if (entry == NULL && cache->global->max_size != 0 && + cache->global->cur_size >= cache->global->max_size) { + // If max_size is set and the cache is full, evict the least recently used entry. + entry = cache->global->lru_list->prev; + } + + if (entry) { // we have eviction candidate + // remove the entry from the frequency list + DL_DELETE(cache->global->lru_list, entry); + // remove the entry from the hash table it belongs to + HASH_DEL(*(entry->hash_table), entry); + cache->global->cur_size -= 1; + evicted_value.mapped_base_ptr = entry->value.mapped_base_ptr; + evicted_value.mapped_size = entry->value.mapped_size; + evicted = true; + } else { // allocate the new entry + entry = umf_ba_alloc(cache->global->cache_allocator); + if (!entry) { + ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + LOG_ERR("Failed to allocate memory for a new IPC cache entry"); + goto exit; + } + if (NULL == utils_mutex_init(&(entry->value.mmap_lock))) { + LOG_ERR("Failed to initialize mutex for the IPC cache entry"); + umf_ba_global_free(entry); + ret = UMF_RESULT_ERROR_UNKNOWN; + goto exit; + } + } + + entry->key = *key; + entry->ref_count = 0; + entry->handle_id = handle_id; + entry->hash_table = &cache->hash_table; + entry->value.mapped_size = 0; + entry->value.mapped_base_ptr = NULL; + + HASH_ADD(hh, cache->hash_table, key, sizeof(entry->key), entry); + DL_PREPEND(cache->global->lru_list, entry); + cache->global->cur_size += 1; + } + +exit: + if (ret == UMF_RESULT_SUCCESS) { + utils_atomic_increment(&entry->ref_count); + *retEntry = &entry->value; + } + + utils_mutex_unlock(&(cache->global->cache_lock)); + + if (evicted) { + cache->eviction_cb(key, &evicted_value); + } + + return ret; +} diff --git a/src/ipc_cache.h b/src/ipc_cache.h new file mode 100644 index 000000000..59ae28787 --- /dev/null +++ b/src/ipc_cache.h @@ -0,0 +1,52 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_IPC_CACHE_H +#define UMF_IPC_CACHE_H 1 + +#include + +#include "utils_concurrency.h" + +typedef struct ipc_mapped_handle_cache_key_t { + void *remote_base_ptr; + umf_memory_provider_handle_t local_provider; + int remote_pid; +} ipc_mapped_handle_cache_key_t; + +typedef struct ipc_mapped_handle_cache_value_t { + void *mapped_base_ptr; + size_t mapped_size; + utils_mutex_t mmap_lock; +} ipc_mapped_handle_cache_value_t; + +struct ipc_mapped_handle_cache_t; + +typedef struct ipc_mapped_handle_cache_t *ipc_mapped_handle_cache_handle_t; + +umf_result_t umfIpcCacheGlobalInit(void); +void umfIpcCacheGlobalTearDown(void); + +// define pointer to the eviction callback function +typedef void (*ipc_mapped_handle_cache_eviction_cb_t)( + const ipc_mapped_handle_cache_key_t *key, + const ipc_mapped_handle_cache_value_t *value); + +ipc_mapped_handle_cache_handle_t umfIpcHandleMappedCacheCreate( + ipc_mapped_handle_cache_eviction_cb_t eviction_cb); + +void umfIpcHandleMappedCacheDestroy(ipc_mapped_handle_cache_handle_t cache); + +umf_result_t +umfIpcHandleMappedCacheGet(ipc_mapped_handle_cache_handle_t cache, + const ipc_mapped_handle_cache_key_t *key, + uint64_t handle_id, + ipc_mapped_handle_cache_value_t **retEntry); + +#endif /* UMF_IPC_CACHE_H */ diff --git a/src/ipc_internal.h b/src/ipc_internal.h index 0f45b24e2..103214407 100644 --- a/src/ipc_internal.h +++ b/src/ipc_internal.h @@ -21,8 +21,10 @@ extern "C" { // providerIpcData is a Flexible Array Member because its size varies // depending on the provider. typedef struct umf_ipc_data_t { - int pid; // process ID of the process that allocated the memory - size_t baseSize; // size of base (coarse-grain) allocation + uint64_t handle_id; // unique ID of this handle + void *base; // base address of the memory + int pid; // process ID of the process that allocated the memory + size_t baseSize; // size of base (coarse-grain) allocation uint64_t offset; char providerIpcData[]; } umf_ipc_data_t; diff --git a/src/libumf.c b/src/libumf.c index 2fcda07a0..3b69ce396 100644 --- a/src/libumf.c +++ b/src/libumf.c @@ -10,6 +10,7 @@ #include #include "base_alloc_global.h" +#include "ipc_cache.h" #include "memspace_internal.h" #include "provider_tracking.h" #include "utils_log.h" @@ -25,9 +26,18 @@ int umfInit(void) { if (utils_fetch_and_add64(&umfRefCount, 1) == 0) { utils_log_init(); TRACKER = umfMemoryTrackerCreate(); + if (!TRACKER) { + LOG_ERR("Failed to create memory tracker"); + return -1; + } + umf_result_t umf_result = umfIpcCacheGlobalInit(); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR("Failed to initialize IPC cache"); + return -1; + } } - return (TRACKER) ? 0 : -1; + return 0; } void umfTearDown(void) { @@ -39,6 +49,7 @@ void umfTearDown(void) { umfMemspaceLowestLatencyDestroy(); umfDestroyTopology(); #endif + umfIpcCacheGlobalTearDown(); // make sure TRACKER is not used after being destroyed umf_memory_tracker_handle_t t = TRACKER; TRACKER = NULL; diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index d058af271..769ed6a94 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -10,6 +10,7 @@ #include "provider_tracking.h" #include "base_alloc_global.h" #include "critnib.h" +#include "ipc_cache.h" #include "ipc_internal.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -25,6 +26,8 @@ #include #include +uint64_t IPC_HANDLE_ID = 0; + typedef struct tracker_value_t { umf_memory_pool_handle_t pool; size_t size; @@ -138,6 +141,7 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, // providerIpcData is a Flexible Array Member because its size varies // depending on the provider. typedef struct ipc_cache_value_t { + uint64_t handle_id; uint64_t ipcDataSize; char providerIpcData[]; } ipc_cache_value_t; @@ -147,6 +151,7 @@ typedef struct umf_tracking_memory_provider_t { umf_memory_tracker_handle_t hTracker; umf_memory_pool_handle_t pool; critnib *ipcCache; + ipc_mapped_handle_cache_handle_t hIpcMappedCache; // the upstream provider does not support the free() operation bool upstreamDoesNotFree; @@ -465,6 +470,8 @@ static void trackingFinalize(void *provider) { umf_tracking_memory_provider_t *p = (umf_tracking_memory_provider_t *)provider; + umfIpcHandleMappedCacheDestroy(p->hIpcMappedCache); + critnib_delete(p->ipcCache); // Do not clear the tracker if we are running in the proxy library, @@ -524,6 +531,11 @@ static umf_result_t trackingGetIpcHandleSize(void *provider, size_t *size) { return umfMemoryProviderGetIPCHandleSize(p->hUpstream, size); } +static inline umf_ipc_data_t *getIpcDataFromIpcHandle(void *providerIpcData) { + return (umf_ipc_data_t *)((uint8_t *)providerIpcData - + sizeof(umf_ipc_data_t)); +} + static umf_result_t trackingGetIpcHandle(void *provider, const void *ptr, size_t size, void *providerIpcData) { umf_tracking_memory_provider_t *p = @@ -531,47 +543,39 @@ static umf_result_t trackingGetIpcHandle(void *provider, const void *ptr, umf_result_t ret = UMF_RESULT_SUCCESS; size_t ipcDataSize = 0; int cached = 0; + ipc_cache_value_t *cache_value = NULL; + umf_ipc_data_t *ipcUmfData = getIpcDataFromIpcHandle(providerIpcData); + do { void *value = critnib_get(p->ipcCache, (uintptr_t)ptr); if (value) { //cache hit - ipc_cache_value_t *cache_value = (ipc_cache_value_t *)value; - memcpy(providerIpcData, cache_value->providerIpcData, - cache_value->ipcDataSize); + cache_value = (ipc_cache_value_t *)value; cached = 1; - } else { - ret = umfMemoryProviderGetIPCHandle(p->hUpstream, ptr, size, - providerIpcData); - if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("upstream provider failed to get IPC handle"); - return ret; - } - + } else { //cache miss ret = umfMemoryProviderGetIPCHandleSize(p->hUpstream, &ipcDataSize); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("upstream provider failed to get the size of IPC " "handle"); - ret = umfMemoryProviderPutIPCHandle(p->hUpstream, - providerIpcData); - if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("upstream provider failed to put IPC handle"); - } return ret; } size_t value_size = sizeof(ipc_cache_value_t) + ipcDataSize; - ipc_cache_value_t *cache_value = umf_ba_global_alloc(value_size); + cache_value = umf_ba_global_alloc(value_size); if (!cache_value) { LOG_ERR("failed to allocate cache_value"); - ret = umfMemoryProviderPutIPCHandle(p->hUpstream, - providerIpcData); - if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("upstream provider failed to put IPC handle"); - } return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } + ret = umfMemoryProviderGetIPCHandle(p->hUpstream, ptr, size, + cache_value->providerIpcData); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("upstream provider failed to get IPC handle"); + umf_ba_global_free(cache_value); + return ret; + } + + cache_value->handle_id = utils_atomic_increment(&IPC_HANDLE_ID); cache_value->ipcDataSize = ipcDataSize; - memcpy(cache_value->providerIpcData, providerIpcData, ipcDataSize); int insRes = critnib_insert(p->ipcCache, (uintptr_t)ptr, (void *)cache_value, 0 /*update*/); @@ -587,9 +591,9 @@ static umf_result_t trackingGetIpcHandle(void *provider, const void *ptr, // But this case should be rare enough. // 2. critnib failed to allocate memory internally. We need // to cleanup and return corresponding error. + ret = umfMemoryProviderPutIPCHandle( + p->hUpstream, cache_value->providerIpcData); umf_ba_global_free(cache_value); - ret = umfMemoryProviderPutIPCHandle(p->hUpstream, - providerIpcData); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("upstream provider failed to put IPC handle"); return ret; @@ -602,6 +606,10 @@ static umf_result_t trackingGetIpcHandle(void *provider, const void *ptr, } } while (!cached); + memcpy(providerIpcData, cache_value->providerIpcData, + cache_value->ipcDataSize); + ipcUmfData->handle_id = cache_value->handle_id; + return ret; } @@ -614,17 +622,65 @@ static umf_result_t trackingPutIpcHandle(void *provider, return UMF_RESULT_SUCCESS; } -static size_t getDataSizeFromIpcHandle(const void *providerIpcData) { - // This is hack to get size of memory pointed by IPC handle. - // tracking memory provider gets only provider-specific data - // pointed by providerIpcData, but the size of allocation tracked - // by umf_ipc_data_t. We use this trick to get pointer to - // umf_ipc_data_t data because the providerIpcData is - // the Flexible Array Member of umf_ipc_data_t. - const umf_ipc_data_t *ipcUmfData = - (const umf_ipc_data_t *)((const uint8_t *)providerIpcData - - sizeof(umf_ipc_data_t)); - return ipcUmfData->baseSize; +static void +ipcMappedCacheEvictionCallback(const ipc_mapped_handle_cache_key_t *key, + const ipc_mapped_handle_cache_value_t *value) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)key->local_provider; + // umfMemoryTrackerRemove should be called before umfMemoryProviderCloseIPCHandle + // to avoid a race condition. If the order would be different, other thread + // could allocate the memory at address `ptr` before a call to umfMemoryTrackerRemove + // resulting in inconsistent state. + if (value->mapped_base_ptr) { + umf_result_t ret = + umfMemoryTrackerRemove(p->hTracker, value->mapped_base_ptr); + if (ret != UMF_RESULT_SUCCESS) { + // DO NOT return an error here, because the tracking provider + // cannot change behaviour of the upstream provider. + LOG_ERR("failed to remove the region from the tracker, ptr=%p, " + "size=%zu, ret = %d", + value->mapped_base_ptr, value->mapped_size, ret); + } + } + umf_result_t ret = umfMemoryProviderCloseIPCHandle( + p->hUpstream, value->mapped_base_ptr, value->mapped_size); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("provider failed to close IPC handle, ptr=%p, size=%zu", + value->mapped_base_ptr, value->mapped_size); + } +} + +static umf_result_t upstreamOpenIPCHandle(umf_tracking_memory_provider_t *p, + void *providerIpcData, + size_t bufferSize, void **ptr) { + void *mapped_ptr = NULL; + assert(p != NULL); + assert(ptr != NULL); + umf_result_t ret = umfMemoryProviderOpenIPCHandle( + p->hUpstream, providerIpcData, &mapped_ptr); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("upstream provider failed to open IPC handle"); + return ret; + } + assert(mapped_ptr != NULL); + + ret = umfMemoryTrackerAdd(p->hTracker, p->pool, mapped_ptr, bufferSize); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("failed to add IPC region to the tracker, ptr=%p, " + "size=%zu, " + "ret = %d", + mapped_ptr, bufferSize, ret); + if (umfMemoryProviderCloseIPCHandle(p->hUpstream, mapped_ptr, + bufferSize)) { + LOG_ERR("upstream provider failed to close IPC handle, " + "ptr=%p, size=%zu", + mapped_ptr, bufferSize); + } + return ret; + } + + *ptr = mapped_ptr; + return UMF_RESULT_SUCCESS; } static umf_result_t trackingOpenIpcHandle(void *provider, void *providerIpcData, @@ -634,47 +690,67 @@ static umf_result_t trackingOpenIpcHandle(void *provider, void *providerIpcData, umf_result_t ret = UMF_RESULT_SUCCESS; assert(p->hUpstream); + assert(p->hIpcMappedCache); + + umf_ipc_data_t *ipcUmfData = getIpcDataFromIpcHandle(providerIpcData); + + // Compiler may add paddings to the ipc_mapped_handle_cache_key_t structure + // so we need to zero it out to avoid false cache miss. + ipc_mapped_handle_cache_key_t key = {0}; + key.remote_base_ptr = ipcUmfData->base; + key.local_provider = provider; + key.remote_pid = ipcUmfData->pid; - ret = umfMemoryProviderOpenIPCHandle(p->hUpstream, providerIpcData, ptr); + ipc_mapped_handle_cache_value_t *cache_entry = NULL; + ret = umfIpcHandleMappedCacheGet(p->hIpcMappedCache, &key, + ipcUmfData->handle_id, &cache_entry); if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("upstream provider failed to open IPC handle"); + LOG_ERR("failed to get cache entry"); return ret; } - size_t bufferSize = getDataSizeFromIpcHandle(providerIpcData); - ret = umfMemoryTrackerAdd(p->hTracker, p->pool, *ptr, bufferSize); - if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("failed to add IPC region to the tracker, ptr=%p, size=%zu, " - "ret = %d", - *ptr, bufferSize, ret); - if (umfMemoryProviderCloseIPCHandle(p->hUpstream, *ptr, bufferSize)) { - LOG_ERR("upstream provider failed to close IPC handle, ptr=%p, " - "size=%zu", - *ptr, bufferSize); + + assert(cache_entry != NULL); + + void *mapped_ptr = NULL; + utils_atomic_load_acquire(&(cache_entry->mapped_base_ptr), &mapped_ptr); + if (mapped_ptr == NULL) { + utils_mutex_lock(&(cache_entry->mmap_lock)); + utils_atomic_load_acquire(&(cache_entry->mapped_base_ptr), &mapped_ptr); + if (mapped_ptr == NULL) { + ret = upstreamOpenIPCHandle(p, providerIpcData, + ipcUmfData->baseSize, &mapped_ptr); + if (ret == UMF_RESULT_SUCCESS) { + // Put to the cache + cache_entry->mapped_size = ipcUmfData->baseSize; + utils_atomic_store_release(&(cache_entry->mapped_base_ptr), + mapped_ptr); + } } + utils_mutex_unlock(&(cache_entry->mmap_lock)); + } + + if (ret == UMF_RESULT_SUCCESS) { + *ptr = mapped_ptr; } + return ret; } static umf_result_t trackingCloseIpcHandle(void *provider, void *ptr, size_t size) { - umf_tracking_memory_provider_t *p = - (umf_tracking_memory_provider_t *)provider; - - // umfMemoryTrackerRemove should be called before umfMemoryProviderCloseIPCHandle - // to avoid a race condition. If the order would be different, other thread - // could allocate the memory at address `ptr` before a call to umfMemoryTrackerRemove - // resulting in inconsistent state. - if (ptr) { - umf_result_t ret = umfMemoryTrackerRemove(p->hTracker, ptr); - if (ret != UMF_RESULT_SUCCESS) { - // DO NOT return an error here, because the tracking provider - // cannot change behaviour of the upstream provider. - LOG_ERR("failed to remove the region from the tracker, ptr=%p, " - "size=%zu, ret = %d", - ptr, size, ret); - } - } - return umfMemoryProviderCloseIPCHandle(p->hUpstream, ptr, size); + (void)provider; + (void)ptr; + (void)size; + // We keep opened IPC handles in the p->hIpcMappedCache. + // IPC handle is closed when it is evicted from the cache + // or when cache is destroyed. + // + // TODO: today the size of the IPC cache is infinite. + // When the threshold for the cache size is implemented + // we need to introduce a reference counting mechanism. + // The trackingOpenIpcHandle will increment the refcount for the corresponding entry. + // The trackingCloseIpcHandle will decrement the refcount for the corresponding cache entry. + return UMF_RESULT_SUCCESS; } umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { @@ -716,10 +792,14 @@ umf_result_t umfTrackingMemoryProviderCreate( return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } + params.hIpcMappedCache = + umfIpcHandleMappedCacheCreate(ipcMappedCacheEvictionCallback); + LOG_DEBUG("upstream=%p, tracker=%p, " - "pool=%p, ipcCache=%p", + "pool=%p, ipcCache=%p, hIpcMappedCache=%p", (void *)params.hUpstream, (void *)params.hTracker, - (void *)params.pool, (void *)params.ipcCache); + (void *)params.pool, (void *)params.ipcCache, + (void *)params.hIpcMappedCache); return umfMemoryProviderCreate(&UMF_TRACKING_MEMORY_PROVIDER_OPS, ¶ms, hTrackingProvider); diff --git a/src/uthash/uthash.h b/src/uthash/uthash.h new file mode 100644 index 000000000..6058e638e --- /dev/null +++ b/src/uthash/uthash.h @@ -0,0 +1,1261 @@ +/* +Copyright (C) 2024 Intel Corporation +Copyright (c) 2003-2022, Troy D. Hanson https://troydhanson.github.io/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* +Modifications by Intel: +- define uthash_malloc unconditional as a umf_ba_global_alloc +- define uthash_free unconditional as a umf_ba_global_free +*/ + +#ifndef UTHASH_H +#define UTHASH_H + +#define UTHASH_VERSION 2.3.0 + +#include /* ptrdiff_t */ +#include /* exit */ +#include /* memcmp, memset, strlen */ + +#include "base_alloc_global.h" + +#if defined(HASH_DEFINE_OWN_STDINT) && HASH_DEFINE_OWN_STDINT +/* This codepath is provided for backward compatibility, but I plan to remove it. */ +#warning \ + "HASH_DEFINE_OWN_STDINT is deprecated; please use HASH_NO_STDINT instead" +typedef unsigned int uint32_t; +typedef unsigned char uint8_t; +#elif defined(HASH_NO_STDINT) && HASH_NO_STDINT +#else +#include /* uint8_t, uint32_t */ +#endif + +/* These macros use decltype or the earlier __typeof GNU extension. + As decltype is only available in newer compilers (VS2010 or gcc 4.3+ + when compiling c++ source) this code uses whatever method is needed + or, for VS2008 where neither is available, uses casting workarounds. */ +#if !defined(DECLTYPE) && !defined(NO_DECLTYPE) +#if defined(_MSC_VER) /* MS compiler */ +#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ +#define DECLTYPE(x) (decltype(x)) +#else /* VS2008 or older (or VS2010 in C mode) */ +#define NO_DECLTYPE +#endif +#elif defined(__MCST__) /* Elbrus C Compiler */ +#define DECLTYPE(x) (__typeof(x)) +#elif defined(__BORLANDC__) || defined(__ICCARM__) || defined(__LCC__) || \ + defined(__WATCOMC__) +#define NO_DECLTYPE +#else /* GNU, Sun and other compilers */ +#define DECLTYPE(x) (__typeof(x)) +#endif +#endif + +#ifdef NO_DECLTYPE +#define DECLTYPE(x) +#define DECLTYPE_ASSIGN(dst, src) \ + do { \ + char **_da_dst = (char **)(&(dst)); \ + *_da_dst = (char *)(src); \ + } while (0) +#else +#define DECLTYPE_ASSIGN(dst, src) \ + do { \ + (dst) = DECLTYPE(dst)(src); \ + } while (0) +#endif + +#define uthash_malloc(sz) umf_ba_global_alloc(sz) /* malloc fcn */ +#define uthash_free(ptr, sz) umf_ba_global_free(ptr) /* free fcn */ + +#ifndef uthash_bzero +#define uthash_bzero(a, n) memset(a, '\0', n) +#endif +#ifndef uthash_strlen +#define uthash_strlen(s) strlen(s) +#endif + +#ifndef HASH_FUNCTION +#define HASH_FUNCTION(keyptr, keylen, hashv) HASH_JEN(keyptr, keylen, hashv) +#endif + +#ifndef HASH_KEYCMP +#define HASH_KEYCMP(a, b, n) memcmp(a, b, n) +#endif + +#ifndef uthash_noexpand_fyi +#define uthash_noexpand_fyi(tbl) /* can be defined to log noexpand */ +#endif +#ifndef uthash_expand_fyi +#define uthash_expand_fyi(tbl) /* can be defined to log expands */ +#endif + +#ifndef HASH_NONFATAL_OOM +#define HASH_NONFATAL_OOM 0 +#endif + +#if HASH_NONFATAL_OOM +/* malloc failures can be recovered from */ + +#ifndef uthash_nonfatal_oom +#define uthash_nonfatal_oom(obj) \ + do { \ + } while (0) /* non-fatal OOM error */ +#endif + +#define HASH_RECORD_OOM(oomed) \ + do { \ + (oomed) = 1; \ + } while (0) +#define IF_HASH_NONFATAL_OOM(x) x + +#else +/* malloc failures result in lost memory, hash tables are unusable */ + +#ifndef uthash_fatal +#define uthash_fatal(msg) exit(-1) /* fatal OOM error */ +#endif + +#define HASH_RECORD_OOM(oomed) uthash_fatal("out of memory") +#define IF_HASH_NONFATAL_OOM(x) + +#endif + +/* initial number of buckets */ +#define HASH_INITIAL_NUM_BUCKETS 32U /* initial number of buckets */ +#define HASH_INITIAL_NUM_BUCKETS_LOG2 5U /* lg2 of initial number of buckets */ +#define HASH_BKT_CAPACITY_THRESH 10U /* expand when bucket count reaches */ + +/* calculate the element whose hash handle address is hhp */ +#define ELMT_FROM_HH(tbl, hhp) ((void *)(((char *)(hhp)) - ((tbl)->hho))) +/* calculate the hash handle from element address elp */ +#define HH_FROM_ELMT(tbl, elp) \ + ((UT_hash_handle *)(void *)(((char *)(elp)) + ((tbl)->hho))) + +#define HASH_ROLLBACK_BKT(hh, head, itemptrhh) \ + do { \ + struct UT_hash_handle *_hd_hh_item = (itemptrhh); \ + unsigned _hd_bkt; \ + HASH_TO_BKT(_hd_hh_item->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \ + (head)->hh.tbl->buckets[_hd_bkt].count++; \ + _hd_hh_item->hh_next = NULL; \ + _hd_hh_item->hh_prev = NULL; \ + } while (0) + +#define HASH_VALUE(keyptr, keylen, hashv) \ + do { \ + HASH_FUNCTION(keyptr, keylen, hashv); \ + } while (0) + +#define HASH_FIND_BYHASHVALUE(hh, head, keyptr, keylen, hashval, out) \ + do { \ + (out) = NULL; \ + if (head) { \ + unsigned _hf_bkt; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _hf_bkt); \ + if (HASH_BLOOM_TEST((head)->hh.tbl, hashval)) { \ + HASH_FIND_IN_BKT((head)->hh.tbl, hh, \ + (head)->hh.tbl->buckets[_hf_bkt], keyptr, \ + keylen, hashval, out); \ + } \ + } \ + } while (0) + +#define HASH_FIND(hh, head, keyptr, keylen, out) \ + do { \ + (out) = NULL; \ + if (head) { \ + unsigned _hf_hashv; \ + HASH_VALUE(keyptr, keylen, _hf_hashv); \ + HASH_FIND_BYHASHVALUE(hh, head, keyptr, keylen, _hf_hashv, out); \ + } \ + } while (0) + +#ifdef HASH_BLOOM +#define HASH_BLOOM_BITLEN (1UL << HASH_BLOOM) +#define HASH_BLOOM_BYTELEN \ + (HASH_BLOOM_BITLEN / 8UL) + (((HASH_BLOOM_BITLEN % 8UL) != 0UL) ? 1UL : 0UL) +#define HASH_BLOOM_MAKE(tbl, oomed) \ + do { \ + (tbl)->bloom_nbits = HASH_BLOOM; \ + (tbl)->bloom_bv = (uint8_t *)uthash_malloc(HASH_BLOOM_BYTELEN); \ + if (!(tbl)->bloom_bv) { \ + HASH_RECORD_OOM(oomed); \ + } else { \ + uthash_bzero((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \ + (tbl)->bloom_sig = HASH_BLOOM_SIGNATURE; \ + } \ + } while (0) + +#define HASH_BLOOM_FREE(tbl) \ + do { \ + uthash_free((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \ + } while (0) + +#define HASH_BLOOM_BITSET(bv, idx) (bv[(idx) / 8U] |= (1U << ((idx) % 8U))) +#define HASH_BLOOM_BITTEST(bv, idx) \ + ((bv[(idx) / 8U] & (1U << ((idx) % 8U))) != 0) + +#define HASH_BLOOM_ADD(tbl, hashv) \ + HASH_BLOOM_BITSET( \ + (tbl)->bloom_bv, \ + ((hashv) & (uint32_t)((1UL << (tbl)->bloom_nbits) - 1U))) + +#define HASH_BLOOM_TEST(tbl, hashv) \ + HASH_BLOOM_BITTEST( \ + (tbl)->bloom_bv, \ + ((hashv) & (uint32_t)((1UL << (tbl)->bloom_nbits) - 1U))) + +#else +#define HASH_BLOOM_MAKE(tbl, oomed) +#define HASH_BLOOM_FREE(tbl) +#define HASH_BLOOM_ADD(tbl, hashv) +#define HASH_BLOOM_TEST(tbl, hashv) 1 +#define HASH_BLOOM_BYTELEN 0U +#endif + +#define HASH_MAKE_TABLE(hh, head, oomed) \ + do { \ + (head)->hh.tbl = \ + (UT_hash_table *)uthash_malloc(sizeof(UT_hash_table)); \ + if (!(head)->hh.tbl) { \ + HASH_RECORD_OOM(oomed); \ + } else { \ + uthash_bzero((head)->hh.tbl, sizeof(UT_hash_table)); \ + (head)->hh.tbl->tail = &((head)->hh); \ + (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS; \ + (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2; \ + (head)->hh.tbl->hho = (char *)(&(head)->hh) - (char *)(head); \ + (head)->hh.tbl->buckets = (UT_hash_bucket *)uthash_malloc( \ + HASH_INITIAL_NUM_BUCKETS * sizeof(struct UT_hash_bucket)); \ + (head)->hh.tbl->signature = HASH_SIGNATURE; \ + if (!(head)->hh.tbl->buckets) { \ + HASH_RECORD_OOM(oomed); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + } else { \ + uthash_bzero((head)->hh.tbl->buckets, \ + HASH_INITIAL_NUM_BUCKETS * \ + sizeof(struct UT_hash_bucket)); \ + HASH_BLOOM_MAKE((head)->hh.tbl, oomed); \ + IF_HASH_NONFATAL_OOM(if (oomed) { \ + uthash_free((head)->hh.tbl->buckets, \ + HASH_INITIAL_NUM_BUCKETS * \ + sizeof(struct UT_hash_bucket)); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + }) \ + } \ + } \ + } while (0) + +#define HASH_REPLACE_BYHASHVALUE_INORDER(hh, head, fieldname, keylen_in, \ + hashval, add, replaced, cmpfcn) \ + do { \ + (replaced) = NULL; \ + HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, \ + hashval, replaced); \ + if (replaced) { \ + HASH_DELETE(hh, head, replaced); \ + } \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), \ + keylen_in, hashval, add, cmpfcn); \ + } while (0) + +#define HASH_REPLACE_BYHASHVALUE(hh, head, fieldname, keylen_in, hashval, add, \ + replaced) \ + do { \ + (replaced) = NULL; \ + HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, \ + hashval, replaced); \ + if (replaced) { \ + HASH_DELETE(hh, head, replaced); \ + } \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, \ + hashval, add); \ + } while (0) + +#define HASH_REPLACE(hh, head, fieldname, keylen_in, add, replaced) \ + do { \ + unsigned _hr_hashv; \ + HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ + HASH_REPLACE_BYHASHVALUE(hh, head, fieldname, keylen_in, _hr_hashv, \ + add, replaced); \ + } while (0) + +#define HASH_REPLACE_INORDER(hh, head, fieldname, keylen_in, add, replaced, \ + cmpfcn) \ + do { \ + unsigned _hr_hashv; \ + HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ + HASH_REPLACE_BYHASHVALUE_INORDER(hh, head, fieldname, keylen_in, \ + _hr_hashv, add, replaced, cmpfcn); \ + } while (0) + +#define HASH_APPEND_LIST(hh, head, add) \ + do { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail); \ + (head)->hh.tbl->tail->next = (add); \ + (head)->hh.tbl->tail = &((add)->hh); \ + } while (0) + +#define HASH_AKBI_INNER_LOOP(hh, head, add, cmpfcn) \ + do { \ + do { \ + if (cmpfcn(DECLTYPE(head)(_hs_iter), add) > 0) { \ + break; \ + } \ + } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \ + } while (0) + +#ifdef NO_DECLTYPE +#undef HASH_AKBI_INNER_LOOP +#define HASH_AKBI_INNER_LOOP(hh, head, add, cmpfcn) \ + do { \ + char *_hs_saved_head = (char *)(head); \ + do { \ + DECLTYPE_ASSIGN(head, _hs_iter); \ + if (cmpfcn(head, add) > 0) { \ + DECLTYPE_ASSIGN(head, _hs_saved_head); \ + break; \ + } \ + DECLTYPE_ASSIGN(head, _hs_saved_head); \ + } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \ + } while (0) +#endif + +#if HASH_NONFATAL_OOM + +#define HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, oomed) \ + do { \ + if (!(oomed)) { \ + unsigned _ha_bkt; \ + (head)->hh.tbl->num_items++; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ + HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], hh, &(add)->hh, \ + oomed); \ + if (oomed) { \ + HASH_ROLLBACK_BKT(hh, head, &(add)->hh); \ + HASH_DELETE_HH(hh, head, &(add)->hh); \ + (add)->hh.tbl = NULL; \ + uthash_nonfatal_oom(add); \ + } else { \ + HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ + HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ + } \ + } else { \ + (add)->hh.tbl = NULL; \ + uthash_nonfatal_oom(add); \ + } \ + } while (0) + +#else + +#define HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, oomed) \ + do { \ + unsigned _ha_bkt; \ + (head)->hh.tbl->num_items++; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ + HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], hh, &(add)->hh, \ + oomed); \ + HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ + HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ + } while (0) + +#endif + +#define HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, keyptr, keylen_in, \ + hashval, add, cmpfcn) \ + do { \ + IF_HASH_NONFATAL_OOM(int _ha_oomed = 0;) \ + (add)->hh.hashv = (hashval); \ + (add)->hh.key = (char *)(keyptr); \ + (add)->hh.keylen = (unsigned)(keylen_in); \ + if (!(head)) { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = NULL; \ + HASH_MAKE_TABLE(hh, add, _ha_oomed); \ + IF_HASH_NONFATAL_OOM(if (!_ha_oomed) { ) \ + (head) = (add); \ + IF_HASH_NONFATAL_OOM( \ + }) \ + } else { \ + void *_hs_iter = (head); \ + (add)->hh.tbl = (head)->hh.tbl; \ + HASH_AKBI_INNER_LOOP(hh, head, add, cmpfcn); \ + if (_hs_iter) { \ + (add)->hh.next = _hs_iter; \ + if (((add)->hh.prev = \ + HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->prev)) { \ + HH_FROM_ELMT((head)->hh.tbl, (add)->hh.prev)->next = \ + (add); \ + } else { \ + (head) = (add); \ + } \ + HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->prev = (add); \ + } else { \ + HASH_APPEND_LIST(hh, head, add); \ + } \ + } \ + HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, \ + _ha_oomed); \ + HASH_FSCK(hh, head, "HASH_ADD_KEYPTR_BYHASHVALUE_INORDER"); \ + } while (0) + +#define HASH_ADD_KEYPTR_INORDER(hh, head, keyptr, keylen_in, add, cmpfcn) \ + do { \ + unsigned _hs_hashv; \ + HASH_VALUE(keyptr, keylen_in, _hs_hashv); \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, keyptr, keylen_in, \ + _hs_hashv, add, cmpfcn); \ + } while (0) + +#define HASH_ADD_BYHASHVALUE_INORDER(hh, head, fieldname, keylen_in, hashval, \ + add, cmpfcn) \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), \ + keylen_in, hashval, add, cmpfcn) + +#define HASH_ADD_INORDER(hh, head, fieldname, keylen_in, add, cmpfcn) \ + HASH_ADD_KEYPTR_INORDER(hh, head, &((add)->fieldname), keylen_in, add, \ + cmpfcn) + +#define HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, keyptr, keylen_in, hashval, add) \ + do { \ + IF_HASH_NONFATAL_OOM(int _ha_oomed = 0;) \ + (add)->hh.hashv = (hashval); \ + (add)->hh.key = (const void *)(keyptr); \ + (add)->hh.keylen = (unsigned)(keylen_in); \ + if (!(head)) { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = NULL; \ + HASH_MAKE_TABLE(hh, add, _ha_oomed); \ + IF_HASH_NONFATAL_OOM(if (!_ha_oomed) { ) \ + (head) = (add); \ + IF_HASH_NONFATAL_OOM( \ + }) \ + } else { \ + (add)->hh.tbl = (head)->hh.tbl; \ + HASH_APPEND_LIST(hh, head, add); \ + } \ + HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, \ + _ha_oomed); \ + HASH_FSCK(hh, head, "HASH_ADD_KEYPTR_BYHASHVALUE"); \ + } while (0) + +#define HASH_ADD_KEYPTR(hh, head, keyptr, keylen_in, add) \ + do { \ + unsigned _ha_hashv; \ + HASH_VALUE(keyptr, keylen_in, _ha_hashv); \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, keyptr, keylen_in, _ha_hashv, \ + add); \ + } while (0) + +#define HASH_ADD_BYHASHVALUE(hh, head, fieldname, keylen_in, hashval, add) \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, \ + hashval, add) + +#define HASH_ADD(hh, head, fieldname, keylen_in, add) \ + HASH_ADD_KEYPTR(hh, head, &((add)->fieldname), keylen_in, add) + +#define HASH_TO_BKT(hashv, num_bkts, bkt) \ + do { \ + bkt = ((hashv) & ((num_bkts)-1U)); \ + } while (0) + +/* delete "delptr" from the hash table. + * "the usual" patch-up process for the app-order doubly-linked-list. + * The use of _hd_hh_del below deserves special explanation. + * These used to be expressed using (delptr) but that led to a bug + * if someone used the same symbol for the head and deletee, like + * HASH_DELETE(hh,users,users); + * We want that to work, but by changing the head (users) below + * we were forfeiting our ability to further refer to the deletee (users) + * in the patch-up process. Solution: use scratch space to + * copy the deletee pointer, then the latter references are via that + * scratch pointer rather than through the repointed (users) symbol. + */ +#define HASH_DELETE(hh, head, delptr) HASH_DELETE_HH(hh, head, &(delptr)->hh) + +#define HASH_DELETE_HH(hh, head, delptrhh) \ + do { \ + const struct UT_hash_handle *_hd_hh_del = (delptrhh); \ + if ((_hd_hh_del->prev == NULL) && (_hd_hh_del->next == NULL)) { \ + HASH_BLOOM_FREE((head)->hh.tbl); \ + uthash_free((head)->hh.tbl->buckets, \ + (head)->hh.tbl->num_buckets * \ + sizeof(struct UT_hash_bucket)); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + (head) = NULL; \ + } else { \ + unsigned _hd_bkt; \ + if (_hd_hh_del == (head)->hh.tbl->tail) { \ + (head)->hh.tbl->tail = \ + HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->prev); \ + } \ + if (_hd_hh_del->prev != NULL) { \ + HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->prev)->next = \ + _hd_hh_del->next; \ + } else { \ + DECLTYPE_ASSIGN(head, _hd_hh_del->next); \ + } \ + if (_hd_hh_del->next != NULL) { \ + HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->next)->prev = \ + _hd_hh_del->prev; \ + } \ + HASH_TO_BKT(_hd_hh_del->hashv, (head)->hh.tbl->num_buckets, \ + _hd_bkt); \ + HASH_DEL_IN_BKT((head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del); \ + (head)->hh.tbl->num_items--; \ + } \ + HASH_FSCK(hh, head, "HASH_DELETE_HH"); \ + } while (0) + +/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */ +#define HASH_FIND_STR(head, findstr, out) \ + do { \ + unsigned _uthash_hfstr_keylen = (unsigned)uthash_strlen(findstr); \ + HASH_FIND(hh, head, findstr, _uthash_hfstr_keylen, out); \ + } while (0) +#define HASH_ADD_STR(head, strfield, add) \ + do { \ + unsigned _uthash_hastr_keylen = \ + (unsigned)uthash_strlen((add)->strfield); \ + HASH_ADD(hh, head, strfield[0], _uthash_hastr_keylen, add); \ + } while (0) +#define HASH_REPLACE_STR(head, strfield, add, replaced) \ + do { \ + unsigned _uthash_hrstr_keylen = \ + (unsigned)uthash_strlen((add)->strfield); \ + HASH_REPLACE(hh, head, strfield[0], _uthash_hrstr_keylen, add, \ + replaced); \ + } while (0) +#define HASH_FIND_INT(head, findint, out) \ + HASH_FIND(hh, head, findint, sizeof(int), out) +#define HASH_ADD_INT(head, intfield, add) \ + HASH_ADD(hh, head, intfield, sizeof(int), add) +#define HASH_REPLACE_INT(head, intfield, add, replaced) \ + HASH_REPLACE(hh, head, intfield, sizeof(int), add, replaced) +#define HASH_FIND_PTR(head, findptr, out) \ + HASH_FIND(hh, head, findptr, sizeof(void *), out) +#define HASH_ADD_PTR(head, ptrfield, add) \ + HASH_ADD(hh, head, ptrfield, sizeof(void *), add) +#define HASH_REPLACE_PTR(head, ptrfield, add, replaced) \ + HASH_REPLACE(hh, head, ptrfield, sizeof(void *), add, replaced) +#define HASH_DEL(head, delptr) HASH_DELETE(hh, head, delptr) + +/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined. + * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined. + */ +#ifdef HASH_DEBUG +#include /* fprintf, stderr */ +#define HASH_OOPS(...) \ + do { \ + fprintf(stderr, __VA_ARGS__); \ + exit(-1); \ + } while (0) +#define HASH_FSCK(hh, head, where) \ + do { \ + struct UT_hash_handle *_thh; \ + if (head) { \ + unsigned _bkt_i; \ + unsigned _count = 0; \ + char *_prev; \ + for (_bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; ++_bkt_i) { \ + unsigned _bkt_count = 0; \ + _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head; \ + _prev = NULL; \ + while (_thh) { \ + if (_prev != (char *)(_thh->hh_prev)) { \ + HASH_OOPS("%s: invalid hh_prev %p, actual %p\n", \ + (where), (void *)_thh->hh_prev, \ + (void *)_prev); \ + } \ + _bkt_count++; \ + _prev = (char *)(_thh); \ + _thh = _thh->hh_next; \ + } \ + _count += _bkt_count; \ + if ((head)->hh.tbl->buckets[_bkt_i].count != _bkt_count) { \ + HASH_OOPS("%s: invalid bucket count %u, actual %u\n", \ + (where), (head)->hh.tbl->buckets[_bkt_i].count, \ + _bkt_count); \ + } \ + } \ + if (_count != (head)->hh.tbl->num_items) { \ + HASH_OOPS("%s: invalid hh item count %u, actual %u\n", \ + (where), (head)->hh.tbl->num_items, _count); \ + } \ + _count = 0; \ + _prev = NULL; \ + _thh = &(head)->hh; \ + while (_thh) { \ + _count++; \ + if (_prev != (char *)_thh->prev) { \ + HASH_OOPS("%s: invalid prev %p, actual %p\n", (where), \ + (void *)_thh->prev, (void *)_prev); \ + } \ + _prev = (char *)ELMT_FROM_HH((head)->hh.tbl, _thh); \ + _thh = (_thh->next ? HH_FROM_ELMT((head)->hh.tbl, _thh->next) \ + : NULL); \ + } \ + if (_count != (head)->hh.tbl->num_items) { \ + HASH_OOPS("%s: invalid app item count %u, actual %u\n", \ + (where), (head)->hh.tbl->num_items, _count); \ + } \ + } \ + } while (0) +#else +#define HASH_FSCK(hh, head, where) +#endif + +/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to + * the descriptor to which this macro is defined for tuning the hash function. + * The app can #include to get the prototype for write(2). */ +#ifdef HASH_EMIT_KEYS +#define HASH_EMIT_KEY(hh, head, keyptr, fieldlen) \ + do { \ + unsigned _klen = fieldlen; \ + write(HASH_EMIT_KEYS, &_klen, sizeof(_klen)); \ + write(HASH_EMIT_KEYS, keyptr, (unsigned long)fieldlen); \ + } while (0) +#else +#define HASH_EMIT_KEY(hh, head, keyptr, fieldlen) +#endif + +/* The Bernstein hash function, used in Perl prior to v5.6. Note (x<<5+x)=x*33. */ +#define HASH_BER(key, keylen, hashv) \ + do { \ + unsigned _hb_keylen = (unsigned)keylen; \ + const unsigned char *_hb_key = (const unsigned char *)(key); \ + (hashv) = 0; \ + while (_hb_keylen-- != 0U) { \ + (hashv) = (((hashv) << 5) + (hashv)) + *_hb_key++; \ + } \ + } while (0) + +/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at + * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx + * (archive link: https://archive.is/Ivcan ) + */ +#define HASH_SAX(key, keylen, hashv) \ + do { \ + unsigned _sx_i; \ + const unsigned char *_hs_key = (const unsigned char *)(key); \ + hashv = 0; \ + for (_sx_i = 0; _sx_i < keylen; _sx_i++) { \ + hashv ^= (hashv << 5) + (hashv >> 2) + _hs_key[_sx_i]; \ + } \ + } while (0) +/* FNV-1a variation */ +#define HASH_FNV(key, keylen, hashv) \ + do { \ + unsigned _fn_i; \ + const unsigned char *_hf_key = (const unsigned char *)(key); \ + (hashv) = 2166136261U; \ + for (_fn_i = 0; _fn_i < keylen; _fn_i++) { \ + hashv = hashv ^ _hf_key[_fn_i]; \ + hashv = hashv * 16777619U; \ + } \ + } while (0) + +#define HASH_OAT(key, keylen, hashv) \ + do { \ + unsigned _ho_i; \ + const unsigned char *_ho_key = (const unsigned char *)(key); \ + hashv = 0; \ + for (_ho_i = 0; _ho_i < keylen; _ho_i++) { \ + hashv += _ho_key[_ho_i]; \ + hashv += (hashv << 10); \ + hashv ^= (hashv >> 6); \ + } \ + hashv += (hashv << 3); \ + hashv ^= (hashv >> 11); \ + hashv += (hashv << 15); \ + } while (0) + +#define HASH_JEN_MIX(a, b, c) \ + do { \ + a -= b; \ + a -= c; \ + a ^= (c >> 13); \ + b -= c; \ + b -= a; \ + b ^= (a << 8); \ + c -= a; \ + c -= b; \ + c ^= (b >> 13); \ + a -= b; \ + a -= c; \ + a ^= (c >> 12); \ + b -= c; \ + b -= a; \ + b ^= (a << 16); \ + c -= a; \ + c -= b; \ + c ^= (b >> 5); \ + a -= b; \ + a -= c; \ + a ^= (c >> 3); \ + b -= c; \ + b -= a; \ + b ^= (a << 10); \ + c -= a; \ + c -= b; \ + c ^= (b >> 15); \ + } while (0) + +#define HASH_JEN(key, keylen, hashv) \ + do { \ + unsigned _hj_i, _hj_j, _hj_k; \ + unsigned const char *_hj_key = (unsigned const char *)(key); \ + hashv = 0xfeedbeefu; \ + _hj_i = _hj_j = 0x9e3779b9u; \ + _hj_k = (unsigned)(keylen); \ + while (_hj_k >= 12U) { \ + _hj_i += \ + (_hj_key[0] + ((unsigned)_hj_key[1] << 8) + \ + ((unsigned)_hj_key[2] << 16) + ((unsigned)_hj_key[3] << 24)); \ + _hj_j += \ + (_hj_key[4] + ((unsigned)_hj_key[5] << 8) + \ + ((unsigned)_hj_key[6] << 16) + ((unsigned)_hj_key[7] << 24)); \ + hashv += (_hj_key[8] + ((unsigned)_hj_key[9] << 8) + \ + ((unsigned)_hj_key[10] << 16) + \ + ((unsigned)_hj_key[11] << 24)); \ + \ + HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ + \ + _hj_key += 12; \ + _hj_k -= 12U; \ + } \ + hashv += (unsigned)(keylen); \ + switch (_hj_k) { \ + case 11: \ + hashv += ((unsigned)_hj_key[10] << 24); /* FALLTHROUGH */ \ + case 10: \ + hashv += ((unsigned)_hj_key[9] << 16); /* FALLTHROUGH */ \ + case 9: \ + hashv += ((unsigned)_hj_key[8] << 8); /* FALLTHROUGH */ \ + case 8: \ + _hj_j += ((unsigned)_hj_key[7] << 24); /* FALLTHROUGH */ \ + case 7: \ + _hj_j += ((unsigned)_hj_key[6] << 16); /* FALLTHROUGH */ \ + case 6: \ + _hj_j += ((unsigned)_hj_key[5] << 8); /* FALLTHROUGH */ \ + case 5: \ + _hj_j += _hj_key[4]; /* FALLTHROUGH */ \ + case 4: \ + _hj_i += ((unsigned)_hj_key[3] << 24); /* FALLTHROUGH */ \ + case 3: \ + _hj_i += ((unsigned)_hj_key[2] << 16); /* FALLTHROUGH */ \ + case 2: \ + _hj_i += ((unsigned)_hj_key[1] << 8); /* FALLTHROUGH */ \ + case 1: \ + _hj_i += _hj_key[0]; /* FALLTHROUGH */ \ + default:; \ + } \ + HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ + } while (0) + +/* The Paul Hsieh hash function */ +#undef get16bits +#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__) || \ + defined(_MSC_VER) || defined(__BORLANDC__) || defined(__TURBOC__) +#define get16bits(d) (*((const uint16_t *)(d))) +#endif + +#if !defined(get16bits) +#define get16bits(d) \ + ((((uint32_t)(((const uint8_t *)(d))[1])) << 8) + \ + (uint32_t)(((const uint8_t *)(d))[0])) +#endif +#define HASH_SFH(key, keylen, hashv) \ + do { \ + unsigned const char *_sfh_key = (unsigned const char *)(key); \ + uint32_t _sfh_tmp, _sfh_len = (uint32_t)keylen; \ + \ + unsigned _sfh_rem = _sfh_len & 3U; \ + _sfh_len >>= 2; \ + hashv = 0xcafebabeu; \ + \ + /* Main loop */ \ + for (; _sfh_len > 0U; _sfh_len--) { \ + hashv += get16bits(_sfh_key); \ + _sfh_tmp = ((uint32_t)(get16bits(_sfh_key + 2)) << 11) ^ hashv; \ + hashv = (hashv << 16) ^ _sfh_tmp; \ + _sfh_key += 2U * sizeof(uint16_t); \ + hashv += hashv >> 11; \ + } \ + \ + /* Handle end cases */ \ + switch (_sfh_rem) { \ + case 3: \ + hashv += get16bits(_sfh_key); \ + hashv ^= hashv << 16; \ + hashv ^= (uint32_t)(_sfh_key[sizeof(uint16_t)]) << 18; \ + hashv += hashv >> 11; \ + break; \ + case 2: \ + hashv += get16bits(_sfh_key); \ + hashv ^= hashv << 11; \ + hashv += hashv >> 17; \ + break; \ + case 1: \ + hashv += *_sfh_key; \ + hashv ^= hashv << 10; \ + hashv += hashv >> 1; \ + break; \ + default:; \ + } \ + \ + /* Force "avalanching" of final 127 bits */ \ + hashv ^= hashv << 3; \ + hashv += hashv >> 5; \ + hashv ^= hashv << 4; \ + hashv += hashv >> 17; \ + hashv ^= hashv << 25; \ + hashv += hashv >> 6; \ + } while (0) + +/* iterate over items in a known bucket to find desired item */ +#define HASH_FIND_IN_BKT(tbl, hh, head, keyptr, keylen_in, hashval, out) \ + do { \ + if ((head).hh_head != NULL) { \ + DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (head).hh_head)); \ + } else { \ + (out) = NULL; \ + } \ + while ((out) != NULL) { \ + if ((out)->hh.hashv == (hashval) && \ + (out)->hh.keylen == (keylen_in)) { \ + if (HASH_KEYCMP((out)->hh.key, keyptr, keylen_in) == 0) { \ + break; \ + } \ + } \ + if ((out)->hh.hh_next != NULL) { \ + DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (out)->hh.hh_next)); \ + } else { \ + (out) = NULL; \ + } \ + } \ + } while (0) + +/* add an item to a bucket */ +#define HASH_ADD_TO_BKT(head, hh, addhh, oomed) \ + do { \ + UT_hash_bucket *_ha_head = &(head); \ + _ha_head->count++; \ + (addhh)->hh_next = _ha_head->hh_head; \ + (addhh)->hh_prev = NULL; \ + if (_ha_head->hh_head != NULL) { \ + _ha_head->hh_head->hh_prev = (addhh); \ + } \ + _ha_head->hh_head = (addhh); \ + if ((_ha_head->count >= \ + ((_ha_head->expand_mult + 1U) * HASH_BKT_CAPACITY_THRESH)) && \ + !(addhh)->tbl->noexpand) { \ + HASH_EXPAND_BUCKETS(addhh, (addhh)->tbl, oomed); \ + IF_HASH_NONFATAL_OOM(if (oomed) { HASH_DEL_IN_BKT(head, addhh); }) \ + } \ + } while (0) + +/* remove an item from a given bucket */ +#define HASH_DEL_IN_BKT(head, delhh) \ + do { \ + UT_hash_bucket *_hd_head = &(head); \ + _hd_head->count--; \ + if (_hd_head->hh_head == (delhh)) { \ + _hd_head->hh_head = (delhh)->hh_next; \ + } \ + if ((delhh)->hh_prev) { \ + (delhh)->hh_prev->hh_next = (delhh)->hh_next; \ + } \ + if ((delhh)->hh_next) { \ + (delhh)->hh_next->hh_prev = (delhh)->hh_prev; \ + } \ + } while (0) + +/* Bucket expansion has the effect of doubling the number of buckets + * and redistributing the items into the new buckets. Ideally the + * items will distribute more or less evenly into the new buckets + * (the extent to which this is true is a measure of the quality of + * the hash function as it applies to the key domain). + * + * With the items distributed into more buckets, the chain length + * (item count) in each bucket is reduced. Thus by expanding buckets + * the hash keeps a bound on the chain length. This bounded chain + * length is the essence of how a hash provides constant time lookup. + * + * The calculation of tbl->ideal_chain_maxlen below deserves some + * explanation. First, keep in mind that we're calculating the ideal + * maximum chain length based on the *new* (doubled) bucket count. + * In fractions this is just n/b (n=number of items,b=new num buckets). + * Since the ideal chain length is an integer, we want to calculate + * ceil(n/b). We don't depend on floating point arithmetic in this + * hash, so to calculate ceil(n/b) with integers we could write + * + * ceil(n/b) = (n/b) + ((n%b)?1:0) + * + * and in fact a previous version of this hash did just that. + * But now we have improved things a bit by recognizing that b is + * always a power of two. We keep its base 2 log handy (call it lb), + * so now we can write this with a bit shift and logical AND: + * + * ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0) + * + */ +#define HASH_EXPAND_BUCKETS(hh, tbl, oomed) \ + do { \ + unsigned _he_bkt; \ + unsigned _he_bkt_i; \ + struct UT_hash_handle *_he_thh, *_he_hh_nxt; \ + UT_hash_bucket *_he_new_buckets, *_he_newbkt; \ + _he_new_buckets = (UT_hash_bucket *)uthash_malloc( \ + sizeof(struct UT_hash_bucket) * (tbl)->num_buckets * 2U); \ + if (!_he_new_buckets) { \ + HASH_RECORD_OOM(oomed); \ + } else { \ + uthash_bzero(_he_new_buckets, sizeof(struct UT_hash_bucket) * \ + (tbl)->num_buckets * 2U); \ + (tbl)->ideal_chain_maxlen = \ + ((tbl)->num_items >> ((tbl)->log2_num_buckets + 1U)) + \ + ((((tbl)->num_items & (((tbl)->num_buckets * 2U) - 1U)) != 0U) \ + ? 1U \ + : 0U); \ + (tbl)->nonideal_items = 0; \ + for (_he_bkt_i = 0; _he_bkt_i < (tbl)->num_buckets; _he_bkt_i++) { \ + _he_thh = (tbl)->buckets[_he_bkt_i].hh_head; \ + while (_he_thh != NULL) { \ + _he_hh_nxt = _he_thh->hh_next; \ + HASH_TO_BKT(_he_thh->hashv, (tbl)->num_buckets * 2U, \ + _he_bkt); \ + _he_newbkt = &(_he_new_buckets[_he_bkt]); \ + if (++(_he_newbkt->count) > (tbl)->ideal_chain_maxlen) { \ + (tbl)->nonideal_items++; \ + if (_he_newbkt->count > \ + _he_newbkt->expand_mult * \ + (tbl)->ideal_chain_maxlen) { \ + _he_newbkt->expand_mult++; \ + } \ + } \ + _he_thh->hh_prev = NULL; \ + _he_thh->hh_next = _he_newbkt->hh_head; \ + if (_he_newbkt->hh_head != NULL) { \ + _he_newbkt->hh_head->hh_prev = _he_thh; \ + } \ + _he_newbkt->hh_head = _he_thh; \ + _he_thh = _he_hh_nxt; \ + } \ + } \ + uthash_free((tbl)->buckets, \ + (tbl)->num_buckets * sizeof(struct UT_hash_bucket)); \ + (tbl)->num_buckets *= 2U; \ + (tbl)->log2_num_buckets++; \ + (tbl)->buckets = _he_new_buckets; \ + (tbl)->ineff_expands = \ + ((tbl)->nonideal_items > ((tbl)->num_items >> 1)) \ + ? ((tbl)->ineff_expands + 1U) \ + : 0U; \ + if ((tbl)->ineff_expands > 1U) { \ + (tbl)->noexpand = 1; \ + uthash_noexpand_fyi(tbl); \ + } \ + uthash_expand_fyi(tbl); \ + } \ + } while (0) + +/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */ +/* Note that HASH_SORT assumes the hash handle name to be hh. + * HASH_SRT was added to allow the hash handle name to be passed in. */ +#define HASH_SORT(head, cmpfcn) HASH_SRT(hh, head, cmpfcn) +#define HASH_SRT(hh, head, cmpfcn) \ + do { \ + unsigned _hs_i; \ + unsigned _hs_looping, _hs_nmerges, _hs_insize, _hs_psize, _hs_qsize; \ + struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail; \ + if (head != NULL) { \ + _hs_insize = 1; \ + _hs_looping = 1; \ + _hs_list = &((head)->hh); \ + while (_hs_looping != 0U) { \ + _hs_p = _hs_list; \ + _hs_list = NULL; \ + _hs_tail = NULL; \ + _hs_nmerges = 0; \ + while (_hs_p != NULL) { \ + _hs_nmerges++; \ + _hs_q = _hs_p; \ + _hs_psize = 0; \ + for (_hs_i = 0; _hs_i < _hs_insize; ++_hs_i) { \ + _hs_psize++; \ + _hs_q = \ + ((_hs_q->next != NULL) \ + ? HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) \ + : NULL); \ + if (_hs_q == NULL) { \ + break; \ + } \ + } \ + _hs_qsize = _hs_insize; \ + while ((_hs_psize != 0U) || \ + ((_hs_qsize != 0U) && (_hs_q != NULL))) { \ + if (_hs_psize == 0U) { \ + _hs_e = _hs_q; \ + _hs_q = ((_hs_q->next != NULL) \ + ? HH_FROM_ELMT((head)->hh.tbl, \ + _hs_q->next) \ + : NULL); \ + _hs_qsize--; \ + } else if ((_hs_qsize == 0U) || (_hs_q == NULL)) { \ + _hs_e = _hs_p; \ + if (_hs_p != NULL) { \ + _hs_p = ((_hs_p->next != NULL) \ + ? HH_FROM_ELMT((head)->hh.tbl, \ + _hs_p->next) \ + : NULL); \ + } \ + _hs_psize--; \ + } else if ((cmpfcn(DECLTYPE(head)(ELMT_FROM_HH( \ + (head)->hh.tbl, _hs_p)), \ + DECLTYPE(head)(ELMT_FROM_HH( \ + (head)->hh.tbl, _hs_q)))) <= \ + 0) { \ + _hs_e = _hs_p; \ + if (_hs_p != NULL) { \ + _hs_p = ((_hs_p->next != NULL) \ + ? HH_FROM_ELMT((head)->hh.tbl, \ + _hs_p->next) \ + : NULL); \ + } \ + _hs_psize--; \ + } else { \ + _hs_e = _hs_q; \ + _hs_q = ((_hs_q->next != NULL) \ + ? HH_FROM_ELMT((head)->hh.tbl, \ + _hs_q->next) \ + : NULL); \ + _hs_qsize--; \ + } \ + if (_hs_tail != NULL) { \ + _hs_tail->next = \ + ((_hs_e != NULL) \ + ? ELMT_FROM_HH((head)->hh.tbl, _hs_e) \ + : NULL); \ + } else { \ + _hs_list = _hs_e; \ + } \ + if (_hs_e != NULL) { \ + _hs_e->prev = \ + ((_hs_tail != NULL) \ + ? ELMT_FROM_HH((head)->hh.tbl, _hs_tail) \ + : NULL); \ + } \ + _hs_tail = _hs_e; \ + } \ + _hs_p = _hs_q; \ + } \ + if (_hs_tail != NULL) { \ + _hs_tail->next = NULL; \ + } \ + if (_hs_nmerges <= 1U) { \ + _hs_looping = 0; \ + (head)->hh.tbl->tail = _hs_tail; \ + DECLTYPE_ASSIGN(head, \ + ELMT_FROM_HH((head)->hh.tbl, _hs_list)); \ + } \ + _hs_insize *= 2U; \ + } \ + HASH_FSCK(hh, head, "HASH_SRT"); \ + } \ + } while (0) + +/* This function selects items from one hash into another hash. + * The end result is that the selected items have dual presence + * in both hashes. There is no copy of the items made; rather + * they are added into the new hash through a secondary hash + * hash handle that must be present in the structure. */ +#define HASH_SELECT(hh_dst, dst, hh_src, src, cond) \ + do { \ + unsigned _src_bkt, _dst_bkt; \ + void *_last_elt = NULL, *_elt; \ + UT_hash_handle *_src_hh, *_dst_hh, *_last_elt_hh = NULL; \ + ptrdiff_t _dst_hho = ((char *)(&(dst)->hh_dst) - (char *)(dst)); \ + if ((src) != NULL) { \ + for (_src_bkt = 0; _src_bkt < (src)->hh_src.tbl->num_buckets; \ + _src_bkt++) { \ + for (_src_hh = (src)->hh_src.tbl->buckets[_src_bkt].hh_head; \ + _src_hh != NULL; _src_hh = _src_hh->hh_next) { \ + _elt = ELMT_FROM_HH((src)->hh_src.tbl, _src_hh); \ + if (cond(_elt)) { \ + IF_HASH_NONFATAL_OOM(int _hs_oomed = 0;) \ + _dst_hh = (UT_hash_handle *)(void *)(((char *)_elt) + \ + _dst_hho); \ + _dst_hh->key = _src_hh->key; \ + _dst_hh->keylen = _src_hh->keylen; \ + _dst_hh->hashv = _src_hh->hashv; \ + _dst_hh->prev = _last_elt; \ + _dst_hh->next = NULL; \ + if (_last_elt_hh != NULL) { \ + _last_elt_hh->next = _elt; \ + } \ + if ((dst) == NULL) { \ + DECLTYPE_ASSIGN(dst, _elt); \ + HASH_MAKE_TABLE(hh_dst, dst, _hs_oomed); \ + IF_HASH_NONFATAL_OOM(if (_hs_oomed) { \ + uthash_nonfatal_oom(_elt); \ + (dst) = NULL; \ + continue; \ + }) \ + } else { \ + _dst_hh->tbl = (dst)->hh_dst.tbl; \ + } \ + HASH_TO_BKT(_dst_hh->hashv, _dst_hh->tbl->num_buckets, \ + _dst_bkt); \ + HASH_ADD_TO_BKT(_dst_hh->tbl->buckets[_dst_bkt], \ + hh_dst, _dst_hh, _hs_oomed); \ + (dst)->hh_dst.tbl->num_items++; \ + IF_HASH_NONFATAL_OOM(if (_hs_oomed) { \ + HASH_ROLLBACK_BKT(hh_dst, dst, _dst_hh); \ + HASH_DELETE_HH(hh_dst, dst, _dst_hh); \ + _dst_hh->tbl = NULL; \ + uthash_nonfatal_oom(_elt); \ + continue; \ + }) \ + HASH_BLOOM_ADD(_dst_hh->tbl, _dst_hh->hashv); \ + _last_elt = _elt; \ + _last_elt_hh = _dst_hh; \ + } \ + } \ + } \ + } \ + HASH_FSCK(hh_dst, dst, "HASH_SELECT"); \ + } while (0) + +#define HASH_CLEAR(hh, head) \ + do { \ + if ((head) != NULL) { \ + HASH_BLOOM_FREE((head)->hh.tbl); \ + uthash_free((head)->hh.tbl->buckets, \ + (head)->hh.tbl->num_buckets * \ + sizeof(struct UT_hash_bucket)); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + (head) = NULL; \ + } \ + } while (0) + +#define HASH_OVERHEAD(hh, head) \ + (((head) != NULL) \ + ? ((size_t)(((head)->hh.tbl->num_items * sizeof(UT_hash_handle)) + \ + ((head)->hh.tbl->num_buckets * sizeof(UT_hash_bucket)) + \ + sizeof(UT_hash_table) + (HASH_BLOOM_BYTELEN))) \ + : 0U) + +#ifdef NO_DECLTYPE +#define HASH_ITER(hh, head, el, tmp) \ + for (((el) = (head)), \ + ((*(char **)(&(tmp))) = \ + (char *)((head != NULL) ? (head)->hh.next : NULL)); \ + (el) != NULL; ((el) = (tmp)), \ + ((*(char **)(&(tmp))) = \ + (char *)((tmp != NULL) ? (tmp)->hh.next : NULL))) +#else +#define HASH_ITER(hh, head, el, tmp) \ + for (((el) = (head)), \ + ((tmp) = DECLTYPE(el)((head != NULL) ? (head)->hh.next : NULL)); \ + (el) != NULL; \ + ((el) = (tmp)), \ + ((tmp) = DECLTYPE(el)((tmp != NULL) ? (tmp)->hh.next : NULL))) +#endif + +/* obtain a count of items in the hash */ +#define HASH_COUNT(head) HASH_CNT(hh, head) +#define HASH_CNT(hh, head) ((head != NULL) ? ((head)->hh.tbl->num_items) : 0U) + +typedef struct UT_hash_bucket { + struct UT_hash_handle *hh_head; + unsigned count; + + /* expand_mult is normally set to 0. In this situation, the max chain length + * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If + * the bucket's chain exceeds this length, bucket expansion is triggered). + * However, setting expand_mult to a non-zero value delays bucket expansion + * (that would be triggered by additions to this particular bucket) + * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH. + * (The multiplier is simply expand_mult+1). The whole idea of this + * multiplier is to reduce bucket expansions, since they are expensive, in + * situations where we know that a particular bucket tends to be overused. + * It is better to let its chain length grow to a longer yet-still-bounded + * value, than to do an O(n) bucket expansion too often. + */ + unsigned expand_mult; + +} UT_hash_bucket; + +/* random signature used only to find hash tables in external analysis */ +#define HASH_SIGNATURE 0xa0111fe1u +#define HASH_BLOOM_SIGNATURE 0xb12220f2u + +typedef struct UT_hash_table { + UT_hash_bucket *buckets; + unsigned num_buckets, log2_num_buckets; + unsigned num_items; + struct UT_hash_handle *tail; /* tail hh in app order, for fast append */ + ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */ + + /* in an ideal situation (all buckets used equally), no bucket would have + * more than ceil(#items/#buckets) items. that's the ideal chain length. */ + unsigned ideal_chain_maxlen; + + /* nonideal_items is the number of items in the hash whose chain position + * exceeds the ideal chain maxlen. these items pay the penalty for an uneven + * hash distribution; reaching them in a chain traversal takes >ideal steps */ + unsigned nonideal_items; + + /* ineffective expands occur when a bucket doubling was performed, but + * afterward, more than half the items in the hash had nonideal chain + * positions. If this happens on two consecutive expansions we inhibit any + * further expansion, as it's not helping; this happens when the hash + * function isn't a good fit for the key domain. When expansion is inhibited + * the hash will still work, albeit no longer in constant time. */ + unsigned ineff_expands, noexpand; + + uint32_t signature; /* used only to find hash tables in external analysis */ +#ifdef HASH_BLOOM + uint32_t + bloom_sig; /* used only to test bloom exists in external analysis */ + uint8_t *bloom_bv; + uint8_t bloom_nbits; +#endif + +} UT_hash_table; + +typedef struct UT_hash_handle { + struct UT_hash_table *tbl; + void *prev; /* prev element in app order */ + void *next; /* next element in app order */ + struct UT_hash_handle *hh_prev; /* previous hh in bucket order */ + struct UT_hash_handle *hh_next; /* next hh in bucket order */ + const void *key; /* ptr to enclosing struct's key */ + unsigned keylen; /* enclosing struct's key len */ + unsigned hashv; /* result of hash-fcn(key) */ +} UT_hash_handle; + +#endif /* UTHASH_H */ diff --git a/src/uthash/utlist.h b/src/uthash/utlist.h new file mode 100644 index 000000000..ed4660b49 --- /dev/null +++ b/src/uthash/utlist.h @@ -0,0 +1,1148 @@ +/* +Copyright (c) 2007-2022, Troy D. Hanson https://troydhanson.github.io/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef UTLIST_H +#define UTLIST_H + +#define UTLIST_VERSION 2.3.0 + +#include + +/* + * This file contains macros to manipulate singly and doubly-linked lists. + * + * 1. LL_ macros: singly-linked lists. + * 2. DL_ macros: doubly-linked lists. + * 3. CDL_ macros: circular doubly-linked lists. + * + * To use singly-linked lists, your structure must have a "next" pointer. + * To use doubly-linked lists, your structure must "prev" and "next" pointers. + * Either way, the pointer to the head of the list must be initialized to NULL. + * + * ----------------.EXAMPLE ------------------------- + * struct item { + * int id; + * struct item *prev, *next; + * } + * + * struct item *list = NULL: + * + * int main() { + * struct item *item; + * ... allocate and populate item ... + * DL_APPEND(list, item); + * } + * -------------------------------------------------- + * + * For doubly-linked lists, the append and delete macros are O(1) + * For singly-linked lists, append and delete are O(n) but prepend is O(1) + * The sort macro is O(n log(n)) for all types of single/double/circular lists. + */ + +/* These macros use decltype or the earlier __typeof GNU extension. + As decltype is only available in newer compilers (VS2010 or gcc 4.3+ + when compiling c++ source) this code uses whatever method is needed + or, for VS2008 where neither is available, uses casting workarounds. */ +#if !defined(LDECLTYPE) && !defined(NO_DECLTYPE) +#if defined(_MSC_VER) /* MS compiler */ +#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ +#define LDECLTYPE(x) decltype(x) +#else /* VS2008 or older (or VS2010 in C mode) */ +#define NO_DECLTYPE +#endif +#elif defined(__MCST__) /* Elbrus C Compiler */ +#define LDECLTYPE(x) __typeof(x) +#elif defined(__BORLANDC__) || defined(__ICCARM__) || defined(__LCC__) || \ + defined(__WATCOMC__) +#define NO_DECLTYPE +#else /* GNU, Sun and other compilers */ +#define LDECLTYPE(x) __typeof(x) +#endif +#endif + +/* for VS2008 we use some workarounds to get around the lack of decltype, + * namely, we always reassign our tmp variable to the list head if we need + * to dereference its prev/next pointers, and save/restore the real head.*/ +#ifdef NO_DECLTYPE +#define IF_NO_DECLTYPE(x) x +#define LDECLTYPE(x) char * +#define UTLIST_SV(elt, list) \ + _tmp = (char *)(list); \ + { \ + char **_alias = (char **)&(list); \ + *_alias = (elt); \ + } +#define UTLIST_NEXT(elt, list, next) ((char *)((list)->next)) +#define UTLIST_NEXTASGN(elt, list, to, next) \ + { \ + char **_alias = (char **)&((list)->next); \ + *_alias = (char *)(to); \ + } +/* #define UTLIST_PREV(elt,list,prev) ((char*)((list)->prev)) */ +#define UTLIST_PREVASGN(elt, list, to, prev) \ + { \ + char **_alias = (char **)&((list)->prev); \ + *_alias = (char *)(to); \ + } +#define UTLIST_RS(list) \ + { \ + char **_alias = (char **)&(list); \ + *_alias = _tmp; \ + } +#define UTLIST_CASTASGN(a, b) \ + { \ + char **_alias = (char **)&(a); \ + *_alias = (char *)(b); \ + } +#else +#define IF_NO_DECLTYPE(x) +#define UTLIST_SV(elt, list) +#define UTLIST_NEXT(elt, list, next) ((elt)->next) +#define UTLIST_NEXTASGN(elt, list, to, next) ((elt)->next) = (to) +/* #define UTLIST_PREV(elt,list,prev) ((elt)->prev) */ +#define UTLIST_PREVASGN(elt, list, to, prev) ((elt)->prev) = (to) +#define UTLIST_RS(list) +#define UTLIST_CASTASGN(a, b) (a) = (b) +#endif + +/****************************************************************************** + * The sort macro is an adaptation of Simon Tatham's O(n log(n)) mergesort * + * Unwieldy variable names used here to avoid shadowing passed-in variables. * + *****************************************************************************/ +#define LL_SORT(list, cmp) LL_SORT2(list, cmp, next) + +#define LL_SORT2(list, cmp, next) \ + do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + UTLIST_CASTASGN(_ls_p, list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + if (!_ls_q) \ + break; \ + } \ + _ls_qsize = _ls_insize; \ + while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + } else if (_ls_qsize == 0 || !_ls_q) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + } else if (cmp(_ls_p, _ls_q) <= 0) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + } else { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + } \ + if (_ls_tail) { \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, _ls_e, next); \ + UTLIST_RS(list); \ + } else { \ + UTLIST_CASTASGN(list, _ls_e); \ + } \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + if (_ls_tail) { \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, NULL, next); \ + UTLIST_RS(list); \ + } \ + if (_ls_nmerges <= 1) { \ + _ls_looping = 0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ + } while (0) + +#define DL_SORT(list, cmp) DL_SORT2(list, cmp, prev, next) + +#define DL_SORT2(list, cmp, prev, next) \ + do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + UTLIST_CASTASGN(_ls_p, list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + if (!_ls_q) \ + break; \ + } \ + _ls_qsize = _ls_insize; \ + while ((_ls_psize > 0) || ((_ls_qsize > 0) && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + } else if ((_ls_qsize == 0) || (!_ls_q)) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + } else if (cmp(_ls_p, _ls_q) <= 0) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + } else { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + } \ + if (_ls_tail) { \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, _ls_e, next); \ + UTLIST_RS(list); \ + } else { \ + UTLIST_CASTASGN(list, _ls_e); \ + } \ + UTLIST_SV(_ls_e, list); \ + UTLIST_PREVASGN(_ls_e, list, _ls_tail, prev); \ + UTLIST_RS(list); \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + UTLIST_CASTASGN((list)->prev, _ls_tail); \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, NULL, next); \ + UTLIST_RS(list); \ + if (_ls_nmerges <= 1) { \ + _ls_looping = 0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ + } while (0) + +#define CDL_SORT(list, cmp) CDL_SORT2(list, cmp, prev, next) + +#define CDL_SORT2(list, cmp, prev, next) \ + do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + LDECLTYPE(list) _ls_oldhead; \ + LDECLTYPE(list) _tmp; \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + UTLIST_CASTASGN(_ls_p, list); \ + UTLIST_CASTASGN(_ls_oldhead, list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + UTLIST_SV(_ls_q, list); \ + if (UTLIST_NEXT(_ls_q, list, next) == _ls_oldhead) { \ + _ls_q = NULL; \ + } else { \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + } \ + UTLIST_RS(list); \ + if (!_ls_q) \ + break; \ + } \ + _ls_qsize = _ls_insize; \ + while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + if (_ls_q == _ls_oldhead) { \ + _ls_q = NULL; \ + } \ + } else if (_ls_qsize == 0 || !_ls_q) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + if (_ls_p == _ls_oldhead) { \ + _ls_p = NULL; \ + } \ + } else if (cmp(_ls_p, _ls_q) <= 0) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + if (_ls_p == _ls_oldhead) { \ + _ls_p = NULL; \ + } \ + } else { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + if (_ls_q == _ls_oldhead) { \ + _ls_q = NULL; \ + } \ + } \ + if (_ls_tail) { \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, _ls_e, next); \ + UTLIST_RS(list); \ + } else { \ + UTLIST_CASTASGN(list, _ls_e); \ + } \ + UTLIST_SV(_ls_e, list); \ + UTLIST_PREVASGN(_ls_e, list, _ls_tail, prev); \ + UTLIST_RS(list); \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + UTLIST_CASTASGN((list)->prev, _ls_tail); \ + UTLIST_CASTASGN(_tmp, list); \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, _tmp, next); \ + UTLIST_RS(list); \ + if (_ls_nmerges <= 1) { \ + _ls_looping = 0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ + } while (0) + +/****************************************************************************** + * singly linked list macros (non-circular) * + *****************************************************************************/ +#define LL_PREPEND(head, add) LL_PREPEND2(head, add, next) + +#define LL_PREPEND2(head, add, next) \ + do { \ + (add)->next = (head); \ + (head) = (add); \ + } while (0) + +#define LL_CONCAT(head1, head2) LL_CONCAT2(head1, head2, next) + +#define LL_CONCAT2(head1, head2, next) \ + do { \ + LDECLTYPE(head1) _tmp; \ + if (head1) { \ + _tmp = (head1); \ + while (_tmp->next) { \ + _tmp = _tmp->next; \ + } \ + _tmp->next = (head2); \ + } else { \ + (head1) = (head2); \ + } \ + } while (0) + +#define LL_APPEND(head, add) LL_APPEND2(head, add, next) + +#define LL_APPEND2(head, add, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + (add)->next = NULL; \ + if (head) { \ + _tmp = (head); \ + while (_tmp->next) { \ + _tmp = _tmp->next; \ + } \ + _tmp->next = (add); \ + } else { \ + (head) = (add); \ + } \ + } while (0) + +#define LL_INSERT_INORDER(head, add, cmp) \ + LL_INSERT_INORDER2(head, add, cmp, next) + +#define LL_INSERT_INORDER2(head, add, cmp, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + if (head) { \ + LL_LOWER_BOUND2(head, _tmp, add, cmp, next); \ + LL_APPEND_ELEM2(head, _tmp, add, next); \ + } else { \ + (head) = (add); \ + (head)->next = NULL; \ + } \ + } while (0) + +#define LL_LOWER_BOUND(head, elt, like, cmp) \ + LL_LOWER_BOUND2(head, elt, like, cmp, next) + +#define LL_LOWER_BOUND2(head, elt, like, cmp, next) \ + do { \ + if ((head) == NULL || (cmp(head, like)) >= 0) { \ + (elt) = NULL; \ + } else { \ + for ((elt) = (head); (elt)->next != NULL; (elt) = (elt)->next) { \ + if (cmp((elt)->next, like) >= 0) { \ + break; \ + } \ + } \ + } \ + } while (0) + +#define LL_DELETE(head, del) LL_DELETE2(head, del, next) + +#define LL_DELETE2(head, del, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + if ((head) == (del)) { \ + (head) = (head)->next; \ + } else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (del))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (del)->next; \ + } \ + } \ + } while (0) + +#define LL_COUNT(head, el, counter) LL_COUNT2(head, el, counter, next) + +#define LL_COUNT2(head, el, counter, next) \ + do { \ + (counter) = 0; \ + LL_FOREACH2(head, el, next) { ++(counter); } \ + } while (0) + +#define LL_FOREACH(head, el) LL_FOREACH2(head, el, next) + +#define LL_FOREACH2(head, el, next) for ((el) = (head); el; (el) = (el)->next) + +#define LL_FOREACH_SAFE(head, el, tmp) LL_FOREACH_SAFE2(head, el, tmp, next) + +#define LL_FOREACH_SAFE2(head, el, tmp, next) \ + for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) + +#define LL_SEARCH_SCALAR(head, out, field, val) \ + LL_SEARCH_SCALAR2(head, out, field, val, next) + +#define LL_SEARCH_SCALAR2(head, out, field, val, next) \ + do { \ + LL_FOREACH2(head, out, next) { \ + if ((out)->field == (val)) \ + break; \ + } \ + } while (0) + +#define LL_SEARCH(head, out, elt, cmp) LL_SEARCH2(head, out, elt, cmp, next) + +#define LL_SEARCH2(head, out, elt, cmp, next) \ + do { \ + LL_FOREACH2(head, out, next) { \ + if ((cmp(out, elt)) == 0) \ + break; \ + } \ + } while (0) + +#define LL_REPLACE_ELEM2(head, el, add, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (el))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (add); \ + } \ + } \ + } while (0) + +#define LL_REPLACE_ELEM(head, el, add) LL_REPLACE_ELEM2(head, el, add, next) + +#define LL_PREPEND_ELEM2(head, el, add, next) \ + do { \ + if (el) { \ + LDECLTYPE(head) _tmp; \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (el))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (add); \ + } \ + } \ + } else { \ + LL_APPEND2(head, add, next); \ + } \ + } while (0) + +#define LL_PREPEND_ELEM(head, el, add) LL_PREPEND_ELEM2(head, el, add, next) + +#define LL_APPEND_ELEM2(head, el, add, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (el)->next = (add); \ + } else { \ + LL_PREPEND2(head, add, next); \ + } \ + } while (0) + +#define LL_APPEND_ELEM(head, el, add) LL_APPEND_ELEM2(head, el, add, next) + +#ifdef NO_DECLTYPE +/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ + +#undef LL_CONCAT2 +#define LL_CONCAT2(head1, head2, next) \ + do { \ + char *_tmp; \ + if (head1) { \ + _tmp = (char *)(head1); \ + while ((head1)->next) { \ + (head1) = (head1)->next; \ + } \ + (head1)->next = (head2); \ + UTLIST_RS(head1); \ + } else { \ + (head1) = (head2); \ + } \ + } while (0) + +#undef LL_APPEND2 +#define LL_APPEND2(head, add, next) \ + do { \ + if (head) { \ + (add)->next = head; /* use add->next as a temp variable */ \ + while ((add)->next->next) { \ + (add)->next = (add)->next->next; \ + } \ + (add)->next->next = (add); \ + } else { \ + (head) = (add); \ + } \ + (add)->next = NULL; \ + } while (0) + +#undef LL_INSERT_INORDER2 +#define LL_INSERT_INORDER2(head, add, cmp, next) \ + do { \ + if ((head) == NULL || (cmp(head, add)) >= 0) { \ + (add)->next = (head); \ + (head) = (add); \ + } else { \ + char *_tmp = (char *)(head); \ + while ((head)->next != NULL && (cmp((head)->next, add)) < 0) { \ + (head) = (head)->next; \ + } \ + (add)->next = (head)->next; \ + (head)->next = (add); \ + UTLIST_RS(head); \ + } \ + } while (0) + +#undef LL_DELETE2 +#define LL_DELETE2(head, del, next) \ + do { \ + if ((head) == (del)) { \ + (head) = (head)->next; \ + } else { \ + char *_tmp = (char *)(head); \ + while ((head)->next && ((head)->next != (del))) { \ + (head) = (head)->next; \ + } \ + if ((head)->next) { \ + (head)->next = ((del)->next); \ + } \ + UTLIST_RS(head); \ + } \ + } while (0) + +#undef LL_REPLACE_ELEM2 +#define LL_REPLACE_ELEM2(head, el, add, next) \ + do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + (add)->next = head; \ + while ((add)->next->next && ((add)->next->next != (el))) { \ + (add)->next = (add)->next->next; \ + } \ + if ((add)->next->next) { \ + (add)->next->next = (add); \ + } \ + } \ + (add)->next = (el)->next; \ + } while (0) + +#undef LL_PREPEND_ELEM2 +#define LL_PREPEND_ELEM2(head, el, add, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + (add)->next = (head); \ + while ((add)->next->next && ((add)->next->next != (el))) { \ + (add)->next = (add)->next->next; \ + } \ + if ((add)->next->next) { \ + (add)->next->next = (add); \ + } \ + } \ + (add)->next = (el); \ + } else { \ + LL_APPEND2(head, add, next); \ + } \ + } while (0) + +#endif /* NO_DECLTYPE */ + +/****************************************************************************** + * doubly linked list macros (non-circular) * + *****************************************************************************/ +#define DL_PREPEND(head, add) DL_PREPEND2(head, add, prev, next) + +#define DL_PREPEND2(head, add, prev, next) \ + do { \ + (add)->next = (head); \ + if (head) { \ + (add)->prev = (head)->prev; \ + (head)->prev = (add); \ + } else { \ + (add)->prev = (add); \ + } \ + (head) = (add); \ + } while (0) + +#define DL_APPEND(head, add) DL_APPEND2(head, add, prev, next) + +#define DL_APPEND2(head, add, prev, next) \ + do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (head)->prev->next = (add); \ + (head)->prev = (add); \ + (add)->next = NULL; \ + } else { \ + (head) = (add); \ + (head)->prev = (head); \ + (head)->next = NULL; \ + } \ + } while (0) + +#define DL_INSERT_INORDER(head, add, cmp) \ + DL_INSERT_INORDER2(head, add, cmp, prev, next) + +#define DL_INSERT_INORDER2(head, add, cmp, prev, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + if (head) { \ + DL_LOWER_BOUND2(head, _tmp, add, cmp, next); \ + DL_APPEND_ELEM2(head, _tmp, add, prev, next); \ + } else { \ + (head) = (add); \ + (head)->prev = (head); \ + (head)->next = NULL; \ + } \ + } while (0) + +#define DL_LOWER_BOUND(head, elt, like, cmp) \ + DL_LOWER_BOUND2(head, elt, like, cmp, next) + +#define DL_LOWER_BOUND2(head, elt, like, cmp, next) \ + do { \ + if ((head) == NULL || (cmp(head, like)) >= 0) { \ + (elt) = NULL; \ + } else { \ + for ((elt) = (head); (elt)->next != NULL; (elt) = (elt)->next) { \ + if ((cmp((elt)->next, like)) >= 0) { \ + break; \ + } \ + } \ + } \ + } while (0) + +#define DL_CONCAT(head1, head2) DL_CONCAT2(head1, head2, prev, next) + +#define DL_CONCAT2(head1, head2, prev, next) \ + do { \ + LDECLTYPE(head1) _tmp; \ + if (head2) { \ + if (head1) { \ + UTLIST_CASTASGN(_tmp, (head2)->prev); \ + (head2)->prev = (head1)->prev; \ + (head1)->prev->next = (head2); \ + UTLIST_CASTASGN((head1)->prev, _tmp); \ + } else { \ + (head1) = (head2); \ + } \ + } \ + } while (0) + +#define DL_DELETE(head, del) DL_DELETE2(head, del, prev, next) + +#define DL_DELETE2(head, del, prev, next) \ + do { \ + assert((head) != NULL); \ + assert((del)->prev != NULL); \ + if ((del)->prev == (del)) { \ + (head) = NULL; \ + } else if ((del) == (head)) { \ + assert((del)->next != NULL); \ + (del)->next->prev = (del)->prev; \ + (head) = (del)->next; \ + } else { \ + (del)->prev->next = (del)->next; \ + if ((del)->next) { \ + (del)->next->prev = (del)->prev; \ + } else { \ + (head)->prev = (del)->prev; \ + } \ + } \ + } while (0) + +#define DL_COUNT(head, el, counter) DL_COUNT2(head, el, counter, next) + +#define DL_COUNT2(head, el, counter, next) \ + do { \ + (counter) = 0; \ + DL_FOREACH2(head, el, next) { ++(counter); } \ + } while (0) + +#define DL_FOREACH(head, el) DL_FOREACH2(head, el, next) + +#define DL_FOREACH2(head, el, next) for ((el) = (head); el; (el) = (el)->next) + +/* this version is safe for deleting the elements during iteration */ +#define DL_FOREACH_SAFE(head, el, tmp) DL_FOREACH_SAFE2(head, el, tmp, next) + +#define DL_FOREACH_SAFE2(head, el, tmp, next) \ + for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) + +/* these are identical to their singly-linked list counterparts */ +#define DL_SEARCH_SCALAR LL_SEARCH_SCALAR +#define DL_SEARCH LL_SEARCH +#define DL_SEARCH_SCALAR2 LL_SEARCH_SCALAR2 +#define DL_SEARCH2 LL_SEARCH2 + +#define DL_REPLACE_ELEM2(head, el, add, prev, next) \ + do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + (add)->next = (el)->next; \ + if ((el)->next == NULL) { \ + (add)->prev = (add); \ + } else { \ + (add)->prev = (el)->prev; \ + (add)->next->prev = (add); \ + } \ + } else { \ + (add)->next = (el)->next; \ + (add)->prev = (el)->prev; \ + (add)->prev->next = (add); \ + if ((el)->next == NULL) { \ + (head)->prev = (add); \ + } else { \ + (add)->next->prev = (add); \ + } \ + } \ + } while (0) + +#define DL_REPLACE_ELEM(head, el, add) \ + DL_REPLACE_ELEM2(head, el, add, prev, next) + +#define DL_PREPEND_ELEM2(head, el, add, prev, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + (add)->prev = (el)->prev; \ + (el)->prev = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + (add)->prev->next = (add); \ + } \ + } else { \ + DL_APPEND2(head, add, prev, next); \ + } \ + } while (0) + +#define DL_PREPEND_ELEM(head, el, add) \ + DL_PREPEND_ELEM2(head, el, add, prev, next) + +#define DL_APPEND_ELEM2(head, el, add, prev, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (add)->prev = (el); \ + (el)->next = (add); \ + if ((add)->next) { \ + (add)->next->prev = (add); \ + } else { \ + (head)->prev = (add); \ + } \ + } else { \ + DL_PREPEND2(head, add, prev, next); \ + } \ + } while (0) + +#define DL_APPEND_ELEM(head, el, add) DL_APPEND_ELEM2(head, el, add, prev, next) + +#ifdef NO_DECLTYPE +/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ + +#undef DL_INSERT_INORDER2 +#define DL_INSERT_INORDER2(head, add, cmp, prev, next) \ + do { \ + if ((head) == NULL) { \ + (add)->prev = (add); \ + (add)->next = NULL; \ + (head) = (add); \ + } else if ((cmp(head, add)) >= 0) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (head) = (add); \ + } else { \ + char *_tmp = (char *)(head); \ + while ((head)->next && (cmp((head)->next, add)) < 0) { \ + (head) = (head)->next; \ + } \ + (add)->prev = (head); \ + (add)->next = (head)->next; \ + (head)->next = (add); \ + UTLIST_RS(head); \ + if ((add)->next) { \ + (add)->next->prev = (add); \ + } else { \ + (head)->prev = (add); \ + } \ + } \ + } while (0) +#endif /* NO_DECLTYPE */ + +/****************************************************************************** + * circular doubly linked list macros * + *****************************************************************************/ +#define CDL_APPEND(head, add) CDL_APPEND2(head, add, prev, next) + +#define CDL_APPEND2(head, add, prev, next) \ + do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (add)->prev->next = (add); \ + } else { \ + (add)->prev = (add); \ + (add)->next = (add); \ + (head) = (add); \ + } \ + } while (0) + +#define CDL_PREPEND(head, add) CDL_PREPEND2(head, add, prev, next) + +#define CDL_PREPEND2(head, add, prev, next) \ + do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (add)->prev->next = (add); \ + } else { \ + (add)->prev = (add); \ + (add)->next = (add); \ + } \ + (head) = (add); \ + } while (0) + +#define CDL_INSERT_INORDER(head, add, cmp) \ + CDL_INSERT_INORDER2(head, add, cmp, prev, next) + +#define CDL_INSERT_INORDER2(head, add, cmp, prev, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + if (head) { \ + CDL_LOWER_BOUND2(head, _tmp, add, cmp, next); \ + CDL_APPEND_ELEM2(head, _tmp, add, prev, next); \ + } else { \ + (head) = (add); \ + (head)->next = (head); \ + (head)->prev = (head); \ + } \ + } while (0) + +#define CDL_LOWER_BOUND(head, elt, like, cmp) \ + CDL_LOWER_BOUND2(head, elt, like, cmp, next) + +#define CDL_LOWER_BOUND2(head, elt, like, cmp, next) \ + do { \ + if ((head) == NULL || (cmp(head, like)) >= 0) { \ + (elt) = NULL; \ + } else { \ + for ((elt) = (head); (elt)->next != (head); (elt) = (elt)->next) { \ + if ((cmp((elt)->next, like)) >= 0) { \ + break; \ + } \ + } \ + } \ + } while (0) + +#define CDL_DELETE(head, del) CDL_DELETE2(head, del, prev, next) + +#define CDL_DELETE2(head, del, prev, next) \ + do { \ + if (((head) == (del)) && ((head)->next == (head))) { \ + (head) = NULL; \ + } else { \ + (del)->next->prev = (del)->prev; \ + (del)->prev->next = (del)->next; \ + if ((del) == (head)) \ + (head) = (del)->next; \ + } \ + } while (0) + +#define CDL_COUNT(head, el, counter) CDL_COUNT2(head, el, counter, next) + +#define CDL_COUNT2(head, el, counter, next) \ + do { \ + (counter) = 0; \ + CDL_FOREACH2(head, el, next) { ++(counter); } \ + } while (0) + +#define CDL_FOREACH(head, el) CDL_FOREACH2(head, el, next) + +#define CDL_FOREACH2(head, el, next) \ + for ((el) = (head); el; (el) = (((el)->next == (head)) ? NULL : (el)->next)) + +#define CDL_FOREACH_SAFE(head, el, tmp1, tmp2) \ + CDL_FOREACH_SAFE2(head, el, tmp1, tmp2, prev, next) + +#define CDL_FOREACH_SAFE2(head, el, tmp1, tmp2, prev, next) \ + for ((el) = (head), (tmp1) = (head) ? (head)->prev : NULL; \ + (el) && ((tmp2) = (el)->next, 1); \ + (el) = ((el) == (tmp1) ? NULL : (tmp2))) + +#define CDL_SEARCH_SCALAR(head, out, field, val) \ + CDL_SEARCH_SCALAR2(head, out, field, val, next) + +#define CDL_SEARCH_SCALAR2(head, out, field, val, next) \ + do { \ + CDL_FOREACH2(head, out, next) { \ + if ((out)->field == (val)) \ + break; \ + } \ + } while (0) + +#define CDL_SEARCH(head, out, elt, cmp) CDL_SEARCH2(head, out, elt, cmp, next) + +#define CDL_SEARCH2(head, out, elt, cmp, next) \ + do { \ + CDL_FOREACH2(head, out, next) { \ + if ((cmp(out, elt)) == 0) \ + break; \ + } \ + } while (0) + +#define CDL_REPLACE_ELEM2(head, el, add, prev, next) \ + do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((el)->next == (el)) { \ + (add)->next = (add); \ + (add)->prev = (add); \ + (head) = (add); \ + } else { \ + (add)->next = (el)->next; \ + (add)->prev = (el)->prev; \ + (add)->next->prev = (add); \ + (add)->prev->next = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + } \ + } while (0) + +#define CDL_REPLACE_ELEM(head, el, add) \ + CDL_REPLACE_ELEM2(head, el, add, prev, next) + +#define CDL_PREPEND_ELEM2(head, el, add, prev, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + (add)->prev = (el)->prev; \ + (el)->prev = (add); \ + (add)->prev->next = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + } else { \ + CDL_APPEND2(head, add, prev, next); \ + } \ + } while (0) + +#define CDL_PREPEND_ELEM(head, el, add) \ + CDL_PREPEND_ELEM2(head, el, add, prev, next) + +#define CDL_APPEND_ELEM2(head, el, add, prev, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (add)->prev = (el); \ + (el)->next = (add); \ + (add)->next->prev = (add); \ + } else { \ + CDL_PREPEND2(head, add, prev, next); \ + } \ + } while (0) + +#define CDL_APPEND_ELEM(head, el, add) \ + CDL_APPEND_ELEM2(head, el, add, prev, next) + +#ifdef NO_DECLTYPE +/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ + +#undef CDL_INSERT_INORDER2 +#define CDL_INSERT_INORDER2(head, add, cmp, prev, next) \ + do { \ + if ((head) == NULL) { \ + (add)->prev = (add); \ + (add)->next = (add); \ + (head) = (add); \ + } else if ((cmp(head, add)) >= 0) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (add)->prev->next = (add); \ + (head)->prev = (add); \ + (head) = (add); \ + } else { \ + char *_tmp = (char *)(head); \ + while ((char *)(head)->next != _tmp && \ + (cmp((head)->next, add)) < 0) { \ + (head) = (head)->next; \ + } \ + (add)->prev = (head); \ + (add)->next = (head)->next; \ + (add)->next->prev = (add); \ + (head)->next = (add); \ + UTLIST_RS(head); \ + } \ + } while (0) +#endif /* NO_DECLTYPE */ + +#endif /* UTLIST_H */ diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index f48c6f910..155184cc4 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -75,13 +75,16 @@ static __inline unsigned char utils_mssb_index(long long value) { // There is no good way to do atomic_load on windows... #define utils_atomic_load_acquire(object, dest) \ do { \ - *dest = InterlockedOr64Acquire((LONG64 volatile *)object, 0); \ + *(LONG64 *)dest = \ + InterlockedOr64Acquire((LONG64 volatile *)object, 0); \ } while (0) #define utils_atomic_store_release(object, desired) \ InterlockedExchange64((LONG64 volatile *)object, (LONG64)desired) #define utils_atomic_increment(object) \ InterlockedIncrement64((LONG64 volatile *)object) +#define utils_atomic_decrement(object) \ + InterlockedDecrement64((LONG64 volatile *)object) #define utils_fetch_and_add64(ptr, value) \ InterlockedExchangeAdd64((LONG64 *)(ptr), value) #else @@ -101,7 +104,10 @@ static __inline unsigned char utils_mssb_index(long long value) { #define utils_atomic_increment(object) \ __atomic_add_fetch(object, 1, __ATOMIC_ACQ_REL) +#define utils_atomic_decrement(object) \ + __atomic_sub_fetch(object, 1, __ATOMIC_ACQ_REL) #define utils_fetch_and_add64 __sync_fetch_and_add + #endif #ifdef __cplusplus diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index df17d9b2c..d24244ab0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -242,7 +242,7 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented add_umf_test( NAME provider_os_memory SRCS provider_os_memory.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST}) + LIBS ${UMF_UTILS_FOR_TEST} ${LIB_JEMALLOC_POOL}) if(UMF_BUILD_LIBUMF_POOL_DISJOINT) target_compile_definitions(umf_test-provider_os_memory PRIVATE UMF_POOL_DISJOINT_ENABLED=1) diff --git a/test/ipcFixtures.hpp b/test/ipcFixtures.hpp index d75e7be7c..161a84844 100644 --- a/test/ipcFixtures.hpp +++ b/test/ipcFixtures.hpp @@ -245,8 +245,7 @@ TEST_P(umfIpcTest, BasicFlow) { pool.reset(nullptr); EXPECT_EQ(stat.getCount, 1); EXPECT_EQ(stat.putCount, stat.getCount); - // TODO: enale check below once cache for open IPC handles is implemented - // EXPECT_EQ(stat.openCount, 1); + EXPECT_EQ(stat.openCount, 1); EXPECT_EQ(stat.closeCount, stat.openCount); } @@ -358,11 +357,8 @@ TEST_P(umfIpcTest, AllocFreeAllocTest) { get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); pool.reset(nullptr); - // TODO fix it - it does not work in case of IPC cache hit - // EXPECT_EQ(stat.allocCount, stat.getCount); EXPECT_EQ(stat.getCount, stat.putCount); - // TODO fix it - it does not work in case of IPC cache hit - // EXPECT_EQ(stat.openCount, stat.getCount); + EXPECT_EQ(stat.openCount, stat.getCount); EXPECT_EQ(stat.openCount, stat.closeCount); } @@ -531,8 +527,7 @@ TEST_P(umfIpcTest, ConcurrentOpenCloseHandles) { pool.reset(nullptr); EXPECT_EQ(stat.getCount, stat.allocCount); EXPECT_EQ(stat.putCount, stat.getCount); - // TODO: enale check below once cache for open IPC handles is implemented - // EXPECT_EQ(stat.openCount, stat.allocCount); + EXPECT_EQ(stat.openCount, stat.allocCount); EXPECT_EQ(stat.openCount, stat.closeCount); } diff --git a/test/provider_os_memory.cpp b/test/provider_os_memory.cpp index 734ebeec9..03128a6b3 100644 --- a/test/provider_os_memory.cpp +++ b/test/provider_os_memory.cpp @@ -11,6 +11,9 @@ #include #include #include +#ifdef UMF_POOL_JEMALLOC_ENABLED +#include +#endif using umf_test::test; @@ -389,6 +392,10 @@ static std::vector ipcTestParamsList = { {umfDisjointPoolOps(), &disjointParams, umfOsMemoryProviderOps(), &os_params, &hostAccessor, false}, #endif +#ifdef UMF_POOL_JEMALLOC_ENABLED + {umfJemallocPoolOps(), nullptr, umfOsMemoryProviderOps(), &os_params, + &hostAccessor, false}, +#endif }; INSTANTIATE_TEST_SUITE_P(osProviderTest, umfIpcTest, diff --git a/test/supp/drd-umf_test-provider_devdax_memory_ipc.supp b/test/supp/drd-umf_test-provider_devdax_memory_ipc.supp new file mode 100644 index 000000000..cd44bb49a --- /dev/null +++ b/test/supp/drd-umf_test-provider_devdax_memory_ipc.supp @@ -0,0 +1,8 @@ +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} diff --git a/test/supp/drd-umf_test-provider_file_memory_ipc.supp b/test/supp/drd-umf_test-provider_file_memory_ipc.supp index 76844585d..7fce24116 100644 --- a/test/supp/drd-umf_test-provider_file_memory_ipc.supp +++ b/test/supp/drd-umf_test-provider_file_memory_ipc.supp @@ -5,3 +5,12 @@ fun:pthread_cond_destroy@* ... } + +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} diff --git a/test/supp/drd-umf_test-provider_os_memory.supp b/test/supp/drd-umf_test-provider_os_memory.supp new file mode 100644 index 000000000..cd44bb49a --- /dev/null +++ b/test/supp/drd-umf_test-provider_os_memory.supp @@ -0,0 +1,8 @@ +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + drd:ConflictingAccess + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} diff --git a/test/supp/helgrind-umf_test-provider_devdax_memory_ipc.supp b/test/supp/helgrind-umf_test-provider_devdax_memory_ipc.supp new file mode 100644 index 000000000..4fcd2786c --- /dev/null +++ b/test/supp/helgrind-umf_test-provider_devdax_memory_ipc.supp @@ -0,0 +1,8 @@ +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} diff --git a/test/supp/helgrind-umf_test-provider_file_memory_ipc.supp b/test/supp/helgrind-umf_test-provider_file_memory_ipc.supp new file mode 100644 index 000000000..4fcd2786c --- /dev/null +++ b/test/supp/helgrind-umf_test-provider_file_memory_ipc.supp @@ -0,0 +1,8 @@ +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +} diff --git a/test/supp/helgrind-umf_test-provider_os_memory.supp b/test/supp/helgrind-umf_test-provider_os_memory.supp new file mode 100644 index 000000000..4fcd2786c --- /dev/null +++ b/test/supp/helgrind-umf_test-provider_os_memory.supp @@ -0,0 +1,8 @@ +{ + [false-positive] Double check locking pattern in trackingOpenIpcHandle + Helgrind:Race + fun:trackingOpenIpcHandle + fun:umfMemoryProviderOpenIPCHandle + fun:umfOpenIPCHandle + ... +}