From e1ccfc6672e4407d2ad59971e2eec4ab9cbe2b04 Mon Sep 17 00:00:00 2001 From: g3gg0 Date: Fri, 16 Aug 2024 00:39:21 +0200 Subject: [PATCH] move caching into separate file --- include/cache.h | 24 ++++ src/cache.c | 315 +++++++++++++++++++++++++++++++++++++++++++++++ src/server.c | 57 ++------- src/toniesJson.c | 93 ++------------ 4 files changed, 357 insertions(+), 132 deletions(-) create mode 100644 include/cache.h create mode 100644 src/cache.c diff --git a/include/cache.h b/include/cache.h new file mode 100644 index 00000000..7b3c4323 --- /dev/null +++ b/include/cache.h @@ -0,0 +1,24 @@ +#ifndef _CACHE_H +#define _CACHE_H + +#include +#include + +typedef struct cache_entry_s cache_entry_t; +struct cache_entry_s +{ + cache_entry_t *next; + uint32_t hash; + bool exists; + const char *original_url; + const char *cached_url; + const char *file_path; +}; + +cache_entry_t *cache_add(const char *url); +bool cache_fetch_entry(cache_entry_t *entry); +cache_entry_t *cache_fetch_by_url(const char *url); +cache_entry_t *cache_fetch_by_uri(const char *uri); +cache_entry_t *cache_fetch_by_cached_url(const char *cached_url); + +#endif // _CACHE_H \ No newline at end of file diff --git a/src/cache.c b/src/cache.c new file mode 100644 index 00000000..0482d3cd --- /dev/null +++ b/src/cache.c @@ -0,0 +1,315 @@ + + +#include "cache.h" +#include "web.h" +#include "fs_port.h" +#include "os_port.h" +#include "server_helpers.h" +#include "hash/sha256.h" // for sha256Update, sha256Final, sha256Init + +cache_entry_t cache_table = {.next = NULL, .hash = 0, .original_url = NULL, .cached_url = NULL, .file_path = NULL}; +uint32_t cache_entries = 0; + +const char *cache_hosturl() +{ + static char *hosturl = NULL; + const char *base_url = settings_get_string("core.host_url"); + + if (!base_url || osStrlen(base_url) < 1) + { + return ""; + } + + char *url = strdup(base_url); + char *end = &url[osStrlen(url) - 1]; + while (end != url) + { + if (*end != '/') + { + break; + } + + *end = '\0'; + end--; + } + + if (hosturl && !osStrcmp(hosturl, url)) + { + osFreeMem(url); + return hosturl; + } + + char *tmp = hosturl; + hosturl = url; + osFreeMem(tmp); + + return hosturl; +} + +void cache_entry_add(cache_entry_t *entry) +{ + if (!entry) + { + TRACE_ERROR("Error: entry is NULL\n"); + return; + } + + cache_entry_t *pos = &cache_table; + + if (!pos) + { + TRACE_ERROR("Error: cache_table is NULL\n"); + return; + } + + TRACE_DEBUG("Starting to add cache entry with the following details:\n"); + TRACE_DEBUG(" Hash: %08X\n", entry->hash); + TRACE_DEBUG(" Original URL: %s\n", entry->original_url ? entry->original_url : "NULL"); + TRACE_DEBUG(" Cached URL: %s\n", entry->cached_url ? entry->cached_url : "NULL"); + TRACE_DEBUG(" File Path: %s\n", entry->file_path ? entry->file_path : "NULL"); + + while (pos) + { + cache_entry_t *next = pos->next; + + if (!next) + { + TRACE_DEBUG("End of list reached, adding entry with hash: %08X at the end\n", entry->hash); + pos->next = entry; + entry->next = NULL; + return; + } + + if (entry->hash < next->hash) + { + TRACE_DEBUG("Inserting entry with hash: %08X before entry with hash: %08X\n", entry->hash, next->hash); + pos->next = entry; + entry->next = next; + return; + } + + if (entry->hash == next->hash) + { + if (!osStrcmp(entry->original_url, next->original_url)) + { + TRACE_DEBUG("Already added: %08X\n", entry->hash); + return; + } + TRACE_DEBUG("Inserting (duplicate short hash) entry with hash: %08X before entry with hash: %08X\n", entry->hash, next->hash); + pos->next = entry; + entry->next = next; + return; + } + + pos = next; + } + + TRACE_DEBUG("Finished adding cache entry with hash: %08X\n", entry->hash); +} + +cache_entry_t *cache_add(const char *url) +{ + const char *cachePath = get_settings()->internal.cachedirfull; + + if (cachePath == NULL || !fsDirExists(cachePath)) + { + TRACE_ERROR("core.cachedirfull not set to a valid path: '%s'", cachePath); + return NULL; + } + + uint8_t sha256_calc[32]; + char sha256_calc_str[65]; + + /* hash the image URL */ + Sha256Context ctx; + sha256Init(&ctx); + sha256Update(&ctx, url, strlen(url)); + sha256Final(&ctx, sha256_calc); + + for (int pos = 0; pos < 32; pos++) + { + osSprintf(&sha256_calc_str[2 * pos], "%02X", sha256_calc[pos]); + } + + /* Find the file extension from the URL */ + const char *ext_pos = strrchr(url, '.'); + char *extension = strdup("jpg"); + + if (ext_pos && !osStrchr(ext_pos, '/')) + { + osFreeMem(extension); + extension = strdup(&ext_pos[1]); + + /* Remove optional HTTP GET parameters */ + char *query_param = osStrchr(extension, '?'); + if (query_param) + { + *query_param = '\0'; + } + } + + cache_entry_t *entry = osAllocMem(sizeof(cache_entry_t)); + + entry->hash = (sha256_calc[0] << 24) | (sha256_calc[1] << 16) | (sha256_calc[2] << 8) | (sha256_calc[3] << 0); + entry->original_url = strdup(url); + entry->file_path = custom_asprintf("%s%c%s.%s", cachePath, PATH_SEPARATOR, sha256_calc_str, extension); + entry->cached_url = custom_asprintf("%s/cache/%s.%s", cache_hosturl(), sha256_calc_str, extension); + entry->exists = fsFileExists(entry->file_path); + + cache_entry_add(entry); + + osFreeMem(extension); + + return entry; +} + +bool cache_fetch_entry(cache_entry_t *entry) +{ + if (entry->exists) + { + return true; + } + + error_t err = web_download(entry->original_url, entry->file_path); + entry->exists = err == NO_ERROR; + + return entry->exists; +} + +cache_entry_t *cache_fetch_by_url(const char *url) +{ + if (url == NULL) + { + TRACE_ERROR("Error: URL is NULL\n"); + return NULL; + } + + cache_entry_t *pos = &cache_table; + + while (pos != NULL) + { + if (pos->original_url && osStrcmp(pos->original_url, url) == 0) + { + TRACE_ERROR("Cache entry found for URL: %s\n", url); + cache_fetch_entry(pos); + return pos; + } + + pos = pos->next; + } + + TRACE_ERROR("No cache entry found for URL: %s\n", url); + return NULL; +} + +cache_entry_t *cache_fetch_by_cached_url(const char *cached_url) +{ + if (cached_url == NULL) + { + TRACE_ERROR("Error: cached_url is NULL\n"); + return NULL; + } + + /* Find the position of "/cache/" in the URL */ + const char *cache_pos = osStrstr(cached_url, "/cache/"); + if (!cache_pos) + { + TRACE_ERROR("Error: '/cache/' not found in cached URL: %s\n", cached_url); + return NULL; + } + + cache_pos += osStrlen("/cache/"); + + if (osStrlen(cache_pos) < 8) + { + TRACE_ERROR("Error: cached URL hash is too short in URL: %s\n", cached_url); + return NULL; + } + + /* Extract the first 8 characters of the hash from the URL */ + char hash_str[9] = {0}; + osStrncpy(hash_str, cache_pos, 8); + + /* Convert the extracted hash part to a uint32_t */ + uint32_t hash_from_url = (uint32_t)osStrtoul(hash_str, NULL, 16); + + cache_entry_t *pos = &cache_table; + + while (pos != NULL) + { + if (pos->hash == hash_from_url) + { + TRACE_INFO("Hash match found for hash: %08X. Checking full cached URL...\n", hash_from_url); + + /* Compare the full cached URL */ + if (strcmp(pos->cached_url, cached_url) == 0) + { + TRACE_INFO("Full cached URL match found for URL: %s\n", cached_url); + cache_fetch_entry(pos); + return pos; + } + } + + pos = pos->next; + } + + TRACE_INFO("No cache entry found for hash: %08X in cached URL: %s\n", hash_from_url, cached_url); + return NULL; +} + +cache_entry_t *cache_fetch_by_uri(const char *uri) +{ + if (uri == NULL) + { + TRACE_ERROR("Error: URI is NULL\n"); + return NULL; + } + + // Find the position of "/cache/" in the URI + const char *cache_pos = strstr(uri, "/cache/"); + if (!cache_pos) + { + TRACE_ERROR("Error: '/cache/' not found in URI: %s\n", uri); + return NULL; + } + + // Move the pointer to the start of the hash part + cache_pos += osStrlen("/cache/"); + + // Ensure that the hash part exists and has enough characters + if (osStrlen(cache_pos) < 8) // 4 bytes of hash = 8 hex characters + { + TRACE_ERROR("Error: URI hash is too short in URI: %s\n", uri); + return NULL; + } + + // Extract the first 8 characters of the hash from the URI + char hash_str[9] = {0}; // 8 characters + 1 for null terminator + osStrncpy(hash_str, cache_pos, 8); + + // Convert the extracted hash part to a uint32_t + uint32_t hash_from_uri = (uint32_t)osStrtoul(hash_str, NULL, 16); + + cache_entry_t *pos = &cache_table; + + while (pos != NULL) + { + if (pos->hash == hash_from_uri) + { + TRACE_DEBUG("Hash match found for hash: %08X. Checking full URI...\n", hash_from_uri); + + // Compare the path "/cache/[hash].[ext]" in the URI + if (osStrstr(pos->cached_url, cache_pos) != NULL) + { + TRACE_DEBUG("Full URI match found for URI: %s\n", uri); + cache_fetch_entry(pos); + return pos; + } + } + + pos = pos->next; + } + + TRACE_ERROR("No cache entry found for URI: %s\n", uri); + return NULL; +} \ No newline at end of file diff --git a/src/server.c b/src/server.c index afcf919e..84c75fcd 100644 --- a/src/server.c +++ b/src/server.c @@ -40,6 +40,7 @@ #include "core/net.h" // for ipStringToAddr, IpAddr #include "core/socket.h" // for _Socket #include "web.h" // for web_download +#include "cache.h" // for image cache functions #include "debug.h" // for TRACE_DEBUG, TRACE_ERROR, TRACE_INFO #include "error.h" // for NO_ERROR, error2text, ERROR_FAILURE #include "fs_port_posix.h" // for fsDirExists @@ -159,62 +160,20 @@ request_type_t request_paths[] = { error_t handleCacheDownload(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx) { - const char *cachePath = get_settings()->internal.cachedirfull; - if (cachePath == NULL || !fsDirExists(cachePath)) + cache_entry_t *entry = cache_fetch_by_uri(uri); + if (!entry) { - char message[128]; - osSnprintf(message, sizeof(message), "core.cachedirfull not set to a valid path: '%s'", cachePath); - TRACE_ERROR("%s\r\n", message); - + TRACE_ERROR("Failed to find cache entry\r\n"); return ERROR_NOT_FOUND; } - const char *fileName = strrchr(uri, '/'); - if (!fileName) + if (!entry->exists) { - char message[128]; - osSnprintf(message, sizeof(message), "file not found in: '%s'", uri); - TRACE_ERROR("%s\r\n", message); - - return ERROR_NOT_FOUND; + TRACE_INFO("Failed, redirecting instead\r\n"); + return httpSendRedirectResponse(connection, 301, entry->original_url); } - char *filePath = custom_asprintf("%s%c%s", cachePath, PATH_SEPARATOR, fileName); - - /* when the file requested does not exist, check for the .url file with the original URL */ - if (!fsFileExists(filePath)) - { - char destUrl[256] = {0}; - size_t destUrlLen = 0; - - /* if it exists, download the file */ - char *urlFilename = custom_asprintf("%s%c%s.url", cachePath, PATH_SEPARATOR, fileName); - FsFile *urlFile = fsOpenFile(urlFilename, FS_FILE_MODE_READ); - if (!urlFile) - { - TRACE_ERROR("failed to open URL file '%s'\r\n", urlFilename); - osFreeMem(urlFilename); - return ERROR_NOT_FOUND; - } - fsReadFile(urlFile, (void *)destUrl, sizeof(destUrl) - 1, &destUrlLen); - destUrl[destUrlLen] = 0; - fsCloseFile(urlFile); - osFreeMem(urlFilename); - - TRACE_INFO("Download image on-demand from '%s'\r\n", destUrl); - error_t err = web_download(destUrl, filePath); - - if (err != NO_ERROR) - { - TRACE_INFO("Failed, redirecting instead\r\n"); - return httpSendRedirectResponse(connection, 301, destUrl); - } - } - - /* either that file existed or was downloaded. return it. */ - - error_t err = httpSendResponseUnsafe(connection, uri, filePath); - osFreeMem(filePath); + error_t err = httpSendResponseUnsafe(connection, uri, entry->file_path); return err; } diff --git a/src/toniesJson.c b/src/toniesJson.c index 3130206e..256a4c23 100644 --- a/src/toniesJson.c +++ b/src/toniesJson.c @@ -3,6 +3,7 @@ #include "fs_port.h" #include "os_port.h" #include "settings.h" +#include "cache.h" #include "debug.h" #include "cJSON.h" #include "handler.h" @@ -374,94 +375,20 @@ void tonies_readJson(char *source, toniesJson_item_t **retCache, size_t *retCoun if (osStrlen(pic_link) > 0 && settings_get_bool("tonie_json.cache_images")) { - const char *cachePath = get_settings()->internal.cachedirfull; - if (cachePath == NULL || !fsDirExists(cachePath)) - { - char message[128]; - osSnprintf(message, sizeof(message), "core.cachedirfull not set to a valid path: '%s'", cachePath); - TRACE_ERROR("%s\r\n", message); - } - else - { - uint8_t sha256_calc[32]; - char sha256_calc_str[65]; - - /* hash the image URL */ - Sha256Context ctx; - sha256Init(&ctx); - sha256Update(&ctx, pic_link, strlen(pic_link)); - sha256Final(&ctx, sha256_calc); - - for (int pos = 0; pos < 32; pos++) - { - osSprintf(&sha256_calc_str[2 * pos], "%02X", sha256_calc[pos]); - } + cache_entry_t *cache = cache_add(pic_link); - /* Find the file extension from the URL */ - const char *ext_pos = strrchr(pic_link, '.'); - char *extension = strdup("jpg"); - - if (ext_pos && !osStrchr(ext_pos, '/')) - { - osFreeMem(extension); - extension = strdup(&ext_pos[1]); - - /* Remove optional HTTP GET parameters */ - char *query_param = osStrchr(extension, '?'); - if (query_param) - { - *query_param = '\0'; - } - } - char *cached_filename = custom_asprintf("%s%c%s.%s", cachePath, PATH_SEPARATOR, sha256_calc_str, extension); - char *cached_url = custom_asprintf("%s%ccache%c%s.%s", settings_get_string("core.host_url"), PATH_SEPARATOR, PATH_SEPARATOR, sha256_calc_str, extension); - - osFreeMem(extension); + if (cache) + { + osFreeMem(pic_link); + pic_link = strdup(cache->cached_url); - // TRACE_INFO("Original URL: '%s'\r\n", pic_link); - // TRACE_INFO("Cache filename would be: '%s'\r\n", cached_filename); - TRACE_INFO("Cache URL would be: '%s'\r\n", cached_url); + TRACE_DEBUG("Cache URL would be: '%s'\r\n", cache->cached_url); - /* check if it is already cached */ - if (fsFileExists(cached_filename)) - { - // TRACE_INFO("File exists, not downloading\r\n"); - osFreeMem(pic_link); - pic_link = strdup(cached_url); - } - else + if (settings_get_bool("tonie_json.cache_preload")) { - if (settings_get_bool("tonie_json.cache_preload")) - { - /* try to download and cache the file */ - // TRACE_INFO("Download file from original URL\r\n"); - - error_t err = web_download(pic_link, cached_filename); - if (err == NO_ERROR) - { - osFreeMem(pic_link); - pic_link = strdup(cached_url); - } - } - else - { - // TRACE_INFO("Link to original URL\r\n"); - char *url_filename = custom_asprintf("%s.url", cached_filename); - FsFile *url_file = fsOpenFile(url_filename, FS_FILE_MODE_WRITE | FS_FILE_MODE_TRUNC); - if (!url_file) - { - TRACE_ERROR("Failed to open file for writing %s\r\n", url_filename); - } - else - { - fsWriteFile(url_file, (void *)pic_link, osStrlen(pic_link)); - fsCloseFile(url_file); - } - osFreeMem(url_filename); - } + /* try to download and cache the file */ + cache_fetch_entry(cache); } - osFreeMem(cached_filename); - osFreeMem(cached_url); } } item->picture = pic_link;