From 8264e662a11eed0526a11c8296afa64dd5ca2305 Mon Sep 17 00:00:00 2001 From: g3gg0 Date: Tue, 13 Aug 2024 01:00:29 +0200 Subject: [PATCH] added on-demand picture downloading --- include/settings.h | 1 + include/web.h | 16 ++++ src/server.c | 138 +++++++++++++++++++++++--------- src/settings.c | 3 +- src/toniesJson.c | 48 ++++++++---- src/web.c | 191 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 344 insertions(+), 53 deletions(-) create mode 100644 include/web.h create mode 100644 src/web.c diff --git a/include/settings.h b/include/settings.h index c533477f..fa01c450 100644 --- a/include/settings.h +++ b/include/settings.h @@ -305,6 +305,7 @@ typedef struct typedef struct { bool cache_images; + bool cache_preload; } settings_tonie_json_t; typedef struct diff --git a/include/web.h b/include/web.h new file mode 100644 index 00000000..44f1207d --- /dev/null +++ b/include/web.h @@ -0,0 +1,16 @@ + +#include "error.h" + +/** + * @brief Download a file from the specified URL to a local file. + * + * This function downloads a file from a given URL and saves it to a specified local file. + * It handles both HTTP and HTTPS protocols based on the URL scheme and uses secure connections + * when required. + * + * @param[in] url The URL of the file to download. The URL should be a valid HTTP or HTTPS URL. + * @param[in] filename The path to the local file where the downloaded content will be saved. + * + * @return NO_ERROR if the download was successful, or an appropriate error code otherwise. + */ +error_t web_download(const char *url, const char *filename); diff --git a/src/server.c b/src/server.c index 2c1ae5fb..00cd1240 100644 --- a/src/server.c +++ b/src/server.c @@ -1,41 +1,42 @@ -#include // for error_t -#include // for uint8_t -#include // for printf -#include // for atoi, exit, free -#include // for NULL, strdup, strlen, strncmp, strcmp -#include // for time_t -#include // for time - -#include "compiler_port.h" // for char_t, PRIuTIME -#include "core/net.h" // for ipStringToAddr, IpAddr -#include "core/socket.h" // for _Socket -#include "debug.h" // for TRACE_DEBUG, TRACE_ERROR, TRACE_INFO -#include "error.h" // for NO_ERROR, error2text, ERROR_FAILURE -#include "fs_port_posix.h" // for fsDirExists -#include "handler_api.h" // for handleApiAssignUnknown, handleApiA... -#include "handler_cloud.h" // for handleCloudClaim, handleCloudConte... -#include "handler_reverse.h" // for handleReverse -#include "handler_rtnl.h" // for handleRtnl -#include "handler_security_mit.h" // for handleSecMitRobotsTxt, checkSecMit... -#include "handler_sse.h" // for handleApiSse, sse_init -#include "http/http_common.h" // for HTTP_AUTH_MODE_DIGEST -#include "http/http_server.h" // for _HttpConnection, HttpServerSettings -#include "mutex_manager.h" // for mutex_unlock, mutex_lock, MUTEX_CL... -#include "net_config.h" // for client_ctx_t, http_connection_priv... -#include "os_port.h" // for osFreeMem, osStrlen, osStrstr, osG... -#include "pcaplog.h" // for pcaplog_close, pcaplog_open -#include "rand.h" // for rand_get_algo, rand_get_context -#include "returncodes.h" // for RETURNCODE_INVALID_CONFIG -#include "server_helpers.h" // for httpServerUriNotFoundCallback, cus... -#include "settings.h" // for settings_t, settings_get_string -#include "stdbool.h" // for true, bool, false -#include "tls.h" // for _TlsContext, tlsLoadCertificate -#include "tls_adapter.h" // for tls_context_key_log_init, tlsCache -#include "toniebox_state.h" // for get_toniebox_state, get_toniebox_s... -#include "toniebox_state_type.h" // for toniebox_state_box_t, toniebox_sta... -#include "toniesJson.h" // for tonieboxes_update, tonies_deinit +#include // for error_t +#include // for uint8_t +#include // for printf +#include // for atoi, exit, free +#include // for NULL, strdup, strlen, strncmp, strcmp +#include // for time_t +#include // for time + +#include "compiler_port.h" // for char_t, PRIuTIME +#include "core/net.h" // for ipStringToAddr, IpAddr +#include "core/socket.h" // for _Socket +#include "web.h" // for web_download +#include "debug.h" // for TRACE_DEBUG, TRACE_ERROR, TRACE_INFO +#include "error.h" // for NO_ERROR, error2text, ERROR_FAILURE +#include "fs_port_posix.h" // for fsDirExists +#include "handler_api.h" // for handleApiAssignUnknown, handleApiA... +#include "handler_cloud.h" // for handleCloudClaim, handleCloudConte... +#include "handler_reverse.h" // for handleReverse +#include "handler_rtnl.h" // for handleRtnl +#include "handler_security_mit.h" // for handleSecMitRobotsTxt, checkSecMit... +#include "handler_sse.h" // for handleApiSse, sse_init +#include "http/http_common.h" // for HTTP_AUTH_MODE_DIGEST +#include "http/http_server.h" // for _HttpConnection, HttpServerSettings +#include "mutex_manager.h" // for mutex_unlock, mutex_lock, MUTEX_CL... +#include "net_config.h" // for client_ctx_t, http_connection_priv... +#include "os_port.h" // for osFreeMem, osStrlen, osStrstr, osG... +#include "pcaplog.h" // for pcaplog_close, pcaplog_open +#include "rand.h" // for rand_get_algo, rand_get_context +#include "returncodes.h" // for RETURNCODE_INVALID_CONFIG +#include "server_helpers.h" // for httpServerUriNotFoundCallback, cus... +#include "settings.h" // for settings_t, settings_get_string +#include "stdbool.h" // for true, bool, false +#include "tls.h" // for _TlsContext, tlsLoadCertificate +#include "tls_adapter.h" // for tls_context_key_log_init, tlsCache +#include "toniebox_state.h" // for get_toniebox_state, get_toniebox_s... +#include "toniebox_state_type.h" // for toniebox_state_box_t, toniebox_sta... +#include "toniesJson.h" // for tonieboxes_update, tonies_deinit #define APP_HTTP_MAX_CONNECTIONS 32 HttpConnection httpConnections[APP_HTTP_MAX_CONNECTIONS]; @@ -66,12 +67,16 @@ typedef struct error_t (*handler)(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx); } request_type_t; +error_t handleCacheDownload(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx); + /* const for now. later maybe dynamic? */ request_type_t request_paths[] = { /*binary handler (rtnl)*/ {REQ_ANY, "*binary", SERTY_BOTH, &handleRtnl}, /* reverse proxy handler */ {REQ_ANY, "/reverse", SERTY_HTTP, &handleReverse}, + /* cached files */ + {REQ_GET, "/cache/", SERTY_HTTP, &handleCacheDownload}, /* web interface directory */ {REQ_GET, "/content/download/", SERTY_HTTP, &handleApiContentDownload}, {REQ_GET, "/content/json/get/", SERTY_HTTP, &handleApiContentJsonGet}, @@ -123,6 +128,67 @@ request_type_t request_paths[] = { {REQ_POST, "/v1/log", SERTY_BOTH, &handleCloudLog}, {REQ_POST, "/v1/cloud-reset", SERTY_BOTH, &handleCloudReset}}; +error_t handleCacheDownload(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx) +{ + const char *cachePath = get_settings()->internal.cachedirfull; + if (cachePath == NULL || !fsDirExists(cachePath)) + { + char message[128]; + osSnprintf(message, sizeof(message), "core.cachedirfull not set to a valid path: '%s'", cachePath); + TRACE_ERROR("%s\r\n", message); + + return ERROR_NOT_FOUND; + } + const char *fileName = strrchr(uri, '/'); + + if (!fileName) + { + char message[128]; + osSnprintf(message, sizeof(message), "file not found in: '%s'", uri); + TRACE_ERROR("%s\r\n", message); + + return ERROR_NOT_FOUND; + } + + char *filePath = custom_asprintf("%s%c%s", cachePath, PATH_SEPARATOR, fileName); + + /* when the file requested does not exist, check for the .url file with the original URL */ + if (!fsFileExists(filePath)) + { + char destUrl[256] = {0}; + size_t destUrlLen = 0; + + /* if it exists, download the file */ + char *urlFilename = custom_asprintf("%s%c%s.url", cachePath, PATH_SEPARATOR, fileName); + FsFile *urlFile = fsOpenFile(urlFilename, FS_FILE_MODE_READ); + if (!urlFile) + { + TRACE_ERROR("failed to open URL file '%s'\r\n", urlFilename); + osFreeMem(urlFilename); + return ERROR_NOT_FOUND; + } + fsReadFile(urlFile, (void *)destUrl, sizeof(destUrl) - 1, &destUrlLen); + destUrl[destUrlLen] = 0; + fsCloseFile(urlFile); + osFreeMem(urlFilename); + + TRACE_INFO("Download image on-demand from '%s'\r\n", destUrl); + error_t err = web_download(destUrl, filePath); + + if (err != NO_ERROR) + { + TRACE_INFO("Failed, redirecting instead\r\n"); + return httpSendRedirectResponse(connection, 301, destUrl); + } + } + + /* either that file existed or was downloaded. return it. */ + + error_t err = httpSendResponseUnsafe(connection, uri, filePath); + osFreeMem(filePath); + return err; +} + error_t resGetData(const char_t *path, const uint8_t **data, size_t *length) { TRACE_DEBUG("resGetData: %s (static response)\n", path); diff --git a/src/settings.c b/src/settings.c index fe0c2386..c5c98a68 100644 --- a/src/settings.c +++ b/src/settings.c @@ -265,7 +265,8 @@ static void option_map_init(uint8_t settingsId) OPTION_STRING("hass.id", &settings->hass.id, "teddyCloud_Server", "Unique ID", "Unique ID to identify this device", LEVEL_DETAIL) OPTION_TREE_DESC("tonie_json", "Tonie JSON", LEVEL_DETAIL) - OPTION_BOOL("tonie_json.cache_images", &settings->tonie_json.cache_images, FALSE, "Cache images", "Download and cache figurine images", LEVEL_DETAIL) + OPTION_BOOL("tonie_json.cache_images", &settings->tonie_json.cache_images, TRUE, "Cache images", "Cache figurine images locally", LEVEL_DETAIL) + OPTION_BOOL("tonie_json.cache_preload", &settings->tonie_json.cache_preload, FALSE, "Preload all images", "Download all figurine images on startup", LEVEL_DETAIL) OPTION_END() settings_size = sizeof(option_map_array) / sizeof(option_map_array[0]) - 1; diff --git a/src/toniesJson.c b/src/toniesJson.c index 979f5470..3130206e 100644 --- a/src/toniesJson.c +++ b/src/toniesJson.c @@ -1,4 +1,5 @@ #include "toniesJson.h" +#include "web.h" #include "fs_port.h" #include "os_port.h" #include "settings.h" @@ -412,37 +413,52 @@ void tonies_readJson(char *source, toniesJson_item_t **retCache, size_t *retCoun *query_param = '\0'; } } - char *cached_filename = custom_asprintf("%s/%s.%s", cachePath, sha256_calc_str, extension); - char *cached_url = custom_asprintf("%s/cache/%s.%s", settings_get_string("core.host_url"), sha256_calc_str, extension); + char *cached_filename = custom_asprintf("%s%c%s.%s", cachePath, PATH_SEPARATOR, sha256_calc_str, extension); + char *cached_url = custom_asprintf("%s%ccache%c%s.%s", settings_get_string("core.host_url"), PATH_SEPARATOR, PATH_SEPARATOR, sha256_calc_str, extension); osFreeMem(extension); - TRACE_INFO("Original URL: '%s'\r\n", pic_link); - TRACE_INFO("Cache filename would be: '%s'\r\n", cached_filename); + // TRACE_INFO("Original URL: '%s'\r\n", pic_link); + // TRACE_INFO("Cache filename would be: '%s'\r\n", cached_filename); TRACE_INFO("Cache URL would be: '%s'\r\n", cached_url); /* check if it is already cached */ if (fsFileExists(cached_filename)) { - TRACE_INFO("File exists, not downloading\r\n"); + // TRACE_INFO("File exists, not downloading\r\n"); osFreeMem(pic_link); pic_link = strdup(cached_url); } else { - /* if not, try to download and cache the file */ - TRACE_INFO("Download file from original URL -> not implemented yet\r\n"); - - /* - fsFile = fsOpenFile(cached_filename, FS_FILE_MODE_WRITE); - if (fsFile != NULL) + if (settings_get_bool("tonie_json.cache_preload")) + { + /* try to download and cache the file */ + // TRACE_INFO("Download file from original URL\r\n"); + + error_t err = web_download(pic_link, cached_filename); + if (err == NO_ERROR) + { + osFreeMem(pic_link); + pic_link = strdup(cached_url); + } + } + else { - fsWriteFile(fsFile, " ", 1); - fsCloseFile(fsFile); - osFreeMem(pic_link); - pic_link = strdup(cached_url); + // TRACE_INFO("Link to original URL\r\n"); + char *url_filename = custom_asprintf("%s.url", cached_filename); + FsFile *url_file = fsOpenFile(url_filename, FS_FILE_MODE_WRITE | FS_FILE_MODE_TRUNC); + if (!url_file) + { + TRACE_ERROR("Failed to open file for writing %s\r\n", url_filename); + } + else + { + fsWriteFile(url_file, (void *)pic_link, osStrlen(pic_link)); + fsCloseFile(url_file); + } + osFreeMem(url_filename); } - */ } osFreeMem(cached_filename); osFreeMem(cached_url); diff --git a/src/web.c b/src/web.c new file mode 100644 index 00000000..35b0ce55 --- /dev/null +++ b/src/web.c @@ -0,0 +1,191 @@ + +#include "fs_port.h" +#include "os_port.h" +#include "settings.h" +#include "handler.h" +#include "cloud_request.h" +#include "debug.h" +#include "core/net.h" +#include "core/ethernet.h" +#include "core/ip.h" +#include "core/tcp.h" +#include "http/http_client.h" +#include "http/http_server.h" +#include "http/http_server_misc.h" + +typedef enum +{ + PROT_HTTP, + PROT_HTTPS +} web_protocol_t; +#define DEFAULT_HTTP_PORT 80 +#define DEFAULT_HTTPS_PORT 443 +#define PORT_MAX 65535 + +bool web_parse_url(const char *url, char **hostname, uint16_t *port, char **uri, web_protocol_t *protocol) +{ + // Check if the URL starts with "http://" or "https://" + if (strncmp(url, "http://", 7) == 0) + { + *protocol = PROT_HTTP; + url += 7; + } + else if (strncmp(url, "https://", 8) == 0) + { + *protocol = PROT_HTTPS; + url += 8; + } + else + { + TRACE_ERROR("Unknown protocol\r\n"); + return false; + } + + // Find the start of the port and path + char *port_start = strchr(url, ':'); + char *path_start = strchr(url, '/'); + + if (path_start == NULL) + { + TRACE_ERROR("URL must contain a path\r\n"); + return false; + } + + // Determine the hostname and port + if (port_start != NULL && port_start < path_start) + { + // Port is specified + size_t hostname_length = port_start - url; + *hostname = (char *)malloc(hostname_length + 1); + if (*hostname == NULL) + { + TRACE_ERROR("Memory allocation error\r\n"); + return false; + } + strncpy(*hostname, url, hostname_length); + (*hostname)[hostname_length] = '\0'; + + // Parse and validate port + long temp = strtol(port_start + 1, &path_start, 10); + if ((temp > 0) && (temp <= PORT_MAX)) + { + *port = (uint16_t)temp; + } + else + { + TRACE_ERROR("Invalid port number\r\n"); + free(*hostname); + return false; + } + } + else + { + // Port is not specified, use default port based on protocol + size_t hostname_length = path_start - url; + *hostname = (char *)malloc(hostname_length + 1); + if (*hostname == NULL) + { + TRACE_ERROR("Memory allocation error\r\n"); + return false; + } + strncpy(*hostname, url, hostname_length); + (*hostname)[hostname_length] = '\0'; + + *port = (*protocol == PROT_HTTP) ? DEFAULT_HTTP_PORT : DEFAULT_HTTPS_PORT; + } + + // Copy the path and query (if any) into the uri + *uri = strdup(path_start); + if (*uri == NULL) + { + TRACE_ERROR("Memory allocation error\r\n"); + free(*hostname); + return false; + } + + return true; +} + +void web_dl_cbr(void *src_ctx, HttpClientContext *cloud_ctx, const char *payload, size_t length, error_t error) +{ + cbr_ctx_t *ctx = (cbr_ctx_t *)src_ctx; + const char *filename = (const char *)ctx->customData; + HttpClientContext *httpClientContext = (HttpClientContext *)cloud_ctx; + + if (httpClientContext->statusCode == 200) + { + if (ctx->file == NULL) + { + TRACE_INFO("Opening file %s\r\n", filename); + ctx->file = fsOpenFile(filename, FS_FILE_MODE_WRITE | FS_FILE_MODE_TRUNC); + if (!ctx->file) + { + TRACE_ERROR("Failed to open file %s\r\n", filename); + return; + } + } + error_t errorWrite = NO_ERROR; + if (length > 0) + { + errorWrite = fsWriteFile(ctx->file, (void *)payload, length); + } + + if (error == ERROR_END_OF_STREAM) + { + fsCloseFile(ctx->file); + } + else if (error != NO_ERROR) + { + fsCloseFile(ctx->file); + TRACE_ERROR("body error=%s\r\n", error2text(error)); + } + if (errorWrite != NO_ERROR) + { + fsCloseFile(ctx->file); + TRACE_ERROR("write error=%s\r\n", error2text(error)); + } + } +} + +error_t web_download(const char *url, const char *filename) +{ + TRACE_INFO("Downloading file from '%s' into local file '%s'\r\n", url, filename); + + char *hostname = NULL; + uint16_t port = 0; + char *uri = NULL; + web_protocol_t protocol; + + // Parse the URL + if (!web_parse_url(url, &hostname, &port, &uri, &protocol)) + { + TRACE_ERROR("Failed to parse URL\r\n"); + return ERROR_INVALID_PARAMETER; + } + + cbr_ctx_t ctx = {0}; + ctx.customData = (void *)filename; + + req_cbr_t cbr = { + .ctx = &ctx, + .body = &web_dl_cbr, + }; + + bool is_secure = (protocol == PROT_HTTPS); + error_t error = web_request(hostname, port, is_secure, uri, NULL, "GET", NULL, 0, NULL, &cbr, false, false); + + if (error == NO_ERROR && fsFileExists(filename)) + { + TRACE_INFO("download successful\r\n"); + } + else + { + TRACE_ERROR("download failed, error=%s\r\n", error2text(error)); + } + + // Free allocated memory + free(hostname); + free(uri); + + return error; +}