Merge branch 'develop' of https://github.com/toniebox-reverse-enginee…

…ring/teddycloud into develop

DockerfileAlpine

@@ -30,7 +30,8 @@ RUN mkdir -p /teddycloud/certs \
     && mkdir -p /teddycloud/data/content/default \
     && mkdir -p /teddycloud/data/library \
     && mkdir -p /teddycloud/data/firmware  \
-    && mkdir -p /teddycloud/data/www   \
+    && mkdir -p /teddycloud/data/www \
+    && mkdir -p /teddycloud/data/cache \
     && mkdir -p /tmp
 
 RUN ln -s /teddycloud /etc/teddycloud
@@ -47,6 +48,7 @@ VOLUME \
     "/teddycloud/data/content" \
     "/teddycloud/data/library" \
     "/teddycloud/data/firmware" \
+    "/teddycloud/data/cache" \
     "/teddycloud/certs" \
     "/teddycloud/config"
 

DockerfileDebian

@@ -24,7 +24,8 @@ RUN mkdir -p /teddycloud/certs \
     && mkdir -p /teddycloud/data/content/default \
     && mkdir -p /teddycloud/data/library \
     && mkdir -p /teddycloud/data/firmware  \
-    && mkdir -p /teddycloud/data/www   \
+    && mkdir -p /teddycloud/data/www \
+    && mkdir -p /teddycloud/data/cache \
     && mkdir -p /tmp
 
 RUN ln -s /teddycloud /etc/teddycloud
@@ -45,6 +46,7 @@ VOLUME \
     "/teddycloud/data/content" \
     "/teddycloud/data/library" \
     "/teddycloud/data/firmware" \
+    "/teddycloud/data/cache" \
     "/teddycloud/certs" \
     "/teddycloud/config"
 

DockerfileUbuntu

@@ -24,7 +24,8 @@ RUN mkdir -p /teddycloud/certs \
     && mkdir -p /teddycloud/data/content/default \
     && mkdir -p /teddycloud/data/library \
     && mkdir -p /teddycloud/data/firmware  \
-    && mkdir -p /teddycloud/data/www   \
+    && mkdir -p /teddycloud/data/www \
+    && mkdir -p /teddycloud/data/cache \
     && mkdir -p /tmp
 
 RUN ln -s /teddycloud /etc/teddycloud
@@ -45,6 +46,7 @@ VOLUME \
     "/teddycloud/data/content" \
     "/teddycloud/data/library" \
     "/teddycloud/data/firmware" \
+    "/teddycloud/data/cache" \
     "/teddycloud/certs" \
     "/teddycloud/config"
 

Makefile

@@ -460,7 +460,7 @@ endif
 $(LINK_LO_FILE): $$(dir $$@)
 	$(file >$@, $(OBJECTS) $(OBJ_ONLY_FILES) )
 
-workdirs: certs/server/ certs/client/ config/ data/www/ data/content/ data/library/ data/www/web/ data/firmware/
+workdirs: certs/server/ certs/client/ config/ data/www/ data/content/ data/library/ data/www/web/ data/firmware/ data/cache/
 	$(QUIET)$(ECHO) '[ ${YELLOW}DIRS${NC}  ] ${CYAN}$@${NC}'
 	$(QUIET)$(CP_R) $(subst /,$(SEP),$(CONTRIB_DIR)/data/www/*) $(subst /,$(SEP),data/www/) 
 

include/settings.h

@@ -307,6 +307,7 @@ typedef struct
 typedef struct
 {
     bool cache_images;
+    bool cache_preload;
 } settings_tonie_json_t;
 
 typedef struct

include/web.h

@@ -0,0 +1,16 @@
+
+#include "error.h"
+
+/**
+ * @brief Download a file from the specified URL to a local file.
+ *
+ * This function downloads a file from a given URL and saves it to a specified local file.
+ * It handles both HTTP and HTTPS protocols based on the URL scheme and uses secure connections
+ * when required.
+ *
+ * @param[in] url      The URL of the file to download. The URL should be a valid HTTP or HTTPS URL.
+ * @param[in] filename The path to the local file where the downloaded content will be saved.
+ *
+ * @return NO_ERROR if the download was successful, or an appropriate error code otherwise.
+ */
+error_t web_download(const char *url, const char *filename);

src/server.c

@@ -36,6 +36,35 @@
 #include "toniebox_state.h"       // for get_toniebox_state, get_toniebox_s...
 #include "toniebox_state_type.h"  // for toniebox_state_box_t, toniebox_sta...
 #include "toniesJson.h"           // for tonieboxes_update, tonies_deinit
+#include "compiler_port.h"        // for char_t, PRIuTIME
+#include "core/net.h"             // for ipStringToAddr, IpAddr
+#include "core/socket.h"          // for _Socket
+#include "web.h"                  // for web_download
+#include "debug.h"                // for TRACE_DEBUG, TRACE_ERROR, TRACE_INFO
+#include "error.h"                // for NO_ERROR, error2text, ERROR_FAILURE
+#include "fs_port_posix.h"        // for fsDirExists
+#include "handler_api.h"          // for handleApiAssignUnknown, handleApiA...
+#include "handler_cloud.h"        // for handleCloudClaim, handleCloudConte...
+#include "handler_reverse.h"      // for handleReverse
+#include "handler_rtnl.h"         // for handleRtnl
+#include "handler_security_mit.h" // for handleSecMitRobotsTxt, checkSecMit...
+#include "handler_sse.h"          // for handleApiSse, sse_init
+#include "http/http_common.h"     // for HTTP_AUTH_MODE_DIGEST
+#include "http/http_server.h"     // for _HttpConnection, HttpServerSettings
+#include "mutex_manager.h"        // for mutex_unlock, mutex_lock, MUTEX_CL...
+#include "net_config.h"           // for client_ctx_t, http_connection_priv...
+#include "os_port.h"              // for osFreeMem, osStrlen, osStrstr, osG...
+#include "pcaplog.h"              // for pcaplog_close, pcaplog_open
+#include "rand.h"                 // for rand_get_algo, rand_get_context
+#include "returncodes.h"          // for RETURNCODE_INVALID_CONFIG
+#include "server_helpers.h"       // for httpServerUriNotFoundCallback, cus...
+#include "settings.h"             // for settings_t, settings_get_string
+#include "stdbool.h"              // for true, bool, false
+#include "tls.h"                  // for _TlsContext, tlsLoadCertificate
+#include "tls_adapter.h"          // for tls_context_key_log_init, tlsCache
+#include "toniebox_state.h"       // for get_toniebox_state, get_toniebox_s...
+#include "toniebox_state_type.h"  // for toniebox_state_box_t, toniebox_sta...
+#include "toniesJson.h"           // for tonieboxes_update, tonies_deinit
 
 #define APP_HTTP_MAX_CONNECTIONS 32
 HttpConnection httpConnections[APP_HTTP_MAX_CONNECTIONS];
@@ -67,12 +96,16 @@ typedef struct
     error_t (*handler)(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx);
 } request_type_t;
 
+error_t handleCacheDownload(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx);
+
 /* const for now. later maybe dynamic? */
 request_type_t request_paths[] = {
     /*binary handler (rtnl)*/
     {REQ_ANY, "*binary", SERTY_BOTH, &handleRtnl},
     /* reverse proxy handler */
     {REQ_ANY, "/reverse", SERTY_WEB, &handleReverse},
+    /* cached files */
+    {REQ_GET, "/cache/", SERTY_WEB, &handleCacheDownload},
     /* web interface directory */
     {REQ_GET, "/content/download/", SERTY_WEB, &handleApiContentDownload},
     {REQ_GET, "/content/json/get/", SERTY_WEB, &handleApiContentJsonGet},
@@ -124,6 +157,67 @@ request_type_t request_paths[] = {
     {REQ_POST, "/v1/log", SERTY_BOTH, &handleCloudLog},
     {REQ_POST, "/v1/cloud-reset", SERTY_BOTH, &handleCloudReset}};
 
+error_t handleCacheDownload(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx)
+{
+    const char *cachePath = get_settings()->internal.cachedirfull;
+    if (cachePath == NULL || !fsDirExists(cachePath))
+    {
+        char message[128];
+        osSnprintf(message, sizeof(message), "core.cachedirfull not set to a valid path: '%s'", cachePath);
+        TRACE_ERROR("%s\r\n", message);
+
+        return ERROR_NOT_FOUND;
+    }
+    const char *fileName = strrchr(uri, '/');
+
+    if (!fileName)
+    {
+        char message[128];
+        osSnprintf(message, sizeof(message), "file not found in: '%s'", uri);
+        TRACE_ERROR("%s\r\n", message);
+
+        return ERROR_NOT_FOUND;
+    }
+
+    char *filePath = custom_asprintf("%s%c%s", cachePath, PATH_SEPARATOR, fileName);
+
+    /* when the file requested does not exist, check for the .url file with the original URL */
+    if (!fsFileExists(filePath))
+    {
+        char destUrl[256] = {0};
+        size_t destUrlLen = 0;
+
+        /* if it exists, download the file */
+        char *urlFilename = custom_asprintf("%s%c%s.url", cachePath, PATH_SEPARATOR, fileName);
+        FsFile *urlFile = fsOpenFile(urlFilename, FS_FILE_MODE_READ);
+        if (!urlFile)
+        {
+            TRACE_ERROR("failed to open URL file '%s'\r\n", urlFilename);
+            osFreeMem(urlFilename);
+            return ERROR_NOT_FOUND;
+        }
+        fsReadFile(urlFile, (void *)destUrl, sizeof(destUrl) - 1, &destUrlLen);
+        destUrl[destUrlLen] = 0;
+        fsCloseFile(urlFile);
+        osFreeMem(urlFilename);
+
+        TRACE_INFO("Download image on-demand from '%s'\r\n", destUrl);
+        error_t err = web_download(destUrl, filePath);
+
+        if (err != NO_ERROR)
+        {
+            TRACE_INFO("Failed, redirecting instead\r\n");
+            return httpSendRedirectResponse(connection, 301, destUrl);
+        }
+    }
+
+    /* either that file existed or was downloaded. return it. */
+
+    error_t err = httpSendResponseUnsafe(connection, uri, filePath);
+    osFreeMem(filePath);
+    return err;
+}
+
 error_t resGetData(const char_t *path, const uint8_t **data, size_t *length)
 {
     TRACE_DEBUG("resGetData: %s (static response)\n", path);

src/settings.c

@@ -266,7 +266,8 @@ static void option_map_init(uint8_t settingsId)
     OPTION_STRING("hass.id", &settings->hass.id, "teddyCloud_Server", "Unique ID", "Unique ID to identify this device", LEVEL_DETAIL)
 
     OPTION_TREE_DESC("tonie_json", "Tonie JSON", LEVEL_DETAIL)
-    OPTION_BOOL("tonie_json.cache_images", &settings->tonie_json.cache_images, FALSE, "Cache images", "Download and cache figurine images", LEVEL_DETAIL)
+    OPTION_BOOL("tonie_json.cache_images", &settings->tonie_json.cache_images, FALSE, "Cache images", "Cache figurine images locally", LEVEL_DETAIL)
+    OPTION_BOOL("tonie_json.cache_preload", &settings->tonie_json.cache_preload, FALSE, "Preload all images", "Download all figurine images on startup", LEVEL_DETAIL)
 
     OPTION_TREE_DESC("debug", "Debug", LEVEL_EXPERT)
     OPTION_BOOL("debug.web.pcm_encode_console_url", &settings->debug.web.pcm_encode_console_url, FALSE, "PCM Console URL", "Caches the PCM of the browser-side encoding and prints a download link to the browser console.", LEVEL_EXPERT)

src/toniesJson.c

@@ -1,4 +1,5 @@
 #include "toniesJson.h"
+#include "web.h"
 #include "fs_port.h"
 #include "os_port.h"
 #include "settings.h"
@@ -412,37 +413,52 @@ void tonies_readJson(char *source, toniesJson_item_t **retCache, size_t *retCoun
                                 *query_param = '\0';
                             }
                         }
-                        char *cached_filename = custom_asprintf("%s/%s.%s", cachePath, sha256_calc_str, extension);
-                        char *cached_url = custom_asprintf("%s/cache/%s", settings_get_string("core.host_url"), sha256_calc_str);
+                        char *cached_filename = custom_asprintf("%s%c%s.%s", cachePath, PATH_SEPARATOR, sha256_calc_str, extension);
+                        char *cached_url = custom_asprintf("%s%ccache%c%s.%s", settings_get_string("core.host_url"), PATH_SEPARATOR, PATH_SEPARATOR, sha256_calc_str, extension);
 
                         osFreeMem(extension);
 
-                        TRACE_INFO("Original URL: '%s'\r\n", pic_link);
-                        TRACE_INFO("Cache filename would be: '%s'\r\n", cached_filename);
+                        // TRACE_INFO("Original URL: '%s'\r\n", pic_link);
+                        // TRACE_INFO("Cache filename would be: '%s'\r\n", cached_filename);
                         TRACE_INFO("Cache URL would be: '%s'\r\n", cached_url);
 
                         /* check if it is already cached */
                         if (fsFileExists(cached_filename))
                         {
-                            TRACE_INFO("File exists, not downloading\r\n");
+                            // TRACE_INFO("File exists, not downloading\r\n");
                             osFreeMem(pic_link);
                             pic_link = strdup(cached_url);
                         }
                         else
                         {
-                            /* if not, try to download and cache the file */
-                            TRACE_INFO("Download file from original URL -> not implemented yet\r\n");
-
-                            /*
-                            fsFile = fsOpenFile(cached_filename, FS_FILE_MODE_WRITE);
-                            if (fsFile != NULL)
+                            if (settings_get_bool("tonie_json.cache_preload"))
+                            {
+                                /* try to download and cache the file */
+                                // TRACE_INFO("Download file from original URL\r\n");
+
+                                error_t err = web_download(pic_link, cached_filename);
+                                if (err == NO_ERROR)
+                                {
+                                    osFreeMem(pic_link);
+                                    pic_link = strdup(cached_url);
+                                }
+                            }
+                            else
                             {
-                                fsWriteFile(fsFile, " ", 1);
-                                fsCloseFile(fsFile);
-                                osFreeMem(pic_link);
-                                pic_link = strdup(cached_url);
+                                // TRACE_INFO("Link to original URL\r\n");
+                                char *url_filename = custom_asprintf("%s.url", cached_filename);
+                                FsFile *url_file = fsOpenFile(url_filename, FS_FILE_MODE_WRITE | FS_FILE_MODE_TRUNC);
+                                if (!url_file)
+                                {
+                                    TRACE_ERROR("Failed to open file for writing %s\r\n", url_filename);
+                                }
+                                else
+                                {
+                                    fsWriteFile(url_file, (void *)pic_link, osStrlen(pic_link));
+                                    fsCloseFile(url_file);
+                                }
+                                osFreeMem(url_filename);
                             }
-                            */
                         }
                         osFreeMem(cached_filename);
                         osFreeMem(cached_url);