From 02ada4d54e39d6eef9bc5dc1cb19dc3e5f284395 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Tue, 3 Dec 2024 12:15:36 +0100 Subject: [PATCH 1/7] Sketch Logger untested sketch for logger hook. ideally we should provide options to pipe into a file and/or insert into a table, etc. --- src/httpserver_extension.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/httpserver_extension.cpp b/src/httpserver_extension.cpp index 88078f7..5e588aa 100644 --- a/src/httpserver_extension.cpp +++ b/src/httpserver_extension.cpp @@ -384,6 +384,20 @@ void HttpServerStart(DatabaseInstance& db, string_t host, int32_t port, string_t string host_str = host.GetString(); + const char* debug_env = std::getenv("DUCKDB_HTTPSERVER_DEBUG"); + if (debug_env != nullptr && std::string(debug_env) == "1") { + global_state.server->set_logger([](const duckdb_httplib_openssl::Request& req, const duckdb_httplib_openssl::Response& res) { + auto now = std::chrono::system_clock::now(); + auto now_time = std::chrono::system_clock::to_time_t(now); + fprintf(stdout, "[%s] %s %s - %d\n", + std::ctime(&now_time), + req.method.c_str(), + req.path.c_str(), + res.status); + fflush(stdout); + }); + } + const char* run_in_same_thread_env = std::getenv("DUCKDB_HTTPSERVER_FOREGROUND"); bool run_in_same_thread = (run_in_same_thread_env != nullptr && std::string(run_in_same_thread_env) == "1"); From 9837b7d30b961ef2e1c1bf384e5248e1a75d39a7 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Tue, 3 Dec 2024 16:19:18 +0100 Subject: [PATCH 2/7] DEBUG or SYSLOG testing --- src/httpserver_extension.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/httpserver_extension.cpp b/src/httpserver_extension.cpp index 5e588aa..5a4cf1c 100644 --- a/src/httpserver_extension.cpp +++ b/src/httpserver_extension.cpp @@ -385,6 +385,8 @@ void HttpServerStart(DatabaseInstance& db, string_t host, int32_t port, string_t string host_str = host.GetString(); const char* debug_env = std::getenv("DUCKDB_HTTPSERVER_DEBUG"); + const char* use_syslog = std::getenv("DUCKDB_HTTPSERVER_SYSLOG"); + if (debug_env != nullptr && std::string(debug_env) == "1") { global_state.server->set_logger([](const duckdb_httplib_openssl::Request& req, const duckdb_httplib_openssl::Response& res) { auto now = std::chrono::system_clock::now(); @@ -396,6 +398,17 @@ void HttpServerStart(DatabaseInstance& db, string_t host, int32_t port, string_t res.status); fflush(stdout); }); + } else if (use_syslog != nullptr && std::string(use_syslog) == "1") { + openlog("duckdb-httpserver", LOG_PID | LOG_NDELAY, LOG_LOCAL0); + global_state.server->set_logger([](const duckdb_httplib_openssl::Request& req, const duckdb_httplib_openssl::Response& res) { + syslog(LOG_INFO, "%s %s - %d", + req.method.c_str(), + req.path.c_str(), + res.status); + }); + std::atexit([]() { + closelog(); + }); } const char* run_in_same_thread_env = std::getenv("DUCKDB_HTTPSERVER_FOREGROUND"); From ee007cc07b66f45eefd759c8a9fbc11952a5717a Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Tue, 3 Dec 2024 16:25:30 +0100 Subject: [PATCH 3/7] syslog --- src/httpserver_extension.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/httpserver_extension.cpp b/src/httpserver_extension.cpp index 5a4cf1c..159c172 100644 --- a/src/httpserver_extension.cpp +++ b/src/httpserver_extension.cpp @@ -9,16 +9,15 @@ #include "duckdb/common/exception/http_exception.hpp" #include "duckdb/common/allocator.hpp" #include +#include +#include +#include +#include #define CPPHTTPLIB_OPENSSL_SUPPORT #include "httplib.hpp" - -// Include yyjson for JSON handling #include "yyjson.hpp" -#include -#include -#include #include "play.h" using namespace duckdb_yyjson; // NOLINT From e5de41d518b66a4598ca4fb1a69baea04d902bb6 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Tue, 3 Dec 2024 16:58:20 +0100 Subject: [PATCH 4/7] No syslog for win32 builds --- src/httpserver_extension.cpp | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/httpserver_extension.cpp b/src/httpserver_extension.cpp index 159c172..bc62388 100644 --- a/src/httpserver_extension.cpp +++ b/src/httpserver_extension.cpp @@ -383,21 +383,11 @@ void HttpServerStart(DatabaseInstance& db, string_t host, int32_t port, string_t string host_str = host.GetString(); - const char* debug_env = std::getenv("DUCKDB_HTTPSERVER_DEBUG"); +#ifndef _WIN32 + // Debug and Syslog Events const char* use_syslog = std::getenv("DUCKDB_HTTPSERVER_SYSLOG"); - - if (debug_env != nullptr && std::string(debug_env) == "1") { - global_state.server->set_logger([](const duckdb_httplib_openssl::Request& req, const duckdb_httplib_openssl::Response& res) { - auto now = std::chrono::system_clock::now(); - auto now_time = std::chrono::system_clock::to_time_t(now); - fprintf(stdout, "[%s] %s %s - %d\n", - std::ctime(&now_time), - req.method.c_str(), - req.path.c_str(), - res.status); - fflush(stdout); - }); - } else if (use_syslog != nullptr && std::string(use_syslog) == "1") { + const char* debug_env = std::getenv("DUCKDB_HTTPSERVER_DEBUG"); + if (use_syslog != nullptr && std::string(use_syslog) == "1") { openlog("duckdb-httpserver", LOG_PID | LOG_NDELAY, LOG_LOCAL0); global_state.server->set_logger([](const duckdb_httplib_openssl::Request& req, const duckdb_httplib_openssl::Response& res) { syslog(LOG_INFO, "%s %s - %d", @@ -408,7 +398,19 @@ void HttpServerStart(DatabaseInstance& db, string_t host, int32_t port, string_t std::atexit([]() { closelog(); }); - } + } else if (debug_env != nullptr && std::string(debug_env) == "1") { + global_state.server->set_logger([](const duckdb_httplib_openssl::Request& req, const duckdb_httplib_openssl::Response& res) { + auto now = std::chrono::system_clock::now(); + auto now_time = std::chrono::system_clock::to_time_t(now); + fprintf(stdout, "[%s] %s %s - %d\n", + std::ctime(&now_time), + req.method.c_str(), + req.path.c_str(), + res.status); + fflush(stdout); + }); + } +#endif const char* run_in_same_thread_env = std::getenv("DUCKDB_HTTPSERVER_FOREGROUND"); bool run_in_same_thread = (run_in_same_thread_env != nullptr && std::string(run_in_same_thread_env) == "1"); From c52f1b500f26ae1418bd8b028dfa41bf776b19dc Mon Sep 17 00:00:00 2001 From: lmangani Date: Tue, 3 Dec 2024 17:03:55 +0000 Subject: [PATCH 5/7] fix debug output --- duckdb | 2 +- src/httpserver_extension.cpp | 46 ++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/duckdb b/duckdb index af39bd0..1986445 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit af39bd0dcf66876e09ac2a7c3baa28fe1b301151 +Subproject commit 19864453f7d0ed095256d848b46e7b8630989bac diff --git a/src/httpserver_extension.cpp b/src/httpserver_extension.cpp index bc62388..12ee33a 100644 --- a/src/httpserver_extension.cpp +++ b/src/httpserver_extension.cpp @@ -12,7 +12,10 @@ #include #include #include + +#ifndef _WIN32 #include +#endif #define CPPHTTPLIB_OPENSSL_SUPPORT #include "httplib.hpp" @@ -383,34 +386,41 @@ void HttpServerStart(DatabaseInstance& db, string_t host, int32_t port, string_t string host_str = host.GetString(); + #ifndef _WIN32 - // Debug and Syslog Events - const char* use_syslog = std::getenv("DUCKDB_HTTPSERVER_SYSLOG"); const char* debug_env = std::getenv("DUCKDB_HTTPSERVER_DEBUG"); - if (use_syslog != nullptr && std::string(use_syslog) == "1") { - openlog("duckdb-httpserver", LOG_PID | LOG_NDELAY, LOG_LOCAL0); + const char* use_syslog = std::getenv("DUCKDB_HTTPSERVER_SYSLOG"); + + if (debug_env != nullptr && std::string(debug_env) == "1") { global_state.server->set_logger([](const duckdb_httplib_openssl::Request& req, const duckdb_httplib_openssl::Response& res) { - syslog(LOG_INFO, "%s %s - %d", + time_t now_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + char timestr[32]; + strftime(timestr, sizeof(timestr), "%Y-%m-%d %H:%M:%S", localtime(&now_time)); + // Use \r\n for consistent line endings + fprintf(stdout, "[%s] %s %s - %d - from %s:%d\r\n", + timestr, req.method.c_str(), req.path.c_str(), - res.status); - }); - std::atexit([]() { - closelog(); + res.status, + req.remote_addr.c_str(), + req.remote_port); + fflush(stdout); }); - } else if (debug_env != nullptr && std::string(debug_env) == "1") { + } else if (use_syslog != nullptr && std::string(use_syslog) == "1") { + openlog("duckdb-httpserver", LOG_PID | LOG_NDELAY, LOG_LOCAL0); global_state.server->set_logger([](const duckdb_httplib_openssl::Request& req, const duckdb_httplib_openssl::Response& res) { - auto now = std::chrono::system_clock::now(); - auto now_time = std::chrono::system_clock::to_time_t(now); - fprintf(stdout, "[%s] %s %s - %d\n", - std::ctime(&now_time), + syslog(LOG_INFO, "%s %s - %d - from %s:%d", req.method.c_str(), req.path.c_str(), - res.status); - fflush(stdout); + res.status, + req.remote_addr.c_str(), + req.remote_port); }); - } -#endif + std::atexit([]() { + closelog(); + }); + } +#endif const char* run_in_same_thread_env = std::getenv("DUCKDB_HTTPSERVER_FOREGROUND"); bool run_in_same_thread = (run_in_same_thread_env != nullptr && std::string(run_in_same_thread_env) == "1"); From 4b6c65c3edafa7586504bf25bba7147375ea119e Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Wed, 4 Dec 2024 20:42:55 +0100 Subject: [PATCH 6/7] Update MainDistributionPipeline.yml --- .github/workflows/MainDistributionPipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index bfb8cb4..77f7de4 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -4,6 +4,9 @@ name: Main Extension Distribution Pipeline on: push: + paths-ignore: + - '**.md' + - '**..yml' pull_request: workflow_dispatch: From df36f8fe5f483230f5eb50fb0c3b971e60532d94 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Wed, 4 Dec 2024 20:45:27 +0100 Subject: [PATCH 7/7] Update README.md --- docs/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index 7211800..532468b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,6 +41,7 @@ LOAD httpserver; Start the HTTP server providing the `host`, `port` and `auth` parameters.
> * If you want no authentication, just pass an empty string as parameter.
> * If you want the API run in foreground set `DUCKDB_HTTPSERVER_FOREGROUND=1` +> * If you want logs set `DUCKDB_HTTPSERVER_DEBUG` or `DUCKDB_HTTPSERVER_SYSLOG` #### Basic Auth ```sql @@ -90,7 +91,7 @@ D SELECT * FROM duck_flock('SELECT version()', ['http://localhost:9999']); │ "version"() │ │ varchar │ ├─────────────┤ -│ v1.1.1 │ +│ v1.1.3 │ └─────────────┘ ``` @@ -122,7 +123,7 @@ curl -X POST -d "SELECT 'hello', version()" "http://localhost:9999/?default_form "data": [ [ "hello", - "v1.1.1" + "v1.1.3" ] ], "rows": 1, @@ -153,7 +154,7 @@ D SELECT * FROM read_json_auto('http://localhost:9999/?q=SELECT version()'); │ "version"() │ │ varchar │ ├─────────────┤ -│ v1.1.1 │ +│ v1.1.3 │ └─────────────┘ ```