From 8de5eca1e994d06c8596692935f5b14b78669a76 Mon Sep 17 00:00:00 2001 From: kannibalox Date: Mon, 20 Jan 2025 00:45:27 -0500 Subject: [PATCH] Add optional gzip compression --- .github/workflows/static-analysis.yml | 2 +- .github/workflows/unit-tests.yml | 1 + configure.ac | 7 +- src/command_network.cc | 14 ++- src/rpc/scgi_task.cc | 157 +++++++++++++++++++------- src/rpc/scgi_task.h | 95 ++++++---------- 6 files changed, 170 insertions(+), 106 deletions(-) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 0806b8c5e..3dddf8b7c 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -37,7 +37,7 @@ jobs: git fetch --no-tags --no-recurse-submodules upstream "${{ github.event.pull_request.base.ref }}" - name: Install Dependencies run: | - sudo apt-get install -y bear clang-tidy libcurl4-openssl-dev + sudo apt-get install -y bear clang-tidy libcurl4-openssl-dev zlib1g-dev - name: Configure Project run: | libtoolize diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index eb46af680..076b1cd97 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -39,6 +39,7 @@ jobs: run: | sudo apt-get install -y \ libcppunit-dev \ + zlib1g-dev \ libcurl4-openssl-dev - name: Configure Project run: | diff --git a/configure.ac b/configure.ac index e4e0dc4c4..de2a312c5 100644 --- a/configure.ac +++ b/configure.ac @@ -48,6 +48,7 @@ if test "x$ax_cv_ncursesw" != xyes && test "x$ax_cv_ncurses" != xyes; then AC_MSG_ERROR([requires either NcursesW or Ncurses library]) fi +PKG_CHECK_MODULES([ZLIB], [zlib]) PKG_CHECK_MODULES([LIBCURL], [libcurl],, [LIBCURL_CHECK_CONFIG]) PKG_CHECK_MODULES([CPPUNIT], [cppunit],, [no_cppunit="yes"]) PKG_CHECK_MODULES([DEPENDENCIES], [libtorrent >= 0.15.1]) @@ -77,9 +78,9 @@ CC_ATTRIBUTE_UNUSED( dnl Only update global build variables immediately before generating the output, dnl to avoid affecting the global build environment for other autoconf checks. -LIBS="$PTHREAD_LIBS $CURSES_LIB $CURSES_LIBS $LIBCURL $LIBCURL_LIBS $DEPENDENCIES_LIBS $LIBS" -CFLAGS="$CFLAGS $PTHREAD_CFLAGS $LIBCURL_CPPFLAGS $LIBCURL_CFLAGS $DEPENDENCIES_CFLAGS $CURSES_CFLAGS" -CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS $LIBCURL_CPPFLAGS $LIBCURL_CFLAGS $DEPENDENCIES_CFLAGS $CURSES_CFLAGS" +LIBS="$PTHREAD_LIBS $ZLIB_LIBS $CURSES_LIB $CURSES_LIBS $LIBCURL $LIBCURL_LIBS $DEPENDENCIES_LIBS $LIBS" +CFLAGS="$CFLAGS $PTHREAD_CFLAGS $ZLIB_CFLAGS $LIBCURL_CPPFLAGS $LIBCURL_CFLAGS $DEPENDENCIES_CFLAGS $CURSES_CFLAGS" +CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS $ZLIB_CFLAGS $LIBCURL_CPPFLAGS $LIBCURL_CFLAGS $DEPENDENCIES_CFLAGS $CURSES_CFLAGS" AC_CONFIG_FILES([ Makefile diff --git a/src/command_network.cc b/src/command_network.cc index e9c9f0a18..ca976a57c 100644 --- a/src/command_network.cc +++ b/src/command_network.cc @@ -54,6 +54,7 @@ #include "core/download.h" #include "core/manager.h" #include "rpc/scgi.h" +#include "rpc/scgi_task.h" #include "ui/root.h" #include "rpc/parse.h" #include "rpc/parse_commands.h" @@ -276,6 +277,8 @@ initialize_command_network() { CMD2_ANY_VALUE_V ("network.send_buffer.size.set", std::bind(&torrent::ConnectionManager::set_send_buffer_size, cm, std::placeholders::_2)); CMD2_ANY ("network.receive_buffer.size", std::bind(&torrent::ConnectionManager::receive_buffer_size, cm)); CMD2_ANY_VALUE_V ("network.receive_buffer.size.set", std::bind(&torrent::ConnectionManager::set_receive_buffer_size, cm, std::placeholders::_2)); + + CMD2_ANY_STRING ("network.tos.set", std::bind(&apply_tos, std::placeholders::_2)); CMD2_ANY ("network.bind_address", std::bind(&core::Manager::bind_address, control->core())); @@ -293,9 +296,14 @@ initialize_command_network() { CMD2_ANY ("network.max_open_sockets", std::bind(&torrent::ConnectionManager::max_size, cm)); CMD2_ANY_VALUE_V ("network.max_open_sockets.set", std::bind(&torrent::ConnectionManager::set_max_size, cm, std::placeholders::_2)); - CMD2_ANY_STRING ("network.scgi.open_port", std::bind(&apply_scgi, std::placeholders::_2, 1)); - CMD2_ANY_STRING ("network.scgi.open_local", std::bind(&apply_scgi, std::placeholders::_2, 2)); - CMD2_VAR_BOOL ("network.scgi.dont_route", false); + CMD2_ANY_STRING ("network.scgi.open_port", std::bind(&apply_scgi, std::placeholders::_2, 1)); + CMD2_ANY_STRING ("network.scgi.open_local", std::bind(&apply_scgi, std::placeholders::_2, 2)); + CMD2_VAR_BOOL ("network.scgi.dont_route", false); + + CMD2_ANY ("network.scgi.gzip.min_size", [](const auto&, const auto&) { return rpc::SCgiTask::gzip_min_size(); }); + CMD2_ANY_VALUE_V ("network.scgi.gzip.min_size.set", [](const auto&, const auto& arg) { return rpc::SCgiTask::set_gzip_min_size(arg); }); + CMD2_ANY ("network.scgi.use_gzip", [](const auto&, const auto&) { return rpc::SCgiTask::gzip_enabled(); }); + CMD2_ANY_VALUE_V ("network.scgi.use_gzip.set", [](const auto&, const auto& arg) { return rpc::SCgiTask::set_gzip_enabled(arg); }); CMD2_ANY_STRING ("network.xmlrpc.dialect.set", [](const auto&, const auto& arg) { return apply_xmlrpc_dialect(arg); }) CMD2_ANY ("network.xmlrpc.size_limit", [](const auto&, const auto&){ return rpc::rpc.size_limit(); }); diff --git a/src/rpc/scgi_task.cc b/src/rpc/scgi_task.cc index 12e87e4a5..bd67beae7 100644 --- a/src/rpc/scgi_task.cc +++ b/src/rpc/scgi_task.cc @@ -1,14 +1,15 @@ +#include "rpc/scgi_task.h" #include "config.h" +#include #include #include -#include -#include -#include #include +#include #include #include #include +#include #include "utils/socket_fd.h" @@ -24,12 +25,16 @@ namespace rpc { +// Disable gzipping by default, but once enabled gzip everything +bool SCgiTask::m_allow_compressed_response = false; +int SCgiTask::m_min_compress_response_size = 0; + // If bufferSize is zero then memcpy won't do anything. inline void -SCgiTask::realloc_buffer(uint32_t size, const char* buffer, uint32_t bufferSize) { +SCgiTask::realloc_buffer(uint32_t size, const char* buffer, uint32_t buffer_size) { char* tmp = rak::cacheline_allocator::alloc_size(size); - std::memcpy(tmp, buffer, bufferSize); + std::memcpy(tmp, buffer, buffer_size); ::free(m_buffer); m_buffer = tmp; } @@ -38,7 +43,7 @@ void SCgiTask::open(SCgi* parent, int fd) { m_parent = parent; m_fileDesc = fd; - m_buffer = rak::cacheline_allocator::alloc_size((m_bufferSize = default_buffer_size) + 1); + m_buffer = rak::cacheline_allocator::alloc_size((m_buffer_size = default_buffer_size) + 1); m_position = m_buffer; m_body = NULL; @@ -46,7 +51,7 @@ SCgiTask::open(SCgi* parent, int fd) { worker_thread->poll()->insert_read(this); worker_thread->poll()->insert_error(this); -// scgiTimer = rak::timer::current(); + // scgiTimer = rak::timer::current(); } void @@ -66,14 +71,14 @@ SCgiTask::close() { m_buffer = NULL; // Test -// char buffer[512]; -// sprintf(buffer, "SCgi system call processed: %i", (int)(rak::timer::current() - scgiTimer).usec()); -// control->core()->push_log(std::string(buffer)); + // char buffer[512]; + // sprintf(buffer, "SCgi system call processed: %i", (int)(rak::timer::current() - scgiTimer).usec()); + // control->core()->push_log(std::string(buffer)); } void SCgiTask::event_read() { - int bytes = ::recv(m_fileDesc, m_position, m_bufferSize - (m_position - m_buffer), 0); + int bytes = ::recv(m_fileDesc, m_position, m_buffer_size - (m_position - m_buffer), 0); if (bytes <= 0) { if (bytes == 0 || !rak::error_number::current().is_blocked_momentary()) @@ -139,6 +144,11 @@ SCgiTask::event_read() { goto event_read_failed; } else if (strcmp(key, "CONTENT_TYPE") == 0) { content_type = value; + } else if (strcmp(key, "ACCEPT_ENCODING") == 0) { + if (strstr(value, "gzip") != nullptr) + // This just marks it as possible to compress, it may not + // actually happen depending on the size of the response + m_client_accepts_compressed_response = true; } } @@ -164,25 +174,25 @@ SCgiTask::event_read() { goto event_read_failed; } - if ((unsigned int)(content_length + header_size) < m_bufferSize) { - m_bufferSize = content_length + header_size; + if ((unsigned int)(content_length + header_size) < m_buffer_size) { + m_buffer_size = content_length + header_size; } else if ((unsigned int)content_length <= default_buffer_size) { - m_bufferSize = content_length; + m_buffer_size = content_length; std::memmove(m_buffer, m_body, std::distance(m_body, m_position)); m_position = m_buffer + std::distance(m_body, m_position); m_body = m_buffer; } else { - realloc_buffer((m_bufferSize = content_length) + 1, m_body, std::distance(m_body, m_position)); + realloc_buffer((m_buffer_size = content_length) + 1, m_body, std::distance(m_body, m_position)); m_position = m_buffer + std::distance(m_body, m_position); m_body = m_buffer; } } - if ((unsigned int)std::distance(m_buffer, m_position) != m_bufferSize) + if ((unsigned int)std::distance(m_buffer, m_position) != m_buffer_size) return; worker_thread->poll()->remove_read(this); @@ -193,14 +203,14 @@ SCgiTask::event_read() { // Clean up logging, this is just plain ugly... // write(m_logFd, "\n---\n", sizeof("\n---\n")); - result = write(m_parent->log_fd(), m_buffer, m_bufferSize); + result = write(m_parent->log_fd(), m_buffer, m_buffer_size); result = write(m_parent->log_fd(), "\n---\n", sizeof("\n---\n")); } - lt_log_print_dump(torrent::LOG_RPC_DUMP, m_body, m_bufferSize - std::distance(m_buffer, m_body), "scgi", "RPC read.", 0); + lt_log_print_dump(torrent::LOG_RPC_DUMP, m_body, m_buffer_size - std::distance(m_buffer, m_body), "scgi", "RPC read.", 0); // Close if the call failed, else stay open to write back data. - if (!m_parent->receive_call(this, m_body, m_bufferSize - std::distance(m_buffer, m_body))) + if (!m_parent->receive_call(this, m_body, m_buffer_size - std::distance(m_buffer, m_body))) close(); return; @@ -212,12 +222,13 @@ SCgiTask::event_read() { void SCgiTask::event_write() { + int bytes; // Apple and Solaris do not support MSG_NOSIGNAL, // so disable this fix until we find a better solution #if defined(__APPLE__) || defined(__sun__) - int bytes = ::send(m_fileDesc, m_position, m_bufferSize, 0); + bytes = ::send(m_fileDesc, m_position, m_bufferSize, 0); #else - int bytes = ::send(m_fileDesc, m_position, m_bufferSize, MSG_NOSIGNAL); + bytes = ::send(m_fileDesc, m_position, m_buffer_size, MSG_NOSIGNAL); #endif if (bytes == -1) { @@ -228,9 +239,9 @@ SCgiTask::event_write() { } m_position += bytes; - m_bufferSize -= bytes; + m_buffer_size -= bytes; - if (bytes == 0 || m_bufferSize == 0) + if (bytes == 0 || m_buffer_size == 0) return close(); } @@ -239,36 +250,102 @@ SCgiTask::event_error() { close(); } +// On failure, returns false and the buffer is left in an +// indeterminate state, but m_position and m_bufferSize remain the +// same. +bool +SCgiTask::gzip_compress_response(const char* buffer, uint32_t length, std::string_view header_template) { + z_stream zs; + zs.zalloc = Z_NULL; + zs.zfree = Z_NULL; + zs.opaque = Z_NULL; + + constexpr int window_bits = 15; + constexpr int gzip_encoding = 16; + constexpr int gzip_level = 6; + constexpr int chunk_size = 16384; + + if (deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits | gzip_encoding, gzip_level, Z_DEFAULT_STRATEGY) != Z_OK) + return false; + + // Calculate the maximum size the buffer could reach, note that the + // max repsonse size will usually be larger than the original length + const auto max_response_size = deflateBound(&zs, length); + if (max_response_size + max_header_size > std::max(m_buffer_size, (unsigned int)default_buffer_size)) + realloc_buffer(max_response_size + max_header_size, NULL, 0); + + auto output = m_buffer + max_header_size; + zs.next_in = (Bytef*)buffer; + zs.avail_in = length; + do { + zs.avail_out = chunk_size; + zs.next_out = (Bytef*)output; + if (deflate(&zs, Z_FINISH) == Z_STREAM_ERROR) + return false; + output += chunk_size - zs.avail_out; + } while (zs.avail_out == 0); + + // Write the header directly to the buffer. If at any point + // max_header_size would be exceeded, fail gracefully. + const std::string_view header_end("Content-Encoding: gzip\r\n\r\n"); + const int response_size = output - (m_buffer + max_header_size); + int header_size = snprintf(m_buffer, max_header_size, header_template.data(), response_size); + if (header_size < 0 || header_end.size() > max_header_size - header_size) + return false; + std::memcpy(m_buffer + header_size, header_end.data(), header_end.size()); + header_size += header_end.size(); + + // Move the response back into position right after the headers + std::memmove(m_buffer + header_size, m_buffer + max_header_size, response_size); + + m_position = m_buffer; + m_buffer_size = header_size + response_size; + return true; +} + bool SCgiTask::receive_write(const char* buffer, uint32_t length) { if (buffer == NULL || length > (100 << 20)) throw torrent::internal_error("SCgiTask::receive_write(...) received bad input."); - // Need to cast due to a bug in MacOSX gcc-4.0.1. - if (length + 256 > std::max(m_bufferSize, (unsigned int)default_buffer_size)) - realloc_buffer(length + 256, NULL, 0); - - const auto header = m_content_type == ContentType::JSON - ? "Status: 200 OK\r\nContent-Type: application/json\r\nContent-Length: %i\r\n\r\n" - : "Status: 200 OK\r\nContent-Type: text/xml\r\nContent-Length: %i\r\n\r\n"; - - // Who ever bothers to check the return value? - int headerSize = sprintf(m_buffer, header, length); - - m_position = m_buffer; - m_bufferSize = length + headerSize; - - std::memcpy(m_buffer + headerSize, buffer, length); + std::string header = m_content_type == ContentType::JSON + ? "Status: 200 OK\r\nContent-Type: application/json\r\nContent-Length: %i\r\n" + : "Status: 200 OK\r\nContent-Type: text/xml\r\nContent-Length: %i\r\n"; + // Write to log prior to possible compression if (m_parent->log_fd() >= 0) { int __UNUSED result; // Clean up logging, this is just plain ugly... // write(m_logFd, "\n---\n", sizeof("\n---\n")); - result = write(m_parent->log_fd(), m_buffer, m_bufferSize); + result = write(m_parent->log_fd(), buffer, length); result = write(m_parent->log_fd(), "\n---\n", sizeof("\n---\n")); } - lt_log_print_dump(torrent::LOG_RPC_DUMP, m_buffer, m_bufferSize, "scgi", "RPC write.", 0); + lt_log_print_dump(torrent::LOG_RPC_DUMP, buffer, length, "scgi", "RPC write.", 0); + + // Compress the response if possible + if (m_client_accepts_compressed_response && + gzip_enabled() && + length > gzip_min_size() && + gzip_compress_response(buffer, length, header)) { + event_write(); + return true; + } + + // Otherwise (or if the compression fails), just copy the bytes + header += "\r\n"; + + int header_size = snprintf(NULL, 0, header.c_str(), length); + + // Need to cast due to a bug in MacOSX gcc-4.0.1. + if (length + header_size > std::max(m_buffer_size, (unsigned int)default_buffer_size)) + realloc_buffer(length + header_size, NULL, 0); + + m_position = m_buffer; + m_buffer_size = length + header_size; + + snprintf(m_buffer, m_buffer_size, header.c_str(), length); + std::memcpy(m_buffer + header_size, buffer, length); event_write(); return true; diff --git a/src/rpc/scgi_task.h b/src/rpc/scgi_task.h index 3aeeb85b6..ce3db0dd2 100644 --- a/src/rpc/scgi_task.h +++ b/src/rpc/scgi_task.h @@ -1,46 +1,11 @@ -// rTorrent - BitTorrent client -// Copyright (C) 2005-2011, Jari Sundell -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations -// including the two. -// -// You must obey the GNU General Public License in all respects for -// all of the code used other than OpenSSL. If you modify file(s) -// with this exception, you may extend this exception to your version -// of the file(s), but you are not obligated to do so. If you do not -// wish to do so, delete this exception statement from your version. -// If you delete this exception statement from all source files in the -// program, then also delete it here. -// -// Contact: Jari Sundell -// -// Skomakerveien 33 -// 3185 Skoppum, NORWAY - #ifndef RTORRENT_RPC_SCGI_TASK_H #define RTORRENT_RPC_SCGI_TASK_H +#include #include namespace utils { - class SocketFd; +class SocketFd; } namespace rpc { @@ -49,44 +14,56 @@ class SCgi; class SCgiTask : public torrent::Event { public: - static const unsigned int default_buffer_size = 2047; - static const int max_header_size = 2000; - static const int max_content_size = (2 << 23); + static constexpr unsigned int default_buffer_size = 2047; + static constexpr int max_header_size = 2000; + static constexpr int max_content_size = (2 << 23); + + static int gzip_min_size() { return m_min_compress_response_size; } + static void set_gzip_min_size(int size) { m_min_compress_response_size = size; } + static bool gzip_enabled() { return m_allow_compressed_response; } + static void set_gzip_enabled(bool enabled) { m_allow_compressed_response = enabled; } - enum ContentType { XML, JSON }; + enum ContentType { XML, + JSON }; SCgiTask() { m_fileDesc = -1; } - bool is_open() const { return m_fileDesc != -1; } - bool is_available() const { return m_fileDesc == -1; } + bool is_open() const { return m_fileDesc != -1; } + bool is_available() const { return m_fileDesc == -1; } - void open(SCgi* parent, int fd); - void close(); + void open(SCgi* parent, int fd); + void close(); - ContentType content_type() const { return m_content_type; } + ContentType content_type() const { return m_content_type; } - virtual void event_read(); - virtual void event_write(); - virtual void event_error(); + virtual void event_read(); + virtual void event_write(); + virtual void event_error(); - bool receive_write(const char* buffer, uint32_t length); + bool receive_write(const char* buffer, uint32_t length); - utils::SocketFd& get_fd() { return *reinterpret_cast(&m_fileDesc); } + utils::SocketFd& get_fd() { return *reinterpret_cast(&m_fileDesc); } private: - inline void realloc_buffer(uint32_t size, const char* buffer, uint32_t bufferSize); + inline void realloc_buffer(uint32_t size, const char* buffer, uint32_t buffer_size); + + bool gzip_compress_response(const char* buffer, uint32_t length, std::string_view header_template); - SCgi* m_parent; + static bool m_allow_compressed_response; + static int m_min_compress_response_size; - char* m_buffer; - char* m_position; - char* m_body; + SCgi* m_parent; - unsigned int m_bufferSize; + char* m_buffer; + char* m_position; + char* m_body; - ContentType m_content_type{ XML }; + unsigned int m_buffer_size; + + ContentType m_content_type{XML}; + bool m_client_accepts_compressed_response = false; }; -} +} // namespace rpc #endif