feat: Implement dynamic loading of libchdb.so via dlopen and auto-download during gem install

Author: wudidapaopao

.rubocop.yml

@@ -4,6 +4,7 @@ AllCops:
     - 'spec/**/*' 
     - 'tmp/**/*'
     - 'ext/**/extconf.rb'
+    - 'pkg/**/*'
 
   NewCops: enable
 

chdb.gemspec

@@ -38,6 +38,18 @@ Gem::Specification.new do |s|
     'INSTALLATION.md',
     'LICENSE',
     'README.md',
+    'dependencies.yml',
+    'ext/chdb/chdb_handle.c',
+    'ext/chdb/chdb_handle.h',
+    'ext/chdb/chdb.c',
+    'ext/chdb/connection.c',
+    'ext/chdb/connection.h',
+    'ext/chdb/constants.h',
+    'ext/chdb/exception.c',
+    'ext/chdb/exception.h',
+    'ext/chdb/extconf.rb',
+    'ext/chdb/local_result.c',
+    'ext/chdb/local_result.h',
     'lib/chdb.rb',
     'lib/chdb/constants.rb',
     'lib/chdb/data_path.rb',
@@ -57,5 +69,5 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'csv', '~> 3.1'
 
-  # s.extensions << 'ext/chdb/extconf.rb'
+  s.extensions << 'ext/chdb/extconf.rb'
 end

ext/chdb/chdb.c

@@ -1,6 +1,7 @@
 #include <ruby.h>
 
 #include "chdb.h"
+#include "chdb_handle.h"
 #include "constants.h"
 #include "connection.h"
 #include "exception.h"
@@ -22,6 +23,7 @@ void Init_chdb_native()
     DEBUG_PRINT("Initializing chdb extension");
 
     init_exception();
+    init_chdb_handle();
     init_chdb_constants();
     init_local_result();
     init_connection();

ext/chdb/chdb_handle.c

@@ -0,0 +1,49 @@
+#include "chdb_handle.h"
+
+#include <dlfcn.h>
+#include <ruby.h>
+
+#include "exception.h"
+
+void *chdb_handle = NULL;
+connect_chdb_func connect_chdb_ptr = NULL;
+close_conn_func close_conn_ptr = NULL;
+query_conn_func query_conn_ptr = NULL;
+
+VALUE get_chdb_rb_path(void)
+{
+    VALUE chdb_module = rb_const_get(rb_cObject, rb_intern("ChDB"));
+    return rb_funcall(chdb_module, rb_intern("lib_file_path"), 0);
+}
+
+void init_chdb_handle()
+{
+    VALUE rb_path = get_chdb_rb_path();
+    VALUE lib_dir = rb_file_dirname(rb_file_dirname(rb_path));
+    VALUE lib_path = rb_str_cat2(lib_dir, "/lib/chdb/lib/libchdb.so");
+    // printf("chdb.rb path from Ruby: %s\n", StringValueCStr(lib_path));
+
+    connect_chdb_ptr = NULL;
+    close_conn_ptr = NULL;
+    query_conn_ptr = NULL;
+
+    chdb_handle = dlopen(RSTRING_PTR(lib_path), RTLD_LAZY | RTLD_GLOBAL);
+    if (!chdb_handle)
+    {
+        rb_raise(cChDBError, "Failed to load chdb library: %s\nCheck if libchdb.so exists at: %s",
+                 dlerror(), RSTRING_PTR(lib_path));
+    }
+
+    connect_chdb_ptr = (connect_chdb_func)dlsym(chdb_handle, "connect_chdb");
+    close_conn_ptr = (close_conn_func)dlsym(chdb_handle, "close_conn");
+    query_conn_ptr = (query_conn_func)dlsym(chdb_handle, "query_conn");
+
+    if (!connect_chdb_ptr || !close_conn_ptr || !query_conn_ptr)
+    {
+        rb_raise(cChDBError, "Symbol loading failed: %s\nMissing functions: connect_chdb(%p) close_conn(%p) query_conn(%p)",
+                 dlerror(),
+                 (void*)connect_chdb_ptr,
+                 (void*)close_conn_ptr,
+                 (void*)query_conn_ptr);
+    }
+}

ext/chdb/chdb_handle.h

@@ -0,0 +1,16 @@
+#ifndef CHDB_HANDLE_H
+#define CHDB_HANDLE_H
+
+typedef struct chdb_conn **(*connect_chdb_func)(int, char**);
+typedef void (*close_conn_func)(struct chdb_conn**);
+typedef struct local_result_v2 *(*query_conn_func)(struct chdb_conn*, const char*, const char*);
+
+extern connect_chdb_func connect_chdb_ptr;
+extern close_conn_func close_conn_ptr;
+extern query_conn_func query_conn_ptr;
+
+extern void *chdb_handle;
+
+void init_chdb_handle();
+
+#endif

ext/chdb/connection.c

@@ -1,10 +1,29 @@
 #include "connection.h"
 
+#include "chdb_handle.h"
 #include "constants.h"
 #include "exception.h"
 #include "include/chdb.h"
 #include "local_result.h"
 
+static void connection_free(void *ptr)
+{
+    Connection *conn = (Connection *)ptr;
+    DEBUG_PRINT("Closing connection: %p", (void*)conn->c_conn);
+    if (conn->c_conn)
+    {
+        close_conn_ptr(conn->c_conn);
+        conn->c_conn = NULL;
+    }
+    free(conn);
+}
+
+const rb_data_type_t ConnectionType =
+{
+    "Connection",
+    {NULL, connection_free, NULL},
+};
+
 void init_connection()
 {
     VALUE mChDB = rb_define_module("ChDB");
@@ -39,7 +58,7 @@ VALUE connection_initialize(VALUE self, VALUE argc, VALUE argv)
 
     Connection *conn;
     TypedData_Get_Struct(self, Connection, &ConnectionType, conn);
-    conn->c_conn = connect_chdb(c_argc, c_argv);
+    conn->c_conn = connect_chdb_ptr(c_argc, c_argv);
 
     if (!conn->c_conn)
     {
@@ -60,7 +79,7 @@ VALUE connection_query(VALUE self, VALUE query, VALUE format)
     Check_Type(query, T_STRING);
     Check_Type(format, T_STRING);
 
-    struct local_result_v2 *c_result = query_conn(
+    struct local_result_v2 *c_result = query_conn_ptr(
                                            *conn->c_conn,
                                            StringValueCStr(query),
                                            StringValueCStr(format)
@@ -92,27 +111,8 @@ VALUE connection_close(VALUE self)
 
     if (conn->c_conn)
     {
-        close_conn(conn->c_conn);
+        close_conn_ptr(conn->c_conn);
         conn->c_conn = NULL;
     }
     return Qnil;
 }
-
-static void connection_free(void *ptr)
-{
-    Connection *conn = (Connection *)ptr;
-    DEBUG_PRINT("Closing connection: %p", (void*)conn->c_conn);
-    if (conn->c_conn)
-    {
-        close_conn(conn->c_conn);
-    }
-    free(conn);
-}
-
-const rb_data_type_t ConnectionType =
-{
-    "Connection",
-    {NULL, connection_free, NULL},
-};
-
-// 其他 Connection 方法保持不变...

ext/chdb/extconf.rb

@@ -18,34 +18,39 @@ def configure
         create_makefile('chdb/chdb_native')
       end
 
+      def compiled?
+        return false if cross_build?
+
+        major_version = RUBY_VERSION.match(/(\d+\.\d+)/)[1]
+        version_dir = File.join(package_root_dir, 'lib', 'chdb', major_version)
+
+        extension = if determine_target_platform.include?('darwin')
+                      'bundle'
+                    else
+                      'so'
+                    end
+        lib_file = "#{libname}.#{extension}"
+
+        File.exist?(File.join(version_dir, lib_file))
+      end
+
       def configure_cross_compiler
         RbConfig::CONFIG['CC'] = RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] if ENV['CC']
         ENV['CC'] = RbConfig::CONFIG['CC']
       end
 
+      def cross_build?
+        enable_config('cross-build')
+      end
+
       def libname
-        'chdb'
+        'chdb_native'
       end
 
       def configure_extension
         include_path = File.expand_path('ext/chdb/include', package_root_dir)
         append_cppflags("-I#{include_path}")
-
-        lib_path = File.expand_path('ext/chdb/lib', package_root_dir)
-        append_ldflags("-L#{lib_path}")
-
-        target_platform = determine_target_platform
-        if target_platform.include?('darwin')
-          append_ldflags('-Wl,-rpath,@loader_path/../lib')
-        else
-          append_ldflags("-Wl,-rpath,'$$ORIGIN/../lib'")
-        end
-
         abort_could_not_find('chdb.h') unless find_header('chdb.h', include_path)
-
-        return if find_library(libname, nil, lib_path)
-
-        abort_could_not_find(libname)
       end
 
       def abort_could_not_find(missing)
@@ -60,9 +65,25 @@ def download_and_extract
         target_platform = determine_target_platform
         version = fetch_chdb_version
         download_dir = determine_download_directory(target_platform, version)
-
-        unless Dir.exist?(download_dir)
+        need_download = false
+
+        if Dir.exist?(download_dir)
+          required_files = [
+            File.join(download_dir, 'chdb.h'),
+            File.join(download_dir, 'libchdb.so')
+          ]
+
+          need_download = !required_files.all? { |f| File.exist?(f) }
+          if need_download
+            puts 'Missing required files, cleaning download directory...'
+            FileUtils.rm_rf(Dir.glob("#{download_dir}/*"))
+          end
+        else
           FileUtils.mkdir_p(download_dir)
+          need_download = true
+        end
+
+        if need_download
           file_name = get_file_name(target_platform)
           url = build_download_url(version, file_name)
           download_tarball(url, download_dir, file_name)
@@ -83,7 +104,7 @@ def determine_target_platform
         when /arm64-darwin/  then 'arm64-darwin'
         when /x86_64-darwin/ then 'x86_64-darwin'
         else
-          'unknown-platform'
+          raise ArgumentError, "Unsupported platform: #{RUBY_PLATFORM}."
         end
       end
 
@@ -114,8 +135,19 @@ def download_tarball(url, download_dir, file_name)
         tarball = File.join(download_dir, file_name)
         puts "Downloading chdb library for #{determine_target_platform}..."
 
-        URI.open(url) do |remote| # rubocop:disable Security/Open
-          IO.copy_stream(remote, tarball)
+        max_retries = 3
+        retries = 0
+
+        begin
+          URI.open(url) do |remote| # rubocop:disable Security/Open
+            IO.copy_stream(remote, tarball)
+          end
+        rescue StandardError => e
+          raise "Failed to download after #{max_retries} attempts: #{e.message}" unless retries < max_retries
+
+          retries += 1
+          puts "Download failed: #{e.message}. Retrying (attempt #{retries}/#{max_retries})..."
+          retry
         end
       end
 
@@ -125,24 +157,27 @@ def extract_tarball(download_dir, file_name)
       end
 
       def copy_files(download_dir, _version)
-        ext_chdb_path = File.join(package_root_dir, 'ext/chdb')
-        [%w[*.h], %w[*.so], %w[*.dylib]].each do |(glob_pattern)|
-          src_dir, pattern = File.split(glob_pattern)
-
+        [%w[*.h], %w[*.so]].each do |(glob_pattern)|
+          # Removed the unused variable src_dir
+          pattern = File.basename(glob_pattern)
           dest_subdir = case pattern
                         when '*.h' then 'include'
                         else 'lib'
                         end
-          src = File.join(download_dir, src_dir, pattern)
-          dest = File.join(ext_chdb_path, dest_subdir)
-          FileUtils.mkdir_p(dest)
-          FileUtils.cp_r(Dir.glob(src), dest, remove_destination: true)
-
-          # target_platform = determine_target_platform
-          # if target_platform.include?('darwin') && (pattern == '*.so')
-          #   system("install_name_tool -id '@rpath/libchdb.so' #{File.join(dest, 'libchdb.so')}")
-          #   system("codesign -f -s - #{File.join(dest, 'libchdb.so')}")
-          # end
+          dest_dir = File.join(package_root_dir, 'ext/chdb', dest_subdir)
+          src_files = Dir.glob(File.join(download_dir, pattern))
+
+          extra_dirs = []
+          extra_dirs << File.join(package_root_dir, 'lib/chdb/lib') if pattern == '*.so'
+
+          ([dest_dir] + extra_dirs).each do |dest|
+            FileUtils.mkdir_p(dest)
+
+            src_files.each do |src_file|
+              dest_file = File.join(dest, File.basename(src_file))
+              FileUtils.ln_s(File.expand_path(src_file), dest_file, force: true)
+            end
+          end
         end
       end
 

ext/chdb/local_result.h

@@ -23,4 +23,4 @@ VALUE local_result_rows_read(VALUE self);
 
 VALUE local_result_bytes_read(VALUE self);
 
-#endif
+#endif

lib/chdb.rb

@@ -1,5 +1,11 @@
 # frozen_string_literal: true
 
+module ChDB
+  def self.lib_file_path
+    @lib_file_path ||= File.expand_path(__FILE__)
+  end
+end
+
 begin
   RUBY_VERSION =~ /(\d+\.\d+)/
   require "chdb/#{Regexp.last_match(1)}/chdb_native"

lib/chdb/version.rb

@@ -2,5 +2,5 @@
 
 module ChDB
   # (String) the version of the chdb gem, e.g. "0.1.0"
-  VERSION = '0.1.0.rc.1'
+  VERSION = '0.1.0.rc.2'
 end

rakelib/check_manifest.rake

@@ -33,17 +33,15 @@ task :check_manifest do # rubocop:disable Metrics/BlockLength
     .rspec_status
     .ruby-version
     .rubocop.yml
-    dependencies.yml
-    ext/chdb/*.{c,h}
     lib/chdb/chdb*.{bundle,so}
+    ext/chdb/include/chdb.h
     Gemfile*
     Rakefile
     [a-z]*.{log,out}
     [0-9]*
     *.gemspec
     *.so
     CHANGELOG.md
-    ext/chdb/extconf.rb
   ]
 
   intended_directories = Dir.children('.')