diff --git a/lib/ruby_wasm/build/product/crossruby.rb b/lib/ruby_wasm/build/product/crossruby.rb
index 946df3703d..cfd5786d7a 100644
--- a/lib/ruby_wasm/build/product/crossruby.rb
+++ b/lib/ruby_wasm/build/product/crossruby.rb
@@ -71,7 +71,6 @@ def do_extconf(executor, crossruby)
         return
       end
       objdir = product_build_dir crossruby
-      source = crossruby.source
       rbconfig_rb = Dir.glob(File.join(crossruby.dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first
       raise "rbconfig.rb not found" unless rbconfig_rb
       extconf_args = [
@@ -359,6 +358,10 @@ def configure_args(build_triple, toolchain)
         args << %Q(wasmoptflags=#{@wasmoptflags.join(" ")})
       end
       args << "--disable-install-doc"
+      unless @params.target.pic?
+        # TODO: Remove this hack after dropping Ruby 3.2 support
+        args << "ac_cv_func_dlopen=no"
+      end
       args
     end
   end
diff --git a/lib/ruby_wasm/packager/core.rb b/lib/ruby_wasm/packager/core.rb
index 6171f9b4f2..6f36eec495 100644
--- a/lib/ruby_wasm/packager/core.rb
+++ b/lib/ruby_wasm/packager/core.rb
@@ -181,6 +181,12 @@ def _link_gem_exts(executor, build, ruby_root, gem_home, module_bytes)
     end
 
     def _build_gem_exts(executor, build, gem_home)
+      build.toolchain.install
+      baseruby = build.baseruby
+      unless Dir.exist?(baseruby.install_dir)
+        baseruby.build(executor)
+      end
+
       exts = specs_with_extensions.flat_map do |spec, exts|
         exts.map do |ext|
           ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
diff --git a/rakelib/packaging.rake b/rakelib/packaging.rake
index 33c0b222a8..a43b87fc38 100644
--- a/rakelib/packaging.rake
+++ b/rakelib/packaging.rake
@@ -25,10 +25,6 @@ def npm_pkg_build_command(pkg)
 end
 
 def npm_pkg_rubies_cache_key(pkg)
-  # FIXME: Now CrossRubyExtProduct depends on just built baseruby, and exts can be
-  # built after restoring the tarball cache. So it can fail to find baseruby when the
-  # cache hit. We need to orchestrate the build dependency graph properly.
-  return nil if pkg[:name] == "ruby-head-wasm-wasi"
   build_command = npm_pkg_build_command(pkg)
   return nil unless build_command
   require "open3"
diff --git a/sig/ruby_wasm/build.rbs b/sig/ruby_wasm/build.rbs
index f63950ef92..4c6376471e 100644
--- a/sig/ruby_wasm/build.rbs
+++ b/sig/ruby_wasm/build.rbs
@@ -33,6 +33,7 @@ module RubyWasm
     @source: BuildSource
 
     attr_reader toolchain: Toolchain
+    attr_reader baseruby: BaseRubyProduct
 
     def initialize: (
       string name,