Merge pull request #361 from ruby/katei/remove-wasi-vfs-cli-dep

Introduce `rbwasm pack` command as an alias of `wasi-vfs pack`

benchmarks/vm_deep_call.rb

@@ -2,8 +2,8 @@
 #
 # Example runs
 # $ ruby vm_deep_call.rb
-# $ RUBY_EXE="wasmtime run --mapdir /::/ head-wasm32-unknown-wasi-minimal/usr/local/bin/ruby --" ruby vm_deep_call.rb
-# $ RUBY_EXE="wasmtime run --env RUBY_FIBER_MACHINE_STACK_SIZE=20971520 --mapdir /::/ head-wasm32-unknown-wasi-minimal/usr/local/bin/ruby --" ruby vm_deep_call.rb
+# $ RUBY_EXE="wasmtime run --dir /::/ head-wasm32-unknown-wasi-minimal/usr/local/bin/ruby --" ruby vm_deep_call.rb
+# $ RUBY_EXE="wasmtime run --env RUBY_FIBER_MACHINE_STACK_SIZE=20971520 --dir /::/ head-wasm32-unknown-wasi-minimal/usr/local/bin/ruby --" ruby vm_deep_call.rb
 
 def vm_rec n
   vm_rec n - 1 if n > 0

ext/ruby_wasm/Cargo.toml

@@ -15,3 +15,4 @@ wizer = "3.0.0"
 wasmtime-wasi = "9.0.4"
 wasi-cap-std-sync = "9.0.4"
 wasi-vfs-cli = { git = "https://github.com/kateinoigakukun/wasi-vfs/", rev = "b1e4e5d9cd6322e8745e67c092b495973835a94f" }
+structopt = "0.3.26"

ext/ruby_wasm/src/lib.rs

@@ -7,6 +7,7 @@ use magnus::{
     wrap, Error, ExceptionClass, RModule, Ruby,
 };
 use wizer::Wizer;
+use structopt::StructOpt;
 
 static RUBY_WASM: value::Lazy<RModule> =
     value::Lazy::new(|ruby| ruby.define_module("RubyWasmExt").unwrap());
@@ -35,6 +36,15 @@ struct WasiVfsInner {
 struct WasiVfs(std::cell::RefCell<WasiVfsInner>);
 
 impl WasiVfs {
+    fn run_cli(args: Vec<String>) -> Result<(), Error> {
+        wasi_vfs_cli::App::from_iter(args).execute().map_err(|e| {
+            Error::new(
+                exception::standard_error(),
+                format!("failed to run wasi vfs cli: {}", e),
+            )
+        })
+    }
+
     fn new() -> Self {
         Self(std::cell::RefCell::new(WasiVfsInner { map_dirs: vec![] }))
     }
@@ -63,6 +73,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
 
     let wasi_vfs = module.define_class("WasiVfs", ruby.class_object())?;
     wasi_vfs.define_singleton_method("new", function!(WasiVfs::new, 0))?;
+    wasi_vfs.define_singleton_method("run_cli", function!(WasiVfs::run_cli, 1))?;
     wasi_vfs.define_method("map_dir", method!(WasiVfs::map_dir, 2))?;
     wasi_vfs.define_method("pack", method!(WasiVfs::pack, 1))?;
     Ok(())

lib/ruby_wasm/build/product/wasi_vfs.rb

@@ -7,11 +7,6 @@ class WasiVfsProduct < BuildProduct
     def initialize(build_dir)
       @build_dir = build_dir
       @need_fetch_lib = ENV["LIB_WASI_VFS_A"].nil?
-      installed_cli_path =
-        ENV["WASI_VFS_CLI"] || Toolchain.find_path("wasi-vfs")
-      @need_fetch_cli = installed_cli_path.nil?
-      @cli_path =
-        installed_cli_path || File.join(cli_product_build_dir, "wasi-vfs")
     end
 
     def lib_product_build_dir
@@ -26,18 +21,6 @@ def lib_wasi_vfs_a
       ENV["LIB_WASI_VFS_A"] || File.join(lib_product_build_dir, "libwasi_vfs.a")
     end
 
-    def cli_product_build_dir
-      File.join(
-        @build_dir,
-        RbConfig::CONFIG["host"],
-        "wasi-vfs-#{WASI_VFS_VERSION}"
-      )
-    end
-
-    def cli_bin_path
-      @cli_path
-    end
-
     def name
       "wasi-vfs-#{WASI_VFS_VERSION}-#{RbConfig::CONFIG["host"]}"
     end
@@ -58,26 +41,5 @@ def build(executor)
         executor.mv File.join(tmpdir, "libwasi_vfs.a"), lib_wasi_vfs_a
       end
     end
-
-    def install_cli
-      return if !@need_fetch_cli || File.exist?(cli_bin_path)
-      FileUtils.mkdir_p cli_product_build_dir
-      zipfile = File.join(cli_product_build_dir, "wasi-vfs-cli.zip")
-      system "curl", "-L", "-o", zipfile, self.cli_download_url
-      system "unzip", zipfile, "-d", cli_product_build_dir
-    end
-
-    def cli_download_url
-      assets = [
-        [/x86_64-linux/, "wasi-vfs-cli-x86_64-unknown-linux-gnu.zip"],
-        [/x86_64-darwin/, "wasi-vfs-cli-x86_64-apple-darwin.zip"],
-        [/arm64e?-darwin/, "wasi-vfs-cli-aarch64-apple-darwin.zip"]
-      ]
-      asset = assets.find { |os, _| os =~ RUBY_PLATFORM }&.at(1)
-      if asset.nil?
-        raise "unsupported platform for fetching wasi-vfs CLI: #{RUBY_PLATFORM}"
-      end
-      "https://github.com/kateinoigakukun/wasi-vfs/releases/download/v#{WASI_VFS_VERSION}/#{asset}"
-    end
   end
 end

lib/ruby_wasm/cli.rb

@@ -9,7 +9,7 @@ def initialize(stdout:, stderr:)
     end
 
     def run(args)
-      available_commands = %w[build]
+      available_commands = %w[build pack]
       parser =
         OptionParser.new do |opts|
           opts.banner = <<~USAGE
@@ -32,6 +32,8 @@ def run(args)
       case command
       when "build"
         build(args)
+      when "pack"
+        pack(args)
       else
         @stderr.puts parser
         exit
@@ -141,6 +143,11 @@ def build(args)
       end
     end
 
+    def pack(args)
+      self.require_extension
+      RubyWasmExt::WasiVfs.run_cli([$0, "pack", *args])
+    end
+
     private
 
     def build_config(options)
@@ -180,7 +187,7 @@ def do_print_ruby_cache_key(packager)
     end
 
     def do_build(executor, tmpdir, packager, options)
-      require_relative "ruby_wasm.so"
+      self.require_extension
       wasm_bytes = packager.package(executor, tmpdir, options)
       RubyWasm.logger.info "Size: #{SizeFormatter.format(wasm_bytes.size)}"
       case options[:output]
@@ -191,5 +198,15 @@ def do_build(executor, tmpdir, packager, options)
         RubyWasm.logger.debug "Wrote #{options[:output]}"
       end
     end
+
+    def require_extension
+      # Tries to require the extension for the given Ruby version first
+      begin
+        RUBY_VERSION =~ /(\d+\.\d+)/
+        require_relative "#{Regexp.last_match(1)}/ruby_wasm.so"
+      rescue LoadError
+        require_relative "ruby_wasm.so"
+      end
+    end
   end
 end

packages/standalone/irb/build-package.sh

@@ -22,6 +22,6 @@ cp -R "$ruby_root" "$workdir/ruby-root"
 (
   cd "$workdir" && \
   "$WASMOPT" --strip-debug ruby-root/usr/local/bin/ruby -o ./ruby-root/ruby.wasm && \
-  "$WASI_VFS_CLI" pack ./ruby-root/ruby.wasm --mapdir /usr::./ruby-root/usr --mapdir /gems::$package_dir/gems -o "$dist_dir/irb.wasm" && \
+  "$WASI_VFS_CLI" pack ./ruby-root/ruby.wasm --dir ./ruby-root/usr::/usr --dir $package_dir/gems::/gems -o "$dist_dir/irb.wasm" && \
   wasi-preset-args "$dist_dir/irb.wasm" -o "$dist_dir/irb.wasm" -- -I/gems/lib /gems/libexec/irb --prompt default
 )

packages/standalone/ruby/build-package.sh

@@ -22,5 +22,5 @@ cp -R "$ruby_root" "$workdir/ruby-root"
 (
   cd "$workdir" && \
   "$WASMOPT" --strip-debug ruby-root/usr/local/bin/ruby -o ./ruby-root/ruby.wasm && \
-  "$WASI_VFS_CLI" pack ./ruby-root/ruby.wasm --mapdir /usr::./ruby-root/usr -o "$dist_dir/ruby.wasm"
+  "$WASI_VFS_CLI" pack ./ruby-root/ruby.wasm --dir ./ruby-root/usr::/usr -o "$dist_dir/ruby.wasm"
 )

rakelib/packaging.rake

@@ -1,7 +1,7 @@
 wasi_vfs = RubyWasm::WasiVfsProduct.new(File.join(Dir.pwd, "build"))
 wasi_sdk = TOOLCHAINS["wasi-sdk"]
 tools = {
-  "WASI_VFS_CLI" => wasi_vfs.cli_bin_path,
+  "WASI_VFS_CLI" => File.expand_path(File.join(__dir__, "..", "exe", "rbwasm")),
   "WASMOPT" => wasi_sdk.wasm_opt
 }
 
@@ -93,7 +93,6 @@ namespace :npm do
 
     desc "Make tarball for npm package #{pkg[:name]}"
     task pkg[:name] do
-      wasi_vfs.install_cli
       wasi_sdk.install_binaryen
       Rake::Task["npm:#{pkg[:name]}:build"].invoke
       sh "npm pack", chdir: pkg_dir
@@ -137,7 +136,6 @@ namespace :standalone do
 
     desc "Build standalone package #{pkg[:name]}"
     task "#{pkg[:name]}" => ["build:#{pkg[:build]}"] do
-      wasi_vfs.install_cli
       wasi_sdk.install_binaryen
       base_dir = Dir.pwd
       sh tools,

sig/ruby_wasm/build.rbs

@@ -135,12 +135,8 @@ module RubyWasm
     def initialize: (String build_dir) -> void
     def lib_product_build_dir: -> String
     def lib_wasi_vfs_a: -> String
-    def cli_product_build_dir: -> String
-    def cli_bin_path: -> String
     def name: -> String
     def build: (BuildExecutor executor) -> void
-    def install_cli: -> bool?
-    def cli_download_url: -> String
   end
 
   class CrossRubyExtProduct < BuildProduct

sig/ruby_wasm/cli.rbs

@@ -8,6 +8,7 @@ module RubyWasm
     def initialize: (stdout: IO, stderr: IO) -> void
 
     def build: (Array[String] args) -> void
+    def pack: (Array[String] args) -> void
 
     private
 
@@ -16,6 +17,8 @@ module RubyWasm
     def derive_packager: (Hash[untyped, untyped] options) -> Packager
     def do_print_ruby_cache_key: (Packager) -> void
     def do_build: (BuildExecutor, string tmpdir, Packager, Hash[untyped, untyped] options) -> void
+
+    def require_extension: () -> void
   end
 
   self.@logger: Logger?

sig/ruby_wasm/ext.rbs

@@ -4,8 +4,10 @@ module RubyWasmExt
   class WasiVfs
     def initialize: () -> void
 
+    def self.run_cli: (Array[String] args) -> void
+
     def map_dir: (String guest_path, String host_path) -> void
 
     def pack: (Array[Integer] module_bytes) -> Array[Integer]
   end
-end
+end