Merge pull request #613 from filmor/discovery

Discover NIFs at startup

CHANGELOG.md

@@ -8,6 +8,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 See [`UPGRADE.md`](./UPGRADE.md) for additional help when upgrading to newer
 versions.
 
+## [unreleased]
+
+### Added
+
+### Fixed
+
+### Changed
+
+- NIF implementations are now discovered automatically and the respective
+  argument of `rustler::init!` is ignored (#613)
+
+### Removed
+
+
 ## [0.33.0] - 2024-05-29
 
 ### Added

README.md

@@ -55,7 +55,7 @@ fn add(a: i64, b: i64) -> i64 {
     a + b
 }
 
-rustler::init!("Elixir.Math", [add]);
+rustler::init!("Elixir.Math");
 ```
 
 #### Minimal Supported Rust Version (MSRV)

UPGRADE.md

@@ -2,6 +2,12 @@
 
 This document is intended to simplify upgrading to newer versions by extending the changelog.
 
+## 0.33 -> 0.34
+
+1. NIF implementations are now discovered automatically, the respective argument
+   in the `rustler::init!` macro should be removed. If a NIF implementation
+   should not be exported, it must be disabled with a `#[cfg]` marker.
+
 ## 0.32 -> 0.33
 
 The macro changes that where already carried out in version `0.22` are now
@@ -109,7 +115,7 @@ rustler::rustler_export_nifs! {
     "Elixir.Math",
     [
         ("add", 2, add),
-	("long_running_operation", 0, long_running_operation, SchedulerFlags::DirtyCpu)
+        ("long_running_operation", 0, long_running_operation, SchedulerFlags::DirtyCpu)
     ],
     None
 }
@@ -184,9 +190,9 @@ NIF called `_long_running_operation`, which used to be declared prior to Rustler
 rustler::rustler_export_nifs! {
     "Elixir.SomeNif",
     [
-	// Note that the function in Rust is long_running_operation, but the NIF is exported as
-	// _long_running_operation!
-	("_long_running_operation", 0, long_running_operation, SchedulerFlags::DirtyCpu)
+        // Note that the function in Rust is long_running_operation, but the NIF is exported as
+        // _long_running_operation!
+        ("_long_running_operation", 0, long_running_operation, SchedulerFlags::DirtyCpu)
     ],
     None
 }

rustler/Cargo.toml

@@ -22,6 +22,7 @@ nif_version_2_17 = ["nif_version_2_16", "rustler_sys/nif_version_2_17"]
 serde = ["dep:serde"]
 
 [dependencies]
+inventory = "0.3"
 rustler_codegen = { path = "../rustler_codegen", version = "0.33.0", optional = true}
 rustler_sys = { path = "../rustler_sys", version = "~2.4.1" }
 num-bigint = { version = "0.4", optional = true }

rustler/src/codegen_runtime.rs

@@ -5,6 +5,9 @@ use std::fmt;
 
 use crate::{Encoder, Env, OwnedBinary, Term};
 
+// Re-export of inventory
+pub use inventory;
+
 // Names used by the `rustler::init!` macro or other generated code.
 pub use crate::wrapper::exception::raise_exception;
 pub use crate::wrapper::{

rustler/src/nif.rs

@@ -1,13 +1,25 @@
 use crate::codegen_runtime::{c_char, c_int, c_uint, DEF_NIF_FUNC, NIF_ENV, NIF_TERM};
 
-pub trait Nif {
-    const NAME: *const c_char;
-    const ARITY: c_uint;
-    const FLAGS: c_uint;
-    const FUNC: DEF_NIF_FUNC;
-    const RAW_FUNC: unsafe extern "C" fn(
-        nif_env: NIF_ENV,
-        argc: c_int,
-        argv: *const NIF_TERM,
-    ) -> NIF_TERM;
+pub struct Nif {
+    pub name: *const c_char,
+    pub arity: c_uint,
+    pub flags: c_uint,
+    // pub func: DEF_NIF_FUNC,
+    pub raw_func:
+        unsafe extern "C" fn(nif_env: NIF_ENV, argc: c_int, argv: *const NIF_TERM) -> NIF_TERM,
 }
+
+impl Nif {
+    pub fn get_def(&self) -> DEF_NIF_FUNC {
+        DEF_NIF_FUNC {
+            arity: self.arity,
+            flags: self.flags,
+            function: self.raw_func,
+            name: self.name,
+        }
+    }
+}
+
+unsafe impl Sync for Nif {}
+
+inventory::collect!(Nif);

rustler/src/serde/mod.rs

@@ -12,7 +12,7 @@ in a better way and allow more Erlang-y configurations).
 use rustler::{self, Encoder, SerdeTerm};
 use serde::{Serialize, Deserialize};
 
-rustler::init!("Elixir.SerdeNif", [readme]);
+rustler::init!("Elixir.SerdeNif");
 
 // NOTE: to serialize to the correct Elixir record, you MUST tell serde to
 // rename the variants to the full Elixir record module atom.

rustler_benchmarks/native/benchmark/src/lib.rs

@@ -2,19 +2,4 @@ mod nif_record;
 mod nif_struct;
 mod nif_various;
 
-rustler::init!(
-    "Elixir.Benchmark",
-    [
-        nif_struct::benchmark,
-        nif_record::benchmark,
-        nif_various::encode_tagged_enum,
-        nif_various::decode_tagged_enum,
-        nif_various::decode_struct,
-        nif_various::decode_struct_string,
-        nif_various::decode_string,
-        nif_various::decode_term,
-        nif_various::void,
-        nif_various::encode_atom,
-        nif_various::compare_atom
-    ]
-);
+rustler::init!("Elixir.Benchmark");

rustler_codegen/Cargo.toml

@@ -17,6 +17,7 @@ syn = { version = "2.0", features = ["full", "extra-traits"] }
 quote = "1.0"
 heck = "0.5"
 proc-macro2 = "1.0"
+inventory = "0.3"
 
 [dev-dependencies]
 trybuild = "1.0"

rustler_codegen/src/init.rs

@@ -1,26 +1,51 @@
 use proc_macro2::{Span, TokenStream};
-use quote::quote;
+use quote::{quote, quote_spanned};
 use syn::parse::{Parse, ParseStream};
-use syn::punctuated::Punctuated;
-use syn::token::Comma;
-use syn::{Expr, Ident, Result, Token};
+use syn::spanned::Spanned;
+use syn::{Ident, Result, Token};
 
 #[derive(Debug)]
 pub struct InitMacroInput {
     name: syn::Lit,
-    funcs: syn::ExprArray,
     load: TokenStream,
+    maybe_warning: TokenStream,
 }
 
 impl Parse for InitMacroInput {
     fn parse(input: ParseStream) -> Result<Self> {
         let name = syn::Lit::parse(input)?;
-        let _comma = <syn::Token![,]>::parse(input)?;
-        let funcs = syn::ExprArray::parse(input)?;
+
+        let maybe_warning = if input.peek(syn::token::Comma) && input.peek2(syn::token::Bracket) {
+            // peeked, must be there
+            let _ = syn::token::Comma::parse(input).unwrap();
+            if let Ok(funcs) = syn::ExprArray::parse(input) {
+                quote_spanned!(funcs.span() =>
+                    #[allow(dead_code)]
+                    fn rustler_init() {
+                        #[deprecated(
+                            since = "0.34.0",
+                            note = "Passing NIF functions explicitly is deprecated and this argument is ignored, please remove it"
+                        )]
+                        #[allow(non_upper_case_globals)]
+                        const explicit_nif_functions: () = ();
+                        let _ = explicit_nif_functions;
+                    }
+                )
+            } else {
+                quote!()
+            }
+        } else {
+            quote!()
+        };
+
         let options = parse_expr_assigns(input);
         let load = extract_option(options, "load");
 
-        Ok(InitMacroInput { name, funcs, load })
+        Ok(InitMacroInput {
+            name,
+            load,
+            maybe_warning,
+        })
     }
 }
 
@@ -55,20 +80,22 @@ fn extract_option(args: Vec<syn::ExprAssign>, name: &str) -> TokenStream {
 impl From<InitMacroInput> for proc_macro2::TokenStream {
     fn from(input: InitMacroInput) -> Self {
         let name = input.name;
-        let num_of_funcs = input.funcs.elems.len();
-        let funcs = nif_funcs(input.funcs.elems);
         let load = input.load;
+        let maybe_warning = input.maybe_warning;
 
         let inner = quote! {
             static mut NIF_ENTRY: Option<rustler::codegen_runtime::DEF_NIF_ENTRY> = None;
-            use rustler::Nif;
+            let nif_funcs: Box<[_]> =
+                rustler::codegen_runtime::inventory::iter::<rustler::Nif>()
+                .map(rustler::Nif::get_def)
+                .collect();
 
             let entry = rustler::codegen_runtime::DEF_NIF_ENTRY {
                 major: rustler::codegen_runtime::NIF_MAJOR_VERSION,
                 minor: rustler::codegen_runtime::NIF_MINOR_VERSION,
                 name: concat!(#name, "\0").as_ptr() as *const rustler::codegen_runtime::c_char,
-                num_of_funcs: #num_of_funcs as rustler::codegen_runtime::c_int,
-                funcs: [#funcs].as_ptr(),
+                num_of_funcs: nif_funcs.len() as rustler::codegen_runtime::c_int,
+                funcs: nif_funcs.as_ptr(),
                 load: {
                     extern "C" fn nif_load(
                         env: rustler::codegen_runtime::NIF_ENV,
@@ -91,12 +118,17 @@ impl From<InitMacroInput> for proc_macro2::TokenStream {
             };
 
             unsafe {
+                // Leak nif_funcs
+                std::mem::forget(nif_funcs);
+
                 NIF_ENTRY = Some(entry);
                 NIF_ENTRY.as_ref().unwrap()
             }
         };
 
         quote! {
+            #maybe_warning
+
             #[cfg(unix)]
             #[no_mangle]
             extern "C" fn nif_init() -> *const rustler::codegen_runtime::DEF_NIF_ENTRY {
@@ -115,17 +147,3 @@ impl From<InitMacroInput> for proc_macro2::TokenStream {
         }
     }
 }
-
-fn nif_funcs(funcs: Punctuated<Expr, Comma>) -> TokenStream {
-    let mut tokens = TokenStream::new();
-
-    for func in funcs.iter() {
-        if let Expr::Path(_) = *func {
-            tokens.extend(quote!(#func::FUNC,));
-        } else {
-            panic!("Expected an expression, found: {}", stringify!(func));
-        }
-    }
-
-    tokens
-}

rustler_codegen/src/lib.rs

@@ -50,7 +50,7 @@ enum RustlerAttr {
 ///     true
 /// }
 ///
-/// rustler::init!("Elixir.Math", [add, sub, mul, div], load = load);
+/// rustler::init!("Elixir.Math", load = load);
 /// ```
 #[proc_macro]
 pub fn init(input: TokenStream) -> TokenStream {

rustler_codegen/src/nif.rs

@@ -55,54 +55,47 @@ pub fn transcoder_decorator(nif_attributes: NifAttributes, fun: syn::ItemFn) ->
     }
 
     quote! {
-        #[allow(non_camel_case_types)]
-        pub struct #name;
-
-        impl rustler::Nif for #name {
-            const NAME: *const rustler::codegen_runtime::c_char = concat!(#erl_func_name, "\0").as_ptr() as *const rustler::codegen_runtime::c_char;
-            const ARITY: rustler::codegen_runtime::c_uint = #arity;
-            const FLAGS: rustler::codegen_runtime::c_uint = #flags as rustler::codegen_runtime::c_uint;
-            const RAW_FUNC: unsafe extern "C" fn(
-                nif_env: rustler::codegen_runtime::NIF_ENV,
-                argc: rustler::codegen_runtime::c_int,
-                argv: *const rustler::codegen_runtime::NIF_TERM
-            ) -> rustler::codegen_runtime::NIF_TERM = {
-                unsafe extern "C" fn nif_func(
-                    nif_env: rustler::codegen_runtime::NIF_ENV,
-                    argc: rustler::codegen_runtime::c_int,
-                    argv: *const rustler::codegen_runtime::NIF_TERM
-                ) -> rustler::codegen_runtime::NIF_TERM {
-                    let lifetime = ();
-                    let env = rustler::Env::new(&lifetime, nif_env);
-
-                    let terms = std::slice::from_raw_parts(argv, argc as usize)
-                        .iter()
-                        .map(|term| rustler::Term::new(env, *term))
-                        .collect::<Vec<rustler::Term>>();
-
-                    fn wrapper<'a>(
-                        env: rustler::Env<'a>,
-                        args: &[rustler::Term<'a>]
-                    ) -> rustler::codegen_runtime::NifReturned {
-                        let result: std::thread::Result<_> = std::panic::catch_unwind(move || {
-                            #decoded_terms
-                            #function
-                            Ok(#name(#argument_names))
-                        });
-
-                        rustler::codegen_runtime::handle_nif_result(result, env)
+        rustler::codegen_runtime::inventory::submit!(
+            rustler::Nif {
+                name: concat!(#erl_func_name, "\0").as_ptr()
+                    as *const rustler::codegen_runtime::c_char,
+                arity: #arity,
+                flags: #flags as rustler::codegen_runtime::c_uint,
+                raw_func: {
+                    unsafe extern "C" fn nif_func(
+                        nif_env: rustler::codegen_runtime::NIF_ENV,
+                        argc: rustler::codegen_runtime::c_int,
+                        argv: *const rustler::codegen_runtime::NIF_TERM
+                    ) -> rustler::codegen_runtime::NIF_TERM {
+                        let lifetime = ();
+                        let env = rustler::Env::new(&lifetime, nif_env);
+
+                        let terms = std::slice::from_raw_parts(argv, argc as usize)
+                            .iter()
+                            .map(|term| rustler::Term::new(env, *term))
+                            .collect::<Vec<rustler::Term>>();
+
+                        fn wrapper<'a>(
+                            env: rustler::Env<'a>,
+                            args: &[rustler::Term<'a>]
+                        ) -> rustler::codegen_runtime::NifReturned {
+                            let result: std::thread::Result<_> =
+                                std::panic::catch_unwind(move || {
+                                    #decoded_terms
+                                    #function
+                                    Ok(#name(#argument_names))
+                                });
+
+                            rustler::codegen_runtime::handle_nif_result(
+                                result, env
+                            )
+                        }
+                        wrapper(env, &terms).apply(env)
                     }
-                    wrapper(env, &terms).apply(env)
+                    nif_func
                 }
-                nif_func
-            };
-            const FUNC: rustler::codegen_runtime::DEF_NIF_FUNC = rustler::codegen_runtime::DEF_NIF_FUNC {
-                arity: Self::ARITY,
-                flags: Self::FLAGS,
-                function: Self::RAW_FUNC,
-                name: Self::NAME
-            };
-        }
+            }
+        );
     }
 }
 

rustler_mix/priv/templates/basic/src/lib.rs

@@ -3,4 +3,4 @@ fn add(a: i64, b: i64) -> i64 {
     a + b
 }
 
-rustler::init!("<%= native_module %>", [add]);
+rustler::init!("<%= native_module %>");

rustler_sys/src/rustler_sys_api.rs

@@ -39,6 +39,7 @@ unsafe impl Send for ErlNifEnv {}
 
 /// See [ErlNifFunc](http://www.erlang.org/doc/man/erl_nif.html#ErlNifFunc) in the Erlang docs.
 // #[allow(missing_copy_implementations)]
+#[derive(Debug)]
 #[repr(C)]
 pub struct ErlNifFunc {
     pub name: *const c_char,