Threadpool: remove stdlib dependencies (#293)

* Address part of #291, detect physical cores, note: Linux limited to 64 cores at the moment.

* change getNumPhysicalCores to return int instead of int32

* renaming cpuinfo -> cpudetect

* system/ansi_c dependency cleanup

* forgot to delete renamed cpuinfo_x86

* dump WIP AMD CPUID detection [skip ci]

* Export available threads at the OS to Nim/C/Rust + Rust constantine-core crate

* cpudetect: CPUID function update

* windows barrier exporting WinBool twice

* fix threadpool.new type changes

* cpudetect: workaround Nim flaky asm interpolation - similar to nim-lang/Nim#23114

* topology: BSD/MacOS fused + no libc dependency + sysctl auto ID / dynamic ID handling

* topology BSD: reorg decls

* cpudetect x86: cleanup

* topology BSD: try to fix import macro define

* topo BSD: defines from C macros needs to be imported as global

Cargo.toml

@@ -2,6 +2,7 @@
 resolver = "2"
 members = [
     "constantine-rust/constantine-sys",
+    "constantine-rust/constantine-core",
     "constantine-rust/constantine-halo2-zal",
     "constantine-rust/constantine-ethereum-kzg",
 ]

benchmarks-threadpool/bouncing_producer_consumer/threadpool_bpc.nim

@@ -1,6 +1,6 @@
 import
   # STD lib
-  system/ansi_c, std/[os, strutils, cpuinfo, strformat, math],
+  std/[os, strutils, cpuinfo, strformat, math],
   # Library
   ../../constantine/threadpool,
   # bench

benchmarks-threadpool/dfs/threadpool_dfs.nim

@@ -7,7 +7,7 @@
 
 import
   # Stdlib
-  system/ansi_c, std/[strformat, os, strutils, cpuinfo],
+  std/[strformat, os, strutils, cpuinfo],
   # Library
   ../../constantine/threadpool
 

benchmarks-threadpool/heat/stdnim_heat.nim

@@ -33,7 +33,7 @@
 
 import
   # Stdlib
-  system/ansi_c, std/[strformat, os, strutils, math, cpuinfo],
+  system/ansi_c, std/[strformat, math, cpuinfo],
   std/threadpool,
   # bench
   ../wtime, ../resources

benchmarks-threadpool/resources.nim

@@ -14,9 +14,9 @@ type
     RusageThread = 1
 
 when defined(debug):
-  var H_RUSAGE_SELF{.importc, header:"<sys/resource.h".}: cint
-  var H_RUSAGE_CHILDREN{.importc, header:"<sys/resource.h".}: cint
-  var H_RUSAGE_THREAD{.importc, header:"<sys/resource.h".}: cint
+  let H_RUSAGE_SELF{.importc, header:"<sys/resource.h".}: cint
+  let H_RUSAGE_CHILDREN{.importc, header:"<sys/resource.h".}: cint
+  let H_RUSAGE_THREAD{.importc, header:"<sys/resource.h".}: cint
   assert H_RUSAGE_SELF == ord(RusageSelf)
   assert H_RUSAGE_CHILDREN = ord(RusageChildren)
   assert H_RUSAGE_THREAD = ord(RusageThread)

benchmarks/bench_ec_msm_pasta.nim

@@ -33,7 +33,7 @@ const AvailableCurves = [
 
 # const testNumPoints = [10, 100, 1000, 10000, 100000]
 # const testNumPoints = [64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072]
-const testNumPoints = [1 shl 8, 1 shl 9, 1 shl 10, 1 shl 11, 1 shl 12, 1 shl 13, 1 shl 14, 1 shl 15, 1 shl 16, 1 shl 17, 1 shl 22]
+const testNumPoints = [1 shl 10, 1 shl 11, 1 shl 12, 1 shl 13, 1 shl 14, 1 shl 15, 1 shl 16, 1 shl 17, 1 shl 18, 1 shl 19, 1 shl 20, 1 shl 21, 1 shl 22]
 
 proc main() =
   separator()

benchmarks/bench_poly1305.nim

@@ -3,9 +3,7 @@ import
   ../constantine/mac/mac_poly1305,
   # Helpers
   ../helpers/prng_unsafe,
-  ./bench_blueprint,
-  # C API
-  system/ansi_c
+  ./bench_blueprint
 
 proc separator*() = separator(69)
 

constantine-go/constantine.go

@@ -35,7 +35,7 @@ type Threadpool struct {
 
 func ThreadpoolNew(numThreads int) Threadpool {
 	return Threadpool{
-		ctx: C.ctt_threadpool_new(C.size_t(numThreads)),
+		ctx: C.ctt_threadpool_new(C.int(numThreads)),
 	}
 }
 

constantine-rust/constantine-core/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "constantine-core"
+version = "0.1.0"
+edition = "2021"
+
+authors = ["Mamy André-Ratsimbazafy"]
+license = "MIT/Apache-2.0"
+repository = "https://github.com/mratsim/constantine"
+
+[dependencies]
+constantine-sys = { path = "../constantine-sys" }

constantine-rust/constantine-core/src/lib.rs

@@ -0,0 +1,69 @@
+//! Constantine
+//! Copyright (c) 2018-2019    Status Research & Development GmbH
+//! Copyright (c) 2020-Present Mamy André-Ratsimbazafy
+//! Licensed and distributed under either of
+//!   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+//!   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+//! at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+use constantine_sys::*;
+
+// Cryptographically secure RNGs
+// ------------------------------------------------------------
+
+pub mod csprngs {
+    use constantine_sys::ctt_csprng_sysrand;
+    use core::ffi::c_void;
+    #[inline(always)]
+    pub fn sysrand(buffer: &mut [u8]) {
+        unsafe {
+            ctt_csprng_sysrand(buffer.as_mut_ptr() as *mut c_void, buffer.len());
+        }
+    }
+}
+
+// Hardware detection
+// ------------------------------------------------------------
+
+pub mod hardware {
+    use constantine_sys::ctt_cpu_get_num_threads_os;
+    #[inline(always)]
+    #[doc = " Query the number of threads available at the OS-level\n  to run computations.\n\n  This takes into account cores disabled at the OS-level, for example in a VM.\n  However this doesn't detect restrictions based on time quotas often used for Docker\n  or taskset / cpuset restrictions from cgroups.\n\n  For Simultaneous-Multithreading (SMT often call HyperThreading),\n  this returns the number of available logical cores."]
+    pub fn get_num_threads_os() -> usize {
+        unsafe { ctt_cpu_get_num_threads_os() }.try_into().unwrap()
+    }
+}
+
+// Threadpool
+// ------------------------------------------------------------
+
+#[derive(Debug)]
+pub struct Threadpool {
+    ctx: *mut ctt_threadpool,
+}
+
+impl Threadpool {
+    /// Instantiate a new Threadpool with `num_threads` threads.
+    /// A single threadpool can be active on a given thread.
+    /// A new threadpool may be instantiated if the previous one has been shutdown.
+    #[inline(always)]
+    pub fn new(num_threads: usize) -> Self {
+        let ctx = unsafe { ctt_threadpool_new(num_threads.try_into().unwrap()) };
+        Self { ctx }
+    }
+
+    /// Access the private context of Threadpool
+    /// For use, only in Constantine's crates.
+    /// No guarantee of continuous support.
+    #[inline(always)]
+    pub fn get_private_context(&self) -> *mut ctt_threadpool {
+        self.ctx
+    }
+}
+
+impl Drop for Threadpool {
+    #[inline(always)]
+    fn drop(&mut self) {
+        unsafe { ctt_threadpool_shutdown(self.ctx) }
+    }
+}

constantine-rust/constantine-ethereum-kzg/Cargo.toml

@@ -9,10 +9,10 @@ repository = "https://github.com/mratsim/constantine"
 
 [dependencies]
 constantine-sys = { path = "../constantine-sys" }
+constantine-core = { path = "../constantine-core" }
 
 [dev-dependencies]
 serde = { version = "1.0", default-features = false, features = ["derive"] }
 serde_yaml = "0.9"
 hex = { version = "0.4", default-features = false, features = ["serde"] }
 glob = "0.3"
-num_cpus = "1.16.0"

constantine-rust/constantine-ethereum-kzg/src/lib.rs

@@ -6,45 +6,12 @@
 //!   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
 //! at your option. This file may not be copied, modified, or distributed except according to those terms.
 
+use constantine_core::Threadpool;
 use constantine_sys::*;
 
 use ::core::mem::MaybeUninit;
-use core::ffi::c_void;
 use std::{ffi::CString, path::Path};
 
-// Cryptographically secure RNG
-// ------------------------------------------------------------
-
-#[inline(always)]
-pub fn csprng_sysrand(buffer: &mut [u8]) {
-    unsafe {
-        ctt_csprng_sysrand(buffer.as_mut_ptr() as *mut c_void, buffer.len());
-    }
-}
-
-// Threadpool
-// ------------------------------------------------------------
-
-#[derive(Debug)]
-pub struct CttThreadpool {
-    ctx: *mut ctt_threadpool,
-}
-
-impl CttThreadpool {
-    #[inline(always)]
-    pub fn new(num_threads: usize) -> CttThreadpool {
-        let ctx = unsafe { ctt_threadpool_new(num_threads) };
-        CttThreadpool { ctx }
-    }
-}
-
-impl Drop for CttThreadpool {
-    #[inline(always)]
-    fn drop(&mut self) {
-        unsafe { ctt_threadpool_shutdown(self.ctx) }
-    }
-}
-
 // Trusted setup
 // ------------------------------------------------------------
 
@@ -248,14 +215,14 @@ impl EthKzgContext {
     #[inline]
     pub fn blob_to_kzg_commitment_parallel(
         &self,
-        tp: &CttThreadpool,
+        tp: &Threadpool,
         blob: &[u8; 4096 * 32],
     ) -> Result<[u8; 48], ctt_eth_kzg_status> {
         let mut result: MaybeUninit<[u8; 48]> = MaybeUninit::uninit();
         unsafe {
             let status = ctt_eth_kzg_blob_to_kzg_commitment_parallel(
                 self.ctx,
-                tp.ctx,
+                tp.get_private_context(),
                 result.as_mut_ptr() as *mut ctt_eth_kzg_commitment,
                 blob.as_ptr() as *const ctt_eth_kzg_blob,
             );
@@ -269,7 +236,7 @@ impl EthKzgContext {
     #[inline]
     pub fn compute_kzg_proof_parallel(
         &self,
-        tp: &CttThreadpool,
+        tp: &Threadpool,
         blob: &[u8; 4096 * 32],
         z_challenge: &[u8; 32],
     ) -> Result<([u8; 48], [u8; 32]), ctt_eth_kzg_status> {
@@ -278,7 +245,7 @@ impl EthKzgContext {
         unsafe {
             let status = ctt_eth_kzg_compute_kzg_proof_parallel(
                 self.ctx,
-                tp.ctx,
+                tp.get_private_context(),
                 proof.as_mut_ptr() as *mut ctt_eth_kzg_proof,
                 y_eval.as_mut_ptr() as *mut ctt_eth_kzg_eval_at_challenge,
                 blob.as_ptr() as *const ctt_eth_kzg_blob,
@@ -296,15 +263,15 @@ impl EthKzgContext {
     #[inline]
     pub fn compute_blob_kzg_proof_parallel(
         &self,
-        tp: &CttThreadpool,
+        tp: &Threadpool,
         blob: &[u8; 4096 * 32],
         commitment: &[u8; 48],
     ) -> Result<[u8; 48], ctt_eth_kzg_status> {
         let mut proof = MaybeUninit::<[u8; 48]>::uninit();
         unsafe {
             let status = ctt_eth_kzg_compute_blob_kzg_proof_parallel(
                 self.ctx,
-                tp.ctx,
+                tp.get_private_context(),
                 proof.as_mut_ptr() as *mut ctt_eth_kzg_proof,
                 blob.as_ptr() as *const ctt_eth_kzg_blob,
                 commitment.as_ptr() as *const ctt_eth_kzg_commitment,
@@ -319,15 +286,15 @@ impl EthKzgContext {
     #[inline]
     pub fn verify_blob_kzg_proof_parallel(
         &self,
-        tp: &CttThreadpool,
+        tp: &Threadpool,
         blob: &[u8; 4096 * 32],
         commitment: &[u8; 48],
         proof: &[u8; 48],
     ) -> Result<bool, ctt_eth_kzg_status> {
         let status = unsafe {
             ctt_eth_kzg_verify_blob_kzg_proof_parallel(
                 self.ctx,
-                tp.ctx,
+                tp.get_private_context(),
                 blob.as_ptr() as *const ctt_eth_kzg_blob,
                 commitment.as_ptr() as *const ctt_eth_kzg_commitment,
                 proof.as_ptr() as *const ctt_eth_kzg_proof,
@@ -343,7 +310,7 @@ impl EthKzgContext {
     #[inline]
     pub fn verify_blob_kzg_proof_batch_parallel(
         &self,
-        tp: &CttThreadpool,
+        tp: &Threadpool,
         blobs: &[[u8; 4096 * 32]],
         commitments: &[[u8; 48]],
         proofs: &[[u8; 48]],
@@ -356,7 +323,7 @@ impl EthKzgContext {
         let status = unsafe {
             ctt_eth_kzg_verify_blob_kzg_proof_batch_parallel(
                 self.ctx,
-                tp.ctx,
+                tp.get_private_context(),
                 blobs.as_ptr() as *const ctt_eth_kzg_blob,
                 commitments.as_ptr() as *const ctt_eth_kzg_commitment,
                 proofs.as_ptr() as *const ctt_eth_kzg_proof,

constantine-rust/constantine-ethereum-kzg/tests/t_ethereum_kzg_vectors.rs

@@ -6,14 +6,14 @@
 //!   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
 //! at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-use constantine_ethereum_kzg::{csprng_sysrand, CttThreadpool, EthKzgContext};
+use constantine_core::{csprngs, hardware, Threadpool};
+use constantine_ethereum_kzg::EthKzgContext;
 
 use std::fs;
 use std::path::{Path, PathBuf};
 
 use glob::glob;
 use hex;
-use num_cpus;
 use serde::Deserialize;
 use serde_yaml;
 
@@ -401,7 +401,7 @@ fn t_verify_blob_kzg_proof_batch() {
         .expect("Trusted setup should be loaded without error.");
 
     let mut secure_random_bytes = [0u8; 32];
-    csprng_sysrand(secure_random_bytes.as_mut_slice());
+    csprngs::sysrand(secure_random_bytes.as_mut_slice());
     assert_ne!(secure_random_bytes, [0u8; 32]);
 
     let test_files: Vec<PathBuf> = glob(VERIFY_BLOB_KZG_PROOF_BATCH_TESTS)
@@ -483,7 +483,7 @@ fn t_blob_to_kzg_commitment_parallel() {
     let ctx = EthKzgContext::load_trusted_setup(Path::new(SRS_PATH))
         .expect("Trusted setup should be loaded without error.");
 
-    let tp = CttThreadpool::new(num_cpus::get());
+    let tp = Threadpool::new(hardware::get_num_threads_os());
 
     let test_files: Vec<PathBuf> = glob(BLOB_TO_KZG_COMMITMENT_TESTS)
         .unwrap()
@@ -543,7 +543,7 @@ fn t_compute_kzg_proof_parallel() {
     let ctx = EthKzgContext::load_trusted_setup(Path::new(SRS_PATH))
         .expect("Trusted setup should be loaded without error.");
 
-    let tp = CttThreadpool::new(num_cpus::get());
+    let tp = Threadpool::new(hardware::get_num_threads_os());
 
     let test_files: Vec<PathBuf> = glob(COMPUTE_KZG_PROOF_TESTS)
         .unwrap()
@@ -605,7 +605,7 @@ fn t_compute_blob_kzg_proof_parallel() {
     let ctx = EthKzgContext::load_trusted_setup(Path::new(SRS_PATH))
         .expect("Trusted setup should be loaded without error.");
 
-    let tp = CttThreadpool::new(num_cpus::get());
+    let tp = Threadpool::new(hardware::get_num_threads_os());
 
     let test_files: Vec<PathBuf> = glob(COMPUTE_BLOB_KZG_PROOF_TESTS)
         .unwrap()
@@ -669,7 +669,7 @@ fn t_verify_blob_kzg_proof_parallel() {
     let ctx = EthKzgContext::load_trusted_setup(Path::new(SRS_PATH))
         .expect("Trusted setup should be loaded without error.");
 
-    let tp = CttThreadpool::new(num_cpus::get());
+    let tp = Threadpool::new(hardware::get_num_threads_os());
 
     let test_files: Vec<PathBuf> = glob(VERIFY_BLOB_KZG_PROOF_TESTS)
         .unwrap()
@@ -738,10 +738,10 @@ fn t_verify_blob_kzg_proof_batch_parallel() {
     let ctx = EthKzgContext::load_trusted_setup(Path::new(SRS_PATH))
         .expect("Trusted setup should be loaded without error.");
 
-    let tp = CttThreadpool::new(num_cpus::get());
+    let tp = Threadpool::new(hardware::get_num_threads_os());
 
     let mut secure_random_bytes = [0u8; 32];
-    csprng_sysrand(secure_random_bytes.as_mut_slice());
+    csprngs::sysrand(secure_random_bytes.as_mut_slice());
     assert_ne!(secure_random_bytes, [0u8; 32]);
 
     let test_files: Vec<PathBuf> = glob(VERIFY_BLOB_KZG_PROOF_BATCH_TESTS)

constantine-rust/constantine-halo2-zal/Cargo.toml

@@ -9,12 +9,12 @@ repository = "https://github.com/mratsim/constantine"
 
 [dependencies]
 constantine-sys = { path = "../constantine-sys" }
+constantine-core = { path = "../constantine-core" }
 halo2curves = { git = 'https://github.com/taikoxyz/halo2curves', branch = "pr-pse-exec-engine" }
 
 [dev-dependencies]
 ark-std = "0.3"
 rand_core = { version = "0.6", default-features = false }
-num_cpus = "1.16.0"
 
 # Benchmark-only dependencies
 criterion = { version = "0.3", features = ["html_reports"] }