build: Use system NSS when possible (#1739)

* build: Use system-installed NSS instead of building our own Fixes #1711 * Update CI * Fix docs * Fix Dockerfile * Fix * build-essential * Try and search for nss * Try to get newest versions * More fixes * Restore Windows link.exe fix * Install pkg-config * Remove MSYS2 linker * Retain ability to build NSS from source * Update Linux instructions * Try and find MSYS2 library path * Retry * Again * Again * Again * Again * Again * Again * Again * Again * Again * Again * Revert many things, keep building NSS from source unless system version is OK * Fixes * Fixes * debug * Debug * Fixes * Compare versions with the `semver` crate * Use NSS version from code in CI * File has other name * Update .github/actions/nss/action.yml Co-authored-by: Martin Thomson <[email protected]> Signed-off-by: Lars Eggert <[email protected]> * Update neqo-crypto/build.rs Co-authored-by: Martin Thomson <[email protected]> Signed-off-by: Lars Eggert <[email protected]> * Update neqo-crypto/build.rs Co-authored-by: Martin Thomson <[email protected]> Signed-off-by: Lars Eggert <[email protected]> * Address code review comments. Not ready yet. Need to determine what to do in `nss_dir()`. See comments. --------- Signed-off-by: Lars Eggert <[email protected]> Co-authored-by: Martin Thomson <[email protected]>
mozilla · Mar 26, 2024 · 8ec74e4 · 8ec74e4
1 parent 76630a5
commit 8ec74e4
Show file tree

Hide file tree

Showing 10 changed files with 132 additions and 28 deletions.
diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml
@@ -16,35 +16,68 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: Check system NSS version
+      shell: bash
+      run: |
+        if ! command -v pkg-config &> /dev/null; then
+          echo "BUILD_NSS=1" >> "$GITHUB_ENV"
+          exit 0
+        fi
+        if ! pkg-config --exists nss; then
+          echo "BUILD_NSS=1" >> "$GITHUB_ENV"
+          exit 0
+        fi
+        NSS_VERSION="$(pkg-config --modversion nss)"
+        if [ "$?" -ne 0 ]; then
+          echo "BUILD_NSS=1" >> "$GITHUB_ENV"
+          exit 0
+        fi
+        NSS_MAJOR=$(echo "$NSS_VERSION" | cut -d. -f1)
+        NSS_MINOR=$(echo "$NSS_VERSION" | cut -d. -f2)
+        REQ_NSS_MAJOR=$(cat neqo-crypto/min_version.txt | cut -d. -f1)
+        REQ_NSS_MINOR=$(cat neqo-crypto/min_version.txt | cut -d. -f2)
+        if [ "$NSS_MAJOR" -lt "REQ_NSS_MAJOR" ] || [ "$NSS_MAJOR" -eq "REQ_NSS_MAJOR" -a "$NSS_MINOR" -lt "REQ_NSS_MINOR"]; then
+          echo "System NSS is too old: $NSS_VERSION"
+          echo "BUILD_NSS=1" >> "$GITHUB_ENV"
+          exit 0
+        fi
+        echo "System NSS is suitable: $NSS_VERSION"
+        echo "BUILD_NSS=0" >> "$GITHUB_ENV"
+
     # Ideally, we'd use this. But things are sufficiently flaky that we're better off
     # trying both hg and git. Leaving this here in case we want to re-try in the future.
     #
     # - name: Checkout NSPR
+    #   if: env.BUILD_NSS == '1'
     #   uses: actions/checkout@v4
     #   with:
     #     repository: "nss-dev/nspr"
     #     path: ${{ github.workspace }}/nspr
 
     # - name: Checkout NSS
+    #   if: env.BUILD_NSS == '1'
     #   uses: actions/checkout@v4
     #   with:
     #     repository: "nss-dev/nss"
     #     path: ${{ github.workspace }}/nss
 
     - name: Checkout NSPR
       shell: bash
+      if: env.BUILD_NSS == '1'
       run: |
         hg clone https://hg.mozilla.org/projects/nspr "${{ github.workspace }}/nspr" || \
           git clone --depth=1 https://github.com/nss-dev/nspr "${{ github.workspace }}/nspr"
 
     - name: Checkout NSS
       shell: bash
+      if: env.BUILD_NSS == '1'
       run: |
         hg clone https://hg.mozilla.org/projects/nss "${{ github.workspace }}/nss" || \
           git clone --depth=1 https://github.com/nss-dev/nss "${{ github.workspace }}/nss"
 
     - name: Build
       shell: bash
+      if: env.BUILD_NSS == '1'
       run: |
         if [ "${{ inputs.type }}" != "Debug" ]; then
           # We want to do an optimized build for accurate CPU profiling, but

diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -49,33 +49,21 @@ jobs:
           sudo apt-get install -y --no-install-recommends gyp mercurial ninja-build lld
           echo "RUSTFLAGS=-C link-arg=-fuse-ld=lld" >> "$GITHUB_ENV"
 
-      # In addition to installing dependencies, first make sure System Integrity Protection (SIP)
-      # is disabled on this MacOS runner. This is needed to allow the NSS libraries to be loaded
-      # from the build directory and avoid various other test failures. This seems to always be
-      # the case on any macos-13 runner, but not consistently on macos-latest (which is currently
-      # macos-12, FWIW).
       - name: Install dependencies (MacOS)
         if: runner.os == 'MacOS'
         run: |
-          csrutil status | grep disabled
-          brew install ninja mercurial llvm
+          brew update
+          brew install llvm nss
           echo "/opt/homebrew/opt/llvm/bin" >> "$GITHUB_PATH"
-          ln -s /opt/homebrew/bin/python3 /opt/homebrew/bin/python
-          # python3 -m pip install gyp-next
-          # Above does not work, since pypi only has gyp 0.15.0, which is too old
-          # for the homebrew python3. Install from source instead.
-          python3 -m pip install git+https://github.com/nodejs/gyp-next
-          python3 -m pip install packaging
-          echo "$(python3 -m site --user-base)/bin" >> "$GITHUB_PATH"
           echo "RUSTFLAGS=-C link-arg=-fuse-ld=lld" >> "$GITHUB_ENV"
 
-      - name: Use MSYS2 environment and install more dependencies (Windows)
+      - name: Install dependencies (Windows)
         if: runner.os == 'Windows'
         run: |
           # shellcheck disable=SC2028
           {
-            echo "C:\\msys64\\usr\\bin"
-            echo "C:\\msys64\\mingw64\\bin"
+            echo C:/msys64/usr/bin
+            echo C:/msys64/mingw64/bin
           } >> "$GITHUB_PATH"
           /c/msys64/usr/bin/pacman -S --noconfirm nsinstall
           python3 -m pip install git+https://github.com/nodejs/gyp-next

diff --git a/neqo-crypto/Cargo.toml b/neqo-crypto/Cargo.toml
@@ -21,6 +21,7 @@ neqo-common = { path = "../neqo-common" }
 # Sync with https://searchfox.org/mozilla-central/source/Cargo.lock 2024-02-08
 bindgen = { version = "0.69", default-features = false, features = ["runtime"] }
 mozbuild = { version = "0.1", default-features = false, optional = true }
+semver = { version = "1.0", default-features = false }
 serde = { version = "1.0", default-features = false }
 serde_derive = { version = "1.0", default-features = false }
 toml = { version = "0.5", default-features = false }

diff --git a/neqo-crypto/bindings/bindings.toml b/neqo-crypto/bindings/bindings.toml
@@ -265,8 +265,3 @@ enums = [
 [nspr_time]
 types = ["PRTime"]
 functions = ["PR_Now"]
-
-[mozpkix]
-cplusplus = true
-types = ["mozilla::pkix::ErrorCode"]
-enums = ["mozilla::pkix::ErrorCode"]
diff --git a/neqo-crypto/bindings/mozpkix.hpp b/neqo-crypto/bindings/mozpkix.hpp
diff --git a/neqo-crypto/build.rs b/neqo-crypto/build.rs
@@ -12,8 +12,11 @@ use std::{
 };
 
 use bindgen::Builder;
+use semver::{Version, VersionReq};
 use serde_derive::Deserialize;
 
+include!("min_version.rs");
+
 const BINDINGS_DIR: &str = "bindings";
 const BINDINGS_CONFIG: &str = "bindings.toml";
 
@@ -295,7 +298,54 @@ fn build_bindings(base: &str, bindings: &Bindings, flags: &[String], gecko: bool
         .expect("couldn't write bindings");
 }
 
-fn setup_standalone() -> Vec<String> {
+fn pkg_config() -> Vec<String> {
+    let modversion = Command::new("pkg-config")
+        .args(["--modversion", "nss"])
+        .output()
+        .expect("pkg-config reports NSS as absent")
+        .stdout;
+    let modversion = String::from_utf8(modversion).expect("non-UTF8 from pkg-config");
+    let modversion = modversion.trim();
+    // The NSS version number does not follow semver numbering, because it omits the patch version
+    // when that's 0. Deal with that.
+    let modversion_for_cmp = if modversion.chars().filter(|c| *c == '.').count() == 1 {
+        modversion.to_owned() + ".0"
+    } else {
+        modversion.to_owned()
+    };
+    let modversion_for_cmp =
+        Version::parse(&modversion_for_cmp).expect("NSS version not in semver format");
+    let version_req = VersionReq::parse(&(">=".to_owned() + MINIMUM_NSS_VERSION.trim())).unwrap();
+    assert!(
+        version_req.matches(&modversion_for_cmp),
+        "neqo has NSS version requirement {version_req}, found {modversion}"
+    );
+
+    let cfg = Command::new("pkg-config")
+        .args(["--cflags", "--libs", "nss"])
+        .output()
+        .expect("NSS flags not returned by pkg-config")
+        .stdout;
+    let cfg_str = String::from_utf8(cfg).expect("non-UTF8 from pkg-config");
+
+    let mut flags: Vec<String> = Vec::new();
+    for f in cfg_str.split(' ') {
+        if let Some(include) = f.strip_prefix("-I") {
+            flags.push(String::from(f));
+            println!("cargo:include={include}");
+        } else if let Some(path) = f.strip_prefix("-L") {
+            println!("cargo:rustc-link-search=native={path}");
+        } else if let Some(lib) = f.strip_prefix("-l") {
+            println!("cargo:rustc-link-lib=dylib={lib}");
+        } else {
+            println!("Warning: Unknown flag from pkg-config: {f}");
+        }
+    }
+
+    flags
+}
+
+fn setup_standalone(_nss: &str) -> Vec<String> {
     setup_clang();
 
     println!("cargo:rerun-if-env-changed=NSS_DIR");
@@ -406,8 +456,10 @@ fn setup_for_gecko() -> Vec<String> {
 fn main() {
     let flags = if cfg!(feature = "gecko") {
         setup_for_gecko()
+    } else if let Ok(nss_dir) = env::var("NSS_DIR") {
+        setup_standalone(&nss_dir)
     } else {
-        setup_standalone()
+        pkg_config()
     };
 
     let config_file = PathBuf::from(BINDINGS_DIR).join(BINDINGS_CONFIG);

diff --git a/neqo-crypto/min_version.rs b/neqo-crypto/min_version.rs
@@ -0,0 +1,9 @@
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+/// The minimum version of NSS that is required by this version of neqo.
+/// Note that the string may contain whitespace at the beginning and/or end.
+const MINIMUM_NSS_VERSION: &str = include_str!("min_version.txt");
diff --git a/neqo-crypto/min_version.txt b/neqo-crypto/min_version.txt
@@ -0,0 +1 @@
+3.98
diff --git a/neqo-crypto/src/err.rs b/neqo-crypto/src/err.rs
@@ -16,13 +16,39 @@ mod codes {
     #![allow(non_snake_case)]
     include!(concat!(env!("OUT_DIR"), "/nss_secerr.rs"));
     include!(concat!(env!("OUT_DIR"), "/nss_sslerr.rs"));
-    include!(concat!(env!("OUT_DIR"), "/mozpkix.rs"));
 }
-pub use codes::{mozilla_pkix_ErrorCode as mozpkix, SECErrorCodes as sec, SSLErrorCodes as ssl};
+pub use codes::{SECErrorCodes as sec, SSLErrorCodes as ssl};
 pub mod nspr {
     include!(concat!(env!("OUT_DIR"), "/nspr_err.rs"));
 }
 
+pub mod mozpkix {
+    // These are manually extracted from the many bindings generated
+    // by bindgen when provided with the simple header:
+    // #include "mozpkix/pkixnss.h"
+
+    #[allow(non_camel_case_types)]
+    pub type mozilla_pkix_ErrorCode = ::std::os::raw::c_int;
+    pub const MOZILLA_PKIX_ERROR_KEY_PINNING_FAILURE: mozilla_pkix_ErrorCode = -16384;
+    pub const MOZILLA_PKIX_ERROR_CA_CERT_USED_AS_END_ENTITY: mozilla_pkix_ErrorCode = -16383;
+    pub const MOZILLA_PKIX_ERROR_INADEQUATE_KEY_SIZE: mozilla_pkix_ErrorCode = -16382;
+    pub const MOZILLA_PKIX_ERROR_V1_CERT_USED_AS_CA: mozilla_pkix_ErrorCode = -16381;
+    pub const MOZILLA_PKIX_ERROR_NO_RFC822NAME_MATCH: mozilla_pkix_ErrorCode = -16380;
+    pub const MOZILLA_PKIX_ERROR_NOT_YET_VALID_CERTIFICATE: mozilla_pkix_ErrorCode = -16379;
+    pub const MOZILLA_PKIX_ERROR_NOT_YET_VALID_ISSUER_CERTIFICATE: mozilla_pkix_ErrorCode = -16378;
+    pub const MOZILLA_PKIX_ERROR_SIGNATURE_ALGORITHM_MISMATCH: mozilla_pkix_ErrorCode = -16377;
+    pub const MOZILLA_PKIX_ERROR_OCSP_RESPONSE_FOR_CERT_MISSING: mozilla_pkix_ErrorCode = -16376;
+    pub const MOZILLA_PKIX_ERROR_VALIDITY_TOO_LONG: mozilla_pkix_ErrorCode = -16375;
+    pub const MOZILLA_PKIX_ERROR_REQUIRED_TLS_FEATURE_MISSING: mozilla_pkix_ErrorCode = -16374;
+    pub const MOZILLA_PKIX_ERROR_INVALID_INTEGER_ENCODING: mozilla_pkix_ErrorCode = -16373;
+    pub const MOZILLA_PKIX_ERROR_EMPTY_ISSUER_NAME: mozilla_pkix_ErrorCode = -16372;
+    pub const MOZILLA_PKIX_ERROR_ADDITIONAL_POLICY_CONSTRAINT_FAILED: mozilla_pkix_ErrorCode =
+        -16371;
+    pub const MOZILLA_PKIX_ERROR_SELF_SIGNED_CERT: mozilla_pkix_ErrorCode = -16370;
+    pub const MOZILLA_PKIX_ERROR_MITM_DETECTED: mozilla_pkix_ErrorCode = -16369;
+    pub const END_OF_LIST: mozilla_pkix_ErrorCode = -16368;
+}
+
 pub type Res<T> = Result<T, Error>;
 
 #[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq)]

diff --git a/neqo-crypto/src/lib.rs b/neqo-crypto/src/lib.rs
@@ -59,7 +59,7 @@ pub use self::{
     ssl::Opt,
 };
 
-const MINIMUM_NSS_VERSION: &str = "3.97";
+include!("../min_version.rs");
 
 #[allow(non_upper_case_globals, clippy::redundant_static_lifetimes)]
 #[allow(clippy::upper_case_acronyms)]