From 8ec74e4cc2255ec90b82508e7bebf41df411a7a8 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 26 Mar 2024 02:49:11 +0200 Subject: [PATCH] build: Use system NSS when possible (#1739) * build: Use system-installed NSS instead of building our own Fixes #1711 * Update CI * Fix docs * Fix Dockerfile * Fix * build-essential * Try and search for nss * Try to get newest versions * More fixes * Restore Windows link.exe fix * Install pkg-config * Remove MSYS2 linker * Retain ability to build NSS from source * Update Linux instructions * Try and find MSYS2 library path * Retry * Again * Again * Again * Again * Again * Again * Again * Again * Again * Again * Revert many things, keep building NSS from source unless system version is OK * Fixes * Fixes * debug * Debug * Fixes * Compare versions with the `semver` crate * Use NSS version from code in CI * File has other name * Update .github/actions/nss/action.yml Co-authored-by: Martin Thomson Signed-off-by: Lars Eggert * Update neqo-crypto/build.rs Co-authored-by: Martin Thomson Signed-off-by: Lars Eggert * Update neqo-crypto/build.rs Co-authored-by: Martin Thomson Signed-off-by: Lars Eggert * Address code review comments. Not ready yet. Need to determine what to do in `nss_dir()`. See comments. --------- Signed-off-by: Lars Eggert Co-authored-by: Martin Thomson --- .github/actions/nss/action.yml | 33 ++++++++++++++++++ .github/workflows/check.yml | 22 +++--------- neqo-crypto/Cargo.toml | 1 + neqo-crypto/bindings/bindings.toml | 5 --- neqo-crypto/bindings/mozpkix.hpp | 1 - neqo-crypto/build.rs | 56 ++++++++++++++++++++++++++++-- neqo-crypto/min_version.rs | 9 +++++ neqo-crypto/min_version.txt | 1 + neqo-crypto/src/err.rs | 30 ++++++++++++++-- neqo-crypto/src/lib.rs | 2 +- 10 files changed, 132 insertions(+), 28 deletions(-) delete mode 100644 neqo-crypto/bindings/mozpkix.hpp create mode 100644 neqo-crypto/min_version.rs create mode 100644 neqo-crypto/min_version.txt diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 23232ebc13..ec6f13eaf8 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -16,16 +16,46 @@ inputs: runs: using: composite steps: + - name: Check system NSS version + shell: bash + run: | + if ! command -v pkg-config &> /dev/null; then + echo "BUILD_NSS=1" >> "$GITHUB_ENV" + exit 0 + fi + if ! pkg-config --exists nss; then + echo "BUILD_NSS=1" >> "$GITHUB_ENV" + exit 0 + fi + NSS_VERSION="$(pkg-config --modversion nss)" + if [ "$?" -ne 0 ]; then + echo "BUILD_NSS=1" >> "$GITHUB_ENV" + exit 0 + fi + NSS_MAJOR=$(echo "$NSS_VERSION" | cut -d. -f1) + NSS_MINOR=$(echo "$NSS_VERSION" | cut -d. -f2) + REQ_NSS_MAJOR=$(cat neqo-crypto/min_version.txt | cut -d. -f1) + REQ_NSS_MINOR=$(cat neqo-crypto/min_version.txt | cut -d. -f2) + if [ "$NSS_MAJOR" -lt "REQ_NSS_MAJOR" ] || [ "$NSS_MAJOR" -eq "REQ_NSS_MAJOR" -a "$NSS_MINOR" -lt "REQ_NSS_MINOR"]; then + echo "System NSS is too old: $NSS_VERSION" + echo "BUILD_NSS=1" >> "$GITHUB_ENV" + exit 0 + fi + echo "System NSS is suitable: $NSS_VERSION" + echo "BUILD_NSS=0" >> "$GITHUB_ENV" + # Ideally, we'd use this. But things are sufficiently flaky that we're better off # trying both hg and git. Leaving this here in case we want to re-try in the future. # # - name: Checkout NSPR + # if: env.BUILD_NSS == '1' # uses: actions/checkout@v4 # with: # repository: "nss-dev/nspr" # path: ${{ github.workspace }}/nspr # - name: Checkout NSS + # if: env.BUILD_NSS == '1' # uses: actions/checkout@v4 # with: # repository: "nss-dev/nss" @@ -33,18 +63,21 @@ runs: - name: Checkout NSPR shell: bash + if: env.BUILD_NSS == '1' run: | hg clone https://hg.mozilla.org/projects/nspr "${{ github.workspace }}/nspr" || \ git clone --depth=1 https://github.com/nss-dev/nspr "${{ github.workspace }}/nspr" - name: Checkout NSS shell: bash + if: env.BUILD_NSS == '1' run: | hg clone https://hg.mozilla.org/projects/nss "${{ github.workspace }}/nss" || \ git clone --depth=1 https://github.com/nss-dev/nss "${{ github.workspace }}/nss" - name: Build shell: bash + if: env.BUILD_NSS == '1' run: | if [ "${{ inputs.type }}" != "Debug" ]; then # We want to do an optimized build for accurate CPU profiling, but diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 9dc8ff2b7f..4e47961d8e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -49,33 +49,21 @@ jobs: sudo apt-get install -y --no-install-recommends gyp mercurial ninja-build lld echo "RUSTFLAGS=-C link-arg=-fuse-ld=lld" >> "$GITHUB_ENV" - # In addition to installing dependencies, first make sure System Integrity Protection (SIP) - # is disabled on this MacOS runner. This is needed to allow the NSS libraries to be loaded - # from the build directory and avoid various other test failures. This seems to always be - # the case on any macos-13 runner, but not consistently on macos-latest (which is currently - # macos-12, FWIW). - name: Install dependencies (MacOS) if: runner.os == 'MacOS' run: | - csrutil status | grep disabled - brew install ninja mercurial llvm + brew update + brew install llvm nss echo "/opt/homebrew/opt/llvm/bin" >> "$GITHUB_PATH" - ln -s /opt/homebrew/bin/python3 /opt/homebrew/bin/python - # python3 -m pip install gyp-next - # Above does not work, since pypi only has gyp 0.15.0, which is too old - # for the homebrew python3. Install from source instead. - python3 -m pip install git+https://github.com/nodejs/gyp-next - python3 -m pip install packaging - echo "$(python3 -m site --user-base)/bin" >> "$GITHUB_PATH" echo "RUSTFLAGS=-C link-arg=-fuse-ld=lld" >> "$GITHUB_ENV" - - name: Use MSYS2 environment and install more dependencies (Windows) + - name: Install dependencies (Windows) if: runner.os == 'Windows' run: | # shellcheck disable=SC2028 { - echo "C:\\msys64\\usr\\bin" - echo "C:\\msys64\\mingw64\\bin" + echo C:/msys64/usr/bin + echo C:/msys64/mingw64/bin } >> "$GITHUB_PATH" /c/msys64/usr/bin/pacman -S --noconfirm nsinstall python3 -m pip install git+https://github.com/nodejs/gyp-next diff --git a/neqo-crypto/Cargo.toml b/neqo-crypto/Cargo.toml index 588d084741..d2f70a5714 100644 --- a/neqo-crypto/Cargo.toml +++ b/neqo-crypto/Cargo.toml @@ -21,6 +21,7 @@ neqo-common = { path = "../neqo-common" } # Sync with https://searchfox.org/mozilla-central/source/Cargo.lock 2024-02-08 bindgen = { version = "0.69", default-features = false, features = ["runtime"] } mozbuild = { version = "0.1", default-features = false, optional = true } +semver = { version = "1.0", default-features = false } serde = { version = "1.0", default-features = false } serde_derive = { version = "1.0", default-features = false } toml = { version = "0.5", default-features = false } diff --git a/neqo-crypto/bindings/bindings.toml b/neqo-crypto/bindings/bindings.toml index 3e5c1fdf7d..72c6d524d5 100644 --- a/neqo-crypto/bindings/bindings.toml +++ b/neqo-crypto/bindings/bindings.toml @@ -265,8 +265,3 @@ enums = [ [nspr_time] types = ["PRTime"] functions = ["PR_Now"] - -[mozpkix] -cplusplus = true -types = ["mozilla::pkix::ErrorCode"] -enums = ["mozilla::pkix::ErrorCode"] diff --git a/neqo-crypto/bindings/mozpkix.hpp b/neqo-crypto/bindings/mozpkix.hpp deleted file mode 100644 index d0a6cb5861..0000000000 --- a/neqo-crypto/bindings/mozpkix.hpp +++ /dev/null @@ -1 +0,0 @@ -#include "mozpkix/pkixnss.h" \ No newline at end of file diff --git a/neqo-crypto/build.rs b/neqo-crypto/build.rs index c4c2a73e75..b0f6cf1f25 100644 --- a/neqo-crypto/build.rs +++ b/neqo-crypto/build.rs @@ -12,8 +12,11 @@ use std::{ }; use bindgen::Builder; +use semver::{Version, VersionReq}; use serde_derive::Deserialize; +include!("min_version.rs"); + const BINDINGS_DIR: &str = "bindings"; const BINDINGS_CONFIG: &str = "bindings.toml"; @@ -295,7 +298,54 @@ fn build_bindings(base: &str, bindings: &Bindings, flags: &[String], gecko: bool .expect("couldn't write bindings"); } -fn setup_standalone() -> Vec { +fn pkg_config() -> Vec { + let modversion = Command::new("pkg-config") + .args(["--modversion", "nss"]) + .output() + .expect("pkg-config reports NSS as absent") + .stdout; + let modversion = String::from_utf8(modversion).expect("non-UTF8 from pkg-config"); + let modversion = modversion.trim(); + // The NSS version number does not follow semver numbering, because it omits the patch version + // when that's 0. Deal with that. + let modversion_for_cmp = if modversion.chars().filter(|c| *c == '.').count() == 1 { + modversion.to_owned() + ".0" + } else { + modversion.to_owned() + }; + let modversion_for_cmp = + Version::parse(&modversion_for_cmp).expect("NSS version not in semver format"); + let version_req = VersionReq::parse(&(">=".to_owned() + MINIMUM_NSS_VERSION.trim())).unwrap(); + assert!( + version_req.matches(&modversion_for_cmp), + "neqo has NSS version requirement {version_req}, found {modversion}" + ); + + let cfg = Command::new("pkg-config") + .args(["--cflags", "--libs", "nss"]) + .output() + .expect("NSS flags not returned by pkg-config") + .stdout; + let cfg_str = String::from_utf8(cfg).expect("non-UTF8 from pkg-config"); + + let mut flags: Vec = Vec::new(); + for f in cfg_str.split(' ') { + if let Some(include) = f.strip_prefix("-I") { + flags.push(String::from(f)); + println!("cargo:include={include}"); + } else if let Some(path) = f.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={path}"); + } else if let Some(lib) = f.strip_prefix("-l") { + println!("cargo:rustc-link-lib=dylib={lib}"); + } else { + println!("Warning: Unknown flag from pkg-config: {f}"); + } + } + + flags +} + +fn setup_standalone(_nss: &str) -> Vec { setup_clang(); println!("cargo:rerun-if-env-changed=NSS_DIR"); @@ -406,8 +456,10 @@ fn setup_for_gecko() -> Vec { fn main() { let flags = if cfg!(feature = "gecko") { setup_for_gecko() + } else if let Ok(nss_dir) = env::var("NSS_DIR") { + setup_standalone(&nss_dir) } else { - setup_standalone() + pkg_config() }; let config_file = PathBuf::from(BINDINGS_DIR).join(BINDINGS_CONFIG); diff --git a/neqo-crypto/min_version.rs b/neqo-crypto/min_version.rs new file mode 100644 index 0000000000..9f1c8ebba4 --- /dev/null +++ b/neqo-crypto/min_version.rs @@ -0,0 +1,9 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +/// The minimum version of NSS that is required by this version of neqo. +/// Note that the string may contain whitespace at the beginning and/or end. +const MINIMUM_NSS_VERSION: &str = include_str!("min_version.txt"); diff --git a/neqo-crypto/min_version.txt b/neqo-crypto/min_version.txt new file mode 100644 index 0000000000..422c9c7093 --- /dev/null +++ b/neqo-crypto/min_version.txt @@ -0,0 +1 @@ +3.98 diff --git a/neqo-crypto/src/err.rs b/neqo-crypto/src/err.rs index 187303d2a9..8d4f239a0b 100644 --- a/neqo-crypto/src/err.rs +++ b/neqo-crypto/src/err.rs @@ -16,13 +16,39 @@ mod codes { #![allow(non_snake_case)] include!(concat!(env!("OUT_DIR"), "/nss_secerr.rs")); include!(concat!(env!("OUT_DIR"), "/nss_sslerr.rs")); - include!(concat!(env!("OUT_DIR"), "/mozpkix.rs")); } -pub use codes::{mozilla_pkix_ErrorCode as mozpkix, SECErrorCodes as sec, SSLErrorCodes as ssl}; +pub use codes::{SECErrorCodes as sec, SSLErrorCodes as ssl}; pub mod nspr { include!(concat!(env!("OUT_DIR"), "/nspr_err.rs")); } +pub mod mozpkix { + // These are manually extracted from the many bindings generated + // by bindgen when provided with the simple header: + // #include "mozpkix/pkixnss.h" + + #[allow(non_camel_case_types)] + pub type mozilla_pkix_ErrorCode = ::std::os::raw::c_int; + pub const MOZILLA_PKIX_ERROR_KEY_PINNING_FAILURE: mozilla_pkix_ErrorCode = -16384; + pub const MOZILLA_PKIX_ERROR_CA_CERT_USED_AS_END_ENTITY: mozilla_pkix_ErrorCode = -16383; + pub const MOZILLA_PKIX_ERROR_INADEQUATE_KEY_SIZE: mozilla_pkix_ErrorCode = -16382; + pub const MOZILLA_PKIX_ERROR_V1_CERT_USED_AS_CA: mozilla_pkix_ErrorCode = -16381; + pub const MOZILLA_PKIX_ERROR_NO_RFC822NAME_MATCH: mozilla_pkix_ErrorCode = -16380; + pub const MOZILLA_PKIX_ERROR_NOT_YET_VALID_CERTIFICATE: mozilla_pkix_ErrorCode = -16379; + pub const MOZILLA_PKIX_ERROR_NOT_YET_VALID_ISSUER_CERTIFICATE: mozilla_pkix_ErrorCode = -16378; + pub const MOZILLA_PKIX_ERROR_SIGNATURE_ALGORITHM_MISMATCH: mozilla_pkix_ErrorCode = -16377; + pub const MOZILLA_PKIX_ERROR_OCSP_RESPONSE_FOR_CERT_MISSING: mozilla_pkix_ErrorCode = -16376; + pub const MOZILLA_PKIX_ERROR_VALIDITY_TOO_LONG: mozilla_pkix_ErrorCode = -16375; + pub const MOZILLA_PKIX_ERROR_REQUIRED_TLS_FEATURE_MISSING: mozilla_pkix_ErrorCode = -16374; + pub const MOZILLA_PKIX_ERROR_INVALID_INTEGER_ENCODING: mozilla_pkix_ErrorCode = -16373; + pub const MOZILLA_PKIX_ERROR_EMPTY_ISSUER_NAME: mozilla_pkix_ErrorCode = -16372; + pub const MOZILLA_PKIX_ERROR_ADDITIONAL_POLICY_CONSTRAINT_FAILED: mozilla_pkix_ErrorCode = + -16371; + pub const MOZILLA_PKIX_ERROR_SELF_SIGNED_CERT: mozilla_pkix_ErrorCode = -16370; + pub const MOZILLA_PKIX_ERROR_MITM_DETECTED: mozilla_pkix_ErrorCode = -16369; + pub const END_OF_LIST: mozilla_pkix_ErrorCode = -16368; +} + pub type Res = Result; #[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq)] diff --git a/neqo-crypto/src/lib.rs b/neqo-crypto/src/lib.rs index 45f61f6127..74215110db 100644 --- a/neqo-crypto/src/lib.rs +++ b/neqo-crypto/src/lib.rs @@ -59,7 +59,7 @@ pub use self::{ ssl::Opt, }; -const MINIMUM_NSS_VERSION: &str = "3.97"; +include!("../min_version.rs"); #[allow(non_upper_case_globals, clippy::redundant_static_lifetimes)] #[allow(clippy::upper_case_acronyms)]