From 5e911d546a0d86bf12821cdac8b489ad7b6bafa0 Mon Sep 17 00:00:00 2001 From: Igor Novgorodov Date: Sat, 25 May 2024 18:12:44 +0200 Subject: [PATCH] Improve metrics, use GET, certificate validity check --- Cargo.lock | 12 +++ Cargo.toml | 5 + README.md | 13 ++- README_CARGO.md | 63 +++++++++++++ src/client.rs | 40 +++++--- src/lib.rs | 94 ++++++++----------- src/stapler.rs | 241 ++++++++++++++++++++++++++++++------------------ 7 files changed, 301 insertions(+), 167 deletions(-) create mode 100644 README_CARGO.md diff --git a/Cargo.lock b/Cargo.lock index 08b3d88..ae345d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -811,6 +811,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1024,10 +1033,13 @@ version = "0.2.0" dependencies = [ "anyhow", "arc-swap", + "base64", "bytes", "chrono", "hex-literal", "http", + "itertools 0.13.0", + "num-bigint", "prometheus", "rasn", "rasn-ocsp", diff --git a/Cargo.toml b/Cargo.toml index 5421e5b..dcf7cbb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,12 +15,16 @@ categories = [ "web-programming::http-server", ] +readme = "README_CARGO.md" + [dependencies] anyhow = "1.0" arc-swap = "1" +base64 = "0.22" bytes = "1.5" chrono = "0.4" http = "1.1" +itertools = "0.13" prometheus = "0.13" rasn = "0.15" rasn-ocsp = "0.15" @@ -43,6 +47,7 @@ x509-parser = "0.16" [dev-dependencies] hex-literal = "0.4" rustls-pemfile = "2" +num-bigint = "0.4" [lib] doctest = false diff --git a/README.md b/README.md index 442be08..0c7c099 100644 --- a/README.md +++ b/README.md @@ -6,29 +6,28 @@ OCSP stapler for Rustls. -- Standalone `Client` that can be used separately +- OCSP `Client` that can be used separately - `Stapler` wraps `Arc` trait object and automatically staples all certificates provided by it `Stapler::new()` spawns background worker using `tokio::spawn` so it must be executed in the Tokio context. -## Example +Please see the [docs](https://docs.rs/ocsp-stapler) for more details. -```rust -// Inner service that provides certificates to Rustls, can be anything that -// implements ResolvesServerCert +## Example +```rust,ignore +// Inner service that provides certificates to Rustls, can be anything let ckey: CertifiedKey = ...; let mut inner = rustls::server::ResolvesServerCertUsingSni::new(); inner.add("crates.io", ckey).unwrap(); let stapler = Arc::new(ocsp_stapler::Stapler::new(inner)); +// Then you can build & use server_config wherever applicable let server_config = rustls::server::ServerConfig::builder() .with_no_client_auth() .with_cert_resolver(stapler.clone()); -// Then you can use server_config wherever applicable - // Stop the background worker to clean up stapler.stop().await; ``` diff --git a/README_CARGO.md b/README_CARGO.md new file mode 100644 index 0000000..e75df29 --- /dev/null +++ b/README_CARGO.md @@ -0,0 +1,63 @@ +# ocsp-stapler + +The `ocsp-stapler` crate provides two structs: `Client` and `Stapler`. + +## `Client` +[`client::Client`](client) is an OCSP client that can be used to query the OCSP responders of the Certificate Authorities. It tries to mostly conform to the [lightweight OCSP profile](https://datatracker.ietf.org/doc/html/rfc5019) + +- Currently only SHA-1 digest for OCSP request is supported since it's the only one that LetsEncrypt uses +- Requests <= 255 bytes will be sent using GET and Base64, otherwise POST + +## `Stapler` +[`stapler::Stapler`](stapler) uses [`client::Client`](client) internally and provides a Rustls-compatible API to attach (staple) OCSP responses to the certificates. + +It wraps whatever that implements Rustls' [`rustls::server::ResolvesServerCert`](https://docs.rs/rustls/latest/rustls/server/trait.ResolvesServerCert.html) trait and also implements the same trait itself. + +The workflow is the following: +- [`stapler::Stapler`](stapler) receives a `ClientHello` from Rustls and forwards it to the wrapped resolver to retrieve the certificate chain +- It calculates the SHA-1 fingerprint over the whole end-entity certificate and uses that to check if it has the same certificate +in the local storage: + - If not, then it sends the certificate to the background worker for eventual processing & stapling. +Meanwhilte it returns to Rustls the original unstapled certificate + - If found, it responds with a stapled version of the certificate + +Since the certificates are only stapled eventually then the `Must-Staple` marked certificates will not work out of the box - first request for them will always be failed by the client. Maybe later an API to pre-staple them will be added. + +Background worker duties: +- Receieves the certificates from `Stapler`, processes them and inserts into the local storage +- Wakes up every minute (or when a new certificate is added) to do the following: +- Obtain OCSP responses for newly added certificates +- Renew the OCSP responses that are already past 50% of their validity interval +- Check for expired certificates & purge them +- Check for expired OCSP responses and clear them +- Post an updated version of storage that is shared with `Stapler` + +Background worker is spawned by `Stapler::new()` using `tokio::spawn` so it must be executed in Tokio context. +It runs indefinitely unless stopped with `Stapler::stop()`. + +Other notes: +- Stapler does not check the certificate validity (i.e. does not traverse the chain up to the root) +- Certificates without the issuer's certificate are passed through as-is since we can't query the OCSP without access to the issuer's public key + +### Metrics + +Stapler supports a few Prometheus metrics - create it using one of `new_..._with_registry()` constructors and provide a Prometheus `Registry` reference to register the metrics in. + +# Example + +```rust,ignore +// Inner service that provides certificates to Rustls, can be anything +let ckey: CertifiedKey = ...; +let mut inner = rustls::server::ResolvesServerCertUsingSni::new(); +inner.add("crates.io", ckey).unwrap(); + +let stapler = Arc::new(ocsp_stapler::Stapler::new(inner)); + +// Then you can build & use server_config wherever applicable +let server_config = rustls::server::ServerConfig::builder() + .with_no_client_auth() + .with_cert_resolver(stapler.clone()); + +// Stop the background worker to clean up +stapler.stop().await; +``` diff --git a/src/client.rs b/src/client.rs index 30637ff..42860ca 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,6 +1,7 @@ use std::time::Duration; use anyhow::{anyhow, Context, Error}; +use base64::prelude::*; use bytes::Bytes; use http::{header::CONTENT_TYPE, StatusCode}; use rasn::types::Oid; @@ -13,13 +14,17 @@ use sha1::{Digest, Sha1}; use url::Url; use x509_parser::{oid_registry::OID_PKIX_ACCESS_DESCRIPTOR_OCSP, prelude::*}; -use super::OcspValidity; +use super::Validity; /// OCSP response pub struct Response { + /// Raw OCSP response body. + /// Useful e.g. for stapling pub raw: Vec, + /// OCSP response validity interval + pub ocsp_validity: Validity, + /// Certificate revocation status pub cert_status: CertStatus, - pub ocsp_validity: OcspValidity, } /// Extracts OCSP responder URL from the given certificate @@ -133,7 +138,8 @@ impl Client { Self { http_client } } - /// Fetches the raw OCSP response for the given certificate chain + /// Fetches the raw OCSP response for the given certificate chain. + /// Certificates must be DER-encoded. pub async fn query_raw(&self, cert: &[u8], issuer: &[u8]) -> Result { // Prepare OCSP request & URL let (ocsp_request, url) = @@ -144,11 +150,22 @@ impl Client { .map_err(|e| anyhow!("unable to serialize OCSP request: {e}"))?; // Execute HTTP request - let response = self - .http_client - .post(url) + // Send using GET if it's <= 255 bytes as required by + // https://datatracker.ietf.org/doc/html/rfc5019 + let request = if ocsp_request.len() <= 255 { + // Encode the request as Base64 and append it to the URL + let ocsp_request = BASE64_STANDARD.encode(ocsp_request); + let url = url + .join(&ocsp_request) + .context("unable to append base64 request")?; + + self.http_client.get(url) + } else { + self.http_client.post(url).body(ocsp_request) + }; + + let response = request .header(CONTENT_TYPE, "application/ocsp-request") - .body(ocsp_request) .send() .await .context("HTTP request failed")?; @@ -169,7 +186,8 @@ impl Client { Ok(ocsp_response) } - /// Fetches the raw OCSP response and returns its validity & status + /// Fetches the raw OCSP response and returns its validity & status. + /// Certificates must be DER-encoded. pub async fn query(&self, cert: &[u8], issuer: &[u8]) -> Result { let ocsp_response = self .query_raw(cert, issuer) @@ -206,9 +224,9 @@ impl Client { Ok(Response { raw, cert_status: resp.cert_status, - ocsp_validity: OcspValidity { - this_update: resp.this_update, - next_update: resp + ocsp_validity: Validity { + not_before: resp.this_update, + not_after: resp .next_update .ok_or_else(|| anyhow!("No next-update field in the response"))?, }, diff --git a/src/lib.rs b/src/lib.rs index 311bf34..c83a8aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,74 +1,52 @@ #![warn(clippy::all)] #![warn(clippy::nursery)] -//! # ocsp-stapler -//! -//! The `ocsp-stapler` crate provides two structs: `Client` and `Stapler`. -//! -//! ## `Client` -//! `Client` is an OCSP client that you can use to do OCSP requests to OCSP responders of the Certificate Authorities. -//! -//! ## `Stapler` -//! `Stapler` uses `Client` internally and provides a Rustls-compatible API to attach (staple) OCSP responses to the certificates. -//! It wraps whatever that implements Rustls' `ResolvesServerCert` trait and also implements the same trait. -//! -//! The workflow is the following: -//! - `Stapler` receives a `ClientHello` from Rustls and forwards it to ther wrapped trait object to get the certificate chain -//! - It calculates the SHA-1 fingerprint over the whole end-entity certificate and uses that to check if it has the same certificate -//! in the local storage -//! - If not - it sends the certificate to the background worker for eventual processing & stapling. -//! Meanwhilte it returns to Rustls the original unstapled certificate -//! - If found - it respondes with a stapled version of the certificate -//! -//! Background worker duties: -//! - Receieves the certificates from `Stapler`, processes them and inserts into the local storage -//! - Wakes up every 60s (or when a new certificate is added) to do the following: -//! - Renew the OCSP responses that are already past 50% of their validity interval -//! - Check for expired certificates & purge them -//! - Check for expired OCSP responses and clear them -//! - Post an updated version of storage that is shared with `Stapler` -//! -//! Background worker is spawned by `Stapler::new()` using `tokio::spawn` so it must be executed in Tokio context. -//! It runs indefinitely unless stopped with `Stapler.stop()`. -//! -//! # Example -//! -//! ```rust,ignore -//! // Inner service that provides certificates to Rustls, can be anything -//! let ckey: CertifiedKey = ...; -//! let mut inner = rustls::server::ResolvesServerCertUsingSni::new(); -//! inner.add("crates.io", ckey).unwrap(); -//! -//! let stapler = Arc::new(ocsp_stapler::Stapler::new(inner)); -//! -//! let server_config = rustls::server::ServerConfig::builder() -//! .with_no_client_auth() -//! .with_cert_resolver(stapler.clone()); -//! -//! // Then you can use server_config wherever applicable -//! -//! // Stop the background worker to clean up -//! stapler.stop().await; -//! ``` - pub mod client; pub mod stapler; pub use client::Client; pub use stapler::Stapler; -use chrono::{DateTime, FixedOffset}; +use anyhow::{anyhow, Error}; +use chrono::{DateTime, FixedOffset, TimeDelta}; +use x509_parser::certificate; + +/// Allow some time inconsistencies +pub(crate) const LEEWAY: TimeDelta = TimeDelta::minutes(5); /// OCSP response validity interval -#[derive(Clone)] -pub struct OcspValidity { - pub this_update: DateTime, - pub next_update: DateTime, +#[derive(Clone, Debug)] +pub struct Validity { + pub not_before: DateTime, + pub not_after: DateTime, } -impl OcspValidity { - // Check if we're already past the half of this validity duration +impl TryFrom<&certificate::Validity> for Validity { + type Error = Error; + fn try_from(v: &certificate::Validity) -> Result { + let not_before = DateTime::from_timestamp(v.not_before.timestamp(), 0) + .ok_or_else(|| anyhow!("unable to parse not_before"))? + .into(); + + let not_after = DateTime::from_timestamp(v.not_after.timestamp(), 0) + .ok_or_else(|| anyhow!("unable to parse not_after"))? + .into(); + + Ok(Self { + not_before, + not_after, + }) + } +} + +impl Validity { + /// Check if we're already past the half of this validity duration pub fn time_to_update(&self, now: DateTime) -> bool { - now >= self.this_update + ((self.next_update - self.this_update) / 2) + now >= self.not_before + ((self.not_after - self.not_before) / 2) + } + + /// Check if it's valid + pub fn valid(&self, now: DateTime) -> bool { + now >= (self.not_before - LEEWAY) && now <= (self.not_after + LEEWAY) } } diff --git a/src/stapler.rs b/src/stapler.rs index bac4513..fd2ba50 100644 --- a/src/stapler.rs +++ b/src/stapler.rs @@ -1,13 +1,14 @@ use std::{ collections::BTreeMap, - fmt, + fmt::{self, Display}, sync::Arc, time::{Duration, Instant}, }; use anyhow::{anyhow, Context, Error}; use arc_swap::ArcSwapOption; -use chrono::{DateTime, FixedOffset, TimeDelta, Utc}; +use chrono::{DateTime, FixedOffset, Utc}; +use itertools::Itertools; use prometheus::{ register_histogram_with_registry, register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry, Histogram, IntCounterVec, IntGaugeVec, Registry, @@ -21,10 +22,10 @@ use rustls::{ use sha1::{Digest, Sha1}; use tokio::sync::mpsc; use tokio_util::{sync::CancellationToken, task::TaskTracker}; -use tracing::warn; +use tracing::{info, warn}; use x509_parser::prelude::*; -use super::{client::Client, OcspValidity}; +use super::{client::Client, Validity, LEEWAY}; type Storage = BTreeMap; @@ -39,15 +40,31 @@ impl From<&CertificateDer<'_>> for Fingerprint { } } +#[derive(PartialEq, Eq)] +enum RefreshResult { + StillValid, + Refreshed, +} + #[derive(Clone)] struct Cert { ckey: Arc, - cert_validity: DateTime, - ocsp_validity: Option, + subject: String, + status: CertStatus, + cert_validity: Validity, + ocsp_validity: Option, +} + +impl Display for Cert { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.subject) + } } +#[derive(Clone)] struct Metrics { - requests_count: IntCounterVec, + resolves: IntCounterVec, + ocsp_requests: IntCounterVec, refresh_duration: Histogram, certificate_count: IntGaugeVec, } @@ -55,10 +72,18 @@ struct Metrics { impl Metrics { fn new(registry: &Registry) -> Self { Self { - requests_count: register_int_counter_vec_with_registry!( + resolves: register_int_counter_vec_with_registry!( + format!("ocsp_resolves_total"), + format!("Counts the number of certificate resolve requests"), + &["stapled"], + registry + ) + .unwrap(), + + ocsp_requests: register_int_counter_vec_with_registry!( format!("ocsp_requests_total"), format!("Counts the number of OCSP requests"), - &["result"], + &["status"], registry ) .unwrap(), @@ -88,6 +113,7 @@ pub struct Stapler { inner: Arc, tracker: TaskTracker, token: CancellationToken, + metrics: Option, } impl Stapler { @@ -111,13 +137,14 @@ impl Stapler { let storage = Arc::new(ArcSwapOption::empty()); let tracker = TaskTracker::new(); let token = CancellationToken::new(); + let metrics = registry.map(Metrics::new); let mut actor = StaplerActor { client, storage: BTreeMap::new(), rx, published: storage.clone(), - metrics: registry.map(Metrics::new), + metrics: metrics.clone(), }; // Spawn the background task @@ -132,6 +159,7 @@ impl Stapler { inner, tracker, token, + metrics, } } @@ -141,42 +169,30 @@ impl Stapler { self.tracker.close(); self.tracker.wait().await; } -} - -/// Debug is required for ResolvesServerCert trait -impl fmt::Debug for Stapler { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "OcspStapler") - } -} - -impl ResolvesServerCert for Stapler { - fn resolve(&self, client_hello: ClientHello) -> Option> { - // Try to get the cert from the inner resolver - let ckey = self.inner.resolve(client_hello)?; + fn staple(&self, ckey: Arc) -> (Arc, bool) { // Check that we have at least two certificates in the chain. // Otherwise we can't staple it since we need an issuer certificate too. // In this case just return it back unstapled. if ckey.cert.len() < 2 { - return Some(ckey); + return (ckey, false); } // Compute the fingerprint let fp = Fingerprint::from(&ckey.cert[0]); - // See if the storage has been published + // See if the storage is already published if let Some(map) = self.storage.load_full() { - // Check if we have a certificate with this fingerprint already + // Check if we have a certificate with this fingerprint if let Some(v) = map.get(&fp) { // Check if its OCSP validity is set // Otherwise it hasn't been yet stapled or OCSP response has expired if v.ocsp_validity.is_some() { - return Some(v.ckey.clone()); + return (v.ckey.clone(), true); } // Return unstapled - return Some(ckey); + return (ckey, false); } } @@ -185,10 +201,73 @@ impl ResolvesServerCert for Stapler { let _ = self.tx.try_send((fp, ckey.clone())); // Return the original unstapled cert + (ckey, false) + } +} + +/// Debug is required for ResolvesServerCert trait +impl fmt::Debug for Stapler { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "OcspStapler") + } +} + +impl ResolvesServerCert for Stapler { + fn resolve(&self, client_hello: ClientHello) -> Option> { + // Try to get the cert from the wrapped resolver + let ckey = self.inner.resolve(client_hello)?; + + // Process it through stapler + let (ckey, stapled) = self.staple(ckey); + + // Record metrics + if let Some(v) = &self.metrics { + v.resolves + .with_label_values(&[if stapled { "yes" } else { "no" }]) + .inc(); + } + Some(ckey) } } +async fn refresh_certificate( + client: &Client, + now: DateTime, + cert: &mut Cert, +) -> Result { + // Check if this OCSP response is still valid + if let Some(x) = &cert.ocsp_validity { + if !x.time_to_update(now) { + return Ok(RefreshResult::StillValid); + } + } + + // Stapler::resolve() makes sure that we have at least two certificates in the chain + let end_entity = cert.ckey.cert[0].as_ref(); + let issuer = cert.ckey.cert[1].as_ref(); + + // Query the OCSP responder + let resp = client + .query(end_entity, issuer) + .await + .context("unable to perform OCSP request")?; + + if !resp.ocsp_validity.valid(now) { + return Err(anyhow!("the OCSP response is not valid at current time")); + } + + // Update the OCSP response on the key + let mut ckey = cert.ckey.as_ref().clone(); + ckey.ocsp = Some(resp.raw); + + // Update values + cert.ckey = Arc::new(ckey); + cert.status = resp.cert_status; + + Ok(RefreshResult::Refreshed) +} + struct StaplerActor { client: Client, storage: Storage, @@ -198,84 +277,59 @@ struct StaplerActor { } impl StaplerActor { - async fn refresh(&mut self) { + async fn refresh(&mut self, now: DateTime) { if self.storage.is_empty() { return; } - let now: DateTime = Utc::now().into(); + let start = Instant::now(); // Remove all expired certificates from the storage to free up resources - self.storage.retain(|_, v| v.cert_validity > now); + self.storage.retain(|_, v| v.cert_validity.valid(now)); - let start = Instant::now(); + for cert in self.storage.values_mut() { + let r = refresh_certificate(&self.client, now, cert).await; - let mut revoked = 0; - for v in self.storage.values_mut() { - if let Some(x) = &v.ocsp_validity { - // See if this OCSP response is still valid - if !x.time_to_update(now) { - continue; - } + // Record the result + if let Some(v) = &self.metrics { + v.ocsp_requests + .with_label_values(&[if r.is_err() { "error" } else { "ok" }]) + .inc() + }; - // If the validity is about to expire - clear it - // This makes sure we don't serve expired OCSP responses in Stapler::resolve() - if x.next_update - now < TimeDelta::hours(1) { - v.ocsp_validity = None + match r { + Ok(v) => { + if v == RefreshResult::Refreshed { + info!("OCSP-Stapler: certificate [{cert}] was refreshed"); + } } + Err(e) => warn!("OCSP-Stapler: unable to refresh certificate [{cert}]: {e:#}"), } - // Stapler::resolve() makes sure that we have at least two certificates in the chain - let cert = v.ckey.cert[0].as_ref(); - let issuer = v.ckey.cert[1].as_ref(); - - // Query the OCSP responder - let resp = self.client.query(cert, issuer).await; - - self.metrics.as_ref().inspect(|x| { - x.requests_count - .with_label_values(&[if resp.is_err() { "error" } else { "ok" }]) - .inc() - }); - - let resp = match resp { - Err(e) => { - warn!("OCSP-Stapler: unable to perform OCSP request: {e:#}"); - - continue; + // If the validity is about to expire for whatever reason - clear it. + // This makes sure we don't serve expired OCSP responses in Stapler::resolve() + if let Some(v) = &cert.ocsp_validity { + if v.not_after - now < LEEWAY { + cert.ocsp_validity = None; } - - Ok(v) => v, - }; - - if let CertStatus::Revoked(x) = resp.cert_status { - warn!("OCSP-Stapler: certificate was revoked: {x:?}"); - revoked += 1; } - - // Update the OCSP response on the key - let mut ckey = v.ckey.as_ref().clone(); - ckey.ocsp = Some(resp.raw); - - // Update values - v.ckey = Arc::new(ckey); - v.ocsp_validity = Some(resp.ocsp_validity); } // Publish the updated storage version let new = Arc::new(self.storage.clone()); self.published.store(Some(new)); - if let Some(v) = &self.metrics { - v.certificate_count - .with_label_values(&["ok"]) - .set(self.storage.len() as i64); + // Record some metrics + if let Some(m) = &self.metrics { + let status = self.storage.values().map(|x| x.status.clone()).counts(); - v.certificate_count - .with_label_values(&["revoked"]) - .set(revoked); + for (k, v) in status { + m.certificate_count + .with_label_values(&[&format!("{k:?}")]) + .set(v as i64); + } - v.refresh_duration.observe(start.elapsed().as_secs_f64()); + m.refresh_duration.observe(start.elapsed().as_secs_f64()); } warn!( @@ -284,7 +338,7 @@ impl StaplerActor { ); } - async fn process_certificate( + async fn add_certificate( &mut self, fp: Fingerprint, ckey: Arc, @@ -298,18 +352,23 @@ impl StaplerActor { .context("unable to parse certificate as X.509")? .1; - let cert_validity = DateTime::from_timestamp(cert.validity.not_after.timestamp(), 0) - .ok_or_else(|| anyhow!("unable to parse NotAfter"))? - .into(); + let cert_validity = + Validity::try_from(&cert.validity).context("unable to parse certificate validity")?; + + if !cert_validity.valid(Utc::now().into()) { + return Err(anyhow!("The certificate is not valid at current time")); + } let cert = Cert { ckey: ckey.clone(), + subject: cert.subject.to_string(), + status: CertStatus::Unknown(()), cert_validity, ocsp_validity: None, }; self.storage.insert(fp, cert); - self.refresh().await; + self.refresh(Utc::now().into()).await; Ok(()) } @@ -327,12 +386,12 @@ impl StaplerActor { } _ = interval.tick() => { - self.refresh().await; + self.refresh(Utc::now().into()).await; }, msg = self.rx.recv() => { if let Some((fp, ckey)) = msg { - if let Err(e) = self.process_certificate(fp, ckey).await { + if let Err(e) = self.add_certificate(fp, ckey).await { warn!("OCSP-Stapler: unable to process certificate: {e:#}"); } }