diff --git a/.github/workflows/ssh-azure-tee-build.yaml b/.github/workflows/ssh-azure-tee-build.yaml new file mode 100644 index 000000000..c8b6f8d15 --- /dev/null +++ b/.github/workflows/ssh-azure-tee-build.yaml @@ -0,0 +1,73 @@ +name: sgx-build-azure-ssh + +on: + push: + branches: [ "quote-presentation" ] + pull_request: + branches: [ "quote-presentation" ] + +# perms needed for attestation workflow +permissions: + id-token: write + attestations: write + +jobs: + build: + runs-on: ubuntu-latest + environment: tee + steps: + - name: azure-tee-build-${{ github.sha }} + uses: appleboy/ssh-action@v1.0.3 + env: + # using personal access token until devops links acct + PAT_TOKEN: ${{ secrets.PAT_SET_ENV }} + with: + host: ${{ secrets.AZURE_TEE_BUILD_HOST }} + username: ${{ secrets.AZURE_BUILD_TEE_USERNAME }} + key: ${{ secrets.AZURE_TEE_BUILD_KEY }} + port: ${{ secrets.SSH_PORT }} + command_timeout: 10m + allenvs: true + envs: PAT_TOKEN + script: | + cd /tmp + source $HOME/.cargo/env + rm -rf ${{ github.sha }} + git clone https://github.com/tlsnotary/tlsn ${{ github.sha }} + cd ${{ github.sha }} + git checkout ${{ github.sha }} + cd crates/notary/server/config/gramine + : # listen on random_port port, set it in config + random_port=$(shuf -i 1024-65535 -n 1) + sed -i '/port*: *7047/s/7047/'$random_port'/' ../config.yaml + : # the makefile compiles the gramine manigest and notary-server + make clean + SGX=1 make start-gramine-server & + : # this next bofh bash script is just to check if the server comes up + notarypid=$! + win=0 + SECONDS=0; set -o pipefail; while [ $SECONDS -lt 300 ]; do echo -e "GET /info HTTP/1.1\r\nHost: localhost\r\nConnection: Close\r\n\r\n" | openssl s_client -quiet localhost:$random_port 2>&1 | tee s.log; if [ $? -eq 0 ]; then win=1; break; fi; sleep 2; done + if [ $win -eq 0 ]; then echo "Build Failed"; kill -SIGTERM $notarypid; exit 1; fi + cat s.log | grep '^{' | jq '. | tostring' > quotejson + : # end ugly bash + mapfile quote < quotejson + : # using http api because we dont have write access to env here + curl -L -X PATCH -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $PAT_TOKEN" -H "Connection: close" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/$GITHUB_REPOSITORY/environments/TEE/variables/AZURE_TEE_BUILD_ATTESTATION -d "{\"name\":\"AZURE_TEE_BUILD_ATTESTATION\",\"value\":$quote}" + : # gramine will keep the sgx process running, we use the gramine setting + : # sys.enable_sigterm_injection, which is insecure and for convenience + kill -SIGTERM $notarypid + : # sleep originated because the next step wouldnt have the updated env + sleep 5 + exit 0 + - name: ✨ fet quotech from gh, write it to runner + #we use http api due to the ssh runner and access to gh envs + run: | + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.PAT_SET_ENV }}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/$GITHUB_REPOSITORY/environments/TEE/variables/AZURE_TEE_BUILD_ATTESTATION > /home/runner/work/_temp/sgx-build-quote.txt + - name: get github to sign our measurement + uses: actions/attest-build-provenance@v1 + with: + subject-path: /home/runner/work/_temp/sgx-build-quote.txt + - name: upload it + uses: actions/upload-artifact@v4 + with: + path: /home/runner/work/_temp/sgx-build-quote.txt diff --git a/.gitignore b/.gitignore index f79cb086d..d73a736ed 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,10 @@ Cargo.lock *.log # metrics -*.csv \ No newline at end of file +*.csv + +# tee +**/*.sgx +**/*.sig +**/*.manifest + diff --git a/crates/notary/server/Cargo.toml b/crates/notary/server/Cargo.toml index 856177014..9c0729269 100644 --- a/crates/notary/server/Cargo.toml +++ b/crates/notary/server/Cargo.toml @@ -3,6 +3,9 @@ name = "notary-server" version = "0.1.0-alpha.6" edition = "2021" +[features] +tee_quote = ["dep:mc-sgx-dcap-types", "dep:hex", "dep:rand_chacha"] + [dependencies] tlsn-core = { workspace = true } tlsn-common = { workspace = true } @@ -48,3 +51,7 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] } uuid = { workspace = true, features = ["v4", "fast-rng"] } ws_stream_tungstenite = { workspace = true, features = ["tokio_io"] } zeroize = { workspace = true } + +mc-sgx-dcap-types = { version = "0.11.0", optional = true } +hex = { workspace = true, optional = true } +rand_chacha = { workspace = true, optional = true } diff --git a/crates/notary/server/config/gramine/Makefile b/crates/notary/server/config/gramine/Makefile new file mode 100644 index 000000000..b9304f341 --- /dev/null +++ b/crates/notary/server/config/gramine/Makefile @@ -0,0 +1,62 @@ +# notary-server testing only +ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) +ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine) + +SELF_EXE = target/release/notary-server + +.PHONY: all +all: $(SELF_EXE) notary-server.manifest +ifeq ($(SGX),1) +all: notary-server.manifest.sgx notary-server.sig +endif + +ifeq ($(DEBUG),1) +GRAMINE_LOG_LEVEL = debug +else +GRAMINE_LOG_LEVEL = error +endif + +# Note that we're compiling in release mode regardless of the DEBUG setting passed +# to Make, as compiling in debug mode results in an order of magnitude's difference in +# performance that makes testing by running a benchmark with ab painful. The primary goal +# of the DEBUG setting is to control Gramine's loglevel. +-include $(SELF_EXE).d # See also: .cargo/config.toml +$(SELF_EXE): $(ROOT_DIR)../../Cargo.toml + cargo build --bin notary-server --release --features tee_quote && ln -s ../../../../../target target + +notary-server.manifest: notary-server.manifest.template + gramine-manifest \ + -Dlog_level=$(GRAMINE_LOG_LEVEL) \ + -Darch_libdir=$(ARCH_LIBDIR) \ + -Dself_exe=$(SELF_EXE) \ + $< $@ + +# Make on Ubuntu <= 20.04 doesn't support "Rules with Grouped Targets" (`&:`), +# see the helloworld example for details on this workaround. +notary-server.manifest.sgx notary-server.sig: sgx_sign + @: + +.INTERMEDIATE: sgx_sign +sgx_sign: notary-server.manifest $(SELF_EXE) + gramine-sgx-sign \ + --manifest $< \ + --output $<.sgx + +ifeq ($(SGX),) +GRAMINE = gramine-direct +else +GRAMINE = gramine-sgx +endif + +.PHONY: start-gramine-server +start-gramine-server: all + $(GRAMINE) notary-server + +.PHONY: clean +clean: + $(RM) -rf *.token *.sig *.manifest.sgx *.manifest result-* OUTPUT + +.PHONY: distclean +distclean: clean + $(RM) -rf $(SELF_EXE) Cargo.lock + diff --git a/crates/notary/server/config/gramine/notary-server.manifest.template b/crates/notary/server/config/gramine/notary-server.manifest.template new file mode 100644 index 000000000..005f10d43 --- /dev/null +++ b/crates/notary/server/config/gramine/notary-server.manifest.template @@ -0,0 +1,48 @@ +libos.entrypoint = "{{ self_exe }}" +loader.log_level = "{{ log_level }}" + +loader.env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}" + + +# See https://gramine.readthedocs.io/en/stable/performance.html#glibc-malloc-tuning +loader.env.MALLOC_ARENA_MAX = "1" + +# For easier debugging — not strictly required to run this workload +loader.env.RUST_BACKTRACE = "full" + + +fs.mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, + { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" }, + { path = "/fixture/notary", uri = "file:../../fixture/notary" }, + { path = "/fixture/tls", uri = "file:../../fixture/tls" }, + { path = "/config", uri = "file:../../config" }, + { type = "encrypted", path = "/vault", uri = "file:vault", key_name = "_sgx_mrenclave" }, + +] + +sgx.trusted_files = [ + "file:{{ self_exe }}", + "file:{{ gramine.runtimedir() }}/", + "file:{{ arch_libdir }}/", + "file:../../config/config.yaml", + "file:../../fixture/notary/notary.pub", + "file:../../fixture/notary/notary.key", + "file:../../fixture/tls/notary.crt", + "file:../../fixture/tls/notary.key", +] +sgx.debug = true +sgx.edmm_enable = false +sgx.remote_attestation = "dcap" +sgx.max_threads = 64 +sgx.enclave_size = "2G" +sys.disallow_subprocesses = true +#### turn this off in prod, +sys.enable_sigterm_injection = true + + +#### tlsn rev +sgx.isvprodid = 7 +#### E +sgx.isvsvn = 45 + diff --git a/crates/notary/server/config/gramine/target b/crates/notary/server/config/gramine/target new file mode 120000 index 000000000..ea5d3c7b6 --- /dev/null +++ b/crates/notary/server/config/gramine/target @@ -0,0 +1 @@ +../../../../../target \ No newline at end of file diff --git a/crates/notary/server/openapi.yaml b/crates/notary/server/openapi.yaml index 9d30898af..c74a1e586 100644 --- a/crates/notary/server/openapi.yaml +++ b/crates/notary/server/openapi.yaml @@ -206,6 +206,9 @@ components: gitCommitTimestamp: description: The git commit timestamp of source code that this notary server is running type: string + quote: + description: a measurement attesting to the hardware, if available + type: object required: - "version" - "publicKey" diff --git a/crates/notary/server/src/domain.rs b/crates/notary/server/src/domain.rs index 5485be620..d649bd8ed 100644 --- a/crates/notary/server/src/domain.rs +++ b/crates/notary/server/src/domain.rs @@ -1,7 +1,8 @@ pub mod auth; pub mod cli; pub mod notary; - +#[cfg(feature = "tee_quote")] +use crate::tee::Quote; use serde::{Deserialize, Serialize}; /// Response object of the /info API @@ -16,4 +17,7 @@ pub struct InfoResponse { pub git_commit_hash: String, /// Current git commit timestamp of notary-server pub git_commit_timestamp: String, + /// hardware attestation + #[cfg(feature = "tee_quote")] + pub quote: Quote, } diff --git a/crates/notary/server/src/lib.rs b/crates/notary/server/src/lib.rs index 9353150db..15dd8a77b 100644 --- a/crates/notary/server/src/lib.rs +++ b/crates/notary/server/src/lib.rs @@ -6,6 +6,8 @@ mod server; mod server_tracing; mod service; mod signing; +#[cfg(feature = "tee_quote")] +mod tee; mod util; pub use config::{ diff --git a/crates/notary/server/src/server.rs b/crates/notary/server/src/server.rs index ac16f258d..f23eba440 100644 --- a/crates/notary/server/src/server.rs +++ b/crates/notary/server/src/server.rs @@ -46,10 +46,18 @@ use crate::{ util::parse_csv_file, }; +#[cfg(feature = "tee_quote")] +use crate::tee::{ephemeral_keypair, quote}; + /// Start a TCP server (with or without TLS) to accept notarization request for both TCP and WebSocket clients #[tracing::instrument(skip(config))] pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotaryServerError> { + //tee uses ephemeral key + #[cfg(feature = "tee_quote")] + let (attestation_key, public_key) = ephemeral_keypair(); + // Load the private key for notarized transcript signing + #[cfg(not(feature = "tee_quote"))] let attestation_key = load_attestation_key(&config.notary_key).await?; let crypto_provider = build_crypto_provider(attestation_key); @@ -106,8 +114,10 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer ); // Parameters needed for the info endpoint + #[cfg(not(feature = "tee_quote"))] let public_key = std::fs::read_to_string(&config.notary_key.public_key_pem_path) .map_err(|err| eyre!("Failed to load notary public signing key for notarization: {err}"))?; + let version = env!("CARGO_PKG_VERSION").to_string(); let git_commit_hash = env!("GIT_COMMIT_HASH").to_string(); let git_commit_timestamp = env!("GIT_COMMIT_TIMESTAMP").to_string(); @@ -141,6 +151,8 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer public_key, git_commit_hash, git_commit_timestamp, + #[cfg(feature = "tee_quote")] + quote: quote().await, }), ) .into_response() @@ -228,6 +240,7 @@ fn build_crypto_provider(attestation_key: AttestationKey) -> CryptoProvider { } /// Load notary signing key for attestations from static file +#[allow(dead_code)] async fn load_attestation_key(config: &NotarySigningKeyProperties) -> Result { debug!("Loading notary server's signing key"); diff --git a/crates/notary/server/src/tee.rs b/crates/notary/server/src/tee.rs new file mode 100644 index 000000000..88aeb3c54 --- /dev/null +++ b/crates/notary/server/src/tee.rs @@ -0,0 +1,149 @@ +use mc_sgx_dcap_types::QlError; +use serde::{Deserialize, Serialize}; + +use crate::signing::AttestationKey; +use p256::{ecdsa::SigningKey, PublicKey}; +use pkcs8::{DecodePrivateKey, EncodePrivateKey, LineEnding}; +use rand_chacha::{ + rand_core::{OsRng, SeedableRng}, + ChaCha20Rng, +}; +use std::{ + fs::{File, OpenOptions}, + io::{self, Read, Write}, + path::Path, +}; +use tracing::{debug, error, instrument}; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Quote { + raw_quote: Option, + mrsigner: Option, + mrenclave: Option, + error: Option, +} + +impl Default for Quote { + fn default() -> Quote { + Quote { + raw_quote: Some("".to_string()), + mrsigner: None, + mrenclave: None, + error: None, + } + } +} + +impl std::fmt::Debug for QuoteError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + QuoteError::IoError(err) => write!(f, "IoError: {:?}", err), + QuoteError::IntelQuoteLibrary(err) => { + write!(f, "IntelQuoteLibrary: {}", err) + } + } + } +} + +impl From for QuoteError { + fn from(err: io::Error) -> QuoteError { + QuoteError::IoError(err) + } +} + +enum QuoteError { + IoError(io::Error), + IntelQuoteLibrary(QlError), +} + +impl From for QuoteError { + fn from(src: QlError) -> Self { + Self::IntelQuoteLibrary(src) + } +} + +#[instrument(level = "debug", skip_all)] +async fn gramine_quote() -> Result { + //// Check if the the gramine pseudo-hardware exists + if !Path::new("/dev/attestation/quote").exists() { + error!("Failed to retrieve quote hardware"); + return Err(QuoteError::IntelQuoteLibrary(QlError::InterfaceUnavailable)); + } + + // Reading attestation type + let mut attestation_file = File::open("/dev/attestation/attestation_type")?; + let mut attestation_type = String::new(); + attestation_file.read_to_string(&mut attestation_type)?; + debug!("Detected attestation type: {}", attestation_type); + + //// Writing 64 zero bytes to the gramine report pseudo-hardware `/dev/attestation/user_report_data` + let mut report_data_file = OpenOptions::new() + .write(true) + .open("/dev/attestation/user_report_data")?; + report_data_file.write_all(&[0u8; 64])?; + + //// Reading from the gramine quote pseudo-hardware `/dev/attestation/quote` + let mut quote_file = File::open("/dev/attestation/quote")?; + let mut quote = Vec::new(); + quote_file.read_to_end(&mut quote)?; + + if quote.len() < 432 { + error!("Quote data is too short, expected at least 432 bytes"); + return Err(QuoteError::IntelQuoteLibrary(QlError::InvalidReport)); + } + + //// Extract mrenclave: enclave image, and mrsigner: identity key bound to enclave + //// https://github.com/intel/linux-sgx/blob/main/common/inc/sgx_quote.h + let mrenclave = hex::encode("e[112..144]); + let mrsigner = hex::encode("e[176..208]); + + debug!("mrenclave: {}", mrenclave); + debug!("mrsigner: {}", mrsigner); + + //// Return the Quote struct with the extracted data + Ok(Quote { + raw_quote: Some(hex::encode(quote)), + mrsigner: Some(mrsigner), + mrenclave: Some(mrenclave), + error: None, + }) +} + +pub fn ephemeral_keypair() -> (AttestationKey, String) { + let mut rng = ChaCha20Rng::from_rng(OsRng).expect("os rng err!"); + let signing_key = SigningKey::random(&mut rng); + let pem_string = signing_key + .clone() + .to_pkcs8_pem(LineEnding::default()) + .expect("to pem"); + let attkey = AttestationKey::from_pkcs8_pem(&pem_string).expect("from pem"); + + return ( + attkey, + PublicKey::from(*signing_key.verifying_key()).to_string(), + ); +} + +pub async fn quote() -> Quote { + //// tee-detection logic will live here, for now its only gramine-sgx + match gramine_quote().await { + Ok(quote) => quote, + Err(err) => { + error!("Failed to retrieve quote: {:?}", err); + match err { + QuoteError::IoError(_) => Quote { + raw_quote: None, + mrsigner: None, + mrenclave: None, + error: Some("io".to_owned()), + }, + QuoteError::IntelQuoteLibrary(_) => Quote { + raw_quote: None, + mrsigner: None, + mrenclave: None, + error: Some("hw".to_owned()), + }, + } + } + } +}