feat: tee quote endpoint

.github/workflows/ssh-azure-tee-build.yaml

@@ -0,0 +1,73 @@
+name: sgx-build-azure-ssh
+
+on:
+  push:
+    branches: [ "quote-presentation" ]
+  pull_request:
+    branches: [ "quote-presentation" ]
+
+# perms needed for attestation workflow
+permissions:
+  id-token: write
+  attestations: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: tee
+    steps:
+    - name: azure-tee-build-${{ github.sha }}
+      uses: appleboy/[email protected]
+      env:
+         # using personal access token until devops links acct
+         PAT_TOKEN: ${{ secrets.PAT_SET_ENV }}
+      with:
+         host: ${{ secrets.AZURE_TEE_BUILD_HOST }}
+         username: ${{ secrets.AZURE_BUILD_TEE_USERNAME }}
+         key: ${{ secrets.AZURE_TEE_BUILD_KEY }}
+         port: ${{ secrets.SSH_PORT }}
+         command_timeout: 10m
+         allenvs: true
+         envs: PAT_TOKEN
+         script: |
+            cd /tmp 
+            source $HOME/.cargo/env
+            rm -rf ${{ github.sha }}
+            git clone https://github.com/tlsnotary/tlsn ${{ github.sha }}
+            cd ${{ github.sha }}
+            git checkout ${{ github.sha }}
+            cd crates/notary/server/config/gramine
+            : # listen on random_port port, set it in config
+            random_port=$(shuf -i 1024-65535 -n 1)
+            sed -i '/port*: *7047/s/7047/'$random_port'/' ../config.yaml
+            : # the makefile compiles the gramine manigest and notary-server
+            make clean
+            SGX=1 make start-gramine-server &
+            : # this next bofh bash script is just to check if the server comes up
+            notarypid=$!
+            win=0
+            SECONDS=0; set -o pipefail; while [ $SECONDS -lt 300 ]; do echo -e "GET /info HTTP/1.1\r\nHost: localhost\r\nConnection: Close\r\n\r\n" | openssl s_client -quiet localhost:$random_port 2>&1 | tee s.log; if [ $? -eq 0 ]; then win=1; break; fi; sleep 2; done
+            if [ $win -eq 0 ]; then echo "Build Failed"; kill -SIGTERM $notarypid; exit 1; fi
+            cat s.log | grep '^{' |  jq '. | tostring' > quotejson
+            : # end ugly bash
+            mapfile quote < quotejson
+            : # using http api because we dont have write access to env here
+            curl -L -X PATCH -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $PAT_TOKEN" -H "Connection: close" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/$GITHUB_REPOSITORY/environments/TEE/variables/AZURE_TEE_BUILD_ATTESTATION -d "{\"name\":\"AZURE_TEE_BUILD_ATTESTATION\",\"value\":$quote}"
+            : # gramine will keep the sgx process running, we use the gramine setting
+            : # sys.enable_sigterm_injection, which is insecure and for convenience 
+            kill -SIGTERM $notarypid
+            : # sleep originated because the next step wouldnt have the updated env
+            sleep 5
+            exit 0
+    - name: ✨ fet quotech from gh, write it to runner
+      #we use http api due to the ssh runner and access to gh envs
+      run:  |
+        curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.PAT_SET_ENV }}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/$GITHUB_REPOSITORY/environments/TEE/variables/AZURE_TEE_BUILD_ATTESTATION > /home/runner/work/_temp/sgx-build-quote.txt
+    - name: get github to sign our measurement
+      uses: actions/attest-build-provenance@v1
+      with:
+        subject-path: /home/runner/work/_temp/sgx-build-quote.txt
+    - name: upload it
+      uses: actions/upload-artifact@v4
+      with:
+        path: /home/runner/work/_temp/sgx-build-quote.txt

.gitignore

@@ -32,4 +32,10 @@ Cargo.lock
 *.log
 
 # metrics
-*.csv
+*.csv
+
+# tee
+**/*.sgx
+**/*.sig
+**/*.manifest
+

crates/notary/server/Cargo.toml

@@ -3,6 +3,9 @@ name = "notary-server"
 version = "0.1.0-alpha.6"
 edition = "2021"
 
+[features]
+tee_quote = ["dep:mc-sgx-dcap-types", "dep:hex", "dep:rand_chacha"]
+
 [dependencies]
 tlsn-core = { workspace = true }
 tlsn-common = { workspace = true }
@@ -48,3 +51,7 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] }
 uuid = { workspace = true, features = ["v4", "fast-rng"] }
 ws_stream_tungstenite = { workspace = true, features = ["tokio_io"] }
 zeroize = { workspace = true }
+
+mc-sgx-dcap-types = { version = "0.11.0", optional = true }
+hex = { workspace = true, optional = true }
+rand_chacha = { workspace = true, optional = true }

crates/notary/server/config/gramine/Makefile

@@ -0,0 +1,62 @@
+# notary-server testing only
+ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
+ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine)
+
+SELF_EXE = target/release/notary-server
+
+.PHONY: all
+all: $(SELF_EXE) notary-server.manifest
+ifeq ($(SGX),1)
+all: notary-server.manifest.sgx notary-server.sig
+endif
+
+ifeq ($(DEBUG),1)
+GRAMINE_LOG_LEVEL = debug
+else
+GRAMINE_LOG_LEVEL = error
+endif
+
+# Note that we're compiling in release mode regardless of the DEBUG setting passed
+# to Make, as compiling in debug mode results in an order of magnitude's difference in
+# performance that makes testing by running a benchmark with ab painful. The primary goal
+# of the DEBUG setting is to control Gramine's loglevel.
+-include $(SELF_EXE).d # See also: .cargo/config.toml
+$(SELF_EXE): $(ROOT_DIR)../../Cargo.toml
+	cargo build --bin notary-server  --release --features tee_quote && ln -s ../../../../../target target
+
+notary-server.manifest: notary-server.manifest.template
+	gramine-manifest \
+		-Dlog_level=$(GRAMINE_LOG_LEVEL) \
+		-Darch_libdir=$(ARCH_LIBDIR) \
+		-Dself_exe=$(SELF_EXE) \
+		$< $@
+
+# Make on Ubuntu <= 20.04 doesn't support "Rules with Grouped Targets" (`&:`),
+# see the helloworld example for details on this workaround.
+notary-server.manifest.sgx notary-server.sig: sgx_sign
+	@:
+
+.INTERMEDIATE: sgx_sign
+sgx_sign: notary-server.manifest $(SELF_EXE)
+	gramine-sgx-sign \
+		--manifest $< \
+		--output $<.sgx
+
+ifeq ($(SGX),)
+GRAMINE = gramine-direct
+else
+GRAMINE = gramine-sgx
+endif
+
+.PHONY: start-gramine-server
+start-gramine-server: all
+	$(GRAMINE) notary-server
+
+.PHONY: clean
+clean:
+	$(RM) -rf *.token *.sig *.manifest.sgx *.manifest result-* OUTPUT
+
+.PHONY: distclean
+distclean: clean
+	$(RM) -rf $(SELF_EXE) Cargo.lock
+

crates/notary/server/config/gramine/notary-server.manifest.template

@@ -0,0 +1,48 @@
+libos.entrypoint = "{{ self_exe }}"
+loader.log_level = "{{ log_level }}"
+
+loader.env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}"
+
+
+# See https://gramine.readthedocs.io/en/stable/performance.html#glibc-malloc-tuning
+loader.env.MALLOC_ARENA_MAX = "1"
+
+# For easier debugging — not strictly required to run this workload
+loader.env.RUST_BACKTRACE = "full"
+
+
+fs.mounts = [
+  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { path = "/fixture/notary", uri = "file:../../fixture/notary" },
+  { path = "/fixture/tls", uri = "file:../../fixture/tls" },
+  { path = "/config", uri = "file:../../config" },
+  { type = "encrypted", path = "/vault", uri = "file:vault", key_name = "_sgx_mrenclave" },
+
+]
+
+sgx.trusted_files = [
+  "file:{{ self_exe }}",
+  "file:{{ gramine.runtimedir() }}/",
+  "file:{{ arch_libdir }}/",
+  "file:../../config/config.yaml",
+  "file:../../fixture/notary/notary.pub",
+  "file:../../fixture/notary/notary.key",
+  "file:../../fixture/tls/notary.crt",
+  "file:../../fixture/tls/notary.key",
+]
+sgx.debug = true
+sgx.edmm_enable = false
+sgx.remote_attestation = "dcap"
+sgx.max_threads = 64
+sgx.enclave_size = "2G"
+sys.disallow_subprocesses = true
+#### turn this off in prod, 
+sys.enable_sigterm_injection = true
+
+
+#### tlsn rev
+sgx.isvprodid = 7
+#### E
+sgx.isvsvn = 45
+

crates/notary/server/config/gramine/target

@@ -0,0 +1 @@
+../../../../../target

crates/notary/server/openapi.yaml

@@ -206,6 +206,9 @@ components:
         gitCommitTimestamp:
           description: The git commit timestamp of source code that this notary server is running
           type: string
+        quote:
+            description: a measurement attesting to the hardware, if available
+          type: object
       required:
         - "version"
         - "publicKey"

crates/notary/server/src/domain.rs

@@ -1,7 +1,8 @@
 pub mod auth;
 pub mod cli;
 pub mod notary;
-
+#[cfg(feature = "tee_quote")]
+use crate::tee::Quote;
 use serde::{Deserialize, Serialize};
 
 /// Response object of the /info API
@@ -16,4 +17,7 @@ pub struct InfoResponse {
     pub git_commit_hash: String,
     /// Current git commit timestamp of notary-server
     pub git_commit_timestamp: String,
+    /// hardware attestation
+    #[cfg(feature = "tee_quote")]
+    pub quote: Quote,
 }

crates/notary/server/src/lib.rs

@@ -6,6 +6,8 @@ mod server;
 mod server_tracing;
 mod service;
 mod signing;
+#[cfg(feature = "tee_quote")]
+mod tee;
 mod util;
 
 pub use config::{

crates/notary/server/src/server.rs

@@ -46,10 +46,18 @@ use crate::{
     util::parse_csv_file,
 };
 
+#[cfg(feature = "tee_quote")]
+use crate::tee::{ephemeral_keypair, quote};
+
 /// Start a TCP server (with or without TLS) to accept notarization request for both TCP and WebSocket clients
 #[tracing::instrument(skip(config))]
 pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotaryServerError> {
+    //tee uses ephemeral key
+    #[cfg(feature = "tee_quote")]
+    let (attestation_key, public_key) = ephemeral_keypair();
+
     // Load the private key for notarized transcript signing
+    #[cfg(not(feature = "tee_quote"))]
     let attestation_key = load_attestation_key(&config.notary_key).await?;
     let crypto_provider = build_crypto_provider(attestation_key);
 
@@ -106,8 +114,10 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer
     );
 
     // Parameters needed for the info endpoint
+    #[cfg(not(feature = "tee_quote"))]
     let public_key = std::fs::read_to_string(&config.notary_key.public_key_pem_path)
         .map_err(|err| eyre!("Failed to load notary public signing key for notarization: {err}"))?;
+
     let version = env!("CARGO_PKG_VERSION").to_string();
     let git_commit_hash = env!("GIT_COMMIT_HASH").to_string();
     let git_commit_timestamp = env!("GIT_COMMIT_TIMESTAMP").to_string();
@@ -141,6 +151,8 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer
                         public_key,
                         git_commit_hash,
                         git_commit_timestamp,
+                        #[cfg(feature = "tee_quote")]
+                        quote: quote().await,
                     }),
                 )
                     .into_response()
@@ -228,6 +240,7 @@ fn build_crypto_provider(attestation_key: AttestationKey) -> CryptoProvider {
 }
 
 /// Load notary signing key for attestations from static file
+#[allow(dead_code)]
 async fn load_attestation_key(config: &NotarySigningKeyProperties) -> Result<AttestationKey> {
     debug!("Loading notary server's signing key");