Notary Server SGX Attestation Template

.github/workflows/gramine-report.yml

@@ -0,0 +1,63 @@
+name: generate a gramine signature, which can be verified later
+
+on:
+  workflow_call:
+
+permissions:
+  attestations: write
+  id-token: write
+  contents: read
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: get src
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+      - name: add gramine key
+        run: |
+               sudo curl --fail --silent --show-error --location --output /usr/share/keyrings/gramine-keyring.gpg https://packages.gramineproject.io/gramine-keyring.gpg
+               echo "deb [arch=amd64 signed-by=/usr/share/keyrings/gramine-keyring.gpg] https://packages.gramineproject.io/ $(lsb_release -sc) main" |  sudo tee /etc/apt/sources.list.d/gramine.list
+      - name: apt get
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages:  cargo gramine cmake clang
+          version: 1.1
+          execute_install_scripts: true
+      - name: Set PATH
+        run: echo "export PATH=\$PATH:/usr/local/bin:/usr/bin" >> $GITHUB_ENV
+      - name: generate manifest and sig
+        run: |
+               make
+               /usr/bin/gramine-sgx-gen-private-key -f
+               /usr/bin/gramine-sgx-sign -v --manifest notary-server.manifest --output notary-server.sgx
+      - name: capture sig
+        id: sigstruct
+        run:  |
+               sigview=`/usr/bin/gramine-sgx-sigstruct-view notary-server.sig`
+               {
+                echo 'SGX_REPORT<<EOF'
+                echo "$sigview"
+                echo EOF
+               } >> "$GITHUB_ENV"
+               echo "$sigview"
+      - name: write report to artifact file
+        run: echo "${{ env.SGX_REPORT }}" > /home/runner/work/_temp/notary-server.txt
+      - name: upload it
+        uses: actions/upload-artifact@v4
+        with:
+            path: /home/runner/work/_temp/notary-server.txt
+      - name: get github to sign our measurement
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-path: /home/runner/work/_temp/notary-server.txt

.github/workflows/release.yml

@@ -0,0 +1,16 @@
+name: on release hook
+on:
+  create:
+    branches:
+      - 'release/*'
+  workflow_dispatch:
+
+permissions:
+  attestations: write
+  contents: read
+  id-token: write
+
+jobs:
+  create-gramine-attestation:
+    uses: maceip/tlsn/.github/workflows/gramine-report.yml@sgx-attest
+    secrets: inherit

.gitignore

@@ -32,4 +32,5 @@ Cargo.lock
 *.log
 
 # metrics
-*.csv
+*.csv
+

README.md

@@ -52,7 +52,6 @@ at your option.
 
 This repository contains the source code for the Rust implementation of the TLSNotary protocol. For additional tools and implementations related to TLSNotary, visit <https://github.com/tlsnotary>. This includes repositories such as [`tlsn-js`](https://github.com/tlsnotary/tlsn-js), [`tlsn-extension`](https://github.com/tlsnotary/tlsn-extension), [`explorer`](https://github.com/tlsnotary/explorer), among others.
 
-
 ## Contribution
 
 Unless you explicitly state otherwise, any contribution intentionally submitted

notary/server/Makefile

@@ -0,0 +1,61 @@
+# Copyright (C) 2023 Gramine contributors
+# SPDX-License-Identifier: BSD-3-Clause
+
+ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine)
+
+SELF_EXE = target/release/notary-server
+
+.PHONY: all
+all: $(SELF_EXE) notary-server.manifest
+ifeq ($(SGX),1)
+all: notary-server.manifest.sgx notary-server.sig
+endif
+
+ifeq ($(DEBUG),1)
+GRAMINE_LOG_LEVEL = debug
+else
+GRAMINE_LOG_LEVEL = error
+endif
+
+# Note that we're compiling in release mode regardless of the DEBUG setting passed
+# to Make, as compiling in debug mode results in an order of magnitude's difference in
+# performance that makes testing by running a benchmark with ab painful. The primary goal
+# of the DEBUG setting is to control Gramine's loglevel.
+-include $(SELF_EXE).d # See also: .cargo/config.toml
+$(SELF_EXE): Cargo.toml
+	cargo build --release
+
+notary-server.manifest: notary-server.manifest.template
+	gramine-manifest \
+		-Dlog_level=$(GRAMINE_LOG_LEVEL) \
+		-Darch_libdir=$(ARCH_LIBDIR) \
+		-Dself_exe=$(SELF_EXE) \
+		$< $@
+
+# Make on Ubuntu <= 20.04 doesn't support "Rules with Grouped Targets" (`&:`),
+# see the helloworld example for details on this workaround.
+notary-server.manifest.sgx notary-server.sig: sgx_sign
+	@:
+
+.INTERMEDIATE: sgx_sign
+sgx_sign: notary-server.manifest $(SELF_EXE)
+	gramine-sgx-sign \
+		--manifest $< \
+		--output $<.sgx
+
+ifeq ($(SGX),)
+GRAMINE = gramine-direct
+else
+GRAMINE = gramine-sgx
+endif
+
+.PHONY: start-gramine-server
+start-gramine-server: all
+	$(GRAMINE) notary-server
+.PHONY: clean
+clean:
+	$(RM) -rf *.token *.sig *.manifest.sgx *.manifest result-* OUTPUT
+
+.PHONY: distclean
+distclean: clean
+	$(RM) -rf target/ Cargo.lock

notary/server/README.md

@@ -134,3 +134,8 @@ Axum is chosen as the framework to serve HTTP and WebSocket requests from the pr
 
 #### WebSocket
 Axum's internal implementation of WebSocket uses [tokio_tungstenite](https://docs.rs/tokio-tungstenite/latest/tokio_tungstenite/), which provides a WebSocket struct that doesn't implement [AsyncRead](https://docs.rs/futures/latest/futures/io/trait.AsyncRead.html) and [AsyncWrite](https://docs.rs/futures/latest/futures/io/trait.AsyncWrite.html). Both these traits are required by the TLSN core libraries for the prover and the notary. To overcome this, a [slight modification](./src/service/axum_websocket.rs) of Axum's implementation of WebSocket is used, where [async_tungstenite](https://docs.rs/async-tungstenite/latest/async_tungstenite/) is used instead so that [ws_stream_tungstenite](https://docs.rs/ws_stream_tungstenite/latest/ws_stream_tungstenite/index.html) can be used to wrap on top of the WebSocket struct to get AsyncRead and AsyncWrite implemented.
+
+## Reproduciple Builds & Attestation
+- We are using [Gramine](https://github.com/gramineproject) to generate SGX reports for notary-server;
+- Each release of tlsn will include a build artifact attested by github, which includes the gramine signature.
+- if you build and run the notary-server with gramine, you should get the same mr_enclave hash as in our release artifact.

notary/server/notary-server.manifest.template

@@ -0,0 +1,54 @@
+[libos]
+entrypoint = "{{ self_exe }}"
+
+
+[loader]
+entrypoint = "file:{{ gramine.libos }}"
+env.MALLOC_ARENA_MAX = "1"
+env.RUST_BACKTRACE = "full"
+env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}"
+log_level = "error"
+uid = 65534
+gid = 65534
+
+[fs]
+mounts = [
+    { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+    { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+    { path = "/fixture", uri = "file:fixture" },
+]
+
+[sgx]
+edmm_enable = true
+
+allowed_files = [
+    "file:fixture/tls",
+]
+trusted_files = [
+    "file:{{ gramine.runtimedir() }}/",
+    "file:{{ arch_libdir }}/",
+    "file:{{ self_exe }}",
+    { uri = "file:config/config.yaml" },
+    { uri = "file:fixture/notary/notary.key" },
+    { uri = "file:fixture/notary/notary.pub" },
+    { uri = "file:fixture/auth/whitelist.csv" },
+    { uri = "file:fixture/tls/notary.crt" },
+]
+max_threads = 32
+isvprodid = 0
+isvsvn = 0
+debug = false
+enable_stats = false
+enclave_size = "1024G"
+use_exinfo = false
+
+[sgx.cpu_features]
+avx = "unspecified"
+avx512 = "unspecified"
+amx = "unspecified"
+mpx = "disabled"
+pkru = "disabled"
+
+[sys]
+enable_sigterm_injection = true
+insecure__allow_eventfd = true