From 58e948716a11dc25b779c9f5096b7b4811061b81 Mon Sep 17 00:00:00 2001 From: maceip Date: Wed, 5 Jun 2024 03:09:49 +0000 Subject: [PATCH 1/6] feat: gramine manifest.template and Makefile to facilitate tee measurement feat: push hook action to generate a gramine sig for notary-server, add output to readme --- .github/workflows/sgx-report.yml | 58 ++++++++++++++++++ .gitignore | 7 ++- README.md | 12 ++++ notary-server/Makefile | 61 +++++++++++++++++++ notary-server/notary-server.manifest.template | 54 ++++++++++++++++ 5 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/sgx-report.yml create mode 100644 notary-server/Makefile create mode 100644 notary-server/notary-server.manifest.template diff --git a/.github/workflows/sgx-report.yml b/.github/workflows/sgx-report.yml new file mode 100644 index 0000000000..2cc5d9c2f8 --- /dev/null +++ b/.github/workflows/sgx-report.yml @@ -0,0 +1,58 @@ +name: Create New SGX Report +on: push +jobs: + build_and_generate_report: + runs-on: ubuntu-latest + permissions: + contents: write + defaults: + run: + shell: bash + working-directory: ./notary-server + name: install Gramine + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + - name: add gramine key + run: | + sudo curl -fsSLo /usr/share/keyrings/gramine-keyring.gpg https://packages.gramineproject.io/gramine-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/gramine-keyring.gpg] https://packages.gramineproject.io/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/gramine.list + - uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: rustc cargo gramine cmake clang gramine + version: 1.1 + execute_install_scripts: true + + - name: Set PATH + run: echo "export PATH=\$PATH:/usr/local/bin:/usr/bin" >> $GITHUB_ENV + - name: generate manifest and sig + run: | + make + /usr/bin/gramine-sgx-gen-private-key -f + /usr/bin/gramine-sgx-sign -v --manifest notary-server.manifest --output notary-server.sgx + - name: capture sig + id: sigstruct + run: | + sigview=`/usr/bin/gramine-sgx-sigstruct-view notary-server.sig` + { + echo 'SGX_REPORT<> "$GITHUB_ENV" + echo "$sigview" + - name: debug path first + run: | + echo "GITHUB_ENV: $GITHUB_ENV" + echo "SGX_REPORT: $SGX_REPORT" + - name: update README + uses: jacobtomlinson/gha-find-replace@v3 + with: + find: "(?si)attributes.*debug_enclave: (true|false)" + replace: "${{ env.SGX_REPORT }}" + include: "README.md" + regex: true + - name: Push changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Automated SGX Report Gen diff --git a/.gitignore b/.gitignore index f79cb086da..444ba891eb 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,9 @@ Cargo.lock *.log # metrics -*.csv \ No newline at end of file +*.csv + +#gramine +notary-server/*.sgx +notary-server/*.manifest +notary-server/*.sig diff --git a/README.md b/README.md index 19bab726ea..a81c21db0e 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,18 @@ at your option. This repository contains the source code for the Rust implementation of the TLSNotary protocol. For additional tools and implementations related to TLSNotary, visit . This includes repositories such as [`tlsn-js`](https://github.com/tlsnotary/tlsn-js), [`tlsn-extension`](https://github.com/tlsnotary/tlsn-extension), [`explorer`](https://github.com/tlsnotary/explorer), among others. +## Reproduciple Builds & Attestation +- We are using [Gramine](https://github.com/gramineproject) to generate SGX reports for notary-server builds; the report below is [dynamically generated](.github/workflows/sgx-report.yml): + ``` + Attributes: + mr_signer: 882f451378e083c19eddb338b19fa7ef02c66f9920db32c9acc6700430a7ad8a + mr_enclave: 57a5750ce03158f4ec665b6a880079900d3258b2f99df60eae995cf4ab46684c + isv_prod_id: 0 + isv_svn: 0 + debug_enclave: False + ``` +- if you build and run the notary-server with gramine, you should get the same mr_enclave hash as the one above. + ## Contribution diff --git a/notary-server/Makefile b/notary-server/Makefile new file mode 100644 index 0000000000..28a53b5fa0 --- /dev/null +++ b/notary-server/Makefile @@ -0,0 +1,61 @@ +# Copyright (C) 2023 Gramine contributors +# SPDX-License-Identifier: BSD-3-Clause + +ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine) + +SELF_EXE = target/release/notary-server + +.PHONY: all +all: $(SELF_EXE) notary-server.manifest +ifeq ($(SGX),1) +all: notary-server.manifest.sgx notary-server.sig +endif + +ifeq ($(DEBUG),1) +GRAMINE_LOG_LEVEL = debug +else +GRAMINE_LOG_LEVEL = error +endif + +# Note that we're compiling in release mode regardless of the DEBUG setting passed +# to Make, as compiling in debug mode results in an order of magnitude's difference in +# performance that makes testing by running a benchmark with ab painful. The primary goal +# of the DEBUG setting is to control Gramine's loglevel. +-include $(SELF_EXE).d # See also: .cargo/config.toml +$(SELF_EXE): Cargo.toml + cargo build --release + +notary-server.manifest: notary-server.manifest.template + gramine-manifest \ + -Dlog_level=$(GRAMINE_LOG_LEVEL) \ + -Darch_libdir=$(ARCH_LIBDIR) \ + -Dself_exe=$(SELF_EXE) \ + $< $@ + +# Make on Ubuntu <= 20.04 doesn't support "Rules with Grouped Targets" (`&:`), +# see the helloworld example for details on this workaround. +notary-server.manifest.sgx notary-server.sig: sgx_sign + @: + +.INTERMEDIATE: sgx_sign +sgx_sign: notary-server.manifest $(SELF_EXE) + gramine-sgx-sign \ + --manifest $< \ + --output $<.sgx + +ifeq ($(SGX),) +GRAMINE = gramine-direct +else +GRAMINE = gramine-sgx +endif + +.PHONY: start-gramine-server +start-gramine-server: all + $(GRAMINE) notary-server +.PHONY: clean +clean: + $(RM) -rf *.token *.sig *.manifest.sgx *.manifest result-* OUTPUT + +.PHONY: distclean +distclean: clean + $(RM) -rf target/ Cargo.lock diff --git a/notary-server/notary-server.manifest.template b/notary-server/notary-server.manifest.template new file mode 100644 index 0000000000..9ddda6ae5b --- /dev/null +++ b/notary-server/notary-server.manifest.template @@ -0,0 +1,54 @@ +[libos] +entrypoint = "{{ self_exe }}" + + +[loader] +entrypoint = "file:{{ gramine.libos }}" +env.MALLOC_ARENA_MAX = "1" +env.RUST_BACKTRACE = "full" +env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}" +log_level = "error" +uid = 65534 +gid = 65534 + +[fs] +mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, + { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" }, + { path = "/fixture", uri = "file:fixture" }, +] + +[sgx] +edmm_enable = true + +allowed_files = [ + "file:fixture/tls", +] +trusted_files = [ + "file:{{ gramine.runtimedir() }}/", + "file:{{ arch_libdir }}/", + "file:{{ self_exe }}", + { uri = "file:config/config.yaml" }, + { uri = "file:fixture/notary/notary.key" }, + { uri = "file:fixture/notary/notary.pub" }, + { uri = "file:fixture/auth/whitelist.csv" }, + { uri = "file:fixture/tls/notary.crt" }, +] +max_threads = 32 +isvprodid = 0 +isvsvn = 0 +debug = false +enable_stats = false +enclave_size = "1024G" +use_exinfo = false + +[sgx.cpu_features] +avx = "unspecified" +avx512 = "unspecified" +amx = "unspecified" +mpx = "disabled" +pkru = "disabled" + +[sys] +enable_sigterm_injection = true +insecure__allow_eventfd = true From 5fd5037f6991eb6cd2ef4b7354315dd0a7cb1658 Mon Sep 17 00:00:00 2001 From: maceip Date: Thu, 6 Jun 2024 08:42:05 +0000 Subject: [PATCH 2/6] Automated SGX Report Gen --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a81c21db0e..31d8fe44d7 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ This repository contains the source code for the Rust implementation of the TLSN - We are using [Gramine](https://github.com/gramineproject) to generate SGX reports for notary-server builds; the report below is [dynamically generated](.github/workflows/sgx-report.yml): ``` Attributes: - mr_signer: 882f451378e083c19eddb338b19fa7ef02c66f9920db32c9acc6700430a7ad8a - mr_enclave: 57a5750ce03158f4ec665b6a880079900d3258b2f99df60eae995cf4ab46684c + mr_signer: b714cf47e702229cffb88e927124c36b16507d83b03b316f78e500b7bfb121a2 + mr_enclave: eddae4733a236d3301a533f872c02422788f50d3de84b098bd101f5001cc43b1 isv_prod_id: 0 isv_svn: 0 debug_enclave: False From 1b0650282d096c8df18d0d7a2d7e88c4a11da693 Mon Sep 17 00:00:00 2001 From: maceip Date: Fri, 28 Jun 2024 21:42:35 -0700 Subject: [PATCH 3/6] [feat] sgx artifacts signed by github --- .../{sgx-report.yml => gramine-report.yml} | 51 ++++++++++--------- .github/workflows/release.yml | 16 ++++++ 2 files changed, 44 insertions(+), 23 deletions(-) rename .github/workflows/{sgx-report.yml => gramine-report.yml} (62%) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/sgx-report.yml b/.github/workflows/gramine-report.yml similarity index 62% rename from .github/workflows/sgx-report.yml rename to .github/workflows/gramine-report.yml index 2cc5d9c2f8..1b34c8339b 100644 --- a/.github/workflows/sgx-report.yml +++ b/.github/workflows/gramine-report.yml @@ -1,29 +1,39 @@ -name: Create New SGX Report -on: push +name: generate a gramine signature, which can be verified later + +on: + workflow_call: + +permissions: + attestations: write + id-token: write + contents: read + + jobs: - build_and_generate_report: + build: runs-on: ubuntu-latest permissions: - contents: write + contents: read + id-token: write + attestations: write defaults: run: shell: bash - working-directory: ./notary-server - name: install Gramine steps: - - uses: actions/checkout@v4 + - name: get src + uses: actions/checkout@v4 with: ref: ${{ github.ref }} - name: add gramine key run: | sudo curl -fsSLo /usr/share/keyrings/gramine-keyring.gpg https://packages.gramineproject.io/gramine-keyring.gpg echo "deb [arch=amd64 signed-by=/usr/share/keyrings/gramine-keyring.gpg] https://packages.gramineproject.io/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/gramine.list - - uses: awalsh128/cache-apt-pkgs-action@latest + - name: apt get + uses: awalsh128/cache-apt-pkgs-action@latest with: packages: rustc cargo gramine cmake clang gramine version: 1.1 execute_install_scripts: true - - name: Set PATH run: echo "export PATH=\$PATH:/usr/local/bin:/usr/bin" >> $GITHUB_ENV - name: generate manifest and sig @@ -41,18 +51,13 @@ jobs: echo EOF } >> "$GITHUB_ENV" echo "$sigview" - - name: debug path first - run: | - echo "GITHUB_ENV: $GITHUB_ENV" - echo "SGX_REPORT: $SGX_REPORT" - - name: update README - uses: jacobtomlinson/gha-find-replace@v3 + - name: create artifact + run: echo "${{ env.SGX_REPORT }}" > /home/runner/work/_temp/notary-server.sig + - name: upload it + uses: actions/upload-artifact@v4 + with: + path: /home/runner/work/_temp/notary-server.sig + - name: get github to sign our measurement + uses: actions/attest-build-provenance@v1 with: - find: "(?si)attributes.*debug_enclave: (true|false)" - replace: "${{ env.SGX_REPORT }}" - include: "README.md" - regex: true - - name: Push changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Automated SGX Report Gen + subject-path: /home/runner/work/_temp/notary-server.sig diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..2ee5fdfc3e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,16 @@ +name: on release hook +on: + create: + branches: + - 'release/*' + workflow_dispatch: + +permissions: + attestations: write + contents: read + id-token: write + +jobs: + create-gramine-attestation: + uses: maceip/tlsn/.github/workflows/gramine-report.yml@sgx-attest + secrets: inherit From 4f148ea8e1ce4830d1c8d3f1885cc82d6c7d835e Mon Sep 17 00:00:00 2001 From: Ryan MacArthur Date: Mon, 1 Jul 2024 18:27:54 -0700 Subject: [PATCH 4/6] [chore] remove attestation from README --- README.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/README.md b/README.md index 7630ce568c..6ccab78b32 100644 --- a/README.md +++ b/README.md @@ -52,19 +52,6 @@ at your option. This repository contains the source code for the Rust implementation of the TLSNotary protocol. For additional tools and implementations related to TLSNotary, visit . This includes repositories such as [`tlsn-js`](https://github.com/tlsnotary/tlsn-js), [`tlsn-extension`](https://github.com/tlsnotary/tlsn-extension), [`explorer`](https://github.com/tlsnotary/explorer), among others. -## Reproduciple Builds & Attestation -- We are using [Gramine](https://github.com/gramineproject) to generate SGX reports for notary-server builds; the report below is [dynamically generated](.github/workflows/sgx-report.yml): - ``` - Attributes: - mr_signer: b714cf47e702229cffb88e927124c36b16507d83b03b316f78e500b7bfb121a2 - mr_enclave: eddae4733a236d3301a533f872c02422788f50d3de84b098bd101f5001cc43b1 - isv_prod_id: 0 - isv_svn: 0 - debug_enclave: False - ``` -- if you build and run the notary-server with gramine, you should get the same mr_enclave hash as the one above. - - ## Contribution Unless you explicitly state otherwise, any contribution intentionally submitted From 0f2ea42f4db401937d7789a51c06ece920ba451b Mon Sep 17 00:00:00 2001 From: Ryan MacArthur Date: Mon, 1 Jul 2024 18:30:47 -0700 Subject: [PATCH 5/6] [chore] update README with sgx attestation info --- notary/server/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/notary/server/README.md b/notary/server/README.md index a79113c085..9f247d5dd1 100644 --- a/notary/server/README.md +++ b/notary/server/README.md @@ -134,3 +134,8 @@ Axum is chosen as the framework to serve HTTP and WebSocket requests from the pr #### WebSocket Axum's internal implementation of WebSocket uses [tokio_tungstenite](https://docs.rs/tokio-tungstenite/latest/tokio_tungstenite/), which provides a WebSocket struct that doesn't implement [AsyncRead](https://docs.rs/futures/latest/futures/io/trait.AsyncRead.html) and [AsyncWrite](https://docs.rs/futures/latest/futures/io/trait.AsyncWrite.html). Both these traits are required by the TLSN core libraries for the prover and the notary. To overcome this, a [slight modification](./src/service/axum_websocket.rs) of Axum's implementation of WebSocket is used, where [async_tungstenite](https://docs.rs/async-tungstenite/latest/async_tungstenite/) is used instead so that [ws_stream_tungstenite](https://docs.rs/ws_stream_tungstenite/latest/ws_stream_tungstenite/index.html) can be used to wrap on top of the WebSocket struct to get AsyncRead and AsyncWrite implemented. + +## Reproduciple Builds & Attestation +- We are using [Gramine](https://github.com/gramineproject) to generate SGX reports for notary-server; +- Each release of tlsn will include a build artifact attested by github, which includes the gramine signature. +- if you build and run the notary-server with gramine, you should get the same mr_enclave hash as in our release artifact. From 97cd381517cea2041698c9634a9d39a9224be36c Mon Sep 17 00:00:00 2001 From: Ryan MacArthur Date: Mon, 1 Jul 2024 18:41:18 -0700 Subject: [PATCH 6/6] [chore] update attestation report to use .txt extension --- .github/workflows/gramine-report.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gramine-report.yml b/.github/workflows/gramine-report.yml index 1b34c8339b..6eb2688cdb 100644 --- a/.github/workflows/gramine-report.yml +++ b/.github/workflows/gramine-report.yml @@ -26,12 +26,12 @@ jobs: ref: ${{ github.ref }} - name: add gramine key run: | - sudo curl -fsSLo /usr/share/keyrings/gramine-keyring.gpg https://packages.gramineproject.io/gramine-keyring.gpg + sudo curl --fail --silent --show-error --location --output /usr/share/keyrings/gramine-keyring.gpg https://packages.gramineproject.io/gramine-keyring.gpg echo "deb [arch=amd64 signed-by=/usr/share/keyrings/gramine-keyring.gpg] https://packages.gramineproject.io/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/gramine.list - name: apt get uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: rustc cargo gramine cmake clang gramine + packages: cargo gramine cmake clang version: 1.1 execute_install_scripts: true - name: Set PATH @@ -51,13 +51,13 @@ jobs: echo EOF } >> "$GITHUB_ENV" echo "$sigview" - - name: create artifact - run: echo "${{ env.SGX_REPORT }}" > /home/runner/work/_temp/notary-server.sig + - name: write report to artifact file + run: echo "${{ env.SGX_REPORT }}" > /home/runner/work/_temp/notary-server.txt - name: upload it uses: actions/upload-artifact@v4 with: - path: /home/runner/work/_temp/notary-server.sig + path: /home/runner/work/_temp/notary-server.txt - name: get github to sign our measurement uses: actions/attest-build-provenance@v1 with: - subject-path: /home/runner/work/_temp/notary-server.sig + subject-path: /home/runner/work/_temp/notary-server.txt