Integrate aws-lc-sys and aws-lc-fips-sys crates (#75)

* Relocate aws-lc-rust crate to subdirectory

* Add Cargo.toml for workspace

* Add aws-lc-sys crate to repository

* Add aws-lc-fips-sys crate to repository

* aws-lc-rust sys crate versions

* Fix git safe directory issues

* Use newer version of Go for docker images

* Makefile updates

* Mark pkcs8 test ignored

* CI Updates

* Feedback

* Use HTTPS URL for submodules

* Fix for coverage

* Adjust working-directory syntax

* Implement docker feedback

* CI & Docker Tweaks

* Allow CMakeLists.txt to handle prefixing library outputs

* Diff target branch and published version

* Fixes for latest clippy, make prefix headers even more stable.

Author: skmcgrail

.github/workflows/ci.yml

@@ -20,8 +20,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          submodules: true
-
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
@@ -40,39 +39,43 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      matrix:
+        crate_dir: [ "aws-lc-sys", "aws-lc-fips-sys", "aws-lc-rust" ]
+        features:
+          - ""
+          - "--features fips"
+        exclude:
+          - crate_dir: aws-lc-sys
+            features: "--features fips"
+          - crate_dir: aws-lc-fips-sys
+            features: "--features fips"
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
           toolchain: ${{ env.RUST_NIGHTLY_TOOLCHAIN }}
           profile: minimal
           override: true
           components: clippy
-
       - name: Run cargo clippy
-        uses: actions-rs/[email protected]
-        with:
-          command: clippy
-          args: --all-targets -- -W clippy::all -W clippy::pedantic -D warnings
-
-      - name: Run cargo clippy on fips
-        uses: actions-rs/[email protected]
-        env:
-          CC: clang
-          CXX: clang++
-        with:
-          command: clippy
-          args: --features fips --all-targets -- -W clippy::all -W clippy::pedantic -D warnings
+        working-directory: ${{ matrix.crate_dir }}
+        run: cargo clippy ${{ matrix.features }} --all-targets -- -W clippy::all -W clippy::pedantic -D warnings
 
   apidiff:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      matrix:
+        crate_dir: [ "aws-lc-sys", "aws-lc-fips-sys", "aws-lc-rust" ]
+        diff_target: [ "branch", "published" ]
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
@@ -84,21 +87,33 @@ jobs:
         with:
           command: install
           args: cargo-public-api
-      - name: Perform API Diff
+      - name: Install rust-script
         uses: actions-rs/[email protected]
         with:
-          command: public-api
-          args: diff --deny changed --deny removed ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
-
-#  dependency-review:
-#    runs-on: ubuntu-latest
-#    steps:
-#      - name: 'Checkout Repository'
-#        uses: actions/checkout@v3
-#      - name: 'Dependency Review'
-#        uses: actions/dependency-review-action@v3
-#        with:
-#          allow-licenses: Apache-2.0, ISC, MIT, MIT-0
+          command: install
+          args: rust-script
+      - name: Perform API Diff (Target Branch)
+        if: matrix.diff_target == 'branch'
+        working-directory: ${{ matrix.crate_dir }}
+        run: cargo public-api diff --deny changed --deny removed ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
+      - name: Perform API Diff (Published)
+        if: matrix.diff_target == 'published'
+        working-directory: ${{ matrix.crate_dir }}
+        shell: bash
+        run: |
+          CRATE_NAME=$("${GITHUB_WORKSPACE}"/scripts/tools/cargo-dig.rs -n)
+          CRATE_VERSION=$(cargo search --limit 1 ${CRATE_NAME} | head -n 1 | sed -e 's/[^"]*"\([^"]*\)".*/\1/')
+          cargo public-api diff --deny changed --deny removed "${CRATE_VERSION}"
+
+  #  dependency-review:
+  #    runs-on: ubuntu-latest
+  #    steps:
+  #      - name: 'Checkout Repository'
+  #        uses: actions/checkout@v3
+  #      - name: 'Dependency Review'
+  #        uses: actions/dependency-review-action@v3
+  #        with:
+  #          allow-licenses: Apache-2.0, ISC, MIT, MIT-0
 
   # This step won't be needed once "dependency-review" is enabled
   audit:
@@ -109,6 +124,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
@@ -134,6 +150,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
@@ -155,7 +172,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          submodules: true
+          submodules: 'recursive'
 
       - uses: actions-rs/[email protected]
         id: toolchain
@@ -175,15 +192,15 @@ jobs:
         env:
           RUSTC_WRAPPER: ""
 
-  test:
-    name: tests
+  aws-lc-rust-test:
+    name: aws-lc-rust tests
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        rust: [stable]
-        os: [ubuntu-latest, macOS-latest]
-        target: [native, aarch64-unknown-linux-gnu, i686-unknown-linux-gnu]
+        rust: [ stable ]
+        os: [ ubuntu-latest, macOS-latest ]
+        target: [ native, aarch64-unknown-linux-gnu, i686-unknown-linux-gnu ]
         args:
           - --all-targets
           - --release --all-targets --features ring-benchmarks
@@ -200,6 +217,8 @@ jobs:
             target: i686-unknown-linux-gnu
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
@@ -215,23 +234,26 @@ jobs:
           args: cross
 
       - name: Run cargo test
-        uses: actions-rs/[email protected]
-        with:
-          command: test
-          args: ${{ matrix.args }} ${{ matrix.target != 'native' && format('--target {0}', matrix.target) || '' }}
-          use-cross: ${{ matrix.target != 'native' }}
+        working-directory: ./aws-lc-rust
+        if: ${{ matrix.target == 'native' }}
+        run: cargo test ${{ matrix.args }} ${{ matrix.target != 'native' && format('--target {0}', matrix.target) || '' }}
+
+      - name: Run cargo cross test
+        working-directory: ./aws-lc-rust
+        if: ${{ matrix.target != 'native' }}
+        run: cross test ${{ matrix.args }} ${{ matrix.target != 'native' && format('--target {0}', matrix.target) || '' }}
 
   fips-test:
-    name: fips-tests
+    name: aws-lc-rust fips-tests
     runs-on: ${{ matrix.os }}
     env:
       CC: clang
       CXX: clang++
     strategy:
       fail-fast: false
       matrix:
-        rust: [stable]
-        os: [ubuntu-latest]
+        rust: [ stable ]
+        os: [ ubuntu-latest ]
         args:
           - --release --all-targets --features fips
           - --no-default-features --features fips
@@ -241,22 +263,24 @@ jobs:
           - --no-default-features --features fips,bindgen
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
           toolchain: ${{ matrix.rust }}
           override: true
       - name: Run cargo test
-        uses: actions-rs/[email protected]
-        with:
-          command: test
-          args: ${{ matrix.args }}
+        working-directory: ./aws-lc-rust
+        run: cargo test ${{ matrix.args }}
 
-  coverage:
+  aws-lc-rust-coverage:
+    name: aws-ls-rust coverage
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:
+          submodules: 'recursive'
           lfs: true
 
       - uses: actions-rs/[email protected]
@@ -276,16 +300,15 @@ jobs:
       # account. Using --html can give us more insight which regions are missing
       # coverage immediately.
       - name: Run coverage
-        uses: actions-rs/[email protected]
-        with:
-          command: llvm-cov
-          args: --no-fail-fast --fail-under-lines 95
+        working-directory: ./aws-lc-rust
+        run: cargo llvm-cov --no-fail-fast --fail-under-lines 95 --ignore-filename-regex "aws-lc-sys/*"
 
   mirai-analysis:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:
+          submodules: 'recursive'
           lfs: true
 
       # Update toolchain when fixed: https://github.com/facebookexperimental/MIRAI/issues/1188
@@ -319,78 +342,52 @@ jobs:
         run: |
           cargo mirai
 
-  asan:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/[email protected]
-        id: toolchain
-        with:
-          toolchain: ${{ env.RUST_NIGHTLY_TOOLCHAIN }}
-          override: true
-      - name: Run address sanitizers
-        uses: actions-rs/[email protected]
-        env:
-          ASAN_OPTIONS: detect_leaks=1
-          RUSTFLAGS: -Zsanitizer=address
-          RUSTDOCFLAGS: -Zsanitizer=address
-        with:
-          command: test
-          args: --lib --bins --tests --examples --target x86_64-unknown-linux-gnu --features asan
-
-  asan-release:
+  aws-lc-rust-asan:
+    name: aws-lc-rust asan
+    strategy:
+      matrix:
+        args:
+          -
+          - --release
+        features:
+          - --features asan
+          - --no-default-features --features fips,asan
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/[email protected]
-        id: toolchain
         with:
-          toolchain: ${{ env.RUST_NIGHTLY_TOOLCHAIN }}
-          override: true
-      - name: Run address sanitizers
-        uses: actions-rs/[email protected]
-        env:
-          ASAN_OPTIONS: detect_leaks=1
-          RUSTFLAGS: -Zsanitizer=address
-          RUSTDOCFLAGS: -Zsanitizer=address
-        with:
-          command: test
-          args: --release --lib --bins --tests --examples --target x86_64-unknown-linux-gnu --features asan
-
-  asan-fips:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
           toolchain: ${{ env.RUST_NIGHTLY_TOOLCHAIN }}
           override: true
       - name: Run address sanitizers
-        uses: actions-rs/[email protected]
         env:
           ASAN_OPTIONS: detect_leaks=1
           RUSTFLAGS: -Zsanitizer=address
           RUSTDOCFLAGS: -Zsanitizer=address
-        with:
-          command: test
-          args: --lib --bins --tests --examples --target x86_64-unknown-linux-gnu --no-default-features --features fips,asan
+        working-directory: ./aws-lc-rust
+        run: cargo test ${{ matrix.args }} --lib --bins --tests --examples --target x86_64-unknown-linux-gnu --features asan
 
   s2n-quic-integration:
     name: s2n-quic-integration
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macOS-latest]
+        os: [ ubuntu-latest, macOS-latest ]
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
           toolchain: stable
           override: true
       - name: Run s2n-quic integration
+        working-directory: ./aws-lc-rust
         run: |
           ./scripts/run-s2n-quic-integration.sh
 
@@ -400,14 +397,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macOS-latest]
+        os: [ ubuntu-latest, macOS-latest ]
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
       - uses: actions-rs/[email protected]
         id: toolchain
         with:
           toolchain: stable
           override: true
       - name: Run rustls integration
+        working-directory: ./aws-lc-rust
         run: |
           ./scripts/run-rustls-integration.sh

.gitmodules

@@ -0,0 +1,8 @@
+[submodule "aws-lc-sys/aws-lc"]
+	path = aws-lc-sys/aws-lc
+	url = https://github.com/aws/aws-lc.git
+	branch = main
+[submodule "aws-lc-fips-sys/aws-lc"]
+	path = aws-lc-fips-sys/aws-lc
+	url = https://github.com/aws/aws-lc.git
+	branch = fips-2022-11-02