feat: add code coverage

.github/workflows/check.yml

@@ -11,6 +11,13 @@ concurrency:
 
 jobs:
   check:
+    permissions:
+      pull-requests: write
+      id-token: write
+      pages: write
+      checks: write
+      contents: write
+
     runs-on: ubuntu-24.04-8core-bakunin
 
     steps:
@@ -44,8 +51,86 @@ jobs:
           args: --color=always --tests -- -D warnings
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Run tests
+      # - name: Run tests
+      #   env:
+      #     RUST_BACKTRACE: 1
+      #     SKIP_WASM_BUILD: 1
+      #   run: cargo test
+
+      - uses: jwalton/gh-find-current-pr@v1
+        id: findPr
+
+      - name: Extract branch name
+        shell: bash
+        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+        id: extractBranch
+
+      - name: Install cargo-llvm-cov
+        if: success() && steps.findPr.outputs.number && steps.extractBranch.outputs.branch
+        uses: taiki-e/install-action@cargo-llvm-cov
+
+      - name: Install cargo-xtask
+        if: success() && steps.findPr.outputs.number && steps.extractBranch.outputs.branch
+        run: cargo install cargo-xtask
+
+      - name: Generate lcov code coverage
+        if: success() && steps.findPr.outputs.number && steps.extractBranch.outputs.branch
+        run: cargo xtask coverage
+        env:
+          RUST_BACKTRACE: 1
+          SKIP_WASM_BUILD: 1
+
+      - name: Generate coverage summary report
+        if: success() && steps.findPr.outputs.number && steps.extractBranch.outputs.branch
+        uses: irongut/[email protected]
+        with:
+          filename: target/cov.xml
+          badge: true
+          format: markdown
+          hide_branch_rate: false
+          hide_complexity: true
+          indicators: true
+          output: both
+
+      - name: Generate html code coverage
+        if: success() && steps.findPr.outputs.number && steps.extractBranch.outputs.branch
+        run: cargo xtask coverage --html
         env:
           RUST_BACKTRACE: 1
           SKIP_WASM_BUILD: 1
-        run: cargo test
+
+      - name: Upload html report to S3 Bucket 
+        if: success() && steps.findPr.outputs.number && steps.extractBranch.outputs.branch
+        id: htmlUpload
+        run: |
+          aws --endpoint-url $ENDPOINT s3 sync ./target/llvm-cov/html s3://$BUCKET_NAME/$BRANCH
+          echo "link=$(aws --endpoint-url $ENDPOINT s3 presign s3://$BUCKET_NAME/$BRANCH/index.html)" >> $GITHUB_OUTPUT
+        env:
+          BRANCH: ${{ steps.extractBranch.outputs.branch }}
+          ENDPOINT: ${{ vars.COV_ENDPOINT_URL }}
+          BUCKET_NAME: ${{ vars.COV_BUCKET_NAME }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.COV_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.COV_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ vars.COV_DEFAULT_REGION }}
+
+      - name: Add coverage PR report comment
+        if: success() && steps.findPr.outputs.number
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: report
+          number: ${{ steps.findPr.outputs.pr }}
+          recreate: true
+          path: code-coverage-results.md
+
+      - name: Add coverage PR html comment
+        if: success() && steps.findPr.outputs.number
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: html
+          number: ${{ steps.findPr.outputs.pr }}
+          recreate: true
+          message: |
+            [Detailed coverage report](${{ steps.htmlUpload.outputs.link }})
+
+
+

.vscode/extensions.json

@@ -1,6 +1,7 @@
 {
   "recommendations": [
     "streetsidesoftware.code-spell-checker",
-    "rust-lang.rust-analyzer"
+    "rust-lang.rust-analyzer",
+    "ryanluker.vscode-coverage-gutters"
   ]
-}
+}

.vscode/settings.json

@@ -2,7 +2,16 @@
   // Editor
   "editor.formatOnSave": true,
   // Rust
-  "rust-analyzer.cargo.extraEnv": { "SKIP_WASM_BUILD": "1" },
-  "rust-analyzer.check.extraEnv": { "SKIP_WASM_BUILD": "1" },
-  "rust-analyzer.cargo.features": ["runtime-benchmarks"]
-}
+  "rust-analyzer.cargo.extraEnv": {
+    "SKIP_WASM_BUILD": "1"
+  },
+  "rust-analyzer.check.extraEnv": {
+    "SKIP_WASM_BUILD": "1"
+  },
+  "rust-analyzer.cargo.features": [
+    "runtime-benchmarks"
+  ],
+  "coverage-gutters.coverageFileNames": [
+    "target/cov.xml",
+  ],
+}

README.md

@@ -88,3 +88,16 @@ Running a local dev node:
 ```sh
 cargo xtask run local --alice
 ```
+
+## Code Coverage
+
+> Code coverage is done via [cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov).  
+
+Run the following command to generate a `Cobertura` xml file on `target/cov.xml` that can be used with the [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) VSCode plugin to display which functions and branches are not covered by tests yet.
+```bash
+cargo xtask coverage
+``` 
+
+If the `--html` attribute is passed to the command, an HTML website will be generated instead. It serves the same purpose as the plugin mentioned and can be accessed on `target/llvm-cov/html/index.html`
+
+**Pull requests must not lower the overall test coverage percentage.**

flake.nix

@@ -37,6 +37,8 @@
           pkgs.act
           # Python
           pkgs.python310
+          # Code coverage tool
+          pkgs.cargo-llvm-cov
         ];
       in
       {

rust-toolchain.toml

@@ -6,6 +6,6 @@
 # but we use the oxalica overlay, which does not fix this yet.
 # Should be fixed by https://github.com/rust-lang/rust/issues/123151.
 channel = "1.82.0"
-components = ["clippy", "rustfmt", "rust-src", "rust-analyzer"]
+components = ["clippy", "rustfmt", "rust-src", "rust-analyzer", "llvm-tools-preview"]
 targets = ["x86_64-unknown-linux-gnu", "wasm32-unknown-unknown"]
 profile = "minimal"

xtask/src/flags.rs

@@ -61,6 +61,10 @@ xflags::xflags! {
 
             optional --sudo sudo_key: String
         }
+
+        cmd coverage {
+            optional --html
+        }
     }
 }
 
@@ -76,6 +80,7 @@ pub struct Xtask {
 pub enum XtaskCmd {
     Run(Run),
     GenerateSpec(GenerateSpec),
+    Coverage(Coverage),
 }
 
 #[derive(Debug)]
@@ -114,6 +119,11 @@ pub struct GenerateSpec {
     pub sudo: Option<String>,
 }
 
+#[derive(Debug)]
+pub struct Coverage {
+    pub html: bool,
+}
+
 impl Xtask {
     #[allow(dead_code)]
     pub fn from_env_or_exit() -> Self {

xtask/src/main.rs

@@ -1,4 +1,4 @@
-use std::{borrow::Cow, net::IpAddr, path::Path};
+use std::{borrow::Cow, net::IpAddr, os::unix::process::CommandExt, path::Path};
 
 use polkadot_sdk::sp_keyring;
 
@@ -75,6 +75,35 @@ fn main() {
 
             std::fs::write(cmd.out, out).expect("failed to write resulting ");
         }
+        flags::XtaskCmd::Coverage(coverage) => {
+            const PALLETS: [&str; 3] = ["pallet-emission0", "pallet-governance", "pallet-torus0"];
+
+            let mut cmd = std::process::Command::new("cargo");
+            let mut args = vec![
+                "llvm-cov",
+                "--no-clean",
+                "--exclude-from-report",
+                "test-utils",
+                "--ignore-filename-regex",
+                "test-utils",
+            ];
+
+            for pallet in PALLETS {
+                args.extend_from_slice(&["-p", pallet]);
+            }
+
+            if coverage.html {
+                let dev_args = ["--html"];
+                args.extend_from_slice(&dev_args);
+            } else {
+                let ci_args = ["--cobertura", "--output-path", "target/cov.xml"];
+                args.extend_from_slice(&ci_args);
+            }
+
+            cmd.args(args);
+            dbg!(&cmd);
+            cmd.exec();
+        }
     }
 }