Merge pull request #2268 from fermyon/run-runtime-tests

Make sure runtime tests run in CI

Author: rylev

.github/workflows/build.yml

@@ -112,7 +112,7 @@ jobs:
           name: spin-static-${{ matrix.config.arch }}
           path: target/${{ matrix.config.target }}/release/spin
 
-  build-rust:
+  build-spin:
     name: Build Spin
     runs-on: ${{ matrix.os }}
     strategy:
@@ -140,7 +140,7 @@ jobs:
           path: target/release/spin${{ matrix.os == 'windows-latest' && '.exe' || '' }}
 
 
-  test-rust:
+  test-spin:
     name: Test Spin
     runs-on: ${{ matrix.runner }}
     strategy:
@@ -230,5 +230,4 @@ jobs:
           echo "$PWD" >> $GITHUB_PATH
 
       - name: Run Full Integration Tests
-        run: |
-          make test-integration-full
+        run: make test-integration-full

Makefile

@@ -49,20 +49,26 @@ update-cargo-locks:
 
 .PHONY: test-unit
 test-unit:
-	$(LOG_LEVEL_VAR) cargo test --all --no-fail-fast -- --skip integration_tests --nocapture
+	$(LOG_LEVEL_VAR) cargo test --all --no-fail-fast -- --skip integration_tests --skip runtime_tests --nocapture
 
-.PHONY: test-crate
-test-crate:
-	$(LOG_LEVEL_VAR) cargo test -p $(crate) --no-fail-fast -- --skip integration_tests --nocapture
+# Run the runtime tests without the tests that use some sort of assumed external depedency (e.g., Docker, a language toolchain, etc.)
+.PHONY: test-runtime
+test-runtime:
+	cargo test --release runtime_tests --no-default-features --no-fail-fast -- --nocapture
+
+# Run all of the runtime tests including those that use some sort of assumed external depedency (e.g., Docker, a language toolchain, etc.)
+.PHONY: test-runtime-full
+test-runtime-full:
+	cargo test --release runtime_tests --no-default-features --features extern-dependencies-tests --no-fail-fast -- --nocapture
 
 # Run the integration tests without the tests that use some sort of assumed external depedency (e.g., Docker, a language toolchain, etc.)
 .PHONY: test-integration
-test-integration:
+test-integration: test-runtime
 	cargo test --release integration_tests --no-default-features --no-fail-fast -- --nocapture
 
 # Run all of the integration tests including those that use some sort of assumed external depedency (e.g., Docker, a language toolchain, etc.)
 .PHONY: test-integration-full
-test-integration-full:
+test-integration-full: test-runtime-full
 	cargo test --release integration_tests --no-default-features --features extern-dependencies-tests --no-fail-fast -- --nocapture
 
 .PHONY: test-sdk-go

tests/README.md

@@ -36,3 +36,9 @@ make test-integration
 ```
 
 Note that this also runs the runtime tests as well.
+
+This will not run the full integration test suite, but a subset that only relies the presence of Rust and Python toolchains. The full integration test suite runs tests that rely on Docker and some additional compiler toolchains (e.g., Swift, Zig, etc.). Eventually, we want to only require the presence of Docker, but we're not quite there yet. You can run the full test suite like so:
+
+```bash
+make test-integration-full
+```

tests/runtime-tests/tests/llm/error.txt

@@ -1 +1 @@
-Could not read model registry directory '.spin/ai-models'
+Local LLM operations are not supported in this version of Spin.