Refactor testsuite

- include a `config.mk` if it exists. This is useful to set local cflags
  while debugging. For example:
    echo "CFLAGS+=-U_FORTIFY_SOURCE" > config.mk
- Remove 'gcc' and 'clang' targets. Let user set that via CC.
- Add a `check` target, so user can run the test suite with
  `make check`.
- set -nostdinc always to avoid pull in system headers

.github/workflows/testsuite.yaml

@@ -12,6 +12,9 @@ jobs:
       matrix:
         version: [9, 10, 11, 12, 13]
         use_native_chk: [true, false]
+    env:
+      CC: ../x86_64-linux-musl-native/bin/gcc
+      SYS_INCLUDES: -I/usr/include/x86_64-linux-musl -I../x86_64-linux-musl-native/include/ -Ix86_64-linux-musl-native/include/
     steps:
       - name: Checking out the code
         uses: actions/checkout@v3
@@ -36,18 +39,19 @@ jobs:
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ matrix.version }} 100
       - name: Build with native chk
         if: ${{ matrix.use_native_chk == true }}
-        run: make CFLAGS=-DFORTIFY_USE_NATIVE_CHK -C tests gcc CC=../x86_64-linux-musl-native/bin/gcc
+        run: make TARGET_DIR=out.gcc.native_chk CFLAGS=-DFORTIFY_USE_NATIVE_CHK -C tests
       - name: Build without native chk, and run the testsuite
         if: ${{ matrix.use_native_chk == false }}
         shell: bash
         run: |
-          make -C tests clean gcc run CC=../x86_64-linux-musl-native/bin/gcc > ./results.txt
+          make -C tests check > ./results.txt
           grep -zvq 'FAIL' ./results.txt
 
   clang:
     runs-on: ubuntu-latest
     env:
-      SYS_INCLUDES: -I/usr/include/x86_64-linux-musl -I../x86_64-linux-musl-native/include/ -Ix86_64-linux-musl-native/include/ -nostdinc
+      SYS_INCLUDES: -I/usr/include/x86_64-linux-musl -I../x86_64-linux-musl-native/include/ -Ix86_64-linux-musl-native/include/
+      CC: clang
     strategy:
       matrix:
         version: [13, 14, 15]
@@ -73,12 +77,12 @@ jobs:
           sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ matrix.version }} 100
       - name: Build with native chk
         if: ${{ matrix.use_native_chk == true }}
-        run: make CFLAGS=-DFORTIFY_USE_NATIVE_CHK SYS_INCLUDES="$SYS_INCLUDES" -C tests clang
+        run: make CFLAGS=-DFORTIFY_USE_NATIVE_CHK SYS_INCLUDES="$SYS_INCLUDES" CC=clang -C tests clang
       - name: Building and running without native chk
         if: ${{ matrix.use_native_chk == false }}
         shell: bash
         run: |
-          make SYS_INCLUDES="$SYS_INCLUDES" -C tests clean clang run > ./results.txt
+          make SYS_INCLUDES="$SYS_INCLUDES" CC=clang TARGET_DIR=out.clang -C tests check > ./results.txt
           grep -zvq 'FAIL' ./results.txt
 
   c_versions:

Makefile

@@ -17,6 +17,8 @@ dist: clean
 	gzip fortify-headers-$(VERSION).tar
 	rm -rf fortify-headers-$(VERSION)
 
+check:
+	$(MAKE) -C tests check
 clean:
 	rm -f fortify-headers-$(VERSION).tar.gz
 

tests/Makefile

@@ -1,5 +1,9 @@
-FORTIFY_CFLAGS= -D_FORTIFY_SOURCE=3
-CFLAGS+=-I../include/ $(FORTIFY_CFLAGS) -static -O2 -DFORTIFY_PEDANTIC_CHECKS -Wno-format -Werror=pointer-arith
+-include config.mk
+
+FORTIFY_LEVEL=3
+FORTIFY_CFLAGS= -D_FORTIFY_SOURCE=$(FORTIFY_LEVEL)
+SYS_INCLUDES= -nostdinc -I/usr/include
+CFLAGS+=-I../include/ $(FORTIFY_CFLAGS) -static -O2 -DFORTIFY_PEDANTIC_CHECKS -Wno-format -Werror=pointer-arith -nostdinc
 
 COMPTIME_BINS= \
 	test_memcpy_overwrite_under  \
@@ -149,33 +153,23 @@ RUNTIME_BINS= \
 	test_write_dynamic           \
 	test_write_static            \
 
-TARGET_DIR=out
+TARGET_DIR=out.$(notdir $(CC))
 RUNTIME_TARGETS = $(addprefix $(TARGET_DIR)/,$(RUNTIME_BINS))
 COMPTIME_TARGETS = $(addprefix $(TARGET_DIR)/,$(COMPTIME_BINS))
 
 .SILENT:
 
-gcc: CC=gcc
-gcc: $(RUNTIME_TARGETS)
-
-clang: CC=clang
-clang: CXX=clang++
-clang: CXXFLAGS+=-I/usr/include/x86_64-linux-musl
-clang: CXXFLAGS+=-I../x86_64-linux-musl-native/include/
-clang: CXXFLAGS+=-Ix86_64-linux-musl-native/include/
-clang: CXXFLAGS+=-nostdinc
-clang: comptime $(RUNTIME_TARGETS) cpp
+check: run comptime cpp
 
 coverage: CFLAGS += -fprofile-arcs -ftest-coverage
-coverage: CC=../x86_64-linux-musl-native/bin/gcc
-coverage: GCOV=../x86_64-linux-musl-native/bin/gcov
+coverage: CC=gcc
+coverage: GCOV=gcov
 coverage: $(RUNTIME_TARGETS) run
 	$(GCOV) *.c
 	lcov --capture --directory . --output-file coverage.info
 	lcov --remove ./coverage.info "*/tests/*" --output-file cleaned-coverage.info
 	genhtml cleaned-coverage.info --output-directory coverage
 
-all: gcc
 
 $(TARGET_DIR):
 	mkdir -p $@