diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8d209a4f974..a826e6f063e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -143,7 +143,7 @@ jobs: FROM ${{ matrix.config.base_image }} WORKDIR /app RUN apk update && apk add make g++ - CMD ["sh", "script.sh"] + CMD ["sh", "src/script.sh"] EOF - name: Download required macOS packages @@ -176,7 +176,7 @@ jobs: $COMPCXX -v else echo "$COMPCXX -v" > script.sh - docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder + docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder fi - name: Test help target @@ -342,8 +342,8 @@ jobs: - name: Test riscv64 build if: matrix.config.run_riscv64_tests run: | - echo "export LDFLAGS='-static' && make clean && make -j4 ARCH=riscv64 build" > script.sh - docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder + echo "cd src && export LDFLAGS='-static' && make clean && make -j4 ARCH=riscv64 build" > script.sh + docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder ../tests/signature.sh $benchref # ppc64 tests @@ -351,8 +351,8 @@ jobs: - name: Test ppc64 build if: matrix.config.run_ppc64_tests run: | - echo "export LDFLAGS='-static' && make clean && make -j4 ARCH=ppc-64 build" > script.sh - docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder + echo "cd src && export LDFLAGS='-static' && make clean && make -j4 ARCH=ppc-64 build" > script.sh + docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder ../tests/signature.sh $benchref # Other tests diff --git a/scripts/net.sh b/scripts/net.sh new file mode 100755 index 00000000000..168fbad66b0 --- /dev/null +++ b/scripts/net.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +wget_or_curl=$( (command -v wget > /dev/null 2>&1 && echo "wget -q") || \ + (command -v curl > /dev/null 2>&1 && echo "curl -L -s -k")) + +if [ -z "$wget_or_curl" ]; then + >&2 printf "%s\n" "Neither wget or curl is installed." \ + "Install one of these tools to download NNUE files automatically." + exit 1 +fi + +sha256sum=$( (command -v shasum > /dev/null 2>&1 && echo "shasum -a 256") || \ + (command -v sha256sum > /dev/null 2>&1 && echo "sha256sum")) + +if [ -z "$sha256sum" ]; then + >&2 echo "sha256sum not found, NNUE files will be assumed valid." +fi + +get_nnue_filename() { + grep "$1" evaluate.h | grep "#define" | sed "s/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/" +} + +validate_network() { + # If no sha256sum command is available, assume the file is always valid. + if [ -n "$sha256sum" ] && [ -f "$1" ]; then + if [ "$1" != "nn-$($sha256sum "$1" | cut -c 1-12).nnue" ]; then + rm -f "$1" + return 1 + fi + fi +} + +fetch_network() { + _filename="$(get_nnue_filename "$1")" + + if [ -z "$_filename" ]; then + >&2 echo "NNUE file name not found for: $1" + return 1 + fi + + if [ -f "$_filename" ]; then + if validate_network "$_filename"; then + echo "Existing $_filename validated, skipping download" + return + else + echo "Removing invalid NNUE file: $_filename" + fi + fi + + for url in \ + "https://tests.stockfishchess.org/api/nn/$_filename" \ + "https://github.com/official-stockfish/networks/raw/master/$_filename"; do + echo "Downloading from $url ..." + if $wget_or_curl "$url"; then + if validate_network "$_filename"; then + echo "Successfully validated $_filename" + else + echo "Downloaded $_filename is invalid" + continue + fi + else + echo "Failed to download from $url" + fi + if [ -f "$_filename" ]; then + return + fi + done + + # Download was not successful in the loop, return false. + >&2 echo "Failed to download $_filename" + return 1 +} + +fetch_network EvalFileDefaultNameBig && \ +fetch_network EvalFileDefaultNameSmall diff --git a/src/Makefile b/src/Makefile index 7142b972745..042d9479cc8 100644 --- a/src/Makefile +++ b/src/Makefile @@ -917,59 +917,9 @@ profileclean: @rm -f stockfish.res @rm -f ./-lstdc++.res -define fetch_network - @echo "Default net: $(nnuenet)" - @if [ "x$(curl_or_wget)" = "x" ]; then \ - echo "Neither curl nor wget is installed. Install one of these tools unless the net has been downloaded manually"; \ - fi - @if [ "x$(shasum_command)" = "x" ]; then \ - echo "shasum / sha256sum not found, skipping net validation"; \ - elif test -f "$(nnuenet)"; then \ - if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \ - echo "Removing invalid network"; rm -f $(nnuenet); \ - fi; \ - fi; - @for nnuedownloadurl in "$(nnuedownloadurl1)" "$(nnuedownloadurl2)"; do \ - if test -f "$(nnuenet)"; then \ - echo "$(nnuenet) available : OK"; break; \ - else \ - if [ "x$(curl_or_wget)" != "x" ]; then \ - echo "Downloading $${nnuedownloadurl}"; $(curl_or_wget) $${nnuedownloadurl} > $(nnuenet);\ - else \ - echo "No net found and download not possible"; exit 1;\ - fi; \ - fi; \ - if [ "x$(shasum_command)" != "x" ]; then \ - if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \ - echo "Removing failed download"; rm -f $(nnuenet); \ - fi; \ - fi; \ - done - @if ! test -f "$(nnuenet)"; then \ - echo "Failed to download $(nnuenet)."; \ - fi; - @if [ "x$(shasum_command)" != "x" ]; then \ - if [ "$(nnuenet)" = "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \ - echo "Network validated"; break; \ - fi; \ - fi; -endef - -# set up shell variables for the net stuff -define netvariables -$(eval nnuenet := $(shell grep $(1) evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/')) -$(eval nnuedownloadurl1 := https://tests.stockfishchess.org/api/nn/$(nnuenet)) -$(eval nnuedownloadurl2 := https://github.com/official-stockfish/networks/raw/master/$(nnuenet)) -$(eval curl_or_wget := $(shell if hash curl 2>/dev/null; then echo "curl -skL"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi)) -$(eval shasum_command := $(shell if hash shasum 2>/dev/null; then echo "shasum -a 256 "; elif hash sha256sum 2>/dev/null; then echo "sha256sum "; fi)) -endef - # evaluation network (nnue) net: - $(call netvariables, EvalFileDefaultNameBig) - $(call fetch_network) - $(call netvariables, EvalFileDefaultNameSmall) - $(call fetch_network) + @$(SHELL) ../scripts/net.sh format: $(CLANG-FORMAT) -i $(SRCS) $(HEADERS) -style=file