From 9d2c6dc68850cc104e5d9f84dda6947aa0921ce1 Mon Sep 17 00:00:00 2001 From: MinetaS Date: Mon, 2 Sep 2024 19:22:18 +0900 Subject: [PATCH 1/2] Export and clean up net downloading script This patch extracts the net downloading script in Makefile into an external script file. Also the script is moderately rewritten for improved readability and speed. * Use wget preferentially over curl, as curl is known to have slight overhead. * Use command instead of hash to check if command exists. Reportedly, hash always returns zero in some POSIX shells even when the command fails. * Command existence checks (wget/curl, sha256sum) are performed only once at the beginning. * Each of common patterns is encapsulated in a function (get_nnue_filename, validate_network). * Print out error/warning messages to stderr. No functional change --- scripts/net.sh | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/Makefile | 52 +--------------------------------- 2 files changed, 76 insertions(+), 51 deletions(-) create mode 100755 scripts/net.sh diff --git a/scripts/net.sh b/scripts/net.sh new file mode 100755 index 00000000000..168fbad66b0 --- /dev/null +++ b/scripts/net.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +wget_or_curl=$( (command -v wget > /dev/null 2>&1 && echo "wget -q") || \ + (command -v curl > /dev/null 2>&1 && echo "curl -L -s -k")) + +if [ -z "$wget_or_curl" ]; then + >&2 printf "%s\n" "Neither wget or curl is installed." \ + "Install one of these tools to download NNUE files automatically." + exit 1 +fi + +sha256sum=$( (command -v shasum > /dev/null 2>&1 && echo "shasum -a 256") || \ + (command -v sha256sum > /dev/null 2>&1 && echo "sha256sum")) + +if [ -z "$sha256sum" ]; then + >&2 echo "sha256sum not found, NNUE files will be assumed valid." +fi + +get_nnue_filename() { + grep "$1" evaluate.h | grep "#define" | sed "s/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/" +} + +validate_network() { + # If no sha256sum command is available, assume the file is always valid. + if [ -n "$sha256sum" ] && [ -f "$1" ]; then + if [ "$1" != "nn-$($sha256sum "$1" | cut -c 1-12).nnue" ]; then + rm -f "$1" + return 1 + fi + fi +} + +fetch_network() { + _filename="$(get_nnue_filename "$1")" + + if [ -z "$_filename" ]; then + >&2 echo "NNUE file name not found for: $1" + return 1 + fi + + if [ -f "$_filename" ]; then + if validate_network "$_filename"; then + echo "Existing $_filename validated, skipping download" + return + else + echo "Removing invalid NNUE file: $_filename" + fi + fi + + for url in \ + "https://tests.stockfishchess.org/api/nn/$_filename" \ + "https://github.com/official-stockfish/networks/raw/master/$_filename"; do + echo "Downloading from $url ..." + if $wget_or_curl "$url"; then + if validate_network "$_filename"; then + echo "Successfully validated $_filename" + else + echo "Downloaded $_filename is invalid" + continue + fi + else + echo "Failed to download from $url" + fi + if [ -f "$_filename" ]; then + return + fi + done + + # Download was not successful in the loop, return false. + >&2 echo "Failed to download $_filename" + return 1 +} + +fetch_network EvalFileDefaultNameBig && \ +fetch_network EvalFileDefaultNameSmall diff --git a/src/Makefile b/src/Makefile index 7142b972745..042d9479cc8 100644 --- a/src/Makefile +++ b/src/Makefile @@ -917,59 +917,9 @@ profileclean: @rm -f stockfish.res @rm -f ./-lstdc++.res -define fetch_network - @echo "Default net: $(nnuenet)" - @if [ "x$(curl_or_wget)" = "x" ]; then \ - echo "Neither curl nor wget is installed. Install one of these tools unless the net has been downloaded manually"; \ - fi - @if [ "x$(shasum_command)" = "x" ]; then \ - echo "shasum / sha256sum not found, skipping net validation"; \ - elif test -f "$(nnuenet)"; then \ - if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \ - echo "Removing invalid network"; rm -f $(nnuenet); \ - fi; \ - fi; - @for nnuedownloadurl in "$(nnuedownloadurl1)" "$(nnuedownloadurl2)"; do \ - if test -f "$(nnuenet)"; then \ - echo "$(nnuenet) available : OK"; break; \ - else \ - if [ "x$(curl_or_wget)" != "x" ]; then \ - echo "Downloading $${nnuedownloadurl}"; $(curl_or_wget) $${nnuedownloadurl} > $(nnuenet);\ - else \ - echo "No net found and download not possible"; exit 1;\ - fi; \ - fi; \ - if [ "x$(shasum_command)" != "x" ]; then \ - if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \ - echo "Removing failed download"; rm -f $(nnuenet); \ - fi; \ - fi; \ - done - @if ! test -f "$(nnuenet)"; then \ - echo "Failed to download $(nnuenet)."; \ - fi; - @if [ "x$(shasum_command)" != "x" ]; then \ - if [ "$(nnuenet)" = "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \ - echo "Network validated"; break; \ - fi; \ - fi; -endef - -# set up shell variables for the net stuff -define netvariables -$(eval nnuenet := $(shell grep $(1) evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/')) -$(eval nnuedownloadurl1 := https://tests.stockfishchess.org/api/nn/$(nnuenet)) -$(eval nnuedownloadurl2 := https://github.com/official-stockfish/networks/raw/master/$(nnuenet)) -$(eval curl_or_wget := $(shell if hash curl 2>/dev/null; then echo "curl -skL"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi)) -$(eval shasum_command := $(shell if hash shasum 2>/dev/null; then echo "shasum -a 256 "; elif hash sha256sum 2>/dev/null; then echo "sha256sum "; fi)) -endef - # evaluation network (nnue) net: - $(call netvariables, EvalFileDefaultNameBig) - $(call fetch_network) - $(call netvariables, EvalFileDefaultNameSmall) - $(call fetch_network) + @$(SHELL) ../scripts/net.sh format: $(CLANG-FORMAT) -i $(SRCS) $(HEADERS) -style=file From b428b5f939b9febd299f9f9d0115f3bb86a9e2f1 Mon Sep 17 00:00:00 2001 From: Disservin Date: Tue, 3 Sep 2024 20:08:03 +0900 Subject: [PATCH 2/2] Fix Docker volume path in GitHub workflow Fixes https://github.com/official-stockfish/Stockfish/issues/5564 No functional change Co-authored-by: MinetaS --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8d209a4f974..a826e6f063e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -143,7 +143,7 @@ jobs: FROM ${{ matrix.config.base_image }} WORKDIR /app RUN apk update && apk add make g++ - CMD ["sh", "script.sh"] + CMD ["sh", "src/script.sh"] EOF - name: Download required macOS packages @@ -176,7 +176,7 @@ jobs: $COMPCXX -v else echo "$COMPCXX -v" > script.sh - docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder + docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder fi - name: Test help target @@ -342,8 +342,8 @@ jobs: - name: Test riscv64 build if: matrix.config.run_riscv64_tests run: | - echo "export LDFLAGS='-static' && make clean && make -j4 ARCH=riscv64 build" > script.sh - docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder + echo "cd src && export LDFLAGS='-static' && make clean && make -j4 ARCH=riscv64 build" > script.sh + docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder ../tests/signature.sh $benchref # ppc64 tests @@ -351,8 +351,8 @@ jobs: - name: Test ppc64 build if: matrix.config.run_ppc64_tests run: | - echo "export LDFLAGS='-static' && make clean && make -j4 ARCH=ppc-64 build" > script.sh - docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder + echo "cd src && export LDFLAGS='-static' && make clean && make -j4 ARCH=ppc-64 build" > script.sh + docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder ../tests/signature.sh $benchref # Other tests