Skip to content

Commit

Permalink
Merge pull request #4248 from pwojcikdev/ci-overhaul-rebased
Browse files Browse the repository at this point in the history
Improvements to CI pipeline
  • Loading branch information
pwojcikdev committed Jun 21, 2023
2 parents 74a9fba + 47a718e commit d33803e
Show file tree
Hide file tree
Showing 23 changed files with 378 additions and 73 deletions.
97 changes: 62 additions & 35 deletions .github/workflows/unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,6 @@ name: Unit Tests

on: [push, pull_request]

env:
RELEASE: 0
artifact: 0

jobs:
macos_test:
name: macOS [${{ matrix.BACKEND }}]
Expand All @@ -17,16 +13,17 @@ jobs:
- ${{ startsWith(github.ref, 'refs/tags/') }}
env:
BACKEND: ${{ matrix.BACKEND }}
TEST_USE_ROCKSDB: ${{ matrix.BACKEND == 'rocksdb' && '1' || '0' }}
RELEASE: ${{ matrix.RELEASE }}
TEST_USE_ROCKSDB: ${{ matrix.BACKEND == 'rocksdb' && '1' || '0' }}
DEADLINE_SCALE_FACTOR: ${{ matrix.BACKEND == 'rocksdb' && '2' || '1' }}
runs-on: macos-12
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
steps:
- name: Checkout
uses: actions/checkout@v3
with:
submodules: "recursive"
fetch-depth: 0 # full history needed for restoring file timestamps
fetch-depth: 0 # Full history needed for restoring file timestamps

- name: Restore Timestamps
uses: ./.github/actions/restore-git-mtimes
Expand All @@ -39,25 +36,33 @@ jobs:
path: build
key: ${{ runner.os }}-build-cache

- name: Fetch Deps
run: TEST=1 ci/actions/osx/install_deps.sh
- name: Prepare
run: ci/prepare/macos/prepare.sh

- name: Build Tests
run: ci/build-ci.sh "/tmp/qt/lib/cmake/Qt5";
id: build
run: ci/build-tests.sh

- name: Save Build Cache
# only save build cache from develop to avoid polluting it by other branches / PRs
# Only save build cache from develop to avoid polluting it by other branches / PRs
if: github.ref == 'refs/heads/develop' && success()
uses: actions/cache/save@v3
continue-on-error: true
with:
path: build
key: ${{ runner.os }}-build-cache

- name: Run Tests
run: cd build && sudo TEST_USE_ROCKSDB=$TEST_USE_ROCKSDB ../ci/test.sh .
env:
DEADLINE_SCALE_FACTOR: ${{ env.TEST_USE_ROCKSDB == 1 && '2' || '1' }}
- name: Core Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-core-tests.sh

- name: RPC Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-rpc-tests.sh

- name: System Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-system-tests.sh

linux_test:
name: Linux [${{ matrix.BACKEND }} | ${{ matrix.COMPILER }}]
Expand All @@ -73,15 +78,16 @@ jobs:
env:
COMPILER: ${{ matrix.COMPILER }}
BACKEND: ${{ matrix.BACKEND }}
TEST_USE_ROCKSDB: ${{ matrix.BACKEND == 'rocksdb' && '1' || '0' }}
RELEASE: ${{ matrix.RELEASE }}
TEST_USE_ROCKSDB: ${{ matrix.BACKEND == 'rocksdb' && '1' || '0' }}
DEADLINE_SCALE_FACTOR: ${{ matrix.BACKEND == 'rocksdb' && '2' || '1' }}
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
steps:
- name: Checkout
uses: actions/checkout@v3
with:
submodules: "recursive"
fetch-depth: 0 # full history needed for restoring file timestamps
fetch-depth: 0 # Full history needed for restoring file timestamps

- name: Restore Timestamps
uses: ./.github/actions/restore-git-mtimes
Expand All @@ -94,25 +100,37 @@ jobs:
path: build
key: ${{ runner.os }}-${{ env.COMPILER }}-build-cache

- name: Fetch Deps
run: ci/actions/linux/install_deps.sh
- name: Prepare
run: sudo -E ci/prepare/linux/prepare.sh

- name: Build Tests
run: docker run -e TEST_USE_ROCKSDB -e RELEASE -v ${PWD}:/workspace nanocurrency/nano-env:${{ matrix.COMPILER }} /bin/bash -c "cd /workspace && ./ci/build-ci.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5"
id: build
run: ci/build-tests.sh

- name: Save Build Cache
# only save build cache from develop to avoid polluting it by other branches / PRs
# Only save build cache from develop to avoid polluting it by other branches / PRs
if: github.ref == 'refs/heads/develop' && success()
uses: actions/cache/save@v3
continue-on-error: true
with:
path: build
key: ${{ runner.os }}-${{ env.COMPILER }}-build-cache

- name: Run Tests
run: docker run -e RELEASE -e TEST_USE_ROCKSDB -e DEADLINE_SCALE_FACTOR -v ${PWD}:/workspace nanocurrency/nano-env:${{ matrix.COMPILER }} /bin/bash -c "cd /workspace/build && ../ci/test.sh ."
env:
DEADLINE_SCALE_FACTOR: ${{ env.TEST_USE_ROCKSDB == 1 && '2' || '1' }}
- name: Core Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-core-tests.sh

- name: RPC Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-rpc-tests.sh

- name: System Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-system-tests.sh

- name: QT Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-qt-tests.sh

windows_test:
name: Windows [${{ matrix.BACKEND }}]
Expand All @@ -126,8 +144,8 @@ jobs:
runs-on: windows-latest
env:
BACKEND: ${{ matrix.BACKEND }}
TEST_USE_ROCKSDB: ${{ matrix.BACKEND == 'rocksdb' && '1' || '0' }}
RELEASE: ${{ matrix.RELEASE }}
TEST_USE_ROCKSDB: ${{ matrix.BACKEND == 'rocksdb' && '1' || '0' }}
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
steps:
- name: Checkout
Expand All @@ -147,14 +165,13 @@ jobs:
path: build
key: ${{ runner.os }}-build-cache

- name: Windows Defender
run: ci/actions/windows/disable_windows_defender.ps1

- name: Fetch Deps
run: ci/actions/windows/install_deps.ps1
- name: Prepare
run: ci/prepare/windows/prepare.ps1

- name: Build Tests
run: ci/actions/windows/build.ps1
id: build
run: ci/build-tests.sh
shell: bash

- name: Save Build Cache
# only save build cache from develop to avoid polluting it by other branches / PRs
Expand All @@ -165,7 +182,17 @@ jobs:
path: build
key: ${{ runner.os }}-build-cache

- name: Run Tests [TEST_USE_ROCKSDB=${{ env.TEST_USE_ROCKSDB }}]
run: ci/actions/windows/run.ps1
env:
DEADLINE_SCALE_FACTOR: ${{ env.TEST_USE_ROCKSDB == 1 && '2' || '1' }}
- name: Core Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-core-tests.sh
shell: bash

- name: RPC Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-rpc-tests.sh
shell: bash

- name: System Tests
if: steps.build.outcome == 'success' && (success() || failure())
run: cd build && ../ci/tests/run-system-tests.sh
shell: bash
23 changes: 11 additions & 12 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,6 @@ if(COVERAGE)
include(CoverageTest)
endif()

set(CI_TEST
0
CACHE STRING "")

if(MSVC)
add_definitions(/MP)
endif()
Expand Down Expand Up @@ -67,6 +63,8 @@ endif()

# Create all libraries and executables in the root binary dir
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR})
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR})

set(NANO_GUI
OFF
Expand Down Expand Up @@ -658,6 +656,12 @@ add_subdirectory(nano/nano_node)
add_subdirectory(nano/rpc)
add_subdirectory(nano/nano_rpc)

add_custom_target(
executables
COMMAND echo "BATCH BUILDING node + rpc"
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
DEPENDS nano_node nano_rpc)

if(NANO_FUZZER_TEST)
if(NOT WIN32)
add_subdirectory(nano/fuzzer_test)
Expand Down Expand Up @@ -696,15 +700,10 @@ if(NANO_TEST OR RAIBLOCKS_TEST)
add_subdirectory(nano/rpc_test)
add_subdirectory(nano/slow_test)
add_custom_target(
build_tests
all_tests
COMMAND echo "BATCH BUILDING TESTS"
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
DEPENDS core_test load_test rpc_test nano_node nano_rpc)
add_custom_target(
run_tests
COMMAND ${PROJECT_SOURCE_DIR}/ci/test.sh ${CMAKE_BINARY_DIR}
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
DEPENDS build_tests)
DEPENDS core_test load_test rpc_test slow_test nano_node nano_rpc)
endif()

if(NANO_TEST OR RAIBLOCKS_TEST)
Expand Down Expand Up @@ -809,7 +808,7 @@ if(NANO_GUI OR RAIBLOCKS_GUI)
set_target_properties(
qt_test PROPERTIES COMPILE_FLAGS
"-DQT_NO_KEYWORDS -DBOOST_ASIO_HAS_STD_ARRAY=1")
add_dependencies(build_tests qt_test)
add_dependencies(all_tests qt_test)
endif()

if(APPLE)
Expand Down
1 change: 0 additions & 1 deletion ci/actions/windows/build.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ else {
}
$env:NETWORK_CFG = "dev"
$env:NANO_TEST = "-DNANO_TEST=ON"
$env:CI = '-DCI_TEST="1"'
$env:RUN = "test"
}

Expand Down
2 changes: 1 addition & 1 deletion ci/build-ci.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -euo pipefail

qt_dir=${1}
build_target=${2:-all}
Expand Down Expand Up @@ -47,7 +48,6 @@ cmake \
-DNANO_WARN_TO_ERR=ON \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE:-Debug} \
-DQt5_DIR=${qt_dir} \
-DCI_TEST="1" \
${BACKTRACE:-} \
${SANITIZERS:-} \
..
Expand Down
4 changes: 4 additions & 0 deletions ci/build-node.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/bin/bash
set -euox pipefail

$(dirname "$BASH_SOURCE")/build.sh executables
13 changes: 13 additions & 0 deletions ci/build-tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/bin/bash
set -euox pipefail

BUILD_TYPE="Debug"
if [[ "${RELEASE:-false}" == "true" ]]; then
BUILD_TYPE="RelWithDebInfo"
fi

BUILD_TYPE=$BUILD_TYPE \
NANO_TEST=ON \
NANO_NETWORK=dev \
NANO_GUI=ON \
$(dirname "$BASH_SOURCE")/build.sh all_tests
93 changes: 93 additions & 0 deletions ci/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
#!/bin/bash
set -euox pipefail

BUILD_TARGET=""
if [[ ${1:-} ]]; then
BUILD_TARGET="--target $1"
fi

SRC=${SRC:-${PWD}}
OS=$(uname)

CMAKE_BACKTRACE=""
if [[ "$OS" == 'Linux' ]]; then
CMAKE_BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON"

if [[ "$COMPILER" == 'clang' ]]; then
CMAKE_BACKTRACE="${CMAKE_BACKTRACE} -DNANO_BACKTRACE_INCLUDE=</tmp/backtrace.h>"
fi
fi

CMAKE_QT_DIR=""
if [[ ${QT_DIR:-} ]]; then
CMAKE_QT_DIR="-DQt5_DIR=${QT_DIR}"
fi

CMAKE_SANITIZER=""
if [[ ${SANITIZER:-} ]]; then
case "${SANITIZER}" in
ASAN)
CMAKE_SANITIZER="-DNANO_ASAN=ON"
;;
ASAN_INT)
CMAKE_SANITIZER="-DNANO_ASAN_INT=ON"
;;
TSAN)
CMAKE_SANITIZER="-DNANO_TSAN=ON"
;;
*)
echo "Unknown sanitizer: '${SANITIZER}'"
exit 1
;;
esac
fi

BUILD_DIR="build"

mkdir -p $BUILD_DIR
pushd $BUILD_DIR

cmake \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE:-"Debug"} \
-DPORTABLE=ON \
-DACTIVE_NETWORK=nano_${NANO_NETWORK:-"live"}_network \
-DNANO_TEST=${NANO_TEST:-OFF} \
-DNANO_GUI=${NANO_GUI:-OFF} \
-DCOVERAGE=${COVERAGE:-OFF} \
${CMAKE_SANITIZER:-} \
${CMAKE_QT_DIR:-} \
${CMAKE_BACKTRACE:-} \
${SRC}

number_of_processors() {
case "$(uname -s)" in
Linux*)
nproc
;;
Darwin*)
sysctl -n hw.ncpu
;;
CYGWIN*|MINGW32*|MSYS*|MINGW*)
echo "${NUMBER_OF_PROCESSORS}"
;;
*)
echo "Unknown OS"
exit 1
;;
esac
}

parallel_build_flag() {
case "$(uname -s)" in
CYGWIN*|MINGW32*|MSYS*|MINGW*)
echo "-- -m"
;;
*)
echo "--parallel $(number_of_processors)"
;;
esac
}

cmake --build ${PWD} ${BUILD_TARGET} $(parallel_build_flag)

popd
Loading

0 comments on commit d33803e

Please sign in to comment.