.gitlab-ci.yml

#############################################################################
# gitlab-runner:docker executor options
image: chapeiro/pelago-build:cuda11.3-llvm12
# - Add support for docker-in-docker tests / build processes

#############################################################################
variables:
 GIT_SUBMODULE_STRATEGY: recursive
 GIT_DEPTH: "1"

#############################################################################
stages:
 - Configure            # Environment configuration

 - Check Out            # When extra steps are needed to checkout the sources

 - Build Tools          # When extra tools are required

 - Package Build Tools  # Build & upload build requirements to registry
 
 - Builder Build Container # Builder image with proteus dependencies

 - Build Libraries      # When extra libraries are required 

 - Build                # Build the project

 - Install              # Install the software so it can be run for tests

 - Test                 # Functional tests

 - Benchmark            # Performance tests (need to be tagged in order to be
                        # run on a noise-free system)

 - Package              # Create archives to be installed

 - Publish              # Publish the packages

 - Build Profile        # Build the project with profiling information

 - Profile              # Profile using known workload

#############################################################################
# Helper Templates

before_script:
 - cache_path=$(pwd | sed "s/^\/builds/\/cache/g")/${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}
 - mkdir -p ${cache_path}/build ${cache_path}/opt
 - ln -s ${cache_path}/build build
 - ln -s ${cache_path}/opt opt
 - cp -p ${cache_path}/.*done ./ || true
 # TODO: we should probably remove the next lines and fix that in executor
 - ln -s /data/ssbm100 src/executor/tests/inputs/ssbm100
 - ln -s /data/tpch1 src/executor/tests/inputs/tpch1

after_script:
 # we have to recalculate cache_path here
 - cache_path=$(pwd | sed "s/^\/builds/\/cache/g")/${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}
 - cp -p .*done ${cache_path}/ || true

#----------------------------------------------------------------------------
# Caching policies templates

# As we keep the build & install marker files, we need to make sure the
# corresponding binaries are kept as well. While this works for all the C/C++
# projects as we build them outside of the sources, SBT does assume its own
# folder layout, which starts in src/<projetc>. Thus we need to keep a cache
# of the src folder as well.
# FIXME: Can we configure SBT to store the generated files under build?
# FIXME: Can we configure SBT to install the generated JAR under opt?
# FIXME: What about npm?
.cache: &cache
 tags:
  - pelago
 cache:
  key: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}"
  paths:
   - src/panorama
   - src/planner
   - src/SQLPlanner

.cache: &cache_profile
 cache:
  key: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}-profile"
  paths:
   - src/panorama
   - src/planner
   - src/SQLPlanner

#----------------------------------------------------------------------------
# Execution restrictions templates
.restrict_precommit: &restrict_precommit
 except:
  - master
  - tags

.restrict_postcommit: &restrict_postcommit
 only:
  - master
  - tags

.restrict_latest: &restrict_latest
 except:
  - tags
 only:
  - master

.restrict_release: &restrict_release
 only:
  - tags

#----------------------------------------------------------------------------
# Job templates
.test: &test
 stage: Test
 cache:
  policy: pull
 <<: *cache
 <<: *restrict_precommit

.benchmark: &benchmark
 stage: Benchmark
 cache:
  policy: pull
 <<: *cache

#############################################################################
# Example jobs
#----------------------------------------------------------------------------

# Workaround until we replace the docker image
.configure-packages:
 stage: Configure
 script:
  - echo "Configure... done"
 <<: *cache
 <<: *restrict_precommit

# Check-out environment
.check-out-submodules:
 stage: Check Out
 script:
  - echo "Check Out... done"
 <<: *cache
 <<: *restrict_precommit

#----------------------------------------------------------------------------
# Setup build environment
build-tools:
 stage: Build Tools
 script:
  - ccache_dir=$(pwd | sed "s/^\/builds/\/cache/g")/.ccache
  - git config --global user.name "Pelago CI"
  - git config --global user.email ""
  - CCACHE_DIR=${ccache_dir} ccache --zero-stats
  - CCACHE_DIR=${ccache_dir} CCACHE_NOHASHDIR=1 CCACHE_BASEDIR=$(pwd) CPACK_PACKAGE_CONTACT=dias-ci@epfl.ch make llvm &> llvm_build_log.txt || (tail -c 512k llvm_build_log.txt && exit 1)
  - CCACHE_DIR=${ccache_dir} ccache --show-stats
  - echo "Building tools... done"
 <<: *cache
 <<: *restrict_precommit

package-build-tools-llvm:
 stage: Package Build Tools
 script:
  - ccache_dir=$(pwd | sed "s/^\/builds/\/cache/g")/.ccache
  - CCACHE_DIR=${ccache_dir} ccache --zero-stats
  - CCACHE_DIR=${ccache_dir} CCACHE_NOHASHDIR=1 CCACHE_BASEDIR=$(pwd) CPACK_PACKAGE_CONTACT=dias-ci@epfl.ch make do-package-llvm
  - CCACHE_DIR=${ccache_dir} ccache --show-stats
  - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build/llvm-stage2/LLVM-14.0.0-Linux.deb "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/LLVM/14.0.0/LLVM-14.0.0-Linux.deb"'
  - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build/llvm-stage2/LLVM-14.0.0-Linux.tar.xz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/LLVM/14.0.0/LLVM-14.0.0-Linux.tar.xz"'
 <<: *cache
 <<: *restrict_precommit

package-build-tools-cmake:
 stage: Package Build Tools
 script:
  - ccache_dir=$(pwd | sed "s/^\/builds/\/cache/g")/.ccache
  - CCACHE_DIR=${ccache_dir} ccache --zero-stats
  - CCACHE_DIR=${ccache_dir} CCACHE_NOHASHDIR=1 CCACHE_BASEDIR=$(pwd) CPACK_PACKAGE_CONTACT=dias-ci@epfl.ch make do-package-cmake
  - CCACHE_DIR=${ccache_dir} ccache --show-stats
  - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build/cmake/cmake-3.23.0-Linux-x86_64.deb "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/cmake/3.23.0/cmake-3.23.0-Linux-x86_64.deb"'
  - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build/cmake/cmake-3.23.0-Linux-x86_64.tar.xz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/cmake/3.23.0/cmake-3.23.0-Linux-x86_64.tar.xz"'
 <<: *cache
 <<: *restrict_precommit

build-dependencies-container:
 image: docker:19.03.12
 stage: Builder Build Container
 script:
  - cp build/llvm-stage2/LLVM-14.0.0-Linux.deb src/executor/tools/docker/proteus-build/
  - cp build/cmake/cmake-3.23.0-Linux-x86_64.deb src/executor/tools/docker/proteus-build/
  # localhost as we use the parent docker
  - cd src/executor/tools/docker/proteus-build && docker build -t localhost:5055/proteus/pelago-build:latest --build-arg LLVM_VERSION=14.0.0 --build-arg CMAKE_VERSION=3.23.0 .
  - docker push localhost:5055/proteus/pelago-build:latest
 <<: *cache
 <<: *restrict_precommit

build-libraries:
 stage: Build Libraries
 script:
  - make external-libs &> external_libs_build_log.txt || (tail -c 512k external_libs_build_log.txt && exit 1)
  - echo "Building external libs... done"
 <<: *cache
 <<: *restrict_precommit

#----------------------------------------------------------------------------
# Setup test environment
build:
 stage: Build
 script:
  - make
  - echo "Building... done"
 <<: *cache
 <<: *restrict_precommit

.install:
 stage: Install
 script:
  - echo "Installing... done"
 <<: *cache
 <<: *restrict_precommit

#----------------------------------------------------------------------------
# Functional Tests
1/10 test - Threads:
 <<: *test
 script:
  - make &> check_installed.txt || (tail -c 512k check_installed.txt && exit 1) # make sure that everything is compiled
  - export CWD=$(pwd)
  - cd opt/pelago && ./unit-tests-threads --gtest_output="xml:$CWD/report-threads.xml" &> thread_test_log.txt || (tail -c 512k thread_test_log.txt && exit 1)
 artifacts:
  paths:
   - report-*.xml
  reports:
   junit: report-*.xml

2/10 test - Plan Parsing:
 <<: *test
 script:
  - make &> check_installed.txt || (tail -c 512k check_installed.txt && exit 1) # make sure that everything is compiled
  - export CWD=$(pwd)
  - cd opt/pelago && ./unit-tests-plan-parsing --gtest_output="xml:$CWD/report-plan-parsing.xml" &> plan_parsing_test_log.txt || (tail -c 512k plan_parsing_test_log.txt && exit 1)
 allow_failure: true
 artifacts:
  paths:
   - report-*.xml
  reports:
   junit: report-*.xml

3/10 test - JSON:
 <<: *test
 script:
  - make &> check_installed.txt || (tail -c 512k check_installed.txt && exit 1) # make sure that everything is compiled
  - export CWD=$(pwd)
  - cd opt/pelago && ./unit-tests-json --gtest_output="xml:$CWD/report-json.xml" &> json_test_log.txt || (tail -c 512k json_test_log.txt && exit 1)
 allow_failure: true
 artifacts:
  paths:
   - report-*.xml
  reports:
   junit: report-*.xml


4/10 test - Joins:
 <<: *test
 script:
  - make &> check_installed.txt || (tail -c 512k check_installed.txt && exit 1) # make sure that everything is compiled
  - export CWD=$(pwd)
  - cd opt/pelago && ./unit-tests-joins --gtest_output="xml:$CWD/report-joins.xml" &> joins_test_log.txt || (tail -c 512k joins_test_log.txt && exit 1)
 allow_failure: true
 artifacts:
  paths:
   - report-*.xml
  reports:
   junit: report-*.xml


5/10 test - Output:
 <<: *test
 script:
  - make &> check_installed.txt || (tail -c 512k check_installed.txt && exit 1) # make sure that everything is compiled
  - export CWD=$(pwd)
  - cd opt/pelago && ./unit-tests-output --gtest_output="xml:$CWD/report-output.xml" &> output_test_log.txt || (tail -c 512k output_test_log.txt && exit 1)
 allow_failure: true
 artifacts:
  paths:
   - report-*.xml
  reports:
   junit: report-*.xml


6/10 test - GPU SSB SF100:
 <<: *test
 script:
  - make &> check_installed.txt || (tail -c 512k check_installed.txt && exit 1) # make sure that everything is compiled
  - export CWD=$(pwd)
  - cd opt/pelago && ./unit-tests-gpu-ssb --gtest_output="xml:$CWD/report-gpu.xml" &> gpu_ssb_test_log.txt || (tail -c 512k gpu_ssb_test_log.txt && exit 1)
 artifacts:
  paths:
   - report-*.xml
  reports:
   junit: report-*.xml


7/10 test - CPU SSB SF100:
 <<: *test
 script:
  - make &> check_installed.txt || (tail -c 512k check_installed.txt && exit 1) # make sure that everything is compiled
  - export CWD=$(pwd)
  - cd opt/pelago && ./unit-tests-cpu-ssb --gtest_output="xml:$CWD/report-cpu.xml" &> cpu_ssb_test_log.txt || (tail -c 512k cpu_ssb_test_log.txt && exit 1)
 artifacts:
  paths:
   - report-*.xml
  reports:
   junit: report-*.xml


8/10 test - Hybrid SSB SF100:
 <<: *test
 script:
  - make &> check_installed.txt || (tail -c 512k check_installed.txt && exit 1) # make sure that everything is compiled
  - export CWD=$(pwd)
  - cd opt/pelago && ./unit-tests-hyb-ssb --gtest_output="xml:$CWD/report-hyb.xml" &> hyb_ssb_test_log.txt || (tail -c 512k hyb_ssb_test_log.txt && exit 1)
 artifacts:
  paths:
   - report-*.xml
  reports:
   junit: report-*.xml

9/10 test - COW:
 <<: *test
 script:
  - make &> check_installed.txt || (tail -c 512k check_installed.txt && exit 1) # make sure that everything is compiled
  - export CWD=$(pwd)
  - cd opt/pelago && ./unit-tests-cow --gtest_output="xml:$CWD/report-cow.xml" &> cow_test_log.txt || (tail -c 512k cow_test_log.txt && exit 1)
 artifacts:
  paths:
   - report-*.xml
  reports:
   junit: report-*.xml

10/10 test - Planner:
 <<: *test
 script:
  - ln -s opt/pelago/inputs build/executor/core/planner/CMakeFiles/planner.dir/inputs
  - export CWD=$(pwd)
  - export OUTPUT=0
  - cd build/executor/core/planner/CMakeFiles/planner.dir && sbt test || export OUTPUT=1
  - mv build/executor/core/planner/CMakeFiles/planner.dir/target/test-reports/TEST-*.xml $CWD
  - rm build/executor/core/planner/CMakeFiles/planner.dir/inputs || true
  - exit $OUTPUT
 allow_failure: true
 artifacts:
  reports:
   junit: TEST-*.xml


#----------------------------------------------------------------------------
# Performance Tests
# unhide the jobs by removing the leading '.'
.bench:
 stage: Benchmark
 script:
  - echo "Bench... done"
 <<: *cache
 <<: *restrict_precommit

.1/2 benchmark:
 <<: *benchmark
 script:
  - echo "Benchmark 2... done"

.2/2 benchmark:
 <<: *benchmark
 script:
  - echo "Benchmark 2... done"

#----------------------------------------------------------------------------
# Execution Profiling 
# unhide the jobs by removing the leading '.'
.build profile:
 stage: Build Profile
 script:
  - echo "Building Dependencies... done"
  - echo "Building... done"
  - echo "Installing... done"
 when: manual
 <<: *cache_profile
 <<: *restrict_precommit

.1/2 profile:
 stage: Profile
 script:
  - echo "Running workload 1... done"
  - echo "Extracting profiling statistics"
 when: manual
 cache:
  policy: pull
 <<: *cache_profile
 <<: *restrict_precommit

.2/2 profile:
 stage: Profile
 script:
  - echo "Running workload 2... done"
  - echo "Extracting profiling statistics"
 when: manual
 cache:
  policy: pull
 <<: *cache_profile
 <<: *restrict_precommit

#----------------------------------------------------------------------------
# Packaging & distribution
.tar:
 stage: Package
 script:
  - echo "Generating tar... done"
 <<: *cache
 <<: *restrict_postcommit

.tar:latest:
 stage: Publish
 script:
  - echo "Publishing HEAD... done"
 artifacts:
  name: "$CI_PROJECT_NAME-g$CI_COMMIT_SHA"
  paths:
   - pkg/
 cache:
  policy: pull
 <<: *cache
 <<: *restrict_latest

.tar:release:
 stage: Publish
 script:
  - echo "Publishing release... done"
 artifacts:
  name: "$CI_PROJECT_NAME-$CI_COMMIT_TAG"
  paths:
   - pkg/
 cache:
  policy: pull
 <<: *cache
 <<: *restrict_release