config.yml

# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: 2.1
workflows:
  version: 2
  dist-compile:
    jobs:
      - linux-build
      - linux-build-options
      - linux-parquet-s3-build
      - macos-build
      - format-check
      - header-check
      - linux-benchmarks-basic

  # Daily documentation update
  nightly:
    triggers:
      - schedule:
         cron: "0 3 * * *"
         filters:
           branches:
             only:
               - main
    jobs:
      - doc-gen-job

  # Hourly build on weekdays
  weekday:
    triggers:
      - schedule:
          cron: "0 0,4,8,12,16,20 * * 1-5"
          filters:
            branches:
              only:
                - main
    jobs:
      - linux-build
      - linux-build-release
      - linux-build-options
      - linux-parquet-s3-build
      - macos-build

executors:
  build:
    docker:
      - image : prestocpp/velox-circleci:kevinwilfong-20220426
    resource_class: 2xlarge
    environment:
        CC:  /opt/rh/gcc-toolset-9/root/bin/gcc
        CXX: /opt/rh/gcc-toolset-9/root/bin/g++
  check:
    docker:
      - image : prestocpp/velox-check:mikesh-20210609
  doc-gen:
    docker:
      - image : prestocpp/velox-circleci:sagarm-20220131

jobs:
  macos-build:
    macos:
      xcode: "11.7.0"
    steps:
      - checkout
      - run:
          name: "Update submodules"
          command: |
            git submodule sync --recursive
            git submodule update --init --recursive
      - restore_cache:
          name: "Restore Dependency Cache"
          key: velox-circleci-macos-deps-{{ checksum ".circleci/config.yml" }}-{{ checksum "scripts/setup-macos.sh" }}
      - run:
          name: "Install dependencies"
          command: |
            set -xu
            if [[ ! -e ~/deps ]]; then
              mkdir ~/deps ~/deps-src
              curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C ~/deps
              PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-macos.sh
              rm -rf ~/deps/.git ~/deps/Library/Taps/  # Reduce cache size by 70%.
            fi
      - save_cache:
          name: "Save Dependency Cache"
          key: velox-circleci-macos-deps-{{ checksum ".circleci/config.yml" }}-{{ checksum "scripts/setup-macos.sh" }}
          paths:
            - ~/deps
      - run:
          name: "Calculate merge-base date for CCache"
          command: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/main HEAD) | tee merge-base-date
      - restore_cache:
          name: "Restore CCache cache"
          keys:
            - velox-ccache-debug-{{ arch }}-{{ checksum "merge-base-date" }}
      - run:
          name: "Build on MacOS"
          command: |
            export PATH=~/deps/bin:${PATH}
            mkdir -p .ccache
            export CCACHE_DIR=$(pwd)/.ccache
            ccache -sz -M 5Gi
            cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=~/deps -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
            ninja -C _build/debug
            ccache -s
          no_output_timeout: 1h
      - save_cache:
          name: "Save CCache cache"
          key: velox-ccache-debug-{{ arch }}-{{ checksum "merge-base-date" }}
          paths:
            - .ccache/

  linux-build:
    executor: build
    steps:
      - checkout
      - run:
          name: "Update submodules"
          command: |
            git submodule sync --recursive
            git submodule update --init --recursive
      - run:
          name: "Calculate merge-base date for CCache"
          command: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/main HEAD) | tee merge-base-date
      - restore_cache:
          name: "Restore CCache cache"
          keys:
            - velox-ccache-debug-{{ arch }}-{{ checksum "merge-base-date" }}
      - run:
          name: Build
          command: |
            mkdir -p .ccache
            export CCACHE_DIR=$(realpath .ccache)
            ccache -sz -M 5Gi
            source /opt/rh/gcc-toolset-9/enable
            make debug NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8
            ccache -s
          no_output_timeout: 1h
      - store_artifacts:
          path: '_build/debug/.ninja_log'
      - save_cache:
          name: "Save CCache cache"
          key: velox-ccache-debug-{{ arch }}-{{ checksum "merge-base-date" }}
          paths:
            - .ccache/
      - run:
          name: "Run Unit Tests"
          command: |
            cd _build/debug && ctest -j 16 -VV --output-on-failure
          no_output_timeout: 1h
      - run:
          name: "Run Fuzzer Tests"
          command: |
            _build/debug/velox/expression/tests/velox_expression_fuzzer_test --steps 100000 --logtostderr=1 --minloglevel=0
          no_output_timeout: 5m
      - run:
         name: "Run the velox examples"
         command: |
           _build/debug/velox/examples/velox_example_expression_eval
           _build/debug/velox/examples/velox_example_opaque_type

  linux-build-release:
    executor: build
    steps:
      - checkout
      - run:
          name: "Update submodules"
          command: |
            git submodule sync --recursive
            git submodule update --init --recursive
      - run:
          name: "Calculate merge-base date for CCache"
          command: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/main HEAD) | tee merge-base-date
      - restore_cache:
          name: "Restore CCache cache"
          keys:
            - velox-ccache-release-{{ arch }}-{{ checksum "merge-base-date" }}
      - run:
          name: Build
          command: |
            mkdir -p .ccache
            export CCACHE_DIR=$(realpath .ccache)
            ccache -sz -M 5Gi
            source /opt/rh/gcc-toolset-9/enable
            make release NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8
            ccache -s
          no_output_timeout: 1h
      - store_artifacts:
          path: '_build/release/.ninja_log'
      - save_cache:
          name: "Save CCache cache"
          key: velox-ccache-release-{{ arch }}-{{ checksum "merge-base-date" }}
          paths:
            - .ccache/
      - run:
          name: "Run Unit Tests"
          command: |
            cd _build/release && ctest -j 16 -VV --output-on-failure
          no_output_timeout: 1h

  linux-benchmarks-basic:
    executor: build
    steps:
      - checkout
      - run:
          name: "Update submodules"
          command: |
            git submodule sync --recursive
            git submodule update --init --recursive
      - run:
          name: "Calculate merge-base date for CCache"
          command: |
            git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/main HEAD) | tee merge-base-date
      - restore_cache:
          name: "Restore CCache cache"
          keys:
            - velox-ccache-release-{{ arch }}-{{ checksum "merge-base-date" }}
      - run:
          name: "Build Benchmarks - Baseline"
          command: |
            git checkout $(git merge-base origin/main HEAD)
            mkdir -p benchmark-basic-dumps/baseline benchmark-basic-dumps/target
            mkdir -p .ccache
            export CCACHE_DIR=$(realpath .ccache)
            ccache -sz -M 5Gi
            source /opt/rh/gcc-toolset-9/enable
            make benchmarks-basic-build NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8
            ccache -s
          no_output_timeout: 1h
      - run:
          name: "Run Benchmarks - Baseline"
          command: |
            make benchmarks-basic-dump NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8 BENCHMARKS_DUMP_DIR=benchmark-basic-dumps/baseline
      - run:
          name: "Build Benchmarks - Target"
          command: |
            git checkout -
            mkdir -p .ccache
            export CCACHE_DIR=$(realpath .ccache)
            ccache -sz -M 5Gi
            source /opt/rh/gcc-toolset-9/enable
            make benchmarks-basic-build NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8
            ccache -s
      - run:
          name: "Run Benchmarks - Target"
          command: |
            make benchmarks-basic-dump NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8 BENCHMARKS_DUMP_DIR=benchmark-basic-dumps/target
      - run:
          name: "Benchmarks Summary"
          command: |
            # We don't want to fail the CI job yet, until we ensure the numbers are stable.
            scripts/benchmark-runner.py compare \
                --baseline-dump-path benchmark-basic-dumps/baseline \
                --target-dump-path benchmark-basic-dumps/target || true
      - store_artifacts:
          path: '_build/debug/.ninja_log'
      - save_cache:
          name: "Save CCache cache"
          key: velox-ccache-release-{{ arch }}-{{ checksum "merge-base-date" }}
          paths:
            - .ccache/

  # Build with different options
  linux-build-options:
    executor: build
    steps:
      - checkout
      - run:
          name: "Update submodules"
          command: |
            git submodule sync --recursive
            git submodule update --init --recursive
      - run:
          name: "Calculate merge-base date for CCache"
          command: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/main HEAD) | tee merge-base-date
      - restore_cache:
          name: "Restore CCache cache"
          keys:
            - velox-ccache-debug-{{ arch }}-{{ checksum "merge-base-date" }}
      - run:
          name: Build minimal velox
          command: |
            mkdir -p .ccache
            export CCACHE_DIR=$(realpath .ccache)
            ccache -sz -M 5Gi
            source /opt/rh/gcc-toolset-9/enable
            make min_debug NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=16
            ccache -s
          no_output_timeout: 1h
      - run:
          name: Build velox without testing
          command: |
            mkdir -p .ccache
            export CCACHE_DIR=$(realpath .ccache)
            ccache -sz -M 5Gi
            source /opt/rh/gcc-toolset-9/enable
            make clean
            make debug EXTRA_CMAKE_FLAGS="-DVELOX_BUILD_TESTING=OFF" NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=16
            ccache -s
          no_output_timeout: 1h
      - store_artifacts:
          path: '_build/debug/.ninja_log'
      - save_cache:
          name: "Save CCache cache"
          key: velox-ccache-debug-{{ arch }}-{{ checksum "merge-base-date" }}
          paths:
            - .ccache/

  linux-parquet-s3-build:
    executor: build
    steps:
      - checkout
      - run:
          name: "Update submodules"
          command: |
            git submodule sync --recursive
            git submodule update --init --recursive
      - run:
          name: "Calculate merge-base date for CCache"
          command: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/main HEAD) | tee merge-base-date
      - restore_cache:
          name: "Restore CCache cache"
          keys:
            - velox-ccache-release-{{ arch }}-{{ checksum "merge-base-date" }}
      - run:
          name: "Install adapter dependencies"
          command: |
            mkdir ~/adapter-deps ~/adapter-deps/install
            source /opt/rh/gcc-toolset-9/enable
            set -xu
            DEPENDENCY_DIR=~/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
      - run:
          name: "Install Minio server"
          command: |
            set -xu
            cd ~/adapter-deps/install/bin/
            wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio-20220526054841.0.0.x86_64.rpm
            rpm -i minio-20220526054841.0.0.x86_64.rpm
      - run:
          name: Build
          command: |
            mkdir -p .ccache
            export CCACHE_DIR=$(realpath .ccache)
            ccache -sz -M 5Gi
            source /opt/rh/gcc-toolset-9/enable
            make release EXTRA_CMAKE_FLAGS=" -DVELOX_ENABLE_PARQUET=ON -DVELOX_ENABLE_S3=ON " AWSSDK_ROOT_DIR=~/adapter-deps/install NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8
            ccache -s
          no_output_timeout: 1h
      - store_artifacts:
          path: '_build/release/.ninja_log'
      - save_cache:
          name: "Save CCache cache"
          key: velox-ccache-release-{{ arch }}-{{ checksum "merge-base-date" }}
          paths:
            - .ccache/
      - run:
          name: "Run Unit Tests"
          command: |
            export PATH=~/adapter-deps/install/bin:${PATH}
            cd _build/release && ctest -j 16 -VV --output-on-failure
          no_output_timeout: 1h

  format-check:
    executor: check
    steps:
      - checkout
      - run:
          name: Check formatting
          command: |
            if ! make format-check; then
              make format-fix
              echo -e "\n==== Apply using:"
              echo "patch -p1 \<<EOF"
              git --no-pager diff
              echo "EOF"
              false
            fi

  header-check:
    executor: check
    steps:
      - checkout
      - run:
          name: Check license headers
          command: |
            if ! make header-check; then
              make header-fix
              echo -e "\n==== Apply using:"
              echo "patch -p1 \<<EOF"
              git --no-pager diff
              echo "EOF"
              false
            fi

  doc-gen-job:
    executor: doc-gen
    steps:
      - checkout
      - add_ssh_keys:
          fingerprints:
            - "ed:40:13:bc:d0:3d:28:26:98:cb:90:31:cc:d3:f1:6b"
      - run:
          name: "Build docs and update gh-pages"
          command: |
            git config --global user.email "velox@users.noreply.github.com"
            git config --global user.name "velox"
            git checkout main
            make -C velox/docs html
            git checkout gh-pages
            cp -R velox/docs/_build/html/* docs
            git add docs
            if [ -n "$(git status --porcelain)" ]
            then
              git commit -m "Update documentation"
              git push
            fi