azure-pipelines.yml

schedules:
- cron: "0 1 * * *"
  displayName: Daily 1:00 AM UTC build
  branches:
    include:
    - 8.0

trigger:
  branches:
    include:
    - '*'
    exclude:
    - 8.0
  paths:
    exclude:
    - doc
    - build-ps
    - man
    - mysql-test
    - packaging
    - policy
    - scripts
    - support-files

pr:
  branches:
    include:
    - '*'
  paths:
    exclude:
    - doc
    - build-ps
    - man
    - mysql-test
    - packaging
    - policy
    - scripts
    - support-files

jobs:
- job: BiDiScan
  pool:
    vmImage: 'ubuntu-20.04'

  steps:
  - checkout: self
    fetchDepth: 32

  - script: |
      git fetch origin 8.0
      CHANGED_FILES=$(git diff --name-only --relative --diff-filter AMR origin/8.0 -- . | tr '\n' ' ')

      if [ -z "${CHANGED_FILES}" ]; then
          echo --- No changed files
      else
          python $(Build.SourcesDirectory)/scripts/find_unicode_control.py -p bidi -v ${CHANGED_FILES}
      fi

- job:
  timeoutInMinutes: 240
  pool:
    vmImage: $(imageName)

  variables:
    UBUNTU_CODE_NAME: focal
    BOOST_VERSION: boost_1_77_0
    BOOST_DIR: $(Pipeline.Workspace)/boost
    USE_CCACHE: 1
    CCACHE_DIR: $(Pipeline.Workspace)/ccache
    CCACHE_COMPRESS: 1
    CCACHE_COMPRESSLEVEL: 9
    CCACHE_CPP2: 1
    CCACHE_MAXSIZE: 2G
    OS_NAME: $(Agent.OS)
    PARENT_BRANCH: 8.0
    INVERTED: OFF

  strategy:
    matrix:
      macOS 12 RelWithDebInfo:
        imageName: 'macOS-12'
        Compiler: clang
        BuildType: RelWithDebInfo

      # skip for a pull request if branch name doesn't contain "fullci"
      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        macOS 12 Debug:
          imageName: 'macOS-12'
          Compiler: clang
          BuildType: Debug

      # clang-6 and newer compilers
      clang-15 RelWithDebInfo [Ubuntu 22.04 Jammy]:
        imageName: 'ubuntu-22.04'
        UBUNTU_CODE_NAME: jammy
        Compiler: clang
        CompilerVer: 15
        BuildType: RelWithDebInfo

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-15 RelWithDebInfo INVERTED=ON [Ubuntu 22.04 Jammy]:
          imageName: 'ubuntu-22.04'
          UBUNTU_CODE_NAME: jammy
          Compiler: clang
          CompilerVer: 15
          BuildType: RelWithDebInfo
          INVERTED: ON

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-15 Debug [Ubuntu 22.04 Jammy]:
          imageName: 'ubuntu-22.04'
          UBUNTU_CODE_NAME: jammy
          Compiler: clang
          CompilerVer: 15
          BuildType: Debug

      clang-15 Debug INVERTED=ON [Ubuntu 22.04 Jammy]:
        imageName: 'ubuntu-22.04'
        UBUNTU_CODE_NAME: jammy
        Compiler: clang
        CompilerVer: 15
        BuildType: Debug
        INVERTED: ON

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-14 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 14
          BuildType: RelWithDebInfo

      clang-14 Debug [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: clang
        CompilerVer: 14
        BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-13 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 13
          BuildType: RelWithDebInfo

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-13 Debug [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 13
          BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-12 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 12
          BuildType: RelWithDebInfo

      clang-12 Debug [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: clang
        CompilerVer: 12
        BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-11 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 11
          BuildType: RelWithDebInfo

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-11 Debug [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 11
          BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-10 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 10
          BuildType: RelWithDebInfo

      clang-10 Debug [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: clang
        CompilerVer: 10
        BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-9 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 9
          BuildType: RelWithDebInfo

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-9 Debug [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 9
          BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-8 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 8
          BuildType: RelWithDebInfo

      clang-8 Debug [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: clang
        CompilerVer: 8
        BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-7 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 7
          BuildType: RelWithDebInfo

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        clang-7 Debug [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: clang
          CompilerVer: 7
          BuildType: Debug

      clang-6 RelWithDebInfo [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: clang
        CompilerVer: 6.0
        BuildType: RelWithDebInfo

      clang-6 Debug [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: clang
        CompilerVer: 6.0
        BuildType: Debug


      # gcc-7 and newer compilers
      gcc-12 RelWithDebInfo [Ubuntu 22.04 Jammy]:
        imageName: 'ubuntu-22.04'
        Compiler: gcc
        CompilerVer: 12
        BuildType: RelWithDebInfo

      gcc-12 Debug [Ubuntu 22.04 Jammy]:
        imageName: 'ubuntu-22.04'
        Compiler: gcc
        CompilerVer: 12
        BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        gcc-11 RelWithDebInfo Ubuntu 20.04 Focal:
          imageName: 'ubuntu-20.04'
          Compiler: gcc
          CompilerVer: 11
          BuildType: RelWithDebInfo

      gcc-11 Debug [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: gcc
        CompilerVer: 11
        BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        gcc-10 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: gcc
          CompilerVer: 10
          BuildType: RelWithDebInfo

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        gcc-10 Debug [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: gcc
          CompilerVer: 10
          BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        gcc-9 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: gcc
          CompilerVer: 9
          BuildType: RelWithDebInfo

      gcc-9 Debug [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: gcc
        CompilerVer: 9
        BuildType: Debug

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        gcc-8 RelWithDebInfo [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: gcc
          CompilerVer: 8
          BuildType: RelWithDebInfo

      ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}:
        gcc-8 Debug [Ubuntu 20.04 Focal]:
          imageName: 'ubuntu-20.04'
          Compiler: gcc
          CompilerVer: 8
          BuildType: Debug

      gcc-7 RelWithDebInfo [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: gcc
        CompilerVer: 7
        BuildType: RelWithDebInfo

      gcc-7 Debug [Ubuntu 20.04 Focal]:
        imageName: 'ubuntu-20.04'
        Compiler: gcc
        CompilerVer: 7
        BuildType: Debug

  steps:
  - script: |
      uname -r
      df -Th
      ls -l ..
      pwd
      ls -l

      if [[ "$OS_NAME" == "Linux" ]]; then
        SELECTED_CC=$(Compiler)-$(CompilerVer)
        if [[ "$(Compiler)" == "clang" ]]; then
          SELECTED_CXX=clang++-$(CompilerVer)
          PACKAGES="$SELECTED_CC $PACKAGES"
        else
          SELECTED_CXX=g++-$(CompilerVer)
          PACKAGES="$SELECTED_CXX $PACKAGES"
        fi
      else
        REAL_COMPILER_VER=`$(Compiler) --version | grep -o "clang version.*" | awk '{print $3}'`
        SELECTED_CC=$(Compiler)
        SELECTED_CXX=clang++
      fi

      echo SELECTED_CC=$SELECTED_CC SELECTED_CXX=$SELECTED_CXX BuildType=$(BuildType) INVERTED=$(INVERTED) imageName=$(imageName) OS_NAME=$OS_NAME
      echo --- Configure required LLVM and Ubuntu Toolchain repositories
      if [[ "$OS_NAME" == "Linux" ]] && [[ "$SELECTED_CC" == "clang"* ]]; then
        PACKAGES="llvm-$(CompilerVer)-dev $PACKAGES"
        curl -sSL "http://apt.llvm.org/llvm-snapshot.gpg.key" | sudo -E apt-key add -
        echo "deb http://apt.llvm.org/$(UBUNTU_CODE_NAME)/ llvm-toolchain-$(UBUNTU_CODE_NAME)-$(CompilerVer) main" | sudo tee -a /etc/apt/sources.list > /dev/null
      fi

      echo --- Update list of packages and download dependencies
      if [[ "$OS_NAME" == "Linux" ]]; then
        sudo -E apt-add-repository -y "ppa:ubuntu-toolchain-r/test"
        sudo -E apt-get -yq update >> ~/apt-get-update.log 2>&1

        sudo -E apt-get -yq --no-install-suggests --no-install-recommends --allow-unauthenticated install $PACKAGES cmake cmake-curses-gui ccache bison libudev-dev libaio-dev libmecab-dev libnuma-dev liblzma-dev libssl-dev libreadline-dev libpam-dev libcurl4-openssl-dev libldap2-dev libkrb5-dev libsasl2-dev libsasl2-modules-gssapi-mit || exit 1;
        if [[ "$(INVERTED)" != "ON" ]]; then
          sudo -E apt-get -yq --no-install-suggests --no-install-recommends --allow-unauthenticated install rapidjson-dev zlib1g-dev libzstd-dev liblz4-dev libfido2-dev libevent-dev libreadline-dev protobuf-compiler libprotobuf-dev libprotoc-dev libicu-dev || exit 1;
        fi

        if [[ "$(Compiler)" == "clang" ]]; then
          REAL_COMPILER_VER=`$SELECTED_CC --version | grep -o "clang version.*" | awk '{print $3}'`
        else
          REAL_COMPILER_VER=`$SELECTED_CC --version | head -1 | awk '{print $4}'`
        fi
      else
         brew update
         brew install ccache protobuf lz4 rapidjson openssl@1.1
      fi

      UPDATE_TIME=$SECONDS
      echo --- Packages updated in $UPDATE_TIME seconds.

      echo "##vso[task.setvariable variable=REAL_COMPILER_VER]${REAL_COMPILER_VER%%-*}"
      echo "##vso[task.setvariable variable=SELECTED_CC]$SELECTED_CC"
      echo "##vso[task.setvariable variable=SELECTED_CXX]$SELECTED_CXX"
      echo "##vso[task.setvariable variable=UPDATE_TIME]$UPDATE_TIME"

    displayName: '*** Install Build Dependencies'

  - task: Cache@2
    continueOnError: true
    inputs:
      key: '"ccache"| "$(PARENT_BRANCH)" | "$(imageName)-$(Compiler)-$(REAL_COMPILER_VER)-$(BuildType)" | "INVERTED=$(INVERTED)" | "$(Build.SourceVersion)"'
      restoreKeys: '"ccache" | "$(PARENT_BRANCH)" | "$(imageName)-$(Compiler)-$(REAL_COMPILER_VER)-$(BuildType)" | "INVERTED=$(INVERTED)"'
      path: $(CCACHE_DIR)
    displayName: '*** Download/upload ccached files'

  - task: Cache@2
    continueOnError: true
    inputs:
      key: $(BOOST_VERSION)
      restoreKeys: $(BOOST_VERSION)
      path: $(BOOST_DIR)
    displayName: '*** Download/upload $(BOOST_VERSION) libraries'

  - checkout: self
    fetchDepth: 32

  - script: |
      df -Th
      git submodule sync
      git submodule update --init --force --depth=256
      git submodule
    displayName: '*** Update git submodules'

  - script: |
      echo "SELECTED_CC=$SELECTED_CC (`which $SELECTED_CC`) SELECTED_CXX=$SELECTED_CXX (`which $SELECTED_CXX`) REAL_COMPILER_VER=$REAL_COMPILER_VER BuildType=$(BuildType)"
      $SELECTED_CC -v
      $SELECTED_CXX -v
      ccache --version
      ccache -p
      ccache --zero-stats
      df -Th
      if [[ "$OS_NAME" == "Linux" ]]; then
        cat /proc/cpuinfo
      fi
    displayName: '*** System and compiler info'

  - script: |
      echo SELECTED_CC=$SELECTED_CC SELECTED_CXX=$SELECTED_CXX REAL_COMPILER_VER=$REAL_COMPILER_VER BuildType=$(BuildType) INVERTED=$(INVERTED) imageName=$(imageName) OS_NAME=$OS_NAME
      echo --- Set cmake parameters
      COMPILE_OPT+=(
        -DCMAKE_C_FLAGS_DEBUG=-g1
        -DCMAKE_CXX_FLAGS_DEBUG=-g1
      )

      CMAKE_OPT="
        -DCMAKE_BUILD_TYPE=$(BuildType)
        -DBUILD_CONFIG=mysql_release
        -DDOWNLOAD_BOOST=1
        -DWITH_BOOST=$(BOOST_DIR)
        -DCMAKE_C_COMPILER=$SELECTED_CC
        -DCMAKE_CXX_COMPILER=$SELECTED_CXX
        -DCMAKE_C_COMPILER_LAUNCHER=ccache
        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
        -DWITH_ROCKSDB=ON
        -DWITH_COREDUMPER=ON
        -DWITH_KEYRING_VAULT=ON
        -DWITH_KEYRING_VAULT_TEST=ON
        -DWITH_PAM=ON
      "

      if [[ ( "$(Compiler)" == "gcc" ) && ( $(bc <<< "$(CompilerVer) < 8.1") -eq 1 ) ]]; then
        CMAKE_OPT+="
          -DWITH_ROUTER=OFF
        "
      fi

      if [[ "$OS_NAME" == "Darwin" ]]; then
        CMAKE_OPT+="
          -DMYSQL_MAINTAINER_MODE=OFF
          -DWITH_TOKUDB=OFF
          -DWITH_PROTOBUF=system
          -DWITH_SYSTEM_LIBS=ON
          -DWITH_ICU=/usr/local/opt/icu4c
          -DWITH_SSL=/usr/local/opt/openssl@1.1
          -DWITH_FIDO=bundled
          -DWITH_ZLIB=bundled
          -DWITH_PERCONA_AUTHENTICATION_LDAP=OFF
        "
      else
        CMAKE_OPT+="
          -DMYSQL_MAINTAINER_MODE=ON
          -DWITH_TOKUDB=ON
        "
        if [[ "$(INVERTED)" != "ON" ]]; then
          CMAKE_OPT+="
            -DWITH_MECAB=system
            -DWITH_NUMA=ON
            -DWITH_READLINE=system
            -DWITH_SYSTEM_LIBS=ON
          "
          `# Required liblz4 1.9.3 and installed version is 1.9.2`
          if [[ "$(imageName)" == "ubuntu-20.04" ]]; then
            CMAKE_OPT+="-DWITH_LZ4=bundled"
          fi
        else
          CMAKE_OPT+="
            -DWITH_NDB=ON
            -DWITH_NDBCLUSTER=ON
            -DWITH_NDB_JAVA=OFF

            -DWITH_ROUTER=OFF
            -DWITH_UNIT_TESTS=OFF
            -DWITH_NUMA=OFF

            -DWITH_EDITLINE=bundled
            -DWITH_FIDO=bundled
            -DWITH_ICU=bundled
            -DWITH_LIBEVENT=bundled
            -DWITH_LZ4=bundled
            -DWITH_PROTOBUF=bundled
            -DWITH_RAPIDJSON=bundled
            -DWITH_ZLIB=bundled
            -DWITH_ZSTD=bundled

            -DWITH_ARCHIVE_STORAGE_ENGINE=OFF
            -DWITH_BLACKHOLE_STORAGE_ENGINE=OFF
            -DWITH_EXAMPLE_STORAGE_ENGINE=ON
            -DWITH_FEDERATED_STORAGE_ENGINE=OFF
            -DWITHOUT_PERFSCHEMA_STORAGE_ENGINE=ON
            -DWITH_INNODB_MEMCACHED=ON
          "
        fi
      fi

      echo --- CMAKE_OPT=\"$CMAKE_OPT\"
      echo --- COMPILE_OPT=\"${COMPILE_OPT[@]}\"
      mkdir bin; cd bin
      cmake .. $CMAKE_OPT "${COMPILE_OPT[@]}" || exit 1
      rm -f $(BOOST_DIR)/$(BOOST_VERSION).tar.gz

      CMAKE_TIME=$SECONDS
      echo --- CMake took $CMAKE_TIME seconds. Packages updated in $UPDATE_TIME seconds.
      cmake -L .

      echo '##vso[task.setvariable variable=CMAKE_TIME]'$CMAKE_TIME

    displayName: '*** cmake -DCMAKE_BUILD_TYPE=$(BuildType)'

  - script: |
      df -Th
      cd bin
      make -j2 || exit 1
      ccache --show-stats

      BUILD_TIME=$SECONDS
      echo --- Total time $(($BUILD_TIME + $UPDATE_TIME + $CMAKE_TIME)) seconds. Build time $BUILD_TIME seconds. CMake took $CMAKE_TIME seconds. Packages updated in $UPDATE_TIME seconds.
      df -Th
      rm -rf *
      df -Th

    displayName: '*** Compile'