.gitlab-ci.yml

#
# Copyright (C) 2005-2021 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
#     https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
# We have to use directly git-lfs wrapping commands to have an
# efficient cloning step.
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)

variables:
  BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
  GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
  DOCKER_DRIVER: overlay2
  GIT_DEPTH: "3"
  # Disable automatic checkout to let us fetch LFS before
  GIT_CHECKOUT: "false"
  # The fetch strategy fails with LFS and GitLab
  GIT_STRATEGY: "clone"

workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
      when: never
    - if: '$CI_COMMIT_BRANCH'

stages:
  - precheck
  - prepare
  - build
  - report
  - deploy
  - docker

.general:
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

.common:
  extends: .general
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  after_script:
    - python3 -u CI/cdash_handler.py

.common-build:
  extends: .common
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
  stage: build
  artifacts:
    when: always
    expire_in: 24 hrs
    paths:
      - build/*/*.log #CMake log
      - log/*.txt # Others
      - CookBook-*-html.tar.gz
      - CookBook-*.pdf
      - OTB-Doxygen-*.tar.bz2
      - OTB-*.run
      - OTB-*.zip
      - build/compile_commands.json
      - build/ctest_report.xml
      - build/cppcheck_report.xml
      - build/coverage_report.xml
      - build/Testing/Temporary/*.txt #DEBUG

.common-prepare:
  extends: .general
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
  stage: prepare
  before_script:
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - export GIT_LFS_SKIP_SMUDGE=0
  artifacts:
    expire_in: 24 hrs
    when: always
    paths:
      - sb_branch.txt # Needed to checkout correct branch in build step
      - build/*/*/*/*.log # Superbuild log
      - build/*/*/*/*.cmake

#-------------------------- precheck job ---------------------------------------
fast-build:
  extends: .common
  only:
    - merge_requests
    - branches
  stage: precheck
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
  before_script:
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - python3 CI/check_twin_pipelines.py
  script:
    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
    - ccache -s

legal-check:
  extends: .common
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
  stage: precheck
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  variables:
    GIT_DEPTH: ""
  allow_failure: true
  script:
    - ./CI/contributors_check.sh
    - ./CI/headers_check.py
  after_script: []

#------------------------- prepare & build jobs --------------------------------

## Ubuntu superbuild
ubuntu-xdk-prepare:
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm

ubuntu-xdk-build-doc:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
    - export QT_QPA_PLATFORM=offscreen
    - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
    - mv build/CookBook-*-html.tar.gz . || true
    - mv build/Documentation/Cookbook/latex/CookBook-*.pdf . || true
    - mv build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2 . || true
    - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk -DNAME_SUFFIX:string=-glibc-2.27
    - mv build_packages/OTB-*.run . || true
  needs:
    - job: ubuntu-xdk-prepare
      artifacts: true

## CentOS superbuild
centos-xdk-prepare:
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:7.9.2009
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=centos-7.9.2009-gcc

centos-xdk-build:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:7.9.2009
  script:
    - export QT_QPA_PLATFORM=offscreen
    - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=centos-7.9.2009-gcc
    - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=centos-7.9.2009-gcc
    - mv build_packages/OTB-*.run . || true
  needs:
    - job: centos-xdk-prepare
      artifacts: true


## MacOS superbuild
macos-xdk-prepare:
  extends: .common-prepare
  tags:
    - macos
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=macos-10.14.6-clang

macos-xdk-build:
  tags:
    - macos
  extends: .common-build
  script:
    - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=macos-10.14.6-clang
    - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=macos-10.14.6-clang
    - mv build_packages/OTB-*.run . || true
  needs:
    - job: macos-xdk-prepare
      artifacts: true

## Windows
.windows-prepare:
  extends: .common-prepare
  before_script:
# This override the previous before_script
    - set GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q %CI_COMMIT_SHA%
    - set GIT_LFS_SKIP_SMUDGE=0

.windows-build:
  extends: .common-build
  before_script:
    - git lfs fetch origin %CI_COMMIT_SHA%
    - git checkout -f -q %CI_COMMIT_SHA%

# - Win10
windows-10-prepare:
  extends: .windows-prepare
  tags:
    - windows10
  script:
    - call ./CI/dev_env.bat x64 xdk 10
    - buildcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - buildcache.exe -s

windows-10-build:
  extends: .windows-build
  tags:
    - windows10
  script:
    - call ./CI/dev_env.bat x64 otb 10
    - buildcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - buildcache.exe -s
    - ctest -V -S CI/main_packages.cmake
    - move "build_packages\OTB-*.zip" . || dir build_packages
  needs:
    - job: windows-10-prepare
      artifacts: true

#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
  extends: .common-build
  only:
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - QT_QPA_PLATFORM=offscreen ctest -V -S CI/main_qa.cmake
                     -DIMAGE_NAME:string=ubuntu-18.04-llvm-qa
                     -DQA:BOOL=ON
    - ./CI/otb_coverage.sh
    - saxon-xslt -o build/ctest_report.xml
                 build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
                 CI/ctest2junit.xsl
  needs:
    - job: ubuntu-xdk-prepare
      artifacts: true

ubuntu-xdk-qa-static-analysis:
  extends: .common-build
  only:
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
          grep -vE '^Modules/ThirdParty/' |
          sort -u > header_directories
    - cppcheck -j 8 -q --xml --xml-version=2 --enable=all
               --language=c++ --std=c++14 --platform=unix64
               --includes-file=header_directories
               -i Modules/ThirdParty Modules
               2> build/cppcheck_report.xml
  after_script: []
  needs:
    - job: ubuntu-xdk-prepare
      artifacts: true

ubuntu-xdk-report:
  extends: .common
  variables:
    GIT_DEPTH: ""
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  stage: report
  only:
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
  script:
    - sonar-scanner -Dproject.settings=sonar-project.properties
                    -Dsonar.host.url=https://sonar.orfeo-toolbox.org
                    -Dsonar.login=$SONAR_OTB_TOKEN
                    -Dsonar.projectKey=$CI_PROJECT_NAMESPACE-$CI_PROJECT_NAME
                    `test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
                    -Dsonar.branch.name=$CI_COMMIT_REF_NAME

  after_script: []
  needs:
    - job: ubuntu-xdk-qa-code-coverage
      artifacts: true
    - job: ubuntu-xdk-qa-static-analysis
      artifacts: true

# #------------------------- Conda packages jobs ---------------------------------
# conda-linux-build:
#   extends: .common
#   stage: build
#   only:
#     - nightly
#     - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
#   image: $BUILD_IMAGE_REGISTRY/otb-conda-build:latest
#   allow_failure: true
#   script:
#     - export otb_tag=${CI_COMMIT_TAG:-$CI_COMMIT_SHORT_SHA}
#     - ./CI/conda_build.sh
#   after_script: []
#   needs:
#     - job: fast-build
#       artifacts: false
#   artifacts:
#     when: always
#     expire_in: 24 hrs
#     paths:
#       - conda-bld/*

# conda-linux-deploy:
#   extends: .general
#   stage: deploy
#   only:
#     - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
#   image: $BUILD_IMAGE_REGISTRY/otb-conda-build:latest
#   before_script:
#     # Provision efficiently the local LFS cache before checkout
#     - git lfs fetch origin $CI_COMMIT_SHA
#     - git checkout -f -q $CI_COMMIT_SHA
#   script:
#     - ./CI/conda_deploy.sh
#   needs:
#     - job: conda-linux-build
#       artifacts: yes

#---------------------------- Deploy job ---------------------------------------
deploy:
  tags:
    - deploy
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    - develop@orfeotoolbox/otb
    - /^release-[0-9]+\.[0-9]+$/
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  script:
    - ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
  needs:
    - job: ubuntu-xdk-build-doc
      artifacts: true
    - job: macos-xdk-build
      artifacts: true
    - job: windows-10-build
      artifacts: true
    - job: centos-xdk-build
      artifacts: true

update-archive:
  tags:
    - deploy
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  stage: deploy
  extends: .general
  only:
    - develop
    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  script:
    - ./CI/SuperbuildDownloadList.sh download build_archive
    - ./CI/deploy-archive.sh build_archive
  needs:
    # don't push the archive if Superbuild was not successful
    - job: ubuntu-xdk-prepare
      artifacts: false


release-container:
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    refs:
      - tags@orfeotoolbox/otb
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
  script:
    - curl --request POST
           --form token=$K8S_SECRET_RELEASE
           --form ref=master
           --form variables[OTB_TAG]=$CI_COMMIT_TAG
           https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline

release-docker:
  image:
      name: gcr.io/kaniko-project/executor:debug
      entrypoint: [""]
  stage: docker
  only:
    refs:
      - /^release-[0-9]+\.[0-9]+$/
  script:
      - /kaniko/executor --cleanup
                       --verbosity warn
                       --context $CI_PROJECT_DIR/Docker
                       --dockerfile $CI_PROJECT_DIR/Docker/Dockerfile
                       --destination $CI_REGISTRY_IMAGE/orfeotoolbox/otb:$CI_COMMIT_BRANCH