.gitlab-ci.yml

# we utilize the images generated by the build-images project, to
# speed up CI runs. We also use ccache and store config.cache
# to speed up compilation. We include a version number in cache
# name to allow expiration of old caches.

stages:
  - tarball
  - stage1-testing

cache:
  key: "$CI_JOB_NAME-wget2"
  paths:
    - cache/

before_script:
  # CCache Config
  - mkdir -p cache
  - export CCACHE_BASEDIR=${PWD}
  - export CCACHE_DIR=${PWD}/cache
  - echo $CCACHE_DIR
  - export CC="ccache gcc"

after_script:
  # somehow after_script looses environment
  - export CCACHE_BASEDIR=${PWD}
  - export CCACHE_DIR=${PWD}/cache
  - ccache -s 2>/dev/null

variables:
  BUILD_IMAGES_PROJECT: gnuwget/build-images
  DEBIAN_BUILD: buildenv-wget2-debian-stable
  DEBIAN_TESTING_BUILD: buildenv-wget2-debian-testing
  DEBIAN_STABLE_BUILD: buildenv-wget2-debian-stable
  DEBIAN_STABLEi386_BUILD: buildenv-debian-stable-i386
  DEBIAN_UNSTABLE_BUILD: buildenv-wget2-debian-unstable
  DEBIAN_HTTP2_BUILD: buildenv-mhd-http2
  FEDORA_BUILD: buildenv-wget2-fedora
  MINGW_BUILD: buildenv-mingw
  MINGW_BUILD_STATIC: buildenv-mingw-static
  ALPINE_BUILD: buildenv-wget2-alpine
  ARCH_BUILD: buildenv-arch
#  BASIC_BUILD: buildenv-wget2-basic
  TARBALL_BUILD: buildenv-wget2-tarball
  GET_SOURCES_ATTEMPTS: "3"
  GIT_DEPTH: "5"
  CONFIGURE_BASE_FLAGS: --enable-assert --cache-file ../cache/config.cache --enable-manywarnings
  CFLAGS_DEFAULT: -O1 -g -ggdb3

sast:
  stage: tarball
  # SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
  variables:
    SAST_EXCLUDED_PATHS: examples, fuzz, tests, unit-tests
include:
  - template: Security/SAST.gitlab-ci.yml

.defaults:
  variables:
    GIT_STRATEGY: none
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./**/config.h
      - ./**/*.log
  dependencies:
    - Build-Tarball
  needs: ["Build-Tarball"]

# We organize the CI runners as thus:
# Stage 'tarball' includes:
#   - commit check
#   - spell check
#   - tarball build including docs (on Debian Testing)
#
# Stage 'stage1-testing' includes:
#   1. Debian Testing Build:
#       * ASan and UBSan builds
#       * make syntax-check
#       * make distcheck
#
#   2. Fedora Build
#       * LLVM/Clang Build
#       * Valgrind Tests
#
#   This way we split most of the different build options across different
#   platforms as well. Other builds we would like to have:
#
#   1. Windows
#   2. OSX
#   3. Documentation Generation and Gitlab pages
#   4. Code coverage stats

CommitCheck:
  stage: tarball
  image: alpine:latest
  before_script:
    - /bin/true
  script:
    - apk add git py3-pip
#    - apk add py3-codespell --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
    - pip install --break-system-packages codespell
    - contrib/commit-check
    - contrib/spell-checker
  after_script:
    - /bin/true
  allow_failure: true
  cache:
      paths:
      policy: push

# Create the tarball for further testing.
# Runs syntax-check, so no need to do this later on.
Build-Tarball:
  stage: tarball
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$TARBALL_BUILD
  extends:
    - .defaults
  variables:
    GIT_STRATEGY: clone
  script:
    - rmdir gnulib && mv /gnulib . && git submodule update gnulib
    - ./bootstrap
    - mkdir vpath && cd vpath
    - export CC=gcc
    - export CFLAGS="-O0"
    - ../configure $CONFIGURE_BASE_FLAGS --with-ssl=none
    - make -j$(nproc) syntax-check
    - make -j$(nproc) -C docs
    - make -j$(nproc) dist-gzip
    - mv wget2-*.gz ..
  artifacts:
    expire_in: 2 weeks
    when: on_success
    paths:
      - wget2-*.gz
  dependencies:
  needs: []

# Debian Testing
# - using dash to detect bashims in configure.ac
DebianTesting:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_TESTING_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - CONFIG_SHELL=/bin/dash dash ./configure $CONFIGURE_BASE_FLAGS
      --enable-manylibs -q 2>&1 >/dev/null
      | grep -v "libhsts was not found" | grep . && exit 1
    - make -j$(nproc) -C lib
    - make check -j$(nproc) CFLAGS="$CFLAGS_DEFAULT -Werror"

# Debian Unstable
Debian/Unstable:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_UNSTABLE_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - ./configure $CONFIGURE_BASE_FLAGS
    - make -j$(nproc) -C lib
    - make -j$(nproc) check CFLAGS="$CFLAGS_DEFAULT -Werror"

Debian/WolfSSL:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_TESTING_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - ./configure $CONFIGURE_BASE_FLAGS --with-ssl=wolfssl
    - make check -j$(nproc)

Debian/OpenSSL:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_TESTING_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - ./configure $CONFIGURE_BASE_FLAGS --with-ssl=openssl
    - make check -j$(nproc)

Fedora:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - ./configure $CONFIGURE_BASE_FLAGS --enable-manywarnings
    - make -j$(nproc) -C lib
    - make -j$(nproc) check CFLAGS="$CFLAGS_DEFAULT -Werror"

Alpine:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$ALPINE_BUILD
  extends:
    - .defaults
  variables:
    GIT_STRATEGY: none
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - ./configure $CONFIGURE_BASE_FLAGS
    - make -j$(nproc) -C lib
    - make -j$(nproc) check CFLAGS="$CFLAGS_DEFAULT -Werror -Wno-cpp"

# In this build we combine
#  * gcc
#  * check, syntax-check
Minimal/Debian:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - ./configure $CONFIGURE_BASE_FLAGS --disable-doc
        --disable-xattr --with-ssl=none --without-libpsl --without-libnghttp2 --without-bzip2
        --without-gpgme --without-zlib --without-lzma --without-brotlidec --without-libidn2
        --without-libidn --without-libpcre2 --without-libpcre --without-plugin-support
    - make -j$(nproc) -C lib
    - make -j$(nproc) check CFLAGS="$CFLAGS_DEFAULT -Werror"

# In this build we combine
#  * clang
#  * ASan, UBSan
#  * check, syntax-check
Sanitizers/Debian:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_TESTING_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - export CC="ccache clang"
    - export UBSAN_OPTIONS=print_stacktrace=1:report_error_type=1
    - export LSAN_OPTIONS=suppressions="$PWD/tests/clang-asan-suppressions"
    - export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer
    - ./configure $CONFIGURE_BASE_FLAGS --enable-fsanitize-asan --enable-fsanitize-ubsan
    - make -j$(nproc) -C lib
    - make -j$(nproc) check CFLAGS="$CFLAGS_DEFAULT -Werror"

Scan-Build/Debian:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_TESTING_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - export CC=clang
    - ./configure $CONFIGURE_BASE_FLAGS
    - make -j$(nproc) -C lib
    - scan-build -v --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability --status-bugs -o scan-build make -j$(nproc)
    - scan-build -v --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability --status-bugs -o scan-build make -j$(nproc) check
  tags:
    - saas-linux-xlarge-amd64
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - scan-build/*

.valgrind:
  stage: stage1-testing
  extends:
    - .defaults
  variables:
    GIT_STRATEGY: clone
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && rm wget2-*.gz
    - cd wget2-*
    - mv ../fuzz/*.in fuzz/
    - ./configure $CONFIGURE_BASE_FLAGS --enable-valgrind-tests --without-gpgme
    - make -j$(nproc)
    - make -j$(nproc) check
  tags:
    - saas-linux-xlarge-amd64

Valgrind/Debian:
  extends:
    - .valgrind
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_STABLE_BUILD

Valgrind/i386:
  extends:
    - .valgrind
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_STABLEi386_BUILD

pages:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_TESTING_BUILD
  script:
    - mkdir -p public
    - apt-get -y update
    - apt-get -y install libcapture-tiny-perl libdatetime-perl pandoc doxygen texinfo
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && mv wget2-*.gz wget2-latest.tar.gz
    - cd wget2-*.*.?
    - ./configure $CONFIGURE_BASE_FLAGS
    - make -j$(nproc) check-coverage

# Tarball
    - mv ../wget2-latest.tar.gz ../public/wget2-latest.tar.gz

# Test suite coverage report
    - coverage=$(grep -m 1 'headerCovTableEntry[a-zA-Z].*[0-9]*' lcov/index.html|sed 's/^.*>\([0-9]\+\.[0-9]\+\).*/\1/')
    - coverage=$(printf %.2f $coverage)
    - inum=$(echo $coverage|cut -d'.' -f1)
    - if [ -z "$inum" ]; then inum="0"; fi
    - ../contrib/make-coverage-badge "coverage" "$coverage%"
    - mv badge.svg ../public/coverage.svg
    - rm -rf ../public/coverage
    - mv lcov ../public/coverage

# Online Documentation
    - make -C docs clean
    - make -C docs
    - rm -rf ../public/reference
    - mv docs/html ../public/reference

# Coverage report for all our fuzz corpora
    - make -j$(nproc) fuzz-coverage
    - coverage=$(grep -m 1 'headerCovTableEntry[a-zA-Z].*[0-9]*' lcov/index.html|sed 's/^.*>\([0-9]\+\.[0-9]\+\).*/\1/')
    - coverage=$(printf %.2f $coverage)
    - inum=$(echo $coverage|cut -d'.' -f1)
    - if [ -z "$inum" ]; then inum="0"; fi
    - ../contrib/make-coverage-badge "fuzz-coverage" "$coverage%"
    - mv badge.svg ../public/fuzz-coverage.svg
    - rm -rf ../public/fuzz-coverage
    - mv lcov ../public/fuzz-coverage
  artifacts:
    when: on_success
    paths:
      - public
  only:
    - master

MinGW64:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$MINGW_BUILD
  extends:
    - .defaults
  script:
    - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
    - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register
    - export CFLAGS="$CFLAGS_DEFAULT"
    - export CC="ccache $PREFIX-gcc"
    - export GCCLIB=$(dirname $(find /usr/lib/gcc/$PREFIX -name libgcc_s_seh-1.dll|grep posix))
    - export WINEPATH="$WINEPATH;/usr/$PREFIX/bin;/usr/$PREFIX/lib;$PWD/libwget/.libs;$GCCLIB"
    - echo "WINEPATH=$WINEPATH"
    - tar xf wget2-*.gz && mv wget2-*.gz wget2-latest.tar.gz
    - cd wget2-*.*.?
    - ./configure $CONFIGURE_BASE_FLAGS --cache-file cache/config.cache
      --build=x86_64-pc-linux-gnu --host=$PREFIX --enable-shared
    - make -j$(nproc)
    - make check -j$(nproc) LOG_COMPILER=wine
  tags:
    - saas-linux-xlarge-amd64
  allow_failure: true

MinGW64-static:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$MINGW_BUILD_STATIC
  script:
    - unset CC
    - tar xf wget2-*.gz && mv wget2-*.gz wget2-latest.tar.gz
    - cd wget2-*.*.?
    - LDFLAGS="-Wl,-Bstatic,--whole-archive -lwinpthread -Wl,--no-whole-archive"
      CFLAGS="$CFLAGS_DEFAULT -DNGHTTP2_STATICLIB"
      ./configure $CONFIGURE_BASE_FLAGS --build=x86_64-pc-linux-gnu --host=$PREFIX
        --disable-shared --enable-static --without-gpgme --enable-threads=windows
    - make -j$(nproc)
    - mv src/wget2.exe .
  tags:
    - saas-linux-xlarge-amd64
  allow_failure: true
  artifacts:
    when: on_success
    paths:
      - wget2.exe
  only:
    - master

Arch:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$ARCH_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && mv wget2-*.gz wget2-latest.tar.gz
    - cd wget2-*.*.?
    - ./configure $CONFIGURE_BASE_FLAGS --cache-file cache/config.cache
    - make -j$(nproc)
    - make check -j$(nproc)

Debian/HTTP2:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_HTTP2_BUILD
  extends:
    - .defaults
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz && mv wget2-*.gz wget2-latest.tar.gz
    - cd wget2-*.*.?
    - ./configure $CONFIGURE_BASE_FLAGS --enable-code-coverage --cache-file cache/config.cache
    - make -j$(nproc)
    - make check -j$(nproc)
    - make code-coverage-capture
  allow_failure: true
  artifacts:
    expire_in: 2 weeks
    when: on_success
    paths:
      - wget2-*-coverage

coverity:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
  script:
    - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
    - tar xfz /tmp/coverity_tool.tgz
    - tar xf wget2-*.gz && mv wget2-*.gz wget2-latest.tar.gz
    - cd wget2-*.*.?
    - CFLAGS="-g -Og" ./configure --cache-file cache/config.cache --disable-doc
    - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
    - tar cfz cov-int.tar.gz cov-int
    - curl https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
      --form token=$COVERITY_SCAN_TOKEN --form email=tim.ruehsen@gmx.de
      --form file=@cov-int.tar.gz --form version="`git describe --tags`"
      --form description="CI build"
  only:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 1 week
    when: on_failure
    paths:
      - cov-int/*.txt