evergreen.yml

#                        🧞    Genny   🧞‍♀️
#
# This file defines how this project is built by evergreen at
#
#   https://evergreen.mongodb.com/waterfall/genny
#
# 1. Indent using 2 spaces. Structures start on indents (e.g. dashes aren't
#    indented). Don't quote unnecessarily.
#
# 2.  Use `f_` prefix for function names and `t_` prefix for task names.
#
#     This allows
#
#     1.  grep for all usage of a function/task without colliding with
#         inline shell etc.
#     2.  grep for definition of a function `FOO` by grepping for `f_FOO:`
#         (with a colon)
#
# 3.  Put comments in yaml not in inline shell.
#
# 4.  Keep yaml anchors/aliases to a minimum and always prefix with `a_`
#     for similar grep-ability. Favor functions and tasks with
#     dependencies.
#
# 5.  Always use functions for all task-bodies and pre/post-bodies rather
#     than inlines. This keeps the overall logic separate from the
#     implementation.
#
# 6.  Maintain one blank line between each unit (each function, task,
#     variant etc). Two lines between each section.
#
# 7.  Don't change the order of the sections. Goal is to start with "entry
#     points" (variants) and then define children in breadth-first order.
#
# 8.  Functions that are non-obvious must have a comment with the existing
#     comment style.
#
# 9.  Any function that requires parameters or `${}`-style expansions must
#     document those parameters appropriately such that callers don't need
#     to read the function definition to figure out what they mean.
#
#                                                                       🧞‍♀️


##                ⚡️ Variants ⚡️

ignore:
- "*.md"  # don't schedule tests if a commit only changes markdown files

buildvariants:
- name: amazon2
  display_name: Amazon Linux 2
  modules: [mongo]
  run_on:
  - amazon2-build
  expansions:
    distro: amazon2
    # latest successful build from master as of 2020-10-01 on "Amazon Linux 2" compile task "Binaries" file
    # commit should match that in get-mongo-source.sh
    mongodb_archive_url:
      https://mciuploads.s3.amazonaws.com/mongodb-mongo-master/amazon2/298d4d6bbb9980b74bded06241067fe6771bef68/binaries/mongo-mongodb_mongo_master_amazon2_298d4d6bbb9980b74bded06241067fe6771bef68_20_10_22_00_55_19.tgz
  tasks:
  - name: tg_compile_and_test_with_server

- name: rhel70
  display_name: RHEL 7
  modules: [mongo]
  expansions:
    distro: rhel70
  run_on:
  - rhel70
  tasks:
  - name: tg_compile

- name: rhel8
  display_name: RHEL 8
  modules: [mongo]
  expansions:
    distro: rhel8
  run_on:
  - rhel80-test
  tasks:
  - name: tg_compile

- name: ubuntu1804
  display_name: Ubuntu 18.04
  modules: [mongo]
  expansions:
    distro: ubuntu1804
  run_on:
  - ubuntu1804-build
  tasks:
  - name: tg_compile

- name: macos-1014
  display_name: macOS Mojave
  modules: [mongo]
  run_on:
  - macos-1014
  tasks:
  - name: tg_compile

- name: centos6-perf
  display_name: CentOS 6 for Performance
  modules: [mongo]
  expansions:
    distro: rhel62
  run_on:
  - centos6-perf
  tasks:
  - name: tg_compile_and_benchmark


##                ⚡️ Tasks ⚡️

tasks:
- name: t_compile
  commands:
  - func: f_fetch_source
  - func: f_compile
  - func: f_dry_run_workloads

- name: t_python_test
  commands:
  - func: f_python_test

- name: t_lint_workloads
  commands:
  - func: f_lint_workloads

- name: t_lint_python
  commands:
  - func: f_lint_python

- name: t_cmake_test
  commands:
  - func: f_cmake_test

- name: t_benchmark_test
  commands:
  - func: f_benchmark_test

  # It's not possible to do a standalone version due to
  # some tests' reliance on an apm callback that doesn't
  # work on standalone. (Specifically, since read concern
  # is irrelevant on a standalone, recent driver versions
  # simply omit it.)
- name: t_integration_test_single_node_replset
  commands:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - resmoke-test
      - --suites
      - src/genny/src/resmokeconfig/genny_single_node_replset.yml

- name: t_integration_test_three_node_replset
  commands:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - resmoke-test
      - --suites
      - ./src/genny/src/resmokeconfig/genny_three_node_replset.yml

- name: t_integration_test_sharded
  commands:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - resmoke-test
      - --suites
      - ./src/genny/src/resmokeconfig/genny_sharded.yml

# Must run this task last to avoid polluting test results with tests generated
# by the new actor.
- name: t_test_create_new_actor_script
  commands:
  - func: f_run_create_new_actor_script
  - func: f_compile
  - func: f_dry_run_workloads  # Test the generated yaml file is valid.
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - resmoke-test
      - --create-new-actor-test-suite

##                ⚡️ Task Groups ⚡️

task_groups:

- &a_compile_and_test
  name: tg_compile_and_test
  max_hosts: 1
  teardown_task:
  # Attaching results is prohibited in "teardown_group".
  # So we call it in "teardown_task" but make the result file optional.
  - func: f_report_test_results
  tasks:
  # Rely on implicit dependencies in task_groups where tasks are executed
  # in the order they're defined. Tasks with explicit "depends_on" tasks
  # may conflict with task_group's implicit dependency, causing
  # unexpected behavior.
  #
  # For this task group, the implicit dependency is that t_*_test has to
  # run after t_compile.
  - t_compile
  - t_python_test
  - t_lint_python
  - t_lint_workloads
  - t_cmake_test

- <<: *a_compile_and_test
  name: tg_compile_and_benchmark
  tasks:
  - t_compile
  - t_benchmark_test

- <<: *a_compile_and_test
  name: tg_compile_and_test_with_server
  tasks:
  - t_compile
  - t_python_test
  - t_lint_python
  - t_lint_workloads
  - t_cmake_test
  - t_integration_test_single_node_replset
  - t_integration_test_three_node_replset
  - t_integration_test_sharded
  - t_test_create_new_actor_script  # Must run this task last.

- name: tg_compile
  max_hosts: 1
  tasks:
  - t_compile


##                ⚡️ Functions ⚡️

functions:

  f_fetch_source:
  - command: manifest.load
  - command: git.get_project
    params:
      directory: src/genny
      revisions:
        mongo: 298d4d6bbb9980b74bded06241067fe6771bef68

  ##
  # Compile the project in src/build.
  #
  # We always remove the toolchain and curator directories because hosts
  # may have incomplete or invalid toolchain directories from
  # previous (unsuccessful) builds in the gennytoolchain project.
  ##
  f_compile:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - install
      - --linux-distro=${distro|not-linux}

  ##
  # Runs each workload as a dry run
  # Requires f_compile to have been run first.
  ##
  f_dry_run_workloads:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - dry-run-workloads

  ##
  # Runs tests via ctest.
  # Requires f_compile to have been run first.
  ##
  f_cmake_test:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - cmake-test

  ##
  # Runs benchmarks via ctest.
  # Requires f_compile to have been run first.
  ##
  f_benchmark_test:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - benchmark-test

  ##
  # Runs tests for the create_new_actor script.
  ##
  f_run_create_new_actor_script:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - create-new-actor
      - SelfTestActor

  ##
  # Runs python unit tests.
  ##
  f_python_test:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      # TODO: rename to pytest
      - self-test

  ##
  # Runs the YAML linter for Genny workloads
  ##
  f_lint_workloads:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - lint-yaml

  ##
  # Lint python
  ##
  f_lint_python:
  - command: subprocess.exec
    params:
      binary: ./src/genny/run-genny
      args:
      - -v
      - lint-python

  ##
  # Reports test results to evergreen API.
  ##
  f_report_test_results:
  - command: shell.exec
    params:
      script: |
        pwd
        find . -iname '*.xml'
        ls build/XUnitXML/*.xml
        cat build/XUnitXML/*.xml
  - command: attach.xunit_results
    params:
      file: build/XUnitXML/*.xml


##                ⚡️ Modules ⚡️

modules:
- name: mongo
  repo: git@github.com:mongodb/mongo.git
  # This prefix is relative to the path in git.get_project
  # I.e., src/genny/../../src = ./src
  prefix: ../../src
  branch: master