From 7e0d388f8822871a919d482a620745f1685c4401 Mon Sep 17 00:00:00 2001 From: Pratik Gupte Date: Thu, 6 Jun 2024 09:23:15 +0100 Subject: [PATCH] Initial touchstone infrastructure --- .Rbuildignore | 1 + .github/workflows/touchstone-comment.yaml | 21 +++++++++ .github/workflows/touchstone-receive.yaml | 56 +++++++++++++++++++++++ touchstone/.gitignore | 6 +++ touchstone/config.json | 8 ++++ touchstone/footer.R | 10 ++++ touchstone/header.R | 13 ++++++ touchstone/script.R | 30 ++++++++++++ 8 files changed, 145 insertions(+) create mode 100644 .github/workflows/touchstone-comment.yaml create mode 100644 .github/workflows/touchstone-receive.yaml create mode 100644 touchstone/.gitignore create mode 100644 touchstone/config.json create mode 100644 touchstone/footer.R create mode 100644 touchstone/header.R create mode 100644 touchstone/script.R diff --git a/.Rbuildignore b/.Rbuildignore index 197e5b9d..29dca5f2 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -15,3 +15,4 @@ ^revdep$ ^CRAN-SUBMISSION$ ^CITATION\.cff$ +^touchstone$ diff --git a/.github/workflows/touchstone-comment.yaml b/.github/workflows/touchstone-comment.yaml new file mode 100644 index 00000000..aa3d40de --- /dev/null +++ b/.github/workflows/touchstone-comment.yaml @@ -0,0 +1,21 @@ +name: Continuous Benchmarks (Comment) + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +on: + workflow_run: + workflows: ["Continuous Benchmarks (Receive)"] + types: + - completed + +jobs: + upload: + runs-on: ubuntu-latest + if: > + ${{ github.event.workflow_run.event == 'pull_request' }} + steps: + - uses: lorenzwalthert/touchstone/actions/comment@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/touchstone-receive.yaml b/.github/workflows/touchstone-receive.yaml new file mode 100644 index 00000000..24b2f6ab --- /dev/null +++ b/.github/workflows/touchstone-receive.yaml @@ -0,0 +1,56 @@ +name: Continuous Benchmarks (Receive) + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +on: + pull_request: + +jobs: + prepare: + runs-on: ubuntu-latest + if: + true && + ( + github.event.pull_request.author_association == 'OWNER' || + github.event.pull_request.author_association == 'MEMBER' || + github.event.pull_request.author_association == 'COLLABORATOR' + ) + outputs: + config: ${{ steps.read_touchstone_config.outputs.config }} + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - id: read_touchstone_config + run: | + content=`cat ./touchstone/config.json` + # the following lines are only required for multi line json + content="${content//'%'/'%25'}" + content="${content//$'\n'/'%0A'}" + content="${content//$'\r'/'%0D'}" + # end of optional handling for multi line json + echo "::set-output name=config::$content" + build: + needs: prepare + runs-on: ${{ matrix.config.os }} + strategy: + fail-fast: false + matrix: + config: + - ${{ fromJson(needs.prepare.outputs.config) }} + env: + R_REMOTES_NO_ERRORS_FROM_WARNINGS: true + RSPM: ${{ matrix.config.rspm }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: lorenzwalthert/touchstone/actions/receive@v1 + with: + cache-version: 1 + benchmarking_repo: ${{ matrix.config.benchmarking_repo }} + benchmarking_ref: ${{ matrix.config.benchmarking_ref }} + benchmarking_path: ${{ matrix.config.benchmarking_path }} + force_upstream: true diff --git a/touchstone/.gitignore b/touchstone/.gitignore new file mode 100644 index 00000000..ee8eacf0 --- /dev/null +++ b/touchstone/.gitignore @@ -0,0 +1,6 @@ +* +!script.R +!config.json +!.gitignore +!header.R +!footer.R diff --git a/touchstone/config.json b/touchstone/config.json new file mode 100644 index 00000000..f9f10879 --- /dev/null +++ b/touchstone/config.json @@ -0,0 +1,8 @@ +{ + "os": "ubuntu-20.04", + "r": "4.1.1", + "rspm": "https://packagemanager.rstudio.com/all/__linux__/focal/2022-01-07+MTo3NDQwNTcyLDI6NDUyNjIxNTs0QzU3NUZBRQ", + // "benchmarking_repo": "lorenzwalthert/here", + // "benchmarking_ref": "ca9c8e69c727def88d8ba1c8b85b0e0bcea87b3f", + "benchmarking_path": "touchstone/sources/here", +} diff --git a/touchstone/footer.R b/touchstone/footer.R new file mode 100644 index 00000000..e827d7b5 --- /dev/null +++ b/touchstone/footer.R @@ -0,0 +1,10 @@ +# You can modify the PR comment footer here. You can use github markdown e.g. +# emojis like :tada:. +# This file will be parsed and evaluate within the context of +# `benchmark_analyze` and should return the comment text as the last value. +# See `?touchstone::pr_comment` +link <- "https://lorenzwalthert.github.io/touchstone/articles/inference.html" +glue::glue( + "\nFurther explanation regarding interpretation and", + " methodology can be found in the [documentation]({link})." +) diff --git a/touchstone/header.R b/touchstone/header.R new file mode 100644 index 00000000..d84b90eb --- /dev/null +++ b/touchstone/header.R @@ -0,0 +1,13 @@ +# You can modify the PR comment header here. You can use github markdown e.g. +# emojis like :tada:. +# This file will be parsed and evaluate within the context of +# `benchmark_analyze` and should return the comment text as the last value. +# Available variables for glue substitution: +# * ci: confidence interval +# * branches: BASE and HEAD branches benchmarked against each other. +# See `?touchstone::pr_comment` +glue::glue( + "This is how benchmark results would change (along with a", + " {100 * ci}% confidence interval in relative change) if ", + "{system2('git', c('rev-parse', 'HEAD'), stdout = TRUE)} is merged into {branches[1]}:\n" +) diff --git a/touchstone/script.R b/touchstone/script.R new file mode 100644 index 00000000..db90b0d4 --- /dev/null +++ b/touchstone/script.R @@ -0,0 +1,30 @@ +# see `help(run_script, package = 'touchstone')` on how to run this +# interactively + +# TODO OPTIONAL Add directories you want to be available in this file or during the +# benchmarks. +# touchstone::pin_assets("some/dir") + +# installs branches to benchmark +touchstone::branch_install() + +# benchmark a function call from your package (two calls per branch) +touchstone::benchmark_run( + # expr_before_benchmark = source("dir/data.R"), #<-- TODO OTPIONAL setup before benchmark + random_test = yourpkg::f(), #<- TODO put the call you want to benchmark here + n = 2 +) + +# TODO OPTIONAL benchmark any R expression (six calls per branch) +# touchstone::benchmark_run( +# more = { +# if (TRUE) { +# y <- yourpkg::f2(x = 3) +# } +# }, #<- TODO put the call you want to benchmark here +# n = 6 +# ) + + +# create artifacts used downstream in the GitHub Action +touchstone::benchmark_analyze()