From 13ea233f7cfbbd4566dcef8f14585faec354f77a Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Sun, 22 Nov 2020 23:25:42 +0100 Subject: [PATCH 1/6] Allow building lesson using GitLab Pages --- .gitlab-ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..86decc77 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,22 @@ +--- +image: + name: carpentries/lesson-docker:latest + entrypoint: [""] + +variables: + LC_ALL: "C.UTF-8" + MAKEFLAGS: "-j 8" + # Override the Gemfile used in the lesson-docker image + # with the one included in the current repository + BUNDLE_GEMFILE: Gemfile + +pages: + stage: deploy + script: + - make site + - mv _site public + artifacts: + paths: + - public + only: + - gh-pages From 8864b22705f8b908ee5755d6abb9f9897a1d005e Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Sat, 15 May 2021 19:06:43 +0000 Subject: [PATCH 2/6] Configure alternatives to GitHub variables while in GitLab --- _includes/gh_variables.html | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/_includes/gh_variables.html b/_includes/gh_variables.html index 3fdae4ac..5966cec8 100644 --- a/_includes/gh_variables.html +++ b/_includes/gh_variables.html @@ -1,4 +1,12 @@ {% comment %} +This file was initially used to provide access to GitHub variables +available when rendering websites through GitHub Pages. +It has since then been expanded to also work with GitLab Pages, +although with some limitations due to platform differences. +Particularly, the variable "repo_info" is set to false when rendering +through GitLab Pages. The value false is the same set by github-pages +plugin when GitHub's API cannot be reached. + When rendering websites locally, `site.github.url` doesn't get resolved properly unless a GitHub Personal Access Token is set up and available in the environment. This leads to warnings and errors when trying to serve the site @@ -7,7 +15,23 @@ GitHub where `site.github.url` is defined. {% endcomment %} -{% if jekyll.environment == "production" %} +{% if site.env.GITLAB_CI %} + +{% comment %} +This block applies when running in a GitLab CI environment +See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html +for GitLab CI specific variables. +{% endcomment %} + +{% assign repo_name = site.env.CI_PROJECT_NAME %} +{% assign repo_info = false %} +{% assign default_branch = site.env.CI_DEFAULT_BRANCH %} +{% assign repo_url = site.env.CI_REPOSITORY_URL %} +{% assign search_domain_url = site.env.CI_PAGES_URL %} +{% assign project_title = site.env.CI_PROJECT_TITLE %} +{% assign source_branch = site.env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME %} + +{% elsif jekyll.environment == "production" and site.github %} {% comment %} First, get the name of the repository From 184ecffecbd39c2e8b685513f955f836bbd4145e Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Sat, 15 May 2021 19:06:06 +0000 Subject: [PATCH 3/6] Use github-pages gem only when building via GitHub Actions Allows inclusion of gems that will be relevant in other CIs such as GitLab. See pull request #521 for additional context --- Gemfile | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 8d69492b..bd6f1d11 100644 --- a/Gemfile +++ b/Gemfile @@ -7,8 +7,50 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } # Synchronize with https://pages.github.com/versions ruby '>=2.7.1' -gem 'github-pages', group: :jekyll_plugins +group :jekyll_plugins do + if ENV["GITHUB_ACTIONS"] + # When building GitHub Pages via GitHub Actions + gem 'github-pages' + else + # Or elsewhere and if additional gems are necessary. + # Note that the 'github-pages' plugin disables any + # non-whitelisted plugin. + # See PR #521 for additional context + + # Dependency list from 'github-pages' + # See: https://github.com/github/pages-gem/blob/75fd58be0f294a6bf55a1990643838d5984a1f62/lib/github-pages/dependencies.rb#L8 + gem 'jekyll', '>= 3.9.0' + gem 'jekyll-sass-converter', '>= 1.5.2' + # Converters + gem 'kramdown', '>= 2.3.1' + gem 'kramdown-parser-gfm', '>= 1.1.0' + gem 'jekyll-commonmark-ghpages', '>= 0.1.6' + # Misc + gem 'liquid', '>= 4.0.3' + gem 'rouge', '>= 3.26.0' + # gem 'github-pages-health-check', '>= 1.17.1' # Not necessary outside GitHub + # Plugins + gem 'jekyll-redirect-from', '>= 0.16.0' + gem 'jekyll-sitemap', '>= 1.4.0' + gem 'jekyll-feed', '>= 0.15.1' + gem 'jekyll-gist', '>= 1.5.0' + gem 'jekyll-paginate', '>= 1.1.0' + gem 'jekyll-coffeescript', '>= 1.1.1' + gem 'jekyll-seo-tag', '>= 2.7.1' + # gem 'jekyll-github-metadata', '>= 2.13.0' # Not necessary outside GitHub + gem 'jekyll-avatar', '>= 0.7.0' + gem 'jekyll-remote-theme', '>= 0.4.3' + # Plugins to match GitHub.com Markdown + gem 'jemoji', '>= 0.12.0' + gem 'jekyll-mentions', '>= 1.6.0' + gem 'jekyll-relative-links', '>= 0.6.1' + gem 'jekyll-optional-front-matter', '>= 0.3.2' + gem 'jekyll-readme-index', '>= 0.3.0' + gem 'jekyll-default-layout', '>= 0.1.4' + gem 'jekyll-titles-from-headings', '>= 0.5.3' + end +end if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.0.0') gem 'webrick', '>= 1.6.1' -end \ No newline at end of file +end From 6bcb65737f20a014c4d4a16909a51f90b4ff6d58 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Sun, 16 May 2021 01:53:09 +0200 Subject: [PATCH 4/6] Add gem to access CI environment variables --- Gemfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Gemfile b/Gemfile index bd6f1d11..3d6ae3b2 100644 --- a/Gemfile +++ b/Gemfile @@ -48,6 +48,8 @@ group :jekyll_plugins do gem 'jekyll-readme-index', '>= 0.3.0' gem 'jekyll-default-layout', '>= 0.1.4' gem 'jekyll-titles-from-headings', '>= 0.5.3' + # Extra dependency that allows accessing ENV variables in other CI environments + gem 'jekyll-environment-variables', '>= 1.0.1' end end From 68f8ba9525d45068ccc3fe605402409934b4d189 Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Sun, 16 May 2021 03:10:38 +0200 Subject: [PATCH 5/6] Use GitHub Actions to test building with GitLab (runner) --- .github/workflows/gitlab.yml | 137 +++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 .github/workflows/gitlab.yml diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml new file mode 100644 index 00000000..655a4f2d --- /dev/null +++ b/.github/workflows/gitlab.yml @@ -0,0 +1,137 @@ +name: GitLab Test template +on: + push: + branches: gh-pages + pull_request: +jobs: + check-template: + name: ${{ matrix.lesson-name }} (${{ matrix.os-name }}) + if: github.repository == 'carpentries/styles' + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + lesson: [carpentries/lesson-example] + os: [ubuntu-20.04] + experimental: [false] + include: + - os: ubuntu-20.04 + os-name: Linux + - lesson: carpentries/lesson-example + lesson-name: (CP) Lesson Example + experimental: false + os: ubuntu-20.04 + os-name: Linux + defaults: + run: + shell: bash # forces 'Git for Windows' on Windows + env: + RSPM: 'https://packagemanager.rstudio.com/cran/__linux__/focal/latest' + steps: + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '2.7' + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install GitHub Pages, Bundler, and kramdown gems + run: | + gem install github-pages bundler kramdown kramdown-parser-gfm + + - name: Install Python modules + run: | + python3 -m pip install --upgrade pip setuptools wheel pyyaml==5.3.1 requests + + - name: Checkout the ${{ matrix.lesson }} lesson + uses: actions/checkout@master + with: + repository: ${{ matrix.lesson }} + path: lesson + fetch-depth: 0 + + - name: Sync lesson with carpentries/styles + working-directory: lesson + run: | + echo "::group::Fetch Styles" + if [[ -n "${{ github.event.pull_request.number }}" ]] + then + ref="refs/pull/${{ github.event.pull_request.number }}/head" + else + ref="gh-pages" + fi + + git config --global user.email "team@carpentries.org" + git config --global user.name "The Carpentries Bot" + + git remote add styles https://github.com/carpentries/styles.git + git fetch styles $ref:styles-ref + echo "::endgroup::" + echo "::group::Synchronize Styles" + # Sync up only if necessary + if [[ $(git rev-list --count HEAD..styles-ref) != 0 ]] + then + + # The merge command below might fail for lessons that use remote theme + # https://github.com/carpentries/carpentries-theme + echo "Testing merge using recursive strategy, accepting upstream changes without committing" + if ! git merge -s recursive -Xtheirs --no-commit styles-ref + then + + # Remove "deleted by us, unmerged" files from the staging area. + # these are the files that were removed from the lesson + # but are still present in the carpentries/styles repo + echo "Removing previously deleted files" + git rm $(git diff --name-only --diff-filter=DU) + + # If there are still "unmerged" files, + # let's raise an error and look into this more closely + if [[ -n $(git diff --name-only --diff-filter=U) ]] + then + echo "There were unmerged files in ${{ matrix.lesson-name }}:" + echo "$(git diff --compact-summary --diff-filter=U)" + exit 1 + fi + fi + + echo "Committing changes" + git commit -m "Sync lesson with carpentries/styles" + fi + echo "::endgroup::" + + - name: Download gitlab-runner + working-directory: lesson + run: | + wget -O gitlab-runner https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64 + chmod +x gitlab-runner + + - name: Build carpentries example lesson using gitlab-runner + working-directory: lesson + run: | + # GitLab expects the final folder to be "public" + # gitlab-runner runs a docker container on which we + # mount _site as the expected output directory + # Chown is necessary only to ensure the files are owned by + # the user outside the container + ./gitlab-runner exec docker pages \ + --docker-volumes "$(pwd)/_site:/public" \ + --env "GITLAB_CI=true" \ + --env "CI_PROJECT_NAME=lesson-example" \ + --env "CI_DEFAULT_BRANCH=gh-pages" \ + --env "CI_REPOSITORY_URL=https://github.com/${{ matrix.lesson }}" \ + --env "CI_PAGES_URL=https://pages.github.com" \ + --env "CI_PROJECT_TITLE=${{ matrix.lesson-name }}" \ + --env "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME=patch-1" \ + --post-build-script "rm -rf /public/* && cp -R public/* /public && chown -R $(id -u):$(id -g) /public" + + - name: List generated files + run: ls -l _site + working-directory: lesson + + - run: make lesson-check-all + working-directory: lesson From 916fae1268a02a39f8cfdbadeb90ac6a24814e1a Mon Sep 17 00:00:00 2001 From: Renato Alves Date: Sun, 16 May 2021 07:09:41 +0200 Subject: [PATCH 6/6] Test that CI ENV vars are being read correctly In particular test that the github-pages plugin isn't interfering with reading environment variables --- .github/workflows/gitlab.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml index 655a4f2d..62b020af 100644 --- a/.github/workflows/gitlab.yml +++ b/.github/workflows/gitlab.yml @@ -118,11 +118,14 @@ jobs: # mount _site as the expected output directory # Chown is necessary only to ensure the files are owned by # the user outside the container + # NOTE when using gitlab-runner CI_REPOSITORY_URL is set + # to /path/to/repo regardless of --env passed below. + # When running in GitLab CI it will be set to the repo URL ./gitlab-runner exec docker pages \ --docker-volumes "$(pwd)/_site:/public" \ --env "GITLAB_CI=true" \ --env "CI_PROJECT_NAME=lesson-example" \ - --env "CI_DEFAULT_BRANCH=gh-pages" \ + --env "CI_DEFAULT_BRANCH=my-custom-branch" \ --env "CI_REPOSITORY_URL=https://github.com/${{ matrix.lesson }}" \ --env "CI_PAGES_URL=https://pages.github.com" \ --env "CI_PROJECT_TITLE=${{ matrix.lesson-name }}" \ @@ -133,5 +136,9 @@ jobs: run: ls -l _site working-directory: lesson + - name: Check that CI_DEFAULT_BRANCH propagated correctly + run: grep '/edit/my-custom-branch/index\.md' _site/index.html + working-directory: lesson + - run: make lesson-check-all working-directory: lesson