diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..1d7d564d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,25 @@ +root = true + +[*] +charset = utf-8 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +indent_size = 2 +indent_style = space +max_line_length = 100 # Please keep this in sync with bin/lesson_check.py! + +[*.r] +max_line_length = 80 + +[*.py] +indent_size = 4 +indent_style = space +max_line_length = 79 + +[*.sh] +end_of_line = lf + +[Makefile] +indent_style = tab diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..2ee9d0ee --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,2 @@ +github: [carpentries, swcarpentry, datacarpentry, librarycarpentry] +custom: ["https://carpentries.wedid.it"] diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..077de4cf --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,21 @@ +
+Instructions + +Thanks for contributing! :heart: + +If this contribution is for instructor training, please email the link to this contribution to +checkout@carpentries.org so we can record your progress. You've completed your contribution +step for instructor checkout by submitting this contribution! + +If this issue is about a specific episode within a lesson, please provide its link or filename. + +Keep in mind that **lesson maintainers are volunteers** and it may take them some time to +respond to your contribution. Although not all contributions can be incorporated into the lesson +materials, we appreciate your time and effort to improve the curriculum. If you have any questions +about the lesson maintenance process or would like to volunteer your time as a contribution +reviewer, please contact The Carpentries Team at team@carpentries.org. + +You may delete these instructions from your comment. + +\- The Carpentries +
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..07aadca2 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,19 @@ +
+Instructions + +Thanks for contributing! :heart: + +If this contribution is for instructor training, please email the link to this contribution to +checkout@carpentries.org so we can record your progress. You've completed your contribution +step for instructor checkout by submitting this contribution! + +Keep in mind that **lesson maintainers are volunteers** and it may take them some time to +respond to your contribution. Although not all contributions can be incorporated into the lesson +materials, we appreciate your time and effort to improve the curriculum. If you have any questions +about the lesson maintenance process or would like to volunteer your time as a contribution +reviewer, please contact The Carpentries Team at team@carpentries.org. + +You may delete these instructions from your comment. + +\- The Carpentries +
diff --git a/.github/workflows/template.yml b/.github/workflows/template.yml new file mode 100644 index 00000000..13f03bcf --- /dev/null +++ b/.github/workflows/template.yml @@ -0,0 +1,118 @@ +name: Template +on: + push: + branches: gh-pages + pull_request: +jobs: + check-template: + name: Test lesson template + if: github.repository == 'carpentries/styles' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + lesson: [swcarpentry/shell-novice, datacarpentry/r-intro-geospatial, librarycarpentry/lc-git] + os: [ubuntu-latest, macos-latest, windows-latest] + defaults: + run: + shell: bash # forces 'Git for Windows' on Windows + env: + RSPM: 'https://packagemanager.rstudio.com/cran/__linux__/bionic/latest' + steps: + - name: Set up Ruby + uses: actions/setup-ruby@main + with: + ruby-version: '2.7.1' + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install GitHub Pages, Bundler, and kramdown gems + run: | + gem install github-pages bundler kramdown + + - name: Install Python modules + run: | + if [[ $RUNNER_OS == macOS || $RUNNER_OS == Linux ]]; then + python3 -m pip install --upgrade pip setuptools wheel pyyaml==5.3.1 requests + elif [[ $RUNNER_OS == Windows ]]; then + python -m pip install --upgrade pip setuptools wheel pyyaml==5.3.1 requests + fi + + - name: Checkout the ${{ matrix.lesson }} lesson + uses: actions/checkout@master + with: + repository: ${{ matrix.lesson }} + path: lesson + fetch-depth: 0 + + - name: Determine the proper reference to use + id: styles-ref + run: | + if [[ -n "${{ github.event.pull_request.number }}" ]]; then + echo "::set-output name=ref::refs/pull/${{ github.event.pull_request.number }}/head" + else + echo "::set-output name=ref::gh-pages" + fi + + - name: Sync lesson with carpentries/styles + working-directory: lesson + run: | + git config --global user.email "team@carpentries.org" + git config --global user.name "The Carpentries Bot" + git remote add styles https://github.com/carpentries/styles.git + git config --local remote.styles.tagOpt --no-tags + git fetch styles ${{ steps.styles-ref.outputs.ref }}:styles-ref + git merge -s recursive -Xtheirs --no-commit styles-ref + git commit -m "Sync lesson with carpentries/styles" + + - name: Look for R-markdown files + id: check-rmd + working-directory: lesson + run: | + echo "::set-output name=count::$(shopt -s nullglob; files=($(find . -iname '*.Rmd')); echo ${#files[@]})" + + - name: Set up R + if: steps.check-rmd.outputs.count != 0 + uses: r-lib/actions/setup-r@master + with: + r-version: 'release' + + - name: Install needed packages + if: steps.check-rmd.outputs.count != 0 + run: | + install.packages(c('remotes', 'rprojroot', 'renv', 'desc', 'rmarkdown', 'knitr')) + shell: Rscript {0} + + - name: Query dependencies + if: steps.check-rmd.outputs.count != 0 + working-directory: lesson + run: | + source('bin/dependencies.R') + deps <- identify_dependencies() + create_description(deps) + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + - name: Cache R packages + if: runner.os != 'Windows' && steps.check-rmd.outputs.count != 0 + uses: actions/cache@v1 + with: + path: ${{ env.R_LIBS_USER }} + key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} + restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1- + + - name: Install system dependencies for R packages + if: runner.os == 'Linux' && steps.check-rmd.outputs.count != 0 + working-directory: lesson + run: | + while read -r cmd + do + eval sudo $cmd + done < <(Rscript -e 'cat(remotes::system_requirements("ubuntu", "18.04"), sep = "\n")') + + - run: make site + working-directory: lesson diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..1a9eadf0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +*.pyc +*~ +.DS_Store +.ipynb_checkpoints +.sass-cache +.jekyll-cache/ +__pycache__ +_site +.Rproj.user +.Rhistory +.RData +.bundle/ +.vendor/ +vendor/ +.docker-vendor/ +Gemfile.lock +.*history diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..f6885e9a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,57 @@ +# Travis CI is only used to check the lesson and is not involved in its deployment +dist: bionic +language: ruby +rvm: + - 2.7.1 + +branches: + only: + - gh-pages + - /.*/ + +cache: + apt: true + bundler: true + directories: + - /home/travis/.rvm/ + - $R_LIBS_USER + - $HOME/.cache/pip + +env: + global: + - NOKOGIRI_USE_SYSTEM_LIBRARIES=true # speeds up installation of html-proofer + - R_LIBS_USER=~/R/Library + - R_LIBS_SITE=/usr/local/lib/R/site-library:/usr/lib/R/site-library + - R_VERSION=4.0.2 + +before_install: + ## Install R + pandoc + dependencies + - sudo add-apt-repository -y "ppa:marutter/rrutter4.0" + - sudo add-apt-repository -y "ppa:c2d4u.team/c2d4u4.0+" + - sudo add-apt-repository -y "ppa:ubuntugis/ppa" + - sudo add-apt-repository -y "ppa:cran/travis" + - travis_apt_get_update + - sudo apt-get install -y --no-install-recommends build-essential gcc g++ libblas-dev liblapack-dev libncurses5-dev libreadline-dev libjpeg-dev libpcre3-dev libpng-dev zlib1g-dev libbz2-dev liblzma-dev libicu-dev cdbs qpdf texinfo libssh2-1-dev gfortran jq python3.5 python3-pip r-base + - export PATH=${TRAVIS_HOME}/R-bin/bin:$PATH + - export LD_LIBRARY_PATH=${TRAVIS_HOME}/R-bin/lib:$LD_LIBRARY_PATH + - sudo mkdir -p /usr/local/lib/R/site-library $R_LIBS_USER + - sudo chmod 2777 /usr/local/lib/R /usr/local/lib/R/site-library $R_LIBS_USER + - echo 'options(repos = c(CRAN = "https://packagemanager.rstudio.com/all/__linux__/bionic/latest"))' > ~/.Rprofile.site + - export R_PROFILE=~/.Rprofile.site + - curl -fLo /tmp/texlive.tar.gz https://github.com/jimhester/ubuntu-bin/releases/download/latest/texlive.tar.gz + - tar xzf /tmp/texlive.tar.gz -C ~ + - export PATH=${TRAVIS_HOME}/texlive/bin/x86_64-linux:$PATH + - tlmgr update --self + - curl -fLo /tmp/pandoc-2.2-1-amd64.deb https://github.com/jgm/pandoc/releases/download/2.2/pandoc-2.2-1-amd64.deb + - sudo dpkg -i /tmp/pandoc-2.2-1-amd64.deb + - sudo apt-get install -f + - rm /tmp/pandoc-2.2-1-amd64.deb + - Rscript -e "install.packages(setdiff(c('renv', 'rprojroot'), installed.packages()), loc = Sys.getenv('R_LIBS_USER')); update.packages(lib.loc = Sys.getenv('R_LIBS_USER'), ask = FALSE, checkBuilt = TRUE)" + - Rscript -e 'sessionInfo()' + ## Install python and dependencies + - python3 -m pip install --upgrade pip setuptools wheel + - python3 -m pip install pyyaml + +script: + - make lesson-check-all + - make --always-make site diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000..04e1f5ab --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +FIXME: list authors' names and email addresses. \ No newline at end of file diff --git a/CITATION b/CITATION new file mode 100644 index 00000000..56ece3c4 --- /dev/null +++ b/CITATION @@ -0,0 +1 @@ +FIXME: describe how to cite this lesson. \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..c3b96690 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,11 @@ +--- +layout: page +title: "Contributor Code of Conduct" +--- +As contributors and maintainers of this project, +we pledge to follow the [Carpentry Code of Conduct][coc]. + +Instances of abusive, harassing, or otherwise unacceptable behavior +may be reported by following our [reporting guidelines][coc-reporting]. + +{% include links.md %} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..7925ceff --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,151 @@ +# Contributing + +[The Carpentries][c-site] ([Software Carpentry][swc-site], [Data Carpentry][dc-site], and [Library Carpentry][lc-site]) are open source projects, +and we welcome contributions of all kinds: +new lessons, +fixes to existing material, +bug reports, +and reviews of proposed changes are all welcome. + +## Contributor Agreement + +By contributing, +you agree that we may redistribute your work under [our license](LICENSE.md). +In exchange, +we will address your issues and/or assess your change proposal as promptly as we can, +and help you become a member of our community. +Everyone involved in [The Carpentries][c-site] +agrees to abide by our [code of conduct](CODE_OF_CONDUCT.md). + +## How to Contribute + +The easiest way to get started is to file an issue +to tell us about a spelling mistake, +some awkward wording, +or a factual error. +This is a good way to introduce yourself +and to meet some of our community members. + +1. If you do not have a [GitHub][github] account, + you can [send us comments by email][email]. + However, + we will be able to respond more quickly if you use one of the other methods described below. + +2. If you have a [GitHub][github] account, + or are willing to [create one][github-join], + but do not know how to use Git, + you can report problems or suggest improvements by [creating an issue][issues]. + This allows us to assign the item to someone + and to respond to it in a threaded discussion. + +3. If you are comfortable with Git, + and would like to add or change material, + you can submit a pull request (PR). + Instructions for doing this are [included below](#using-github). + +## Where to Contribute + +1. If you wish to change this lesson, + please work in , + which can be viewed at . + +2. If you wish to change the example lesson, + please work in , + which documents the format of our lessons + and can be viewed at . + +3. If you wish to change the template used for workshop websites, + please work in . + The home page of that repository explains how to set up workshop websites, + while the extra pages in + provide more background on our design choices. + +4. If you wish to change CSS style files, tools, + or HTML boilerplate for lessons or workshops stored in `_includes` or `_layouts`, + please work in . + +## What to Contribute + +There are many ways to contribute, +from writing new exercises and improving existing ones +to updating or filling in the documentation +and submitting [bug reports][issues] +about things that don't work, aren't clear, or are missing. +If you are looking for ideas, please see the 'Issues' tab for +a list of issues associated with this repository, +or you may also look at the issues for [Data Carpentry][dc-issues], +[Software Carpentry][swc-issues], and [Library Carpentry][lc-issues] projects. + +Comments on issues and reviews of pull requests are just as welcome: +we are smarter together than we are on our own. +Reviews from novices and newcomers are particularly valuable: +it's easy for people who have been using these lessons for a while +to forget how impenetrable some of this material can be, +so fresh eyes are always welcome. + +## What *Not* to Contribute + +Our lessons already contain more material than we can cover in a typical workshop, +so we are usually *not* looking for more concepts or tools to add to them. +As a rule, +if you want to introduce a new idea, +you must (a) estimate how long it will take to teach +and (b) explain what you would take out to make room for it. +The first encourages contributors to be honest about requirements; +the second, to think hard about priorities. + +We are also not looking for exercises or other material that only run on one platform. +Our workshops typically contain a mixture of Windows, macOS, and Linux users; +in order to be usable, +our lessons must run equally well on all three. + +## Using GitHub + +If you choose to contribute via GitHub, you may want to look at +[How to Contribute to an Open Source Project on GitHub][how-contribute]. +To manage changes, we follow [GitHub flow][github-flow]. +Each lesson has two maintainers who review issues and pull requests or encourage others to do so. +The maintainers are community volunteers and have final say over what gets merged into the lesson. +To use the web interface for contributing to a lesson: + +1. Fork the originating repository to your GitHub profile. +2. Within your version of the forked repository, move to the `gh-pages` branch and +create a new branch for each significant change being made. +3. Navigate to the file(s) you wish to change within the new branches and make revisions as required. +4. Commit all changed files within the appropriate branches. +5. Create individual pull requests from each of your changed branches +to the `gh-pages` branch within the originating repository. +6. If you receive feedback, make changes using your issue-specific branches of the forked +repository and the pull requests will update automatically. +7. Repeat as needed until all feedback has been addressed. + +When starting work, please make sure your clone of the originating `gh-pages` branch is up-to-date +before creating your own revision-specific branch(es) from there. +Additionally, please only work from your newly-created branch(es) and *not* +your clone of the originating `gh-pages` branch. +Lastly, published copies of all the lessons are available in the `gh-pages` branch of the originating +repository for reference while revising. + +## Other Resources + +General discussion of [Software Carpentry][swc-site] and [Data Carpentry][dc-site] +happens on the [discussion mailing list][discuss-list], +which everyone is welcome to join. +You can also [reach us by email][email]. + +[email]: mailto:admin@software-carpentry.org +[dc-issues]: https://github.com/issues?q=user%3Adatacarpentry +[dc-lessons]: http://datacarpentry.org/lessons/ +[dc-site]: http://datacarpentry.org/ +[discuss-list]: http://lists.software-carpentry.org/listinfo/discuss +[github]: https://github.com +[github-flow]: https://guides.github.com/introduction/flow/ +[github-join]: https://github.com/join +[how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github +[issues]: https://guides.github.com/features/issues/ +[swc-issues]: https://github.com/issues?q=user%3Aswcarpentry +[swc-lessons]: https://software-carpentry.org/lessons/ +[swc-site]: https://software-carpentry.org/ +[c-site]: https://carpentries.org/ +[lc-site]: https://librarycarpentry.org/ +[lc-issues]: https://github.com/issues?q=user%3Alibrarycarpentry diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..fb25ae48 --- /dev/null +++ b/Gemfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } + +# Synchronize with https://pages.github.com/versions +ruby '>=2.5.8' + +gem 'github-pages', group: :jekyll_plugins diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 00000000..e6a3398d --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,83 @@ +--- +layout: page +title: "Licenses" +root: . +--- +## Instructional Material + +All Software Carpentry, Data Carpentry, and Library Carpentry instructional material is +made available under the [Creative Commons Attribution +license][cc-by-human]. The following is a human-readable summary of +(and not a substitute for) the [full legal text of the CC BY 4.0 +license][cc-by-legal]. + +You are free: + +* to **Share**---copy and redistribute the material in any medium or format +* to **Adapt**---remix, transform, and build upon the material + +for any purpose, even commercially. + +The licensor cannot revoke these freedoms as long as you follow the +license terms. + +Under the following terms: + +* **Attribution**---You must give appropriate credit (mentioning that + your work is derived from work that is Copyright © Software + Carpentry and, where practical, linking to + http://software-carpentry.org/), provide a [link to the + license][cc-by-human], and indicate if changes were made. You may do + so in any reasonable manner, but not in any way that suggests the + licensor endorses you or your use. + +**No additional restrictions**---You may not apply legal terms or +technological measures that legally restrict others from doing +anything the license permits. With the understanding that: + +Notices: + +* You do not have to comply with the license for elements of the + material in the public domain or where your use is permitted by an + applicable exception or limitation. +* No warranties are given. The license may not give you all of the + permissions necessary for your intended use. For example, other + rights such as publicity, privacy, or moral rights may limit how you + use the material. + +## Software + +Except where otherwise noted, the example programs and other software +provided by Software Carpentry and Data Carpentry are made available under the +[OSI][osi]-approved +[MIT license][mit-license]. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +## Trademark + +"Software Carpentry" and "Data Carpentry" and their respective logos +are registered trademarks of [Community Initiatives][CI]. + +[cc-by-human]: https://creativecommons.org/licenses/by/4.0/ +[cc-by-legal]: https://creativecommons.org/licenses/by/4.0/legalcode +[mit-license]: https://opensource.org/licenses/mit-license.html +[ci]: http://communityin.org/ +[osi]: https://opensource.org diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..d35f08f5 --- /dev/null +++ b/Makefile @@ -0,0 +1,162 @@ +## ======================================== +## Commands for both workshop and lesson websites. + +# Settings +MAKEFILES=Makefile $(wildcard *.mk) +JEKYLL=bundle config --local set path .vendor/bundle && bundle install && bundle update && bundle exec jekyll +PARSER=bin/markdown_ast.rb +DST=_site + +# Check Python 3 is installed and determine if it's called via python3 or python +# (https://stackoverflow.com/a/4933395) +PYTHON3_EXE := $(shell which python3 2>/dev/null) +ifneq (, $(PYTHON3_EXE)) + ifeq (,$(findstring Microsoft/WindowsApps/python3,$(subst \,/,$(PYTHON3_EXE)))) + PYTHON := python3 + endif +endif + +ifeq (,$(PYTHON)) + PYTHON_EXE := $(shell which python 2>/dev/null) + ifneq (, $(PYTHON_EXE)) + PYTHON_VERSION_FULL := $(wordlist 2,4,$(subst ., ,$(shell python --version 2>&1))) + PYTHON_VERSION_MAJOR := $(word 1,${PYTHON_VERSION_FULL}) + ifneq (3, ${PYTHON_VERSION_MAJOR}) + $(error "Your system does not appear to have Python 3 installed.") + endif + PYTHON := python + else + $(error "Your system does not appear to have any Python installed.") + endif +endif + + +# Controls +.PHONY : commands clean files + +# Default target +.DEFAULT_GOAL := commands + +## I. Commands for both workshop and lesson websites +## ================================================= + +## * serve : render website and run a local server +serve : lesson-md + ${JEKYLL} serve + +## * site : build website but do not run a server +site : lesson-md + ${JEKYLL} build + +## * docker-serve : use Docker to serve the site +docker-serve : + docker pull carpentries/lesson-docker:latest + docker run --rm -it \ + -v $${PWD}:/home/rstudio \ + -p 4000:4000 \ + -p 8787:8787 \ + -e USERID=$$(id -u) \ + -e GROUPID=$$(id -g) \ + carpentries/lesson-docker:latest + +## * repo-check : check repository settings +repo-check : + @${PYTHON} bin/repo_check.py -s . + +## * clean : clean up junk files +clean : + @rm -rf ${DST} + @rm -rf .sass-cache + @rm -rf bin/__pycache__ + @find . -name .DS_Store -exec rm {} \; + @find . -name '*~' -exec rm {} \; + @find . -name '*.pyc' -exec rm {} \; + +## * clean-rmd : clean intermediate R files (that need to be committed to the repo) +clean-rmd : + @rm -rf ${RMD_DST} + @rm -rf fig/rmd-* + + +## +## II. Commands specific to workshop websites +## ================================================= + +.PHONY : workshop-check + +## * workshop-check : check workshop homepage +workshop-check : + @${PYTHON} bin/workshop_check.py . + + +## +## III. Commands specific to lesson websites +## ================================================= + +.PHONY : lesson-check lesson-md lesson-files lesson-fixme install-rmd-deps + +# RMarkdown files +RMD_SRC = $(wildcard _episodes_rmd/??-*.Rmd) +RMD_DST = $(patsubst _episodes_rmd/%.Rmd,_episodes/%.md,$(RMD_SRC)) + +# Lesson source files in the order they appear in the navigation menu. +MARKDOWN_SRC = \ + index.md \ + CODE_OF_CONDUCT.md \ + setup.md \ + $(sort $(wildcard _episodes/*.md)) \ + reference.md \ + $(sort $(wildcard _extras/*.md)) \ + LICENSE.md + +# Generated lesson files in the order they appear in the navigation menu. +HTML_DST = \ + ${DST}/index.html \ + ${DST}/conduct/index.html \ + ${DST}/setup/index.html \ + $(patsubst _episodes/%.md,${DST}/%/index.html,$(sort $(wildcard _episodes/*.md))) \ + ${DST}/reference/index.html \ + $(patsubst _extras/%.md,${DST}/%/index.html,$(sort $(wildcard _extras/*.md))) \ + ${DST}/license/index.html + +## * install-rmd-deps : Install R packages dependencies to build the RMarkdown lesson +install-rmd-deps: + @${SHELL} bin/install_r_deps.sh + +## * lesson-md : convert Rmarkdown files to markdown +lesson-md : ${RMD_DST} + +_episodes/%.md: _episodes_rmd/%.Rmd install-rmd-deps + @mkdir -p _episodes + @bin/knit_lessons.sh $< $@ + +## * lesson-check : validate lesson Markdown +lesson-check : lesson-fixme + @${PYTHON} bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md + +## * lesson-check-all : validate lesson Markdown, checking line lengths and trailing whitespace +lesson-check-all : + @${PYTHON} bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md -l -w --permissive + +## * unittest : run unit tests on checking tools +unittest : + @${PYTHON} bin/test_lesson_check.py + +## * lesson-files : show expected names of generated files for debugging +lesson-files : + @echo 'RMD_SRC:' ${RMD_SRC} + @echo 'RMD_DST:' ${RMD_DST} + @echo 'MARKDOWN_SRC:' ${MARKDOWN_SRC} + @echo 'HTML_DST:' ${HTML_DST} + +## * lesson-fixme : show FIXME markers embedded in source files +lesson-fixme : + @grep --fixed-strings --word-regexp --line-number --no-messages FIXME ${MARKDOWN_SRC} || true + +## +## IV. Auxililary (plumbing) commands +## ================================================= + +## * commands : show all commands. +commands : + @sed -n -e '/^##/s|^##[[:space:]]*||p' $(MAKEFILE_LIST) diff --git a/README.md b/README.md new file mode 100644 index 00000000..9a004e29 --- /dev/null +++ b/README.md @@ -0,0 +1,75 @@ +# FIXME Lesson title + +[![Create a Slack Account with us](https://img.shields.io/badge/Create_Slack_Account-The_Carpentries-071159.svg)](https://swc-slack-invite.herokuapp.com/) + +**Thanks for contributing to The Carpentries Incubator!** +This repository provides a blank starting point for lessons to be developed here. + +A member of the [Carpentries Curriculum Team](https://carpentries.org/team/) +will work with you to get your lesson listed on the +[Community Developed Lessons page][community-lessons] +and make sure you have everything you need to begin developing your new lesson. + +## What to do next + +Before you begin developing your new lesson, +here are a few things we recommend you do: + +* [ ] Decide on a title for your new lesson! + Once you've chosen a new title, you can set the value for `lesson_title` + in [`_config.yml`](_config.yml) +* [ ] Add the URL to your built lesson pages to the repository description\* +* [ ] Fill in the fields marked `FIXME` in: + * this README + * [`_config.yml`](_config.yml) +* [ ] If you're going to be developing lesson material for the first time + according to our design principles, + consider reading the [Carpentries Curriculum Development Handbook][cdh] +* [ ] Consult the [Lesson Example][lesson-example] website to find out more about + working with the lesson template +* [ ] Update this README with relevant information about your lesson + and delete this section + +\* To set the URL on GitHub, click the gear wheel button next to **About** +on the right of the repository landing page. +The lesson URL structure is **https://carpentries-incubator.github.io/**: +a repository at https://github.com/carpentries-incubator/new-lesson/ will have pages at +the lesson URL https://carpentries-incubator.github.io/new-lesson/. + + +## Contributing + +We welcome all contributions to improve the lesson! Maintainers will do their best to help you if you have any +questions, concerns, or experience any difficulties along the way. + +We'd like to ask you to familiarize yourself with our [Contribution Guide](CONTRIBUTING.md) and have a look at +the [more detailed guidelines][lesson-example] on proper formatting, ways to render the lesson locally, and even +how to write new episodes. + +Please see the current list of [issues][FIXME] for ideas for contributing to this +repository. For making your contribution, we use the GitHub flow, which is +nicely explained in the chapter [Contributing to a Project](http://git-scm.com/book/en/v2/GitHub-Contributing-to-a-Project) in Pro Git +by Scott Chacon. +Look for the tag ![good_first_issue](https://img.shields.io/badge/-good%20first%20issue-gold.svg). This indicates that the maintainers will welcome a pull request fixing this issue. + + +## Maintainer(s) + +Current maintainers of this lesson are + +* FIXME +* FIXME +* FIXME + + +## Authors + +A list of contributors to the lesson can be found in [AUTHORS](AUTHORS) + +## Citation + +To cite this lesson, please consult with [CITATION](CITATION) + +[cdh]: https://cdh.carpentries.org +[community-lessons]: https://carpentries.org/community-lessons +[lesson-example]: https://carpentries.github.io/lesson-example diff --git a/_config.yml b/_config.yml new file mode 100644 index 00000000..1662c9d1 --- /dev/null +++ b/_config.yml @@ -0,0 +1,104 @@ +#------------------------------------------------------------ +# Values for this lesson. +#------------------------------------------------------------ + +# Which carpentry is this ("swc", "dc", "lc", or "cp")? +# swc: Software Carpentry +# dc: Data Carpentry +# lc: Library Carpentry +# cp: Carpentries (to use for instructor traning for instance) +carpentry: "incubator" + +# Overall title for pages. +title: "Lesson Title" # FIXME + +# Life cycle stage of the lesson +# See this page for more details: https://cdh.carpentries.org/the-lesson-life-cycle.html +# Possible values: "pre-alpha", "alpha", "beta", "stable" +life_cycle: "pre-alpha" + +#------------------------------------------------------------ +# Generic settings (should not need to change). +#------------------------------------------------------------ + +# What kind of thing is this ("workshop" or "lesson")? +kind: "lesson" + +# Magic to make URLs resolve both locally and on GitHub. +# See https://help.github.com/articles/repository-metadata-on-github-pages/. +# Please don't change it: / is correct. +repository: / + +# Email address, no mailto: +email: "team@carpentries.org" # FIXME + +# Sites. +amy_site: "https://amy.carpentries.org/" +carpentries_github: "https://github.com/carpentries" +carpentries_pages: "https://carpentries.github.io" +carpentries_site: "https://carpentries.org/" +dc_site: "https://datacarpentry.org" +example_repo: "https://github.com/carpentries/lesson-example" +example_site: "https://carpentries.github.io/lesson-example" +lc_site: "https://librarycarpentry.org/" +swc_github: "https://github.com/swcarpentry" +swc_pages: "https://swcarpentry.github.io" +swc_site: "https://software-carpentry.org" +template_repo: "https://github.com/carpentries/styles" +training_site: "https://carpentries.github.io/instructor-training" +workshop_repo: "https://github.com/carpentries/workshop-template" +workshop_site: "https://carpentries.github.io/workshop-template" +cc_by_human: "https://creativecommons.org/licenses/by/4.0/" + +# Surveys. +pre_survey: "https://carpentries.typeform.com/to/wi32rS?slug=" +post_survey: "https://carpentries.typeform.com/to/UgVdRQ?slug=" +instructor_pre_survey: "https://www.surveymonkey.com/r/instructor_training_pre_survey?workshop_id=" +instructor_post_survey: "https://www.surveymonkey.com/r/instructor_training_post_survey?workshop_id=" + + +# Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). +start_time: 0 + +# Specify that things in the episodes collection should be output. +collections: + episodes: + output: true + permalink: /:path/index.html + extras: + output: true + permalink: /:path/index.html + +# Set the default layout for things in the episodes collection. +defaults: + - values: + root: . + layout: page + - scope: + path: "" + type: episodes + values: + root: .. + layout: episode + - scope: + path: "" + type: extras + values: + root: .. + layout: page + +# Files and directories that are not to be copied. +exclude: + - Makefile + - bin/ + - .Rproj.user/ + - .vendor/ + - vendor/ + - .docker-vendor/ + +# Turn on built-in syntax highlighting. +highlighter: rouge + + +# Remote theme +remote_theme: carpentries/carpentries-theme diff --git a/_episodes/.gitkeep b/_episodes/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/_episodes/01-introduction.md b/_episodes/01-introduction.md new file mode 100644 index 00000000..2e156c26 --- /dev/null +++ b/_episodes/01-introduction.md @@ -0,0 +1,15 @@ +--- +title: "Introduction" +teaching: 0 +exercises: 0 +questions: +- "Key question (FIXME)" +objectives: +- "First learning objective. (FIXME)" +keypoints: +- "First key point. Brief Answer to questions. (FIXME)" +--- +FIXME + +{% include links.md %} + diff --git a/_episodes_rmd/.gitkeep b/_episodes_rmd/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/_episodes_rmd/data/.gitkeep b/_episodes_rmd/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/_extras/.gitkeep b/_extras/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/_extras/about.md b/_extras/about.md new file mode 100644 index 00000000..5f07f659 --- /dev/null +++ b/_extras/about.md @@ -0,0 +1,5 @@ +--- +title: About +--- +{% include carpentries.html %} +{% include links.md %} diff --git a/_extras/discuss.md b/_extras/discuss.md new file mode 100644 index 00000000..bfc33c50 --- /dev/null +++ b/_extras/discuss.md @@ -0,0 +1,6 @@ +--- +title: Discussion +--- +FIXME + +{% include links.md %} diff --git a/_extras/figures.md b/_extras/figures.md new file mode 100644 index 00000000..0012c88e --- /dev/null +++ b/_extras/figures.md @@ -0,0 +1,79 @@ +--- +title: Figures +--- + +{% include base_path.html %} +{% include manual_episode_order.html %} + + + +{% comment %} Create anchor for each one of the episodes. {% endcomment %} + +{% for lesson_episode in lesson_episodes %} + {% if site.episode_order %} + {% assign episode = site.episodes | where: "slug", lesson_episode | first %} + {% else %} + {% assign episode = lesson_episode %} + {% endif %} +
+{% endfor %} + +{% include links.md %} diff --git a/_extras/guide.md b/_extras/guide.md new file mode 100644 index 00000000..50f266f8 --- /dev/null +++ b/_extras/guide.md @@ -0,0 +1,6 @@ +--- +title: "Instructor Notes" +--- +FIXME + +{% include links.md %} diff --git a/aio.md b/aio.md new file mode 100644 index 00000000..0ab72ea5 --- /dev/null +++ b/aio.md @@ -0,0 +1,13 @@ +--- +permalink: /aio/index.html +--- + +{% comment %} +As a maintainer, you don't need to edit this file. +If you notice that something doesn't work, please +open an issue: https://github.com/carpentries/styles/issues/new +{% endcomment %} + +{% include base_path.html %} + +{% include aio-script.md %} diff --git a/bin/boilerplate/.travis.yml b/bin/boilerplate/.travis.yml new file mode 100644 index 00000000..f6885e9a --- /dev/null +++ b/bin/boilerplate/.travis.yml @@ -0,0 +1,57 @@ +# Travis CI is only used to check the lesson and is not involved in its deployment +dist: bionic +language: ruby +rvm: + - 2.7.1 + +branches: + only: + - gh-pages + - /.*/ + +cache: + apt: true + bundler: true + directories: + - /home/travis/.rvm/ + - $R_LIBS_USER + - $HOME/.cache/pip + +env: + global: + - NOKOGIRI_USE_SYSTEM_LIBRARIES=true # speeds up installation of html-proofer + - R_LIBS_USER=~/R/Library + - R_LIBS_SITE=/usr/local/lib/R/site-library:/usr/lib/R/site-library + - R_VERSION=4.0.2 + +before_install: + ## Install R + pandoc + dependencies + - sudo add-apt-repository -y "ppa:marutter/rrutter4.0" + - sudo add-apt-repository -y "ppa:c2d4u.team/c2d4u4.0+" + - sudo add-apt-repository -y "ppa:ubuntugis/ppa" + - sudo add-apt-repository -y "ppa:cran/travis" + - travis_apt_get_update + - sudo apt-get install -y --no-install-recommends build-essential gcc g++ libblas-dev liblapack-dev libncurses5-dev libreadline-dev libjpeg-dev libpcre3-dev libpng-dev zlib1g-dev libbz2-dev liblzma-dev libicu-dev cdbs qpdf texinfo libssh2-1-dev gfortran jq python3.5 python3-pip r-base + - export PATH=${TRAVIS_HOME}/R-bin/bin:$PATH + - export LD_LIBRARY_PATH=${TRAVIS_HOME}/R-bin/lib:$LD_LIBRARY_PATH + - sudo mkdir -p /usr/local/lib/R/site-library $R_LIBS_USER + - sudo chmod 2777 /usr/local/lib/R /usr/local/lib/R/site-library $R_LIBS_USER + - echo 'options(repos = c(CRAN = "https://packagemanager.rstudio.com/all/__linux__/bionic/latest"))' > ~/.Rprofile.site + - export R_PROFILE=~/.Rprofile.site + - curl -fLo /tmp/texlive.tar.gz https://github.com/jimhester/ubuntu-bin/releases/download/latest/texlive.tar.gz + - tar xzf /tmp/texlive.tar.gz -C ~ + - export PATH=${TRAVIS_HOME}/texlive/bin/x86_64-linux:$PATH + - tlmgr update --self + - curl -fLo /tmp/pandoc-2.2-1-amd64.deb https://github.com/jgm/pandoc/releases/download/2.2/pandoc-2.2-1-amd64.deb + - sudo dpkg -i /tmp/pandoc-2.2-1-amd64.deb + - sudo apt-get install -f + - rm /tmp/pandoc-2.2-1-amd64.deb + - Rscript -e "install.packages(setdiff(c('renv', 'rprojroot'), installed.packages()), loc = Sys.getenv('R_LIBS_USER')); update.packages(lib.loc = Sys.getenv('R_LIBS_USER'), ask = FALSE, checkBuilt = TRUE)" + - Rscript -e 'sessionInfo()' + ## Install python and dependencies + - python3 -m pip install --upgrade pip setuptools wheel + - python3 -m pip install pyyaml + +script: + - make lesson-check-all + - make --always-make site diff --git a/bin/boilerplate/AUTHORS b/bin/boilerplate/AUTHORS new file mode 100644 index 00000000..04e1f5ab --- /dev/null +++ b/bin/boilerplate/AUTHORS @@ -0,0 +1 @@ +FIXME: list authors' names and email addresses. \ No newline at end of file diff --git a/bin/boilerplate/CITATION b/bin/boilerplate/CITATION new file mode 100644 index 00000000..56ece3c4 --- /dev/null +++ b/bin/boilerplate/CITATION @@ -0,0 +1 @@ +FIXME: describe how to cite this lesson. \ No newline at end of file diff --git a/bin/boilerplate/CONTRIBUTING.md b/bin/boilerplate/CONTRIBUTING.md new file mode 100644 index 00000000..7925ceff --- /dev/null +++ b/bin/boilerplate/CONTRIBUTING.md @@ -0,0 +1,151 @@ +# Contributing + +[The Carpentries][c-site] ([Software Carpentry][swc-site], [Data Carpentry][dc-site], and [Library Carpentry][lc-site]) are open source projects, +and we welcome contributions of all kinds: +new lessons, +fixes to existing material, +bug reports, +and reviews of proposed changes are all welcome. + +## Contributor Agreement + +By contributing, +you agree that we may redistribute your work under [our license](LICENSE.md). +In exchange, +we will address your issues and/or assess your change proposal as promptly as we can, +and help you become a member of our community. +Everyone involved in [The Carpentries][c-site] +agrees to abide by our [code of conduct](CODE_OF_CONDUCT.md). + +## How to Contribute + +The easiest way to get started is to file an issue +to tell us about a spelling mistake, +some awkward wording, +or a factual error. +This is a good way to introduce yourself +and to meet some of our community members. + +1. If you do not have a [GitHub][github] account, + you can [send us comments by email][email]. + However, + we will be able to respond more quickly if you use one of the other methods described below. + +2. If you have a [GitHub][github] account, + or are willing to [create one][github-join], + but do not know how to use Git, + you can report problems or suggest improvements by [creating an issue][issues]. + This allows us to assign the item to someone + and to respond to it in a threaded discussion. + +3. If you are comfortable with Git, + and would like to add or change material, + you can submit a pull request (PR). + Instructions for doing this are [included below](#using-github). + +## Where to Contribute + +1. If you wish to change this lesson, + please work in , + which can be viewed at . + +2. If you wish to change the example lesson, + please work in , + which documents the format of our lessons + and can be viewed at . + +3. If you wish to change the template used for workshop websites, + please work in . + The home page of that repository explains how to set up workshop websites, + while the extra pages in + provide more background on our design choices. + +4. If you wish to change CSS style files, tools, + or HTML boilerplate for lessons or workshops stored in `_includes` or `_layouts`, + please work in . + +## What to Contribute + +There are many ways to contribute, +from writing new exercises and improving existing ones +to updating or filling in the documentation +and submitting [bug reports][issues] +about things that don't work, aren't clear, or are missing. +If you are looking for ideas, please see the 'Issues' tab for +a list of issues associated with this repository, +or you may also look at the issues for [Data Carpentry][dc-issues], +[Software Carpentry][swc-issues], and [Library Carpentry][lc-issues] projects. + +Comments on issues and reviews of pull requests are just as welcome: +we are smarter together than we are on our own. +Reviews from novices and newcomers are particularly valuable: +it's easy for people who have been using these lessons for a while +to forget how impenetrable some of this material can be, +so fresh eyes are always welcome. + +## What *Not* to Contribute + +Our lessons already contain more material than we can cover in a typical workshop, +so we are usually *not* looking for more concepts or tools to add to them. +As a rule, +if you want to introduce a new idea, +you must (a) estimate how long it will take to teach +and (b) explain what you would take out to make room for it. +The first encourages contributors to be honest about requirements; +the second, to think hard about priorities. + +We are also not looking for exercises or other material that only run on one platform. +Our workshops typically contain a mixture of Windows, macOS, and Linux users; +in order to be usable, +our lessons must run equally well on all three. + +## Using GitHub + +If you choose to contribute via GitHub, you may want to look at +[How to Contribute to an Open Source Project on GitHub][how-contribute]. +To manage changes, we follow [GitHub flow][github-flow]. +Each lesson has two maintainers who review issues and pull requests or encourage others to do so. +The maintainers are community volunteers and have final say over what gets merged into the lesson. +To use the web interface for contributing to a lesson: + +1. Fork the originating repository to your GitHub profile. +2. Within your version of the forked repository, move to the `gh-pages` branch and +create a new branch for each significant change being made. +3. Navigate to the file(s) you wish to change within the new branches and make revisions as required. +4. Commit all changed files within the appropriate branches. +5. Create individual pull requests from each of your changed branches +to the `gh-pages` branch within the originating repository. +6. If you receive feedback, make changes using your issue-specific branches of the forked +repository and the pull requests will update automatically. +7. Repeat as needed until all feedback has been addressed. + +When starting work, please make sure your clone of the originating `gh-pages` branch is up-to-date +before creating your own revision-specific branch(es) from there. +Additionally, please only work from your newly-created branch(es) and *not* +your clone of the originating `gh-pages` branch. +Lastly, published copies of all the lessons are available in the `gh-pages` branch of the originating +repository for reference while revising. + +## Other Resources + +General discussion of [Software Carpentry][swc-site] and [Data Carpentry][dc-site] +happens on the [discussion mailing list][discuss-list], +which everyone is welcome to join. +You can also [reach us by email][email]. + +[email]: mailto:admin@software-carpentry.org +[dc-issues]: https://github.com/issues?q=user%3Adatacarpentry +[dc-lessons]: http://datacarpentry.org/lessons/ +[dc-site]: http://datacarpentry.org/ +[discuss-list]: http://lists.software-carpentry.org/listinfo/discuss +[github]: https://github.com +[github-flow]: https://guides.github.com/introduction/flow/ +[github-join]: https://github.com/join +[how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github +[issues]: https://guides.github.com/features/issues/ +[swc-issues]: https://github.com/issues?q=user%3Aswcarpentry +[swc-lessons]: https://software-carpentry.org/lessons/ +[swc-site]: https://software-carpentry.org/ +[c-site]: https://carpentries.org/ +[lc-site]: https://librarycarpentry.org/ +[lc-issues]: https://github.com/issues?q=user%3Alibrarycarpentry diff --git a/bin/boilerplate/README.md b/bin/boilerplate/README.md new file mode 100644 index 00000000..060994ae --- /dev/null +++ b/bin/boilerplate/README.md @@ -0,0 +1,40 @@ +# FIXME Lesson title + +[![Create a Slack Account with us](https://img.shields.io/badge/Create_Slack_Account-The_Carpentries-071159.svg)](https://swc-slack-invite.herokuapp.com/) + +This repository generates the corresponding lesson website from [The Carpentries](https://carpentries.org/) repertoire of lessons. + +## Contributing + +We welcome all contributions to improve the lesson! Maintainers will do their best to help you if you have any +questions, concerns, or experience any difficulties along the way. + +We'd like to ask you to familiarize yourself with our [Contribution Guide](CONTRIBUTING.md) and have a look at +the [more detailed guidelines][lesson-example] on proper formatting, ways to render the lesson locally, and even +how to write new episodes. + +Please see the current list of [issues][FIXME] for ideas for contributing to this +repository. For making your contribution, we use the GitHub flow, which is +nicely explained in the chapter [Contributing to a Project](http://git-scm.com/book/en/v2/GitHub-Contributing-to-a-Project) in Pro Git +by Scott Chacon. +Look for the tag ![good_first_issue](https://img.shields.io/badge/-good%20first%20issue-gold.svg). This indicates that the maintainers will welcome a pull request fixing this issue. + + +## Maintainer(s) + +Current maintainers of this lesson are + +* FIXME +* FIXME +* FIXME + + +## Authors + +A list of contributors to the lesson can be found in [AUTHORS](AUTHORS) + +## Citation + +To cite this lesson, please consult with [CITATION](CITATION) + +[lesson-example]: https://carpentries.github.io/lesson-example diff --git a/bin/boilerplate/_config.yml b/bin/boilerplate/_config.yml new file mode 100644 index 00000000..795f788e --- /dev/null +++ b/bin/boilerplate/_config.yml @@ -0,0 +1,100 @@ +#------------------------------------------------------------ +# Values for this lesson. +#------------------------------------------------------------ + +# Which carpentry is this ("swc", "dc", "lc", or "cp")? +# swc: Software Carpentry +# dc: Data Carpentry +# lc: Library Carpentry +# cp: Carpentries (to use for instructor traning for instance) +carpentry: "swc" + +# Overall title for pages. +title: "Lesson Title" + +# Life cycle stage of the lesson +# See this page for more details: https://cdh.carpentries.org/the-lesson-life-cycle.html +# Possible values: "pre-alpha", "alpha", "beta", "stable" +life_cycle: "pre-alpha" + +#------------------------------------------------------------ +# Generic settings (should not need to change). +#------------------------------------------------------------ + +# What kind of thing is this ("workshop" or "lesson")? +kind: "lesson" + +# Magic to make URLs resolve both locally and on GitHub. +# See https://help.github.com/articles/repository-metadata-on-github-pages/. +# Please don't change it: / is correct. +repository: / + +# Email address, no mailto: +email: "team@carpentries.org" + +# Sites. +amy_site: "https://amy.carpentries.org/" +carpentries_github: "https://github.com/carpentries" +carpentries_pages: "https://carpentries.github.io" +carpentries_site: "https://carpentries.org/" +dc_site: "https://datacarpentry.org" +example_repo: "https://github.com/carpentries/lesson-example" +example_site: "https://carpentries.github.io/lesson-example" +lc_site: "https://librarycarpentry.org/" +swc_github: "https://github.com/swcarpentry" +swc_pages: "https://swcarpentry.github.io" +swc_site: "https://software-carpentry.org" +template_repo: "https://github.com/carpentries/styles" +training_site: "https://carpentries.github.io/instructor-training" +workshop_repo: "https://github.com/carpentries/workshop-template" +workshop_site: "https://carpentries.github.io/workshop-template" +cc_by_human: "https://creativecommons.org/licenses/by/4.0/" + +# Surveys. +pre_survey: "https://carpentries.typeform.com/to/wi32rS?slug=" +post_survey: "https://carpentries.typeform.com/to/UgVdRQ?slug=" +instructor_pre_survey: "https://www.surveymonkey.com/r/instructor_training_pre_survey?workshop_id=" +instructor_post_survey: "https://www.surveymonkey.com/r/instructor_training_post_survey?workshop_id=" + + +# Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). +start_time: 0 + +# Specify that things in the episodes collection should be output. +collections: + episodes: + output: true + permalink: /:path/index.html + extras: + output: true + permalink: /:path/index.html + +# Set the default layout for things in the episodes collection. +defaults: + - values: + root: . + layout: page + - scope: + path: "" + type: episodes + values: + root: .. + layout: episode + - scope: + path: "" + type: extras + values: + root: .. + layout: page + +# Files and directories that are not to be copied. +exclude: + - Makefile + - bin/ + - .Rproj.user/ + - .vendor/ + - vendor/ + - .docker-vendor/ + +# Turn on built-in syntax highlighting. +highlighter: rouge diff --git a/bin/boilerplate/_episodes/01-introduction.md b/bin/boilerplate/_episodes/01-introduction.md new file mode 100644 index 00000000..2e156c26 --- /dev/null +++ b/bin/boilerplate/_episodes/01-introduction.md @@ -0,0 +1,15 @@ +--- +title: "Introduction" +teaching: 0 +exercises: 0 +questions: +- "Key question (FIXME)" +objectives: +- "First learning objective. (FIXME)" +keypoints: +- "First key point. Brief Answer to questions. (FIXME)" +--- +FIXME + +{% include links.md %} + diff --git a/bin/boilerplate/_extras/about.md b/bin/boilerplate/_extras/about.md new file mode 100644 index 00000000..5f07f659 --- /dev/null +++ b/bin/boilerplate/_extras/about.md @@ -0,0 +1,5 @@ +--- +title: About +--- +{% include carpentries.html %} +{% include links.md %} diff --git a/bin/boilerplate/_extras/discuss.md b/bin/boilerplate/_extras/discuss.md new file mode 100644 index 00000000..bfc33c50 --- /dev/null +++ b/bin/boilerplate/_extras/discuss.md @@ -0,0 +1,6 @@ +--- +title: Discussion +--- +FIXME + +{% include links.md %} diff --git a/bin/boilerplate/_extras/figures.md b/bin/boilerplate/_extras/figures.md new file mode 100644 index 00000000..0012c88e --- /dev/null +++ b/bin/boilerplate/_extras/figures.md @@ -0,0 +1,79 @@ +--- +title: Figures +--- + +{% include base_path.html %} +{% include manual_episode_order.html %} + + + +{% comment %} Create anchor for each one of the episodes. {% endcomment %} + +{% for lesson_episode in lesson_episodes %} + {% if site.episode_order %} + {% assign episode = site.episodes | where: "slug", lesson_episode | first %} + {% else %} + {% assign episode = lesson_episode %} + {% endif %} +
+{% endfor %} + +{% include links.md %} diff --git a/bin/boilerplate/_extras/guide.md b/bin/boilerplate/_extras/guide.md new file mode 100644 index 00000000..50f266f8 --- /dev/null +++ b/bin/boilerplate/_extras/guide.md @@ -0,0 +1,6 @@ +--- +title: "Instructor Notes" +--- +FIXME + +{% include links.md %} diff --git a/bin/boilerplate/index.md b/bin/boilerplate/index.md new file mode 100644 index 00000000..95ccdbdc --- /dev/null +++ b/bin/boilerplate/index.md @@ -0,0 +1,17 @@ +--- +layout: lesson +root: . # Is the only page that doesn't follow the pattern /:path/index.html +permalink: index.html # Is the only page that doesn't follow the pattern /:path/index.html +--- +FIXME: home page introduction + + + +{% comment %} This is a comment in Liquid {% endcomment %} + +> ## Prerequisites +> +> FIXME +{: .prereq} + +{% include links.md %} diff --git a/bin/boilerplate/reference.md b/bin/boilerplate/reference.md new file mode 100644 index 00000000..8c826167 --- /dev/null +++ b/bin/boilerplate/reference.md @@ -0,0 +1,9 @@ +--- +layout: reference +--- + +## Glossary + +FIXME + +{% include links.md %} diff --git a/bin/boilerplate/setup.md b/bin/boilerplate/setup.md new file mode 100644 index 00000000..b8c50321 --- /dev/null +++ b/bin/boilerplate/setup.md @@ -0,0 +1,7 @@ +--- +title: Setup +--- +FIXME + + +{% include links.md %} diff --git a/bin/chunk-options.R b/bin/chunk-options.R new file mode 100644 index 00000000..8e0d62af --- /dev/null +++ b/bin/chunk-options.R @@ -0,0 +1,70 @@ +# These settings control the behavior of all chunks in the novice R materials. +# For example, to generate the lessons with all the output hidden, simply change +# `results` from "markup" to "hide". +# For more information on available chunk options, see +# http://yihui.name/knitr/options#chunk_options + +library("knitr") + +fix_fig_path <- function(pth) file.path("..", pth) + + +## We set the path for the figures globally below, so if we want to +## customize it for individual episodes, we can append a prefix to the +## global path. For instance, if we call knitr_fig_path("01-") in the +## first episode of the lesson, it will generate the figures in +## `fig/rmd-01-` +knitr_fig_path <- function(prefix) { + new_path <- paste0(opts_chunk$get("fig.path"), + prefix) + opts_chunk$set(fig.path = new_path) +} + +## We use the rmd- prefix for the figures generated by the lessons so +## they can be easily identified and deleted by `make clean-rmd`. The +## working directory when the lessons are generated is the root so the +## figures need to be saved in fig/, but when the site is generated, +## the episodes will be one level down. We fix the path using the +## `fig.process` option. + +opts_chunk$set(tidy = FALSE, results = "markup", comment = NA, + fig.align = "center", fig.path = "fig/rmd-", + fig.process = fix_fig_path, + fig.width = 8.5, fig.height = 8.5, + fig.retina = 2) + +# The hooks below add html tags to the code chunks and their output so that they +# are properly formatted when the site is built. + +hook_in <- function(x, options) { + lg <- tolower(options$engine) + style <- paste0(".language-", lg) + + stringr::str_c("\n\n~~~\n", + paste0(x, collapse="\n"), + "\n~~~\n{: ", style, "}\n\n") +} + +hook_out <- function(x, options) { + x <- gsub("\n$", "", x) + stringr::str_c("\n\n~~~\n", + paste0(x, collapse="\n"), + "\n~~~\n{: .output}\n\n") +} + +hook_error <- function(x, options) { + x <- gsub("\n$", "", x) + stringr::str_c("\n\n~~~\n", + paste0(x, collapse="\n"), + "\n~~~\n{: .error}\n\n") +} + +hook_warning <- function(x, options) { + x <- gsub("\n$", "", x) + stringr::str_c("\n\n~~~\n", + paste0(x, collapse = "\n"), + "\n~~~\n{: .warning}\n\n") +} + +knit_hooks$set(source = hook_in, output = hook_out, warning = hook_warning, + error = hook_error, message = hook_out) diff --git a/bin/dependencies.R b/bin/dependencies.R new file mode 100644 index 00000000..676b0505 --- /dev/null +++ b/bin/dependencies.R @@ -0,0 +1,55 @@ +install_required_packages <- function(lib = NULL, repos = getOption("repos")) { + + if (is.null(lib)) { + lib <- .libPaths() + } + + message("lib paths: ", paste(lib, collapse = ", ")) + missing_pkgs <- setdiff( + c("rprojroot", "desc", "remotes", "renv"), + rownames(installed.packages(lib.loc = lib)) + ) + + install.packages(missing_pkgs, lib = lib, repos = repos) + +} + +find_root <- function() { + + cfg <- rprojroot::has_file_pattern("^_config.y*ml$") + root <- rprojroot::find_root(cfg) + + root +} + +identify_dependencies <- function() { + + root <- find_root() + + required_pkgs <- unique(c( + ## Packages for episodes + renv::dependencies(file.path(root, "_episodes_rmd"), progress = FALSE, error = "ignore")$Package, + ## Packages for tools + renv::dependencies(file.path(root, "bin"), progress = FALSE, error = "ignore")$Package + )) + + required_pkgs +} + +create_description <- function(required_pkgs) { + d <- desc::description$new("!new") + lapply(required_pkgs, function(x) d$set_dep(x)) + d$write("DESCRIPTION") +} + +install_dependencies <- function(required_pkgs, ...) { + + create_description(required_pkgs) + on.exit(file.remove("DESCRIPTION")) + remotes::install_deps(dependencies = TRUE, ...) + + if (require("knitr") && packageVersion("knitr") < '1.9.19') { + stop("knitr must be version 1.9.20 or higher") + } + +} diff --git a/bin/generate_md_episodes.R b/bin/generate_md_episodes.R new file mode 100644 index 00000000..ce9e8aa6 --- /dev/null +++ b/bin/generate_md_episodes.R @@ -0,0 +1,40 @@ +generate_md_episodes <- function() { + + ## get the Rmd file to process from the command line, and generate the path + ## for their respective outputs + args <- commandArgs(trailingOnly = TRUE) + if (!identical(length(args), 2L)) { + stop("input and output file must be passed to the script") + } + + src_rmd <- args[1] + dest_md <- args[2] + + ## knit the Rmd into markdown + knitr::knit(src_rmd, output = dest_md) + + # Read the generated md files and add comments advising not to edit them + add_no_edit_comment <- function(y) { + con <- file(y) + mdfile <- readLines(con) + if (mdfile[1] != "---") + stop("Input file does not have a valid header") + mdfile <- append( + mdfile, + "# Please do not edit this file directly; it is auto generated.", + after = 1 + ) + mdfile <- append( + mdfile, + paste("# Instead, please edit", basename(y), "in _episodes_rmd/"), + after = 2 + ) + writeLines(mdfile, con) + close(con) + return(paste("Warning added to YAML header of", y)) + } + + vapply(dest_md, add_no_edit_comment, character(1)) +} + +generate_md_episodes() diff --git a/bin/install_r_deps.sh b/bin/install_r_deps.sh new file mode 100755 index 00000000..0280f241 --- /dev/null +++ b/bin/install_r_deps.sh @@ -0,0 +1 @@ +Rscript -e "source(file.path('bin', 'dependencies.R')); install_required_packages(); install_dependencies(identify_dependencies())" diff --git a/bin/knit_lessons.sh b/bin/knit_lessons.sh new file mode 100755 index 00000000..141c136a --- /dev/null +++ b/bin/knit_lessons.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Only try running R to translate files if there are some files present. +# The Makefile passes in the names of files. + +if [ $# -eq 2 ] ; then + Rscript -e "source('bin/generate_md_episodes.R')" "$@" +fi diff --git a/bin/lesson_check.py b/bin/lesson_check.py new file mode 100644 index 00000000..618f322a --- /dev/null +++ b/bin/lesson_check.py @@ -0,0 +1,569 @@ +""" +Check lesson files and their contents. +""" + + +import os +import glob +import re +from argparse import ArgumentParser + +from util import (Reporter, read_markdown, load_yaml, check_unwanted_files, + require) + +__version__ = '0.3' + +# Where to look for source Markdown files. +SOURCE_DIRS = ['', '_episodes', '_extras'] + +# Where to look for source Rmd files. +SOURCE_RMD_DIRS = ['_episodes_rmd'] + +# Required files: each entry is ('path': YAML_required). +# FIXME: We do not yet validate whether any files have the required +# YAML headers, but should in the future. +# The '%' is replaced with the source directory path for checking. +# Episodes are handled specially, and extra files in '_extras' are also handled +# specially. This list must include all the Markdown files listed in the +# 'bin/initialize' script. +REQUIRED_FILES = { + 'CODE_OF_CONDUCT.md': True, + 'CONTRIBUTING.md': False, + 'LICENSE.md': True, + 'README.md': False, + os.path.join('_extras', 'discuss.md'): True, + os.path.join('_extras', 'guide.md'): True, + 'index.md': True, + 'reference.md': True, + 'setup.md': True, +} + +# Episode filename pattern. +P_EPISODE_FILENAME = re.compile(r'(\d\d)-[-\w]+.md$') + +# Pattern to match lines ending with whitespace. +P_TRAILING_WHITESPACE = re.compile(r'\s+$') + +# Pattern to match figure references in HTML. +P_FIGURE_REFS = re.compile(r']+src="([^"]+)"[^>]*>') + +# Pattern to match internally-defined Markdown links. +P_INTERNAL_LINK_REF = re.compile(r'\[([^\]]+)\]\[([^\]]+)\]') + +# Pattern to match reference links (to resolve internally-defined references). +P_INTERNAL_LINK_DEF = re.compile(r'^\[([^\]]+)\]:\s*(.+)') + +# Pattern to match {% include ... %} statements +P_INTERNAL_INCLUDE_LINK = re.compile(r'^{% include ([^ ]*) %}$') + +# What kinds of blockquotes are allowed? +KNOWN_BLOCKQUOTES = { + 'callout', + 'challenge', + 'checklist', + 'discussion', + 'keypoints', + 'objectives', + 'prereq', + 'quotation', + 'solution', + 'testimonial' +} + +# What kinds of code fragments are allowed? +KNOWN_CODEBLOCKS = { + 'error', + 'output', + 'source', + 'language-bash', + 'html', + 'language-c', + 'language-cmake', + 'language-cpp', + 'language-make', + 'language-matlab', + 'language-python', + 'language-r', + 'language-shell', + 'language-sql' +} + +# What fields are required in teaching episode metadata? +TEACHING_METADATA_FIELDS = { + ('title', str), + ('teaching', int), + ('exercises', int), + ('questions', list), + ('objectives', list), + ('keypoints', list) +} + +# What fields are required in break episode metadata? +BREAK_METADATA_FIELDS = { + ('layout', str), + ('title', str), + ('break', int) +} + +# How long are lines allowed to be? +# Please keep this in sync with .editorconfig! +MAX_LINE_LEN = 100 + + +def main(): + """Main driver.""" + + args = parse_args() + args.reporter = Reporter() + check_config(args.reporter, args.source_dir) + check_source_rmd(args.reporter, args.source_dir, args.parser) + args.references = read_references(args.reporter, args.reference_path) + + docs = read_all_markdown(args.source_dir, args.parser) + check_fileset(args.source_dir, args.reporter, list(docs.keys())) + check_unwanted_files(args.source_dir, args.reporter) + for filename in list(docs.keys()): + checker = create_checker(args, filename, docs[filename]) + checker.check() + + args.reporter.report() + if args.reporter.messages and not args.permissive: + exit(1) + + +def parse_args(): + """Parse command-line arguments.""" + + parser = ArgumentParser(description="""Check episode files in a lesson.""") + parser.add_argument('-l', '--linelen', + default=False, + action="store_true", + dest='line_lengths', + help='Check line lengths') + parser.add_argument('-p', '--parser', + default=None, + dest='parser', + help='path to Markdown parser') + parser.add_argument('-r', '--references', + default=None, + dest='reference_path', + help='path to Markdown file of external references') + parser.add_argument('-s', '--source', + default=os.curdir, + dest='source_dir', + help='source directory') + parser.add_argument('-w', '--whitespace', + default=False, + action="store_true", + dest='trailing_whitespace', + help='Check for trailing whitespace') + parser.add_argument('--permissive', + default=False, + action="store_true", + dest='permissive', + help='Do not raise an error even if issues are detected') + + args, extras = parser.parse_known_args() + require(args.parser is not None, + 'Path to Markdown parser not provided') + require(not extras, + 'Unexpected trailing command-line arguments "{0}"'.format(extras)) + + return args + + +def check_config(reporter, source_dir): + """Check configuration file.""" + + config_file = os.path.join(source_dir, '_config.yml') + config = load_yaml(config_file) + reporter.check_field(config_file, 'configuration', + config, 'kind', 'lesson') + reporter.check_field(config_file, 'configuration', + config, 'carpentry', ('swc', 'dc', 'lc', 'cp')) + reporter.check_field(config_file, 'configuration', config, 'title') + reporter.check_field(config_file, 'configuration', config, 'email') + + for defaults in [ + {'values': {'root': '.', 'layout': 'page'}}, + {'values': {'root': '..', 'layout': 'episode'}, 'scope': {'type': 'episodes', 'path': ''}}, + {'values': {'root': '..', 'layout': 'page'}, 'scope': {'type': 'extras', 'path': ''}} + ]: + reporter.check(defaults in config.get('defaults', []), + 'configuration', + '"root" not set to "." in configuration') + +def check_source_rmd(reporter, source_dir, parser): + """Check that Rmd episode files include `source: Rmd`""" + + episode_rmd_dir = [os.path.join(source_dir, d) for d in SOURCE_RMD_DIRS] + episode_rmd_files = [os.path.join(d, '*.Rmd') for d in episode_rmd_dir] + results = {} + for pat in episode_rmd_files: + for f in glob.glob(pat): + data = read_markdown(parser, f) + dy = data['metadata'] + if dy: + reporter.check_field(f, 'episode_rmd', + dy, 'source', 'Rmd') + +def read_references(reporter, ref_path): + """Read shared file of reference links, returning dictionary of valid references + {symbolic_name : URL} + """ + + if not ref_path: + raise Warning("No filename has been provided.") + + result = {} + urls_seen = set() + + with open(ref_path, 'r', encoding='utf-8') as reader: + for (num, line) in enumerate(reader, 1): + + if P_INTERNAL_INCLUDE_LINK.search(line): continue + + m = P_INTERNAL_LINK_DEF.search(line) + + message = '{}: {} not a valid reference: {}' + require(m, message.format(ref_path, num, line.rstrip())) + + name = m.group(1) + url = m.group(2) + + message = 'Empty reference at {0}:{1}' + require(name, message.format(ref_path, num)) + + unique_name = name not in result + unique_url = url not in urls_seen + + reporter.check(unique_name, + ref_path, + 'Duplicate reference name {0} at line {1}', + name, num) + + reporter.check(unique_url, + ref_path, + 'Duplicate definition of URL {0} at line {1}', + url, num) + + result[name] = url + urls_seen.add(url) + + return result + + +def read_all_markdown(source_dir, parser): + """Read source files, returning + {path : {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}} + """ + + all_dirs = [os.path.join(source_dir, d) for d in SOURCE_DIRS] + all_patterns = [os.path.join(d, '*.md') for d in all_dirs] + result = {} + for pat in all_patterns: + for filename in glob.glob(pat): + data = read_markdown(parser, filename) + if data: + result[filename] = data + return result + + +def check_fileset(source_dir, reporter, filenames_present): + """Are all required files present? Are extraneous files present?""" + + # Check files with predictable names. + required = [os.path.join(source_dir, p) for p in REQUIRED_FILES] + missing = set(required) - set(filenames_present) + for m in missing: + reporter.add(None, 'Missing required file {0}', m) + + # Check episode files' names. + seen = [] + for filename in filenames_present: + if '_episodes' not in filename: + continue + + # split path to check episode name + base_name = os.path.basename(filename) + m = P_EPISODE_FILENAME.search(base_name) + if m and m.group(1): + seen.append(m.group(1)) + else: + reporter.add( + None, 'Episode {0} has badly-formatted filename', filename) + + # Check for duplicate episode numbers. + reporter.check(len(seen) == len(set(seen)), + None, + 'Duplicate episode numbers {0} vs {1}', + sorted(seen), sorted(set(seen))) + + # Check that numbers are consecutive. + seen = sorted([int(s) for s in seen]) + clean = True + for i in range(len(seen) - 1): + clean = clean and ((seen[i+1] - seen[i]) == 1) + reporter.check(clean, + None, + 'Missing or non-consecutive episode numbers {0}', + seen) + + +def create_checker(args, filename, info): + """Create appropriate checker for file.""" + + for (pat, cls) in CHECKERS: + if pat.search(filename): + return cls(args, filename, **info) + return NotImplemented + +class CheckBase: + """Base class for checking Markdown files.""" + + def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): + """Cache arguments for checking.""" + + self.args = args + self.reporter = self.args.reporter # for convenience + self.filename = filename + self.metadata = metadata + self.metadata_len = metadata_len + self.text = text + self.lines = lines + self.doc = doc + + self.layout = None + + def check(self): + """Run tests.""" + + self.check_metadata() + self.check_line_lengths() + self.check_trailing_whitespace() + self.check_blockquote_classes() + self.check_codeblock_classes() + self.check_defined_link_references() + + def check_metadata(self): + """Check the YAML metadata.""" + + self.reporter.check(self.metadata is not None, + self.filename, + 'Missing metadata entirely') + + if self.metadata and (self.layout is not None): + self.reporter.check_field( + self.filename, 'metadata', self.metadata, 'layout', self.layout) + + def check_line_lengths(self): + """Check the raw text of the lesson body.""" + + if self.args.line_lengths: + over = [i for (i, l, n) in self.lines if ( + n > MAX_LINE_LEN) and (not l.startswith('!'))] + self.reporter.check(not over, + self.filename, + 'Line(s) too long: {0}', + ', '.join([str(i) for i in over])) + + def check_trailing_whitespace(self): + """Check for whitespace at the ends of lines.""" + + if self.args.trailing_whitespace: + trailing = [ + i for (i, l, n) in self.lines if P_TRAILING_WHITESPACE.match(l)] + self.reporter.check(not trailing, + self.filename, + 'Line(s) end with whitespace: {0}', + ', '.join([str(i) for i in trailing])) + + def check_blockquote_classes(self): + """Check that all blockquotes have known classes.""" + + for node in self.find_all(self.doc, {'type': 'blockquote'}): + cls = self.get_val(node, 'attr', 'class') + self.reporter.check(cls in KNOWN_BLOCKQUOTES, + (self.filename, self.get_loc(node)), + 'Unknown or missing blockquote type {0}', + cls) + + def check_codeblock_classes(self): + """Check that all code blocks have known classes.""" + + for node in self.find_all(self.doc, {'type': 'codeblock'}): + cls = self.get_val(node, 'attr', 'class') + self.reporter.check(cls in KNOWN_CODEBLOCKS, + (self.filename, self.get_loc(node)), + 'Unknown or missing code block type {0}', + cls) + + def check_defined_link_references(self): + """Check that defined links resolve in the file. + + Internally-defined links match the pattern [text][label]. + """ + + result = set() + for node in self.find_all(self.doc, {'type': 'text'}): + for match in P_INTERNAL_LINK_REF.findall(node['value']): + text = match[0] + link = match[1] + if link not in self.args.references: + result.add('"{0}"=>"{1}"'.format(text, link)) + self.reporter.check(not result, + self.filename, + 'Internally-defined links may be missing definitions: {0}', + ', '.join(sorted(result))) + + def find_all(self, node, pattern, accum=None): + """Find all matches for a pattern.""" + + assert isinstance(pattern, dict), 'Patterns must be dictionaries' + if accum is None: + accum = [] + if self.match(node, pattern): + accum.append(node) + for child in node.get('children', []): + self.find_all(child, pattern, accum) + return accum + + def match(self, node, pattern): + """Does this node match the given pattern?""" + + for key in pattern: + if key not in node: + return False + val = pattern[key] + if isinstance(val, str): + if node[key] != val: + return False + elif isinstance(val, dict): + if not self.match(node[key], val): + return False + return True + + @staticmethod + def get_val(node, *chain): + """Get value one or more levels down.""" + + curr = node + for selector in chain: + curr = curr.get(selector, None) + if curr is None: + break + return curr + + def get_loc(self, node): + """Convenience method to get node's line number.""" + + result = self.get_val(node, 'options', 'location') + if self.metadata_len is not None: + result += self.metadata_len + return result + + +class CheckNonJekyll(CheckBase): + """Check a file that isn't translated by Jekyll.""" + + def check_metadata(self): + self.reporter.check(self.metadata is None, + self.filename, + 'Unexpected metadata') + + +class CheckIndex(CheckBase): + """Check the main index page.""" + + def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): + super().__init__(args, filename, metadata, metadata_len, text, lines, doc) + self.layout = 'lesson' + + def check_metadata(self): + super().check_metadata() + self.reporter.check(self.metadata.get('root', '') == '.', + self.filename, + 'Root not set to "."') + + +class CheckEpisode(CheckBase): + """Check an episode page.""" + + def check(self): + """Run extra tests.""" + + super().check() + self.check_reference_inclusion() + + def check_metadata(self): + super().check_metadata() + if self.metadata: + if 'layout' in self.metadata: + if self.metadata['layout'] == 'break': + self.check_metadata_fields(BREAK_METADATA_FIELDS) + else: + self.reporter.add(self.filename, + 'Unknown episode layout "{0}"', + self.metadata['layout']) + else: + self.check_metadata_fields(TEACHING_METADATA_FIELDS) + + def check_metadata_fields(self, expected): + """Check metadata fields.""" + for (name, type_) in expected: + if name not in self.metadata: + self.reporter.add(self.filename, + 'Missing metadata field {0}', + name) + elif not isinstance(self.metadata[name], type_): + self.reporter.add(self.filename, + '"{0}" has wrong type in metadata ({1} instead of {2})', + name, type(self.metadata[name]), type_) + + def check_reference_inclusion(self): + """Check that links file has been included.""" + + if not self.args.reference_path: + return + + for (i, last_line, line_len) in reversed(self.lines): + if last_line: + break + + require(last_line, + 'No non-empty lines in {0}'.format(self.filename)) + + include_filename = os.path.split(self.args.reference_path)[-1] + if include_filename not in last_line: + self.reporter.add(self.filename, + 'episode does not include "{0}"', + include_filename) + + +class CheckReference(CheckBase): + """Check the reference page.""" + + def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): + super().__init__(args, filename, metadata, metadata_len, text, lines, doc) + self.layout = 'reference' + + +class CheckGeneric(CheckBase): + """Check a generic page.""" + + def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): + super().__init__(args, filename, metadata, metadata_len, text, lines, doc) + + +CHECKERS = [ + (re.compile(r'CONTRIBUTING\.md'), CheckNonJekyll), + (re.compile(r'README\.md'), CheckNonJekyll), + (re.compile(r'index\.md'), CheckIndex), + (re.compile(r'reference\.md'), CheckReference), + (re.compile(os.path.join('_episodes', '*\.md')), CheckEpisode), + (re.compile(r'.*\.md'), CheckGeneric) +] + + +if __name__ == '__main__': + main() diff --git a/bin/lesson_initialize.py b/bin/lesson_initialize.py new file mode 100644 index 00000000..2f7b8e67 --- /dev/null +++ b/bin/lesson_initialize.py @@ -0,0 +1,48 @@ +"""Initialize a newly-created repository.""" + + +import sys +import os +import shutil + +BOILERPLATE = ( + '.travis.yml', + 'AUTHORS', + 'CITATION', + 'CONTRIBUTING.md', + 'README.md', + '_config.yml', + os.path.join('_episodes', '01-introduction.md'), + os.path.join('_extras', 'about.md'), + os.path.join('_extras', 'discuss.md'), + os.path.join('_extras', 'figures.md'), + os.path.join('_extras', 'guide.md'), + 'index.md', + 'reference.md', + 'setup.md', +) + + +def main(): + """Check for collisions, then create.""" + + # Check. + errors = False + for path in BOILERPLATE: + if os.path.exists(path): + print('Warning: {0} already exists.'.format(path), file=sys.stderr) + errors = True + if errors: + print('**Exiting without creating files.**', file=sys.stderr) + sys.exit(1) + + # Create. + for path in BOILERPLATE: + shutil.copyfile( + os.path.join('bin', 'boilerplate', path), + path + ) + + +if __name__ == '__main__': + main() diff --git a/bin/markdown_ast.rb b/bin/markdown_ast.rb new file mode 100755 index 00000000..c3fd0b5e --- /dev/null +++ b/bin/markdown_ast.rb @@ -0,0 +1,11 @@ +#!/usr/bin/env ruby + +# Use Kramdown parser to produce AST for Markdown document. + +require "kramdown" +require "json" + +markdown = STDIN.read() +doc = Kramdown::Document.new(markdown) +tree = doc.to_hash_a_s_t +puts JSON.pretty_generate(tree) diff --git a/bin/repo_check.py b/bin/repo_check.py new file mode 100644 index 00000000..9bf5c597 --- /dev/null +++ b/bin/repo_check.py @@ -0,0 +1,180 @@ +""" +Check repository settings. +""" + + +import sys +import os +from subprocess import Popen, PIPE +import re +from argparse import ArgumentParser + +from util import Reporter, require + +# Import this way to produce a more useful error message. +try: + import requests +except ImportError: + print('Unable to import requests module: please install requests', file=sys.stderr) + sys.exit(1) + + +# Pattern to match Git command-line output for remotes => (user name, project name). +P_GIT_REMOTE = re.compile(r'upstream\s+(?:https://|git@)github.com[:/]([^/]+)/([^.]+)(\.git)?\s+\(fetch\)') + +# Repository URL format string. +F_REPO_URL = 'https://github.com/{0}/{1}/' + +# Pattern to match repository URLs => (user name, project name) +P_REPO_URL = re.compile(r'https?://github\.com/([^.]+)/([^/]+)/?') + +# API URL format string. +F_API_URL = 'https://api.github.com/repos/{0}/{1}/labels' + +# Expected labels and colors. +EXPECTED = { + 'help wanted': 'dcecc7', + 'status:in progress': '9bcc65', + 'status:changes requested': '679f38', + 'status:wait': 'fff2df', + 'status:refer to cac': 'ffdfb2', + 'status:need more info': 'ee6c00', + 'status:blocked': 'e55100', + 'status:out of scope': 'eeeeee', + 'status:duplicate': 'bdbdbd', + 'type:typo text': 'f8bad0', + 'type:bug': 'eb3f79', + 'type:formatting': 'ac1357', + 'type:template and tools': '7985cb', + 'type:instructor guide': '00887a', + 'type:discussion': 'b2e5fc', + 'type:enhancement': '7fdeea', + 'type:clarification': '00acc0', + 'type:teaching example': 'ced8dc', + 'good first issue': 'ffeb3a', + 'high priority': 'd22e2e' +} + + +def main(): + """ + Main driver. + """ + + args = parse_args() + reporter = Reporter() + repo_url = get_repo_url(args.repo_url) + check_labels(reporter, repo_url) + reporter.report() + + +def parse_args(): + """ + Parse command-line arguments. + """ + + parser = ArgumentParser(description="""Check repository settings.""") + parser.add_argument('-r', '--repo', + default=None, + dest='repo_url', + help='repository URL') + parser.add_argument('-s', '--source', + default=os.curdir, + dest='source_dir', + help='source directory') + + args, extras = parser.parse_known_args() + require(not extras, + 'Unexpected trailing command-line arguments "{0}"'.format(extras)) + + return args + + +def get_repo_url(repo_url): + """ + Figure out which repository to query. + """ + + # Explicitly specified. + if repo_url is not None: + return repo_url + + # Guess. + cmd = 'git remote -v' + p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, + close_fds=True, universal_newlines=True, encoding='utf-8') + stdout_data, stderr_data = p.communicate() + stdout_data = stdout_data.split('\n') + matches = [P_GIT_REMOTE.match(line) for line in stdout_data] + matches = [m for m in matches if m is not None] + require(len(matches) == 1, + 'Unexpected output from git remote command: "{0}"'.format(matches)) + + username = matches[0].group(1) + require( + username, 'empty username in git remote output {0}'.format(matches[0])) + + project_name = matches[0].group(2) + require( + username, 'empty project name in git remote output {0}'.format(matches[0])) + + url = F_REPO_URL.format(username, project_name) + return url + + +def check_labels(reporter, repo_url): + """ + Check labels in repository. + """ + + actual = get_labels(repo_url) + extra = set(actual.keys()) - set(EXPECTED.keys()) + + reporter.check(not extra, + None, + 'Extra label(s) in repository {0}: {1}', + repo_url, ', '.join(sorted(extra))) + + missing = set(EXPECTED.keys()) - set(actual.keys()) + reporter.check(not missing, + None, + 'Missing label(s) in repository {0}: {1}', + repo_url, ', '.join(sorted(missing))) + + overlap = set(EXPECTED.keys()).intersection(set(actual.keys())) + for name in sorted(overlap): + reporter.check(EXPECTED[name].lower() == actual[name].lower(), + None, + 'Color mis-match for label {0} in {1}: expected {2}, found {3}', + name, repo_url, EXPECTED[name], actual[name]) + + +def get_labels(repo_url): + """ + Get actual labels from repository. + """ + + m = P_REPO_URL.match(repo_url) + require( + m, 'repository URL {0} does not match expected pattern'.format(repo_url)) + + username = m.group(1) + require(username, 'empty username in repository URL {0}'.format(repo_url)) + + project_name = m.group(2) + require( + username, 'empty project name in repository URL {0}'.format(repo_url)) + + url = F_API_URL.format(username, project_name) + r = requests.get(url) + require(r.status_code == 200, + 'Request for {0} failed with {1}'.format(url, r.status_code)) + + result = {} + for entry in r.json(): + result[entry['name']] = entry['color'] + return result + + +if __name__ == '__main__': + main() diff --git a/bin/run-make-docker-serve.sh b/bin/run-make-docker-serve.sh new file mode 100755 index 00000000..1e091781 --- /dev/null +++ b/bin/run-make-docker-serve.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + + +bundle install +bundle update +exec bundle exec jekyll serve --host 0.0.0.0 diff --git a/bin/test_lesson_check.py b/bin/test_lesson_check.py new file mode 100644 index 00000000..0981720a --- /dev/null +++ b/bin/test_lesson_check.py @@ -0,0 +1,19 @@ +import unittest + +import lesson_check +import util + + +class TestFileList(unittest.TestCase): + def setUp(self): + self.reporter = util.Reporter() # TODO: refactor reporter class. + + def test_file_list_has_expected_entries(self): + # For first pass, simply assume that all required files are present + + lesson_check.check_fileset('', self.reporter, lesson_check.REQUIRED_FILES) + self.assertEqual(len(self.reporter.messages), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/bin/util.py b/bin/util.py new file mode 100644 index 00000000..0e16d869 --- /dev/null +++ b/bin/util.py @@ -0,0 +1,188 @@ +import sys +import os +import json +from subprocess import Popen, PIPE + +# Import this way to produce a more useful error message. +try: + import yaml +except ImportError: + print('Unable to import YAML module: please install PyYAML', file=sys.stderr) + sys.exit(1) + + +# Things an image file's name can end with. +IMAGE_FILE_SUFFIX = { + '.gif', + '.jpg', + '.png', + '.svg' +} + +# Files that shouldn't be present. +UNWANTED_FILES = [ + '.nojekyll' +] + +# Marker to show that an expected value hasn't been provided. +# (Can't use 'None' because that might be a legitimate value.) +REPORTER_NOT_SET = [] + + +class Reporter: + """Collect and report errors.""" + + def __init__(self): + """Constructor.""" + self.messages = [] + + def check_field(self, filename, name, values, key, expected=REPORTER_NOT_SET): + """Check that a dictionary has an expected value.""" + + if key not in values: + self.add(filename, '{0} does not contain {1}', name, key) + elif expected is REPORTER_NOT_SET: + pass + elif type(expected) in (tuple, set, list): + if values[key] not in expected: + self.add( + filename, '{0} {1} value {2} is not in {3}', name, key, values[key], expected) + elif values[key] != expected: + self.add(filename, '{0} {1} is {2} not {3}', + name, key, values[key], expected) + + def check(self, condition, location, fmt, *args): + """Append error if condition not met.""" + + if not condition: + self.add(location, fmt, *args) + + def add(self, location, fmt, *args): + """Append error unilaterally.""" + + self.messages.append((location, fmt.format(*args))) + + @staticmethod + def pretty(item): + location, message = item + if isinstance(location, type(None)): + return message + elif isinstance(location, str): + return location + ': ' + message + elif isinstance(location, tuple): + return '{0}:{1}: '.format(*location) + message + + print('Unknown item "{0}"'.format(item), file=sys.stderr) + return NotImplemented + + @staticmethod + def key(item): + location, message = item + if isinstance(location, type(None)): + return ('', -1, message) + elif isinstance(location, str): + return (location, -1, message) + elif isinstance(location, tuple): + return (location[0], location[1], message) + + print('Unknown item "{0}"'.format(item), file=sys.stderr) + return NotImplemented + + def report(self, stream=sys.stdout): + """Report all messages in order.""" + + if not self.messages: + return + + for m in sorted(self.messages, key=self.key): + print(self.pretty(m), file=stream) + + +def read_markdown(parser, path): + """ + Get YAML and AST for Markdown file, returning + {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}. + """ + + # Split and extract YAML (if present). + with open(path, 'r', encoding='utf-8') as reader: + body = reader.read() + metadata_raw, metadata_yaml, body = split_metadata(path, body) + + # Split into lines. + metadata_len = 0 if metadata_raw is None else metadata_raw.count('\n') + lines = [(metadata_len+i+1, line, len(line)) + for (i, line) in enumerate(body.split('\n'))] + + # Parse Markdown. + cmd = 'ruby {0}'.format(parser) + p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, + close_fds=True, universal_newlines=True, encoding='utf-8') + stdout_data, stderr_data = p.communicate(body) + doc = json.loads(stdout_data) + + return { + 'metadata': metadata_yaml, + 'metadata_len': metadata_len, + 'text': body, + 'lines': lines, + 'doc': doc + } + + +def split_metadata(path, text): + """ + Get raw (text) metadata, metadata as YAML, and rest of body. + If no metadata, return (None, None, body). + """ + + metadata_raw = None + metadata_yaml = None + + pieces = text.split('---', 2) + if len(pieces) == 3: + metadata_raw = pieces[1] + text = pieces[2] + try: + metadata_yaml = yaml.load(metadata_raw, Loader=yaml.SafeLoader) + except yaml.YAMLError as e: + print('Unable to parse YAML header in {0}:\n{1}'.format( + path, e), file=sys.stderr) + sys.exit(1) + + return metadata_raw, metadata_yaml, text + + +def load_yaml(filename): + """ + Wrapper around YAML loading so that 'import yaml' is only needed + in one file. + """ + + try: + with open(filename, 'r', encoding='utf-8') as reader: + return yaml.load(reader, Loader=yaml.SafeLoader) + except (yaml.YAMLError, IOError) as e: + print('Unable to load YAML file {0}:\n{1}'.format( + filename, e), file=sys.stderr) + sys.exit(1) + + +def check_unwanted_files(dir_path, reporter): + """ + Check that unwanted files are not present. + """ + + for filename in UNWANTED_FILES: + path = os.path.join(dir_path, filename) + reporter.check(not os.path.exists(path), + path, + "Unwanted file found") + + +def require(condition, message): + """Fail if condition not met.""" + + if not condition: + print(message, file=sys.stderr) + sys.exit(1) diff --git a/bin/workshop_check.py b/bin/workshop_check.py new file mode 100644 index 00000000..15d954a6 --- /dev/null +++ b/bin/workshop_check.py @@ -0,0 +1,418 @@ +'''Check that a workshop's index.html metadata is valid. See the +docstrings on the checking functions for a summary of the checks. +''' + + +import sys +import os +import re +from datetime import date +from util import Reporter, split_metadata, load_yaml, check_unwanted_files + +# Metadata field patterns. +EMAIL_PATTERN = r'[^@]+@[^@]+\.[^@]+' +HUMANTIME_PATTERN = r'((0?[1-9]|1[0-2]):[0-5]\d(am|pm)(-|to)(0?[1-9]|1[0-2]):[0-5]\d(am|pm))|((0?\d|1\d|2[0-3]):[0-5]\d(-|to)(0?\d|1\d|2[0-3]):[0-5]\d)' +EVENTBRITE_PATTERN = r'\d{9,10}' +URL_PATTERN = r'https?://.+' + +# Defaults. +CARPENTRIES = ("dc", "swc", "lc", "cp") +DEFAULT_CONTACT_EMAIL = 'admin@software-carpentry.org' + +USAGE = 'Usage: "workshop_check.py path/to/root/directory"' + +# Country and language codes. Note that codes mean different things: 'ar' +# is 'Arabic' as a language but 'Argentina' as a country. + +ISO_COUNTRY = [ + 'ad', 'ae', 'af', 'ag', 'ai', 'al', 'am', 'an', 'ao', 'aq', 'ar', 'as', + 'at', 'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd', 'be', 'bf', 'bg', 'bh', + 'bi', 'bj', 'bm', 'bn', 'bo', 'br', 'bs', 'bt', 'bv', 'bw', 'by', 'bz', + 'ca', 'cc', 'cd', 'cf', 'cg', 'ch', 'ci', 'ck', 'cl', 'cm', 'cn', 'co', + 'cr', 'cu', 'cv', 'cx', 'cy', 'cz', 'de', 'dj', 'dk', 'dm', 'do', 'dz', + 'ec', 'ee', 'eg', 'eh', 'er', 'es', 'et', 'eu', 'fi', 'fj', 'fk', 'fm', + 'fo', 'fr', 'ga', 'gb', 'gd', 'ge', 'gf', 'gg', 'gh', 'gi', 'gl', 'gm', + 'gn', 'gp', 'gq', 'gr', 'gs', 'gt', 'gu', 'gw', 'gy', 'hk', 'hm', 'hn', + 'hr', 'ht', 'hu', 'id', 'ie', 'il', 'im', 'in', 'io', 'iq', 'ir', 'is', + 'it', 'je', 'jm', 'jo', 'jp', 'ke', 'kg', 'kh', 'ki', 'km', 'kn', 'kp', + 'kr', 'kw', 'ky', 'kz', 'la', 'lb', 'lc', 'li', 'lk', 'lr', 'ls', 'lt', + 'lu', 'lv', 'ly', 'ma', 'mc', 'md', 'me', 'mg', 'mh', 'mk', 'ml', 'mm', + 'mn', 'mo', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu', 'mv', 'mw', 'mx', 'my', + 'mz', 'na', 'nc', 'ne', 'nf', 'ng', 'ni', 'nl', 'no', 'np', 'nr', 'nu', + 'nz', 'om', 'pa', 'pe', 'pf', 'pg', 'ph', 'pk', 'pl', 'pm', 'pn', 'pr', + 'ps', 'pt', 'pw', 'py', 'qa', 're', 'ro', 'rs', 'ru', 'rw', 'sa', 'sb', + 'sc', 'sd', 'se', 'sg', 'sh', 'si', 'sj', 'sk', 'sl', 'sm', 'sn', 'so', + 'sr', 'st', 'sv', 'sy', 'sz', 'tc', 'td', 'tf', 'tg', 'th', 'tj', 'tk', + 'tl', 'tm', 'tn', 'to', 'tr', 'tt', 'tv', 'tw', 'tz', 'ua', 'ug', 'um', + 'us', 'uy', 'uz', 'va', 'vc', 've', 'vg', 'vi', 'vn', 'vu', 'wf', 'ws', + 'ye', 'yt', 'za', 'zm', 'zw' +] + +ISO_LANGUAGE = [ + 'aa', 'ab', 'ae', 'af', 'ak', 'am', 'an', 'ar', 'as', 'av', 'ay', 'az', + 'ba', 'be', 'bg', 'bh', 'bi', 'bm', 'bn', 'bo', 'br', 'bs', 'ca', 'ce', + 'ch', 'co', 'cr', 'cs', 'cu', 'cv', 'cy', 'da', 'de', 'dv', 'dz', 'ee', + 'el', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'ff', 'fi', 'fj', 'fo', 'fr', + 'fy', 'ga', 'gd', 'gl', 'gn', 'gu', 'gv', 'ha', 'he', 'hi', 'ho', 'hr', + 'ht', 'hu', 'hy', 'hz', 'ia', 'id', 'ie', 'ig', 'ii', 'ik', 'io', 'is', + 'it', 'iu', 'ja', 'jv', 'ka', 'kg', 'ki', 'kj', 'kk', 'kl', 'km', 'kn', + 'ko', 'kr', 'ks', 'ku', 'kv', 'kw', 'ky', 'la', 'lb', 'lg', 'li', 'ln', + 'lo', 'lt', 'lu', 'lv', 'mg', 'mh', 'mi', 'mk', 'ml', 'mn', 'mr', 'ms', + 'mt', 'my', 'na', 'nb', 'nd', 'ne', 'ng', 'nl', 'nn', 'no', 'nr', 'nv', + 'ny', 'oc', 'oj', 'om', 'or', 'os', 'pa', 'pi', 'pl', 'ps', 'pt', 'qu', + 'rm', 'rn', 'ro', 'ru', 'rw', 'sa', 'sc', 'sd', 'se', 'sg', 'si', 'sk', + 'sl', 'sm', 'sn', 'so', 'sq', 'sr', 'ss', 'st', 'su', 'sv', 'sw', 'ta', + 'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tn', 'to', 'tr', 'ts', 'tt', 'tw', + 'ty', 'ug', 'uk', 'ur', 'uz', 've', 'vi', 'vo', 'wa', 'wo', 'xh', 'yi', + 'yo', 'za', 'zh', 'zu' +] + + +def look_for_fixme(func): + """Decorator to fail test if text argument starts with "FIXME".""" + + def inner(arg): + if (arg is not None) and \ + isinstance(arg, str) and \ + arg.lstrip().startswith('FIXME'): + return False + return func(arg) + return inner + + +@look_for_fixme +def check_layout(layout): + '''"layout" in YAML header must be "workshop".''' + + return layout == 'workshop' + + +@look_for_fixme +def check_carpentry(layout): + '''"carpentry" in YAML header must be "dc", "swc", "lc", or "cp".''' + + return layout in CARPENTRIES + + +@look_for_fixme +def check_country(country): + '''"country" must be a lowercase ISO-3166 two-letter code.''' + + return country in ISO_COUNTRY + + +@look_for_fixme +def check_language(language): + '''"language" must be a lowercase ISO-639 two-letter code.''' + + return language in ISO_LANGUAGE + + +@look_for_fixme +def check_humandate(date): + """ + 'humandate' must be a human-readable date with a 3-letter month + and 4-digit year. Examples include 'Feb 18-20, 2025' and 'Feb 18 + and 20, 2025'. It may be in languages other than English, but the + month name should be kept short to aid formatting of the main + Carpentries web site. + """ + + if ',' not in date: + return False + + month_dates, year = date.split(',') + + # The first three characters of month_dates are not empty + month = month_dates[:3] + if any(char == ' ' for char in month): + return False + + # But the fourth character is empty ("February" is illegal) + if month_dates[3] != ' ': + return False + + # year contains *only* numbers + try: + int(year) + except: + return False + + return True + + +@look_for_fixme +def check_humantime(time): + """ + 'humantime' is a human-readable start and end time for the + workshop, such as '09:00 - 16:00'. + """ + + return bool(re.match(HUMANTIME_PATTERN, time.replace(' ', ''))) + + +def check_date(this_date): + """ + 'startdate' and 'enddate' are machine-readable start and end dates + for the workshop, and must be in YYYY-MM-DD format, e.g., + '2015-07-01'. + """ + + # YAML automatically loads valid dates as datetime.date. + return isinstance(this_date, date) + + +@look_for_fixme +def check_latitude_longitude(latlng): + """ + 'latlng' must be a valid latitude and longitude represented as two + floating-point numbers separated by a comma. + """ + + try: + lat, lng = latlng.split(',') + lat = float(lat) + lng = float(lng) + return (-90.0 <= lat <= 90.0) and (-180.0 <= lng <= 180.0) + except ValueError: + return False + + +def check_instructors(instructors): + """ + 'instructor' must be a non-empty comma-separated list of quoted + names, e.g. ['First name', 'Second name', ...']. Do not use 'TBD' + or other placeholders. + """ + + # YAML automatically loads list-like strings as lists. + return isinstance(instructors, list) and len(instructors) > 0 + + +def check_helpers(helpers): + """ + 'helper' must be a comma-separated list of quoted names, + e.g. ['First name', 'Second name', ...']. The list may be empty. + Do not use 'TBD' or other placeholders. + """ + + # YAML automatically loads list-like strings as lists. + return isinstance(helpers, list) and len(helpers) >= 0 + + +@look_for_fixme +def check_emails(emails): + """ + 'emails' must be a comma-separated list of valid email addresses. + The list may be empty. A valid email address consists of characters, + an '@', and more characters. It should not contain the default contact + """ + + # YAML automatically loads list-like strings as lists. + if (isinstance(emails, list) and len(emails) >= 0): + for email in emails: + if ((not bool(re.match(EMAIL_PATTERN, email))) or (email == DEFAULT_CONTACT_EMAIL)): + return False + else: + return False + + return True + + +def check_eventbrite(eventbrite): + """ + 'eventbrite' (the Eventbrite registration key) must be 9 or more + digits. It may appear as an integer or as a string. + """ + + if isinstance(eventbrite, int): + return True + else: + return bool(re.match(EVENTBRITE_PATTERN, eventbrite)) + + +@look_for_fixme +def check_collaborative_notes(collaborative_notes): + """ + 'collaborative_notes' must be a valid URL. + """ + + return bool(re.match(URL_PATTERN, collaborative_notes)) + + +@look_for_fixme +def check_pass(value): + """ + This test always passes (it is used for 'checking' things like the + workshop address, for which no sensible validation is feasible). + """ + + return True + + +HANDLERS = { + 'layout': (True, check_layout, 'layout isn\'t "workshop"'), + + 'carpentry': (True, check_carpentry, 'carpentry isn\'t in ' + + ', '.join(CARPENTRIES)), + + 'country': (True, check_country, + 'country invalid: must use lowercase two-letter ISO code ' + + 'from ' + ', '.join(ISO_COUNTRY)), + + 'language': (False, check_language, + 'language invalid: must use lowercase two-letter ISO code' + + ' from ' + ', '.join(ISO_LANGUAGE)), + + 'humandate': (True, check_humandate, + 'humandate invalid. Please use three-letter months like ' + + '"Jan" and four-letter years like "2025"'), + + 'humantime': (True, check_humantime, + 'humantime doesn\'t include numbers'), + + 'startdate': (True, check_date, + 'startdate invalid. Must be of format year-month-day, ' + + 'i.e., 2014-01-31'), + + 'enddate': (False, check_date, + 'enddate invalid. Must be of format year-month-day, i.e.,' + + ' 2014-01-31'), + + 'latlng': (True, check_latitude_longitude, + 'latlng invalid. Check that it is two floating point ' + + 'numbers, separated by a comma'), + + 'instructor': (True, check_instructors, + 'instructor list isn\'t a valid list of format ' + + '["First instructor", "Second instructor",..]'), + + 'helper': (True, check_helpers, + 'helper list isn\'t a valid list of format ' + + '["First helper", "Second helper",..]'), + + 'email': (True, check_emails, + 'contact email list isn\'t a valid list of format ' + + '["me@example.org", "you@example.org",..] or contains incorrectly formatted email addresses or ' + + '"{0}".'.format(DEFAULT_CONTACT_EMAIL)), + + 'eventbrite': (False, check_eventbrite, 'Eventbrite key appears invalid'), + + 'collaborative_notes': (False, check_collaborative_notes, 'Collaborative Notes URL appears invalid'), + + 'venue': (False, check_pass, 'venue name not specified'), + + 'address': (False, check_pass, 'address not specified') +} + +# REQUIRED is all required categories. +REQUIRED = {k for k in HANDLERS if HANDLERS[k][0]} + +# OPTIONAL is all optional categories. +OPTIONAL = {k for k in HANDLERS if not HANDLERS[k][0]} + + +def check_blank_lines(reporter, raw): + """ + Blank lines are not allowed in category headers. + """ + + lines = [(i, x) for (i, x) in enumerate( + raw.strip().split('\n')) if not x.strip()] + reporter.check(not lines, + None, + 'Blank line(s) in header: {0}', + ', '.join(["{0}: {1}".format(i, x.rstrip()) for (i, x) in lines])) + + +def check_categories(reporter, left, right, msg): + """ + Report differences (if any) between two sets of categories. + """ + + diff = left - right + reporter.check(len(diff) == 0, + None, + '{0}: offending entries {1}', + msg, sorted(list(diff))) + + +def check_file(reporter, path, data): + """ + Get header from file, call all other functions, and check file for + validity. + """ + + # Get metadata as text and as YAML. + raw, header, body = split_metadata(path, data) + + # Do we have any blank lines in the header? + check_blank_lines(reporter, raw) + + # Look through all header entries. If the category is in the input + # file and is either required or we have actual data (as opposed to + # a commented-out entry), we check it. If it *isn't* in the header + # but is required, report an error. + for category in HANDLERS: + required, handler, message = HANDLERS[category] + if category in header: + if required or header[category]: + reporter.check(handler(header[category]), + None, + '{0}\n actual value "{1}"', + message, header[category]) + elif required: + reporter.add(None, + 'Missing mandatory key "{0}"', + category) + + # Check whether we have missing or too many categories + seen_categories = set(header.keys()) + check_categories(reporter, REQUIRED, seen_categories, + 'Missing categories') + check_categories(reporter, seen_categories, REQUIRED.union(OPTIONAL), + 'Superfluous categories') + + +def check_config(reporter, filename): + """ + Check YAML configuration file. + """ + + config = load_yaml(filename) + + kind = config.get('kind', None) + reporter.check(kind == 'workshop', + filename, + 'Missing or unknown kind of event: {0}', + kind) + + carpentry = config.get('carpentry', None) + reporter.check(carpentry in ('swc', 'dc', 'lc', 'cp'), + filename, + 'Missing or unknown carpentry: {0}', + carpentry) + + +def main(): + '''Run as the main program.''' + + if len(sys.argv) != 2: + print(USAGE, file=sys.stderr) + sys.exit(1) + + root_dir = sys.argv[1] + index_file = os.path.join(root_dir, 'index.html') + config_file = os.path.join(root_dir, '_config.yml') + + reporter = Reporter() + check_config(reporter, config_file) + check_unwanted_files(root_dir, reporter) + with open(index_file, encoding='utf-8') as reader: + data = reader.read() + check_file(reporter, index_file, data) + reporter.report() + + +if __name__ == '__main__': + main() diff --git a/code/.gitkeep b/code/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/fig/.gitkeep b/fig/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/files/.gitkeep b/files/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/index.md b/index.md new file mode 100644 index 00000000..95ccdbdc --- /dev/null +++ b/index.md @@ -0,0 +1,17 @@ +--- +layout: lesson +root: . # Is the only page that doesn't follow the pattern /:path/index.html +permalink: index.html # Is the only page that doesn't follow the pattern /:path/index.html +--- +FIXME: home page introduction + + + +{% comment %} This is a comment in Liquid {% endcomment %} + +> ## Prerequisites +> +> FIXME +{: .prereq} + +{% include links.md %} diff --git a/reference.md b/reference.md new file mode 100644 index 00000000..8c826167 --- /dev/null +++ b/reference.md @@ -0,0 +1,9 @@ +--- +layout: reference +--- + +## Glossary + +FIXME + +{% include links.md %} diff --git a/setup.md b/setup.md new file mode 100644 index 00000000..b8c50321 --- /dev/null +++ b/setup.md @@ -0,0 +1,7 @@ +--- +title: Setup +--- +FIXME + + +{% include links.md %}