Never cache main page, always recalc book stats as needed. #906

Workflow file for this run

	name: ci

	on:
	push:
	# A branch github-ci-updates can be created and used for ci
	# experiments and tweaks.
	branches: [ "develop", "master", "github-ci", "windows", "pr_413_continued_plugins" ]
	pull_request:
	branches: [ "develop", "master" ]

	permissions:
	contents: read

	jobs:

	# Run all tests.
	base-ci:
	runs-on: ubuntu-latest
	timeout-minutes: 30

	strategy:
	matrix:
	python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]

	steps:

	- name: Install Dependencies
	run: \|
	sudo apt-get update -y
	sudo apt-get install -y mecab mecab-ipadic-utf8
	# Helper checks to find the mecab library path,
	# so it can be exported before running tests.
	# Without the export, natto-py fails on github.
	# echo FIND THE LIB:
	# which mecab
	# actual=`readlink -f /usr/bin/mecab`
	# ldd $actual

	- uses: actions/checkout@v4
	with:
	submodules: true

	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python_version }}
	cache: 'pip' # caching pip dependencies

	- run: pip install -r requirements.txt

	- name: Setup config
	run: \|
	mkdir ${{ github.workspace }}/data
	echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
	echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
	echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
	ls ${{ github.workspace }}
	cat ${{ github.workspace }}/lute/config/config.yml

	- name: Test
	run: \|
	set -e
	# Have to explicitly set MECAB_PATH for natto-py.
	export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2
	pytest

	- name: Acceptance testing
	run: \|
	set -e
	# Have to explicitly set MECAB_PATH for natto-py.
	export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2
	inv accept --show --verbose

	# Playwright tests were hanging far too often on github ci,
	# but not consistently. No idea what was going wrong.
	# TODO github ci: fix playwright hanging.
	- name: Playwright install
	run: playwright install

	- name: Playwright smoke test
	run: \|
	set -x
	set -e
	inv playwright

	- name: Check flit package
	run: \|
	mkdir ${{ github.workspace }}/../lute_flit
	cd ${{ github.workspace }}/../lute_flit
	flit -f ${{ github.workspace }}/pyproject.toml install

	python -m lute.main & # Start in background process
	sleep 2

	# Verify with utils script back in the workspace
	# (verify.py is not included in flit install)
	pushd ${{ github.workspace }}
	python -m utils.verify 5000
	popd

	pkill -f "python -m lute.main" # Kill that process.


	# Lute should still work if the user doesn't have mecab installed.
	no-mecab-check:
	runs-on: ubuntu-latest
	timeout-minutes: 10

	strategy:
	matrix:
	# Only checking early and late versions.
	python_version: [ '3.8', '3.11' ]

	steps:

	- uses: actions/checkout@v4
	with:
	submodules: true

	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python_version }}
	cache: 'pip' # caching pip dependencies

	- run: pip install -r requirements.txt

	- name: Setup config
	run: \|
	mkdir ${{ github.workspace }}/data
	echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
	echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
	echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
	ls ${{ github.workspace }}
	cat ${{ github.workspace }}/lute/config/config.yml

	# Run one particular sanity check.
	#
	# Note this isn't _completely_ valid because the test loads
	# only supported language stories, whereas a prod release
	# comes with _all_ stories pre-loaded and the invalid ones
	# are deleted ...
	- name: Smoke test no mecab
	run: inv accept -s -k disabled_data_is_hidden


	# Run all plugin tests.
	#
	# For each plugin:
	# - install Lute requirements
	# - install plugin reqs
	# - run tests.
	#
	# The Lute requirements are installed first b/c the plugins may come
	# with their own conflicting requirements. Doing a full req install
	# will (hopefully) uncover conflicts.
	plugins:
	runs-on: ubuntu-latest
	timeout-minutes: 30

	strategy:
	matrix:
	python_version: [ '3.8', '3.9', '3.10', '3.11' ]

	steps:

	- uses: actions/checkout@v4
	with:
	submodules: true

	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python_version }}
	cache: 'pip' # caching pip dependencies

	# Plugins likely won't need this config file, but just in case ...
	- name: Setup config
	run: \|
	mkdir ${{ github.workspace }}/data
	echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
	echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
	echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
	ls ${{ github.workspace }}
	cat ${{ github.workspace }}/lute/config/config.yml

	- name: test all plugins
	run: \|
	for plugin in $(ls plugins); do
	# Lute reqs, such as pytest.
	pip install -r requirements.txt

	# Lute itself, via toml, so that it can be found
	# by each plugin's own "pip install ."
	pip install .

	pushd plugins/$plugin
	pip install .
	# Note for future: some plugins may have extra reqs not covered by pip
	# (e.g. mecab uses apt-get and exports etc). Idea for future: plugin
	# could have a .github folder as well with additional setup scripts.

	pytest tests

	# pip uninstall $plugin -y
	# NOTE: Not bothering to do an uninstall!
	# if multiple plugins have different/clashing version requirements,
	# perhaps it is best to run into problems in ci.
	# This may ultimately come back to haunt me, but it will do for now.
	popd
	done


	code-quality:
	runs-on: ubuntu-latest
	timeout-minutes: 30

	strategy:
	matrix:
	python_version: [ '3.11' ]

	steps:

	- uses: actions/checkout@v4
	with:
	submodules: true


	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python_version }}
	cache: 'pip' # caching pip dependencies

	- run: pip install -r requirements.txt

	- name: Lint
	run: inv lint

	- name: Coding style
	run: \|
	black --check .


	# Test build docker container and try running.
	# Slightly wasteful re-setup of node.
	docker-build:
	runs-on: ubuntu-latest
	timeout-minutes: 30
	needs: base-ci
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: true

	- uses: actions/setup-python@v4
	with:
	python-version: 3.11
	cache: 'pip' # caching pip dependencies
	- run: pip install -r requirements.txt
	- name: Test docker build
	run: \|
	docker build -f docker/Dockerfile --build-arg INSTALL_EVERYTHING=false -t lute3 .
	# Run container in the background, and check.
	docker run -d -p 5000:5000 -v ./my_data:/lute_data -v ./my_backups:/lute_backup --name my-lute lute3:latest
	sleep 10 # Give it a moment to start.
	python -m utils.verify 5000
	docker stop my-lute
	docker rm my-lute


	# Ensure that basic things work correctly on Windows,
	# particularly PlatformDirs package.
	windows-ci:

	runs-on: windows-latest
	timeout-minutes: 30

	strategy:
	matrix:
	python_version: [ '3.8', '3.9', '3.10', '3.11' ]

	steps:

	- uses: actions/checkout@v4
	with:
	submodules: true


	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python_version }}
	cache: 'pip' # caching pip dependencies

	- name: Setup config
	run: \|
	mkdir ${{ github.workspace }}\data
	echo ENV: dev > ${{ github.workspace }}\lute\config\config.yml
	echo DATAPATH: ${{ github.workspace }}\data >> ${{ github.workspace }}\lute\config\config.yml
	echo DBNAME: test_lute.db >> ${{ github.workspace }}\lute\config\config.yml
	shell: cmd

	- name: Replace config slashes.
	run: \|
	$content = [System.IO.File]::ReadAllText("${{ github.workspace }}\lute\config\config.yml").Replace('\','/')
	[System.IO.File]::WriteAllText("${{ github.workspace }}\lute\config\config.yml", $content)
	Get-Content -Path "${{ github.workspace }}\lute\config\config.yml"

	- run: pip install -r requirements.txt

	# Can't get playwright to run reliably on Windows.
	# - name: Playwright install
	# run: playwright install
	# - name: Playwright smoke test
	# run: inv playwright \|\| exit /b

	# Now having problems with tests not working on windows ...
	# getting failure message:
	# javascript error: clear_datatable_state is not defined
	#
	# The above message is called from lute_test_client to clear book
	# datatables state. This _used_ to work (e.g. in v3.3.0), and
	# I can't track it down at the moment!!!!!!
	#
	# TODO ci: RESTORE AT LEAST ONE SANITY CHECK TEST ON WINDOWS.
	#
	# Run specific sanity check.
	# Old tests no longer run -- datatables may have timing issues on Windows tests,
	# tests were far too flaky.
	# inv accept -s -k test_unsupported_language_not_shown \|\| exit /b
	# inv accept -s -k import_a_valid_term_file \|\| exit /b
	# - name: Smoke tests
	# run: \|
	# inv accept -s -k test_updating_term_status_updates_the_reading_frame \|\| exit /b

	- name: Remove config to force using prod config
	run: del ${{ github.workspace }}\lute\config\config.yml
	shell: cmd

	- name: Install prod-like flit package
	run: \|
	mkdir ${{ github.workspace }}\..\lute_flit
	cd ${{ github.workspace }}\..\lute_flit
	flit -f ${{ github.workspace }}\pyproject.toml install
	shell: cmd

	- name: Run Lute application
	run: \|
	echo "python -m lute.main" > run-server.bat
	cmd /c "START /b run-server.bat"

	- name: Wait for application to start
	run: Start-Sleep -Seconds 10

	- name: Verify flit install
	run: \|
	cd ${{ github.workspace }}
	python -m utils.verify 5000

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Never cache main page, always recalc book stats as needed. #906

Workflow file

Never cache main page, always recalc book stats as needed. #906

Jobs

Run details

Workflow file for this run