From 12c816fd3efaa96b7814ae9ab2f7c75a38ce128a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Gandra=C3=9F?= Date: Tue, 13 Aug 2024 12:41:23 +0200 Subject: [PATCH 1/4] Ignore .github and test directories in Docker image build --- .dockerignore | 2 ++ CHANGELOG.md | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/.dockerignore b/.dockerignore index 799d2f1..03c60b5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ .git/ +.github/ .gitignore .venv/ .idea/ out/ +tests/ *.iml \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index ad6c070..4527d96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Version X.X.X (XXXX-XX-XX) + +- Ignore `.github` and `test` directories in Docker image build + + ## Version 1.6.0 (2024-07-29) - Implement support for passing additional status values (statusextras) to Moodle From 2ca38b34856b2e02803f030342e487d518e4a325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Gandra=C3=9F?= Date: Tue, 13 Aug 2024 12:43:13 +0200 Subject: [PATCH 2/4] Wait for GeoGebra applets to be fully rendered before exporting page and improve page export readiness detection logic --- CHANGELOG.md | 2 + archiveworker/quiz_archive_job.py | 28 +------ res/readysignal.js | 124 ++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 26 deletions(-) create mode 100644 res/readysignal.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 4527d96..0c9e8dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Version X.X.X (XXXX-XX-XX) +- Add custom readiness probe for GeoGebra applets +- Improve page export readiness detection and add support for multiple readiness probes - Ignore `.github` and `test` directories in Docker image build diff --git a/archiveworker/quiz_archive_job.py b/archiveworker/quiz_archive_job.py index 49bb7dc..012cebe 100644 --- a/archiveworker/quiz_archive_job.py +++ b/archiveworker/quiz_archive_job.py @@ -38,6 +38,7 @@ from .custom_types import JobStatus, JobArchiveRequest, ReportSignal, BackupStatus from .moodle_api import MoodleAPI +READYSIGNAL_JAVASCRIPT = open(os.path.join(os.path.dirname(__file__), '../res/readysignal.js')).read() class QuizArchiveJob: """ @@ -405,32 +406,7 @@ async def _wait_for_page_ready_signal(self, page) -> None: timeout=Config.REPORT_WAIT_FOR_READY_SIGNAL_TIMEOUT_SEC * 1000 ) as cmsg_handler: self.logger.debug('Injecting JS to wait for page rendering ...') - await page.evaluate(''' - setTimeout(function() { - const SIGNAL_PAGE_READY_FOR_EXPORT = "x-quiz-archiver-page-ready-for-export"; - const SIGNAL_MATHJAX_FOUND = "x-quiz-archiver-mathjax-found"; - const SIGNAL_MATHJAX_NOT_FOUND = "x-quiz-archiver-mathjax-not-found"; - const SIGNAL_MATHJAX_NO_FORMULAS_ON_PAGE = "x-quiz-archiver-mathjax-no-formulas-on-page"; - - if (typeof window.MathJax !== 'undefined') { - console.log(SIGNAL_MATHJAX_FOUND); - - if (document.getElementsByClassName('filter_mathjaxloader_equation').length == 0) { - console.log(SIGNAL_MATHJAX_NO_FORMULAS_ON_PAGE); - console.log(SIGNAL_PAGE_READY_FOR_EXPORT); - return; - } - - window.MathJax.Hub.Queue(function () { - console.log(SIGNAL_PAGE_READY_FOR_EXPORT); - }); - window.MathJax.Hub.processSectionDelay = 0; - } else { - console.log(SIGNAL_MATHJAX_NOT_FOUND); - console.log(SIGNAL_PAGE_READY_FOR_EXPORT); - } - }, 1000); - ''') + await page.evaluate(READYSIGNAL_JAVASCRIPT) self.logger.debug(f'Waiting for ready signal: {ReportSignal.READY_FOR_EXPORT}') cmsg = await cmsg_handler.value diff --git a/res/readysignal.js b/res/readysignal.js new file mode 100644 index 0000000..0f51757 --- /dev/null +++ b/res/readysignal.js @@ -0,0 +1,124 @@ +/** + * This script is injected into the page to check if the page is ready for export. + */ + +/** + * Interval in milliseconds the readiness probe checks are executed after the initial delay. + * @type {number} + */ +const QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS = 250; + +const SIGNAL_PAGE_READY_FOR_EXPORT = "x-quiz-archiver-page-ready-for-export"; +const SIGNAL_GEOGEBRA_FOUND = "x-quiz-archiver-geogebra-found"; +const SIGNAL_GEOGEBRA_NOT_FOUND = "x-quiz-archiver-geogebra-not-found"; +const SIGNAL_GEOGEBRA_READY_FOR_EXPORT = "x-quiz-archiver-geogebra-ready-for-export"; +const SIGNAL_MATHJAX_FOUND = "x-quiz-archiver-mathjax-found"; +const SIGNAL_MATHJAX_NOT_FOUND = "x-quiz-archiver-mathjax-not-found"; +const SIGNAL_MATHJAX_NO_FORMULAS_ON_PAGE = "x-quiz-archiver-mathjax-no-formulas-on-page"; +const SIGNAL_MATHJAX_READY_FOR_EXPORT = "x-quiz-archiver-mathjax-ready-for-export"; + +/** + * Global object to store readiness signals for different components. + * + * @type {{readySignals: {geogebra: null, mathjax: null}}} + */ +window.MoodleQuizArchiver = { + initialized: false, + readySignals: { + mathjax: null, + geogebra: null + } +}; + +/** + * Detects and prepares readiness signals for all tracked components. + * This function must be called prior to checkReadiness(). + */ +function detectAndPrepareReadinessComponents() { + // MathJax + if (typeof window.MathJax !== 'undefined') { + window.MoodleQuizArchiver.readySignals.mathjax = false; + console.log(SIGNAL_MATHJAX_FOUND); + + if (document.getElementsByClassName('filter_mathjaxloader_equation').length === 0) { + window.MoodleQuizArchiver.readySignals.mathjax = true; + console.log(SIGNAL_MATHJAX_NO_FORMULAS_ON_PAGE); + console.log(SIGNAL_MATHJAX_READY_FOR_EXPORT); + } else { + window.MathJax.Hub.Queue(function () { + window.MoodleQuizArchiver.readySignals.mathjax = true; + console.log(SIGNAL_MATHJAX_READY_FOR_EXPORT); + }); + window.MathJax.Hub.processSectionDelay = 0; + } + } else { + console.log(SIGNAL_MATHJAX_NOT_FOUND); + } + + // GeoGebra + if (typeof window.GGBApplet !== 'undefined') { + window.MoodleQuizArchiver.readySignals.geogebra = false; + console.log(SIGNAL_GEOGEBRA_FOUND); + + detectGeogebraFinishedRendering(); + } else { + console.log(SIGNAL_GEOGEBRA_NOT_FOUND); + } + + window.MoodleQuizArchiver.initialized = true; +} + +/** + * Detects when GeoGebra applets have finished rendering. This function calls + * itself periodically until all applets are rendered. + * + * Results are stored inside window.MoodleQuizArchiver.readySignals.geogebra. + */ +function detectGeogebraFinishedRendering() { + // Detect GeoGebraFrames + let ggbFrames = document.getElementsByClassName('GeoGebraFrame'); + let numLoadingFrames = 0; + + // Count the number of loading images in each GeoGebra frame + ggbFrames.forEach(ggbFrame => { + numLoadingFrames += ggbFrame.querySelectorAll("img.gwt-Image").length; + }) + + // Declare GeoGebra to be ready for export if all GeoGebraFrames appear loaded + if (ggbFrames.length > 0 && numLoadingFrames === 0) { + window.MoodleQuizArchiver.readySignals.geogebra = true; + console.log(SIGNAL_GEOGEBRA_READY_FOR_EXPORT); + } else { + setTimeout(detectGeogebraFinishedRendering, QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS); + } +} + +/** + * Checks if all components are ready for export. If not, this function will + * call itself periodically until all components are ready. + */ +function checkReadiness() { + if (!window.MoodleQuizArchiver.initialized) { + console.error("Failed to check component export readiness before initialization."); + setTimeout(checkReadiness, QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS); + return; + } + + for (const [component, ready] of Object.entries(window.MoodleQuizArchiver.readySignals)) { + if (ready === null) { + continue; + } + if (ready !== true) { + setTimeout(checkReadiness, QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS); + return; + } + } + + console.log(SIGNAL_PAGE_READY_FOR_EXPORT); +} + +// Ignite the readiness detection process. +setTimeout(function() { + detectAndPrepareReadinessComponents(); + checkReadiness(); +}, 1000); From 98b70594215cac8d985006af0a66e69a7f05d4f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Gandra=C3=9F?= Date: Tue, 13 Aug 2024 15:06:39 +0200 Subject: [PATCH 3/4] GeoGebra readiness probe: Replace fragile loading icon detection with proper MutationObserver for all GeoGebra frames on the page --- res/readysignal.js | 71 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/res/readysignal.js b/res/readysignal.js index 0f51757..1a35c5b 100644 --- a/res/readysignal.js +++ b/res/readysignal.js @@ -1,5 +1,23 @@ +/* + * Moodle Quiz Archive Worker + * Copyright (C) 2024 Niels Gandraß + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + /** - * This script is injected into the page to check if the page is ready for export. + * This script is injected into each page to check if the page is ready for export. */ /** @@ -8,9 +26,17 @@ */ const QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS = 250; +/** + * Number of milliseconds to wait after the last mutation of a GeoGebra applet before + * considering it stable and ready for export. + * @type {number} + */ +const QUIZ_ARCHIVER_GEOGEBRA_MUTATION_STABLE_PERIOD_MS = 1000; + const SIGNAL_PAGE_READY_FOR_EXPORT = "x-quiz-archiver-page-ready-for-export"; const SIGNAL_GEOGEBRA_FOUND = "x-quiz-archiver-geogebra-found"; const SIGNAL_GEOGEBRA_NOT_FOUND = "x-quiz-archiver-geogebra-not-found"; +const SIGNAL_GEOGEBRA_MUTATED = "x-quiz-archiver-geogebra-mutated"; const SIGNAL_GEOGEBRA_READY_FOR_EXPORT = "x-quiz-archiver-geogebra-ready-for-export"; const SIGNAL_MATHJAX_FOUND = "x-quiz-archiver-mathjax-found"; const SIGNAL_MATHJAX_NOT_FOUND = "x-quiz-archiver-mathjax-not-found"; @@ -23,10 +49,15 @@ const SIGNAL_MATHJAX_READY_FOR_EXPORT = "x-quiz-archiver-mathjax-ready-for-expor * @type {{readySignals: {geogebra: null, mathjax: null}}} */ window.MoodleQuizArchiver = { - initialized: false, + initialized: false, // True if the readiness detection process has been initialized readySignals: { - mathjax: null, - geogebra: null + mathjax: null, // True if MathJax is ready for export, null if MathJax is not found + geogebra: null // True if GeoGebra is ready for export, null if GeoGebra is not found + }, + states: { // Optional stateful data for different components + geogebra: { + last_mutation: null // Timestamp of the last mutation of a GeoGebra applet + } } }; @@ -40,11 +71,13 @@ function detectAndPrepareReadinessComponents() { window.MoodleQuizArchiver.readySignals.mathjax = false; console.log(SIGNAL_MATHJAX_FOUND); + // Check if MathJax is not just loaded but the page also has formulas on it if (document.getElementsByClassName('filter_mathjaxloader_equation').length === 0) { window.MoodleQuizArchiver.readySignals.mathjax = true; console.log(SIGNAL_MATHJAX_NO_FORMULAS_ON_PAGE); console.log(SIGNAL_MATHJAX_READY_FOR_EXPORT); } else { + // Formulas found. Wait for MathJax to process them. window.MathJax.Hub.Queue(function () { window.MoodleQuizArchiver.readySignals.mathjax = true; console.log(SIGNAL_MATHJAX_READY_FOR_EXPORT); @@ -60,7 +93,19 @@ function detectAndPrepareReadinessComponents() { window.MoodleQuizArchiver.readySignals.geogebra = false; console.log(SIGNAL_GEOGEBRA_FOUND); - detectGeogebraFinishedRendering(); + // Attach mutation listener to GeoGebra frames + var mutationObserver = new (window.MutationObserver || window.WebKitMutationObserver)(() => { + window.MoodleQuizArchiver.states.geogebra.last_mutation = new Date(); + console.log(SIGNAL_GEOGEBRA_MUTATED); + }); + + document.getElementsByClassName('GeoGebraFrame').forEach(ggbFrame => { + mutationObserver.observe(ggbFrame, {childList: true, subtree: true}); + }); + window.MoodleQuizArchiver.states.geogebra.last_mutation = new Date() + + // Ignite periodic readiness check + setTimeout(detectGeogebraFinishedRendering, QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS); } else { console.log(SIGNAL_GEOGEBRA_NOT_FOUND); } @@ -69,23 +114,15 @@ function detectAndPrepareReadinessComponents() { } /** - * Detects when GeoGebra applets have finished rendering. This function calls + * Detects when GeoGebra instances have finished rendering. This function calls * itself periodically until all applets are rendered. * * Results are stored inside window.MoodleQuizArchiver.readySignals.geogebra. */ function detectGeogebraFinishedRendering() { - // Detect GeoGebraFrames - let ggbFrames = document.getElementsByClassName('GeoGebraFrame'); - let numLoadingFrames = 0; - - // Count the number of loading images in each GeoGebra frame - ggbFrames.forEach(ggbFrame => { - numLoadingFrames += ggbFrame.querySelectorAll("img.gwt-Image").length; - }) - - // Declare GeoGebra to be ready for export if all GeoGebraFrames appear loaded - if (ggbFrames.length > 0 && numLoadingFrames === 0) { + // Declare GeoGebra to be ready for export if no mutation has occurred since the given time + const lastMutationMs = window.MoodleQuizArchiver.states.geogebra.last_mutation.getTime(); + if (new Date().getTime() >= lastMutationMs + QUIZ_ARCHIVER_GEOGEBRA_MUTATION_STABLE_PERIOD_MS) { window.MoodleQuizArchiver.readySignals.geogebra = true; console.log(SIGNAL_GEOGEBRA_READY_FOR_EXPORT); } else { From d7345b60d0202c6f972f2059d1c4fd6bd44d36c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Gandra=C3=9F?= Date: Tue, 20 Aug 2024 15:39:22 +0200 Subject: [PATCH 4/4] GeoGebra readiness probe: Wait for GeoGebra main process to render final applet frames before attaching mutation observers. Prevents race conditions on slow hardware. --- res/readysignal.js | 67 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/res/readysignal.js b/res/readysignal.js index 1a35c5b..50b6ae5 100644 --- a/res/readysignal.js +++ b/res/readysignal.js @@ -31,7 +31,7 @@ const QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS = 250; * considering it stable and ready for export. * @type {number} */ -const QUIZ_ARCHIVER_GEOGEBRA_MUTATION_STABLE_PERIOD_MS = 1000; +const QUIZ_ARCHIVER_GEOGEBRA_MUTATION_STABLE_PERIOD_MS = 2000; const SIGNAL_PAGE_READY_FOR_EXPORT = "x-quiz-archiver-page-ready-for-export"; const SIGNAL_GEOGEBRA_FOUND = "x-quiz-archiver-geogebra-found"; @@ -91,21 +91,11 @@ function detectAndPrepareReadinessComponents() { // GeoGebra if (typeof window.GGBApplet !== 'undefined') { window.MoodleQuizArchiver.readySignals.geogebra = false; + window.MoodleQuizArchiver.states.geogebra.last_mutation = new Date(9999, 1, 1); // Far future console.log(SIGNAL_GEOGEBRA_FOUND); - // Attach mutation listener to GeoGebra frames - var mutationObserver = new (window.MutationObserver || window.WebKitMutationObserver)(() => { - window.MoodleQuizArchiver.states.geogebra.last_mutation = new Date(); - console.log(SIGNAL_GEOGEBRA_MUTATED); - }); - - document.getElementsByClassName('GeoGebraFrame').forEach(ggbFrame => { - mutationObserver.observe(ggbFrame, {childList: true, subtree: true}); - }); - window.MoodleQuizArchiver.states.geogebra.last_mutation = new Date() - - // Ignite periodic readiness check - setTimeout(detectGeogebraFinishedRendering, QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS); + // Attach mutation observer to GeoGebra frames once available + attachGeogebraMutationObserver(); } else { console.log(SIGNAL_GEOGEBRA_NOT_FOUND); } @@ -113,6 +103,54 @@ function detectAndPrepareReadinessComponents() { window.MoodleQuizArchiver.initialized = true; } +/** + * Waits for GeoGebra to be initialized to the point where it rendered its final + * applet frames and attach mutation observers to them. + * + * This also ignites the readiness detection process for GeoGebra. + */ +function attachGeogebraMutationObserver() { + // Check if GeoGebra is initialized to the point where it created its target applet frames + try { + if (typeof window.GGBApplet().getAppletObject === 'function') { + if (typeof window.GGBApplet().getAppletObject().getFrame === 'function') { + if (window.GGBApplet().getAppletObject().getFrame().classList.contains('jsloaded')) { + // Attach mutation listener to GeoGebra frames + var mutationObserver = new (window.MutationObserver || window.WebKitMutationObserver)(() => { + window.MoodleQuizArchiver.states.geogebra.last_mutation = new Date(); + console.log(SIGNAL_GEOGEBRA_MUTATED); + }); + + document.getElementsByClassName('GeoGebraFrame').forEach(ggbFrame => { + mutationObserver.observe(ggbFrame, {childList: true, subtree: true}); + console.log("Attached mutation observer to GeoGebra frame."); + }); + window.MoodleQuizArchiver.states.geogebra.last_mutation = new Date(); + + // Ignite periodic readiness check + setTimeout(detectGeogebraFinishedRendering, QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS); + return; + } else { + console.log("GeoGebra frame not fully initialized yet. Waiting ..."); + } + } else { + console.log("GeoGebra frame object not yet ready. Waiting ..."); + } + } else { + console.log("GeoGebra applet object not yet ready. Waiting ..."); + } + } catch (e) { + if (e instanceof TypeError) { + console.log("GeoGebra applet/frames not yet ready. Waiting ..."); + } else { + console.log("Failed to attach mutation observer to GeoGebra frames: " + e); + } + } + + // If we got here, GeoGebra is not ready yet. Retry in a bit. + setTimeout(attachGeogebraMutationObserver, QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS); +} + /** * Detects when GeoGebra instances have finished rendering. This function calls * itself periodically until all applets are rendered. @@ -126,6 +164,7 @@ function detectGeogebraFinishedRendering() { window.MoodleQuizArchiver.readySignals.geogebra = true; console.log(SIGNAL_GEOGEBRA_READY_FOR_EXPORT); } else { + window.MoodleQuizArchiver.readySignals.geogebra = false; setTimeout(detectGeogebraFinishedRendering, QUIZ_ARCHIVER_READINESS_PROBE_INTERVAL_MS); } }