From eb5da6470431868611cd9e21310dd60837567fb2 Mon Sep 17 00:00:00 2001
From: Julian Waller <git@julusian.co.uk>
Date: Sat, 27 Jan 2024 21:49:01 +0000
Subject: [PATCH] fix: retry drawing if workers crash

---
 lib/Graphics/Controller.js        | 37 ++++++++++++++++++++++++++-----
 lib/Resources/EventDefinitions.js |  1 -
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/lib/Graphics/Controller.js b/lib/Graphics/Controller.js
index 99de32ba0..b7600dad2 100644
--- a/lib/Graphics/Controller.js
+++ b/lib/Graphics/Controller.js
@@ -27,6 +27,8 @@ import { isPackaged } from '../Resources/Util.js'
 import { fileURLToPath } from 'url'
 import path from 'path'
 
+const CRASHED_WORKER_RETRY_COUNT = 10
+
 /**
  * @typedef {{
  *   page_direction_flipped: boolean,
@@ -159,12 +161,12 @@ class GraphicsController extends CoreBase {
 						render = this.#renderLRUCache.get(key)
 
 						if (!render) {
-							const { buffer, width, height, dataUrl, draw_style } = await this.#pool.exec('drawButtonImage', [
-								this.#drawOptions,
+							const { buffer, width, height, dataUrl, draw_style } = await this.#executePoolDrawButtonImage(
 								buttonStyle,
 								location,
 								pagename,
-							])
+								CRASHED_WORKER_RETRY_COUNT
+							)
 							render = GraphicsRenderer.wrapDrawButtonImage(buffer, width, height, dataUrl, draw_style, buttonStyle)
 						}
 					} else {
@@ -252,10 +254,12 @@ class GraphicsController extends CoreBase {
 			size: buttonStyle.size === 'auto' ? 'auto' : Number(buttonStyle.size),
 		}
 
-		const { buffer, width, height, dataUrl, draw_style } = await this.#pool.exec('drawButtonImage', [
-			this.#drawOptions,
+		const { buffer, width, height, dataUrl, draw_style } = await this.#executePoolDrawButtonImage(
 			drawStyle,
-		])
+			undefined,
+			undefined,
+			CRASHED_WORKER_RETRY_COUNT
+		)
 		return GraphicsRenderer.wrapDrawButtonImage(buffer, width, height, dataUrl, draw_style, drawStyle)
 	}
 
@@ -397,6 +401,27 @@ class GraphicsController extends CoreBase {
 
 		return GraphicsRenderer.drawBlank(this.#drawOptions, location)
 	}
+
+	/**
+	 * Draw a button image in the worker pool
+	 * @param {import('../Shared/Model/StyleModel.js').DrawStyleModel} drawStyle The style to draw
+	 * @param {import('../Resources/Util.js').ControlLocation | undefined} location
+	 * @param {string | undefined} pagename
+	 * @param {number} remainingAttempts
+	 * @returns {Promise<{ buffer: Buffer, width: number, height: number, dataUrl: string, draw_style: import('../Shared/Model/StyleModel.js').DrawStyleModel['style'] | undefined}>} Image render object
+	 */
+	async #executePoolDrawButtonImage(drawStyle, location, pagename, remainingAttempts) {
+		try {
+			return this.#pool.exec('drawButtonImage', [this.#drawOptions, drawStyle, location, pagename])
+		} catch (/** @type {any} */ e) {
+			// if a worker crashes, the first attempt will fail, retry when that happens, but not infinitely
+			if (remainingAttempts > 1 && e?.message?.includes('Worker is terminated')) {
+				return this.#executePoolDrawButtonImage(drawStyle, location, pagename, remainingAttempts - 1)
+			} else {
+				throw e
+			}
+		}
+	}
 }
 
 export default GraphicsController
diff --git a/lib/Resources/EventDefinitions.js b/lib/Resources/EventDefinitions.js
index 5867abfbb..a7c61ab2a 100644
--- a/lib/Resources/EventDefinitions.js
+++ b/lib/Resources/EventDefinitions.js
@@ -189,7 +189,6 @@ export function visitEventOptions(visitor, event) {
 	}
 
 	for (const key of Object.keys(event.options || {})) {
-		console.log('check', key, event.options[key])
 		visitor.visitString(event.options, key)
 	}
 }