ported MQTT Client to PsychicMqttClient

Author: theelims

.gitignore

@@ -15,3 +15,4 @@ node_modules
 /lib/framework/WWWData.h
 *WWWData.h
 lib/framework/WWWData.h
+ssl_certs/cacert.pem

CHANGELOG.md

@@ -6,43 +6,42 @@ All notable changes to this project will be documented in this file.
 
 > **Warning**: This update has breaking changes!
 
-### ToDo's for this release
-
-- [ ] Provide CORS preflight confirmation
-- [ ] Switch AsyncMqttClient to ESP-IDF MQTT Client
-- [ ] Fix glitch in UI that System Status page is not reactive to 2sec polling of status
-- [ ] LightState NimBLE Demo --> Switch LED on and off with BLE HID device. Binding procedure with BLE HID device and BLE HID status page.
-- [ ] Update docs and check for consistency
-
 ### Added
 
 - Added postscript to platform.io build process to copy, rename and calculate MD5 checksum of \*.bin file. These files are ready for uploading to the Github Release page.
 - Added more information to SystemStatus API
 - Added generateToken API for security settings
 - Added Multi-WiFi capability. Add up to five WiFi configurations and connect to either strongest network (default), or by priority.
 - Added InfoDialog as a simpler version of the ConfirmDialog for a simple notification modal.
+- Added Adafruit certificate repository as the default choice for the X509 certificate bundle.
 
 ### Changed
 
 - Better route protection for user page with deep link.
 - Changed build_interface.py script to check for modified files in the interface sources before re-building the interface. Saves some time on the compilation process.
 - Upload firmware binary allows uploading of MD5 checksum file in advance to verify downloaded firmware package.
 - GithubFirmwareManager checks against PIO build_target in filename to support Github OTA for binaries build for various targets. You should rename your old release \*.bin files on the Github release pages for backward compatibility.
+- Changed MQTT Client to an ESP-IDF backed one which supports SSL/TLS X509 root CA bundles and transport over WS.
+- Changed the PROGMEM_WWW flag to EMBED_WWW as there is technically speaking no PROGMEM on ESP32's.
+
+### Fixed
+
+- Fixed reactivity of System Status page.
 
 ### Removed
 
 - Removed support for Arduino ESP OTA.
 - HttpEndpoints and Websocket Server without a securityManager are no longer possible.
 
-### Migrate from ESPAsyncWebServer
+### Migrate from ESPAsyncWebServer to PsychicHttp
 
 #### Migrate `main.cpp`
 
 Change the server and ESPSvelteKit instances to PsychicHttpServer and give the ESP32SvelteKit constructor the number of http endpoints of your project.
 
 ```
 PsychicHttpServer server;
-ESP32SvelteKit esp32sveltekit(&server, 115);
+ESP32SvelteKit esp32sveltekit(&server, 125);
 ```
 
 Remove `server.begin();` in `void setup()`. This is handled by ESP32SvelteKit now.
@@ -57,6 +56,7 @@ Remove the following `build_flags`:
     -D WS_MAX_QUEUED_MESSAGES=64
     -D CONFIG_ASYNC_TCP_RUNNING_CORE=0
     -D NO_GLOBAL_ARDUINOOTA
+    -D PROGMEM_WWW
 ```
 
 Add the following `build_flags` and adjust to your app, if needed:
@@ -65,9 +65,29 @@ Add the following `build_flags` and adjust to your app, if needed:
     -D BUILD_TARGET=\"$PIOENV\"
     -D APP_NAME=\"ESP32-Sveltekit\" ; Must only contain characters from [a-zA-Z0-9-_] as this is converted into a filename
     -D APP_VERSION=\"0.3.0\" ; semver compatible version string
+    -D EMBED_WWW
+```
+
+Remove the lib dependency `esphome/AsyncTCP-esphome @ ^2.0.0` and add `https://github.com/theelims/PsychicMqttClient.git`
+
+Consider adjusting `board_ssl_cert_source = adafruit`, so that the new MQTT client has universal SSL/TLS support with a wide range of CA root certificates.
+
+#### Migrate `factory_settings.ini`
+
+The new MQTT client has slightly renamed factory settings:
+
+```ini
+  ; MQTT settings
+  -D FACTORY_MQTT_ENABLED=false
+  -D FACTORY_MQTT_URI=\"mqtts://mqtt.eclipseprojects.io:8883\"
+  -D FACTORY_MQTT_USERNAME=\"\" ; supports placeholders
+  -D FACTORY_MQTT_PASSWORD=\"\"
+  -D FACTORY_MQTT_CLIENT_ID=\"#{platform}-#{unique_id}\" ; supports placeholders
+  -D FACTORY_MQTT_KEEP_ALIVE=120
+  -D FACTORY_MQTT_CLEAN_SESSION=true
 ```
 
-Remove the lib dependency `esphome/AsyncTCP-esphome @ ^2.0.0`.
+Max Topic Length is no longer needed.
 
 #### Custom Stateful Services
 

factory_settings.ini

@@ -37,14 +37,12 @@ build_flags =
 
   ; MQTT settings
   -D FACTORY_MQTT_ENABLED=false
-  -D FACTORY_MQTT_HOST=\"test.mosquitto.org\"
-  -D FACTORY_MQTT_PORT=1883
+  -D FACTORY_MQTT_URI=\"mqtts://mqtt.eclipseprojects.io:8883\"
   -D FACTORY_MQTT_USERNAME=\"\" ; supports placeholders
   -D FACTORY_MQTT_PASSWORD=\"\"
   -D FACTORY_MQTT_CLIENT_ID=\"#{platform}-#{unique_id}\" ; supports placeholders
-  -D FACTORY_MQTT_KEEP_ALIVE=60
+  -D FACTORY_MQTT_KEEP_ALIVE=120
   -D FACTORY_MQTT_CLEAN_SESSION=true
-  -D FACTORY_MQTT_MAX_TOPIC_LENGTH=128
 
   ; JWT Secret
   -D FACTORY_JWT_SECRET=\"#{random}-#{random}\" ; supports placeholders

features.ini

@@ -1,6 +1,6 @@
 [features]
 build_flags = 
-  -D FT_SECURITY=0
+  -D FT_SECURITY=1
   -D FT_MQTT=1
   -D FT_NTP=1
   -D FT_UPLOAD_FIRMWARE=1

interface/src/routes/connections/mqtt/MQTT.svelte

@@ -16,37 +16,24 @@
 		enabled: boolean;
 		connected: boolean;
 		client_id: string;
-		disconnect_reason: number;
+		last_error: string;
 	};
 
 	type MQTTSettings = {
 		enabled: boolean;
-		host: string;
-		port: number;
+		uri: string;
 		username: string;
 		password: string;
 		client_id: string;
 		keep_alive: number;
 		clean_session: boolean;
-		max_topic_length: number;
 	};
 
 	let mqttSettings: MQTTSettings;
 	let mqttStatus: MQTTStatus;
 
 	let formField: any;
 
-	const disconnectReason = [
-		'TCP Disconnected',
-		'Unacceptable Protocol Version',
-		'Identifier Rejected',
-		'Server unavailable',
-		'Malformed Credentials',
-		'Not Authorized',
-		'Not Enough Memory',
-		'TLS Rejected'
-	];
-
 	async function getMQTTStatus() {
 		try {
 			const response = await fetch('/rest/mqttStatus', {
@@ -123,29 +110,17 @@
 	function handleSubmitMQTT() {
 		let valid = true;
 
-		// Validate Server
-		// RegEx for IPv4
-		const regexExpIPv4 =
-			/\b(?:(?:2(?:[0-4][0-9]|5[0-5])|[0-1]?[0-9]?[0-9])\.){3}(?:(?:2([0-4][0-9]|5[0-5])|[0-1]?[0-9]?[0-9]))\b/;
+		// Validate Server URI
 		const regexExpURL =
-			/[-a-zA-Z0-9@:%_\+.~#?&//=]{2,256}\.[a-z]{2,4}\b(\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?/i;
+			/(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,256}\.[a-z]{2,4})|(\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b))(\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?/i;
 
-		if (!regexExpURL.test(mqttSettings.host) && !regexExpIPv4.test(mqttSettings.host)) {
+		if (!regexExpURL.test(mqttSettings.uri)) {
 			valid = false;
 			formErrors.host = true;
 		} else {
 			formErrors.host = false;
 		}
 
-		// Validate if port is a number and within the right range
-		let port = Number(mqttSettings.port);
-		if (0 <= port && port <= 65536) {
-			formErrors.port = false;
-		} else {
-			formErrors.port = true;
-			valid = false;
-		}
-
 		// Validate if port is a number and within the right range
 		let keepalive = Number(mqttSettings.keep_alive);
 		if (1 <= keepalive && keepalive <= 600) {
@@ -194,7 +169,7 @@
 							{:else if !mqttStatus.enabled}
 								MQTT Disabled
 							{:else}
-								{disconnectReason[mqttStatus.disconnect_reason]}
+								{mqttStatus.last_error}
 							{/if}
 						</div>
 					</div>
@@ -231,50 +206,28 @@
 						<span>Enable MQTT</span>
 					</label>
 					<div class="hidden sm:block" />
-					<!-- Host -->
-					<div>
+					<!-- URI -->
+					<div class="sm:col-span-2">
 						<label class="label" for="host">
-							<span class="label-text text-md">Host</span>
+							<span class="label-text text-md">URI</span>
 						</label>
 						<input
 							type="text"
 							class="input input-bordered invalid:border-error w-full invalid:border-2 {formErrors.host
 								? 'border-error border-2'
 								: ''}"
-							bind:value={mqttSettings.host}
+							bind:value={mqttSettings.uri}
 							id="host"
 							min="3"
 							max="64"
 							required
 						/>
 						<label class="label" for="host">
 							<span class="label-text-alt text-error {formErrors.host ? '' : 'hidden'}"
-								>Must be a valid IPv4 address or URL</span
+								>Must be a valid URI</span
 							>
 						</label>
 					</div>
-					<!-- Port -->
-					<div>
-						<label class="label" for="port">
-							<span class="label-text text-md">Port</span>
-						</label>
-						<input
-							type="number"
-							min="0"
-							max="65536"
-							class="input input-bordered invalid:border-error w-full invalid:border-2 {formErrors.port
-								? 'border-error border-2'
-								: ''}"
-							bind:value={mqttSettings.port}
-							id="port"
-							required
-						/>
-						<label for="port" class="label"
-							><span class="label-text-alt text-error {formErrors.port ? '' : 'hidden'}"
-								>Port number must be between 0 and 65536</span
-							></label
-						>
-					</div>
 					<!-- Username -->
 					<div>
 						<label class="label" for="user">
@@ -306,15 +259,6 @@
 							id="clientid"
 						/>
 					</div>
-					<!-- Clean Session -->
-					<label class="label inline-flex cursor-pointer content-end justify-start gap-4">
-						<input
-							type="checkbox"
-							bind:checked={mqttSettings.clean_session}
-							class="checkbox checkbox-primary mt-2 sm:-mb-8 sm:mt-0"
-						/>
-						<span class="mt-2 sm:-mb-8 sm:mt-0">Clean Session?</span>
-					</label>
 					<!-- Keep Alive -->
 					<div>
 						<label class="label" for="keepalive">
@@ -340,31 +284,15 @@
 							></label
 						>
 					</div>
-					<!-- Max Topic Length -->
-					<div>
-						<label class="label" for="maxtopic">
-							<span class="label-text text-md">Max Topic Length</span>
-						</label>
-						<label for="maxtopic" class="input-group">
-							<input
-								type="number"
-								min="64"
-								max="4096"
-								class="input input-bordered invalid:border-error w-full invalid:border-2 {formErrors.topic_length
-									? 'border-error border-2'
-									: ''}"
-								bind:value={mqttSettings.max_topic_length}
-								id="maxtopic"
-								required
-							/>
-							<span>Chars</span>
-						</label>
-						<label for="maxtopic" class="label"
-							><span class="label-text-alt text-error {formErrors.topic_length ? '' : 'hidden'}"
-								>Must be between 64 and 4096 characters</span
-							></label
-						>
-					</div>
+					<!-- Clean Session -->
+					<label class="label inline-flex cursor-pointer content-end justify-start gap-4">
+						<input
+							type="checkbox"
+							bind:checked={mqttSettings.clean_session}
+							class="checkbox checkbox-primary"
+						/>
+						<span class="">Clean Session?</span>
+					</label>
 				</div>
 				<div class="divider mb-2 mt-0" />
 				<div class="mx-4 flex flex-wrap justify-end gap-2">

interface/src/routes/system/status/SystemStatus.svelte

@@ -50,6 +50,8 @@
 		uptime: number;
 	};
 
+	let systemStatus: SystemStatus;
+
 	async function getSystemStatus() {
 		try {
 			const response = await fetch('/rest/systemStatus', {
@@ -59,10 +61,11 @@
 					'Content-Type': 'application/json'
 				}
 			});
-			return await response.json();
+			systemStatus = await response.json();
 		} catch (error) {
 			console.log('Error:', error);
 		}
+		return systemStatus;
 	}
 
 	const interval = setInterval(async () => {
@@ -180,7 +183,7 @@
 	<div class="w-full overflow-x-auto">
 		{#await getSystemStatus()}
 			<Spinner />
-		{:then systemStatus}
+		{:then nothing}
 			<div
 				class="flex w-full flex-col space-y-1"
 				transition:slide|local={{ duration: 300, easing: cubicOut }}
@@ -329,9 +332,7 @@
 					<div>
 						<div class="font-bold">Core Temperature</div>
 						<div class="text-sm opacity-75">
-							{systemStatus.core_temp.toFixed(2) == 53.33
-								? 'NaN'
-								: systemStatus.core_temp.toFixed(2) + ' °C'}
+							{systemStatus.core_temp == 53.33 ? 'NaN' : systemStatus.core_temp.toFixed(2) + ' °C'}
 						</div>
 					</div>
 				</div>