v1.1.90/v1.1.91 changes + Fix validator timeout

.github/workflows/test_and_deploy.yml

@@ -62,10 +62,12 @@ jobs:
           key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
 
       # Configure Rust for 32-bit builds
+      # Please use fixed versions of rust so that installs build consistently
+      # (So they don't randomly trigger Windows Defender)
       - name: Configure Rust for 32-bit
         run: |
-          rustup target add i686-pc-windows-msvc
-          rustup default stable-i686-pc-windows-msvc
+          rustup install 1.52.1-i686-pc-windows-msvc
+          rustup default 1.52.1-i686-pc-windows-msvc
 
       # Run Python Deploy Script
       # This also installs and scans .exe with virustotal on Windows (to try prevent .exe virus false positives)
@@ -86,7 +88,7 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: windows-loader-exe
-          path: travis_installer_output/*.exe
+          path: travis_installer_output/*.*
           if-no-files-found: error
 
   # Linux/Mac Build
@@ -112,7 +114,7 @@ jobs:
       - name: Validate JSON
         run: bash travis_validate_json.sh
 
-      # Download windows .exe artifact
+      # Download Windows artifacts
       - name: Download all Windows .exe artifacts
         uses: actions/download-artifact@v2
         with:
@@ -130,11 +132,9 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/') # only publish tagged commits
         with:
           files: |
-            travis_installer_output/07th-Mod.Installer.mac.zip
-            travis_installer_output/07th-Mod.Installer.linux.tar.gz
-            travis_installer_output/07th-Mod.Installer.Windows.exe
-            travis_installer_output/07th-Mod.Installer.Windows.NoAdmin.exe
-            travis_installer_output/07th-Mod.Installer.Windows.SafeMode.exe
+            travis_installer_output/*.tar.gz
+            travis_installer_output/*.exe
+            travis_installer_output/*.zip
           body_path: github_actions_changelog_template_generated.md
           draft: true
         env:

JSONValidator/Tests/JSONValidatorTests/JSONValidatorTests.swift

@@ -99,13 +99,92 @@ final class JSONValidatorTests: XCTestCase {
 		}
 	}
 
+	class URLTester: NSObject, URLSessionDataDelegate {
+		var session: URLSession!
+		var requests: [URLSessionTask: (req: URLRequest, url: URL, tries: Int, path: String, exp: XCTestExpectation)] = [:]
+
+		override init() {
+			super.init()
+			let config = URLSessionConfiguration.ephemeral
+			config.httpMaximumConnectionsPerHost = 1
+			session = URLSession(configuration: config, delegate: self, delegateQueue: .main)
+		}
+
+		func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) {
+			guard dataTask.state != .canceling else { return }
+			if dataTask.countOfBytesExpectedToReceive > (1024 * 1024) {
+				// Server not honoring range request for a large (multi-megabyte) file!
+				let mb = dataTask.countOfBytesExpectedToReceive / (1024 * 1024)
+				if let info = requests[dataTask] {
+					print("::warning::Server did not honor range request when downloading the \(mb)mb file \(info.url) (path: \(info.path))")
+				} else {
+					let name = dataTask.currentRequest?.url?.absoluteString ?? "<unknown>"
+					print("::warning::Server did not honor range request when downloading the \(mb)mb file \(name)")
+				}
+				dataTask.cancel()
+			}
+		}
+
+		func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
+			guard let (request, url, tries, codingPath, expectation) = requests[task] else {
+				print("Received callback for unregistered task!")
+				return
+			}
+			requests[task] = nil
+			if let error = error, (error as NSError).code != NSURLErrorCancelled {
+				if tries > 1, (error as NSError).code == NSURLErrorTimedOut {
+					// If it times out, try again
+					print("Attempt to download \(url) timed out, \(tries - 1) retries left")
+					tryDownload(request, fulfilling: expectation, url: url, codingPath: codingPath, tries: tries - 1)
+					return
+				}
+				XCTFail("Failed to download \(url) (at \(codingPath)): \(error)")
+			}
+			else if let response = task.response as? HTTPURLResponse {
+				// Open source Foundation doesn't properly follow HTTP 300s
+				if response.statusCode / 100 == 3, let loc = response.allHeaderFields["location"].flatMap({ $0 as? String }).flatMap(URL.init(string:)) {
+					var request = request
+					request.url = loc
+					tryDownload(request, fulfilling: expectation, url: loc, codingPath: codingPath, tries: tries - 1)
+					return
+				}
+				if response.statusCode != 200 && response.statusCode != 206 {
+					XCTFail("Failed to download \(url) (at \(codingPath)): response code was \(response.statusCode)")
+				}
+			}
+			else if let response = task.response {
+				XCTFail("Failed to download \(url) (at \(codingPath)): unexpected response: \(response)")
+			}
+			else {
+				XCTFail("Failed to download \(url) (at \(codingPath)): got nil response with no error")
+			}
+			expectation.fulfill()
+		}
+
+		/// Attempts to download the given file, fulfilling the given expectation if it succeeds
+		/// If the request times out, the download will be retried for a total of up to `tries` tries.
+		func tryDownload(
+			_ request: URLRequest,
+			fulfilling expectation: XCTestExpectation,
+			url: URL,
+			codingPath: String,
+			tries: Int
+		) {
+			let task = session.dataTask(with: request)
+			requests[task] = (request, url, tries, codingPath, expectation)
+			task.resume()
+		}
+	}
+
 	func testURLsExist() {
 		let decoder = PedanticJSONDecoder()
 		guard let installData = try? decoder.decode(InstallDataDefinition.self, from: Data(contentsOf: installData)) else {
 			XCTFail("Failed to decode install data, look at other tests for details")
 			return
 		}
 
+		let tester = URLTester()
+
 		func testDownload(_ urlString: String, codingPath: String) {
 			guard let url = URL(string: urlString) else {
 				XCTFail("The url \"\(urlString)\" was invalid")
@@ -116,51 +195,9 @@ final class JSONValidatorTests: XCTestCase {
 			request.setValue("bytes=0-1023", forHTTPHeaderField: "Range")
 			request.timeoutInterval = Double.random(in: 4...6) // Use a random interval so if the timeout reason was that the server didn't like our request spam, subsequent requests will be more and more spread out
 
-			tryDownload(request, fulfilling: e, url: url, codingPath: codingPath, tries: 8)
-		}
+			tester.tryDownload(request, fulfilling: e, url: url, codingPath: codingPath, tries: 8)
 
-		/// Attempts to download the given file, fulfilling the given expectation if it succeeds
-		/// If the request times out, the download will be retried for a total of up to `tries` tries.
-		func tryDownload(
-			_ request: URLRequest,
-			fulfilling expectation: XCTestExpectation,
-			url: URL,
-			codingPath: String,
-			tries: Int
-		) {
-			var task: URLSessionDataTask? = nil
-			task = URLSession.shared.dataTask(with: request) { [weak task] (data, response, error) in
-				task?.cancel()
-				if let error = error {
-					if tries > 1, (error as NSError).code == NSURLErrorTimedOut {
-						// If it times out, try again
-						print("Attempt to download \(url) timed out, \(tries - 1) retries left")
-						tryDownload(request, fulfilling: expectation, url: url, codingPath: codingPath, tries: tries - 1)
-						return
-					}
-					XCTFail("Failed to download \(url) (at \(codingPath)): \(error)")
-				}
-				else if let response = response as? HTTPURLResponse {
-					// Open source Foundation doesn't properly follow HTTP 300s
-					if response.statusCode / 100 == 3, let loc = response.allHeaderFields["location"].flatMap({ $0 as? String }).flatMap(URL.init(string:)) {
-						var request = request
-						request.url = loc
-						tryDownload(request, fulfilling: expectation, url: loc, codingPath: codingPath, tries: tries - 1)
-						return
-					}
-					if response.statusCode != 200 && response.statusCode != 206 {
-						XCTFail("Failed to download \(url) (at \(codingPath)): response code was \(response.statusCode)")
-					}
-				}
-				else if let response = response {
-					XCTFail("Failed to download \(url) (at \(codingPath)): unexpected response: \(response)")
-				}
-				else {
-					XCTFail("Failed to download \(url) (at \(codingPath)): got nil response with no error")
-				}
-				expectation.fulfill()
-			}
-			task!.resume()
+			waitForExpectations(timeout: 60) // Workaround for URLSession bug, normally we'd dispatch them all and wait for them all at once.  See https://bugs.swift.org/browse/SR-15214 for details
 		}
 
 		for mod in installData.mods {
@@ -182,8 +219,6 @@ final class JSONValidatorTests: XCTestCase {
 				}
 			}
 		}
-
-		waitForExpectations(timeout: 60)
 	}
 
 	static var allTests = [

common.py

@@ -643,16 +643,27 @@ def getModList(jsonURI, isURL):
 	"""
 	info, exception = getJSON(jsonURI, isURL)
 	if info is None:
-		raise Exception("""------------------------------------------------------------------------
-Error: Couldn't reach Github to download mod list! ({})
+		if exception is HTTPError:
+			raise Exception("""------------------------------------------------------------------------
+	Error: Couldn't reach Github to download mod list! ({})
 
-Please check the following:
-- You have a working internet connection
-- Check if you can manually download the following file ({})
-- Check our Wiki for more solutions: https://www.07th-mod.com/wiki/Installer/faq/
+	Please check the following:
+	- You have a working internet connection
+	- Check if you can manually download the following file ({})
+	- Check our Wiki for more solutions: https://www.07th-mod.com/wiki/Installer/faq/
 
-Detailed Error: {}
-------------------------------------------------------------------------""".format(jsonURI, jsonURI, exception))
+	Detailed Error: {}
+	------------------------------------------------------------------------""".format(jsonURI, jsonURI, exception))
+		else:
+			raise Exception("""------------------------------------------------------------------------
+	Error while reading JSON: {}
+
+	Please check the following:
+	- Tell the developers
+	- You have a working internet connection
+	- Check if you can manually download the following file ({})
+	- Check our Wiki for more solutions: https://www.07th-mod.com/wiki/Installer/faq/
+	------------------------------------------------------------------------""".format(exception, jsonURI))
 
 	try:
 		version = info["version"]

fileVersionManagement.py

@@ -114,7 +114,7 @@ def __init__(self, subMod, modFileList, localVersionFolder, _testRemoteSubModVer
 			print("Full Update: {} ({}/{}) excluding mod options".format(self.fullUpdateRequired(), self.numUpdatesRequired, self.totalNumUpdates))
 
 	def fullUpdateRequired(self):
-		return self.numUpdatesRequired == self.totalNumUpdates
+		return self.localVersionInfo is None
 
 	def getFilesRequiringUpdate(self):
 		#type: () -> List[installConfiguration.ModFile]
@@ -144,7 +144,7 @@ def saveVersionInstallFinished(self, forcedSaveFolder=None):
 		self.remoteVersionInfo.serialize(versionSavePath, lastAttemptedInstallID=self.remoteVersionInfo.id)
 
 	@staticmethod
-	def tryDeleteLocalVersionFile(localVersionFolder):
+	def deleteLocalVersionFileIfExists(localVersionFolder):
 		localVersionFilePathToDelete = os.path.join(localVersionFolder, VersionManager.localVersionFileName)
 		if os.path.exists(localVersionFilePathToDelete):
 			os.remove(localVersionFilePathToDelete)

github_actions_changelog_template.md

@@ -28,9 +28,23 @@ Please make sure you can launch the base game before applying any mods (please d
 
 ### Explanation of Windows Versions of the Installer
 
-1. `07th-Mod.Installer.Windows.exe` - Try this first. Runs the installer in administrator mode.
-2. `07th-Mod.Installer.Windows.NoAdmin.exe` - Use this if you don't want to run the installer in administrator mode.
-3. `07th-Mod.Installer.Windows.SafeMode.exe` - If the installer does not start up at all, or you have other problems, try this version of the installer. It uses a text-based interface for the launcher.
+1. `07th-Mod.Installer.Windows.exe` (Requires Administrator) - Try this first.
+2. `07th-Mod.Installer.Windows.SafeMode.exe` (Requires Administrator) - If the installer does not start up at all, or you have other problems, try this version of the installer. It uses a text-based interface for the launcher.
+3. `07th-Mod.Installer.Windows.NoLauncher.zip` (No Administrator Required): See below instructions
+
+#### No Launcher instructions
+
+- Use this version if the above two do not work. It may be useful if:
+  - You do not have administrator privileges
+  - Your Antivirus software refuses to launch the above two .exes
+  - The above two .exes immediately crash, so you're unable to run the installer
+
+- Usage Instructions
+  - You might need to install the [Visual C++ Redistributable (x86)](https://aka.ms/vs/16/release/vc_redist.x86.exe) ([linked from this page](https://support.microsoft.com/en-au/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0)), but most people will already have it.
+  - Extract the `.zip` file to your desktop or downloads folder (NOT program files or the game root, as this may cause permissions issues)
+  - Double click on the `install.bat` file.
+    - A terminal will open, followed by a special page in your web browser (you use this webpage to install the mod).
+    - If the `install.bat` does not work, close the terminal, then double click the `install_safe_mode.bat` for the text-only installer
 
 ### Important Changes
 

higurashiInstaller.py

@@ -243,22 +243,42 @@ def moveFilesIntoPlace(self):
 		Moves files from the directory they were extracted to
 		to the game data folder
 		"""
+		# On MacOS, the datadirectory has a different path than in the archive file:
+		#
+		# datadirectory in archive file : "Higurashi_Ep0X"
+		# datadirectory on Linux/Windows: "Higurashi_Ep0X"
+		#         datadirectory on Macos: "Contents/Resources/Data"
+		#
+		# To account for this, we rename the "Higurashi_Ep0X" folder to "Contents/Resources/Data" below
+		# (self.dataDirectory should equal "Contents/Resources/Data" on MacOS)
 		self._moveDirectoryIntoPlace(
 			fromDir = os.path.join(self.extractDir, self.info.subModConfig.dataName),
-			toDir = self.dataDirectory
+			toDir = self.dataDirectory,
+			log = True,
 		)
+
 		if common.Globals.IS_WINDOWS:
 			exePath = self.info.subModConfig.dataName[:-5] + ".exe"
 			self._moveFileIntoPlace(
 				fromPath = os.path.join(self.extractDir, exePath),
 				toPath = os.path.join(self.directory, exePath),
+				log=True,
 			)
 		elif common.Globals.IS_MAC:
 			self._moveFileIntoPlace(
 				fromPath = os.path.join(self.extractDir, "Contents/Resources/PlayerIcon.icns"),
-				toPath = os.path.join(self.directory, "Contents/Resources/PlayerIcon.icns")
+				toPath = os.path.join(self.directory, "Contents/Resources/PlayerIcon.icns"),
+				log = True,
 			)
 
+		# If any files still remain, just move them directly into the game directory,
+		# keeping the same folder structure as inside the archive
+		self._moveDirectoryIntoPlace(
+			fromDir = self.extractDir,
+			toDir = self.directory,
+			log = True,
+		)
+
 	def _applyLanguageSpecificSharedAssets(self, folderToApply):
 		"""Helper function which applies language specific assets.
 		Returns False if there was an error during the proccess.
@@ -322,11 +342,14 @@ def applyLanguagePatchFixesIfNecessary(self):
 			                'this error so we can fix it:\n\n'
 			                '"Invalid Language Specific Asset files found: {}"'.format(invalidUIFileList))
 
-	def _moveDirectoryIntoPlace(self, fromDir, toDir):
-		# type: (str, str) -> None
+	def _moveDirectoryIntoPlace(self, fromDir, toDir, log=False):
+		# type: (str, str, Optional[bool]) -> None
 		"""
 		Recursive function that does the actual moving for `moveFilesIntoPlace`
 		"""
+		if log:
+			print("_moveDirectoryIntoPlace: '{}' -> '{}'".format(fromDir, toDir))
+
 		for file in os.listdir(fromDir):
 			src = path.join(fromDir, file)
 			target = path.join(toDir, file)
@@ -340,11 +363,14 @@ def _moveDirectoryIntoPlace(self, fromDir, toDir):
 				shutil.move(src, target)
 		forceRemoveDir(fromDir)
 
-	def _moveFileIntoPlace(self, fromPath, toPath):
-		# type: (str, str) -> None
+	def _moveFileIntoPlace(self, fromPath, toPath, log=False):
+		# type: (str, str, Optional[bool]) -> None
 		"""
 		Moves a single file from `fromPath` to `toPath`
 		"""
+		if log:
+			print("_moveFileIntoPlace: '{}' -> '{}'".format(fromPath, toPath))
+
 		if path.exists(fromPath):
 			if path.exists(toPath):
 				forceRemove(toPath)

httpGUI.py

@@ -829,7 +829,7 @@ def startInstallHandler(requestData):
 				retval = { 'installStarted': installValid }
 				if installValid:
 					if deleteVersionInformation:
-						fileVersionManagement.VersionManager.tryDeleteLocalVersionFile(fullInstallConfiguration.installPath)
+						fileVersionManagement.VersionManager.deleteLocalVersionFileIfExists(fullInstallConfiguration.installPath)
 
 					downloadItemsPreview, totalDownloadSize, numUpdatesRequired, fullUpdateRequired, partialReinstallDetected, scriptNeedsUpdate = getDownloadPreview(fullInstallConfiguration, verbosePrinting=not allowCache)
 					haveEnoughFreeSpace, freeSpaceAdvisoryString = common.checkFreeSpace(

installData.json

@@ -478,7 +478,13 @@
 							["sharedassets0.assets", "A200EC2A85349BC03B59C8E2F106B99ED0CBAAA25FC50928BB8BA2E2AA90FCE9"]
 						]},
 						{"name": "ui",    "id": "ui-windows-old", "os": ["windows"],      "unity": "5.6.7f1", "url":"https://07th-mod.com/ui.php?chapter=matsuribayashi&os=win&unity=2017.2.5"},
+						{"name": "ui",    "id": "ui-windows-gog-upgraded", "os": ["windows"], "unity": "2017.2.5", "url":"https://07th-mod.com/ui.php?chapter=matsuribayashi&os=win&unity=2017.2.5", "targetChecksums": [
+							["sharedassets0.assets.backup", "22890F97997B7B7BD21F5E64F69E1A6FB2E0FF82B3DF6C59860D8D1A56B23BBB"]
+						]},
 						{"name": "ui",    "id": "ui-linux-old", "os": ["linux"], "unity": "5.6.7f1", "url":"https://07th-mod.com/ui.php?chapter=matsuribayashi&os=unix&unity=2017.2.5"},
+						{"name": "ui",    "id": "ui-linux-gog-upgraded", "os": ["linux"], "unity": "2017.2.5", "url":"https://07th-mod.com/ui.php?chapter=matsuribayashi&os=unix&unity=2017.2.5", "targetChecksums": [
+							["sharedassets0.assets.backup", "a1f7b39665d63d704bc14275dcdf71cf4cca9532ac61bee23b5b18b62a5cd3af"]
+						]},
 						{"name": "system", "id": "system-gog-linux", "os": ["linux"], "unity": "5.6.7f1", "url": "https://07th-mod.com/rikachama/matsuri-system/Matsuribayashi-System_unix.7z"},
 						{"name": "system", "id": "system-gog-windows", "os": ["windows"], "unity": "5.6.7f1", "url": "https://07th-mod.com/rikachama/matsuri-system/Matsuribayashi-System.7z"}
 					]