diff --git a/commands/unpackGameData.js b/commands/unpackGameData.js index f24e0c87..a1e2d0eb 100644 --- a/commands/unpackGameData.js +++ b/commands/unpackGameData.js @@ -24,81 +24,40 @@ const { processPak } = require('../support_files/process_pak.js'); const { isMainThread, MessageChannel, Worker } = require('node:worker_threads'); const comsPorts = new MessageChannel(); -const { jobs, buildPathArrays } = require('../support_files/conversion_junction'); +const { createConversionWorkers } = require('../support_files/conversion_junction'); -let temp_dir; +let unpackedGameDataDirectory; async function getTempDir() { - temp_dir = await vscode.window.showOpenDialog({ + unpackedGameDataDirectory = await vscode.window.showOpenDialog({ canSelectFiles: false, canSelectFolders: true, canSelectMany: false, title: 'Unpacked game data folder. Needs at least 160GB free space.' }) - if (!temp_dir || temp_dir.length === 0) { - vscode.window.showInformationMessage('No folder selected.'); + if (!unpackedGameDataDirectory || unpackedGameDataDirectory.length === 0) { + vscode.window.showErrorMessage('No folder selected.'); return false; } else { return true; } } - -const unpackGameData = vscode.commands.registerCommand('bg3-mod-helper.unpackGameDataCommand', async function () { - // raiseInfo("hi dipshit! 💩"); +// eventually we can add specific game data files for unpacking, but atm it'll just grab everything. +const unpackGameData = vscode.commands.registerCommand('bg3-mod-helper.unpackGameDataCommand', async function () { let filesToConvert = await FIND_FILES(pak, path.join(gameInstallLocation, "Data")); - - // an array of workers - let workerArray = []; let workerConfig = JSON.parse(loadConfigFile(true)); - // the jobs function returns the lower of half the user's cpu cores or the number of things being unpacked - let jobsTotal = jobs(filesToConvert.length); - - let jobsFinished = 0; - // made this a boolean so things don't get converted unless an unpack location is selected if (isMainThread && await getTempDir()) { // console.log(filesToConvert); - filesToConvert = buildPathArrays(filesToConvert); + createConversionWorkers(filesToConvert, workerConfig, unpackedGameDataDirectory); raiseInfo(`Game data unpacking started. Please allow a few hours for this to complete, it's a big game.`); - - // nested .map() functionality :o - let convertFileNames = filesToConvert.map(fileArray => fileArray.map(file => path.basename(file))); - - // for each job, which is either a single file or an array of files sorted from smallest to largest file size, create a worker and give it the data it needs to unpack those files via workerData. - for (let i = 0; i < jobsTotal; i++) { - raiseInfo(`${convertFileNames[i]}\n`, false); - workerArray.push(new Worker(__dirname + "/conversion_worker.js", { - workerData: { - // passes the crystallized configuration settings to each of the workers - workerConfig, - // adding 2 to the workerId because 0 is the extension host and 1 is the main window - workerId: i + 2, - // passes the path of the file that needs to be converted - jobsTotal, - task: filesToConvert[i], - jobDestPath: temp_dir, - } - })); - - workerArray[i].on('message', (message) => { - if (message.includes("done.")) { - raiseInfo(message); - jobsFinished++; - } - - if (jobsFinished === jobsTotal) { - raiseInfo("All game data unpacked!") - } - }) - } } }); - module.exports = { unpackGameData } diff --git a/support_files/conversion_junction.js b/support_files/conversion_junction.js index 758392f8..af0e0539 100644 --- a/support_files/conversion_junction.js +++ b/support_files/conversion_junction.js @@ -3,20 +3,21 @@ const path = require('path'); const os = require('os'); const fs = require('fs'); -const { FIND_FILES, getFormats, compatRootModPath } = require('./lslib_utils'); +const { FIND_FILES, getFormats } = require('./lslib_utils'); const { lsx, xml, pak } = getFormats(); -const { CREATE_LOGGER, raiseError, raiseInfo } = require('./log_utils'); -const bg3mh_logger = CREATE_LOGGER(); +const { raiseError, raiseInfo } = require('./log_utils'); -const { isLoca, processLoca, getLocaOutputPath } = require('./loca_convert'); -const { isLsf, processLsf, getLsfOutputPath } = require('./lsf_convert'); +const { isLoca, processLoca } = require('./loca_convert'); +const { isLsf, processLsf } = require('./lsf_convert'); const { processPak, prepareTempDir } = require('./process_pak'); -const { isMainThread, workerData } = require('node:worker_threads'); +const { isMainThread, workerData, Worker } = require('node:worker_threads'); let getConfig, getModName; + +// need this outside a function so it's run on load. if (isMainThread) { getConfig = require('./config.js').getConfig(); getModName = require('./config.js').getModName(); @@ -25,7 +26,7 @@ if (isMainThread) { getModName = require('./lslib_utils.js').getModName(); } - +// this functionality can't work on a worker thread, so i don't think it's possible for it to return undefined. i still need it to not attempt to load the vscode module while in a worker thread, though. function getActiveTabPath() { if (isMainThread) { const vscode = require('vscode'); @@ -33,9 +34,9 @@ function getActiveTabPath() { } else { return undefined; } - } + function jobs(filesNum) { let halfCoreCount = Math.floor(os.availableParallelism() / 2); @@ -43,6 +44,7 @@ function jobs(filesNum) { } +// sorts the files into nested arrays if the amount of files is greater than the number of cores. if not, will just return the original array. function buildPathArrays(filesToConvert) { let filesNum = filesToConvert.length; let jobsNum = jobs(filesNum); @@ -50,18 +52,19 @@ function buildPathArrays(filesToConvert) { let temp_array = []; let fileArrays = []; + // if these are equal, it will just be one file per core anyways if (jobsNum >= filesNum) { return filesToConvert; + // divides files into equal arrrays and sorts them smallest to largest } else if (jobsNum < filesNum) { for (let i = 0; i < jobsNum; i++) { temp_array = []; for (let j = 0; j < filesNum; j += jobsNum) { let temp_index = i + j; + // makes sure we're not trying to read beyond the bounds of the number of jobs if (filesToConvert[temp_index] !== undefined) { - // console.log(filesToConvert[temp_index]); temp_array.push(filesToConvert[temp_index]); - // console.log(temp_array); } } fileArrays.push(temp_array.sort((file1, file2) => fs.statSync(file1).size - fs.statSync(file2).size)); @@ -71,9 +74,57 @@ function buildPathArrays(filesToConvert) { } +async function createConversionWorkers(filesToConvert, workerConfig, unpackedGameDataDirectory) { + // an array of workers + const workerArray = []; + + filesToConvert = buildPathArrays(filesToConvert); + + // the jobs function returns the lower of half the user's cpu cores or the number of things being unpacked. + const jobsTotal = jobs(filesToConvert.length); + + // tracks the number of jobs reported completed by the worker threads. + let jobsFinished = 0; + + // nested .map() functionality :o + const convertFileNames = filesToConvert.map(fileArray => fileArray.map(file => path.basename(file))); + + // for each job, which is either a single file or an array of files sorted from smallest to largest file size, create a worker and give it the data it needs to unpack those files via workerData. + for (let i = 0; i < jobsTotal; i++) { + raiseInfo(`${convertFileNames[i]}\n`, false); + workerArray.push(new Worker(__dirname + "/conversion_worker.js", { + workerData: { + // passes the crystallized configuration settings to each of the workers + workerConfig, + // adding 2 to the workerId because 0 is the extension host and 1 is the main window + workerId: i + 2, + // passes the path of the file that needs to be converted + task: filesToConvert[i], + jobDestPath: unpackedGameDataDirectory + } + })); + + // creates a listener for the newly pushed worker. if the worker sends back a message containing "done.", adds 1 to the jobsFinished tracker variable. + workerArray[i].on('message', (message) => { + if (message.includes("done.")) { + raiseInfo(message); + jobsFinished++; + } + + // once the jobsFinished variable is equal to the amount of jobs we started with, we know all jobs are done and can tell the user that. + if (jobsFinished === jobsTotal) { + raiseInfo("All game data unpacked!"); + } + }) + } +} + + +// basically just type-checking for the convertPath arg so path.extname doesn't throw a tantrum function getDynamicPath(filePath) { let temp_path; + //this bit is important since we will only be handing off arrays of specific types to convert() if (Array.isArray(filePath) && filePath != []) { temp_path = filePath[0]; } @@ -91,14 +142,16 @@ function getDynamicPath(filePath) { } +// at the moment this has all the functionality i planned for it, ie lsf, loca, and paks. for any other conversions we can make separate functions async function convert(convertPath, targetExt = path.extname(getDynamicPath(convertPath)), modName = getModName) { let rootModPath = getConfig.rootModPath; - // console.log('targetExt:' + targetExt); + // checks if the convertPath was undefined and halts the function before it goes any further if (targetExt === "empty") { return; } + // if the targetExt indicates a pack, we need to know what to do with it: are we packing up or unpacking? if (targetExt === pak) { if (fs.statSync(convertPath).isDirectory()) { prepareTempDir(); @@ -111,6 +164,7 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv processPak(rootModPath, modName); } + // has a check for main thread here because when a worker thread calls this function, it's batch unpacking and has a specific place it needs the files inside to go, but can't rely on vscode module functions to get them else if (fs.statSync(convertPath).isFile()) { if (isMainThread) { processPak(convertPath, modName); @@ -120,17 +174,20 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv } } + // this function works best on single files, so we need to process that array and re-run this function with each of its elements else if (Array.isArray(convertPath)) { // console.log('array1') for (let i = 0; i < convertPath.length; i++) { convert(convertPath[i], path.extname(convertPath[i])); } } + // if this function is passsed a directory, parse it into an array of files of the specified type in targetExt, then send that back through else if (fs.statSync(convertPath).isDirectory()) { // console.log('plz1') const filesToConvert = await FIND_FILES(targetExt); convert(filesToConvert); } + // finally, if this function is handed a file, convert that bitch :catyes: else if (fs.statSync(convertPath).isFile()) { // console.log('plz2') if (isLoca(targetExt)) { @@ -155,4 +212,4 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv } -module.exports = { convert, jobs, buildPathArrays }; \ No newline at end of file +module.exports = { convert, jobs, createConversionWorkers }; \ No newline at end of file diff --git a/commands/conversion_worker.js b/support_files/conversion_worker.js similarity index 72% rename from commands/conversion_worker.js rename to support_files/conversion_worker.js index 146eef19..81ff9eb1 100644 --- a/commands/conversion_worker.js +++ b/support_files/conversion_worker.js @@ -1,20 +1,20 @@ -const { CREATE_LOGGER, raiseError, raiseInfo } = require('../support_files/log_utils.js'); +const { CREATE_LOGGER, raiseError, raiseInfo } = require('./log_utils.js'); const bg3mh_logger = CREATE_LOGGER(); const { parentPort, workerData } = require('node:worker_threads'); const path = require('node:path'); const fs = require('node:fs'); -const { convert } = require('../support_files/conversion_junction.js') +const { convert } = require('./conversion_junction.js') function taskIntake() { - // console.log(workerData.task) + // basic array or file handling stuff if (Array.isArray(workerData.task)) { for (let i = 0; i < workerData.task.length; i++) { try { raiseInfo(`converting ${workerData.task[i]}`); - // convert(workerData.task[i]); + convert(workerData.task[i]); } catch (Error) { raiseError(`converting ${workerData.task[i]}\n failed with error ${Error}`); } @@ -22,13 +22,15 @@ function taskIntake() { } else if (typeof(workerData.task) == 'string') { try { raiseInfo(`converting ${workerData.task}`); - // convert(workerData.task); + convert(workerData.task); } catch (Error) { raiseError(`converting ${workerData.task}\n failed with error ${Error}`); } } + // let the main thread know you're done parentPort.postMessage(`worker ${workerData.workerId} done.`); } +// start it up babyyyyy taskIntake(); \ No newline at end of file