diff --git a/commands/debug2.js b/commands/debug2.js index 238c809a..e3b1e634 100644 --- a/commands/debug2.js +++ b/commands/debug2.js @@ -23,29 +23,45 @@ const { processPak } = require('../support_files/process_pak.js'); const { isMainThread, parentPort, Worker } = require('node:worker_threads') +const { jobs, buildPathArrays } = require('../support_files/conversion_junction'); + const debug2 = vscode.commands.registerCommand('bg3-mod-helper.debug2Command', async function () { // raiseInfo("hi dipshit! 💩"); - let halfCoreCount = os.availableParallelism() / 2; + let filesToConvert = await FIND_FILES(pak, path.join(gameInstallLocation, "Data")); + let workerArray = []; let workerConfig = JSON.parse(loadConfigFile(true)); - // let workerLSLIB = await LOAD_LSLIB(); - if (isMainThread) { - for (let i = 0; i < halfCoreCount; i++) { - workerArray.push(new Worker(__dirname + "/worker_test.js", { workerData: workerConfig })); + let jobsTotal = jobs(filesToConvert.length); - workerArray[i].on('message', (message) => { - console.log(`${message} received from worker ${workerArray[i].threadId}!`) - }); + let temp_dir = path.normalize("W:\\Libraries\\Documents\\Baldur's Gate 3 Mods\\unpacking_tests\\unpacked_game_data"); - workerArray[i].postMessage(workerConfig); + if (isMainThread) { + // console.log(filesToConvert); + filesToConvert = buildPathArrays(filesToConvert); + raiseInfo(`unpacking ${filesToConvert} started.`) + + // will only unpack + for (let i = 0; i < jobsTotal; i++) { + workerArray.push(new Worker(__dirname + "/worker_test.js", { + workerData: { + // passes the crystallized configuration settings to each of the workers + workerConfig, + // adding 2 to the workerId because 0 is the extension host and 1 is the main window + workerId: i + 2, + // passes the path of the file that needs to be converted + jobsTotal, + task: filesToConvert[i], + jobDestPath: temp_dir + } + })); } - } - }); + + module.exports = { debug2 } diff --git a/commands/worker_test.js b/commands/worker_test.js index fc9e4959..a720044a 100644 --- a/commands/worker_test.js +++ b/commands/worker_test.js @@ -3,16 +3,21 @@ const bg3mh_logger = CREATE_LOGGER(); const { parentPort, workerData } = require('node:worker_threads'); const path = require('node:path'); const fs = require('node:fs'); -const { LOAD_LSLIB } = require('../support_files/lslib_utils.js') -parentPort.once('message', async () => { - raiseInfo(`lookee lslib @ ${workerData.lslibPath}`) - parentPort.postMessage("i got the workerData bitchhhhhhh"); +const { convert } = require('../support_files/conversion_junction.js') - /*try { - let LSLIB = await LOAD_LSLIB(); - } catch (Error) { - raiseError(Error); - }*/ -}); \ No newline at end of file +function taskIntake() { + // console.log(workerData.task) + if (Array.isArray(workerData.task)) { + for (let i = 0; i < workerData.task.length; i++) { + console.log(`converting ${workerData.task[i]}`) + convert(workerData.task[i]); + } + } else if (typeof(workerData.task) == 'string') { + convert(workerData.task); + } + +} + +taskIntake(); \ No newline at end of file diff --git a/extension.js b/extension.js index ecd3789a..967cb99c 100644 --- a/extension.js +++ b/extension.js @@ -101,6 +101,7 @@ function activate(context) { }); } + console.log(loadConfigFile(true)); if (loadConfigFile(true) === undefined) { saveConfigFile(); } diff --git a/support_files/conversion_junction.js b/support_files/conversion_junction.js index d39f4935..2ed210a5 100644 --- a/support_files/conversion_junction.js +++ b/support_files/conversion_junction.js @@ -1,23 +1,73 @@ // what's your function const path = require('path'); +const os = require('os'); const fs = require('fs'); -const vscode = require('vscode'); -const { FIND_FILES, getFormats } = require('./lslib_utils'); +const { FIND_FILES, getFormats, compatRootModPath } = require('./lslib_utils'); const { lsx, xml, pak } = getFormats(); const { CREATE_LOGGER, raiseError, raiseInfo } = require('./log_utils'); const bg3mh_logger = CREATE_LOGGER(); -const { getConfig, getModName } = require('./config.js'); - const { isLoca, processLoca, getLocaOutputPath } = require('./loca_convert'); const { isLsf, processLsf, getLsfOutputPath } = require('./lsf_convert'); const { processPak, prepareTempDir } = require('./process_pak'); +const { isMainThread, workerData } = require('node:worker_threads'); + +let getConfig, getModName; + +if (isMainThread) { + getConfig = require('./config.js').getConfig(); + getModName = require('./config.js').getModName(); +} else { + getConfig = workerData.workerConfig; + getModName = require('./lslib_utils.js').getModName(); +} + function getActiveTabPath() { - return vscode.window.activeTextEditor.document.fileName; + if (isMainThread) { + const vscode = require('vscode'); + return vscode.window.activeTextEditor.document.fileName; + } else { + return undefined; + } + +} + +function jobs(filesNum) { + let halfCoreCount = Math.floor(os.availableParallelism() / 2); + + return Math.min(filesNum, halfCoreCount); +} + + +function buildPathArrays(filesToConvert) { + let filesNum = filesToConvert.length; + let jobsNum = jobs(filesNum); + + let temp_array = []; + let fileArrays = []; + + if (jobsNum >= filesNum) { + return filesToConvert; + } else if (jobsNum < filesNum) { + for (let i = 0; i < jobsNum; i++) { + temp_array = []; + for (let j = 0; j < filesNum; j += jobsNum) { + let temp_index = i + j; + + if (filesToConvert[temp_index] !== undefined) { + // console.log(filesToConvert[temp_index]); + temp_array.push(filesToConvert[temp_index]); + // console.log(temp_array); + } + } + fileArrays.push(temp_array); + } + return fileArrays; + } } @@ -41,10 +91,10 @@ function getDynamicPath(filePath) { } -async function convert(convertPath, targetExt = path.extname(getDynamicPath(convertPath)), modName = getModName()) { - const { rootModPath } = getConfig(); +async function convert(convertPath, targetExt = path.extname(getDynamicPath(convertPath)), modName = getModName) { + let rootModPath = getConfig.rootModPath; - console.log('targetExt:' + targetExt); + // console.log('targetExt:' + targetExt); if (targetExt === "empty") { return; } @@ -62,7 +112,12 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv processPak(rootModPath, modName); } else if (fs.statSync(convertPath).isFile()) { - processPak(convertPath, modName, 'n/a'); + if (isMainThread) { + processPak(convertPath, modName); + } else { + processPak(convertPath, modName, workerData.jobDestPath); + } + } } else if (Array.isArray(convertPath)) { @@ -100,4 +155,4 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv } -module.exports = { convert }; \ No newline at end of file +module.exports = { convert, jobs, buildPathArrays }; \ No newline at end of file diff --git a/support_files/loca_convert.js b/support_files/loca_convert.js index 1f57d937..e126ccb4 100644 --- a/support_files/loca_convert.js +++ b/support_files/loca_convert.js @@ -1,5 +1,4 @@ const path = require('path'); -const vscode = require('vscode'); const { getFormats, baseNamePath, LOAD_LSLIB } = require('./lslib_utils'); const { CREATE_LOGGER, raiseError, raiseInfo } = require('./log_utils'); diff --git a/support_files/log_utils.js b/support_files/log_utils.js index c1ff4cab..a29218a2 100644 --- a/support_files/log_utils.js +++ b/support_files/log_utils.js @@ -87,8 +87,8 @@ if (isMainThread) { bg3mh_logger.info(info); } } else { - lslibPath = workerData.lslibPath; - logPath = path.normalize(lslibPath + "\\logs\\bg3mh_log_" + LOGDATE() + ".log"); + lslibPath = workerData.workerConfig.lslibPath; + logPath = path.normalize(lslibPath + "\\logs\\bg3mh_log_" + LOGDATE() + "_worker_thread.log"); CREATE_LOGGER = () => { var log4js = require('log4js'); @@ -99,7 +99,7 @@ if (isMainThread) { filename: logPath, layout: { type: "pattern", - pattern: "[%d] %f{1}:%l in %M [%p]: %m" + pattern: "[%d]" + "[from worker_thread_" + workerData.workerId + "] %f{1}:%l in %M [%p]: %m" } } }, @@ -128,14 +128,15 @@ if (isMainThread) { return log4js.getLogger("bg3mh_logger"); } - raiseError = (error) => { + // extra arg for compat between main and worker threads + raiseError = (error, popup = Boolean) => { var bg3mh_logger = CREATE_LOGGER(); console.error(error); bg3mh_logger.error(error); } - raiseInfo = (info) => { + raiseInfo = (info, popup = Boolean) => { var bg3mh_logger = CREATE_LOGGER(); console.info(info); diff --git a/support_files/lsf_convert.js b/support_files/lsf_convert.js index 55ae2c2e..4a3b22ba 100644 --- a/support_files/lsf_convert.js +++ b/support_files/lsf_convert.js @@ -1,6 +1,5 @@ const path = require('path'); const fs = require('fs'); -const vscode = require('vscode'); const { getFormats, baseNamePath, LOAD_LSLIB } = require('./lslib_utils'); diff --git a/support_files/lslib_utils.js b/support_files/lslib_utils.js index b3e2e90d..299c7458 100644 --- a/support_files/lslib_utils.js +++ b/support_files/lslib_utils.js @@ -32,6 +32,10 @@ const convertDirs = ["[PAK]_UI", "[PAK]_Armor", "RootTemplates", "MultiEffectInf // excluding this because it will match to "UI" in convertDirs const illegalFiles = ["Icons_Items.lsx"]; +// excluding these packs because lslib uses something else to unpack them +const virtualTexturePaks = ["VirtualTextures_"]; +const hotfixPatchPaks = ["Patch", "Hotfix"] + // tools to test where the process is const { isMainThread, workerData } = require('node:worker_threads'); @@ -49,7 +53,7 @@ if (isMainThread) { findFiles = vscode.workspace.findFiles; parse = vscode.Uri.parse; } else { - getConfig = workerData; + getConfig = workerData.workerConfig; lslibPath = getConfig.lslibPath; compatRootModPath = path.join(getConfig.rootModPath + "\\"); lslibToolsPath = path.join(getConfig.lslibPath, TOOL_SUBDIR); @@ -145,8 +149,11 @@ async function LOAD_LSLIB() { processDllPaths(); await loadDlls(); - bg3mh_logger.info(`${DLL_PATHS} \n.dlls loaded`); - console.log(`${DLL_PATHS} \n.dlls loaded`); + if (isMainThread) { + raiseInfo(`${DLL_PATHS} \n.dlls loaded`, false); + } else { + // raiseInfo(`.dlls loaded into worker ${workerData.workerId}`) + } // have to ignore this because the ts-linter doesn't know 'LSLib' exists :starege: // @ts-ignore @@ -237,6 +244,9 @@ function FILTER_PATHS(filesPath) { // these statements could technically be combined, but that doesn't make it very readable. if (temp_ext === getFormats().dll && !illegalDlls.includes(path.basename(filesPath))) { return filesPath; + } + else if (filesPath.includes(virtualTexturePaks) || hotfixPatchPaks.includes(temp) { + } else if ( ( @@ -280,9 +290,53 @@ function moveFileAcrossDevices(sourcePath, destPath, raiseError) { raiseInfo(infoMsg, false); vscode.window.showInformationMessage(infoMsg); } else { - raiseInfo(infoMsg); + raiseInfo(infoMsg, false); + } +} + +// i don't like putting this here but i need a worker_thread friendly version +function getModName() { + let rootModPath; + + if (isMainThread) { + rootModPath = getConfig().rootModPath; + } else { + rootModPath = getConfig.rootModPath; + } + + let modsDirPath = path.join(rootModPath, 'Mods'); + + try { + if (!fs.existsSync(modsDirPath)) { + if (isMainThread) { + vscode.window.showErrorMessage('Mods directory does not exist.'); + } else { + raiseError('Mods directory does not exist.'); + } + + return ''; + } + + const files = fs.readdirSync(modsDirPath); + const directories = files.filter(file => + fs.statSync(path.join(modsDirPath, file)).isDirectory() + ); + + if (directories.length === 1) { + return directories[0]; + } else { + return ''; + } + } catch (error) { + if (isMainThread) { + vscode.window.showErrorMessage(`Error reading directories in ${modsDirPath}: ${error}`); + } else { + raiseError(`Error reading directories in ${modsDirPath}: ${error}`); + } + + return ''; } } -module.exports = { LSLIB, LOAD_LSLIB, FIND_FILES, FIND_FILES_SYNC, FILTER_PATHS, getFormats, moveFileAcrossDevices, baseNamePath, dirSeparator, compatRootModPath }; \ No newline at end of file +module.exports = { LSLIB, LOAD_LSLIB, FIND_FILES, FIND_FILES_SYNC, FILTER_PATHS, getFormats, moveFileAcrossDevices, baseNamePath, dirSeparator, compatRootModPath, getModName }; \ No newline at end of file diff --git a/support_files/process_pak.js b/support_files/process_pak.js index 2e50deab..156c87e9 100644 --- a/support_files/process_pak.js +++ b/support_files/process_pak.js @@ -1,27 +1,46 @@ const path = require('path'); -const vscode = require('vscode'); const fs = require('fs'); const { getFormats, moveFileAcrossDevices, compatRootModPath, LOAD_LSLIB } = require('./lslib_utils'); const { pak } = getFormats(); +const { isMainThread, workerData } = require('node:worker_threads'); + const { CREATE_LOGGER, raiseError, raiseInfo } = require('./log_utils'); var bg3mh_logger = CREATE_LOGGER(); -const { getConfig } = require('./config.js'); - const temp_folder = "\\temp_folder"; // const temp_path = path.join(rootParentPath, temp_folder); -var LSLIB; +var LSLIB, getConfig, vscode; async function lslib_load() { - LSLIB = await LOAD_LSLIB(); + if (LSLIB === undefined) { + console.log("lslib not found. loading..."); + LSLIB = await LOAD_LSLIB(); + // console.log(typeof(LSLIB)) + } else { + console.log("lslib is already loaded!"); + } +} + + +if (isMainThread) { + vscode = require('vscode'); + getConfig = require('./config.js').getConfig(); +} else { + getConfig = workerData.workerConfig; } // this function is getting redone for next release function prepareTempDir(movedPak = false) { - const { rootModPath } = getConfig(); + let rootModPath; + if (isMainThread) { + rootModPath = getConfig().rootModPath; + } else { + rootModPath = getConfig.rootModPath; + } + const rootParentPath = path.dirname(rootModPath); const temp_path = path.join(rootParentPath, temp_folder); @@ -48,18 +67,28 @@ function prepareTempDir(movedPak = false) { // btw, sometimes this will log things before others because it's async. -async function processPak(modPath, modName_, unpackLocation = '') { +async function processPak(modPath, modName, unpackLocation = path.join(path.dirname(modPath), path.basename(modPath, pak))) { + await lslib_load(); var build = new LSLIB.PackageBuildData(); var Packager = new LSLIB.Packager(); - const { rootModPath, modDestPath } = getConfig(); + let rootModPath, modDestPath; + + if (isMainThread) { + rootModPath = getConfig().rootModPath; + modDestPath = getConfig().modDestPath; + + } else { + rootModPath = getConfig.rootModPath; + modDestPath = getConfig.modDestPath; + } const lastFolderName = path.basename(rootModPath); const rootParentPath = path.dirname(rootModPath); const temp_path = path.join(rootParentPath, temp_folder); const modFinalDestPath = path.join(modDestPath, lastFolderName + pak); const modTempDestPath = path.join(temp_path, lastFolderName + pak); - const metaPath = path.join(rootModPath, 'Mods', modName_, 'meta.lsx'); + const metaPath = path.join(rootModPath, 'Mods', modName, 'meta.lsx'); try { @@ -84,10 +113,13 @@ async function processPak(modPath, modName_, unpackLocation = '') { // Write the updated XML back to the file fs.writeFileSync(metaPath, xmlContent, 'utf8'); bg3mh_logger.info('meta.lsx updated successfully.'); + await Packager.CreatePackage(modTempDestPath, modPath, build); raiseInfo(lastFolderName + pak + " packed", false); - vscode.window.showInformationMessage(`${lastFolderName + pak} packed`); + if (isMainThread) { + vscode.window.showInformationMessage(`${lastFolderName + pak} packed`); + } // move files to chosen path and [in progress] clean up the empty directory moveFileAcrossDevices(modTempDestPath, modFinalDestPath); @@ -98,5 +130,4 @@ async function processPak(modPath, modName_, unpackLocation = '') { } } -lslib_load(); module.exports = { processPak, prepareTempDir }; \ No newline at end of file