sleepy but working on pak filtering

commands/debug2.js

@@ -23,29 +23,45 @@ const { processPak } = require('../support_files/process_pak.js');
 
 const { isMainThread, parentPort, Worker } = require('node:worker_threads')
 
+const { jobs, buildPathArrays } = require('../support_files/conversion_junction');
+
 
 const debug2 = vscode.commands.registerCommand('bg3-mod-helper.debug2Command', async function () {
     // raiseInfo("hi dipshit! 💩");
 
-    let halfCoreCount = os.availableParallelism() / 2;
+    let filesToConvert = await FIND_FILES(pak, path.join(gameInstallLocation, "Data"));
+
     let workerArray = [];
     let workerConfig = JSON.parse(loadConfigFile(true));
-    // let workerLSLIB = await LOAD_LSLIB();
 
-    if (isMainThread) {
-        for (let i = 0; i < halfCoreCount; i++) {
-            workerArray.push(new Worker(__dirname + "/worker_test.js", { workerData: workerConfig }));
+    let jobsTotal = jobs(filesToConvert.length);
 
-            workerArray[i].on('message', (message) => {
-                console.log(`${message} received from worker ${workerArray[i].threadId}!`)
-            });
+    let temp_dir = path.normalize("W:\\Libraries\\Documents\\Baldur's Gate 3 Mods\\unpacking_tests\\unpacked_game_data");
 
-            workerArray[i].postMessage(workerConfig);
+    if (isMainThread) {
+        // console.log(filesToConvert);
+        filesToConvert = buildPathArrays(filesToConvert);
+        raiseInfo(`unpacking ${filesToConvert} started.`)
+
+        // will only unpack 
+        for (let i = 0; i < jobsTotal; i++) {
+            workerArray.push(new Worker(__dirname + "/worker_test.js", { 
+                workerData:  { 
+                    // passes the crystallized configuration settings to each of the workers
+                    workerConfig, 
+                    // adding 2 to the workerId because 0 is the extension host and 1 is the main window
+                    workerId: i + 2, 
+                    // passes the path of the file that needs to be converted
+                    jobsTotal, 
+                    task: filesToConvert[i],
+                    jobDestPath: temp_dir
+                }
+            }));
         }
-
     }
 
-
 });
 
+
+
 module.exports = { debug2 }

commands/worker_test.js

@@ -3,16 +3,21 @@ const bg3mh_logger = CREATE_LOGGER();
 const { parentPort, workerData } = require('node:worker_threads');
 const path = require('node:path');
 const fs = require('node:fs');
-const { LOAD_LSLIB } = require('../support_files/lslib_utils.js')
 
 
-parentPort.once('message', async () => {
-    raiseInfo(`lookee lslib @ ${workerData.lslibPath}`)
-    parentPort.postMessage("i got the workerData bitchhhhhhh");
+const { convert } = require('../support_files/conversion_junction.js')
 
-    /*try {
-        let LSLIB = await LOAD_LSLIB();
-    } catch (Error) {
-        raiseError(Error);
-    }*/
-});
+function taskIntake() {
+    // console.log(workerData.task)
+    if (Array.isArray(workerData.task)) {
+        for (let i = 0; i < workerData.task.length; i++) {
+            console.log(`converting ${workerData.task[i]}`)
+            convert(workerData.task[i]);
+        }
+    } else if (typeof(workerData.task) == 'string') {
+        convert(workerData.task);
+    }
+
+}
+
+taskIntake();

extension.js

@@ -101,6 +101,7 @@ function activate(context) {
             });
     }
 
+    console.log(loadConfigFile(true));
     if (loadConfigFile(true) === undefined) {
         saveConfigFile();
     }

support_files/conversion_junction.js

@@ -1,23 +1,73 @@
 // what's your function
 const path = require('path');
+const os = require('os');
 const fs = require('fs');
-const vscode = require('vscode');
 
-const { FIND_FILES, getFormats } = require('./lslib_utils');
+const { FIND_FILES, getFormats, compatRootModPath } = require('./lslib_utils');
 const { lsx, xml, pak } = getFormats();
 
 const { CREATE_LOGGER, raiseError, raiseInfo } = require('./log_utils');
 const bg3mh_logger = CREATE_LOGGER(); 
 
-const { getConfig, getModName } = require('./config.js');
-
 const { isLoca, processLoca, getLocaOutputPath } = require('./loca_convert');
 const { isLsf, processLsf, getLsfOutputPath } = require('./lsf_convert');
 const { processPak, prepareTempDir } = require('./process_pak');
 
+const { isMainThread, workerData } = require('node:worker_threads');
+
+let getConfig, getModName;
+
+if (isMainThread) {
+    getConfig = require('./config.js').getConfig();
+    getModName = require('./config.js').getModName();
+} else {
+    getConfig = workerData.workerConfig;
+    getModName = require('./lslib_utils.js').getModName();
+}
+
 
 function getActiveTabPath() {
-    return vscode.window.activeTextEditor.document.fileName;
+    if (isMainThread) {
+        const vscode = require('vscode');
+        return vscode.window.activeTextEditor.document.fileName;
+    } else {
+        return undefined;
+    }
+
+}
+
+function jobs(filesNum) {
+    let halfCoreCount = Math.floor(os.availableParallelism() / 2);
+
+    return Math.min(filesNum, halfCoreCount);
+}
+
+
+function buildPathArrays(filesToConvert) {
+    let filesNum = filesToConvert.length;
+    let jobsNum = jobs(filesNum);
+
+    let temp_array = [];
+    let fileArrays = [];
+
+    if (jobsNum >= filesNum) {
+        return filesToConvert;
+    } else if (jobsNum < filesNum) {
+        for (let i = 0; i < jobsNum; i++) {
+            temp_array = [];
+            for (let j = 0; j < filesNum; j += jobsNum) {
+                let temp_index = i + j;
+
+                if (filesToConvert[temp_index] !== undefined) {
+                    // console.log(filesToConvert[temp_index]);
+                    temp_array.push(filesToConvert[temp_index]);
+                    // console.log(temp_array);
+                }
+            }
+            fileArrays.push(temp_array);
+        }
+        return fileArrays;
+    }
 }
 
 
@@ -41,10 +91,10 @@ function getDynamicPath(filePath) {
 }
 
 
-async function convert(convertPath, targetExt = path.extname(getDynamicPath(convertPath)), modName = getModName()) {
-    const { rootModPath } = getConfig();
+async function convert(convertPath, targetExt = path.extname(getDynamicPath(convertPath)), modName = getModName) {
+    let rootModPath = getConfig.rootModPath;
 
-    console.log('targetExt:' + targetExt);
+    // console.log('targetExt:' + targetExt);
     if (targetExt === "empty") {
         return;
     }
@@ -62,7 +112,12 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv
             processPak(rootModPath, modName);
         }
         else if (fs.statSync(convertPath).isFile()) {
-            processPak(convertPath, modName, 'n/a');
+            if (isMainThread) {
+                processPak(convertPath, modName);
+            } else {
+                processPak(convertPath, modName, workerData.jobDestPath);
+            }
+
         }
     } 
     else if (Array.isArray(convertPath)) {
@@ -100,4 +155,4 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv
 }
 
 
-module.exports = { convert };
+module.exports = { convert, jobs, buildPathArrays };

support_files/loca_convert.js

@@ -1,5 +1,4 @@
 const path = require('path');
-const vscode = require('vscode');
 
 const { getFormats, baseNamePath, LOAD_LSLIB } = require('./lslib_utils');
 const { CREATE_LOGGER, raiseError, raiseInfo } = require('./log_utils');

support_files/log_utils.js

@@ -87,8 +87,8 @@ if (isMainThread) {
         bg3mh_logger.info(info);
     }
 } else {
-    lslibPath = workerData.lslibPath;
-    logPath = path.normalize(lslibPath + "\\logs\\bg3mh_log_" + LOGDATE() + ".log");
+    lslibPath = workerData.workerConfig.lslibPath;
+    logPath = path.normalize(lslibPath + "\\logs\\bg3mh_log_" + LOGDATE() + "_worker_thread.log");
 
     CREATE_LOGGER = () => {
         var log4js = require('log4js');
@@ -99,7 +99,7 @@ if (isMainThread) {
                     filename: logPath,
                     layout: {
                         type: "pattern",
-                        pattern: "[%d] %f{1}:%l in %M [%p]: %m"
+                        pattern: "[%d]" + "[from worker_thread_" + workerData.workerId + "] %f{1}:%l in %M [%p]: %m"
                     }
                 } 
             },
@@ -128,14 +128,15 @@ if (isMainThread) {
         return log4js.getLogger("bg3mh_logger");
     }
 
-    raiseError = (error) => {
+    // extra arg for compat between main and worker threads
+    raiseError = (error, popup = Boolean) => {
         var bg3mh_logger = CREATE_LOGGER();
 
         console.error(error);
         bg3mh_logger.error(error);
     }
 
-    raiseInfo = (info) => {
+    raiseInfo = (info, popup = Boolean) => {
         var bg3mh_logger = CREATE_LOGGER();
 
         console.info(info);

support_files/lsf_convert.js

@@ -1,6 +1,5 @@
 const path = require('path');
 const fs = require('fs');
-const vscode = require('vscode');
 
 const { getFormats, baseNamePath, LOAD_LSLIB } = require('./lslib_utils');
 

support_files/lslib_utils.js

@@ -32,6 +32,10 @@ const convertDirs = ["[PAK]_UI", "[PAK]_Armor", "RootTemplates", "MultiEffectInf
 // excluding this because it will match to "UI" in convertDirs
 const illegalFiles = ["Icons_Items.lsx"];
 
+// excluding these packs because lslib uses something else to unpack them
+const virtualTexturePaks = ["VirtualTextures_"];
+const hotfixPatchPaks = ["Patch", "Hotfix"]
+
 // tools to test where the process is
 const { isMainThread, workerData } = require('node:worker_threads');
 
@@ -49,7 +53,7 @@ if (isMainThread) {
     findFiles = vscode.workspace.findFiles;
     parse = vscode.Uri.parse;
 } else {
-    getConfig = workerData;
+    getConfig = workerData.workerConfig;
     lslibPath = getConfig.lslibPath;
     compatRootModPath = path.join(getConfig.rootModPath + "\\");
     lslibToolsPath = path.join(getConfig.lslibPath, TOOL_SUBDIR);
@@ -145,8 +149,11 @@ async function LOAD_LSLIB() {
 
     processDllPaths();    
     await loadDlls();
-    bg3mh_logger.info(`${DLL_PATHS} \n.dlls loaded`);
-    console.log(`${DLL_PATHS} \n.dlls loaded`);
+    if (isMainThread) {
+        raiseInfo(`${DLL_PATHS} \n.dlls loaded`, false);
+    } else {
+        // raiseInfo(`.dlls loaded into worker ${workerData.workerId}`)
+    }
 
     // have to ignore this because the ts-linter doesn't know 'LSLib' exists :starege:
     // @ts-ignore 
@@ -237,6 +244,9 @@ function FILTER_PATHS(filesPath) {
             // these statements could technically be combined, but that doesn't make it very readable.
             if (temp_ext === getFormats().dll && !illegalDlls.includes(path.basename(filesPath))) {
                 return filesPath;
+            }
+            else if (filesPath.includes(virtualTexturePaks) || hotfixPatchPaks.includes(temp) {
+
             }
             else if (
                 (
@@ -280,9 +290,53 @@ function moveFileAcrossDevices(sourcePath, destPath, raiseError) {
         raiseInfo(infoMsg, false);
         vscode.window.showInformationMessage(infoMsg);
     } else {
-        raiseInfo(infoMsg);
+        raiseInfo(infoMsg, false);
+    }
+}
+
+// i don't like putting this here but i need a worker_thread friendly version
+function getModName() {
+    let rootModPath;
+
+    if (isMainThread) {
+        rootModPath = getConfig().rootModPath;
+    } else {
+        rootModPath = getConfig.rootModPath;
+    }
+
+    let modsDirPath = path.join(rootModPath, 'Mods');
+
+    try {
+        if (!fs.existsSync(modsDirPath)) {
+            if (isMainThread) {
+                vscode.window.showErrorMessage('Mods directory does not exist.');
+            } else {
+                raiseError('Mods directory does not exist.');
+            }
+
+            return '';
+        }
+
+        const files = fs.readdirSync(modsDirPath);
+        const directories = files.filter(file => 
+            fs.statSync(path.join(modsDirPath, file)).isDirectory()
+        );
+
+        if (directories.length === 1) {
+            return directories[0];
+        } else {
+            return '';
+        }
+    } catch (error) {
+        if (isMainThread) {
+            vscode.window.showErrorMessage(`Error reading directories in ${modsDirPath}: ${error}`);
+        } else {
+            raiseError(`Error reading directories in ${modsDirPath}: ${error}`);
+        }
+
+        return '';
     }
 }
 
 
-module.exports = { LSLIB, LOAD_LSLIB, FIND_FILES, FIND_FILES_SYNC, FILTER_PATHS, getFormats, moveFileAcrossDevices, baseNamePath, dirSeparator, compatRootModPath };
+module.exports = { LSLIB, LOAD_LSLIB, FIND_FILES, FIND_FILES_SYNC, FILTER_PATHS, getFormats, moveFileAcrossDevices, baseNamePath, dirSeparator, compatRootModPath, getModName };