Skip to content

Commit

Permalink
polishing and lots of comments. moved most of the worker thread stuff…
Browse files Browse the repository at this point in the history 
… into conversion_junction.js
  • Loading branch information
@khbsd
khbsd committed May 31, 2024
1 parent ddbf5e1 commit 478261a
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 66 deletions.
57 changes: 8 additions & 49 deletions commands/unpackGameData.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,81 +24,40 @@ const { processPak } = require('../support_files/process_pak.js');
const { isMainThread, MessageChannel, Worker } = require('node:worker_threads');
const comsPorts = new MessageChannel();

const { jobs, buildPathArrays } = require('../support_files/conversion_junction');
const { createConversionWorkers } = require('../support_files/conversion_junction');

let temp_dir;
let unpackedGameDataDirectory;


async function getTempDir() {
temp_dir = await vscode.window.showOpenDialog({
unpackedGameDataDirectory = await vscode.window.showOpenDialog({
canSelectFiles: false,
canSelectFolders: true,
canSelectMany: false,
title: 'Unpacked game data folder. Needs at least 160GB free space.'
})

if (!temp_dir || temp_dir.length === 0) {
vscode.window.showInformationMessage('No folder selected.');
if (!unpackedGameDataDirectory || unpackedGameDataDirectory.length === 0) {
vscode.window.showErrorMessage('No folder selected.');
return false;
} else {
return true;
}
}


const unpackGameData = vscode.commands.registerCommand('bg3-mod-helper.unpackGameDataCommand', async function () {
// raiseInfo("hi dipshit! 💩");

// eventually we can add specific game data files for unpacking, but atm it'll just grab everything.
const unpackGameData = vscode.commands.registerCommand('bg3-mod-helper.unpackGameDataCommand', async function () {
let filesToConvert = await FIND_FILES(pak, path.join(gameInstallLocation, "Data"));

// an array of workers
let workerArray = [];
let workerConfig = JSON.parse(loadConfigFile(true));

// the jobs function returns the lower of half the user's cpu cores or the number of things being unpacked
let jobsTotal = jobs(filesToConvert.length);

let jobsFinished = 0;

// made this a boolean so things don't get converted unless an unpack location is selected
if (isMainThread && await getTempDir()) {
// console.log(filesToConvert);
filesToConvert = buildPathArrays(filesToConvert);
createConversionWorkers(filesToConvert, workerConfig, unpackedGameDataDirectory);
raiseInfo(`Game data unpacking started. Please allow a few hours for this to complete, it's a big game.`);

// nested .map() functionality :o
let convertFileNames = filesToConvert.map(fileArray => fileArray.map(file => path.basename(file)));

// for each job, which is either a single file or an array of files sorted from smallest to largest file size, create a worker and give it the data it needs to unpack those files via workerData.
for (let i = 0; i < jobsTotal; i++) {
raiseInfo(`${convertFileNames[i]}\n`, false);
workerArray.push(new Worker(__dirname + "/conversion_worker.js", {
workerData: {
// passes the crystallized configuration settings to each of the workers
workerConfig,
// adding 2 to the workerId because 0 is the extension host and 1 is the main window
workerId: i + 2,
// passes the path of the file that needs to be converted
jobsTotal,
task: filesToConvert[i],
jobDestPath: temp_dir,
}
}));

workerArray[i].on('message', (message) => {
if (message.includes("done.")) {
raiseInfo(message);
jobsFinished++;
}

if (jobsFinished === jobsTotal) {
raiseInfo("All game data unpacked!")
}
})
}
}
});



module.exports = { unpackGameData }
81 changes: 69 additions & 12 deletions support_files/conversion_junction.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,21 @@ const path = require('path');
const os = require('os');
const fs = require('fs');

const { FIND_FILES, getFormats, compatRootModPath } = require('./lslib_utils');
const { FIND_FILES, getFormats } = require('./lslib_utils');
const { lsx, xml, pak } = getFormats();

const { CREATE_LOGGER, raiseError, raiseInfo } = require('./log_utils');
const bg3mh_logger = CREATE_LOGGER();
const { raiseError, raiseInfo } = require('./log_utils');

const { isLoca, processLoca, getLocaOutputPath } = require('./loca_convert');
const { isLsf, processLsf, getLsfOutputPath } = require('./lsf_convert');
const { isLoca, processLoca } = require('./loca_convert');
const { isLsf, processLsf } = require('./lsf_convert');
const { processPak, prepareTempDir } = require('./process_pak');

const { isMainThread, workerData } = require('node:worker_threads');
const { isMainThread, workerData, Worker } = require('node:worker_threads');

let getConfig, getModName;


// need this outside a function so it's run on load.
if (isMainThread) {
getConfig = require('./config.js').getConfig();
getModName = require('./config.js').getModName();
Expand All @@ -25,43 +26,45 @@ if (isMainThread) {
getModName = require('./lslib_utils.js').getModName();
}


// this functionality can't work on a worker thread, so i don't think it's possible for it to return undefined. i still need it to not attempt to load the vscode module while in a worker thread, though.
function getActiveTabPath() {
if (isMainThread) {
const vscode = require('vscode');
return vscode.window.activeTextEditor.document.fileName;
} else {
return undefined;
}

}


function jobs(filesNum) {
let halfCoreCount = Math.floor(os.availableParallelism() / 2);

return Math.min(filesNum, halfCoreCount);
}


// sorts the files into nested arrays if the amount of files is greater than the number of cores. if not, will just return the original array.
function buildPathArrays(filesToConvert) {
let filesNum = filesToConvert.length;
let jobsNum = jobs(filesNum);

let temp_array = [];
let fileArrays = [];

// if these are equal, it will just be one file per core anyways
if (jobsNum >= filesNum) {
return filesToConvert;
// divides files into equal arrrays and sorts them smallest to largest
} else if (jobsNum < filesNum) {
for (let i = 0; i < jobsNum; i++) {
temp_array = [];
for (let j = 0; j < filesNum; j += jobsNum) {
let temp_index = i + j;

// makes sure we're not trying to read beyond the bounds of the number of jobs
if (filesToConvert[temp_index] !== undefined) {
// console.log(filesToConvert[temp_index]);
temp_array.push(filesToConvert[temp_index]);
// console.log(temp_array);
}
}
fileArrays.push(temp_array.sort((file1, file2) => fs.statSync(file1).size - fs.statSync(file2).size));
Expand All @@ -71,9 +74,57 @@ function buildPathArrays(filesToConvert) {
}


async function createConversionWorkers(filesToConvert, workerConfig, unpackedGameDataDirectory) {
// an array of workers
const workerArray = [];

filesToConvert = buildPathArrays(filesToConvert);

// the jobs function returns the lower of half the user's cpu cores or the number of things being unpacked.
const jobsTotal = jobs(filesToConvert.length);

// tracks the number of jobs reported completed by the worker threads.
let jobsFinished = 0;

// nested .map() functionality :o
const convertFileNames = filesToConvert.map(fileArray => fileArray.map(file => path.basename(file)));

// for each job, which is either a single file or an array of files sorted from smallest to largest file size, create a worker and give it the data it needs to unpack those files via workerData.
for (let i = 0; i < jobsTotal; i++) {
raiseInfo(`${convertFileNames[i]}\n`, false);
workerArray.push(new Worker(__dirname + "/conversion_worker.js", {
workerData: {
// passes the crystallized configuration settings to each of the workers
workerConfig,
// adding 2 to the workerId because 0 is the extension host and 1 is the main window
workerId: i + 2,
// passes the path of the file that needs to be converted
task: filesToConvert[i],
jobDestPath: unpackedGameDataDirectory
}
}));

// creates a listener for the newly pushed worker. if the worker sends back a message containing "done.", adds 1 to the jobsFinished tracker variable.
workerArray[i].on('message', (message) => {
if (message.includes("done.")) {
raiseInfo(message);
jobsFinished++;
}

// once the jobsFinished variable is equal to the amount of jobs we started with, we know all jobs are done and can tell the user that.
if (jobsFinished === jobsTotal) {
raiseInfo("All game data unpacked!");
}
})
}
}


// basically just type-checking for the convertPath arg so path.extname doesn't throw a tantrum
function getDynamicPath(filePath) {
let temp_path;

//this bit is important since we will only be handing off arrays of specific types to convert()
if (Array.isArray(filePath) && filePath != []) {
temp_path = filePath[0];
}
Expand All @@ -91,14 +142,16 @@ function getDynamicPath(filePath) {
}


// at the moment this has all the functionality i planned for it, ie lsf, loca, and paks. for any other conversions we can make separate functions
async function convert(convertPath, targetExt = path.extname(getDynamicPath(convertPath)), modName = getModName) {
let rootModPath = getConfig.rootModPath;

// console.log('targetExt:' + targetExt);
// checks if the convertPath was undefined and halts the function before it goes any further
if (targetExt === "empty") {
return;
}

// if the targetExt indicates a pack, we need to know what to do with it: are we packing up or unpacking?
if (targetExt === pak) {
if (fs.statSync(convertPath).isDirectory()) {
prepareTempDir();
Expand All @@ -111,6 +164,7 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv

processPak(rootModPath, modName);
}
// has a check for main thread here because when a worker thread calls this function, it's batch unpacking and has a specific place it needs the files inside to go, but can't rely on vscode module functions to get them
else if (fs.statSync(convertPath).isFile()) {
if (isMainThread) {
processPak(convertPath, modName);
Expand All @@ -120,17 +174,20 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv

}
}
// this function works best on single files, so we need to process that array and re-run this function with each of its elements
else if (Array.isArray(convertPath)) {
// console.log('array1')
for (let i = 0; i < convertPath.length; i++) {
convert(convertPath[i], path.extname(convertPath[i]));
}
}
// if this function is passsed a directory, parse it into an array of files of the specified type in targetExt, then send that back through
else if (fs.statSync(convertPath).isDirectory()) {
// console.log('plz1')
const filesToConvert = await FIND_FILES(targetExt);
convert(filesToConvert);
}
// finally, if this function is handed a file, convert that bitch :catyes:
else if (fs.statSync(convertPath).isFile()) {
// console.log('plz2')
if (isLoca(targetExt)) {
Expand All @@ -155,4 +212,4 @@ async function convert(convertPath, targetExt = path.extname(getDynamicPath(conv
}


module.exports = { convert, jobs, buildPathArrays };
module.exports = { convert, jobs, createConversionWorkers };
12 changes: 7 additions & 5 deletions commands/conversion_worker.js → support_files/conversion_worker.js
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,36 @@
const { CREATE_LOGGER, raiseError, raiseInfo } = require('../support_files/log_utils.js');
const { CREATE_LOGGER, raiseError, raiseInfo } = require('./log_utils.js');
const bg3mh_logger = CREATE_LOGGER();
const { parentPort, workerData } = require('node:worker_threads');
const path = require('node:path');
const fs = require('node:fs');


const { convert } = require('../support_files/conversion_junction.js')
const { convert } = require('./conversion_junction.js')

function taskIntake() {
// console.log(workerData.task)
// basic array or file handling stuff
if (Array.isArray(workerData.task)) {
for (let i = 0; i < workerData.task.length; i++) {

try {
raiseInfo(`converting ${workerData.task[i]}`);
// convert(workerData.task[i]);
convert(workerData.task[i]);
} catch (Error) {
raiseError(`converting ${workerData.task[i]}\n failed with error ${Error}`);
}
}
} else if (typeof(workerData.task) == 'string') {
try {
raiseInfo(`converting ${workerData.task}`);
// convert(workerData.task);
convert(workerData.task);
} catch (Error) {
raiseError(`converting ${workerData.task}\n failed with error ${Error}`);
}
}

// let the main thread know you're done
parentPort.postMessage(`worker ${workerData.workerId} done.`);
}

// start it up babyyyyy
taskIntake();

0 comments on commit 478261a

Please sign in to comment.