Reverts Location Compression Updates

This commit reverts the compression code that was added to lower the amount of LFS bandwidth that is used. We have spun up a custom LFS server, so we can safely bring back the raw JSON files as we're not as concerned about bandwidth.

.gitattributes

@@ -1,4 +1,22 @@
-# Normalize Line Endings
+# Normalize Line Endings.
 * text=auto eol=lf
 
-project/assets/compressed/database/locations.7z filter=lfs diff=lfs merge=lfs -text
+# LFS Tracking for large (10MB+) JSON files.
+#
+# Can be found with the following command:
+# find project/assets/database -type f -name "*.json" -size +5M | while read file; do
+#     git lfs track "$file"
+# done
+project/assets/database/locations/bigmap/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/laboratory/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/interchange/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/factory4_night/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/shoreline/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/tarkovstreets/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/sandbox/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/sandbox_high/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/lighthouse/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/rezervbase/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/woods/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/locations/factory4_day/looseLoot.json filter=lfs diff=lfs merge=lfs -text
+project/assets/database/templates/items.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -2,7 +2,6 @@
 *.exe
 *.zip
 *.d.ts
-project/assets/database/locations/
 project/build/
 project/obj/
 project/dist/

.lfsconfig

@@ -0,0 +1,3 @@
+[lfs]
+    url = https://lfs.sp-tarkov.com/sp-tarkov/server
+    locksverify = false

README.md

@@ -72,8 +72,6 @@ The following commands are available after the initial setup. Run them with `npm
 | `run:profiler`        | Run the project in profiler mode.                                         |
 | `gen:types`           | Generate types for the project.                                           |
 | `gen:docs`            | Generate documentation for the project.                                   |
-| `database:compress`   | Compress database files from working directory into archive.              |
-| `database:decompress` | Decompress database archives into working directory.                      |
 
 ### Debugging
 

project/gulpfile.mjs

@@ -1,6 +1,5 @@
 import crypto from "node:crypto";
 import path from "node:path";
-import { path7za } from "7zip-bin";
 import pkg from "@yao-pkg/pkg";
 import pkgfetch from "@yao-pkg/pkg-fetch";
 import fs from "fs-extra";
@@ -10,7 +9,6 @@ import download from "gulp-download";
 import { exec } from "gulp-execa";
 import rename from "gulp-rename";
 import minimist from "minimist";
-import Seven from "node-7z";
 import * as ResEdit from "resedit";
 import manifest from "./package.json" assert { type: "json" };
 
@@ -36,69 +34,6 @@ const entries = {
 };
 const licenseFile = "../LICENSE.md";
 
-/**
- * Decompresses the database archives from the assets/compressed/database directory into the assets/database directory.
- */
-const decompressArchives = async () => {
-    const compressedDir = path.resolve("assets", "compressed", "database");
-    const assetsDir = path.resolve("assets", "database");
-
-    // Read the compressed directory and filter out only the 7z files.
-    let compressedFiles = [];
-    try {
-        const files = await fs.readdir(compressedDir);
-        compressedFiles = files.filter((file) => file.endsWith(".7z"));
-    } catch (error) {
-        console.error(`Error reading compressed directory: ${error}`);
-        return;
-    }
-
-    if (compressedFiles.length === 0) {
-        console.log("No compressed database archives found.");
-        return;
-    }
-
-    for (const compressedFile of compressedFiles) {
-        const compressedFilePath = path.join(compressedDir, compressedFile);
-        const relativeTargetPath = compressedFile.replace(".7z", "");
-        const targetDir = path.join(assetsDir, relativeTargetPath);
-
-        console.log(`Processing archive: ${compressedFile}`);
-
-        // Clean the target directory before extracting the archive.
-        try {
-            await fs.remove(targetDir);
-            console.log(`Cleaned target directory: ${targetDir}`);
-        } catch (error) {
-            console.error(`Error cleaning target directory ${targetDir}: ${error}`);
-            continue;
-        }
-
-        // Extract the archive.
-        await new Promise((resolve, reject) => {
-            const stream = Seven.extractFull(compressedFilePath, targetDir, {
-                $bin: path7za,
-                overwrite: "a",
-            });
-
-            let hadError = false;
-
-            stream.on("end", () => {
-                if (!hadError) {
-                    console.log(`Successfully decompressed: ${compressedFile}`);
-                    resolve();
-                }
-            });
-
-            stream.on("error", (err) => {
-                hadError = true;
-                console.error(`Error decompressing ${compressedFile}: ${err}`);
-                reject(err);
-            });
-        });
-    }
-};
-
 /**
  * Transpile src files into Javascript with SWC
  */
@@ -255,7 +190,6 @@ const createHashFile = async () => {
 
 // Combine all tasks into addAssets
 const addAssets = gulp.series(
-    decompressArchives,
     copyAssets,
     downloadPnpm,
     copyLicense,

project/package.json

@@ -1,6 +1,6 @@
 {
     "name": "spt-server",
-    "version": "3.10.0",
+    "version": "3.10.2",
     "author": "SPT Server",
     "license": "NCSA",
     "main": "obj/bundle.js",
@@ -11,7 +11,6 @@
         "node": "20.11.1"
     },
     "scripts": {
-        "postinstall": "node scripts/fix-7za-permissions.js",
         "check:circular": "madge --circular --ts-config tsconfig.json --extensions ts ./src/",
         "lint": "npx @biomejs/biome lint ./",
         "lint:fix": "npx @biomejs/biome lint --write ./",
@@ -32,12 +31,9 @@
         "gen:types": "tsc -p tsconfig.typedef.json --resolveJsonModule",
         "gen:docs": "typedoc --options ./typedoc.json --entryPointStrategy expand ./src",
         "gen:items": "ts-node -r tsconfig-paths/register ./src/tools/ItemTplGenerator/ItemTplGeneratorProgram.ts",
-        "gen:productionquests": "ts-node -r tsconfig-paths/register ./src/tools/ProductionQuestsGen/ProductionQuestsGenProgram.ts",
-        "database:compress": "node scripts/databaseCompress.js",
-        "database:decompress": "node scripts/databaseDecompress.js"
+        "gen:productionquests": "ts-node -r tsconfig-paths/register ./src/tools/ProductionQuestsGen/ProductionQuestsGenProgram.ts"
     },
     "dependencies": {
-        "7zip-bin": "^5.2.0",
         "atomically": "~1.7",
         "buffer-crc32": "~1.0",
         "date-fns": "~3.6",
@@ -47,7 +43,6 @@
         "json5": "~2.2",
         "jsonc": "~2.0",
         "mongoid-js": "~1.3",
-        "node-7z": "^3.0.0",
         "proper-lockfile": "~4.1",
         "reflect-metadata": "~0.2",
         "semver": "~7.6",

project/src/Program.ts

@@ -2,7 +2,6 @@ import { ErrorHandler } from "@spt/ErrorHandler";
 import { Container } from "@spt/di/Container";
 import type { PreSptModLoader } from "@spt/loaders/PreSptModLoader";
 import { App } from "@spt/utils/App";
-import { DatabaseDecompressionUtil } from "@spt/utils/DatabaseDecompressionUtil";
 import { Watermark } from "@spt/utils/Watermark";
 import { container } from "tsyringe";
 
@@ -22,10 +21,6 @@ export class Program {
             const watermark = childContainer.resolve<Watermark>("Watermark");
             watermark.initialize();
 
-            const databaseDecompressionUtil =
-                childContainer.resolve<DatabaseDecompressionUtil>("DatabaseDecompressionUtil");
-            await databaseDecompressionUtil.initialize();
-
             const preSptModLoader = childContainer.resolve<PreSptModLoader>("PreSptModLoader");
             Container.registerListTypes(childContainer);
             await preSptModLoader.load(childContainer);

project/src/di/Container.ts

@@ -253,7 +253,6 @@ import { StaticRouterModService } from "@spt/services/mod/staticRouter/StaticRou
 import { App } from "@spt/utils/App";
 import { AsyncQueue } from "@spt/utils/AsyncQueue";
 import { CompareUtil } from "@spt/utils/CompareUtil";
-import { DatabaseDecompressionUtil } from "@spt/utils/DatabaseDecompressionUtil";
 import { DatabaseImporter } from "@spt/utils/DatabaseImporter";
 import { EncodingUtil } from "@spt/utils/EncodingUtil";
 import { HashUtil } from "@spt/utils/HashUtil";
@@ -420,9 +419,6 @@ export class Container {
     private static registerUtils(depContainer: DependencyContainer): void {
         // Utils
         depContainer.register<App>("App", App, { lifecycle: Lifecycle.Singleton });
-        depContainer.register<DatabaseDecompressionUtil>("DatabaseDecompressionUtil", DatabaseDecompressionUtil, {
-            lifecycle: Lifecycle.Singleton,
-        });
         depContainer.register<DatabaseImporter>("DatabaseImporter", DatabaseImporter, {
             lifecycle: Lifecycle.Singleton,
         });