Merge branch 'main' into ub-fixes

.env.example

@@ -3,12 +3,20 @@ CLIENT_PATH=./client
 # Updates NET_VERSION in CMakeVariables.txt
 NET_VERSION=171022
 # make sure this is a long random string
-# grab a "SHA 256-bit Key" from here: https://keygen.io/
+# generate a "SHA 256-bit Key" from here: https://gchq.github.io/CyberChef/#recipe=Pseudo-Random_Number_Generator(256,'Hex')
 ACCOUNT_MANAGER_SECRET=
 # Should be the externally facing IP of your server host
 EXTERNAL_IP=localhost
+
+# The database type that will be used.
+# Acceptable values are `sqlite`, `mysql`, `mariadb`, `maria`.
+# Case insensitive.
+DATABASE_TYPE=mariadb
+SQLITE_DATABASE_PATH=resServer/dlu.sqlite
+
 # Database values
 # Be careful with special characters here. It is more safe to use normal characters and/or numbers.
 MARIADB_USER=darkflame
 MARIADB_PASSWORD=
 MARIADB_DATABASE=darkflame
+SKIP_ACCOUNT_CREATION=1

.github/workflows/build-and-test.yml

@@ -43,6 +43,7 @@ jobs:
           build/*/*.ini
           build/*/*.so
           build/*/*.dll
+          build/*/*.dylib
           build/*/vanity/
           build/*/navmeshes/
           build/*/migrations/

.gitignore

@@ -7,7 +7,6 @@ valgrind-out.txt
 # Third party libraries
 thirdparty/mysql/
 thirdparty/mysql_linux/
-CMakeVariables.txt
 
 # Build folders
 build/
@@ -96,6 +95,7 @@ ipch/
 
 # Exceptions:
 CMakeSettings.json
+CMakeUserPresets.json
 *.vcxproj
 *.filters
 *.cmake

CMakeLists.txt

@@ -78,6 +78,7 @@ set(RECASTNAVIGATION_EXAMPLES OFF CACHE BOOL "" FORCE)
 # Disabled no-register
 # Disabled unknown pragmas because Linux doesn't understand Windows pragmas.
 if(UNIX)
+	add_link_options("-Wl,-rpath,$ORIGIN/")
 	add_compile_options("-fPIC")
 	add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0 _GLIBCXX_USE_CXX17_ABI=0)
 
@@ -186,16 +187,18 @@ foreach(resource_file ${RESOURCE_FILES})
 				list(GET line_split 0 variable_name)
 
 				if(NOT ${parsed_current_file_contents} MATCHES ${variable_name})
-					message(STATUS "Adding missing config option " ${variable_name} " to " ${resource_file})
-					set(line_to_add ${line_to_add} ${line})
+					# For backwards compatibility with older setup versions, dont add this option.
+					if(NOT ${variable_name} MATCHES "database_type")
+						message(STATUS "Adding missing config option " ${variable_name} " to " ${resource_file})
+						set(line_to_add ${line_to_add} ${line})
 
-					foreach(line_to_append ${line_to_add})
-						file(APPEND ${DLU_CONFIG_DIR}/${resource_file} "\n" ${line_to_append})
-					endforeach()
+						foreach(line_to_append ${line_to_add})
+							file(APPEND ${DLU_CONFIG_DIR}/${resource_file} "\n" ${line_to_append})
+						endforeach()
 
-					file(APPEND ${DLU_CONFIG_DIR}/${resource_file} "\n")
+						file(APPEND ${DLU_CONFIG_DIR}/${resource_file} "\n")
+					endif()
 				endif()
-
 				set(line_to_add "")
 			else()
 				set(line_to_add ${line_to_add} ${line})
@@ -225,21 +228,8 @@ foreach(file ${VANITY_FILES})
 endforeach()
 
 # Move our migrations for MasterServer to run
-file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/dlu/)
-file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/dlu/*.sql)
-
-foreach(file ${SQL_FILES})
-	get_filename_component(file ${file} NAME)
-	configure_file(${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/dlu/${file})
-endforeach()
-
-file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/cdserver/)
-file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/cdserver/*.sql)
-
-foreach(file ${SQL_FILES})
-	get_filename_component(file ${file} NAME)
-	configure_file(${CMAKE_SOURCE_DIR}/migrations/cdserver/${file} ${PROJECT_BINARY_DIR}/migrations/cdserver/${file})
-endforeach()
+file(REMOVE_RECURSE ${PROJECT_BINARY_DIR}/migrations)
+file(COPY ${CMAKE_SOURCE_DIR}/migrations DESTINATION ${CMAKE_BINARY_DIR})
 
 # Add system specfic includes for Apple, Windows and Other Unix OS' (including Linux)
 if (APPLE)
@@ -324,7 +314,7 @@ add_subdirectory(dPhysics)
 add_subdirectory(dServer)
 
 # Create a list of common libraries shared between all binaries
-set(COMMON_LIBRARIES "dCommon" "dDatabase" "dNet" "raknet" "MariaDB::ConnCpp" "magic_enum")
+set(COMMON_LIBRARIES "dCommon" "dDatabase" "dNet" "raknet" "magic_enum")
 
 # Add platform specific common libraries
 if(UNIX)

CMakePresets.json

@@ -11,9 +11,6 @@
 			"displayName": "Default configure step",
 			"description": "Use 'build' dir and Unix makefiles",
 			"binaryDir": "${sourceDir}/build",
-			"environment": {
-				"DLU_CONFIG_DIR": "${sourceDir}/build"
-			},
 			"generator": "Unix Makefiles"
 		},
 		{

README.md

@@ -13,21 +13,33 @@ Darkflame Universe is licensed under AGPLv3, please read [LICENSE](LICENSE). Som
 * You must disclose any changes you make to the code when you distribute it
 * Hosting a server for others counts as distribution
 
-## Disclaimers
-### Setup difficulty
-Throughout the entire build and setup process a level of familiarity with the command line and preferably a Unix-like development environment is greatly advantageous.
-
 ### Hosting a server
 We do not recommend hosting public servers. Darkflame Universe is intended for small scale deployment, for example within a group of friends. It has not been tested for large scale deployment which comes with additional security risks.
 
 ### Supply of resource files
 Darkflame Universe is a server emulator and does not distribute any LEGO® Universe files. A separate game client is required to setup this server emulator and play the game, which we cannot supply. Users are strongly suggested to refer to the safe checksums listed [here](#verifying-your-client-files) to see if a client will work.
 
-## Step by step walkthrough for a single-player server
-If you would like a setup for a single player server only on a Windows machine, use the [Native Windows Setup Guide by HailStorm](https://gist.github.com/HailStorm32/169df65a47a104199b5cc57d10fa57de) and skip this README.
-
-## Steps to setup server
+## Setting up a single player server
+* If you don't know what WSL is, skip this warning.
+	Warning: WSL version 1 does NOT support using sqlite as a database due to how it handles filesystem synchronization.
+	You must use Version 2 if you must run the server under WSL. Not doing so will result in save data loss.
+* Single player installs now no longer require building the server from source or installing development tools.
+* Download the [latest windows release](https://github.com/DarkflameUniverse/DarkflameServer/releases) (or whichever release you need) and extract the files into a folder inside your client. Note that this setup is expecting that when double clicking the folder that you put in the same folder as `legouniverse.exe`, the file `MasterServer.exe` is in there.
+* You should be able to see the folder with the server files in the same folder as `legouniverse.exe`.
+* Go into the server files folder and open `sharedconfig.ini`. Find the line that says `client_location` and put `..` after it so the line reads `client_location=..`.
+* To run the server, double-click `MasterServer.exe`.
+* You will be asked to create an account the first time you run the server. After you have created the account, the server will shutdown and need to be restarted.
+* To connect to the server, either delete the file `boot.cfg` which is found in your LEGO Universe client, rename the file `boot.cfg` to something else or follow the steps [here](#allowing-a-user-to-connect-to-your-server) if you wish to keep the file.
+* When shutting down the server, it is highly recommended to click the `MasterServer.exe` window and hold `ctrl` while pressing `c` to stop the server.
+* We are working on a way to make it so when you close the game, the server stops automatically alongside when you open the game, the server starts automatically.
+
+<font size="32">**If you are not planning on hosting a server for others, working in the codebase or wanting to use MariaDB for a database, you can stop reading here.**</font>
+
+If you would like to use a MariaDB as a database instead of the default of sqlite, follow the steps [here](#database-setup).
+
+# Steps to setup a development environment
 * [Clone this repository](#clone-the-repository)
+* [Setting up a development environment](#setting-up-a-development-environment)
 * [Install dependencies](#install-dependencies)
 * [Database setup](#database-setup)
 * [Build the server](#build-the-server)
@@ -39,6 +51,13 @@ If you would like a setup for a single player server only on a Windows machine,
 * [User Guide](#user-guide)
 * [Docker](#docker)
 
+## Disclaimers
+### Setup difficulty
+Throughout the entire build and setup process a level of familiarity with the command line and preferably a Unix-like development environment is greatly advantageous.
+
+## Step by step walkthrough for building a single-player Windows server from source
+If you would like a setup for a single player server only on a Windows machine built from source, use the [Native Windows Setup Guide by HailStorm](https://gist.github.com/HailStorm32/169df65a47a104199b5cc57d10fa57de) and skip this README.
+
 ## Clone the repository
 If you are on Windows, you will need to download and install git from [here](https://git-scm.com/download/win)
 
@@ -266,8 +285,8 @@ systemctl stop darkflame.service
 journalctl -xeu darkflame.service
 ```
 
-### First admin user
-Run `MasterServer -a` to get prompted to create an admin account. This method is only intended for the system administrator as a means to get started, do NOT use this method to create accounts for other users!
+### First user or adding more users.
+The first time you run `MasterServer`, you will be prompted to create an account. To create more accounts from the command line, `MasterServer -a` to get prompted to create an admin account. This method is only intended for the system administrator as a means to get started, do NOT use this method to create accounts for other users!
 
 ### Account management tool (Nexus Dashboard)
 **If you are just using this server for yourself, you can skip setting up Nexus Dashboard**
@@ -371,7 +390,7 @@ at once. For that:
 - Download the [.env.example](.env.example) file and place it next to `client` with the file name `.env`
   - You may get warnings that this name starts with a dot, acknowledge those, this is intentional. Depending on your operating system, you may need to activate showing hidden files (e.g. Ctrl-H in Gnome on Linux) and/or file extensions ("File name extensions" in the "View" tab on Windows).
   - Update the `ACCOUNT_MANAGER_SECRET` and `MARIADB_PASSWORD` with strong random passwords.
-    - Use a password generator like <https://keygen.io>
+    - Use a password generator <https://gchq.github.io/CyberChef/#recipe=Pseudo-Random_Number_Generator(256,'Hex')>
     - Avoid `:` and `@` characters
     - Once the database user is created, changing the password will not update it, so the server will just fail to connect.
   - Set `EXTERNAL_IP` to your LAN IP or public IP if you want to host the game for friends & family

dAuthServer/AuthServer.cpp

@@ -60,7 +60,7 @@ int main(int argc, char** argv) {
 
 	try {
 		Database::Connect();
-	} catch (sql::SQLException& ex) {
+	} catch (std::exception& ex) {
 		LOG("Got an error while connecting to the database: %s", ex.what());
 		Database::Destroy("AuthServer");
 		delete Game::server;

dChatServer/ChatServer.cpp

@@ -81,7 +81,7 @@ int main(int argc, char** argv) {
 	//Connect to the MySQL Database
 	try {
 		Database::Connect();
-	} catch (sql::SQLException& ex) {
+	} catch (std::exception& ex) {
 		LOG("Got an error while connecting to the database: %s", ex.what());
 		Database::Destroy("ChatServer");
 		delete Game::server;

dCommon/BrickByBrickFix.cpp

@@ -123,7 +123,7 @@ uint32_t BrickByBrickFix::UpdateBrickByBrickModelsToSd0() {
 				Database::Get()->UpdateUgcModelData(model.id, outputStringStream);
 				LOG("Updated model %i to sd0", model.id);
 				updatedModels++;
-			} catch (sql::SQLException exception) {
+			} catch (std::exception& exception) {
 				LOG("Failed to update model %i.  This model should be inspected manually to see why."
 					"The database error is %s", model.id, exception.what());
 			}

dCommon/CMakeLists.txt

@@ -37,7 +37,6 @@ target_include_directories(dCommon
 	"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase"
 	"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/ITables"
 	"${PROJECT_SOURCE_DIR}/dDatabase/CDClientDatabase"
-	"${PROJECT_SOURCE_DIR}/thirdparty/mariadb-connector-cpp/include"
 )
 
 if (UNIX)

dCommon/GeneralUtils.cpp

@@ -291,11 +291,12 @@ std::u16string GeneralUtils::ReadWString(RakNet::BitStream& inStream) {
 
 std::vector<std::string> GeneralUtils::GetSqlFileNamesFromFolder(const std::string_view folder) {
 	// Because we dont know how large the initial number before the first _ is we need to make it a map like so.
-    std::map<uint32_t, std::string> filenames{};
+	std::map<uint32_t, std::string> filenames{};
 	for (const auto& t : std::filesystem::directory_iterator(folder)) {
-        auto filename = t.path().filename().string();
-        const auto index = std::stoi(GeneralUtils::SplitString(filename, '_').at(0));
-        filenames.emplace(index, std::move(filename));
+		if (t.is_directory() || t.is_symlink()) continue;
+		auto filename = t.path().filename().string();
+		const auto index = std::stoi(GeneralUtils::SplitString(filename, '_').at(0));
+		filenames.emplace(index, std::move(filename));
 	}
 
 	// Now sort the map by the oldest migration.

dDatabase/CMakeLists.txt

@@ -2,6 +2,12 @@ add_subdirectory(CDClientDatabase)
 add_subdirectory(GameDatabase)
 
 add_library(dDatabase STATIC "MigrationRunner.cpp")
+
+add_custom_target(conncpp_dylib
+	${CMAKE_COMMAND} -E copy $<TARGET_FILE:MariaDB::ConnCpp> ${PROJECT_BINARY_DIR})
+
+add_dependencies(dDatabase conncpp_dylib)
+
 target_include_directories(dDatabase PUBLIC ".")
 target_link_libraries(dDatabase
 	PUBLIC dDatabaseCDClient dDatabaseGame)

dDatabase/GameDatabase/CMakeLists.txt

@@ -8,6 +8,12 @@ foreach(file ${DDATABSE_DATABSES_MYSQL_SOURCES})
 	set(DDATABASE_GAMEDATABASE_SOURCES ${DDATABASE_GAMEDATABASE_SOURCES} "MySQL/${file}")
 endforeach()
 
+add_subdirectory(SQLite)
+
+foreach(file ${DDATABSE_DATABSES_SQLITE_SOURCES})
+	set(DDATABASE_GAMEDATABASE_SOURCES ${DDATABASE_GAMEDATABASE_SOURCES} "SQLite/${file}")
+endforeach()
+
 add_subdirectory(TestSQL)
 
 foreach(file ${DDATABSE_DATABSES_TEST_SQL_SOURCES})
@@ -16,13 +22,14 @@ endforeach()
 
 add_library(dDatabaseGame STATIC ${DDATABASE_GAMEDATABASE_SOURCES})
 target_include_directories(dDatabaseGame PUBLIC "."
-	"ITables" PRIVATE "MySQL" "TestSQL"
+	"ITables" PRIVATE "MySQL" "SQLite" "TestSQL"
 	"${PROJECT_SOURCE_DIR}/dCommon"
 	"${PROJECT_SOURCE_DIR}/dCommon/dEnums"
 )
+
 target_link_libraries(dDatabaseGame
-	PUBLIC MariaDB::ConnCpp
-	INTERFACE dCommon)
+	INTERFACE dCommon
+	PRIVATE sqlite3 MariaDB::ConnCpp)
 
 # Glob together all headers that need to be precompiled
 file(

dDatabase/GameDatabase/Database.cpp

@@ -2,22 +2,46 @@
 #include "Game.h"
 #include "dConfig.h"
 #include "Logger.h"
-#include "MySQLDatabase.h"
 #include "DluAssert.h"
 
+#include "SQLiteDatabase.h"
+#include "MySQLDatabase.h"
+
+#include <ranges>
+
 #pragma warning (disable:4251) //Disables SQL warnings
 
 namespace {
 	GameDatabase* database = nullptr;
 }
 
+std::string Database::GetMigrationFolder() {
+	const std::set<std::string> validMysqlTypes = { "mysql", "mariadb", "maria" };
+	auto databaseType = Game::config->GetValue("database_type");
+	std::ranges::transform(databaseType, databaseType.begin(), ::tolower);
+	if (databaseType == "sqlite") return "sqlite";
+	else if (validMysqlTypes.contains(databaseType)) return "mysql";
+	else {
+		LOG("No database specified, using MySQL");
+		return "mysql";
+	}
+}
+
 void Database::Connect() {
 	if (database) {
 		LOG("Tried to connect to database when it's already connected!");
 		return;
 	}
 
-	database = new MySQLDatabase();
+	const auto databaseType = GetMigrationFolder();
+
+	if (databaseType == "sqlite") database = new SQLiteDatabase();
+	else if (databaseType == "mysql") database = new MySQLDatabase();
+	else {
+		LOG("Invalid database type specified in config, using MySQL");
+		database = new MySQLDatabase();
+	}
+
 	database->Connect();
 }
 

dDatabase/GameDatabase/Database.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <string>
-#include <conncpp.hpp>
 
 #include "GameDatabase.h"
 
@@ -13,4 +12,6 @@ namespace Database {
 	// Used for assigning a test database as the handler for database logic.
 	// Do not use in production code.
 	void _setDatabase(GameDatabase* const db);
+
+	std::string GetMigrationFolder();
 };

dDatabase/GameDatabase/GameDatabase.h

@@ -24,14 +24,10 @@
 #include "IIgnoreList.h"
 #include "IAccountsRewardCodes.h"
 #include "IBehaviors.h"
-
-namespace sql {
-	class Statement;
-	class PreparedStatement;
-};
+#include "IUgcModularBuild.h"
 
 #ifdef _DEBUG
-#  define DLU_SQL_TRY_CATCH_RETHROW(x) do { try { x; } catch (sql::SQLException& ex) { LOG("SQL Error: %s", ex.what()); throw; } } while(0)
+#  define DLU_SQL_TRY_CATCH_RETHROW(x) do { try { x; } catch (std::exception& ex) { LOG("SQL Error: %s", ex.what()); throw; } } while(0)
 #else
 #  define DLU_SQL_TRY_CATCH_RETHROW(x) x
 #endif // _DEBUG
@@ -42,14 +38,13 @@ class GameDatabase :
 	public IPropertyContents, public IProperty, public IPetNames, public ICharXml,
 	public IMigrationHistory, public IUgc, public IFriends, public ICharInfo,
 	public IAccounts, public IActivityLog, public IAccountsRewardCodes, public IIgnoreList,
-	public IBehaviors {
+	public IBehaviors, public IUgcModularBuild {
 public:
 	virtual ~GameDatabase() = default;
 	// TODO: These should be made private.
 	virtual void Connect() = 0;
 	virtual void Destroy(std::string source = "") = 0;
 	virtual void ExecuteCustomQuery(const std::string_view query) = 0;
-	virtual sql::PreparedStatement* CreatePreppedStmt(const std::string& query) = 0;
 	virtual void Commit() = 0;
 	virtual bool GetAutoCommit() = 0;
 	virtual void SetAutoCommit(bool value) = 0;