Rust-g IconForge: Lightning fast spritesheet generation (BeeStation#1…

…0404)

* Initial testing

* Rustg icon generator

* this thing.. fast....

* Fix

* Finish porting spritesheets and add GAGS support

* Remove 'moving' attribute

* Refactor the DM component

* Update rustg DM and add cleanup call

* vendor_icon_preview -> icon_state_preview

* Revert tracy change

* Re-add GAGS to design spritesheet

* Fix crafting spritesheet size

* Correct webroot args

* Adds caching

* Reset rustg

* Move cleanup step where it belongs

* Add crosscompiled rustg dll

* Log cache invalidations

* Fix player panel and orbit spritesheets

* Fix reading legacy sheets from cache

* Enables local caching of legacy assets, adds Regenerate Asset Cache debug verb

* Fix universal_icon creation being slow

* Protect against cache deletion

* Fix hand icons not displaying properly

* Remove this very incorrect assumption

* Restore rust_g to master

* Update defs

* Add rustg version checker to smart cache

* Move legacy cache to data/spritesheets so it persists on prod

* Update config comment to match

* Copy all icons so CI works

* Try fix CI again

* Fix chat and designs not displaying properly when loaded from cache

* Fix mistake

* Fix double encoding

* Inline uni_icon, saving unnecessary proc-call overhead

* Create a unit test for the smart cache

beestation.dme

@@ -38,6 +38,7 @@
 #include "code\__DEFINES\areas.dm"
 #include "code\__DEFINES\armor.dm"
 #include "code\__DEFINES\art.dm"
+#include "code\__DEFINES\assets.dm"
 #include "code\__DEFINES\async.dm"
 #include "code\__DEFINES\atmospherics.dm"
 #include "code\__DEFINES\atom_hud.dm"
@@ -2135,6 +2136,8 @@
 #include "code\modules\asset_cache\asset_cache_item.dm"
 #include "code\modules\asset_cache\asset_list.dm"
 #include "code\modules\asset_cache\asset_list_items.dm"
+#include "code\modules\asset_cache\spritesheet\batched\batched_spritesheet.dm"
+#include "code\modules\asset_cache\spritesheet\batched\universal_icon.dm"
 #include "code\modules\asset_cache\transports\asset_transport.dm"
 #include "code\modules\asset_cache\transports\webroot_transport.dm"
 #include "code\modules\atmospherics\auxgm\breathing_classes.dm"

code/__DEFINES/assets.dm

@@ -0,0 +1,16 @@
+#define ASSET_CROSS_ROUND_CACHE_DIRECTORY "data/spritesheets/legacy_cache"
+#define ASSET_CROSS_ROUND_SMART_CACHE_DIRECTORY "data/spritesheets/smart_cache"
+
+/// When sending mutiple assets, how many before we give the client a quaint little sending resources message
+#define ASSET_CACHE_TELL_CLIENT_AMOUNT 8
+
+/// How many assets can be sent at once during legacy asset transport
+#define SLOW_ASSET_SEND_RATE 6
+
+/// Constructs a universal icon. This is done in the same manner as the icon() BYOND proc.
+/// "color" will not do anything if a transform is provided. Blend it yourself or use color_transform().
+/// Do note that transforms are NOT COPIED, and are internally lists. So take care not to re-use transforms.
+/// This is a DEFINE for performance reasons.
+/// Parameters (in order):
+/// icon_file, icon_state, dir, frame, transform, color
+#define uni_icon(I, icon_state, rest...) new /datum/universal_icon(I, icon_state, ##rest)

code/__HELPERS/_lists.dm

@@ -654,6 +654,21 @@
 			.[i] = key
 			.[key] = value
 
+/// A version of deep_copy_list that actually supports associative list nesting: list(list(list("a" = "b"))) will actually copy correctly.
+/proc/deep_copy_list_alt(list/inserted_list)
+	if(!islist(inserted_list))
+		return inserted_list
+	var/copied_list = inserted_list.Copy()
+	. = copied_list
+	for(var/key_or_value in inserted_list)
+		if(isnum_safe(key_or_value) || !inserted_list[key_or_value])
+			continue
+		var/value = inserted_list[key_or_value]
+		var/new_value = value
+		if(islist(value))
+			new_value = deep_copy_list_alt(value)
+		copied_list[key_or_value] = new_value
+
 /**
  takes an input_key, as text, and the list of keys already used, outputting a replacement key in the format of "[input_key] ([number_of_duplicates])" if it finds a duplicate
 

code/__HELPERS/files.dm

@@ -85,6 +85,7 @@ GLOBAL_VAR_INIT(fileaccess_timer, 0)
 
 /// Save file as an external file then md5 it.
 /// Used because md5ing files stored in the rsc sometimes gives incorrect md5 results.
+/// https://www.byond.com/forum/post/2611357
 /proc/md5asfile(file)
 	var/static/notch = 0
 	// its importaint this code can handle md5filepath sleeping instead of hard blocking, if it's converted to use rust_g.

code/_compile_options.dm

@@ -80,7 +80,9 @@
 #endif				// 1 to use the default behaviour;
 					// 2 for preloading absolutely everything;
 
+//#define LOWMEMORYMODE
 #ifdef LOWMEMORYMODE
+#warn WARNING: Compiling with LOWMEMORYMODE.
 #define FORCE_MAP "runtimestation"
 #endif
 

code/controllers/configuration/entries/resources.dm

@@ -30,3 +30,5 @@
 	return ..(str_var)
 
 /datum/config_entry/flag/cache_assets
+
+/datum/config_entry/flag/smart_cache_assets

code/controllers/subsystem/asset_loading.dm

@@ -7,6 +7,7 @@ SUBSYSTEM_DEF(asset_loading)
 	flags = SS_NO_INIT
 	runlevels = RUNLEVEL_LOBBY|RUNLEVELS_DEFAULT
 	var/list/datum/asset/generate_queue = list()
+	var/last_queue_len = 0
 
 /datum/controller/subsystem/asset_loading/fire(resumed)
 	while(length(generate_queue))
@@ -16,7 +17,12 @@ SUBSYSTEM_DEF(asset_loading)
 
 		if(MC_TICK_CHECK)
 			return
+		last_queue_len = length(generate_queue)
 		generate_queue.len--
+	// We just emptied the queue
+	if(last_queue_len && !length(generate_queue))
+		// Clean up cached icons, freeing memory.
+		rustg_iconforge_cleanup()
 
 /datum/controller/subsystem/asset_loading/proc/queue_asset(datum/asset/queue)
 #ifdef DO_NOT_DEFER_ASSETS

code/controllers/subsystem/processing/greyscale.dm

@@ -43,6 +43,16 @@ PROCESSING_SUBSYSTEM_DEF(greyscale)
 		CRASH("Invalid colors were given to `GetColoredIconByType()`: [colors]")
 	return configurations[type].Generate(colors)
 
+/datum/controller/subsystem/processing/greyscale/proc/GetColoredIconEntryByType(type, list/colors, target_icon_state)
+	if(!ispath(type, /datum/greyscale_config))
+		CRASH("An invalid greyscale configuration was given to `GetColoredIconEntryByType()`: [type]")
+	type = "[type]"
+	if(istype(colors)) // It's the color list format
+		colors = colors.Join()
+	else if(!istext(colors))
+		CRASH("Invalid colors were given to `GetColoredIconEntryByType()`: [colors]")
+	return configurations[type].Generate_entry(colors, target_icon_state)
+
 /datum/controller/subsystem/processing/greyscale/proc/ParseColorString(color_string)
 	. = list()
 	var/list/split_colors = splittext(color_string, "#")

code/datums/browser.dm

@@ -42,6 +42,9 @@
 	if (istype(name, /datum/asset/spritesheet))
 		var/datum/asset/spritesheet/sheet = name
 		stylesheets["spritesheet_[sheet.name].css"] = "data/spritesheets/[sheet.name]"
+	else if (istype(name, /datum/asset/spritesheet_batched))
+		var/datum/asset/spritesheet_batched/sheet = name
+		stylesheets["spritesheet_[sheet.name].css"] = "data/spritesheets/[sheet.name]"
 	else
 		var/asset_name = "[name].css"
 

code/datums/components/crafting/crafting.dm

@@ -142,7 +142,7 @@
 
 /datum/component/personal_crafting/ui_assets(mob/user)
 	return list(
-		get_asset_datum(/datum/asset/spritesheet/crafting),
+		get_asset_datum(/datum/asset/spritesheet_batched/crafting),
 	)
 
 /datum/component/personal_crafting/proc/check_tools(atom/a, datum/crafting_recipe/R, list/contents)

code/datums/greyscale/_greyscale_config.dm

@@ -303,4 +303,38 @@
 	output["icon"] = GenerateBundle(colors, debug_steps)
 	return output
 
+/datum/greyscale_config/proc/Generate_entry(color_string, target_bundle_state, datum/universal_icon/last_external_icon)
+	return GenerateBundle_entry(color_string, target_bundle_state, last_external_icon=last_external_icon)
+
+/// Handles the actual icon manipulation to create the spritesheet
+/datum/greyscale_config/proc/GenerateBundle_entry(list/colors, target_bundle_state, datum/universal_icon/last_external_icon)
+	if(!istype(colors))
+		colors = SSgreyscale.ParseColorString(colors)
+	if(length(colors) != expected_colors)
+		CRASH("[DebugName()] expected [expected_colors] color arguments but received [length(colors)]")
+
+	if(!(target_bundle_state in icon_states))
+		CRASH("Invalid target bundle icon_state \"[target_bundle_state]\"! Valid icon_states: [icon_states.Join(", ")]")
+
+	var/datum/universal_icon/icon_bundle = GenerateLayerGroup_entry(colors, icon_states[target_bundle_state], last_external_icon) || uni_icon('icons/effects/effects.dmi', "nothing")
+	icon_bundle.scale(width, height)
+	return icon_bundle
+
+/// Internal recursive proc to handle nested layer groups
+/datum/greyscale_config/proc/GenerateLayerGroup_entry(list/colors, list/group, datum/universal_icon/last_external_icon)
+	var/datum/universal_icon/new_icon
+	for(var/datum/greyscale_layer/layer as anything in group)
+		var/datum/universal_icon/layer_icon
+		if(islist(layer))
+			layer_icon = GenerateLayerGroup_entry(colors, layer, new_icon || last_external_icon)
+			layer = layer[1] // When there are multiple layers in a group like this we use the first one's blend mode
+		else
+			layer_icon = layer.Generate_entry(colors, new_icon || last_external_icon)
+
+		if(!new_icon)
+			new_icon = layer_icon
+		else
+			new_icon.blend_icon(layer_icon, layer.blend_mode)
+	return new_icon
+
 #undef MAX_SANE_LAYERS

code/datums/greyscale/layer.dm

@@ -76,10 +76,27 @@
 	var/icon/copy_of_new_icon = icon(new_icon) // Layers shouldn't be modifying it directly, this is just for them to reference
 	return InternalGenerate(processed_colors, render_steps, copy_of_new_icon)
 
+/// Used to actualy create the layer using the given colors
+/// Do not override, use InternalGenerate instead
+/datum/greyscale_layer/proc/Generate_entry(list/colors, datum/universal_icon/new_icon)
+	var/list/processed_colors = list()
+	for(var/i in color_ids)
+		if(isnum(i))
+			processed_colors += colors[i]
+		else
+			processed_colors += i
+	var/datum/universal_icon/copy_of_new_icon = isnull(new_icon) ? uni_icon('icons/effects/effects.dmi', "nothing") : new_icon.copy() // Layers shouldn't be modifying it directly, this is just for them to reference
+	return InternalGenerate_entry(processed_colors, copy_of_new_icon)
+
 /// Override this to implement layers.
 /// The colors var will only contain colors that this layer is configured to use.
 /datum/greyscale_layer/proc/InternalGenerate(list/colors, list/render_steps, icon/new_icon)
 
+/// Override this to implement layers.
+/// The colors var will only contain colors that this layer is configured to use.
+/datum/greyscale_layer/proc/InternalGenerate_entry(list/colors, datum/universal_icon/new_icon)
+	return new_icon
+
 ////////////////////////////////////////////////////////
 // Subtypes
 
@@ -88,10 +105,12 @@
 	layer_type = "icon_state"
 	var/icon_state
 	var/icon/icon
+	var/icon_file
 	var/color_id
 
 /datum/greyscale_layer/icon_state/Initialize(icon_file)
 	. = ..()
+	src.icon_file = icon_file
 	var/list/icon_states = icon_states(icon_file)
 	if(!(icon_state in icon_states))
 		CRASH("Configured icon state \[[icon_state]\] was not found in [icon_file]. Double check your json configuration.")
@@ -111,6 +130,13 @@
 		generated_icon.Blend(colors[1], ICON_MULTIPLY)
 	return generated_icon
 
+/datum/greyscale_layer/icon_state/InternalGenerate_entry(list/colors, datum/universal_icon/new_icon)
+	. = ..()
+	var/datum/universal_icon/generated_icon = uni_icon(icon_file, icon_state)
+	if(length(colors))
+		generated_icon.blend_color(colors[1], ICON_MULTIPLY)
+	return generated_icon
+
 /// A layer to modify the previous layer's colors with a color matrix
 /datum/greyscale_layer/color_matrix
 	layer_type = "color_matrix"
@@ -154,3 +180,9 @@
 	else
 		generated_icon = reference_type.Generate(colors.Join(), new_icon)
 	return icon(generated_icon, icon_state)
+
+/datum/greyscale_layer/reference/InternalGenerate_entry(list/colors, datum/universal_icon/new_icon)
+	var/datum/universal_icon/generated_icon = reference_type.Generate_entry(colors.Join(), new_icon)
+	generated_icon = generated_icon.copy()
+	generated_icon.icon_state = icon_state
+	return generated_icon

code/game/atoms.dm

@@ -138,8 +138,6 @@
 	///Used for changing icon states for different base sprites.
 	var/base_icon_state
 
-	///Used to show a specific icon state in certain situations - i.e.) crafting menu
-	var/icon_state_preview
 	// This veriable exists BECAUSE animating sprite (bola) has an issue to render to TGUI crafting window - it shows wrong icons.
 
 	///LazyList of all balloon alerts currently on this atom

code/game/machinery/telecomms/computers/message.dm

@@ -87,7 +87,7 @@
 
 /obj/machinery/computer/message_monitor/ui_assets(mob/user)
 	return list(
-		get_asset_datum(/datum/asset/spritesheet/chat),
+		get_asset_datum(/datum/asset/spritesheet_batched/chat),
 	)
 
 /obj/machinery/computer/message_monitor/ui_static_data(mob/user)

code/game/objects/items.dm

@@ -224,7 +224,7 @@ GLOBAL_VAR_INIT(rpg_loot_items, FALSE)
 	var/canMouseDown = FALSE
 
 	///Icons used to show the item in vendors instead of the item's actual icon, drawn from the item's icon file (just chemical.dm for now)
-	var/vendor_icon_preview = null
+	var/icon_state_preview = null
 
 
 /obj/item/Initialize(mapload)

code/game/objects/items/RPD.dm

@@ -332,7 +332,7 @@ GLOBAL_LIST_INIT(fluid_duct_recipes, list(
 
 /obj/item/pipe_dispenser/ui_assets(mob/user)
 	return list(
-		get_asset_datum(/datum/asset/spritesheet/pipes),
+		get_asset_datum(/datum/asset/spritesheet_batched/pipes),
 	)
 
 

code/game/objects/items/airlock_painter.dm

@@ -200,7 +200,7 @@
 	/// The full icon state of the decal being printed.
 	var/stored_decal_total = "warningline"
 	/// The type path of the spritesheet being used for the frontend.
-	var/spritesheet_type = /datum/asset/spritesheet/decals // spritesheet containing previews
+	var/spritesheet_type = /datum/asset/spritesheet_batched/decals // spritesheet containing previews
 	/// Does this printer implementation support custom colors?
 	var/supports_custom_color = FALSE
 	/// Current custom color
@@ -293,7 +293,7 @@
 
 /obj/item/airlock_painter/decal/ui_static_data(mob/user)
 	. = ..()
-	var/datum/asset/spritesheet/icon_assets = get_asset_datum(spritesheet_type)
+	var/datum/asset/spritesheet_batched/icon_assets = get_asset_datum(spritesheet_type)
 
 	.["icon_prefix"] = "[icon_assets.name]32x32"
 	.["supports_custom_color"] = supports_custom_color
@@ -350,28 +350,15 @@
 	update_decal_path()
 	. = TRUE
 
-/datum/asset/spritesheet/decals
+/datum/asset/spritesheet_batched/decals
 	name = "floor_decals"
-	cross_round_cachable = TRUE
+	ignore_dir_errors = TRUE
 
-	/// The floor icon used for blend_preview_floor()
 	var/preview_floor_icon = 'icons/turf/floors.dmi'
-	/// The floor icon state used for blend_preview_floor()
 	var/preview_floor_state = "floor"
 	/// The associated decal painter type to grab decals, colors, etc from.
 	var/painter_type = /obj/item/airlock_painter/decal
 
-/**
- * Underlay an example floor for preview purposes, and return the new icon.
- *
- * Arguments:
- * * decal - the decal to place over the example floor tile
- */
-/datum/asset/spritesheet/decals/proc/blend_preview_floor(icon/decal)
-	var/icon/final = icon(preview_floor_icon, preview_floor_state)
-	final.Blend(decal, ICON_OVERLAY)
-	return final
-
 /**
  * Insert a specific state into the spritesheet.
  *
@@ -380,14 +367,15 @@
  * * dir - the given direction.
  * * color - the given color.
  */
-/datum/asset/spritesheet/decals/proc/insert_state(decal, dir, color)
+/datum/asset/spritesheet_batched/decals/proc/insert_state(decal, dir, color)
 	// Special case due to icon_state names
 	var/icon_state_color = color == "yellow" ? "" : color
 
-	var/icon/final = blend_preview_floor(icon('icons/turf/decals.dmi', "[decal][icon_state_color ? "_" : ""][icon_state_color]", dir))
-	Insert("[decal]_[dir]_[color]", final)
+	var/datum/universal_icon/floor = uni_icon(preview_floor_icon, preview_floor_state)
+	floor.blend_icon(uni_icon('icons/turf/decals.dmi', "[decal][icon_state_color ? "_" : ""][icon_state_color]", dir), ICON_OVERLAY)
+	insert_icon("[decal]_[dir]_[color]", floor)
 
-/datum/asset/spritesheet/decals/create_spritesheets()
+/datum/asset/spritesheet_batched/decals/create_spritesheets()
 	// Must actually create because initial(type) doesn't work for /lists for some reason.
 	var/obj/item/airlock_painter/decal/painter = new painter_type()
 
@@ -412,7 +400,7 @@
 	stored_dir = 2
 	stored_color = "#D4D4D432"
 	stored_decal = "tile_corner"
-	spritesheet_type = /datum/asset/spritesheet/decals/tiles
+	spritesheet_type = /datum/asset/spritesheet_batched/decals/tiles
 	supports_custom_color = TRUE
 	color_list = list(
 		list("Neutral", "#D4D4D432"),
@@ -467,11 +455,12 @@
 
 	target.AddElement(/datum/element/decal, 'icons/turf/decals.dmi', source_decal, source_dir, FALSE, decal_color, null, null, decal_alpha)
 
-/datum/asset/spritesheet/decals/tiles
+/datum/asset/spritesheet_batched/decals/tiles
 	name = "floor_tile_decals"
+	ignore_dir_errors = TRUE
 	painter_type = /obj/item/airlock_painter/decal/tile
 
-/datum/asset/spritesheet/decals/tiles/insert_state(decal, dir, color)
+/datum/asset/spritesheet_batched/decals/tiles/insert_state(decal, dir, color)
 	// Account for 8-sided decals.
 	var/source_decal = decal
 	var/source_dir = dir
@@ -487,12 +476,13 @@
 		render_color = tile_type.rgba_regex.group[1]
 		render_alpha = text2num(tile_type.rgba_regex.group[2], 16)
 
-	var/icon/colored_icon = icon('icons/turf/decals.dmi', source_decal, dir=source_dir)
-	colored_icon.ChangeOpacity(render_alpha * 0.008)
+	var/datum/universal_icon/colored_icon = uni_icon('icons/turf/decals.dmi', source_decal, dir=source_dir)
+	colored_icon.blend_color("#ffffff" + num2hex(render_alpha * 0.008, 2), ICON_MULTIPLY)
 	if(color == "custom")
-		colored_icon.Blend("#0e0f0f", ICON_MULTIPLY)
+		colored_icon.blend_color("#0e0f0f", ICON_MULTIPLY)
 	else
-		colored_icon.Blend(render_color, ICON_MULTIPLY)
+		colored_icon.blend_color(render_color, ICON_MULTIPLY)
 
-	colored_icon = blend_preview_floor(colored_icon)
-	Insert("[decal]_[dir]_[replacetext(color, "#", "")]", colored_icon)
+	var/datum/universal_icon/floor = uni_icon(preview_floor_icon, preview_floor_state)
+	floor.blend_icon(colored_icon, ICON_OVERLAY)
+	insert_icon("[decal]_[dir]_[replacetext(color, "#", "")]", floor)

code/game/objects/items/debug_items.dm

@@ -126,7 +126,7 @@
 
 /obj/item/debug/omnitool/ui_assets(mob/user)
 	return list(
-		get_asset_datum(/datum/asset/spritesheet/tools)
+		get_asset_datum(/datum/asset/spritesheet_batched/tools)
 	)
 
 /obj/item/debug/omnitool/ui_data(mob/user)