From 3626611e687c7be2f00507d917666c9c6d9a9f2f Mon Sep 17 00:00:00 2001 From: Myk Taylor Date: Mon, 7 Aug 2023 03:18:09 -0700 Subject: [PATCH 1/2] add overlays to integrate into legends UI --- changelog.txt | 1 + docs/exportlegends.rst | 41 +++++-- exportlegends.lua | 267 ++++++++++++++++++++++++++++++++--------- 3 files changed, 244 insertions(+), 65 deletions(-) diff --git a/changelog.txt b/changelog.txt index 4dfa0db3b1..a33fc46be2 100644 --- a/changelog.txt +++ b/changelog.txt @@ -27,6 +27,7 @@ Template for new versions: # Future ## New Features +- `exportlegends`: new overlay that integrates with the vanilla "Export XML" button. Now you can generate both the vanilla export and the extended data export with a single click! ## Fixes - `suspendmanager`: Fix the overlay enabling/disabling `suspendmanager` unexpectedly diff --git a/docs/exportlegends.rst b/docs/exportlegends.rst index 6f0088178b..2d7c4702d6 100644 --- a/docs/exportlegends.rst +++ b/docs/exportlegends.rst @@ -5,17 +5,25 @@ exportlegends :summary: Exports extended legends data for external viewing. :tags: legends inspection -When run from the legends mode screen, you can export detailed data about your -world so that it can be browsed with external programs like -:forums:`Legends Browser <179848>` and other similar utilities. The data -exported with this tool is more detailed than what you can get with vanilla -export functionality, and some external tools depend on this extra information. +When run from the legends mode screen, this tool will export detailed data +about your world so that it can be browsed with external programs like +:forums:`Legends Browser <179848>`. The data is more detailed than what you can +get with vanilla export functionality, and many external tools depend on this +extra information. + +By default, ``exportlegends`` hooks into the standard vanilla ``Export XML`` button and runs in the background when you click it, allowing both the vanilla export and the extended data export to execute simultaneously. You can continue to browse legends mode via the vanilla UI while the export is running. To use: -- enter legends mode -- click the vanilla "Export XML" button to get the standard export -- run this command (``exportlegends``) to get the extended export +- Enter legends by "Starting a new game" in an existing world and selecting + Legends mode +- Ensure the toggle for "Also export extended legends data" is on (which is the + default) +- Click the "Export XML" button to generate both the standard export and the + extended data export + +You can also generate just the extended data export by manually running the +``exportlegends`` command while legends mode is open. Usage ----- @@ -23,3 +31,20 @@ Usage :: exportlegends + +Overlay +------- + +This script also provides an overlay that is managed by the `overlay` framework. +When the overlay is enabled, a toggle for exporting extended legends data will +appear below the vanilla "Export XML" button. If the toggle is enabled when the +"Export XML" button is clicked, then ``exportlegends`` will run alongside the +vanilla data export. + +While the extended data is being exported, a status line will appear in place +of the toggle, reporting the current export target and the overall percent +complete. + +There is an additional overlay that masks out the "Done" button while the +extended export is running. This prevents the player from exiting legends mode +before the export is complete. diff --git a/exportlegends.lua b/exportlegends.lua index 97e3281de9..82a85f3409 100644 --- a/exportlegends.lua +++ b/exportlegends.lua @@ -1,7 +1,12 @@ -- Export everything from legends mode --luacheck-flags: strictsubtype +--@ module=true local gui = require('gui') +local overlay = require('plugins.overlay') +local script = require('gui.script') +local widgets = require('gui.widgets') + local args = {...} -- Get the date of the world as a string @@ -39,15 +44,34 @@ local function table_containskey(self, key) return false end +progress_item = progress_item or '' +step_size = step_size or 1 +step_percent = -1 +progress_percent = progress_percent or -1 +last_update_ms = 0 + +local function yield_if_timeout() + local now_ms = dfhack.getTickCount() + if now_ms - last_update_ms > 10 then + script.sleep(1, 'frames') + last_update_ms = dfhack.getTickCount() + end +end + --luacheck: skip local function progress_ipairs(vector, desc, interval) desc = desc or 'item' interval = interval or 10000 local cb = ipairs(vector) return function(vector, k, ...) - if k and #vector >= interval and (k % interval == 0 or k == #vector - 1) then - print((' %s %i/%i (%0.f%%)'):format(desc, k, #vector, k * 100 / #vector)) + if k then + local prev_progress_percent = progress_percent + progress_percent = math.max(progress_percent, step_percent + ((k * step_size) // #vector)) + if #vector >= interval and (k % interval == 0 or k == #vector - 1) then + print((' %s %i/%i (%0.f%%)'):format(desc, k, #vector, (k * 100) / #vector)) + end end + yield_if_timeout() return cb(vector, k) end, vector, nil end @@ -96,7 +120,17 @@ local function export_more_legends_xml() file:write(""..escape_xml(dfhack.df2utf(dfhack.TranslateName(df.global.world.world_data.name))).."\n") file:write(""..escape_xml(dfhack.df2utf(dfhack.TranslateName(df.global.world.world_data.name,1))).."\n") - file:write("\n") + local function write_chunk(name, fn) + progress_item = name + yield_if_timeout() + file:write("<" .. name .. ">\n") + fn() + file:write("\n") + end + + local chunks = {} + + table.insert(chunks, {name='landmasses', fn=function() for landmassK, landmassV in progress_ipairs(df.global.world.world_data.landmasses, 'landmass') do file:write("\t\n") file:write("\t\t"..landmassV.index.."\n") @@ -105,9 +139,9 @@ local function export_more_legends_xml() file:write("\t\t"..landmassV.max_x..","..landmassV.max_y.."\n") file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='mountain_peaks', fn=function() for mountainK, mountainV in progress_ipairs(df.global.world.world_data.mountain_peaks, 'mountain') do file:write("\t\n") file:write("\t\t"..mountainK.."\n") @@ -119,9 +153,9 @@ local function export_more_legends_xml() end file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='regions', fn=function() for regionK, regionV in progress_ipairs(df.global.world.world_data.regions, 'region') do file:write("\t\n") file:write("\t\t"..regionV.index.."\n") @@ -142,9 +176,9 @@ local function export_more_legends_xml() end file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='underground_regions', fn=function() for regionK, regionV in progress_ipairs(df.global.world.world_data.underground_regions, 'underground region') do file:write("\t\n") file:write("\t\t"..regionV.index.."\n") @@ -155,9 +189,9 @@ local function export_more_legends_xml() file:write("\n") file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='rivers', fn=function() for riverK, riverV in progress_ipairs(df.global.world.world_data.rivers, 'river') do file:write("\t\n") file:write("\t\t"..escape_xml(dfhack.df2utf(dfhack.TranslateName(riverV.name, 1))).."\n") @@ -172,9 +206,9 @@ local function export_more_legends_xml() file:write("\t\t"..riverV.end_pos.x..","..riverV.end_pos.y.."\n") file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='creature_raw', fn=function() for creatureK, creatureV in ipairs (df.global.world.raws.creatures.all) do file:write("\t\n") file:write("\t\t"..creatureV.creature_id.."\n") @@ -187,15 +221,15 @@ local function export_more_legends_xml() end file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='sites', fn=function() for siteK, siteV in progress_ipairs(df.global.world.world_data.sites, 'site') do file:write("\t\n") for k,v in pairs(siteV) do if (k == "id" or k == "civ_id" or k == "cur_owner_id") then printifvalue(file, 2, k, v) --- file:write("\t\t<"..k..">"..tostring(v).."\n") + -- file:write("\t\t<"..k..">"..tostring(v).."\n") elseif (k == "buildings") then if (#siteV.buildings > 0) then file:write("\t\t\n") @@ -229,9 +263,9 @@ local function export_more_legends_xml() end file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='world_constructions', fn=function() for wcK, wcV in progress_ipairs(df.global.world.world_data.constructions.list, 'construction') do file:write("\t\n") file:write("\t\t"..wcV.id.."\n") @@ -244,9 +278,9 @@ local function export_more_legends_xml() file:write("\n") file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='artifacts', fn=function() for artifactK, artifactV in progress_ipairs(df.global.world.artifacts.all, 'artifact') do file:write("\t\n") file:write("\t\t"..artifactV.id.."\n") @@ -277,9 +311,9 @@ local function export_more_legends_xml() end file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='historical_figures', fn=function() for hfK, hfV in progress_ipairs(df.global.world.history.figures, 'historical figure') do file:write("\t\n") file:write("\t\t"..hfV.id.."\n") @@ -287,9 +321,9 @@ local function export_more_legends_xml() if hfV.race >= 0 then file:write("\t\t"..escape_xml(dfhack.df2utf(df.creature_raw.find(hfV.race).name[0])).."\n") end file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='identities', fn=function() for idK, idV in progress_ipairs(df.global.world.identities.all, 'identity') do file:write("\t\n") file:write("\t\t"..idV.id.."\n") @@ -308,9 +342,9 @@ local function export_more_legends_xml() file:write("\t\t"..idV.entity_id.."\n") file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='entity_populations', fn=function() for entityPopK, entityPopV in progress_ipairs(df.global.world.entity_populations, 'entity population') do file:write("\t\n") file:write("\t\t"..entityPopV.id.."\n") @@ -321,9 +355,9 @@ local function export_more_legends_xml() file:write("\t\t"..entityPopV.civ_id.."\n") file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='entities', fn=function() for entityK, entityV in progress_ipairs(df.global.world.entities.all, 'entity') do file:write("\t\n") file:write("\t\t"..entityV.id.."\n") @@ -431,9 +465,9 @@ local function export_more_legends_xml() end file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='historical_events', fn=function() for ID, event in progress_ipairs(df.global.world.history.events, 'event') do if df.history_event_add_hf_entity_linkst:is_instance(event) or df.history_event_add_hf_site_linkst:is_instance(event) @@ -847,8 +881,9 @@ local function export_more_legends_xml() file:write("\t\n") end end - file:write("\n") - file:write("\n") + end}) + + table.insert(chunks, {name='historical_event_relationships', fn=function() for ID, set in progress_ipairs(df.global.world.history.relationship_events, 'relationship_event') do for k = 0, set.next_element - 1 do file:write("\t\n") @@ -860,8 +895,9 @@ local function export_more_legends_xml() file:write("\t\n") end end - file:write("\n") - file:write("\n") + end}) + + table.insert(chunks, {name='historical_event_relationship_supplements', fn=function() for ID, event in progress_ipairs(df.global.world.history.relationship_event_supplements, 'relationship_event_supplement') do file:write("\t\n") file:write("\t\t"..event.event.."\n") @@ -870,13 +906,13 @@ local function export_more_legends_xml() file:write("\t\t"..event.unk_1.."\n") file:write("\t\n") end - file:write("\n") - file:write("\n") - file:write("\n") - file:write("\n") - file:write("\n") + end}) + + table.insert(chunks, {name='historical_event_collections', fn=function() end}) + + table.insert(chunks, {name='historical_eras', fn=function() end}) - file:write("\n") + table.insert(chunks, {name='written_contents', fn=function() for wcK, wcV in progress_ipairs(df.global.world.written_contents.all) do file:write("\t\n") file:write("\t\t"..wcV.id.."\n") @@ -914,57 +950,174 @@ local function export_more_legends_xml() file:write("\t\t"..wcV.author.."\n") file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='poetic_forms', fn=function() for formK, formV in progress_ipairs(df.global.world.poetic_forms.all, 'poetic form') do file:write("\t\n") file:write("\t\t"..formV.id.."\n") file:write("\t\t"..escape_xml(dfhack.df2utf(dfhack.TranslateName(formV.name,1))).."\n") file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='musical_forms', fn=function() for formK, formV in progress_ipairs(df.global.world.musical_forms.all, 'musical form') do file:write("\t\n") file:write("\t\t"..formV.id.."\n") file:write("\t\t"..escape_xml(dfhack.df2utf(dfhack.TranslateName(formV.name,1))).."\n") file:write("\t\n") end - file:write("\n") + end}) - file:write("\n") + table.insert(chunks, {name='dance_forms', fn=function() for formK, formV in progress_ipairs(df.global.world.dance_forms.all, 'dance form') do file:write("\t\n") file:write("\t\t"..formV.id.."\n") file:write("\t\t"..escape_xml(dfhack.df2utf(dfhack.TranslateName(formV.name,1))).."\n") file:write("\t\n") end - file:write("\n") + end}) + + step_size = math.max(1, 100 // #chunks) + for k, chunk in ipairs(chunks) do + progress_percent = math.max(progress_percent, (100 * k) // #chunks) + step_percent = progress_percent + write_chunk(chunk.name, chunk.fn) + end file:write("\n") file:close() local problem_elements_exist = false - for i, element in pairs (problem_elements) do - for k, field in pairs (element) do - dfhack.printerr (i.." element '"..k.."' attempted to be processed as simple type.") + for i, element in pairs(problem_elements) do + for k, field in pairs(element) do + dfhack.printerr(i.." element '"..k.."' attempted to be processed as simple type.") end problem_elements_exist = true end if problem_elements_exist then - dfhack.printerr ("Some elements could not be interpreted correctly because they were not simple elements.") - dfhack.printerr ("These elements are reported above. Please notify the DFHack community of these value pairs.") - dfhack.printerr ("Note that these issues have not invalidated the XML file: it ought to still be usable.") + dfhack.printerr("Some elements could not be interpreted correctly because they were not simple elements.") + dfhack.printerr("These elements are reported above. Please notify the DFHack community of these value pairs.") + dfhack.printerr("Note that these issues have not invalidated the XML file: it ought to still be usable.") + end + + print("Done exporting extended legends data to: " .. filename) +end + +local function wrap_export() + if progress_percent >= 0 then + qerror('exportlegends already in progress') + end + progress_percent = 0 + step_size = 1 + progress_item = 'basic info' + yield_if_timeout() + local ok, err = pcall(export_more_legends_xml) + if not ok then + dfhack.printerr(err) end + progress_percent = -1 + step_size = 1 + progress_item = '' +end + +-- ------------------- +-- LegendsOverlay +-- + +LegendsOverlay = defclass(LegendsOverlay, overlay.OverlayWidget) +LegendsOverlay.ATTRS{ + default_pos={x=2, y=2}, + default_enabled=true, + viewscreens='legends/Default', + frame={w=70, h=5}, +} + +function LegendsOverlay:init() + self:addviews{ + widgets.Panel{ + view_id='button_mask', + frame={t=0, l=0, w=15, h=3}, + }, + widgets.Panel{ + frame={b=0, l=0, r=0, h=1}, + subviews={ + widgets.ToggleHotkeyLabel{ + view_id='do_export', + frame={t=0, l=0, w=48}, + label='Also export extended legends data:', + key='CUSTOM_CTRL_D', + visible=function() return progress_percent < 0 end, + }, + widgets.Label{ + frame={t=0, l=0}, + text={ + 'Exporting ', + {text=function() return progress_item end}, + ' (', + {text=function() return progress_percent end, pen=COLOR_YELLOW}, + '% complete)' + }, + visible=function() return progress_percent >= 0 end, + }, + }, + }, + } +end + +function LegendsOverlay:onInput(keys) + if keys._MOUSE_L_DOWN and progress_percent < 0 and + self.subviews.button_mask:getMousePos() and + self.subviews.do_export:getOptionValue() + then + script.start(wrap_export) + end + return LegendsOverlay.super.onInput(self, keys) +end + +-- ------------------- +-- DoneMaskOverlay +-- + +DoneMaskOverlay = defclass(DoneMaskOverlay, overlay.OverlayWidget) +DoneMaskOverlay.ATTRS{ + default_pos={x=-2, y=2}, + default_enabled=true, + viewscreens='legends', + frame={w=9, h=3}, +} + +function DoneMaskOverlay:init() + self:addviews{ + widgets.Panel{ + frame_background=gui.CLEAR_PEN, + visible=function() return progress_percent >= 0 end, + } + } +end + +function DoneMaskOverlay:onInput(keys) + if progress_percent >= 0 then + if keys.LEAVESCREEN or (keys._MOUSE_L_DOWN and self:getMousePos()) then + return true + end + end + return DoneMaskOverlay.super.onInput(self, keys) +end + +OVERLAY_WIDGETS = { + export=LegendsOverlay, + mask=DoneMaskOverlay, +} + +if dfhack_flags.module then + return end -- Check if on legends screen and trigger the export if so -if dfhack.gui.matchFocusString('legends') then - export_more_legends_xml() -else +if not dfhack.gui.matchFocusString('legends') then qerror('exportlegends must be run from the main legends view') end -print("Exported files can be found in the top-level DF game folder.") +script.start(wrap_export) From 8ed047aa299c95ee23e118f46dafab8b8f336834 Mon Sep 17 00:00:00 2001 From: Myk Taylor Date: Mon, 7 Aug 2023 03:55:14 -0700 Subject: [PATCH 2/2] document yield latency --- exportlegends.lua | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/exportlegends.lua b/exportlegends.lua index 82a85f3409..ab8b585b67 100644 --- a/exportlegends.lua +++ b/exportlegends.lua @@ -50,9 +50,13 @@ step_percent = -1 progress_percent = progress_percent or -1 last_update_ms = 0 +-- should be frequent enough so that user can still effectively use +-- the vanilla legends UI to browse while export is in progress +local YIELD_TIMEOUT_MS = 10 + local function yield_if_timeout() local now_ms = dfhack.getTickCount() - if now_ms - last_update_ms > 10 then + if now_ms - last_update_ms > YIELD_TIMEOUT_MS then script.sleep(1, 'frames') last_update_ms = dfhack.getTickCount() end @@ -65,7 +69,6 @@ local function progress_ipairs(vector, desc, interval) local cb = ipairs(vector) return function(vector, k, ...) if k then - local prev_progress_percent = progress_percent progress_percent = math.max(progress_percent, step_percent + ((k * step_size) // #vector)) if #vector >= interval and (k % interval == 0 or k == #vector - 1) then print((' %s %i/%i (%0.f%%)'):format(desc, k, #vector, (k * 100) / #vector))