Merge pull request #276 from joshua-smith-12/gdb

support GDB debugging on overlays via custom GDB build

.gitignore

@@ -78,7 +78,8 @@ cmake-build-*
 diff.txt
 
 .vs/
-.vscode/
+.vscode/*
+!.vscode/launch.json
 
 temp_asm/
 asm_old/

.vscode/launch.json

@@ -0,0 +1,39 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Debug in melonDS",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/build/debug.nef",
+            "MIMode": "gdb",
+            "cwd": "${workspaceFolder}",
+            "externalConsole": true,
+            "miDebuggerServerAddress": "localhost:3333",
+            // point this to your own gdb path...
+            "miDebuggerPath": "/path/to/binutils-gdb/gdb/gdb",
+            //"miDebuggerPath": "gdb-multiarch",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                },
+                {
+                    "description": "Set architecture",
+                    "text": "set architecture armv5te"
+                },
+                {
+                    "description": "Enable overlays",
+                    "text": "overlay auto"
+                },
+                {
+                    "description": "Enable overlay map",
+                    "text": "overlay map build/overlay.map"
+                }
+            ],
+            "stopAtConnect": false,
+            "stopAtEntry": false
+        }
+    ]
+}

build.sh

@@ -18,10 +18,14 @@ export NINJA_STATUS="[%p %f/%t] "
 export MESON_RSP_THRESHOLD=16387
 
 # Build the project
+"${MESON:-meson}" configure build "-Dgdb_debugging=false"
 if [ "$target" = test ]; then
     "${MESON:-meson}" test -C build "$@"
 elif [ "$target" = rom ]; then
     "${MESON:-meson}" compile -C build "pokeplatinum.us.nds"
+elif [ "$target" = debug ]; then
+    "${MESON:-meson}" configure build "-Dgdb_debugging=true"
+    "${MESON:-meson}" compile -C build "pokeplatinum.us.nds" "debug.nef" "overlay.map"
 else
     "${MESON:-meson}" compile -C build "$target" "$@"
 fi

include/game_overlay.h

@@ -13,4 +13,24 @@ void Overlay_UnloadByID(const FSOverlayID param0);
 int Overlay_GetLoadDestination(const FSOverlayID param0);
 BOOL Overlay_LoadByID(const FSOverlayID param0, int param1);
 
+#ifdef GDB_DEBUGGING
+// describes a single overlay entry, which GDB can inspect to determine which overlays are loaded.
+typedef struct {
+    unsigned long vma;
+    unsigned long size;
+    FSOverlayID id;
+    unsigned long mapped;
+} struct_overlayTable;
+
+// this is set based on the current number of overlays, other projects might need more!
+#define MAX_OVERLAYS 128
+
+// externs required for GDB to access overlay state
+extern unsigned long _novlys;
+extern struct_overlayTable _ovly_table[MAX_OVERLAYS];
+
+// event callback which GDB will hook and use to refresh overlay state
+static void _ovly_debug_event(void);
+#endif // GDB_DEBUGGING
+
 #endif // POKEPLATINUM_UNK_020064F0_H

meson.build

@@ -19,7 +19,6 @@ public_includes = include_directories('include', 'asm', 'res')
 ###                    COMPILER FLAGS                    ###
 ############################################################
 c_args = [
-    '-O4,p',
     '-proc', 'arm946e',
     '-enum', 'int',
     '-lang', 'c99',
@@ -35,6 +34,15 @@ c_args = [
     '-sym', 'on'
 ]
 
+if get_option('gdb_debugging')
+    c_args += [
+        '-O1,p',
+        '-inline', 'off'
+    ]
+else
+    c_args += '-O4,p'
+endif
+
 add_global_arguments(c_args,
     language: 'c',
     native: false
@@ -46,6 +54,10 @@ pokeplatinum_args = [
     '-DGAME_LANGUAGE=ENGLISH'
 ]
 
+if get_option('gdb_debugging')
+    pokeplatinum_args += '-DGDB_DEBUGGING'
+endif
+
 asm_args = [
     '-proc', 'arm5TE',
     '-16',
@@ -228,6 +240,33 @@ pokeplatinum_nds = custom_target('pokeplatinum.us.nds',
     build_by_default: true
 )
 
+############################################################
+###                   GDB HELPERS                        ###
+############################################################
+nef_fixer = custom_target('debug.nef',
+    output: [
+        'debug.nef'
+    ],
+    input: [
+        main
+    ],
+    command : [
+        nef_fixer_py, '@INPUT@', '@OUTPUT@'
+    ]
+)
+
+ovly_mapper = custom_target('overlay.map',
+    output: [
+        'overlay.map'
+    ],
+    input: [
+        main_lsf
+    ],
+    command : [
+        overlay_mapper_py, '@INPUT@', '@OUTPUT@'
+    ]
+)
+
 
 ############################################################
 ###                        TESTS                         ###

meson.options

@@ -0,0 +1 @@
+option('gdb_debugging', type : 'boolean', value : false)

src/game_overlay.c

@@ -34,6 +34,47 @@ BOOL Overlay_LoadByID(const FSOverlayID param0, int param1);
 
 static UnkStruct_021BF370 Unk_021BF370;
 
+#ifdef GDB_DEBUGGING
+
+/* Added to support GDB overlay debugging. */
+unsigned long _novlys = MAX_OVERLAYS;
+struct_overlayTable _ovly_table[MAX_OVERLAYS] = {};
+// this does nothing, but needs to be defined for GDB to refresh overlay state automatically.
+static void _ovly_debug_event(void)
+{
+}
+
+// helper function to mark a specific overlay as unmapped.
+void UnloadOverlayGDB(const FSOverlayID overlayID)
+{
+    GF_ASSERT(overlayID < _novlys);
+    _ovly_table[overlayID].mapped--;
+    _ovly_debug_event();
+}
+// helper function to mark a specific overlay as mapped, and provide its RAM address and size to GDB.
+void LoadOverlayGDB(const FSOverlayID overlayID)
+{
+    FSOverlayInfo overlayInfo;
+
+    GF_ASSERT(overlayID < _novlys);
+
+    // 1. fetch overlay info to identify vma
+    GF_ASSERT(FS_LoadOverlayInfo(&overlayInfo, MI_PROCESSOR_ARM9, overlayID) == TRUE);
+
+    // 2. add entry to _ovly_table
+    // note that this is a little hacky. the VMA is correct but the LMA is not exposed by the OverlayManager
+    // and the size field is not correct compared to what's stored in the NEF.
+    // the standard overlay manager in GDB bases comparisons on VMA and LMA, so it's not viable here.
+    // requires a custom GDB build which maps based on section ID and can override section size.
+    // see https://github.com/joshua-smith-12/binutils-gdb-nds
+    _ovly_table[overlayID].vma = overlayInfo.header.ram_address;
+    _ovly_table[overlayID].id = overlayID;
+    _ovly_table[overlayID].size = overlayInfo.header.ram_size;
+    _ovly_table[overlayID].mapped++;
+    _ovly_debug_event();
+}
+#endif // GDB_DEBUGGING
+
 static void FreeOverlayAllocation(PMiLoadedOverlay *param0)
 {
     GF_ASSERT(param0->isActive == 1);
@@ -52,6 +93,9 @@ void Overlay_UnloadByID(const FSOverlayID overlayID)
     for (i = 0; i < 8; i++) {
         if ((table[i].isActive == 1) && (table[i].id == overlayID)) {
             FreeOverlayAllocation(&table[i]);
+#ifdef GDB_DEBUGGING
+            UnloadOverlayGDB(overlayID);
+#endif
             return;
         }
     }
@@ -195,6 +239,9 @@ static BOOL GetOverlayRamBounds(const FSOverlayID overlayID, u32 *start, u32 *en
 
 static BOOL LoadOverlayNormal(MIProcessor proc, FSOverlayID overlayID)
 {
+#ifdef GDB_DEBUGGING
+    LoadOverlayGDB(overlayID);
+#endif
     return FS_LoadOverlay(proc, overlayID);
 }
 
@@ -210,6 +257,10 @@ static BOOL LoadOverlayNoInit(MIProcessor proc, FSOverlayID overlayID)
         return FALSE;
     }
 
+#ifdef GDB_DEBUGGING
+    LoadOverlayGDB(overlayID);
+#endif
+
     FS_StartOverlay(&info);
     return TRUE;
 }
@@ -223,6 +274,10 @@ static BOOL LoadOverlayNoInitAsync(MIProcessor proc, FSOverlayID overlayID)
         return FALSE;
     }
 
+#ifdef GDB_DEBUGGING
+    LoadOverlayGDB(overlayID);
+#endif
+
     FS_InitFile(&file);
     FS_LoadOverlayImageAsync(&info, &file);
     FS_WaitAsync(&file);

tools/debug/meson.build

@@ -0,0 +1,2 @@
+nef_fixer_py = find_program('nef_fixer.py', native: true)
+overlay_mapper_py = find_program('overlay_mapper.py', native: true)

tools/debug/nef_fixer.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+### This tool is responsible for using `debugedit` to fix Wine paths in the debuginfo file to real filesystem paths.
+### VSC requires the debuginfo paths to map to real filesystem paths in order for VSC to correctly open sources during a debugging session.
+### (This works fine on my Linux system, but may or may not work on WSL, and will certainly not work on other systems)
+import subprocess
+import os
+import shutil
+import sys
+
+try:
+    os.remove("sources.txt")
+except:
+    pass
+
+source = sys.argv[1]
+dest = sys.argv[2]
+
+print(f"Copying {source} to {dest}")
+shutil.copyfile(source, dest)
+
+print("Retrieving source file list via debugedit")
+subprocess.run(["debugedit", "-l", "sources.txt", dest])
+
+with open("sources.txt") as f:
+    content = f.read()
+    all_sources = content.split('\0')
+
+print("Identifying unique source directories")
+source_paths = []
+for source in all_sources:
+    source_parts = source.split(":")
+    if len(source_parts) != 3: continue
+
+    source_original = source_parts[0] + ":" + source_parts[1]
+    source_original = source_original[:-2]
+
+    if source_original not in source_paths:
+        source_paths.append(source_original)
+
+print(f"Identified {len(source_paths)} unique source directories from source file list")
+for source in source_paths:
+    remapped = source.replace("\\", "/")[2:]
+    print(f"Remapping source path from {source} to {remapped} using debugedit.")
+    subprocess.run(["debugedit", "-b", source, "-d", remapped, dest])

tools/debug/overlay_mapper.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+### This tool creates a file which maps overlay section names and source file names to overlay IDs.
+### custom GDB build then processes this source map and uses it to identify what overlay a source file belongs to.
+### (This makes it a lot easier to identify which overlapping overlay is used for a given input location).
+import subprocess
+import os
+import shutil
+import sys
+
+try:
+    os.remove("overlay.map")
+except:
+    pass
+
+source = sys.argv[1]
+dest = sys.argv[2]
+
+# TERRIBLE but necessary to allow using the LSF file
+def base_path(path):
+    splitted = path.split("_")
+    maybe_path = ""
+    found_dir = ".." # need to escape builddir because this sucks
+    while len(splitted) > 1:
+        segment = splitted[0]
+        splitted = splitted[1:]
+        maybe_path += segment
+        if os.path.isdir(found_dir + "/" + maybe_path):
+            found_dir = found_dir + "/" + maybe_path
+            maybe_path = ""
+        else:
+            maybe_path += "_"
+    maybe_path += splitted[0]
+    return maybe_path[:-2] # remove '.o'
+
+# VERY inflexible parsing!
+overlays = {}
+sources = {}
+overlay_index = 0
+current_overlay = None
+in_block = False
+with open(source) as f:
+    for line in f.readlines():
+        if current_overlay == None and not in_block and line.strip().startswith("Overlay"):
+            current_overlay = line.strip().replace("Overlay ", "")
+            overlays[current_overlay] = overlay_index
+            overlay_index += 1
+        elif not in_block and line.strip() == "{":
+            in_block = True
+        elif in_block and line.strip() == "}":
+            in_block = False
+            current_overlay = None
+        elif current_overlay != None and in_block and line.strip().startswith("Object"):
+            obj_name = base_path(line.strip().replace("Object ", "").split("/")[-1])
+            sources[obj_name] = current_overlay
+
+with open(dest, mode="w") as f:
+    for ovly in overlays:
+        f.write(f"OVERLAY {ovly}:{overlays[ovly]}\n")
+    f.write("\n")
+    for s in sources:
+        f.write(f"SOURCE {s}:{sources[s]}\n")

tools/meson.build

@@ -5,6 +5,7 @@ subdir('json2bin')
 subdir('msgenc')
 subdir('postconf')
 subdir('scripts')
+subdir('debug')
 
 # Prebuilt tools
 mwrap_exe = find_program('cw/mwrap', native: true)

tools/scripts/meson.build

@@ -9,4 +9,4 @@ make_pl_growtbl_py = find_program('make_pl_growtbl.py', native: true)
 make_species_tables_py = find_program('make_species_tables.py', native: true)
 make_tutorable_moves_py = find_program('make_tutorable_moves.py', native: true)
 make_pokedex_data_py = find_program('make_pokedex_data.py', native: true)
-make_pokedex_message_banks_py = find_program('make_pokedex_message_banks.py', native: true)
+make_pokedex_message_banks_py = find_program('make_pokedex_message_banks.py', native: true)