djdev64: allow to request dlmopen() explicitly

It appears asan doesn't support RTLD_DEEPBIND flag, it just aborts on it. To avoid RTLD_DEEPBIND we need dlmopen. In case of missing dlmopen, asan support is impossible.
stsp · Jul 28, 2024 · 5a092b3 · 5a092b3
1 parent 99c94c5
commit 5a092b3
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 11 deletions.
diff --git a/src/djdev64/djdev64.c b/src/djdev64/djdev64.c
@@ -16,11 +16,7 @@
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define WANT_DLMOPEN 0
-
-#if WANT_DLMOPEN
 #define _GNU_SOURCE
-#endif
 #include <dlfcn.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -31,11 +27,9 @@
 #include "djdev64/djdev64.h"
 #include "elf_priv.h"
 
-#if WANT_DLMOPEN
 #ifdef __GLIBC__
 #define HAVE_DLMOPEN 1
 #endif
-#endif
 
 static int handles;
 #define HNDL_MAX 5
@@ -161,18 +155,32 @@ static int _djdev64_open(const char *path, const struct dj64_api *api,
         fprintf(stderr, "out of handles\n");
         return -1;
     }
-    /* RTLD_DEEPBIND has similar effect to -Wl,-Bsymbolic */
-    if (flags & FLG_STATIC) {
+    /* RTLD_DEEPBIND has similar effect to -Wl,-Bsymbolic, but we don't
+     * use it with dlmopen() as asan doesn't work with that flag.
+     * So if we don't have dlmopen() then no asan support is possible. */
+    if (flags & DJ64F_DLMOPEN) {
+#ifdef HAVE_DLMOPEN
+        use_dlm = 1;
+        dlh = dlmopen(LM_ID_NEWLM, path, RTLD_LOCAL | RTLD_NOW);
+#else
+        fprintf(stderr, "dlmopen() not supported, asan won't work\n");
+        return -1;
+#endif
+    } else if (flags & FLG_STATIC) {
         dlh = emu_dlmopen(handles, path, RTLD_LOCAL | RTLD_NOW | RTLD_DEEPBIND,
                 &path2);
     } else {
+#define WANT_DLMOPEN 0
+#if WANT_DLMOPEN
 #ifdef HAVE_DLMOPEN
         use_dlm = 1;
-        dlh = dlmopen(LM_ID_NEWLM, path, RTLD_LOCAL | RTLD_NOW | RTLD_DEEPBIND);
+        dlh = dlmopen(LM_ID_NEWLM, path, RTLD_LOCAL | RTLD_NOW);
 #else
-#if WANT_DLMOPEN
         fprintf(stderr, "dlmopen() not supported, use static linking\n");
+        dlh = emu_dlmopen(handles, path, RTLD_LOCAL | RTLD_NOW | RTLD_DEEPBIND,
+                &path2);
 #endif
+#else
         dlh = emu_dlmopen(handles, path, RTLD_LOCAL | RTLD_NOW | RTLD_DEEPBIND,
                 &path2);
 #endif

diff --git a/src/djdev64/include/djdev64/dj64init.h b/src/djdev64/include/djdev64/dj64init.h
@@ -24,7 +24,7 @@
 
 typedef int (dj64cdispatch_t)(int handle, int libid, int fn, unsigned esi,
         uint8_t *sp);
-#define DJ64_API_VER 8
+#define DJ64_API_VER 9
 enum { DJ64_PRINT_LOG, DJ64_PRINT_TERMINAL, DJ64_PRINT_SCREEN };
 
 /* pushal */

diff --git a/src/djdev64/include/djdev64/djdev64.h b/src/djdev64/include/djdev64/djdev64.h
@@ -21,6 +21,9 @@
 
 #include "djdev64/dj64init.h"
 
+/* first 16 bits are for internal use */
+#define DJ64F_DLMOPEN (1 << 16)
+
 int djdev64_open(const char *path, const struct dj64_api *api, int api_ver,
         unsigned flags);
 int djdev64_call(int handle, int libid, int fn, unsigned esi,