R2RDump improvements (#46833)

* Fix handling of module override token in signature parser. When the override is present, a new SignatureDecoder is created and used as the decoder for the final signature with the fixup kind (and module override flag which is stored in the upper bit of the fixup kind byte) already parsed. This causes the remainder of the signature to be parsed as a full R2R signature which is now missing the fixup type.
* Instead of creating a new decoder when a module override is present, set up the initial decoder's metadata reader in the constructor by detecting the module override up front.
* Fix `TryLocateNativeReadyToRunHeader` to swallow `BadImageFormatException` and return true / false whether the image has a native R2R header (for composite images).
* Running R2RDump on IL assemblies with no R2R now emits an error message that the assembly is not R2R instead of an unhelpful error about some RVA offset conversion failing.

Author: nattress

src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunReader.cs

@@ -481,12 +481,21 @@ public IReadOnlyDictionary<TMethod, ReadyToRunMethod> GetCustomMethodToRuntimeFu
 
         private bool TryLocateNativeReadyToRunHeader()
         {
-            PEExportTable exportTable = CompositeReader.GetExportTable();
-            if (exportTable.TryGetValue("RTR_HEADER", out _readyToRunHeaderRVA))
+            try
             {
-                _composite = true;
-                return true;
+                PEExportTable exportTable = CompositeReader.GetExportTable();
+                if (exportTable.TryGetValue("RTR_HEADER", out _readyToRunHeaderRVA))
+                {
+                    _composite = true;
+                    return true;
+                }
             }
+            catch (BadImageFormatException)
+            {
+                // MSIL assemblies with no ready-to-run payload typically have no export table
+                return false;
+            }
+
             return false;
         }
 

src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs

@@ -458,13 +458,15 @@ protected void UpdateOffset(int offset)
         /// <param name="offset">Signature offset within the PE file byte array</param>
         public R2RSignatureDecoder(IR2RSignatureTypeProvider<TType, TMethod, TGenericContext> provider, TGenericContext context, MetadataReader metadataReader, ReadyToRunReader r2rReader, int offset)
         {
-            _metadataReader = metadataReader;
-            _outerReader = metadataReader;
             Context = context;
             _provider = provider;
             _image = r2rReader.Image;
             _originalOffset = _offset = offset;
             _contextReader = r2rReader;
+            MetadataReader moduleOverrideMetadataReader = TryGetModuleOverrideMetadataReader();
+            _metadataReader = moduleOverrideMetadataReader ?? metadataReader;
+            _outerReader = moduleOverrideMetadataReader ?? metadataReader;
+            Reset();
         }
 
         /// <summary>
@@ -480,11 +482,27 @@ public R2RSignatureDecoder(IR2RSignatureTypeProvider<TType, TMethod, TGenericCon
         {
             Context = context;
             _provider = provider;
-            _metadataReader = metadataReader;
             _image = signature;
             _originalOffset = _offset = offset;
-            _outerReader = outerReader;
             _contextReader = contextReader;
+            MetadataReader moduleOverrideMetadataReader = TryGetModuleOverrideMetadataReader();
+            _metadataReader = moduleOverrideMetadataReader ?? metadataReader;
+            _outerReader = moduleOverrideMetadataReader ?? outerReader;
+            Reset();
+        }
+
+        private MetadataReader TryGetModuleOverrideMetadataReader()
+        {
+            bool moduleOverride = (ReadByte() & (byte)ReadyToRunFixupKind.ModuleOverride) != 0;
+            // Check first byte for a module override being encoded
+            if (moduleOverride)
+            {
+                int moduleIndex = (int)ReadUInt();
+                IAssemblyMetadata refAsmEcmaReader = _contextReader.OpenReferenceAssembly(moduleIndex);
+                return refAsmEcmaReader.MetadataReader;
+            }
+
+            return null;
         }
 
         /// <summary>
@@ -1098,17 +1116,15 @@ private ReadyToRunSignature ParseSignature(StringBuilder builder)
             bool moduleOverride = (fixupType & (byte)ReadyToRunFixupKind.ModuleOverride) != 0;
             SignatureDecoder moduleDecoder = this;
 
-            // Check first byte for a module override being encoded
+            // Check first byte for a module override being encoded. The metadata reader for the module
+            // override is configured in the R2RSignatureDecoder constructor.
             if (moduleOverride)
             {
                 fixupType &= ~(uint)ReadyToRunFixupKind.ModuleOverride;
-                int moduleIndex = (int)ReadUIntAndEmitInlineSignatureBinary(builder);
-                IAssemblyMetadata refAsmEcmaReader = _contextReader.OpenReferenceAssembly(moduleIndex);
-                moduleDecoder = new SignatureDecoder(Context.AssemblyResolver, Context.Options, refAsmEcmaReader.MetadataReader, _image, Offset, refAsmEcmaReader.MetadataReader, _contextReader);
+                ReadUIntAndEmitInlineSignatureBinary(builder);
             }
 
-            ReadyToRunSignature result = moduleDecoder.ParseSignature((ReadyToRunFixupKind)fixupType, builder);
-            UpdateOffset(moduleDecoder.Offset);
+            ReadyToRunSignature result = ParseSignature((ReadyToRunFixupKind)fixupType, builder);
             return result;
         }