Core: Some changes to reduce memory usage of call analysis + native m…

…ethod detection

Should help with #306

.editorconfig

@@ -1,11 +1,14 @@
-# top-most EditorConfig file
 root = true
+# top-most EditorConfig file
 
 # Unix-style newlines with a newline ending every file
 [*]
 end_of_line = lf
 insert_final_newline = true
 
+# ReSharper properties
+resharper_indent_inside_namespace = true
+
 # Matches multiple files with brace expansion notation
 # Set default charset
 [*.{cs,yml,config}]

Cpp2IL.Core/ISIL/InstructionSetIndependentInstruction.cs

@@ -19,7 +19,7 @@ public InstructionSetIndependentInstruction(InstructionSetIndependentOpCode opCo
         OpCode.Validate(this);
     }
 
-    public override string ToString() => $"{InstructionIndex:000} {OpCode} {string.Join(", ", (IEnumerable<InstructionSetIndependentOperand>) Operands)}";
+    public override string ToString() => $"{InstructionIndex:000} {OpCode} {string.Join(", ", Operands)}";
 
     /// <summary>
     /// Marks the instruction as <see cref="IsilMnemonic.Invalid"/>.

Cpp2IL.Core/Il2CppApiFunctions/BaseKeyFunctionAddresses.cs

@@ -66,7 +66,7 @@ private void FindExport(string name, out ulong ptr)
             Logger.VerboseNewline(ptr == 0 ? "Not found" : $"Found at 0x{ptr:X}");
         }
 
-        public void Find(ApplicationAnalysisContext applicationAnalysisContext)
+        public virtual void Find(ApplicationAnalysisContext applicationAnalysisContext)
         {
             _appContext = applicationAnalysisContext;
             Init(applicationAnalysisContext);

Cpp2IL.Core/Il2CppApiFunctions/X86KeyFunctionAddresses.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using Cpp2IL.Core.Logging;
+using Cpp2IL.Core.Model.Contexts;
 using Cpp2IL.Core.Utils;
 using Iced.Intel;
 using LibCpp2IL;
@@ -24,6 +25,13 @@ private InstructionList DisassembleTextSection()
             return _cachedDisassembledBytes;
         }
 
+        public override void Find(ApplicationAnalysisContext applicationAnalysisContext)
+        {
+            base.Find(applicationAnalysisContext);
+
+            _cachedDisassembledBytes = null; //Clean up once we're done finding everything
+        }
+
         protected override IEnumerable<ulong> FindAllThunkFunctions(ulong addr, uint maxBytesBack = 0, params ulong[] addressesToIgnore)
         {
             //Disassemble .text
@@ -132,4 +140,4 @@ protected override int GetCallerCount(ulong toWhere)
             return allInstructions.Count(i => i.Mnemonic == Mnemonic.Jmp || i.Mnemonic == Mnemonic.Call && i.NearBranchTarget == toWhere);
         }
     }
-}
+}

Cpp2IL.Core/InstructionSets/X86InstructionSet.cs

@@ -13,25 +13,42 @@ namespace Cpp2IL.Core.InstructionSets;
 
 public class X86InstructionSet : Cpp2IlInstructionSet
 {
+    private static readonly MasmFormatter Formatter = new();
+    private static readonly StringOutput Output = new();
+
+    private static string FormatInstructionInternal(Instruction instruction)
+    {
+        Formatter.Format(instruction, Output);
+        return Output.ToStringAndReset();
+    }
+
+    public static string FormatInstruction(Instruction instruction)
+    {
+        lock (Formatter)
+        {
+            return FormatInstructionInternal(instruction);
+        }
+    }
 
     public override Memory<byte> GetRawBytesForMethod(MethodAnalysisContext context, bool isAttributeGenerator) => X86Utils.GetRawManagedOrCaCacheGenMethodBody(context.UnderlyingPointer, isAttributeGenerator);
 
     public override BaseKeyFunctionAddresses CreateKeyFunctionAddressesInstance() => new X86KeyFunctionAddresses();
 
     public override string PrintAssembly(MethodAnalysisContext context)
     {
-        var insns = X86Utils.Disassemble(X86Utils.GetRawManagedOrCaCacheGenMethodBody(context.UnderlyingPointer, false), context.UnderlyingPointer);
+        lock (Formatter)
+        {
+            var insns = X86Utils.Iterate(X86Utils.GetRawManagedOrCaCacheGenMethodBody(context.UnderlyingPointer, false), context.UnderlyingPointer);
 
-        return string.Join("\n", insns);
+            return string.Join("\n", insns.Select(FormatInstructionInternal));
+        }
     }
 
     public override List<InstructionSetIndependentInstruction> GetIsilFromMethod(MethodAnalysisContext context)
     {
-        var insns = X86Utils.Disassemble(context.RawBytes, context.UnderlyingPointer);
-
         var builder = new IsilBuilder();
 
-        foreach (var instruction in insns)
+        foreach (var instruction in X86Utils.Iterate(context.RawBytes, context.UnderlyingPointer))
         {
             ConvertInstructionStatement(instruction, builder, context);
         }
@@ -163,7 +180,7 @@ private void ConvertInstructionStatement(Instruction instruction, IsilBuilder bu
 
                         ret++; //For MethodInfo arg
                         return ret;
-                    }).ToArray();
+                    });
 
                     // if (parameterCounts.Max() != parameterCounts.Min())
                     // throw new("Cannot handle call to address with multiple managed methods of different parameter counts");
@@ -296,7 +313,7 @@ private void ConvertInstructionStatement(Instruction instruction, IsilBuilder bu
                 builder.Interrupt(instruction.IP); // We'll add it but eliminate later
                 break;
             default:
-                builder.NotImplemented(instruction.IP, instruction.ToString());
+                builder.NotImplemented(instruction.IP, FormatInstruction(instruction));
                 break;
         }
     }

Cpp2IL.Core/Model/Contexts/MethodAnalysisContext.cs

@@ -147,6 +147,12 @@ public void Analyze()
         // ControlFlowGraph.Run();
         // InstructionSetIndependentNodes = AppContext.InstructionSet.ControlFlowGraphToISIL(ControlFlowGraph, this);
     }
+
+    public void ReleaseAnalysisData()
+    {
+        ConvertedIsil = null;
+        ControlFlowGraph = null;
+    }
 
     public override string ToString() => $"Method: {Definition?.DeclaringType!.Name}::{Definition?.Name ?? "No definition"}";
 

Cpp2IL.Core/OutputFormats/IsilDumpOutputFormat.cs

@@ -48,12 +48,13 @@ public override void DoOutput(ApplicationAnalysisContext context, string outputR
 
                     try
                     {
-                        method.Analyze();
 
                         typeDump.AppendLine("Disassembly:");
                         typeDump.Append('\t').AppendLine(context.InstructionSet.PrintAssembly(method).Replace("\n", "\n\t"));
 
                         typeDump.AppendLine().AppendLine("ISIL:");
+
+                        method.Analyze();
 
                         if (method.ConvertedIsil == null || method.ConvertedIsil.Count == 0)
                         {
@@ -65,6 +66,8 @@ public override void DoOutput(ApplicationAnalysisContext context, string outputR
                         {
                             typeDump.Append('\t').Append(isilInsn).AppendLine();
                         }
+
+                        method.ReleaseAnalysisData();
 
                         typeDump.AppendLine();
                     }

Cpp2IL.Core/ProcessingLayers/CallAnalysisProcessingLayer.cs

@@ -75,6 +75,7 @@ private static void InjectAttribute(ApplicationAnalysisContext appContext)
                     {
                         AttributeInjectionUtils.AddZeroParameterAttribute(m, analysisNotSupportedConstructor);
                     }
+                    m.ReleaseAnalysisData();
                     continue;
                 }
 
@@ -124,6 +125,8 @@ private static void InjectAttribute(ApplicationAnalysisContext appContext)
                         unknownCalls[m] = unknownCalls.GetOrDefault(m, 0) + 1;
                     }
                 }
+
+                m.ReleaseAnalysisData();
             }
         }
 

Cpp2IL.Core/ProcessingLayers/NativeMethodDetectionProcessingLayer.cs

@@ -52,6 +52,7 @@ private static void AnalyzeMethod(ApplicationAnalysisContext appContext, MethodA
 
         if (m.ConvertedIsil is { Count: 0 })
         {
+            m.ReleaseAnalysisData();
             return;
         }
 
@@ -72,6 +73,8 @@ private static void AnalyzeMethod(ApplicationAnalysisContext appContext, MethodA
                 }
             }
         }
+
+        m.ReleaseAnalysisData();
     }
 
     private static bool TryGetAddressFromInstruction(InstructionSetIndependentInstruction instruction, out ulong address)

Cpp2IL.Core/Utils/MiscUtils.cs

@@ -317,5 +317,55 @@ public static string CleanPathElement(string input)
 
             return InvalidPathElements.Contains(input) ? $"__invalidwin32name_{input}__" : input;
         }
+
+        public static MemoryEnumerator<T> GetEnumerator<T>(this Memory<T> memory) => new(memory);
+
+        public static MemoryEnumerable<T> AsEnumerable<T>(this Memory<T> memory) => new(memory);
+
+        public class MemoryEnumerable<T> : IEnumerable<T>
+        {
+            private readonly Memory<T> _memory;
+
+            public MemoryEnumerable(Memory<T> memory)
+            {
+                _memory = memory;
+            }
+
+            public IEnumerator<T> GetEnumerator() => new MemoryEnumerator<T>(_memory);
+
+            IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+        }
+
+        public class MemoryEnumerator<T> : IEnumerator<T>
+        {
+            private readonly Memory<T> _memory;
+
+            private int _index = -1;
+
+            public MemoryEnumerator(Memory<T> memory)
+            {
+                _memory = memory;
+            }
+
+            public bool MoveNext()
+            {
+                _index++;
+                return _index < _memory.Length;
+            }
+
+            public void Reset()
+            {
+                _index = -1;
+            }
+
+            public T Current => _memory.Span[_index];
+
+            object? IEnumerator.Current => Current;
+
+            public void Dispose()
+            {
+                // Nothing to dispose
+            }
+        }
     }
 }

Cpp2IL.Core/Utils/X86Utils.cs

@@ -26,14 +26,33 @@ public static InstructionList Disassemble(byte[] bytes, ulong methodBase)
             var decoder = Decoder.Create(LibCpp2IlMain.Binary!.is32Bit ? 32 : 64, codeReader);
             decoder.IP = methodBase;
             var instructions = new InstructionList();
-            var endRip = decoder.IP + (uint) bytes.Length;
+            var endRip = decoder.IP + (uint)bytes.Length;
 
             while (decoder.IP < endRip)
                 instructions.Add(decoder.Decode());
 
             return instructions;
         }
 
+        public static IEnumerable<Instruction> Iterate(Memory<byte> bytes, ulong methodBase)
+        {
+            return Iterate(bytes.AsEnumerable(), methodBase);
+        }
+
+        public static IEnumerable<Instruction> Iterate(IEnumerable<byte> bytes, ulong methodBase)
+        {
+            var codeReader = new EnumerableCodeReader(bytes);
+            var decoder = Decoder.Create(LibCpp2IlMain.Binary!.is32Bit ? 32 : 64, codeReader);
+            decoder.IP = methodBase;
+
+            decoder.Decode(out var instruction);
+            while (!instruction.IsInvalid)
+            {
+                yield return instruction;
+                decoder.Decode(out instruction);
+            }
+        }
+
         public static Memory<byte> GetRawManagedOrCaCacheGenMethodBody(ulong ptr, bool isCaGen)
         {
             var rawAddr = LibCpp2IlMain.Binary!.MapVirtualAddressToRaw(ptr, false);
@@ -80,8 +99,8 @@ public static Memory<byte> GetRawManagedOrCaCacheGenMethodBody(ulong ptr, bool i
 
             while (rawArray[lastPos] == 0xCC && lastPos > rawAddr)
                 lastPos--;
-            var memArray =  rawArray.AsMemory((int) rawAddr, (int) (lastPos - rawAddr + 1));
-            
+            var memArray = rawArray.AsMemory((int)rawAddr, (int)(lastPos - rawAddr + 1));
+
             if (TryFindJumpTableStart(memArray, ptr, virtStartNextFunc, out var startIndex, out var jumpTableElements))
             {
                 // TODO: Figure out what to do with jumpTableElements, how do we handle returning it from this function?
@@ -93,18 +112,18 @@ public static Memory<byte> GetRawManagedOrCaCacheGenMethodBody(ulong ptr, bool i
                 */
                 memArray = memArray.Slice(0, startIndex);
             }
-            return memArray;
 
+            return memArray;
         }
 
         private static bool TryFindJumpTableStart(Memory<byte> methodBytes, ulong methodPtr, ulong nextMethodPtr, out int startIndex, out List<ulong> jumpTableElements)
         {
             bool foundTable = false;
             startIndex = 0;
             jumpTableElements = new List<ulong>();
-            for (int i = (int) (methodPtr % 4); i < methodBytes.Length; i += 4)
+            for (int i = (int)(methodPtr % 4); i < methodBytes.Length; i += 4)
             {
-                var result = (ulong) methodBytes.Span.ReadUInt(i); 
+                var result = (ulong)methodBytes.Span.ReadUInt(i);
                 var possibleJumpAddress = result + 0x180000000; // image base
                 if (possibleJumpAddress > methodPtr && possibleJumpAddress < nextMethodPtr)
                 {
@@ -114,6 +133,7 @@ private static bool TryFindJumpTableStart(Memory<byte> methodBytes, ulong method
                         startIndex = i;
                         foundTable = true;
                     }
+
                     jumpTableElements.Add(result);
                 }
             }
@@ -144,7 +164,7 @@ public static InstructionList GetMethodBodyAtVirtAddressNew(ulong addr, bool pee
                 if (addr >= startOfNextFunc)
                     break;
 
-                buff.Add(LibCpp2IlMain.Binary.GetByteAtRawAddress((ulong) rawAddr));
+                buff.Add(LibCpp2IlMain.Binary.GetByteAtRawAddress((ulong)rawAddr));
 
                 ret = X86Utils.Disassemble(buff.ToArray(), functionStart);
 
@@ -229,5 +249,23 @@ public static string GetRegisterName(Register register)
 
             return ret;
         }
+
+        private class EnumerableCodeReader : CodeReader
+        {
+            private readonly IEnumerator<byte> _enumerator;
+
+            public EnumerableCodeReader(IEnumerable<byte> bytes)
+            {
+                _enumerator = bytes.GetEnumerator();
+            }
+
+            public override int ReadByte()
+            {
+                if (_enumerator.MoveNext())
+                    return _enumerator.Current;
+
+                return -1;
+            }
+        }
     }
 }