From f38644253e9bc6268defccd4f7aa22d731f062df Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 15 Feb 2024 08:44:24 +0100 Subject: [PATCH] TestEngine: Get instruction coverage and move to net standard (#898) * Get execution coverage * Improve coverage comments * Get coverage * GetCoverage extension * format * Net standard and CoveredPercentage * AbiMethod * clean * Allow coverage on Read&Write properties * format * Update README.md * Allow to sum coverages * clean * Allow to sum coverage contracts * Allow to join coverage from multiple sources * change to public and ready to review * Format and load methods in contract * LF * Rename to CoverageHit * Enable or disable recover coverage * Fix hit --- .../Coverage/AbiMethod.cs | 94 ++++++++ .../Coverage/CoverageBase.cs | 79 ++++++ .../Coverage/CoverageHit.cs | 125 ++++++++++ .../Coverage/CoveredCollection.cs | 38 +++ .../Coverage/CoveredContract.cs | 167 +++++++++++++ .../Coverage/CoveredMethod.cs | 64 +++++ src/Neo.SmartContract.Testing/CustomMock.cs | 15 +- .../Extensions/ArtifactExtensions.cs | 2 +- .../Extensions/SmartContractExtensions.cs | 44 ++++ .../Neo.SmartContract.Testing.csproj | 2 + src/Neo.SmartContract.Testing/README.md | 57 +++++ .../SmartContractInitialize.cs | 19 +- src/Neo.SmartContract.Testing/TestEngine.cs | 98 +++++++- .../TestingApplicationEngine.cs | 55 +++++ .../Coverage/CoverageDataTests.cs | 228 ++++++++++++++++++ .../Extensions/ArtifactExtensionsTests.cs | 2 +- .../NativeArtifactsTests.cs | 7 + 17 files changed, 1087 insertions(+), 9 deletions(-) create mode 100644 src/Neo.SmartContract.Testing/Coverage/AbiMethod.cs create mode 100644 src/Neo.SmartContract.Testing/Coverage/CoverageBase.cs create mode 100644 src/Neo.SmartContract.Testing/Coverage/CoverageHit.cs create mode 100644 src/Neo.SmartContract.Testing/Coverage/CoveredCollection.cs create mode 100644 src/Neo.SmartContract.Testing/Coverage/CoveredContract.cs create mode 100644 src/Neo.SmartContract.Testing/Coverage/CoveredMethod.cs create mode 100644 src/Neo.SmartContract.Testing/Extensions/SmartContractExtensions.cs create mode 100644 tests/Neo.SmartContract.Testing.UnitTests/Coverage/CoverageDataTests.cs diff --git a/src/Neo.SmartContract.Testing/Coverage/AbiMethod.cs b/src/Neo.SmartContract.Testing/Coverage/AbiMethod.cs new file mode 100644 index 000000000..c0e743c44 --- /dev/null +++ b/src/Neo.SmartContract.Testing/Coverage/AbiMethod.cs @@ -0,0 +1,94 @@ +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Linq.Expressions; +using System.Reflection; + +namespace Neo.SmartContract.Testing.Coverage +{ + [DebuggerDisplay("{Name},{PCount}")] + public class AbiMethod : IEquatable + { + /// + /// Method name + /// + public string Name { get; } + + /// + /// Parameters count + /// + public int PCount { get; } + + /// + /// Constructor + /// + /// Method name + /// Parameters count + public AbiMethod(string name, int pCount) + { + Name = name; + PCount = pCount; + } + + /// + /// Create from expression + /// + /// Expression + /// AbiMethod + public static AbiMethod[] CreateFromExpression(Expression expression) + { + if (expression is MemberExpression memberExpression) + { + if (memberExpression.Member is PropertyInfo pInfo) + { + if (pInfo.CanRead) + { + var display = pInfo.GetGetMethod()?.GetCustomAttribute(); + var nameRead = display is not null ? display.DisplayName : memberExpression.Member.Name; + + if (pInfo.CanWrite) + { + // If Property CanWrite, we return both methods + + display = pInfo.GetSetMethod()?.GetCustomAttribute(); + var nameWrite = display is not null ? display.DisplayName : memberExpression.Member.Name; + + return new AbiMethod[] + { + new AbiMethod(nameRead, 0), + new AbiMethod(nameWrite, 1) + }; + } + + // Only read property + + return new AbiMethod[] { new AbiMethod(nameRead, 0) }; + } + } + } + else if (expression is MethodCallExpression methodExpression) + { + if (methodExpression.Method is MethodInfo mInfo) + { + var display = mInfo.GetCustomAttribute(); + var name = display is not null ? display.DisplayName : mInfo.Name; + + return new AbiMethod[] { new AbiMethod(name, mInfo.GetParameters().Length) }; + } + } + + return Array.Empty(); + } + + public override bool Equals(object obj) + { + if (obj is not AbiMethod other) return false; + + return PCount == other.PCount && Name == other.Name; + } + + bool IEquatable.Equals(AbiMethod other) => PCount == other.PCount && Name == other.Name; + public override int GetHashCode() => HashCode.Combine(PCount, Name); + public override string ToString() => $"{Name},{PCount}"; + } +} diff --git a/src/Neo.SmartContract.Testing/Coverage/CoverageBase.cs b/src/Neo.SmartContract.Testing/Coverage/CoverageBase.cs new file mode 100644 index 000000000..ed7cbc132 --- /dev/null +++ b/src/Neo.SmartContract.Testing/Coverage/CoverageBase.cs @@ -0,0 +1,79 @@ +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace Neo.SmartContract.Testing.Coverage +{ + public abstract class CoverageBase : IEnumerable + { + /// + /// Coverage + /// + public abstract IEnumerable Coverage { get; } + + /// + /// Total instructions (could be different from Coverage.Count if, for example, a contract JUMPS to PUSHDATA content) + /// + public virtual int TotalInstructions => Coverage.Where(u => !u.OutOfScript).Count(); + + /// + /// Covered Instructions (OutOfScript are not taken into account) + /// + public virtual int CoveredInstructions => Coverage.Where(u => !u.OutOfScript && u.Hits > 0).Count(); + + /// + /// All instructions that have been touched + /// + public virtual int HitsInstructions => Coverage.Where(u => u.Hits > 0).Count(); + + /// + /// Covered Percentage + /// + public float CoveredPercentage + { + get + { + var total = TotalInstructions; + if (total == 0) return 0F; + + return (float)CoveredInstructions / total * 100F; + } + } + + /// + /// Get Coverage from the Contract coverage + /// + /// Offset + /// Length + /// Coverage + public IEnumerable GetCoverageFrom(int offset, int length) + { + var to = offset + length; + + foreach (var kvp in Coverage) + { + if (kvp.Offset >= offset && kvp.Offset <= to) + { + yield return kvp; + } + } + } + + #region IEnumerable + + public IEnumerator GetEnumerator() => Coverage.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => Coverage.GetEnumerator(); + + #endregion + + // Allow to sum coverages + + public static CoverageBase? operator +(CoverageBase? a, CoverageBase? b) + { + if (a is null) return b; + if (b is null) return a; + + return new CoveredCollection(a, b); + } + } +} diff --git a/src/Neo.SmartContract.Testing/Coverage/CoverageHit.cs b/src/Neo.SmartContract.Testing/Coverage/CoverageHit.cs new file mode 100644 index 000000000..d419da70c --- /dev/null +++ b/src/Neo.SmartContract.Testing/Coverage/CoverageHit.cs @@ -0,0 +1,125 @@ +using System; +using System.Diagnostics; + +namespace Neo.SmartContract.Testing.Coverage +{ + [DebuggerDisplay("Offset:{Offset}, OutOfScript:{OutOfScript}, Hits:{Hits}, GasTotal:{GasTotal}, GasMin:{GasMin}, GasMax:{GasMax}, GasAvg:{GasAvg}")] + public class CoverageHit + { + /// + /// The instruction offset + /// + public int Offset { get; } + + /// + /// The instruction is out of the script + /// + public bool OutOfScript { get; } + + /// + /// Hits + /// + public int Hits { get; private set; } + + /// + /// Minimum used gas + /// + public long GasMin { get; private set; } + + /// + /// Minimum used gas + /// + public long GasMax { get; private set; } + + /// + /// Total used gas + /// + public long GasTotal { get; private set; } + + /// + /// Average used gas + /// + public long GasAvg => Hits == 0 ? 0 : GasTotal / Hits; + + /// + /// Constructor + /// + /// Offset + /// Out of script + public CoverageHit(int offset, bool outOfScript = false) + { + Offset = offset; + OutOfScript = outOfScript; + } + + /// + /// Hits + /// + /// Gas + public void Hit(long gas) + { + Hits++; + + if (Hits == 1) + { + GasMin = gas; + GasMax = gas; + } + else + { + GasMin = Math.Min(GasMin, gas); + GasMax = Math.Max(GasMax, gas); + } + + GasTotal += gas; + } + + /// + /// Hits + /// + /// Value + public void Hit(CoverageHit value) + { + if (value.Hits == 0) return; + + Hits += value.Hits; + + if (Hits == 1) + { + GasMin = value.GasMin; + GasMax = value.GasMax; + } + else + { + GasMin = Math.Min(GasMin, value.GasMin); + GasMax = Math.Max(GasMax, value.GasMax); + } + + GasTotal += value.GasTotal; + } + + /// + /// Clone data + /// + /// CoverageData + public CoverageHit Clone() + { + return new CoverageHit(Offset, OutOfScript) + { + GasMax = GasMax, + GasMin = GasMin, + GasTotal = GasTotal, + Hits = Hits + }; + } + + /// + /// String representation + /// + /// + public override string ToString() + { + return $"Offset:{Offset}, OutOfScript:{OutOfScript}, Hits:{Hits}, GasTotal:{GasTotal}, GasMin:{GasMin}, GasMax:{GasMax}, GasAvg:{GasAvg}"; + } + } +} diff --git a/src/Neo.SmartContract.Testing/Coverage/CoveredCollection.cs b/src/Neo.SmartContract.Testing/Coverage/CoveredCollection.cs new file mode 100644 index 000000000..d5ebb27fc --- /dev/null +++ b/src/Neo.SmartContract.Testing/Coverage/CoveredCollection.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; + +namespace Neo.SmartContract.Testing.Coverage +{ + public class CoveredCollection : CoverageBase + { + /// + /// Entries + /// + public CoverageBase[] Entries { get; } + + /// + /// Coverage + /// + public override IEnumerable Coverage + { + get + { + foreach (var method in Entries) + { + foreach (var entry in method.Coverage) + { + yield return entry; + } + } + } + } + + /// + /// Constructor + /// + /// Entries + public CoveredCollection(params CoverageBase[] entries) + { + Entries = entries; + } + } +} diff --git a/src/Neo.SmartContract.Testing/Coverage/CoveredContract.cs b/src/Neo.SmartContract.Testing/Coverage/CoveredContract.cs new file mode 100644 index 000000000..06e9e8a4b --- /dev/null +++ b/src/Neo.SmartContract.Testing/Coverage/CoveredContract.cs @@ -0,0 +1,167 @@ +using Neo.SmartContract.Manifest; +using Neo.VM; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; + +namespace Neo.SmartContract.Testing.Coverage +{ + [DebuggerDisplay("{Hash.ToString()}")] + public class CoveredContract : CoverageBase + { + #region Internal + + /// + /// Coverage Data + /// + internal Dictionary CoverageData { get; } = new(); + + #endregion + + /// + /// Contract Hash + /// + public UInt160 Hash { get; } + + /// + /// Methods + /// + public CoveredMethod[] Methods { get; } + + /// + /// Coverage + /// + public override IEnumerable Coverage => CoverageData.Values; + + /// + /// CoveredContract + /// + /// Hash + /// Contract abi + /// Script + public CoveredContract(UInt160 hash, ContractAbi? abi, Script? script) + { + Hash = hash; + Methods = Array.Empty(); + + if (script is null) return; + + // Extract all methods + + if (abi is not null) + { + Methods = abi.Methods + .Select(u => CreateMethod(abi, script, u)) + .Where(u => u is not null) + .OrderBy(u => u!.Offset) + .ToArray()!; + } + + // Iterate all valid instructions + + int ip = 0; + + while (ip < script.Length) + { + var instruction = script.GetInstruction(ip); + CoverageData[ip] = new CoverageHit(ip, false); + ip += instruction.Size; + } + } + + private CoveredMethod? CreateMethod(ContractAbi abi, Script script, ContractMethodDescriptor abiMethod) + { + var to = script.Length - 1; + var next = abi.Methods.OrderBy(u => u.Offset).Where(u => u.Offset > abiMethod.Offset).FirstOrDefault(); + + if (next is not null) to = next.Offset - 1; + + // Return method coverage + + return new CoveredMethod(this, abiMethod, to - abiMethod.Offset); + } + + /// + /// Get method coverage + /// + /// Method name + /// Parameter count + /// CoveredMethod + public CoveredMethod? GetCoverage(string methodName, int pcount) + { + return GetCoverage(new AbiMethod(methodName, pcount)); + } + + /// + /// Get method coverage + /// + /// Method + /// CoveredMethod + public CoveredMethod? GetCoverage(AbiMethod? method = null) + { + if (method is null) return null; + + return Methods.FirstOrDefault(m => m.Method.Equals(method)); + } + + /// + /// Join coverage + /// + /// Coverage + public void Join(IEnumerable coverage) + { + // Join the coverage between them + + foreach (var c in coverage) + { + if (c.Hits == 0) continue; + + if (CoverageData.TryGetValue(c.Offset, out var kvpValue)) + { + kvpValue.Hit(c); + } + else + { + CoverageData.Add(c.Offset, c.Clone()); + } + } + } + + /// + /// Dump coverage + /// + /// Coverage dump + public string Dump() + { + // TODO: improve dump later + + var builder = new StringBuilder(); + using var sourceCode = new StringWriter(builder) + { + NewLine = "\n" + }; + + var cover = CoveredPercentage.ToString("0.00").ToString(); + sourceCode.WriteLine($"| {Hash,-50} | {cover,7}% |"); + + foreach (var method in Methods) + { + sourceCode.WriteLine(method.Dump()); + } + + return builder.ToString(); + } + + /// + /// String representation + /// + /// + public override string ToString() + { + return $"Hash:{Hash}"; + } + } +} diff --git a/src/Neo.SmartContract.Testing/Coverage/CoveredMethod.cs b/src/Neo.SmartContract.Testing/Coverage/CoveredMethod.cs new file mode 100644 index 000000000..2dd1708b0 --- /dev/null +++ b/src/Neo.SmartContract.Testing/Coverage/CoveredMethod.cs @@ -0,0 +1,64 @@ +using Neo.SmartContract.Manifest; +using System.Collections.Generic; +using System.Diagnostics; + +namespace Neo.SmartContract.Testing.Coverage +{ + [DebuggerDisplay("{Method}")] + public class CoveredMethod : CoverageBase + { + /// + /// Contract + /// + public CoveredContract Contract { get; } + + /// + /// Method + /// + public AbiMethod Method { get; } + + /// + /// Offset + /// + public int Offset { get; } + + /// + /// Method length + /// + public int MethodLength { get; } + + /// + /// Coverage + /// + public override IEnumerable Coverage => Contract.GetCoverageFrom(Offset, MethodLength); + + /// + /// Constructor + /// + /// Contract + /// Method + /// Method length + public CoveredMethod(CoveredContract contract, ContractMethodDescriptor method, int methodLength) + { + Contract = contract; + Method = new AbiMethod(method.Name, method.Parameters.Length); + Offset = method.Offset; + MethodLength = methodLength; + } + + /// + /// Dump coverage + /// + /// Coverage dump + public string Dump() + { + // TODO: improve dump later + + var cover = CoveredPercentage.ToString("0.00").ToString(); + + return $"| {Method,50} | {cover,7}% |"; + } + + public override string ToString() => Method.ToString(); + } +} diff --git a/src/Neo.SmartContract.Testing/CustomMock.cs b/src/Neo.SmartContract.Testing/CustomMock.cs index 903f702c1..1327ac7f8 100644 --- a/src/Neo.SmartContract.Testing/CustomMock.cs +++ b/src/Neo.SmartContract.Testing/CustomMock.cs @@ -7,11 +7,22 @@ internal class CustomMock /// /// Mocked contract /// - public required SmartContract Contract { get; init; } + public SmartContract Contract { get; } /// /// Mocked method /// - public required MethodInfo Method { get; init; } + public MethodInfo Method { get; } + + /// + /// Constructor + /// + /// Contract + /// Method + public CustomMock(SmartContract contract, MethodInfo method) + { + Contract = contract; + Method = method; + } } } diff --git a/src/Neo.SmartContract.Testing/Extensions/ArtifactExtensions.cs b/src/Neo.SmartContract.Testing/Extensions/ArtifactExtensions.cs index e91810bac..0b260af0b 100644 --- a/src/Neo.SmartContract.Testing/Extensions/ArtifactExtensions.cs +++ b/src/Neo.SmartContract.Testing/Extensions/ArtifactExtensions.cs @@ -124,7 +124,7 @@ public static string GetArtifactsSource(this ContractAbi abi, string name, bool sourceCode.WriteLine("}"); - return sourceCode.ToString().TrimEnd(); + return sourceCode.ToString(); } private static (ContractMethodDescriptor[] methods, (ContractMethodDescriptor getter, ContractMethodDescriptor? setter)[] properties) diff --git a/src/Neo.SmartContract.Testing/Extensions/SmartContractExtensions.cs b/src/Neo.SmartContract.Testing/Extensions/SmartContractExtensions.cs new file mode 100644 index 000000000..84d95da3c --- /dev/null +++ b/src/Neo.SmartContract.Testing/Extensions/SmartContractExtensions.cs @@ -0,0 +1,44 @@ +using Neo.SmartContract.Testing.Coverage; +using System; +using System.Linq.Expressions; + +namespace Neo.SmartContract.Testing +{ + public static class SmartContractExtensions + { + /// + /// Get Coverage by contract + /// + /// Contract + /// CoveredContract + public static CoveredContract? GetCoverage(this SmartContract contract) + { + return contract.Engine.GetCoverage(contract); + } + + /// + /// Get Coverage by method + /// + /// Contract + /// Contract + /// Method + /// CoveredContract + public static CoverageBase? GetCoverage(this T contract, Expression> method) where T : SmartContract + { + return contract.Engine.GetCoverage(contract, method); + } + + /// + /// Get Coverage by method + /// + /// Contract + /// Result + /// Contract + /// Method + /// CoveredContract + public static CoverageBase? GetCoverage(this T contract, Expression> method) where T : SmartContract + { + return contract.Engine.GetCoverage(contract, method); + } + } +} diff --git a/src/Neo.SmartContract.Testing/Neo.SmartContract.Testing.csproj b/src/Neo.SmartContract.Testing/Neo.SmartContract.Testing.csproj index 33e7bcd3c..15676ae7b 100644 --- a/src/Neo.SmartContract.Testing/Neo.SmartContract.Testing.csproj +++ b/src/Neo.SmartContract.Testing/Neo.SmartContract.Testing.csproj @@ -2,6 +2,8 @@ Neo.SmartContract.Testing + netstandard2.1;net7.0 + latest Neo.SmartContract.Testing NEO;Blockchain;Smart Contract TestEngine for NEO smart contract testing. diff --git a/src/Neo.SmartContract.Testing/README.md b/src/Neo.SmartContract.Testing/README.md index 7cd03e004..aab24d859 100644 --- a/src/Neo.SmartContract.Testing/README.md +++ b/src/Neo.SmartContract.Testing/README.md @@ -23,6 +23,8 @@ The **Neo.SmartContract.Testing** project is designed to facilitate the developm - [Example of use](#example-of-use) - [Event testing](#event-testing) - [Example of use](#example-of-use) +- [Coverage Calculation](#coverage-calculation) + - [Example of use](#example-of-use) - [Known limitations](#known-limitations) ### Installation and configuration @@ -70,6 +72,7 @@ The publicly exposed read-only properties are as follows: - **CommitteeAddress**: Returns the address of the current chain's committee. - **Transaction**: Defines the transaction that will be used as `ScriptContainer` for the neo virtual machine, by default it updates the script of the same as calls are composed and executed, and the `Signers` will be used as validators for the `CheckWitness`, regardless of whether the signature is correct or not, so if you want to test with different wallets or scopes, you do not need to sign the transaction correctly, just set the desired signers. - **CurrentBlock**: Defaults to `Genesis` for the defined `ProtocolSettings`, but the height has been incremented by 1 to avoid issues related to the generation of gas from native contracts. +- **EnableCoverageCapture**: Enables or disables the coverage capture. For initialize, we have: @@ -283,8 +286,62 @@ Assert.IsTrue(raisedEvent); Assert.AreEqual(123, engine.Native.NEO.BalanceOf(addressTo)); ``` +### Coverage Calculation + +To calculate the coverage of a contract, it is enough to call the `GetCoverage` method of our `TestEngine`, it require the `EnableCoverageCapture` property of the `TestEngine` to be enabled. + +```csharp +var engine = new TestEngine(true); + +// Get NEO Coverage (NULL) + +Assert.IsNull(engine.GetCoverage(engine.Native.NEO)); + +// Call NEO.TotalSupply + +Assert.AreEqual(100_000_000, engine.Native.NEO.TotalSupply); + +// Check that the 3 instructions has been covered + +Assert.AreEqual(3, engine.GetCoverage(engine.Native.NEO)?.CoveredInstructions); +Assert.AreEqual(3, engine.GetCoverage(engine.Native.NEO)?.HitsInstructions); +``` + +It is also possible to call it to obtain the specific coverage of a method, either through an expression or manually. + +```csharp +var engine = new TestEngine(true); + +// Call NEO.TotalSupply + +Assert.AreEqual(100_000_000, engine.Native.NEO.TotalSupply); + +// Oracle was not called + +var methodCovered = engine.GetCoverage(engine.Native.Oracle, o => o.Finish()); +Assert.IsNull(methodCovered); + +// NEO.TotalSupply is covered + +methodCovered = engine.GetCoverage(engine.Native.NEO, o => o.TotalSupply); +Assert.AreEqual(3, methodCovered?.TotalInstructions); +Assert.AreEqual(3, methodCovered?.CoveredInstructions); + +// Check coverage by raw method + +methodCovered = engine.GetCoverage(engine.Native.Oracle, "finish", 0); +Assert.IsNull(methodCovered); + +methodCovered = engine.GetCoverage(engine.Native.NEO, "totalSupply", 0); +Assert.AreEqual(3, methodCovered?.TotalInstructions); +Assert.AreEqual(3, methodCovered?.CoveredInstructions); +``` + +Keep in mind that the coverage is at the instruction level. + ### Known limitations The currently known limitations are: - Receive events during the deploy, because the object is returned after performing the deploy, it is not possible to intercept notifications for the deploy unless the contract is previously created with `FromHash` knowing the hash of the contract to be created. +- It is possible that if the contract is updated, the coverage calculation may be incorrect. diff --git a/src/Neo.SmartContract.Testing/SmartContractInitialize.cs b/src/Neo.SmartContract.Testing/SmartContractInitialize.cs index 9a7d72609..dbcebad5d 100644 --- a/src/Neo.SmartContract.Testing/SmartContractInitialize.cs +++ b/src/Neo.SmartContract.Testing/SmartContractInitialize.cs @@ -5,16 +5,29 @@ public class SmartContractInitialize /// /// Engine /// - public required TestEngine Engine { get; init; } + public TestEngine Engine { get; } /// /// Hash /// - public required UInt160 Hash { get; init; } + public UInt160 Hash { get; } /// /// ContractId /// - internal int? ContractId { get; init; } + internal int? ContractId { get; } + + /// + /// Constructor + /// + /// Engine + /// Hash + /// Contract Id + internal SmartContractInitialize(TestEngine engine, UInt160 hash, int? contractId = null) + { + Engine = engine; + Hash = hash; + ContractId = contractId; + } } } diff --git a/src/Neo.SmartContract.Testing/TestEngine.cs b/src/Neo.SmartContract.Testing/TestEngine.cs index 37e807a2f..57fada30f 100644 --- a/src/Neo.SmartContract.Testing/TestEngine.cs +++ b/src/Neo.SmartContract.Testing/TestEngine.cs @@ -4,6 +4,7 @@ using Neo.Network.P2P.Payloads; using Neo.Persistence; using Neo.SmartContract.Manifest; +using Neo.SmartContract.Testing.Coverage; using Neo.SmartContract.Testing.Extensions; using Neo.VM; using Neo.VM.Types; @@ -11,6 +12,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Linq.Expressions; using System.Reflection; using System.Text; @@ -18,6 +20,7 @@ namespace Neo.SmartContract.Testing { public class TestEngine { + internal readonly Dictionary Coverage = new(); private readonly Dictionary> _contracts = new(); private readonly Dictionary> _customMocks = new(); private NativeArtifacts? _native; @@ -75,6 +78,11 @@ public class TestEngine /// public ProtocolSettings ProtocolSettings { get; } + /// + /// Enable coverage capture + /// + public bool EnableCoverageCapture { get; set; } = true; + /// /// Validators Address /// @@ -306,7 +314,7 @@ internal T FromHash(UInt160 hash, int? contractId = null) where T : SmartCont private T MockContract(UInt160 hash, int? contractId = null, Action>? customMock = null) where T : SmartContract { - var mock = new Mock(new SmartContractInitialize() { Engine = this, Hash = hash, ContractId = contractId }) + var mock = new Mock(new SmartContractInitialize(this, hash, contractId)) { CallBase = true }; @@ -326,7 +334,7 @@ private T MockContract(UInt160 hash, int? contractId = null, Action>? if (mock.IsMocked(method)) { var mockName = method.Name + ";" + method.GetParameters().Length; - var cm = new CustomMock() { Contract = mock.Object, Method = method }; + var cm = new CustomMock(mock.Object, method); if (_customMocks.TryGetValue(hash, out var mocks)) { @@ -439,6 +447,92 @@ public StackItem Execute(Script script) return engine.ResultStack.Pop(); } + /// + /// Get contract coverage + /// + /// Contract + /// Contract + /// CoveredContract + public CoveredContract? GetCoverage(T contract) where T : SmartContract + { + if (!Coverage.TryGetValue(contract.Hash, out var coveredContract)) + { + return null; + } + + return coveredContract; + } + + /// + /// Get method coverage by contract + /// + /// Contract + /// Contract + /// CoveredContract + public CoverageBase? GetCoverage(T contract, string methodName, int pcount) where T : SmartContract + { + var coveredContract = GetCoverage(contract); + + return coveredContract?.GetCoverage(methodName, pcount); + } + + /// + /// Get method coverage + /// + /// Contract + /// Contract + /// Method + /// CoveredContract + public CoverageBase? GetCoverage(T contract, Expression> method) where T : SmartContract + { + if (!Coverage.TryGetValue(contract.Hash, out var coveredContract)) + { + return null; + } + + var abiMethods = AbiMethod.CreateFromExpression(method.Body) + .Select(coveredContract.GetCoverage) + .Where(u => u != null) + .Cast() + .ToArray(); + + return abiMethods.Length switch + { + 0 => null, + 1 => abiMethods[0], + _ => new CoveredCollection(abiMethods), + }; + } + + /// + /// Get method coverage + /// + /// Contract + /// Result + /// Contract + /// Method + /// CoveredContract + public CoverageBase? GetCoverage(T contract, Expression> method) where T : SmartContract + { + if (!Coverage.TryGetValue(contract.Hash, out var coveredContract)) + { + return null; + } + + var abiMethods = AbiMethod.CreateFromExpression(method.Body) + .Select(coveredContract.GetCoverage) + .Where(u => u != null) + .Cast() + .ToArray(); + + return abiMethods.Length switch + { + 0 => null, + 1 => abiMethods[0], + _ => new CoveredCollection(abiMethods), + }; + } + /// /// Set Transaction signers /// diff --git a/src/Neo.SmartContract.Testing/TestingApplicationEngine.cs b/src/Neo.SmartContract.Testing/TestingApplicationEngine.cs index 3110c7a68..72b1df211 100644 --- a/src/Neo.SmartContract.Testing/TestingApplicationEngine.cs +++ b/src/Neo.SmartContract.Testing/TestingApplicationEngine.cs @@ -1,6 +1,8 @@ using Neo.Network.P2P.Payloads; using Neo.Persistence; +using Neo.SmartContract.Testing.Coverage; using Neo.SmartContract.Testing.Extensions; +using Neo.VM; using System; namespace Neo.SmartContract.Testing @@ -10,6 +12,10 @@ namespace Neo.SmartContract.Testing /// internal class TestingApplicationEngine : ApplicationEngine { + private ExecutionContext? InstructionContext; + private int? InstructionPointer; + private long PreExecuteInstructionGasConsumed; + /// /// Testing engine /// @@ -21,6 +27,55 @@ public TestingApplicationEngine(TestEngine engine, TriggerType trigger, IVerifia Engine = engine; } + protected override void PreExecuteInstruction(Instruction instruction) + { + // Cache coverage data + + if (Engine.EnableCoverageCapture) + { + PreExecuteInstructionGasConsumed = GasConsumed; + InstructionContext = CurrentContext; + InstructionPointer = InstructionContext?.InstructionPointer; + } + + // Regular action + + base.PreExecuteInstruction(instruction); + } + + protected override void PostExecuteInstruction(Instruction instruction) + { + base.PostExecuteInstruction(instruction); + + // We need the script to know the offset + + if (InstructionContext is null) return; + + // Compute coverage + + var contractHash = InstructionContext.GetScriptHash(); + + if (!Engine.Coverage.TryGetValue(contractHash, out var coveredContract)) + { + // We need the contract state without pay gas + + var state = Native.NativeContract.ContractManagement.GetContract(Engine.Storage.Snapshot, contractHash); + + Engine.Coverage[contractHash] = coveredContract = new(contractHash, state?.Manifest.Abi, InstructionContext.Script); + } + + if (InstructionPointer is null) return; + + if (!coveredContract.CoverageData.TryGetValue(InstructionPointer.Value, out var coverage)) + { + // Note: This call is unusual, out of the expected + + coveredContract.CoverageData[InstructionPointer.Value] = coverage = new CoverageHit(InstructionPointer.Value, true); + } + + coverage.Hit(GasConsumed - PreExecuteInstructionGasConsumed); + } + protected override void OnSysCall(InteropDescriptor descriptor) { // Check if the syscall is a contract call and we need to mock it because it was defined by the user diff --git a/tests/Neo.SmartContract.Testing.UnitTests/Coverage/CoverageDataTests.cs b/tests/Neo.SmartContract.Testing.UnitTests/Coverage/CoverageDataTests.cs new file mode 100644 index 000000000..1c1ca6ca2 --- /dev/null +++ b/tests/Neo.SmartContract.Testing.UnitTests/Coverage/CoverageDataTests.cs @@ -0,0 +1,228 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Neo.SmartContract.Testing.Coverage; +using System.Numerics; + +namespace Neo.SmartContract.Testing.UnitTests.Coverage +{ + [TestClass] + public class CoverageDataTests + { + [TestMethod] + public void TestDump() + { + var engine = new TestEngine(true); + + // Check totalSupply + + Assert.AreEqual(100_000_000, engine.Native.NEO.TotalSupply); + + Assert.AreEqual(@" +| 0xef4073a0f2b305a38ec4050e4d3d28bc40ea63f5 | 5.26% | +| balanceOf,1 | 0.00% | +| decimals,0 | 0.00% | +| getAccountState,1 | 0.00% | +| getAllCandidates,0 | 0.00% | +| getCandidateVote,1 | 0.00% | +| getCandidates,0 | 0.00% | +| getCommittee,0 | 0.00% | +| getGasPerBlock,0 | 0.00% | +| getNextBlockValidators,0 | 0.00% | +| getRegisterPrice,0 | 0.00% | +| registerCandidate,1 | 0.00% | +| setGasPerBlock,1 | 0.00% | +| setRegisterPrice,1 | 0.00% | +| symbol,0 | 0.00% | +| totalSupply,0 | 100.00% | +| transfer,4 | 0.00% | +| unclaimedGas,2 | 0.00% | +| unregisterCandidate,1 | 0.00% | +| vote,2 | 0.00% | + + +".Trim(), engine.GetCoverage(engine.Native.NEO)?.Dump().Trim()); + } + + [TestMethod] + public void TestCoverageByEngine() + { + // Create the engine initializing the native contracts + // Native contracts use 3 opcodes per method + + //{ + // sb.EmitPush(0); //version + // sb.EmitSysCall(ApplicationEngine.System_Contract_CallNative); + // sb.Emit(OpCode.RET); + //} + + var engine = new TestEngine(true); + + // Check totalSupply + + Assert.IsNull(engine.GetCoverage(engine.Native.NEO)); + Assert.AreEqual(100_000_000, engine.Native.NEO.TotalSupply); + + Assert.AreEqual(engine.Native.NEO.Hash, engine.GetCoverage(engine.Native.NEO)?.Hash); + Assert.AreEqual(57, engine.GetCoverage(engine.Native.NEO)?.TotalInstructions); + Assert.AreEqual(3, engine.GetCoverage(engine.Native.NEO)?.CoveredInstructions); + Assert.AreEqual(3, engine.GetCoverage(engine.Native.NEO)?.HitsInstructions); + + // Check balanceOf + + Assert.AreEqual(0, engine.Native.NEO.BalanceOf(engine.Native.NEO.Hash)); + + Assert.AreEqual(57, engine.GetCoverage(engine.Native.NEO)?.TotalInstructions); + Assert.AreEqual(6, engine.GetCoverage(engine.Native.NEO)?.CoveredInstructions); + Assert.AreEqual(6, engine.GetCoverage(engine.Native.NEO)?.HitsInstructions); + + // Check coverage by method and expression + + var methodCovered = engine.GetCoverage(engine.Native.Oracle, o => o.Finish()); + Assert.IsNull(methodCovered); + + methodCovered = engine.GetCoverage(engine.Native.NEO, o => o.TotalSupply); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(3, methodCovered?.CoveredInstructions); + + methodCovered = engine.GetCoverage(engine.Native.NEO, o => o.RegisterPrice); + Assert.AreEqual(6, methodCovered?.TotalInstructions); + Assert.AreEqual(0, methodCovered?.CoveredInstructions); + + methodCovered = engine.GetCoverage(engine.Native.NEO, o => o.BalanceOf(It.IsAny())); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(3, methodCovered?.CoveredInstructions); + + methodCovered = engine.GetCoverage(engine.Native.NEO, o => o.Transfer(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(0, methodCovered?.CoveredInstructions); + + // Check coverage by raw method + + methodCovered = engine.GetCoverage(engine.Native.Oracle, "finish", 0); + Assert.IsNull(methodCovered); + + methodCovered = engine.GetCoverage(engine.Native.NEO, "totalSupply", 0); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(3, methodCovered?.CoveredInstructions); + + methodCovered = engine.GetCoverage(engine.Native.NEO, "balanceOf", 1); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(3, methodCovered?.CoveredInstructions); + + methodCovered = engine.GetCoverage(engine.Native.NEO, "transfer", 4); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(0, methodCovered?.CoveredInstructions); + } + + [TestMethod] + public void TestCoverageByExtension() + { + // Create the engine initializing the native contracts + // Native contracts use 3 opcodes per method + + //{ + // sb.EmitPush(0); //version + // sb.EmitSysCall(ApplicationEngine.System_Contract_CallNative); + // sb.Emit(OpCode.RET); + //} + + var engine = new TestEngine(true); + + // Check totalSupply + + Assert.IsNull(engine.Native.NEO.GetCoverage()); + Assert.AreEqual(100_000_000, engine.Native.NEO.TotalSupply); + + Assert.AreEqual(engine.Native.NEO.Hash, engine.Native.NEO.GetCoverage()?.Hash); + Assert.AreEqual(57, engine.Native.NEO.GetCoverage()?.TotalInstructions); + Assert.AreEqual(3, engine.Native.NEO.GetCoverage()?.CoveredInstructions); + Assert.AreEqual(3, engine.Native.NEO.GetCoverage()?.HitsInstructions); + + // Check balanceOf + + Assert.AreEqual(0, engine.Native.NEO.BalanceOf(engine.Native.NEO.Hash)); + + Assert.AreEqual(57, engine.Native.NEO.GetCoverage()?.TotalInstructions); + Assert.AreEqual(6, engine.Native.NEO.GetCoverage()?.CoveredInstructions); + Assert.AreEqual(6, engine.Native.NEO.GetCoverage()?.HitsInstructions); + + // Check coverage by method and expression + + var methodCovered = engine.Native.Oracle.GetCoverage(o => o.Finish()); + Assert.IsNull(methodCovered); + + methodCovered = engine.Native.NEO.GetCoverage(o => o.TotalSupply); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(3, methodCovered?.CoveredInstructions); + + methodCovered = engine.Native.NEO.GetCoverage(o => o.BalanceOf(It.IsAny())); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(3, methodCovered?.CoveredInstructions); + + methodCovered = engine.Native.NEO.GetCoverage(o => o.Transfer(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(0, methodCovered?.CoveredInstructions); + + // Check coverage by raw method + + methodCovered = engine.GetCoverage(engine.Native.Oracle, "finish", 0); + Assert.IsNull(methodCovered); + + methodCovered = engine.GetCoverage(engine.Native.NEO, "totalSupply", 0); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(3, methodCovered?.CoveredInstructions); + + methodCovered = engine.GetCoverage(engine.Native.NEO, "balanceOf", 1); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(3, methodCovered?.CoveredInstructions); + + methodCovered = engine.GetCoverage(engine.Native.NEO, "transfer", 4); + Assert.AreEqual(3, methodCovered?.TotalInstructions); + Assert.AreEqual(0, methodCovered?.CoveredInstructions); + } + + [TestMethod] + public void TestHits() + { + var coverage = new CoverageHit(0); + + Assert.AreEqual(0, coverage.Hits); + Assert.AreEqual(0, coverage.GasAvg); + Assert.AreEqual(0, coverage.GasMax); + Assert.AreEqual(0, coverage.GasMin); + Assert.AreEqual(0, coverage.GasTotal); + + coverage.Hit(123); + + Assert.AreEqual(1, coverage.Hits); + Assert.AreEqual(123, coverage.GasAvg); + Assert.AreEqual(123, coverage.GasMax); + Assert.AreEqual(123, coverage.GasMin); + Assert.AreEqual(123, coverage.GasTotal); + + coverage.Hit(377); + + Assert.AreEqual(2, coverage.Hits); + Assert.AreEqual(250, coverage.GasAvg); + Assert.AreEqual(377, coverage.GasMax); + Assert.AreEqual(123, coverage.GasMin); + Assert.AreEqual(500, coverage.GasTotal); + + coverage.Hit(500); + + Assert.AreEqual(3, coverage.Hits); + Assert.AreEqual(333, coverage.GasAvg); + Assert.AreEqual(500, coverage.GasMax); + Assert.AreEqual(123, coverage.GasMin); + Assert.AreEqual(1000, coverage.GasTotal); + + coverage.Hit(0); + + Assert.AreEqual(4, coverage.Hits); + Assert.AreEqual(250, coverage.GasAvg); + Assert.AreEqual(500, coverage.GasMax); + Assert.AreEqual(0, coverage.GasMin); + Assert.AreEqual(1000, coverage.GasTotal); + } + } +} diff --git a/tests/Neo.SmartContract.Testing.UnitTests/Extensions/ArtifactExtensionsTests.cs b/tests/Neo.SmartContract.Testing.UnitTests/Extensions/ArtifactExtensionsTests.cs index 2062dfa40..66fa2633c 100644 --- a/tests/Neo.SmartContract.Testing.UnitTests/Extensions/ArtifactExtensionsTests.cs +++ b/tests/Neo.SmartContract.Testing.UnitTests/Extensions/ArtifactExtensionsTests.cs @@ -91,7 +91,7 @@ public abstract class Contract1 : Neo.SmartContract.Testing.SmartContract protected Contract1(Neo.SmartContract.Testing.SmartContractInitialize initialize) : base(initialize) { } #endregion } -".Replace("\r\n", "\n").Trim()); +".Replace("\r\n", "\n").TrimStart()); } } } diff --git a/tests/Neo.SmartContract.Testing.UnitTests/NativeArtifactsTests.cs b/tests/Neo.SmartContract.Testing.UnitTests/NativeArtifactsTests.cs index f670241d8..949391761 100644 --- a/tests/Neo.SmartContract.Testing.UnitTests/NativeArtifactsTests.cs +++ b/tests/Neo.SmartContract.Testing.UnitTests/NativeArtifactsTests.cs @@ -1,4 +1,5 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; using System.Linq; using System.Numerics; using System.Reflection; @@ -30,6 +31,12 @@ public void TestInitialize() Assert.AreEqual(100_000_000, engine.Native.NEO.TotalSupply); Assert.AreEqual(engine.Native.NEO.TotalSupply, engine.Native.NEO.BalanceOf(engine.ValidatorsAddress)); + + // Check coverage + + Assert.AreEqual(100.0F, engine.Native.NEO.GetCoverage(o => o.Symbol).CoveredPercentage); + Assert.AreEqual(100.0F, engine.Native.NEO.GetCoverage(o => o.TotalSupply).CoveredPercentage); + Assert.AreEqual(100.0F, engine.Native.NEO.GetCoverage(o => o.BalanceOf(It.IsAny())).CoveredPercentage); } [TestMethod]