Skip to content

Commit 43b22a8

Browse files
lambdageekAaronRobinsonMSFTam11elinor-fung
authored
[cdac] Implement a JSON contract reader (#100966)
Implement a parser for the "compact" JSON contract descriptor format specified in [data_contracts.md](https://github.com/dotnet/runtime/blob/main/docs/design/datacontracts/data_descriptor.md) The data model is a WIP - it's likely we will want something a bit more abstract (and less mutable). This is not wired up to consume a real contract descriptor from target process memory at the moment. It's only exercised by unit tests for now. --- * compact descriptor format json parser * suggestions from reviews; remove FieldDescriptor wrong conversion we incorrectly allowed `[number]` as a field descriptor conversion. that's not allowed. removed it. * Make test project like the nativeoat+linker tests Dont' use libraries test infrastructure. Just normal arcade xunit support. * add tools.cdacreadertests subset; add to CLR_Tools_Tests test leg * no duplicate fields/sizes in types * Make all cdacreader.csproj ProjectReferences use the same AdditionalProperties Since we set Configuration and RuntimeIdentifier, if we don't pass the same AdditionalProperties in all ProjectReferences, we bad project.assets.json files * Don't share the native compilation AdditionalProperties --------- Co-authored-by: Aaron Robinson <[email protected]> Co-authored-by: Adeel Mujahid <[email protected]> Co-authored-by: Elinor Fung <[email protected]>
1 parent a344abd commit 43b22a8

8 files changed

+584
-12
lines changed

‎eng/Subsets.props

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@
173173
<SubsetName Include="Tools.ILLink" Description="The projects that produce illink and analyzer tools for trimming." />
174174
<SubsetName Include="Tools.ILLinkTests" OnDemand="true" Description="Unit tests for the tools.illink subset." />
175175

176+
<SubsetName Include="Tools.CdacReaderTests" OnDemand="true" Description="Units tests for the cDAC reader." />
177+
176178
<!-- Host -->
177179
<SubsetName Include="Host" Description="The .NET hosts, packages, hosting libraries, and tests. Equivalent to: $(DefaultHostSubsets)" />
178180
<SubsetName Include="Host.Native" Description="The .NET hosts." />
@@ -369,6 +371,10 @@
369371
Test="true" Category="clr" Condition="'$(DotNetBuildSourceOnly)' != 'true' and '$(NativeAotSupported)' == 'true'"/>
370372
</ItemGroup>
371373

374+
<ItemGroup Condition="$(_subset.Contains('+tools.cdacreadertests+'))">
375+
<ProjectToBuild Include="$(SharedNativeRoot)managed\cdacreader\tests\Microsoft.Diagnostics.DataContractReader.Tests.csproj" Test="true" Category="tools"/>
376+
</ItemGroup>
377+
372378
<ItemGroup Condition="$(_subset.Contains('+tools.illink+'))">
373379
<ProjectToBuild Include="$(ToolsProjectRoot)illink\src\linker\Mono.Linker.csproj" Category="tools" />
374380
<ProjectToBuild Include="$(ToolsProjectRoot)illink\src\ILLink.Tasks\ILLink.Tasks.csproj" Category="tools" />

‎eng/pipelines/common/evaluate-default-paths.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,10 @@ jobs:
164164
- src/tools/illink/*
165165
- global.json
166166

167+
- subset: tools_cdacreader
168+
include:
169+
- src/native/managed/cdacreader/*
170+
167171
- subset: installer
168172
include:
169173
exclude:

‎eng/pipelines/runtime.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -713,14 +713,15 @@ extends:
713713
jobParameters:
714714
timeoutInMinutes: 120
715715
nameSuffix: CLR_Tools_Tests
716-
buildArgs: -s clr.aot+clr.iltools+libs.sfx+clr.toolstests -c $(_BuildConfig) -test
716+
buildArgs: -s clr.aot+clr.iltools+libs.sfx+clr.toolstests+tools.cdacreadertests -c $(_BuildConfig) -test
717717
enablePublishTestResults: true
718718
testResultsFormat: 'xunit'
719719
# We want to run AOT tests when illink changes because there's share code and tests from illink which are used by AOT
720720
condition: >-
721721
or(
722722
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true),
723723
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_tools_illink.containsChange'], true),
724+
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_tools_cdacreader.containsChange'], true),
724725
eq(variables['isRollingBuild'], true))
725726
#
726727
# Build CrossDacs

‎src/native/managed/cdacreader/src/ContractDescriptorParser.cs

Lines changed: 327 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Diagnostics.Contracts;
7+
using System.Text.Json;
8+
using System.Text.Json.Serialization;
9+
10+
namespace Microsoft.Diagnostics.DataContractReader;
11+
12+
/// <summary>
13+
/// A parser for the JSON representation of a contract descriptor.
14+
/// </summary>
15+
/// <remarks>
16+
/// <see href="https://github.com/dotnet/runtime/blob/main/docs/design/datacontracts/data_descriptor.md">See design doc</see> for the format.
17+
/// </remarks>
18+
public partial class ContractDescriptorParser
19+
{
20+
// data_descriptor.md uses a distinguished property name to indicate the size of a type
21+
public const string TypeDescriptorSizeSigil = "!";
22+
23+
/// <summary>
24+
/// Parses the "compact" representation of a contract descriptor.
25+
/// </summary>
26+
public static ContractDescriptor? ParseCompact(ReadOnlySpan<byte> json)
27+
{
28+
return JsonSerializer.Deserialize(json, ContractDescriptorContext.Default.ContractDescriptor);
29+
}
30+
31+
[JsonSerializable(typeof(ContractDescriptor))]
32+
[JsonSerializable(typeof(int))]
33+
[JsonSerializable(typeof(string))]
34+
[JsonSerializable(typeof(Dictionary<string, int>))]
35+
[JsonSerializable(typeof(Dictionary<string, TypeDescriptor>))]
36+
[JsonSerializable(typeof(Dictionary<string, FieldDescriptor>))]
37+
[JsonSerializable(typeof(Dictionary<string, GlobalDescriptor>))]
38+
[JsonSerializable(typeof(TypeDescriptor))]
39+
[JsonSerializable(typeof(FieldDescriptor))]
40+
[JsonSerializable(typeof(GlobalDescriptor))]
41+
[JsonSourceGenerationOptions(AllowTrailingCommas = true,
42+
DictionaryKeyPolicy = JsonKnownNamingPolicy.Unspecified, // contracts, types and globals are case sensitive
43+
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
44+
NumberHandling = JsonNumberHandling.AllowReadingFromString,
45+
ReadCommentHandling = JsonCommentHandling.Skip)]
46+
internal sealed partial class ContractDescriptorContext : JsonSerializerContext
47+
{
48+
}
49+
50+
public class ContractDescriptor
51+
{
52+
public int? Version { get; set; }
53+
public string? Baseline { get; set; }
54+
public Dictionary<string, int>? Contracts { get; set; }
55+
56+
public Dictionary<string, TypeDescriptor>? Types { get; set; }
57+
58+
public Dictionary<string, GlobalDescriptor>? Globals { get; set; }
59+
60+
[JsonExtensionData]
61+
public Dictionary<string, object?>? Extras { get; set; }
62+
}
63+
64+
[JsonConverter(typeof(TypeDescriptorConverter))]
65+
public class TypeDescriptor
66+
{
67+
public uint? Size { get; set; }
68+
public Dictionary<string, FieldDescriptor>? Fields { get; set; }
69+
}
70+
71+
[JsonConverter(typeof(FieldDescriptorConverter))]
72+
public class FieldDescriptor
73+
{
74+
public string? Type { get; set; }
75+
public int Offset { get; set; }
76+
}
77+
78+
[JsonConverter(typeof(GlobalDescriptorConverter))]
79+
public class GlobalDescriptor
80+
{
81+
public string? Type { get; set; }
82+
public ulong Value { get; set; }
83+
public bool Indirect { get; set; }
84+
}
85+
86+
internal sealed class TypeDescriptorConverter : JsonConverter<TypeDescriptor>
87+
{
88+
// Almost a normal dictionary converter except:
89+
// 1. looks for a special key "!" to set the Size property
90+
// 2. field names are property names, but treated case-sensitively
91+
public override TypeDescriptor Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
92+
{
93+
if (reader.TokenType != JsonTokenType.StartObject)
94+
throw new JsonException();
95+
uint? size = null;
96+
Dictionary<string, FieldDescriptor>? fields = new();
97+
while (reader.Read())
98+
{
99+
switch (reader.TokenType)
100+
{
101+
case JsonTokenType.EndObject:
102+
return new TypeDescriptor { Size = size, Fields = fields };
103+
case JsonTokenType.PropertyName:
104+
string? fieldNameOrSizeSigil = reader.GetString();
105+
reader.Read(); // read the next value: either a number or a field descriptor
106+
if (fieldNameOrSizeSigil == TypeDescriptorSizeSigil)
107+
{
108+
uint newSize = reader.GetUInt32();
109+
if (size is not null)
110+
{
111+
throw new JsonException($"Size specified multiple times: {size} and {newSize}");
112+
}
113+
size = newSize;
114+
}
115+
else
116+
{
117+
string? fieldName = fieldNameOrSizeSigil;
118+
var field = JsonSerializer.Deserialize(ref reader, ContractDescriptorContext.Default.FieldDescriptor);
119+
if (fieldName is null || field is null)
120+
throw new JsonException();
121+
if (!fields.TryAdd(fieldName, field))
122+
{
123+
throw new JsonException($"Duplicate field name: {fieldName}");
124+
}
125+
}
126+
break;
127+
case JsonTokenType.Comment:
128+
// unexpected - we specified to skip comments. but let's ignore anyway
129+
break;
130+
default:
131+
throw new JsonException();
132+
}
133+
}
134+
throw new JsonException();
135+
}
136+
137+
public override void Write(Utf8JsonWriter writer, TypeDescriptor value, JsonSerializerOptions options)
138+
{
139+
throw new NotImplementedException();
140+
}
141+
}
142+
143+
internal sealed class FieldDescriptorConverter : JsonConverter<FieldDescriptor>
144+
{
145+
// Compact Field descriptors are either a number or a two element array
146+
// 1. number - no type, offset is given as the number
147+
// 2. [number, string] - offset is given as the number, type name is given as the string
148+
public override FieldDescriptor Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
149+
{
150+
if (TryGetInt32FromToken(ref reader, out int offset))
151+
return new FieldDescriptor { Offset = offset };
152+
if (reader.TokenType != JsonTokenType.StartArray)
153+
throw new JsonException();
154+
reader.Read();
155+
// [number, string]
156+
// ^ we're here
157+
if (!TryGetInt32FromToken(ref reader, out offset))
158+
throw new JsonException();
159+
reader.Read(); // string
160+
if (reader.TokenType != JsonTokenType.String)
161+
throw new JsonException();
162+
string? type = reader.GetString();
163+
reader.Read(); // end of array
164+
if (reader.TokenType != JsonTokenType.EndArray)
165+
throw new JsonException();
166+
return new FieldDescriptor { Type = type, Offset = offset };
167+
}
168+
169+
public override void Write(Utf8JsonWriter writer, FieldDescriptor value, JsonSerializerOptions options)
170+
{
171+
throw new JsonException();
172+
}
173+
}
174+
175+
internal sealed class GlobalDescriptorConverter : JsonConverter<GlobalDescriptor>
176+
{
177+
public override GlobalDescriptor Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
178+
{
179+
// four cases:
180+
// 1. number - no type, direct value, given value
181+
// 2. [number] - no type, indirect value, given aux data ptr
182+
// 3. [number, string] - type, direct value, given value
183+
// 4. [[number], string] - type, indirect value, given aux data ptr
184+
185+
// Case 1: number
186+
if (TryGetUInt64FromToken(ref reader, out ulong valueCase1))
187+
return new GlobalDescriptor { Value = valueCase1 };
188+
if (reader.TokenType != JsonTokenType.StartArray)
189+
throw new JsonException();
190+
reader.Read();
191+
// we're in case 2 or 3 or 4:
192+
// case 2: [number]
193+
// ^ we're here
194+
// case 3: [number, string]
195+
// ^ we're here
196+
// case 4: [[number], string]
197+
// ^ we're here
198+
if (TryGetUInt64FromToken(ref reader, out ulong valueCase2or3))
199+
{
200+
// case 2 or 3
201+
// case 2: [number]
202+
// ^ we're here
203+
// case 3: [number, string]
204+
// ^ we're here
205+
reader.Read(); // end of array (case 2) or string (case 3)
206+
if (reader.TokenType == JsonTokenType.EndArray) // it was case 2
207+
{
208+
return new GlobalDescriptor { Value = valueCase2or3, Indirect = true };
209+
}
210+
if (reader.TokenType == JsonTokenType.String) // it was case 3
211+
{
212+
string? type = reader.GetString();
213+
reader.Read(); // end of array for case 3
214+
if (reader.TokenType != JsonTokenType.EndArray)
215+
throw new JsonException();
216+
return new GlobalDescriptor { Type = type, Value = valueCase2or3 };
217+
}
218+
throw new JsonException();
219+
}
220+
if (reader.TokenType == JsonTokenType.StartArray)
221+
{
222+
// case 4: [[number], string]
223+
// ^ we're here
224+
reader.Read(); // number
225+
if (!TryGetUInt64FromToken(ref reader, out ulong value))
226+
throw new JsonException();
227+
reader.Read(); // end of inner array
228+
if (reader.TokenType != JsonTokenType.EndArray)
229+
throw new JsonException();
230+
reader.Read(); // string
231+
if (reader.TokenType != JsonTokenType.String)
232+
throw new JsonException();
233+
string? type = reader.GetString();
234+
reader.Read(); // end of outer array
235+
if (reader.TokenType != JsonTokenType.EndArray)
236+
throw new JsonException();
237+
return new GlobalDescriptor { Type = type, Value = value, Indirect = true };
238+
}
239+
throw new JsonException();
240+
}
241+
242+
public override void Write(Utf8JsonWriter writer, GlobalDescriptor value, JsonSerializerOptions options)
243+
{
244+
throw new JsonException();
245+
}
246+
}
247+
248+
// Somewhat flexible parsing of numbers, allowing json number tokens or strings as decimal or hex, possibly negatated.
249+
private static bool TryGetUInt64FromToken(ref Utf8JsonReader reader, out ulong value)
250+
{
251+
if (reader.TokenType == JsonTokenType.Number)
252+
{
253+
if (reader.TryGetUInt64(out value))
254+
return true;
255+
if (reader.TryGetInt64(out long signedValue))
256+
{
257+
value = (ulong)signedValue;
258+
return true;
259+
}
260+
}
261+
if (reader.TokenType == JsonTokenType.String)
262+
{
263+
var s = reader.GetString();
264+
if (s == null)
265+
{
266+
value = 0u;
267+
return false;
268+
}
269+
if (ulong.TryParse(s, out value))
270+
return true;
271+
if (long.TryParse(s, out long signedValue))
272+
{
273+
value = (ulong)signedValue;
274+
return true;
275+
}
276+
if (s.StartsWith("0x", StringComparison.OrdinalIgnoreCase) &&
277+
ulong.TryParse(s.AsSpan(2), System.Globalization.NumberStyles.HexNumber, null, out value))
278+
{
279+
return true;
280+
}
281+
if (s.StartsWith("-0x", StringComparison.OrdinalIgnoreCase) &&
282+
ulong.TryParse(s.AsSpan(3), System.Globalization.NumberStyles.HexNumber, null, out ulong negValue))
283+
{
284+
value = ~negValue + 1; // two's complement
285+
return true;
286+
}
287+
}
288+
value = 0;
289+
return false;
290+
}
291+
292+
// Somewhat flexible parsing of numbers, allowing json number tokens or strings as either decimal or hex, possibly negated
293+
private static bool TryGetInt32FromToken(ref Utf8JsonReader reader, out int value)
294+
{
295+
if (reader.TokenType == JsonTokenType.Number)
296+
{
297+
value = reader.GetInt32();
298+
return true;
299+
}
300+
if (reader.TokenType == JsonTokenType.String)
301+
{
302+
var s = reader.GetString();
303+
if (s == null)
304+
{
305+
value = 0;
306+
return false;
307+
}
308+
if (int.TryParse(s, out value))
309+
{
310+
return true;
311+
}
312+
if (s.StartsWith("0x", StringComparison.OrdinalIgnoreCase) &&
313+
int.TryParse(s.AsSpan(2), System.Globalization.NumberStyles.HexNumber, null, out value))
314+
{
315+
return true;
316+
}
317+
if (s.StartsWith("-0x", StringComparison.OrdinalIgnoreCase) &&
318+
int.TryParse(s.AsSpan(3), System.Globalization.NumberStyles.HexNumber, null, out int negValue))
319+
{
320+
value = -negValue;
321+
return true;
322+
}
323+
}
324+
value = 0;
325+
return false;
326+
}
327+
}

0 commit comments

Comments
 (0)