Add option to ignore reference cycles on serialization (#46101)

* Add option to ignore reference cycles on serialization

* Fix whitespace

* Combine IsValueType check on WriteCoreAsObject

* Add missing null-forgiving operator

* Fix perf regression by using ReferenceHandlerStrategy field in JsonSerializerOptions

* Address suggestions

* Rename API to IgnoreCycles

* Add missing letter to test class name

* Fix CI issue with nullable annotation

Author: jozkee

src/libraries/System.Text.Json/ref/System.Text.Json.cs

@@ -559,6 +559,7 @@ public abstract partial class ReferenceHandler
     {
         protected ReferenceHandler() { }
         public static System.Text.Json.Serialization.ReferenceHandler Preserve { get { throw null; } }
+        public static System.Text.Json.Serialization.ReferenceHandler IgnoreCycles { get { throw null; } }
         public abstract System.Text.Json.Serialization.ReferenceResolver CreateResolver();
     }
     public sealed partial class ReferenceHandler<T> : System.Text.Json.Serialization.ReferenceHandler where T : System.Text.Json.Serialization.ReferenceResolver, new()

src/libraries/System.Text.Json/src/System.Text.Json.csproj

@@ -64,6 +64,8 @@
     <Compile Include="System\Text\Json\Serialization\Attributes\JsonIncludeAttribute.cs" />
     <Compile Include="System\Text\Json\Serialization\Attributes\JsonNumberHandlingAttribute.cs" />
     <Compile Include="System\Text\Json\Serialization\Attributes\JsonPropertyNameAttribute.cs" />
+    <Compile Include="System\Text\Json\Serialization\IgnoreReferenceResolver.cs" />
+    <Compile Include="System\Text\Json\Serialization\ReferenceEqualsWrapper.cs" />
     <Compile Include="System\Text\Json\Serialization\ClassType.cs" />
     <Compile Include="System\Text\Json\Serialization\ConverterList.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\Collection\ArrayConverter.cs" />
@@ -128,6 +130,7 @@
     <Compile Include="System\Text\Json\Serialization\Converters\Value\UInt64Converter.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\Value\UriConverter.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\Value\VersionConverter.cs" />
+    <Compile Include="System\Text\Json\Serialization\IgnoreReferenceHandler.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonCamelCaseNamingPolicy.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonClassInfo.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonClassInfo.Cache.cs" />
@@ -174,6 +177,7 @@
     <Compile Include="System\Text\Json\Serialization\ReadStackFrame.cs" />
     <Compile Include="System\Text\Json\Serialization\ReferenceHandler.cs" />
     <Compile Include="System\Text\Json\Serialization\ReferenceHandlerOfT.cs" />
+    <Compile Include="System\Text\Json\Serialization\ReferenceHandlingStrategy.cs" />
     <Compile Include="System\Text\Json\Serialization\ReferenceResolver.cs" />
     <Compile Include="System\Text\Json\Serialization\ReflectionEmitMemberAccessor.cs" />
     <Compile Include="System\Text\Json\Serialization\ReflectionMemberAccessor.cs" />

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/DictionaryDefaultConverter.cs

@@ -133,7 +133,7 @@ internal sealed override bool OnTryRead(
                 }
 
                 // Handle the metadata properties.
-                bool preserveReferences = options.ReferenceHandler != null;
+                bool preserveReferences = options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve;
                 if (preserveReferences && state.Current.ObjectState < StackFrameObjectState.PropertyValue)
                 {
                     if (JsonSerializer.ResolveMetadataForJsonObject<TCollection>(ref reader, ref state, options))
@@ -272,8 +272,7 @@ internal sealed override bool OnTryWrite(
             {
                 state.Current.ProcessedStartToken = true;
                 writer.WriteStartObject();
-
-                if (options.ReferenceHandler != null)
+                if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve)
                 {
                     if (JsonSerializer.WriteReferenceForObject(this, dictionary, ref state, writer) == MetadataPropertyName.Ref)
                     {

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IEnumerableDefaultConverter.cs

@@ -90,7 +90,7 @@ internal override bool OnTryRead(
             {
                 // Slower path that supports continuation and preserved references.
 
-                bool preserveReferences = options.ReferenceHandler != null;
+                bool preserveReferences = options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve;
                 if (state.Current.ObjectState == StackFrameObjectState.None)
                 {
                     if (reader.TokenType == JsonTokenType.StartArray)
@@ -236,12 +236,7 @@ internal sealed override bool OnTryWrite(
                 if (!state.Current.ProcessedStartToken)
                 {
                     state.Current.ProcessedStartToken = true;
-
-                    if (options.ReferenceHandler == null)
-                    {
-                        writer.WriteStartArray();
-                    }
-                    else
+                    if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve)
                     {
                         MetadataPropertyName metadata = JsonSerializer.WriteReferenceForCollection(this, value, ref state, writer);
                         if (metadata == MetadataPropertyName.Ref)
@@ -251,6 +246,10 @@ internal sealed override bool OnTryWrite(
 
                         state.Current.MetadataPropertyName = metadata;
                     }
+                    else
+                    {
+                        writer.WriteStartArray();
+                    }
 
                     state.Current.DeclaredJsonPropertyInfo = state.Current.JsonClassInfo.ElementClassInfo!.PropertyInfoForClassInfo;
                 }

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/ObjectDefaultConverter.cs

@@ -75,7 +75,7 @@ internal override bool OnTryRead(ref Utf8JsonReader reader, Type typeToConvert,
                 // Handle the metadata properties.
                 if (state.Current.ObjectState < StackFrameObjectState.PropertyValue)
                 {
-                    if (options.ReferenceHandler != null)
+                    if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve)
                     {
                         if (JsonSerializer.ResolveMetadataForJsonObject<T>(ref reader, ref state, options))
                         {
@@ -233,8 +233,7 @@ internal sealed override bool OnTryWrite(
             if (!state.SupportContinuation)
             {
                 writer.WriteStartObject();
-
-                if (options.ReferenceHandler != null)
+                if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve)
                 {
                     if (JsonSerializer.WriteReferenceForObject(this, objectValue, ref state, writer) == MetadataPropertyName.Ref)
                     {
@@ -289,8 +288,7 @@ internal sealed override bool OnTryWrite(
                 if (!state.Current.ProcessedStartToken)
                 {
                     writer.WriteStartObject();
-
-                    if (options.ReferenceHandler != null)
+                    if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve)
                     {
                         if (JsonSerializer.WriteReferenceForObject(this, objectValue, ref state, writer) == MetadataPropertyName.Ref)
                         {

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/IgnoreReferenceHandler.cs

@@ -0,0 +1,12 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Text.Json.Serialization
+{
+    internal sealed class IgnoreReferenceHandler : ReferenceHandler
+    {
+        public IgnoreReferenceHandler() => HandlingStrategy = ReferenceHandlingStrategy.IgnoreCycles;
+
+        public override ReferenceResolver CreateResolver() => new IgnoreReferenceResolver();
+    }
+}

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/IgnoreReferenceResolver.cs

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace System.Text.Json.Serialization
+{
+    internal sealed class IgnoreReferenceResolver : ReferenceResolver
+    {
+        // The stack of references on the branch of the current object graph, used to detect reference cycles.
+        private Stack<ReferenceEqualsWrapper>? _stackForCycleDetection;
+
+        internal override void PopReferenceForCycleDetection()
+        {
+            Debug.Assert(_stackForCycleDetection != null);
+            _stackForCycleDetection.Pop();
+        }
+
+        internal override bool ContainsReferenceForCycleDetection(object value)
+            => _stackForCycleDetection?.Contains(new ReferenceEqualsWrapper(value)) ?? false;
+
+        internal override void PushReferenceForCycleDetection(object value)
+        {
+            var wrappedValue = new ReferenceEqualsWrapper(value);
+
+            if (_stackForCycleDetection is null)
+            {
+                _stackForCycleDetection = new Stack<ReferenceEqualsWrapper>();
+            }
+
+            Debug.Assert(!_stackForCycleDetection.Contains(wrappedValue));
+            _stackForCycleDetection.Push(wrappedValue);
+        }
+
+        public override void AddReference(string referenceId, object value) => throw new InvalidOperationException();
+
+        public override string GetReference(object value, out bool alreadyExists) => throw new InvalidOperationException();
+
+        public override object ResolveReference(string referenceId) => throw new InvalidOperationException();
+    }
+}

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.WriteCore.cs

@@ -11,10 +11,19 @@ internal sealed override bool WriteCoreAsObject(
             JsonSerializerOptions options,
             ref WriteStack state)
         {
-            // Value types can never have a null except for Nullable<T>.
-            if (value == null && IsValueType && Nullable.GetUnderlyingType(TypeToConvert) == null)
+            if (IsValueType)
             {
-                ThrowHelper.ThrowJsonException_DeserializeUnableToConvertValue(TypeToConvert);
+                // Value types can never have a null except for Nullable<T>.
+                if (value == null && Nullable.GetUnderlyingType(TypeToConvert) == null)
+                {
+                    ThrowHelper.ThrowJsonException_DeserializeUnableToConvertValue(TypeToConvert);
+                }
+
+                // Root object is a boxed value type, we need to push it to the reference stack before it gets unboxed here.
+                if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.IgnoreCycles && value != null)
+                {
+                    state.ReferenceResolver.PushReferenceForCycleDetection(value);
+                }
             }
 
             T actualValue = (T)value!;

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs

@@ -196,7 +196,8 @@ internal bool TryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSeriali
                         ref reader);
                 }
 
-                if (CanBePolymorphic && options.ReferenceHandler != null && value is JsonElement element)
+                if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve &&
+                    CanBePolymorphic && value is JsonElement element)
                 {
                     // Edge case where we want to lookup for a reference when parsing into typeof(object)
                     // instead of return `value` as a JsonElement.
@@ -303,23 +304,48 @@ internal bool TryWrite(Utf8JsonWriter writer, in T value, JsonSerializerOptions
                 ThrowHelper.ThrowJsonException_SerializerCycleDetected(options.EffectiveMaxDepth);
             }
 
+            if (CanBeNull && !HandleNullOnWrite && IsNull(value))
+            {
+                // We do not pass null values to converters unless HandleNullOnWrite is true. Null values for properties were
+                // already handled in GetMemberAndWriteJson() so we don't need to check for IgnoreNullValues here.
+                writer.WriteNullValue();
+                return true;
+            }
+
+            bool ignoreCyclesPopReference = false;
+            if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.IgnoreCycles &&
+                !IsValueType && !IsNull(value))
+            {
+                Debug.Assert(value != null);
+                ReferenceResolver resolver = state.ReferenceResolver;
+
+                // Write null to break reference cycles.
+                if (resolver.ContainsReferenceForCycleDetection(value))
+                {
+                    writer.WriteNullValue();
+                    return true;
+                }
+
+                // For boxed reference types: do not push when boxed in order to avoid false positives
+                //   when we run the ContainsReferenceForCycleDetection check for the converter of the unboxed value.
+                if (!CanBePolymorphic)
+                {
+                    resolver.PushReferenceForCycleDetection(value);
+                    ignoreCyclesPopReference = true;
+                }
+            }
+
             if (CanBePolymorphic)
             {
                 if (value == null)
                 {
-                    if (!HandleNullOnWrite)
-                    {
-                        writer.WriteNullValue();
-                    }
-                    else
-                    {
-                        Debug.Assert(ClassType == ClassType.Value);
-                        Debug.Assert(!state.IsContinuation);
+                    Debug.Assert(ClassType == ClassType.Value);
+                    Debug.Assert(!state.IsContinuation);
+                    Debug.Assert(HandleNullOnWrite);
 
-                        int originalPropertyDepth = writer.CurrentDepth;
-                        Write(writer, value, options);
-                        VerifyWrite(originalPropertyDepth, writer);
-                    }
+                    int originalPropertyDepth = writer.CurrentDepth;
+                    Write(writer, value, options);
+                    VerifyWrite(originalPropertyDepth, writer);
 
                     return true;
                 }
@@ -339,18 +365,26 @@ internal bool TryWrite(Utf8JsonWriter writer, in T value, JsonSerializerOptions
                     JsonConverter jsonConverter = state.Current.InitializeReEntry(type, options);
                     if (jsonConverter != this)
                     {
+                        if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.IgnoreCycles &&
+                            jsonConverter.IsValueType)
+                        {
+                            // For boxed value types: push the value before it gets unboxed on TryWriteAsObject.
+                            state.ReferenceResolver.PushReferenceForCycleDetection(value);
+                            ignoreCyclesPopReference = true;
+                        }
+
                         // We found a different converter; forward to that.
-                        return jsonConverter.TryWriteAsObject(writer, value, options, ref state);
+                        bool success2 = jsonConverter.TryWriteAsObject(writer, value, options, ref state);
+
+                        if (ignoreCyclesPopReference)
+                        {
+                            state.ReferenceResolver.PopReferenceForCycleDetection();
+                        }
+
+                        return success2;
                     }
                 }
             }
-            else if (CanBeNull && !HandleNullOnWrite && IsNull(value))
-            {
-                // We do not pass null values to converters unless HandleNullOnWrite is true. Null values for properties were
-                // already handled in GetMemberAndWriteJson() so we don't need to check for IgnoreNullValues here.
-                writer.WriteNullValue();
-                return true;
-            }
 
             if (ClassType == ClassType.Value)
             {
@@ -390,6 +424,11 @@ internal bool TryWrite(Utf8JsonWriter writer, in T value, JsonSerializerOptions
 
             state.Pop(success);
 
+            if (ignoreCyclesPopReference)
+            {
+                state.ReferenceResolver.PopReferenceForCycleDetection();
+            }
+
             return success;
         }
 

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonPropertyInfoOfT.cs

@@ -138,6 +138,15 @@ public override bool GetMemberAndWriteJson(object obj, ref WriteStack state, Utf
         {
             T value = Get!(obj);
 
+            if (Options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.IgnoreCycles &&
+                !Converter.IsValueType && value != null &&
+                state.ReferenceResolver.ContainsReferenceForCycleDetection(value))
+            {
+                // If a reference cycle is detected, treat value as null.
+                value = default!;
+                Debug.Assert(value == null);
+            }
+
             if (IgnoreDefaultValuesOnWrite)
             {
                 // If value is null, it is a reference type or nullable<T>.

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.HandlePropertyName.cs

@@ -79,7 +79,7 @@ internal static ReadOnlySpan<byte> GetPropertyName(
                 unescapedPropertyName = propertyName;
             }
 
-            if (options.ReferenceHandler != null)
+            if (options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve)
             {
                 if (propertyName.Length > 0 && propertyName[0] == '$')
                 {

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializerOptions.cs

@@ -89,6 +89,7 @@ public JsonSerializerOptions(JsonSerializerOptions options)
 
             Converters = new ConverterList(this, (ConverterList)options.Converters);
             EffectiveMaxDepth = options.EffectiveMaxDepth;
+            ReferenceHandlingStrategy = options.ReferenceHandlingStrategy;
 
             // _classes is not copied as sharing the JsonClassInfo and JsonPropertyInfo caches can result in
             // unnecessary references to type metadata, potentially hindering garbage collection on the source options.
@@ -487,9 +488,13 @@ public ReferenceHandler? ReferenceHandler
             {
                 VerifyMutable();
                 _referenceHandler = value;
+                ReferenceHandlingStrategy = value?.HandlingStrategy ?? ReferenceHandlingStrategy.None;
             }
         }
 
+        // The cached value used to determine if ReferenceHandler should use Preserve or IgnoreCycles semanitcs or None of them.
+        internal ReferenceHandlingStrategy ReferenceHandlingStrategy = ReferenceHandlingStrategy.None;
+
         internal MemberAccessor MemberAccessorStrategy
         {
             get

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadStack.cs

@@ -89,7 +89,7 @@ public void Initialize(Type type, JsonSerializerOptions options, bool supportCon
 
             Current.NumberHandling = Current.JsonPropertyInfo.NumberHandling;
 
-            bool preserveReferences = options.ReferenceHandler != null;
+            bool preserveReferences = options.ReferenceHandlingStrategy == ReferenceHandlingStrategy.Preserve;
             if (preserveReferences)
             {
                 ReferenceResolver = options.ReferenceHandler!.CreateResolver(writing: false);

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReferenceEqualsWrapper.cs

@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+
+namespace System.Text.Json.Serialization
+{
+    internal struct ReferenceEqualsWrapper : IEquatable<ReferenceEqualsWrapper>
+    {
+        private object _object;
+        public ReferenceEqualsWrapper(object obj) => _object = obj;
+        public override bool Equals(object? obj) => obj is ReferenceEqualsWrapper otherObj && Equals(otherObj);
+        public bool Equals(ReferenceEqualsWrapper obj) => ReferenceEquals(_object, obj._object);
+        public override int GetHashCode() => RuntimeHelpers.GetHashCode(_object);
+    }
+}