Use RegexCaseEquivalence table for case-insensitive backreferences (#67977)

Author: joperezr

src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs

@@ -181,27 +181,6 @@ private static void EmitRegexDerivedTypeRunnerFactory(IndentedTextWriter writer,
             writer.WriteLine($"    /// <summary>Provides the runner that contains the custom logic implementing the specified regular expression.</summary>");
             writer.WriteLine($"    private sealed class Runner : RegexRunner");
             writer.WriteLine($"    {{");
-            if (rm.Tree.Culture != null)
-            {
-                // If the RegexTree has Culture set, then we need to emit the _textInfo field that should be used at match time for casing operations.
-                // Instead of just using the culture set on the tree, we use it to check which behavior corresponds to the culture, and based on that we
-                // create a TextInfo based on a well-known culture that has the same behavior. This is done in order to ensure that the culture being used at
-                // runtime will be supported no matter where the code ends up running.
-                writer.WriteLine($"        /// <summary>TextInfo that will be used for Backreference case comparisons.</summary>");
-                switch (RegexCaseEquivalences.GetRegexBehavior(rm.Tree.Culture))
-                {
-                    case RegexCaseBehavior.Invariant:
-                        writer.WriteLine($"        private readonly TextInfo _textInfo = CultureInfo.InvariantCulture.TextInfo;");
-                        break;
-                    case RegexCaseBehavior.NonTurkish:
-                        writer.WriteLine($"        private readonly TextInfo _textInfo = CultureInfo.GetCultureInfo(\"en-US\").TextInfo;");
-                        break;
-                    case RegexCaseBehavior.Turkish:
-                        writer.WriteLine($"        private readonly TextInfo _textInfo = CultureInfo.GetCultureInfo(\"tr-TR\").TextInfo;");
-                        break;
-                }
-                writer.WriteLine();
-            }
             if (rm.MatchTimeout is null)
             {
                 // We need to emit timeout checks for everything other than the developer explicitly setting Timeout.Infinite.
@@ -1625,38 +1604,14 @@ void EmitWhenHasCapture()
                     additionalDeclarations.Add("int matchLength = 0;");
                     writer.WriteLine($"matchLength = base.MatchLength({capnum});");
 
-                    bool caseInsensitive = (node.Options & RegexOptions.IgnoreCase) != 0;
-
                     if ((node.Options & RegexOptions.RightToLeft) == 0)
                     {
-                        if (!caseInsensitive)
+                        // Validate that the remaining length of the slice is sufficient
+                        // to possibly match, and then do a SequenceEqual against the matched text.
+                        writer.WriteLine($"if ({sliceSpan}.Length < matchLength || ");
+                        using (EmitBlock(writer, $"    !inputSpan.Slice(base.MatchIndex({capnum}), matchLength).SequenceEqual({sliceSpan}.Slice(0, matchLength)))"))
                         {
-                            // If we're case-sensitive, we can simply validate that the remaining length of the slice is sufficient
-                            // to possibly match, and then do a SequenceEqual against the matched text.
-                            writer.WriteLine($"if ({sliceSpan}.Length < matchLength || ");
-                            using (EmitBlock(writer, $"    !inputSpan.Slice(base.MatchIndex({capnum}), matchLength).SequenceEqual({sliceSpan}.Slice(0, matchLength)))"))
-                            {
-                                Goto(doneLabel);
-                            }
-                        }
-                        else
-                        {
-                            // For case-insensitive, we have to walk each character individually.
-                            using (EmitBlock(writer, $"if ({sliceSpan}.Length < matchLength)"))
-                            {
-                                Goto(doneLabel);
-                            }
-                            writer.WriteLine();
-
-                            additionalDeclarations.Add("int matchIndex = 0;");
-                            writer.WriteLine($"matchIndex = base.MatchIndex({capnum});");
-                            using (EmitBlock(writer, $"for (int i = 0; i < matchLength; i++)"))
-                            {
-                                using (EmitBlock(writer, $"if (_textInfo.ToLower(inputSpan[matchIndex + i]) != _textInfo.ToLower({sliceSpan}[i]))"))
-                                {
-                                    Goto(doneLabel);
-                                }
-                            }
+                            Goto(doneLabel);
                         }
 
                         writer.WriteLine();
@@ -1675,9 +1630,7 @@ void EmitWhenHasCapture()
                         writer.WriteLine($"matchIndex = base.MatchIndex({capnum});");
                         using (EmitBlock(writer, $"for (int i = 0; i < matchLength; i++)"))
                         {
-                            using (EmitBlock(writer, caseInsensitive ?
-                                $"if (_textInfo.ToLower(inputSpan[matchIndex + i]) != _textInfo.ToLower(inputSpan[pos - matchLength + i]))" :
-                                $"if (inputSpan[matchIndex + i] != inputSpan[pos - matchLength + i])"))
+                            using (EmitBlock(writer, $"if (inputSpan[matchIndex + i] != inputSpan[pos - matchLength + i])"))
                             {
                                 Goto(doneLabel);
                             }

src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.cs

@@ -65,7 +65,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
 
                     // If we're unable to generate a full implementation for this regex, report a diagnostic.
                     // We'll still output a limited implementation that just caches a new Regex(...).
-                    if (!regexMethod.Tree.Root.SupportsCompilation(out string? reason))
+                    if (!SupportsCodeGeneration(regexMethod.Tree.Root, out string? reason))
                     {
                         return (regexMethod, reason, Diagnostic.Create(DiagnosticDescriptors.LimitedSourceGeneration, regexMethod.MethodSyntax.GetLocation()));
                     }
@@ -275,6 +275,55 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
             });
         }
 
+        // Determines whether the passed in node supports code generation strategy based on walking the tree.
+        // Also returns a human-readable string to explain the reason (it will be emitted by the source generator, hence
+        // there's no need to localize).
+        private static bool SupportsCodeGeneration(RegexNode node, [NotNullWhen(false)] out string? reason)
+        {
+            if (!node.SupportsCompilation(out reason))
+            {
+                // If the pattern doesn't support Compilation, then code generation won't be supported either.
+                return false;
+            }
+
+            if (HasCaseInsensitiveBackReferences(node))
+            {
+                // For case-insensitive patterns, we use our internal Regex case equivalence table when doing character comparisons.
+                // Most of the use of this table is done at Regex construction time by substituting all characters that are involved in
+                // case conversions into sets that contain all possible characters that could match. That said, there is still one case
+                // where you may need to do case-insensitive comparisons at match time which is the case for backreferences. For that reason,
+                // and given the Regex case equivalence table is internal and can't be called by the source generated emitted type, if
+                // the pattern contains case-insensitive backreferences, we won't try to create a source generated Regex-derived type.
+                reason = "the expression contains case-insensitive backreferences which are not supported by the source generator";
+                return false;
+            }
+
+            // If Compilation is supported and pattern doesn't have case insensitive backreferences, then code generation is supported.
+            reason = null;
+            return true;
+
+            static bool HasCaseInsensitiveBackReferences(RegexNode node)
+            {
+                if (node.Kind is RegexNodeKind.Backreference && (node.Options & RegexOptions.IgnoreCase) != 0)
+                {
+                    return true;
+                }
+
+                int childCount = node.ChildCount();
+                for (int i = 0; i < childCount; i++)
+                {
+                    // This recursion shouldn't hit issues with stack depth since this gets checked after
+                    // SupportCompilation has ensured that the max depth is not greater than 40.
+                    if (HasCaseInsensitiveBackReferences(node.Child(i)))
+                    {
+                        return true;
+                    }
+                }
+
+                return false;
+            }
+        }
+
         /// <summary>Computes a hash of the string.</summary>
         /// <remarks>
         /// Currently an FNV-1a hash function. The actual algorithm used doesn't matter; just something

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/CompiledRegexRunner.cs

@@ -9,14 +9,20 @@ internal sealed class CompiledRegexRunner : RegexRunner
     {
         private readonly ScanDelegate _scanMethod;
         /// <summary>This field will only be set if the pattern contains backreferences and has RegexOptions.IgnoreCase</summary>
-        private readonly TextInfo? _textInfo;
+        private readonly CultureInfo? _culture;
+
+#pragma warning disable CA1823 // Avoid unused private fields. Justification: Used via reflection to cache the Case behavior if needed.
+#pragma warning disable CS0169
+        private RegexCaseBehavior _caseBehavior;
+#pragma warning restore CS0169
+#pragma warning restore CA1823
 
         internal delegate void ScanDelegate(RegexRunner runner, ReadOnlySpan<char> text);
 
         public CompiledRegexRunner(ScanDelegate scan, CultureInfo? culture)
         {
             _scanMethod = scan;
-            _textInfo = culture?.TextInfo;
+            _culture = culture;
         }
 
         protected internal override void Scan(ReadOnlySpan<char> text)

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCaseBehavior.cs

@@ -12,12 +12,16 @@ namespace System.Text.RegularExpressions
     /// same character. Note that we don't consider a mapping when the only relationship between 'A' and 'B' is that one is the ToUpper() representation of the other. This
     /// is for backwards compatibility since, in Regex, we have only consider ToLower() for case insensitive comparisons. Given the case mappings vary depending on the culture,
     /// Regex supports 3 main different behaviors or mappings: Invariant, NonTurkish, and Turkish. This is in order to match the behavior of all .NET supported cultures
-    /// current behavior for ToLower(). As a side note, there should be no cases where 'A'.ToLower() == 'B' but 'A'.ToLower() != 'B'.ToLower(). This aspect is important since
-    /// for backreferences we make use a.ToLower() == b.ToLower() for comparisons so if there was such a case then it would lead to inconsistencies between how we handle
-    /// backreferences vs how we handle other case insensitive comparisons.
+    /// current behavior for ToLower(). As a side note, there should be no cases where 'A'.ToLower() == 'B' but 'A'.ToLower() != 'B'.ToLower().
     /// </summary>
     internal enum RegexCaseBehavior
     {
+        /// <summary>
+        /// This means that the RegexCaseBehavior hasn't been calculated based on a passed in culture yet, so it will need to be calculated before the first
+        /// equivalence check by calling <see cref="RegexCaseEquivalences.GetRegexBehavior(CultureInfo)"/>
+        /// </summary>
+        NotSet,
+
         /// <summary>
         /// Invariant case-mappings are used. This includes all of the common mappings across cultures. This behavior is used when either the  user
         /// specified <see cref="RegexOptions.CultureInvariant"/> or when the CurrentCulture is <see cref="CultureInfo.InvariantCulture"/>.

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCaseEquivalences.cs

@@ -28,15 +28,20 @@ internal static partial class RegexCaseEquivalences
         /// culture and will also factor in the current culture in order to handle the special cases which are different between cultures.
         /// </summary>
         /// <param name="c">The character being analyzed</param>
-        /// <param name="culture">The <see cref="CultureInfo"/> to be used to determine the equivalences.</param>
+        /// <param name="culture">The <see cref="CultureInfo"/> to be used to calculate <paramref name="mappingBehavior"/> in case it hasn't been cached.</param>
+        /// <param name="mappingBehavior">The behavior to be used for case comparisons. If the value hasn't been set yet, it will get initialized in the first lookup.</param>
         /// <param name="equivalences">If <paramref name="c"/> is involved in case conversion, then equivalences will contain the
         /// span of character which should be considered equal to <paramref name="c"/> in a case-insensitive comparison.</param>
         /// <returns><see langword="true"/> if <paramref name="c"/> is involved in case conversion; otherwise, <see langword="false"/></returns>
-        public static bool TryFindCaseEquivalencesForCharWithIBehavior(char c, CultureInfo culture, out ReadOnlySpan<char> equivalences)
+        public static bool TryFindCaseEquivalencesForCharWithIBehavior(char c, CultureInfo culture, ref RegexCaseBehavior mappingBehavior, out ReadOnlySpan<char> equivalences)
         {
             if ((c | 0x20) == 'i' || (c | 0x01) == '\u0131')
             {
-                RegexCaseBehavior mappingBehavior = GetRegexBehavior(culture);
+                // If this is the first time that this method is being called then mappingBehavior will be set to default, so we calculate
+                // the behavior to use and cache the value for future lookups.
+                if (mappingBehavior == RegexCaseBehavior.NotSet)
+                    mappingBehavior = GetRegexBehavior(culture);
+
                 equivalences = c switch
                 {
                     // Invariant mappings

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs

@@ -276,6 +276,7 @@ internal sealed partial class RegexCharClass
         private StringBuilder? _categories;
         private RegexCharClass? _subtractor;
         private bool _negate;
+        private RegexCaseBehavior _caseBehavior;
 
 #if DEBUG
         static RegexCharClass()
@@ -440,7 +441,7 @@ public void AddCaseEquivalences(CultureInfo culture)
                     (char First, char Last) range = rangeList[i];
                     if (range.First == range.Last)
                     {
-                        if (RegexCaseEquivalences.TryFindCaseEquivalencesForCharWithIBehavior(range.First, culture, out ReadOnlySpan<char> equivalences))
+                        if (RegexCaseEquivalences.TryFindCaseEquivalencesForCharWithIBehavior(range.First, culture, ref _caseBehavior, out ReadOnlySpan<char> equivalences))
                         {
                             foreach (char equivalence in equivalences)
                             {
@@ -464,7 +465,7 @@ private void AddCaseEquivalenceRange(char chMin, char chMax, CultureInfo culture
         {
             for (int i = chMin; i <= chMax; i++)
             {
-                if (RegexCaseEquivalences.TryFindCaseEquivalencesForCharWithIBehavior((char)i, culture, out ReadOnlySpan<char> equivalences))
+                if (RegexCaseEquivalences.TryFindCaseEquivalencesForCharWithIBehavior((char)i, culture, ref _caseBehavior, out ReadOnlySpan<char> equivalences))
                 {
                     foreach (char equivalence in equivalences)
                     {