TfsExportUsersForMappingProcessor: allow matching users by email (#2526)

Our scenario is:

- Source server is on-premise TFS 2018 connected to on-premise Active
Directory.
- Target server is Azure DevOps connected to Azure Entra ID (formerly
Azure Active Directory).

Even when the users are synchronized between the Active Directiories,
the users' display names are often different between the two. So default
matchind by display name does not work for us. I implemented the
possibility to match users by email (at least for us it works for most
cases). This matching is turned off by default. When turned on, it has
higher precedence than matching by display name. So first, match by
email is looked for and if no match is found, default matching by
display name is used.

Because of this I expanded properties of the users loaded from server. I
still do not use all of them, but I want to keep them for future use (I
have another PR prepared). Because of this, I also added mapper
([Riok.Mapperly](https://github.com/riok/mapperly)), so I do not need to
copy properties one by one when creating `IdentityItemData` object from
`Identity` object.

Directory.Packages.props

@@ -40,6 +40,7 @@
     <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
     <PackageVersion Include="OxyPlot.Core" Version="2.2.0" />
     <PackageVersion Include="OxyPlot.ImageSharp" Version="2.2.0" />
+    <PackageVersion Include="Riok.Mapperly" Version="4.1.0" />
     <PackageVersion Include="Serilog" Version="4.0.1" />
     <PackageVersion Include="Serilog.Enrichers.Environment" Version="3.0.1" />
     <PackageVersion Include="Serilog.Enrichers.Process" Version="3.0.0" />

src/MigrationTools.Clients.TfsObjectModel/MigrationTools.Clients.TfsObjectModel.csproj

@@ -40,6 +40,7 @@
     <PackageReference Include="OpenTelemetry.Exporter.Console" />
     <PackageReference Include="OpenTelemetry.Extensions.Hosting" />
     <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" />
+    <PackageReference Include="Riok.Mapperly" />
     <PackageReference Include="TfsUrlParser" />
   </ItemGroup>
 

src/MigrationTools.Clients.TfsObjectModel/Processors/TfsExportUsersForMappingProcessor.cs

@@ -65,14 +65,14 @@ protected override void InternalExecute()
                 Log.LogInformation("Found {usersToMap} total mapped", usersToMap.Count);
             }
 
-            usersToMap = usersToMap.Where(x => x.Source.FriendlyName != x.Target?.FriendlyName).ToList();
+            usersToMap = usersToMap.Where(x => x.Source.DisplayName != x.Target?.DisplayName).ToList();
             Log.LogInformation("Filtered to {usersToMap} total viable mappings", usersToMap.Count);
             Dictionary<string, string> usermappings = [];
             foreach (IdentityMapData userMapping in usersToMap)
             {
-                // We cannot use ToDictionary(), because there can be multiple users with the same friendly name and so
+                // We cannot use ToDictionary(), because there can be multiple users with the same display name and so
                 // it would throw with duplicate key. This way we just overwrite the value – last item in source wins.
-                usermappings[userMapping.Source.FriendlyName] = userMapping.Target?.FriendlyName;
+                usermappings[userMapping.Source.DisplayName] = userMapping.Target?.DisplayName;
             }
             System.IO.File.WriteAllText(CommonTools.UserMapping.Options.UserMappingFile, JsonConvert.SerializeObject(usermappings, Formatting.Indented));
             Log.LogInformation("Writen to: {LocalExportJsonFile}", CommonTools.UserMapping.Options.UserMappingFile);

src/MigrationTools.Clients.TfsObjectModel/Tools/TfsUserMappingTool.cs

@@ -8,6 +8,7 @@
 using MigrationTools.DataContracts;
 using MigrationTools.Processors.Infrastructure;
 using MigrationTools.Tools.Infrastructure;
+using Riok.Mapperly.Abstractions;
 
 namespace MigrationTools.Tools
 {
@@ -22,16 +23,19 @@ public TfsUserMappingTool(IOptions<TfsUserMappingToolOptions> options, IServiceP
         {
         }
 
-        private List<string> GetUsersFromWorkItems(List<WorkItemData> workitems, List<string> identityFieldsToCheck)
+        private readonly CaseInsensitiveStringComparer _workItemNameComparer = new();
+        private readonly TfsUserMappingToolMapper _mapper = new();
+
+        private HashSet<string> GetUsersFromWorkItems(List<WorkItemData> workitems, List<string> identityFieldsToCheck)
         {
-            List<string> foundUsers = new List<string>();
+            HashSet<string> foundUsers = new(StringComparer.CurrentCultureIgnoreCase);
             foreach (var wItem in workitems)
             {
                 foreach (var rItem in wItem.Revisions.Values)
                 {
                     foreach (var fItem in rItem.Fields.Values)
                     {
-                        if (identityFieldsToCheck.Contains(fItem.ReferenceName, new CaseInsensativeStringComparer()))
+                        if (identityFieldsToCheck.Contains(fItem.ReferenceName, _workItemNameComparer))
                         {
                             if (!foundUsers.Contains(fItem.Value) && !string.IsNullOrEmpty((string)fItem.Value))
                             {
@@ -74,7 +78,7 @@ private Dictionary<string, string> GetMappingFileData()
                 try
                 {
                     var fileMaps = Newtonsoft.Json.JsonConvert.DeserializeObject<List<IdentityMapData>>(fileData);
-                    _UserMappings = fileMaps.ToDictionary(x => x.Source.FriendlyName, x => x.Target?.FriendlyName);
+                    _UserMappings = fileMaps.ToDictionary(x => x.Source.DisplayName, x => x.Target?.DisplayName);
                 }
                 catch (Exception)
                 {
@@ -104,11 +108,7 @@ private List<IdentityItemData> GetUsersListFromServer(IGroupSecurityService gss)
                     else if ((identity.Type == IdentityType.WindowsUser) || (identity.Type == IdentityType.UnknownIdentityType))
                     {
                         // UnknownIdentityType is set for users in Azure Entra ID.
-                        foundUsers.Add(new IdentityItemData()
-                        {
-                            FriendlyName = identity.DisplayName,
-                            AccountName = identity.AccountName
-                        });
+                        foundUsers.Add(_mapper.IdentityToIdentityItemData(identity));
                     }
                     else
                     {
@@ -134,12 +134,40 @@ public List<IdentityMapData> GetUsersInSourceMappedToTarget(TfsProcessor process
                 var sourceUsers = GetUsersListFromServer(processor.Source.GetService<IGroupSecurityService>());
                 Log.LogInformation($"TfsUserMappingTool::GetUsersInSourceMappedToTarget Loading identities from target server");
                 var targetUsers = GetUsersListFromServer(processor.Target.GetService<IGroupSecurityService>());
-                return sourceUsers.Select(sUser => new IdentityMapData { Source = sUser, Target = targetUsers.SingleOrDefault(tUser => tUser.FriendlyName == sUser.FriendlyName) }).ToList();
+
+                if (Options.MatchUsersByEmail)
+                {
+                    Log.LogInformation("TfsUserMappingTool::GetUsersInSourceMappedToTarget "
+                        + "Matching users between source and target by email is enabled. In no match by email is found, "
+                        + "matching by display name will be used.");
+                }
+
+                List<IdentityMapData> identityMap = [];
+                foreach (var sourceUser in sourceUsers)
+                {
+                    IdentityItemData targetUser = null;
+                    if (Options.MatchUsersByEmail && !string.IsNullOrEmpty(sourceUser.MailAddress))
+                    {
+                        var candidates = targetUsers
+                            .Where(tu => tu.MailAddress.Equals(sourceUser.MailAddress, StringComparison.OrdinalIgnoreCase))
+                            .ToList();
+                        if (candidates.Count == 1)
+                        {
+                            // If there are more than one user with the same email address, we can't be sure which one is
+                            // the correct one, so mapping will match either by display name, or will be skipped and
+                            // exported for manual mapping.
+                            targetUser = candidates[0];
+                        }
+                    }
+                    targetUser ??= targetUsers.SingleOrDefault(x => x.DisplayName == sourceUser.DisplayName);
+                    identityMap.Add(new IdentityMapData { Source = sourceUser, Target = targetUser });
+                }
+                return identityMap;
             }
             else
             {
                 Log.LogWarning("TfsUserMappingTool is disabled in settings. You may have users in the source that are not mapped to the target. ");
-                return null;
+                return [];
             }
         }
 
@@ -148,11 +176,11 @@ public List<IdentityMapData> GetUsersInSourceMappedToTargetForWorkItems(TfsProce
             if (Options.Enabled)
             {
                 Dictionary<string, string> result = new Dictionary<string, string>();
-                List<string> workItemUsers = GetUsersFromWorkItems(sourceWorkItems, Options.IdentityFieldsToCheck);
+                HashSet<string> workItemUsers = GetUsersFromWorkItems(sourceWorkItems, Options.IdentityFieldsToCheck);
                 Log.LogDebug($"TfsUserMappingTool::GetUsersInSourceMappedToTargetForWorkItems [workItemUsers|{workItemUsers.Count}]");
                 List<IdentityMapData> mappedUsers = GetUsersInSourceMappedToTarget(processor);
                 Log.LogDebug($"TfsUserMappingTool::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|{mappedUsers.Count}]");
-                return mappedUsers.Where(x => workItemUsers.Contains(x.Source.FriendlyName)).ToList();
+                return mappedUsers.Where(x => workItemUsers.Contains(x.Source.DisplayName)).ToList();
             }
             else
             {
@@ -162,7 +190,7 @@ public List<IdentityMapData> GetUsersInSourceMappedToTargetForWorkItems(TfsProce
         }
     }
 
-    public class CaseInsensativeStringComparer : IEqualityComparer<string>
+    internal class CaseInsensitiveStringComparer : IEqualityComparer<string>
     {
         public bool Equals(string x, string y)
         {
@@ -174,4 +202,12 @@ public int GetHashCode(string obj)
             return obj.GetHashCode();
         }
     }
+
+    [Mapper]
+    internal partial class TfsUserMappingToolMapper
+    {
+#pragma warning disable RMG020 // Source member is not mapped to any target member
+        public partial IdentityItemData IdentityToIdentityItemData(Identity identity);
+#pragma warning restore RMG020 // Source member is not mapped to any target member
+    }
 }

src/MigrationTools.Clients.TfsObjectModel/Tools/TfsUserMappingToolOptions.cs

@@ -1,7 +1,4 @@
-using System;
-using System.Collections.Generic;
-using Microsoft.TeamFoundation.Build.Client;
-using MigrationTools.Enrichers;
+using System.Collections.Generic;
 using MigrationTools.Tools.Infrastructure;
 
 namespace MigrationTools.Tools
@@ -10,7 +7,7 @@ public class TfsUserMappingToolOptions : ToolOptions, ITfsUserMappingToolOptions
     {
 
         /// <summary>
-        /// This is a list of the Identiy fields in the Source to check for user mapping purposes. You should list all identiy fields that you wan to map.
+        /// This is a list of the Identiy fields in the Source to check for user mapping purposes. You should list all identiy fields that you want to map.
         /// </summary>
         public List<string> IdentityFieldsToCheck { get; set; }
 
@@ -19,11 +16,17 @@ public class TfsUserMappingToolOptions : ToolOptions, ITfsUserMappingToolOptions
         /// </summary>
         public string UserMappingFile { get; set; }
 
+        /// <summary>
+        /// By default, users in source are mapped to target users by their display name. If this is set to true, then the
+        /// users will be mapped by their email address first. If no match is found, then the display name will be used.
+        /// </summary>
+        public bool MatchUsersByEmail { get; set; }
     }
 
     public interface ITfsUserMappingToolOptions
     {
         List<string> IdentityFieldsToCheck { get; set; }
         string UserMappingFile { get; set; }
+        bool MatchUsersByEmail { get; set; }
     }
-}
+}

src/MigrationTools/DataContracts/IdentityItemData.cs

@@ -2,8 +2,11 @@
 {
     public class IdentityItemData
     {
-        public string FriendlyName { get; set; }
+        public string Sid { get; set; }
+        public string DisplayName { get; set; }
+        public string Domain { get; set; }
         public string AccountName { get; set; }
+        public string MailAddress { get; set; }
     }
 
     public class IdentityMapData