TfsExportUsersForMappingProcessor: export all users in source and tar…

…get server to JSON file (#2527)

Our scenario is:

- Source server is on-premise TFS 2018 connected to on-premise Active
Directory.
- Target server is Azure DevOps connected to Azure Entra ID (formerly
Azure Active Directory).

Even when the users are synchronized between the Active Directiories,
the users' display names are often different and also quite a bunch of
email addresses are different. So a lot of people are not matched at all
and in exported mapping file, the target name is `null`. So the mapping
file needs to be manually edited to add missing users. Usually (almost
always), the user is in target server, but has different email and
display name. To help find the corresponding user, this PR allows to
export all users in source and target server into separate JSON. We can
than find the user we need and use his correct display name in mapping
file.

Two properties are added to `TfsExportUsersForMappingProcessorOptions`:

- `ExportAllUsers` – turns this feature on.
- `UserExportFile` – path to file, where users will be exported.

## Example of export

``` json
{
    "SourceUsers": [
        {
            "Sid": "24b644ca-b413-4bd5-bdea-466534a69c72:Build:1cf84e9e-e068-4a9a-b12e-bf058294c1f5",
            "DisplayName": "Lorem Ipsum",
            "Domain": "Build",
            "AccountName": "lorem",
            "MailAddress": "lorem@example.com"
        },
        ...
    "TargetUsers": [
        {
            "Sid": "751af7c1-c13a-4f1b-888e-da6f6db86a01:Build:0132d9af-7fab-4f97-8170-b28466e07427",
            "DisplayName": "Dolor Sit",
            "Domain": "Build",
            "AccountName": "dolor",
            "MailAddress": "dolor@example.com"
        },
        ...
}
```

src/MigrationTools.Clients.TfsObjectModel/Processors/TfsExportUsersForMappingProcessor.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -42,30 +43,26 @@ protected override void InternalExecute()
         {
             Stopwatch stopwatch = Stopwatch.StartNew();
 
-            if (string.IsNullOrEmpty(CommonTools.UserMapping.Options.UserMappingFile))
-            {
-                Log.LogError("UserMappingFile is not set");
-                throw new ArgumentNullException("UserMappingFile must be set on the TfsUserMappingToolOptions in CommonEnrichersConfig.");
-            }
+            CheckOptions();
 
-            List<IdentityMapData> usersToMap = new List<IdentityMapData>();
+            IdentityMapResult data;
             if (Options.OnlyListUsersInWorkItems)
             {
                 Log.LogInformation("OnlyListUsersInWorkItems is true, only users in work items will be listed");
                 List<WorkItemData> sourceWorkItems = Source.WorkItems.GetWorkItems(Options.WIQLQuery);
                 Log.LogInformation("Processed {0} work items from Source", sourceWorkItems.Count);
 
-                usersToMap = CommonTools.UserMapping.GetUsersInSourceMappedToTargetForWorkItems(this, sourceWorkItems);
-                Log.LogInformation("Found {usersToMap} total mapped", usersToMap.Count);
+                data = CommonTools.UserMapping.GetUsersInSourceMappedToTargetForWorkItems(this, sourceWorkItems);
+                Log.LogInformation("Found {usersToMap} total mapped", data.IdentityMap.Count);
             }
             else
             {
                 Log.LogInformation("OnlyListUsersInWorkItems is false, all users will be listed");
-                usersToMap = CommonTools.UserMapping.GetUsersInSourceMappedToTarget(this);
-                Log.LogInformation("Found {usersToMap} total mapped", usersToMap.Count);
+                data = CommonTools.UserMapping.GetUsersInSourceMappedToTarget(this);
+                Log.LogInformation("Found {usersToMap} total mapped", data.IdentityMap.Count);
             }
 
-            usersToMap = usersToMap.Where(x => x.Source.DisplayName != x.Target?.DisplayName).ToList();
+            List<IdentityMapData> usersToMap = data.IdentityMap.Where(x => x.Source.DisplayName != x.Target?.DisplayName).ToList();
             Log.LogInformation("Filtered to {usersToMap} total viable mappings", usersToMap.Count);
             Dictionary<string, string> usermappings = [];
             foreach (IdentityMapData userMapping in usersToMap)
@@ -74,11 +71,40 @@ protected override void InternalExecute()
                 // it would throw with duplicate key. This way we just overwrite the value – last item in source wins.
                 usermappings[userMapping.Source.DisplayName] = userMapping.Target?.DisplayName;
             }
-            System.IO.File.WriteAllText(CommonTools.UserMapping.Options.UserMappingFile, JsonConvert.SerializeObject(usermappings, Formatting.Indented));
-            Log.LogInformation("Writen to: {LocalExportJsonFile}", CommonTools.UserMapping.Options.UserMappingFile);
+            File.WriteAllText(CommonTools.UserMapping.Options.UserMappingFile, JsonConvert.SerializeObject(usermappings, Formatting.Indented));
+            Log.LogInformation("User mappings writen to: {LocalExportJsonFile}", CommonTools.UserMapping.Options.UserMappingFile);
+            if (Options.ExportAllUsers)
+            {
+                ExportAllUsers(data);
+            }
 
             stopwatch.Stop();
             Log.LogInformation("DONE in {Elapsed} seconds", stopwatch.Elapsed);
         }
+
+        private void ExportAllUsers(IdentityMapResult data)
+        {
+            var allUsers = new
+            {
+                data.SourceUsers,
+                data.TargetUsers
+            };
+            File.WriteAllText(Options.UserExportFile, JsonConvert.SerializeObject(allUsers, Formatting.Indented));
+            Log.LogInformation("All user writen to: {exportFile}", Options.UserExportFile);
+        }
+
+        private void CheckOptions()
+        {
+            if (string.IsNullOrEmpty(CommonTools.UserMapping.Options.UserMappingFile))
+            {
+                Log.LogError("UserMappingFile is not set");
+                throw new ArgumentNullException("UserMappingFile must be set on the TfsUserMappingToolOptions in CommonTools.");
+            }
+            if (Options.ExportAllUsers && string.IsNullOrEmpty(Options.UserExportFile))
+            {
+                Log.LogError($"Flag ExportAllUsers is set but export file UserExportFile is not set.");
+                throw new ArgumentNullException("UserExportFile must be set on the TfsExportUsersForMappingProcessorOptions in Processors.");
+            }
+        }
     }
 }

src/MigrationTools.Clients.TfsObjectModel/Processors/TfsExportUsersForMappingProcessorOptions.cs

@@ -1,7 +1,4 @@
-using System.Collections.Generic;
-using MigrationTools._EngineV1.Configuration;
-using MigrationTools.Enrichers;
-using MigrationTools.Processors.Infrastructure;
+using MigrationTools.Processors.Infrastructure;
 
 namespace MigrationTools.Processors
 {
@@ -16,5 +13,17 @@ public class TfsExportUsersForMappingProcessorOptions : ProcessorOptions
         /// <default>true</default>
         public bool OnlyListUsersInWorkItems { get; set; } = true;
 
+        /// <summary>
+        /// Set to <see langword="true"/>, if you want to export all users in source and target server.
+        /// The lists of user can be useful, if you need tu manually edit mapping file.
+        /// Users will be exported to file set in <see cref="UserExportFile"/>.
+        /// </summary>
+        public bool ExportAllUsers { get; set; }
+
+        /// <summary>
+        /// Path to export file where all source and target servers' users will be exported.
+        /// Users are exported only if <see cref="ExportAllUsers"/> is set to <see langword="true"/>.
+        /// </summary>
+        public string UserExportFile { get; set; }
     }
 }

src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs

@@ -227,15 +227,15 @@ protected override void InternalExecute()
 
         private void ValidateAllUsersExistOrAreMapped(List<WorkItemData> sourceWorkItems)
         {
-
             contextLog.Information("Validating::Check that all users in the source exist in the target or are mapped!");
-            List<IdentityMapData> usersToMap = new List<IdentityMapData>();
-            usersToMap = CommonTools.UserMapping.GetUsersInSourceMappedToTargetForWorkItems(this, sourceWorkItems);
-            if (usersToMap != null && usersToMap?.Count > 0)
+            IdentityMapResult usersToMap = CommonTools.UserMapping.GetUsersInSourceMappedToTargetForWorkItems(this, sourceWorkItems);
+            if (usersToMap.IdentityMap != null && usersToMap.IdentityMap.Count > 0)
             {
-                Log.LogWarning("Validating Failed! There are {usersToMap} users that exist in the source that do not exist in the target. This will not cause any errors, but may result in disconnected users that could have been mapped. Use the ExportUsersForMapping processor to create a list of mappable users. Then Import using ", usersToMap.Count);
+                Log.LogWarning("Validating Failed! There are {usersToMap} users that exist in the source that do not exist "
+                    + "in the target. This will not cause any errors, but may result in disconnected users that could have "
+                    + "been mapped. Use the ExportUsersForMapping processor to create a list of mappable users.",
+                    usersToMap.IdentityMap.Count);
             }
-
         }
 
         //private void ValidateAllNodesExistOrAreMapped(List<WorkItemData> sourceWorkItems)

src/MigrationTools.Clients.TfsObjectModel/Tools/TfsUserMappingTool.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using Microsoft.Extensions.Logging;
@@ -121,11 +121,12 @@ private List<IdentityItemData> GetUsersListFromServer(IGroupSecurityService gss)
                     Log.LogWarning("TfsUserMappingTool::GetUsersListFromServer::[user:{user}] Failed With {Exception}", sid, ex.Message);
                 }
             }
+            foundUsers.Sort((x, y) => x.AccountName.CompareTo(y.AccountName));
             Log.LogInformation("TfsUserMappingTool::GetUsersListFromServer {count} user identities are applicable for mapping", foundUsers.Count);
             return foundUsers;
         }
 
-        public List<IdentityMapData> GetUsersInSourceMappedToTarget(TfsProcessor processor)
+        public IdentityMapResult GetUsersInSourceMappedToTarget(TfsProcessor processor)
         {
             Log.LogDebug("TfsUserMappingTool::GetUsersInSourceMappedToTarget");
             if (Options.Enabled)
@@ -162,25 +163,31 @@ public List<IdentityMapData> GetUsersInSourceMappedToTarget(TfsProcessor process
                     targetUser ??= targetUsers.SingleOrDefault(x => x.DisplayName == sourceUser.DisplayName);
                     identityMap.Add(new IdentityMapData { Source = sourceUser, Target = targetUser });
                 }
-                return identityMap;
+                return new()
+                {
+                    IdentityMap = identityMap,
+                    SourceUsers = sourceUsers,
+                    TargetUsers = targetUsers
+                };
             }
             else
             {
                 Log.LogWarning("TfsUserMappingTool is disabled in settings. You may have users in the source that are not mapped to the target. ");
-                return [];
+                return new();
             }
         }
 
-        public List<IdentityMapData> GetUsersInSourceMappedToTargetForWorkItems(TfsProcessor processor, List<WorkItemData> sourceWorkItems)
+        public IdentityMapResult GetUsersInSourceMappedToTargetForWorkItems(TfsProcessor processor, List<WorkItemData> sourceWorkItems)
         {
             if (Options.Enabled)
             {
                 Dictionary<string, string> result = new Dictionary<string, string>();
                 HashSet<string> workItemUsers = GetUsersFromWorkItems(sourceWorkItems, Options.IdentityFieldsToCheck);
                 Log.LogDebug($"TfsUserMappingTool::GetUsersInSourceMappedToTargetForWorkItems [workItemUsers|{workItemUsers.Count}]");
-                List<IdentityMapData> mappedUsers = GetUsersInSourceMappedToTarget(processor);
-                Log.LogDebug($"TfsUserMappingTool::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|{mappedUsers.Count}]");
-                return mappedUsers.Where(x => workItemUsers.Contains(x.Source.DisplayName)).ToList();
+                IdentityMapResult mappedUsers = GetUsersInSourceMappedToTarget(processor);
+                Log.LogDebug($"TfsUserMappingTool::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|{mappedUsers.IdentityMap.Count}]");
+                mappedUsers.IdentityMap = mappedUsers.IdentityMap.Where(x => workItemUsers.Contains(x.Source.DisplayName)).ToList();
+                return mappedUsers;
             }
             else
             {

src/MigrationTools/DataContracts/IdentityItemData.cs

@@ -1,4 +1,6 @@
-namespace MigrationTools.DataContracts
+using System.Collections.Generic;
+
+namespace MigrationTools.DataContracts
 {
     public class IdentityItemData
     {
@@ -14,4 +16,11 @@ public class IdentityMapData
         public IdentityItemData Source { get; set; }
         public IdentityItemData Target { get; set; }
     }
+
+    public class IdentityMapResult
+    {
+        public List<IdentityMapData> IdentityMap { get; set; } = [];
+        public List<IdentityItemData> SourceUsers { get; set; } = [];
+        public List<IdentityItemData> TargetUsers { get; set; } = [];
+    }
 }