Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Bug]: Export User file empty when Source = DevOps Server 2022 #2085

Open
2 tasks done
LudekStipal opened this issue Jun 10, 2024 · 7 comments
Open
2 tasks done

[Bug]: Export User file empty when Source = DevOps Server 2022 #2085

LudekStipal opened this issue Jun 10, 2024 · 7 comments

Comments

@LudekStipal
Copy link

Version

  • I confirm that I am using the latest version

Source Version

Azure DevOps Server 2022

Target Version

Azure DevOps Service

Relevant configuration

{
  "ChangeSetMappingFile": null,
  "Source": {
    "$type": "TfsTeamProjectConfig",
    "Collection": "https://devops/tfs/VWFSAG/",
    "Project": "IBATeam",
    "ReflectedWorkItemIDFieldName": "ReflectedWorkItemId",
    "AllowCrossProjectLinking": false,
    "AuthenticationMode": "AccessToken",
    "PersonalAccessToken": "xxx",
    "PersonalAccessTokenVariableName": "",
    "LanguageMaps": {
      "AreaPath": "Area",
      "IterationPath": "Iteration"
    }
  },
  "Target": {
    "$type": "TfsTeamProjectConfig",
    "Collection": "https://dev.azure.com/vwfs",
    "Project": "Azubis.AuroraTeam",
    "ReflectedWorkItemIDFieldName": "ReflectedWorkItemId",
    "AllowCrossProjectLinking": false,
    "AuthenticationMode": "AccessToken",
    "PersonalAccessToken": "xxx",
    "PersonalAccessTokenVariableName": "",
    "LanguageMaps": {
      "AreaPath": "Area",
      "IterationPath": "Iteration"
    }
  },
  "FieldMaps": [],
  "GitRepoMapping": null,
  "LogLevel": "Debug",
  "CommonEnrichersConfig": [
    {
      "$type": "TfsNodeStructureOptions",
      "RefName": "TfsNodeStructure",
      "Enabled": false,
      "NodeBasePaths": [],
      "AreaMaps": {},
      "IterationMaps": {},
      "ShouldCreateMissingRevisionPaths": true,
      "ReplicateAllExistingNodes": true
    },
    {
      "$type": "TfsTeamSettingsEnricherOptions",
      "Enabled": false,
      "MigrateTeamSettings": true,
      "UpdateTeamSettings": true,
      "MigrateTeamCapacities": true,
      "Teams": null
    },
    {
      "$type": "TfsWorkItemLinkEnricherOptions",
      "RefName": "TfsWorkItemLinkEnricher",
      "Enabled": false,
      "FilterIfLinkCountMatches": true,
      "SaveAfterEachLinkIsAdded": false
    },
    {
      "$type": "TfsRevisionManagerOptions",
      "RefName": "TfsRevisionManager",
      "Enabled": false,
      "ReplayRevisions": true,
      "MaxRevisions": 0
    },
    {
      "$type": "TfsAttachmentEnricherOptions",
      "RefName": "TfsAttachmentEnricher",
      "Enabled": false,
      "ExportBasePath": "c:\\temp\\WorkItemAttachmentExport",
      "MaxRevisions": 480000000
    },
    {
      "$type": "StringManipulatorEnricherOptions",
      "RefName": "StringManipulator",
      "Enabled": false,
      "MaxStringLength": 1000000,
      "Manipulators": [
        {
          "$type": "RegexStringManipulator",
          "Enabled": true,
          "Pattern": "[^( -~)\n\r\t]+",
          "Replacement": "",
          "Description": "Remove invalid characters from the end of the string"
        }
      ]
    },
    {
      "$type": "TfsUserMappingEnricherOptions",
      "RefName": "TfsUserMappingEnricher",
      "Enabled": true,
      "UserMappingFile": "C:\\Users\\dkx8myt\\downloads\\Users_IBATeam.json",
      "IdentityFieldsToCheck": [
        "System.AssignedTo",
        "System.ChangedBy",
        "System.CreatedBy",
        "Microsoft.VSTS.Common.ActivatedBy",
        "Microsoft.VSTS.Common.ResolvedBy",
        "Microsoft.VSTS.Common.ClosedBy"
      ]
    }
  ],
  "Processors": [
    {
      "$type": "WorkItemMigrationConfig",
      "Enabled": false,
      "UpdateCreatedDate": true,
      "UpdateCreatedBy": true,
      "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc",
      "FixHtmlAttachmentLinks": false,
      "WorkItemCreateRetryLimit": 5,
      "FilterWorkItemsThatAlreadyExistInTarget": false,
      "PauseAfterEachWorkItem": false,
      "AttachRevisionHistory": false,
      "GenerateMigrationComment": true,
      "WorkItemIDs": null,
      "MaxGracefulFailures": 0,
      "SkipRevisionWithInvalidIterationPath": false,
      "SkipRevisionWithInvalidAreaPath": false
    },
    {
      "$type": "WorkItemPostProcessingConfig",
      "Enabled": false,
      "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc",
      "WorkItemCreateRetryLimit": 5,
      "FilterWorkItemsThatAlreadyExistInTarget": false,
      "PauseAfterEachWorkItem": false
    },
    {
      "$type": "ExportUsersForMappingConfig",
      "Enabled": true,
      "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') AND [System.ChangedDate] > '02-06-2024' ORDER BY [System.ChangedDate] desc",
      "OnlyListUsersInWorkItems": true
    }
  ],
  "Version": "15.0",
  "workaroundForQuerySOAPBugEnabled": false,
  "WorkItemTypeDefinition": {
    "User Story": "Product Backlog Item"
  },
  "Endpoints": {
    "InMemoryWorkItemEndpoints": [
      {
        "Name": "Source",
        "EndpointEnrichers": null
      },
      {
        "Name": "Target",
        "EndpointEnrichers": null
      }
    ]
  }
}

Relevant log output

[12:22:15 DBG] [v15.0.5] TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [targetUsersCount|1316]
[12:22:15 DBG] [v15.0.5] TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|0]
[12:22:15 INF] [v15.0.5] Found 0 total mapped
[12:22:15 INF] [v15.0.5] Filtered to 0 total viable mappings
[12:22:15 INF] [v15.0.5] Writen to: C:\Users\dkx8myt\downloads\Users_IBATeam.json
[12:22:15 INF] [v15.0.5] DONE in {Elapsed} seconds
[12:22:15 INF] [v15.0.5]  Migration Context Complete ExportUsersForMappingContext
[12:22:15 INF] [v15.0.5] Application is shutting down...
[12:22:15 DBG] [v15.0.5] Hosting stopping
[12:22:15 DBG] [v15.0.5] Exiting with return code: 0
[12:22:15 INF] [v15.0.5] Application Ending
[12:22:15 INF] [v15.0.5] The application ran in 00:02:46.3304176 and finished at 06/10/2024 12:22:15
[12:22:15 DBG] [v15.0.5] Hosting stopped

What happened?

All versions of Tool with current TfsUserMappingEnricher (tried 15.0.4 Release, 1.0.5 Preview 72, 1.0.5 Preview 168) are incapable of exporting any Users from DevOps Server (Domain logins). Even if only TfsUserMappingEnricherOptions and ExportUsersForMappingConfig are enabled, it always "filters to 0 viable mappings".

I do not expect the EXPORT of SOURCE users to do any "mapping" to target identities. All I want is a JSON of all Users present in Source WorkItems, which can be used as basis for preparing our own Mapping file (from DevOps Server AD identity do DevOps Services e-mail identity). This worked in Tool version 14.x, where all that happened was:

src/VstsSyncMigrator.Core/Execution/MigrationContext/ExportUsersForMapping.cs

Dictionary<string,string> usersToMap = _TfsUserMappingEnricher.findUsersToMap(sourceWorkItems, _config.IdentityFieldsToCheck);
System.IO.File.WriteAllText(_config.LocalExportJsonFile, Newtonsoft.Json.JsonConvert.SerializeObject(usersToMap));

However, in current implementation, no matter what I configure (I even tried setting the Target DevOps to the same Server instance, no luck - the Enricher is unable to work with AD identities properly, and output is always an empty file).

Debug in Visual Studio

  • Visual Studio Debug
@LudekStipal
Copy link
Author

LudekStipal commented Jun 24, 2024

Still present in current Preview 15.1.1-Preview.11:

[09:35:07 DBG] [v15.1.1] TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [targetUsersCount|1342]
[09:35:07 DBG] [v15.1.1] TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|0]
[09:35:07 INF] [v15.1.1] Found 0 total mapped
[09:35:07 INF] [v15.1.1] Filtered to 0 total viable mappings
[09:35:07 INF] [v15.1.1] Writen to: C:\Users\dkx8myt\downloads\Users_IBATeam.json
[09:35:07 INF] [v15.1.1] DONE in {Elapsed} seconds
[09:35:07 INF] [v15.1.1]  Migration Context Complete ExportUsersForMappingContext
[09:35:07 INF] [v15.1.1] Application is shutting down...
[09:35:07 DBG] [v15.1.1] Hosting stopping
[09:35:07 DBG] [v15.1.1] Exiting with return code: 0
[09:35:07 INF] [v15.1.1] Application Ending
[09:35:07 INF] [v15.1.1] The application ran in 00:05:00.0167004 and finished at 07/01/2024 09:35:07
[09:35:07 DBG] [v15.1.1] Hosting stopped

@LudekStipal
Copy link
Author

LudekStipal commented Jul 15, 2024

The same behaviour in 15.1.3-Preview.2:

[11:39:46 DBG] [v15.1.3.0-Preview.2] TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [targetUsersCount|1343]
[11:39:46 DBG] [v15.1.3.0-Preview.2] TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|0]
[11:39:46 INF] [v15.1.3.0-Preview.2] Found 0 total mapped
[11:39:46 INF] [v15.1.3.0-Preview.2] Filtered to 0 total viable mappings
[11:39:46 INF] [v15.1.3.0-Preview.2] Writen to: C:\Users\dkx8myt\downloads\Users_IBATeam.json
[11:39:46 INF] [v15.1.3.0-Preview.2] DONE in {Elapsed} seconds
[11:39:46 INF] [v15.1.3.0-Preview.2]  Migration Context Complete ExportUsersForMappingContext
[11:39:46 INF] [v15.1.3.0-Preview.2] Application is shutting down...
[11:39:46 DBG] [v15.1.3.0-Preview.2] Hosting stopping
[11:39:46 DBG] [v15.1.3.0-Preview.2] Exiting with return code: 0
[11:39:46 INF] [v15.1.3.0-Preview.2] Application Ending
[11:39:46 INF] [v15.1.3.0-Preview.2] The application ran in 00:06:10.1517818 and finished at 07/15/2024 11:39:46

and 15.1.4-Preview.6:

[10:29:28 DBG] [v15.1.4.0-Preview.6] TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [targetUsersCount|1343]
[10:29:28 DBG] [v15.1.4.0-Preview.6] TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|0]
[10:29:28 INF] [v15.1.4.0-Preview.6] Found 0 total mapped
[10:29:28 INF] [v15.1.4.0-Preview.6] Filtered to 0 total viable mappings
[10:29:28 INF] [v15.1.4.0-Preview.6] Writen to: C:\Users\dkx8myt\downloads\Users_IBATeam.json
[10:29:28 INF] [v15.1.4.0-Preview.6] DONE in {Elapsed} seconds
[10:29:28 INF] [v15.1.4.0-Preview.6]  Migration Context Complete ExportUsersForMappingContext
[10:29:28 INF] [v15.1.4.0-Preview.6] Application is shutting down...
[10:29:28 DBG] [v15.1.4.0-Preview.6] Hosting stopping
[10:29:28 DBG] [v15.1.4.0-Preview.6] Exiting with return code: 0
[10:29:28 INF] [v15.1.4.0-Preview.6] Application Ending
[10:29:28 INF] [v15.1.4.0-Preview.6] The application ran in 00:04:00.5554693 and finished at 07/18/2024 10:29:28
[10:29:28 DBG] [v15.1.4.0-Preview.6] Hosting stopped

@MrHinsh
Copy link
Member

MrHinsh commented Jul 25, 2024

@LudekStipal I think you are going to have to debug. I do not have access to a TFS server connected to AD!

We can run a debug session together if you need!

@LudekStipal
Copy link
Author

Hi @MrHinsh , sorry for the late reply, I was on planned holiday and just returned. I ran Debug session using the latest Code version (Repo clone, reports itself as [15.1.8-Local.9]) and was able to narrow down the issue: the Identities from Source (DevOps Server 2022) are never loaded (empty), so there can be no mapping file produced.

The problem lies within:

\src\MigrationTools.Clients.AzureDevops.ObjectModel\ProcessorEnrichers\TfsUserMappingEnricher.cs

private List<IdentityItemData> GetUsersListFromServer(IGroupSecurityService gss)

This works fine for Azure DevOps Services (Cloud); However, when executing gss.ReadIdentity() against Azure DevOps Server 2022 (Update 2, but same behaviour was in Update 1), the filter:

var people = SIDS.Members.ToList().Where(x => x.Contains("\\")).Select(x => x);

always returns empty array, so nothing can be mapped / exported.

The SIDS return from DevOps Server do not contain a single entry with "backslash", they are all SIDs - either starting with S-1-5-xxx, S-1-9-xxx, or in [Guid]:Build:[Guid] Format:

DevOpsServer_SIDs

So the following logic - ReadIdentity with SaerchFactor.AccountName - is also not usable for this case.

I am currently not sure which "SIDs" belong tu Users, but it seems this modified function works for both DevOps Services and DevOps Server:

        private List<IdentityItemData> GetUsersListFromServer(IGroupSecurityService gss)
        {
            Identity SIDS = gss.ReadIdentity(SearchFactor.AccountName, "Project Collection Valid Users", QueryMembership.Expanded);
            var people = SIDS.Members.ToList().Where(x => x.Contains("\\") || x.StartsWith("S-1-5-")).Select(x => x);

            List<IdentityItemData> foundUsers = new List<IdentityItemData>();
            Log.LogTrace("TfsUserMappingEnricher::GetUsersListFromServer:foundUsers\\ {@foundUsers}", foundUsers);
            foreach (string user in people)
            {
                Log.LogDebug("TfsUserMappingEnricher::GetUsersListFromServer::[user:{user}] Atempting to load user", user);
                try
                {
                    var bits = user.Split('\\');
                    Identity sids = bits.Length > 1 ? gss.ReadIdentity(SearchFactor.AccountName, bits[1], QueryMembership.Expanded) : gss.ReadIdentity(SearchFactor.Sid, bits[0], QueryMembership.Expanded);
 
                    if (sids != null)
                    {
                        if (!foundUsers.Where(x => x.FriendlyName == sids.DisplayName).Any())
                            foundUsers.Add(new IdentityItemData() { FriendlyName = sids.DisplayName, AccountName = sids.AccountName });
                    }
                    else
                    {
                        Log.LogDebug("TfsUserMappingEnricher::GetUsersListFromServer::[user:{user}] ReadIdentity returned null for {@bits}", user, bits);
                    }

                }
                catch (Exception ex)
                {
                    Telemetry.TrackException(ex, null, null);
                    Log.LogWarning("TfsUserMappingEnricher::GetUsersListFromServer::[user:{user}] Failed With {Exception}", user, ex.Message);
                }

            }
            return foundUsers;
        }
  • People filter: added x.StartsWith("S-1-5-")
  • Read Identities either by AccountName, or SID:

Identity sids = bits.Length > 1 ? gss.ReadIdentity(SearchFactor.AccountName, bits[1], QueryMembership.Expanded) : gss.ReadIdentity(SearchFactor.Sid, bits[0], QueryMembership.Expanded);

  • Also adding a simple "distinct" filter when adding to output array was necessary (most likely due to technical accounts / users in multiple Domains with the same DisplayName), otherwise following mapping to targetUsers failed:

if (!foundUsers.Where(x => x.FriendlyName == sids.DisplayName).Any())

With this modification, the output file contains expected result (DisplayNames of Users in WorkItems mapped to "null": this is what we need, as the Projects need to supply their own correct Identities (e-mail addresses) for the Mapping).

@LudekStipal
Copy link
Author

New Path for file containing GetUsersListFromServer function in MigrationTools v16.0.5-Preview.4

src\MigrationTools.Clients.TfsObjectModel\Tools\TfsUserMappingTool.cs

The problem / possible solution remains the same as in my previous post.

@satano
Copy link
Contributor

satano commented Nov 22, 2024

@LudekStipal I have fixed this in #2522 Not sure if it is released out in latest preview version, but from the sources it will work for you.

@MrHinsh I think you can close this issue.

@MrHinsh
Copy link
Member

MrHinsh commented Nov 22, 2024

It's in preview. Not sure if it's shipped yet as Winget are quite slow... But choco is usually quick.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
Development

No branches or pull requests

4 participants
@satano @MrHinsh @LudekStipal and others