[PM-19145] refactor organization service.import async (#5800)

* initial lift and shift

* extract function RemoveExistingExternalUsers

* Extract function RemoveExistingUsers()

* extract function OverwriteExisting()

* create new model for sync data

* extract add users to function, rename

* rename OrganizatinUserInvite for command, implement command

* implement command

* refactor groups logic

* fix imports

* remove old tests, fix imports

* fix namespace

* fix CommandResult useage

* tests wip

* wip

* wip

* remove redundant code, remove looping db call, refactor tests

* clean up

* remove looping db call with bulk method

* clean up

* remove orgId param to use id already in request

* change param

* cleanup params

* remove IReferenceEventService

* fix test

* fix tests

* cr feedback

* remove _timeProvider

* add xmldoc, refactor to make InviteOrganizationUsersCommand vNext instead of default

* switch back to command

* re-add old ImportAsync impl

* fix test

* add feature flag

* cleanup

* clean up

* fix tests

* wip

* wip

* add api integration tests for users WIP

* groups integration tests

* cleanup

* fix error from merging main

* fix tests

* cr feedback

* fix test

* fix test

test/Core.Test/AdminConsole/Models/InviteOrganizationUsersRequestTests.cs

@@ -16,7 +16,7 @@ public class InviteOrganizationUsersRequestTests
     public void Constructor_WhenPassedInvalidEmail_ThrowsException(string email, OrganizationUserType type, Permissions permissions, string externalId)
     {
         var exception = Assert.Throws<BadRequestException>(() =>
-            new OrganizationUserInvite(email, [], [], type, permissions, externalId, false));
+            new OrganizationUserInviteCommandModel(email, [], [], type, permissions, externalId, false));
 
         Assert.Contains(InvalidEmailErrorMessage, exception.Message);
     }
@@ -33,7 +33,7 @@ public class InviteOrganizationUsersRequestTests
         };
 
         var exception = Assert.Throws<BadRequestException>(() =>
-            new OrganizationUserInvite(
+            new OrganizationUserInviteCommandModel(
                 email: validEmail,
                 assignedCollections: [invalidCollectionConfiguration],
                 groups: [],
@@ -51,7 +51,7 @@ public class InviteOrganizationUsersRequestTests
         const string validEmail = "test@email.com";
         var validCollectionConfiguration = new CollectionAccessSelection { Id = Guid.NewGuid(), Manage = true };
 
-        var invite = new OrganizationUserInvite(
+        var invite = new OrganizationUserInviteCommandModel(
             email: validEmail,
             assignedCollections: [validCollectionConfiguration],
             groups: [],

test/Core.Test/AdminConsole/OrganizationFeatures/Import/ImportOrganizationUsersAndGroupsCommandTests.cs

@@ -0,0 +1,215 @@
+using Bit.Core.AdminConsole.Models.Business;
+using Bit.Core.AdminConsole.OrganizationFeatures.OrganizationUsers;
+using Bit.Core.Auth.Models.Business.Tokenables;
+using Bit.Core.Context;
+using Bit.Core.Entities;
+using Bit.Core.Enums;
+using Bit.Core.Models.Business;
+using Bit.Core.Models.Data;
+using Bit.Core.Models.Data.Organizations.OrganizationUsers;
+using Bit.Core.Repositories;
+using Bit.Core.Services;
+using Bit.Core.Test.AutoFixture.OrganizationFixtures;
+using Bit.Core.Tokens;
+using Bit.Test.Common.AutoFixture;
+using Bit.Test.Common.AutoFixture.Attributes;
+using Bit.Test.Common.Fakes;
+using NSubstitute;
+using Xunit;
+using Organization = Bit.Core.AdminConsole.Entities.Organization;
+
+namespace Bit.Core.Test.OrganizationFeatures.OrganizationUsers;
+
+public class ImportOrganizationUsersAndGroupsCommandTests
+{
+
+    private readonly IDataProtectorTokenFactory<OrgUserInviteTokenable> _orgUserInviteTokenDataFactory = new FakeDataProtectorTokenFactory<OrgUserInviteTokenable>();
+
+    [Theory, PaidOrganizationCustomize, BitAutoData]
+    public async Task OrgImportCallsInviteOrgUserCommand(
+            SutProvider<ImportOrganizationUsersAndGroupsCommand> sutProvider,
+            Organization org,
+            List<OrganizationUserUserDetails> existingUsers,
+            List<ImportedOrganizationUser> importedUsers,
+            List<ImportedGroup> newGroups)
+    {
+        SetupOrganizationConfigForImport(sutProvider, org, existingUsers, importedUsers);
+
+        var orgUsers = new List<OrganizationUser>();
+
+        // fix mocked email format, mock OrganizationUsers.
+        foreach (var u in importedUsers)
+        {
+            u.Email += "@bitwardentest.com";
+            orgUsers.Add(new OrganizationUser { Email = u.Email, ExternalId = u.ExternalId });
+        }
+
+        importedUsers.Add(new ImportedOrganizationUser
+        {
+            Email = existingUsers.First().Email,
+            ExternalId = existingUsers.First().ExternalId
+        });
+
+
+        existingUsers.First().Type = OrganizationUserType.Owner;
+
+        sutProvider.GetDependency<IOrganizationRepository>().GetByIdAsync(org.Id).Returns(org);
+
+        var organizationUserRepository = sutProvider.GetDependency<IOrganizationUserRepository>();
+        SetupOrgUserRepositoryCreateManyAsyncMock(organizationUserRepository);
+
+        sutProvider.GetDependency<IPaymentService>().HasSecretsManagerStandalone(org).Returns(true);
+        sutProvider.GetDependency<IOrganizationUserRepository>().GetManyDetailsByOrganizationAsync(org.Id).Returns(existingUsers);
+        sutProvider.GetDependency<IOrganizationRepository>().GetOccupiedSeatCountByOrganizationIdAsync(org.Id).Returns(
+            new OrganizationSeatCounts
+            {
+                Users = existingUsers.Count,
+                Sponsored = 0
+            });
+        sutProvider.GetDependency<ICurrentContext>().ManageUsers(org.Id).Returns(true);
+        sutProvider.GetDependency<IOrganizationService>().InviteUsersAsync(org.Id, Guid.Empty, EventSystemUser.PublicApi,
+                Arg.Any<IEnumerable<(OrganizationUserInvite, string)>>())
+            .Returns(orgUsers);
+
+        await sutProvider.Sut.ImportAsync(org.Id, newGroups, importedUsers, new List<string>(), false);
+
+        var expectedNewUsersCount = importedUsers.Count - 1;
+
+        await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
+            .UpsertAsync(default);
+        await sutProvider.GetDependency<IOrganizationUserRepository>().Received(1)
+            .UpsertManyAsync(Arg.Is<IEnumerable<OrganizationUser>>(users => !users.Any()));
+        await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
+            .CreateAsync(default);
+
+        // Send Invites
+        await sutProvider.GetDependency<IOrganizationService>().Received(1).
+            InviteUsersAsync(org.Id, Guid.Empty, EventSystemUser.PublicApi,
+                    Arg.Is<IEnumerable<(OrganizationUserInvite, string)>>(invites => invites.Count() == expectedNewUsersCount));
+
+        // Send events
+        await sutProvider.GetDependency<IEventService>().Received(1)
+            .LogOrganizationUserEventsAsync(Arg.Any<IEnumerable<(OrganizationUserUserDetails, EventType, EventSystemUser, DateTime?)>>());
+    }
+
+    [Theory, PaidOrganizationCustomize, BitAutoData]
+    public async Task OrgImportCreateNewUsersAndMarryExistingUser(
+            SutProvider<ImportOrganizationUsersAndGroupsCommand> sutProvider,
+            Organization org,
+            List<OrganizationUserUserDetails> existingUsers,
+            List<ImportedOrganizationUser> importedUsers,
+            List<ImportedGroup> newGroups)
+    {
+        SetupOrganizationConfigForImport(sutProvider, org, existingUsers, importedUsers);
+
+        var orgUsers = new List<OrganizationUser>();
+        var reInvitedUser = existingUsers.First();
+        // Existing user has no external ID. This will make the SUT call UpsertManyAsync
+        reInvitedUser.ExternalId = "";
+
+        // Mock an existing org user for this "existing" user
+        var reInvitedOrgUser = new OrganizationUser { Email = reInvitedUser.Email, Id = reInvitedUser.Id };
+
+        // fix email formatting, mock orgUsers to be returned
+        foreach (var u in existingUsers)
+        {
+            u.Email += "@bitwardentest.com";
+            orgUsers.Add(new OrganizationUser { Email = u.Email, ExternalId = u.ExternalId });
+        }
+        foreach (var u in importedUsers)
+        {
+            u.Email += "@bitwardentest.com";
+            orgUsers.Add(new OrganizationUser { Email = u.Email, ExternalId = u.ExternalId });
+        }
+
+        // add the existing user to be re-imported
+        importedUsers.Add(new ImportedOrganizationUser
+        {
+            Email = reInvitedUser.Email,
+            ExternalId = reInvitedUser.Email,
+        });
+
+        var expectedNewUsersCount = importedUsers.Count - 1;
+
+        sutProvider.GetDependency<IOrganizationRepository>().GetByIdAsync(org.Id).Returns(org);
+
+        var organizationUserRepository = sutProvider.GetDependency<IOrganizationUserRepository>();
+        SetupOrgUserRepositoryCreateManyAsyncMock(organizationUserRepository);
+
+        sutProvider.GetDependency<IOrganizationUserRepository>().GetManyAsync(Arg.Any<IEnumerable<Guid>>())
+            .Returns(new List<OrganizationUser>([reInvitedOrgUser]));
+        sutProvider.GetDependency<IOrganizationUserRepository>().GetManyDetailsByOrganizationAsync(org.Id).Returns(existingUsers);
+        sutProvider.GetDependency<IOrganizationRepository>().GetOccupiedSeatCountByOrganizationIdAsync(org.Id).Returns(
+            new OrganizationSeatCounts
+            {
+                Users = existingUsers.Count,
+                Sponsored = 0
+            });
+
+        sutProvider.GetDependency<IOrganizationService>().InviteUsersAsync(org.Id, Guid.Empty, EventSystemUser.PublicApi,
+                Arg.Any<IEnumerable<(OrganizationUserInvite, string)>>())
+            .Returns(orgUsers);
+
+        await sutProvider.Sut.ImportAsync(org.Id, newGroups, importedUsers, new List<string>(), false);
+
+        await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
+            .UpsertAsync(default);
+        await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
+            .CreateAsync(default);
+        await sutProvider.GetDependency<IOrganizationUserRepository>().DidNotReceiveWithAnyArgs()
+            .CreateAsync(default, default);
+
+        // Upserted existing user
+        await sutProvider.GetDependency<IOrganizationUserRepository>().Received(1)
+            .UpsertManyAsync(Arg.Is<IEnumerable<OrganizationUser>>(users => users.Count() == 1 && users.First() == reInvitedOrgUser));
+
+        // Send Invites
+        await sutProvider.GetDependency<IOrganizationService>().Received(1).
+            InviteUsersAsync(org.Id, Guid.Empty, EventSystemUser.PublicApi,
+                    Arg.Is<IEnumerable<(OrganizationUserInvite, string)>>(invites => invites.Count() == expectedNewUsersCount));
+
+        // Send events
+        await sutProvider.GetDependency<IEventService>().Received(1)
+            .LogOrganizationUserEventsAsync(Arg.Any<IEnumerable<(OrganizationUserUserDetails, EventType, EventSystemUser, DateTime?)>>());
+    }
+
+    private void SetupOrganizationConfigForImport(
+            SutProvider<ImportOrganizationUsersAndGroupsCommand> sutProvider,
+            Organization org,
+            List<OrganizationUserUserDetails> existingUsers,
+            List<ImportedOrganizationUser> importedUsers)
+    {
+        // Setup FakeDataProtectorTokenFactory for creating new tokens - this must come first in order to avoid resetting mocks
+        sutProvider.SetDependency(_orgUserInviteTokenDataFactory, "orgUserInviteTokenDataFactory");
+        sutProvider.Create();
+
+        org.UseDirectory = true;
+        org.Seats = importedUsers.Count + existingUsers.Count + 1;
+    }
+
+    // Must set real guids in order for dictionary of guids to not throw aggregate exceptions
+    private void SetupOrgUserRepositoryCreateManyAsyncMock(IOrganizationUserRepository organizationUserRepository)
+    {
+        organizationUserRepository.CreateManyAsync(Arg.Any<IEnumerable<OrganizationUser>>()).Returns(
+            info =>
+            {
+                var orgUsers = info.Arg<IEnumerable<OrganizationUser>>();
+                foreach (var orgUser in orgUsers)
+                {
+                    orgUser.Id = Guid.NewGuid();
+                }
+
+                return Task.FromResult<ICollection<Guid>>(orgUsers.Select(u => u.Id).ToList());
+            }
+        );
+
+        organizationUserRepository.CreateAsync(Arg.Any<OrganizationUser>(), Arg.Any<IEnumerable<CollectionAccessSelection>>()).Returns(
+            info =>
+            {
+                var orgUser = info.Arg<OrganizationUser>();
+                orgUser.Id = Guid.NewGuid();
+                return Task.FromResult<Guid>(orgUser.Id);
+            }
+        );
+    }
+}

test/Core.Test/AdminConsole/OrganizationFeatures/OrganizationUsers/InviteUsers/InviteOrganizationUserCommandTests.cs

@@ -30,7 +30,6 @@ using NSubstitute;
 using NSubstitute.ExceptionExtensions;
 using Xunit;
 using static Bit.Core.Test.AdminConsole.OrganizationFeatures.OrganizationUsers.InviteUsers.Helpers.InviteUserOrganizationValidationRequestHelpers;
-using OrganizationUserInvite = Bit.Core.AdminConsole.OrganizationFeatures.OrganizationUsers.InviteUsers.Models.OrganizationUserInvite;
 
 namespace Bit.Core.Test.AdminConsole.OrganizationFeatures.OrganizationUsers.InviteUsers;
 
@@ -54,7 +53,7 @@ public class InviteOrganizationUserCommandTests
 
         var request = new InviteOrganizationUsersRequest(
             invites: [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -112,7 +111,7 @@ public class InviteOrganizationUserCommandTests
 
         var request = new InviteOrganizationUsersRequest(
             invites: [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: orgUser.Email,
                     assignedCollections: [],
                     groups: [],
@@ -182,7 +181,7 @@ public class InviteOrganizationUserCommandTests
 
         var request = new InviteOrganizationUsersRequest(
             invites: [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -257,7 +256,7 @@ public class InviteOrganizationUserCommandTests
 
         var request = new InviteOrganizationUsersRequest(
             invites: [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -334,7 +333,7 @@ public class InviteOrganizationUserCommandTests
         var request = new InviteOrganizationUsersRequest(
             invites:
             [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -411,7 +410,7 @@ public class InviteOrganizationUserCommandTests
 
         var request = new InviteOrganizationUsersRequest(
             invites: [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -492,7 +491,7 @@ public class InviteOrganizationUserCommandTests
 
         var request = new InviteOrganizationUsersRequest(
             invites: [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -566,7 +565,7 @@ public class InviteOrganizationUserCommandTests
 
         var request = new InviteOrganizationUsersRequest(
             invites: [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -669,7 +668,7 @@ public class InviteOrganizationUserCommandTests
 
         var request = new InviteOrganizationUsersRequest(
             invites: [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -768,7 +767,7 @@ public class InviteOrganizationUserCommandTests
 
         var request = new InviteOrganizationUsersRequest(
             invites: [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -863,7 +862,7 @@ public class InviteOrganizationUserCommandTests
         var request = new InviteOrganizationUsersRequest(
             invites:
             [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],
@@ -942,7 +941,7 @@ public class InviteOrganizationUserCommandTests
         var request = new InviteOrganizationUsersRequest(
             invites:
             [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: user.Email,
                     assignedCollections: [],
                     groups: [],

test/Core.Test/AdminConsole/OrganizationFeatures/OrganizationUsers/InviteUsers/Validation/InviteOrganizationUsersValidatorTests.cs

@@ -14,7 +14,6 @@ using Bit.Test.Common.AutoFixture.Attributes;
 using NSubstitute;
 using NSubstitute.ExceptionExtensions;
 using Xunit;
-using OrganizationUserInvite = Bit.Core.AdminConsole.OrganizationFeatures.OrganizationUsers.InviteUsers.Models.OrganizationUserInvite;
 
 namespace Bit.Core.Test.AdminConsole.OrganizationFeatures.OrganizationUsers.InviteUsers.Validation;
 
@@ -36,13 +35,13 @@ public class InviteOrganizationUsersValidatorTests
         {
             Invites =
             [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: "test@email.com",
                     externalId: "test-external-id"),
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: "test2@email.com",
                     externalId: "test-external-id2"),
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: "test3@email.com",
                     externalId: "test-external-id3")
             ],
@@ -82,13 +81,13 @@ public class InviteOrganizationUsersValidatorTests
         {
             Invites =
             [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: "test@email.com",
                     externalId: "test-external-id"),
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: "test2@email.com",
                     externalId: "test-external-id2"),
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: "test3@email.com",
                     externalId: "test-external-id3")
             ],
@@ -126,13 +125,13 @@ public class InviteOrganizationUsersValidatorTests
         {
             Invites =
             [
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: "test@email.com",
                     externalId: "test-external-id"),
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: "test2@email.com",
                     externalId: "test-external-id2"),
-                new OrganizationUserInvite(
+                new OrganizationUserInviteCommandModel(
                     email: "test3@email.com",
                     externalId: "test-external-id3")
             ],