[PM-22219] - [Vault] [Server] Exclude items in default collections from Admin Console (#5992)

* add GetAllOrganizationCiphersExcludingDefaultUserCollections

* add sproc

* update sproc and feature flag name

* add sproc. update tests

* rename sproc

* rename sproc

* use single sproc

* revert change

* remove unused code. update sproc

* remove joins from proc

* update migration filename

* fix syntax

* fix indentation

* remove unnecessary feature flag and go statements. clean up code

* update sproc, view, and index

* update sproc

* update index

* update timestamp

* update filename. update sproc to match EF filter

* match only enabled organizations. make index creation idempotent

* update file timestamp

* update timestamp

* use square brackets

* add square brackets

* formatting fixes

* rename view

* remove index

src/Api/Vault/Controllers/CiphersController.cs

@@ -48,6 +48,7 @@ public class CiphersController : Controller
     private readonly IOrganizationCiphersQuery _organizationCiphersQuery;
     private readonly IApplicationCacheService _applicationCacheService;
     private readonly ICollectionRepository _collectionRepository;
+    private readonly IFeatureService _featureService;
 
     public CiphersController(
         ICipherRepository cipherRepository,
@@ -61,7 +62,8 @@ public class CiphersController : Controller
         GlobalSettings globalSettings,
         IOrganizationCiphersQuery organizationCiphersQuery,
         IApplicationCacheService applicationCacheService,
-        ICollectionRepository collectionRepository)
+        ICollectionRepository collectionRepository,
+        IFeatureService featureService)
     {
         _cipherRepository = cipherRepository;
         _collectionCipherRepository = collectionCipherRepository;
@@ -75,6 +77,7 @@ public class CiphersController : Controller
         _organizationCiphersQuery = organizationCiphersQuery;
         _applicationCacheService = applicationCacheService;
         _collectionRepository = collectionRepository;
+        _featureService = featureService;
     }
 
     [HttpGet("{id}")]
@@ -314,8 +317,11 @@ public class CiphersController : Controller
         {
             throw new NotFoundException();
         }
-
+        var allOrganizationCiphers = _featureService.IsEnabled(FeatureFlagKeys.CreateDefaultLocation)
-        var allOrganizationCiphers = await _organizationCiphersQuery.GetAllOrganizationCiphers(organizationId);
+        ?
+            await _organizationCiphersQuery.GetAllOrganizationCiphersExcludingDefaultUserCollections(organizationId)
+        :
+            await _organizationCiphersQuery.GetAllOrganizationCiphers(organizationId);
 
         var allOrganizationCipherResponses =
             allOrganizationCiphers.Select(c =>

src/Core/Vault/Queries/IOrganizationCiphersQuery.cs

@@ -37,4 +37,10 @@ public interface IOrganizationCiphersQuery
     /// </remarks>
     public Task<IEnumerable<CipherOrganizationDetailsWithCollections>> GetOrganizationCiphersByCollectionIds(
         Guid organizationId, IEnumerable<Guid> collectionIds);
+
+    /// <summary>
+    /// Returns all organization ciphers except those in default user collections.
+    /// </summary>
+    public Task<IEnumerable<CipherOrganizationDetailsWithCollections>>
+        GetAllOrganizationCiphersExcludingDefaultUserCollections(Guid organizationId);
 }

src/Core/Vault/Queries/OrganizationCiphersQuery.cs

@@ -61,4 +61,10 @@ public class OrganizationCiphersQuery : IOrganizationCiphersQuery
         var allOrganizationCiphers = await GetAllOrganizationCiphers(organizationId);
         return allOrganizationCiphers.Where(c => c.CollectionIds.Intersect(managedCollectionIds).Any());
     }
+
+    public async Task<IEnumerable<CipherOrganizationDetailsWithCollections>>
+        GetAllOrganizationCiphersExcludingDefaultUserCollections(Guid orgId)
+    {
+        return (await _cipherRepository.GetManyCipherOrganizationDetailsExcludingDefaultCollectionsAsync(orgId)).ToList();
+    }
 }

src/Core/Vault/Repositories/ICipherRepository.cs

@@ -84,6 +84,12 @@ public interface ICipherRepository : IRepository<Cipher, Guid>
     /// <param name="ciphers">A list of ciphers with updated data</param>
     UpdateEncryptedDataForKeyRotation UpdateForKeyRotation(Guid userId,
         IEnumerable<Cipher> ciphers);
+
+    /// <summary>
+    /// Returns all ciphers belonging to the organization excluding those with default collections
+    /// </summary>
+    Task<IEnumerable<CipherOrganizationDetailsWithCollections>>
+    GetManyCipherOrganizationDetailsExcludingDefaultCollectionsAsync(Guid organizationId);
     /// <inheritdoc cref="UpdateForKeyRotation(Guid, IEnumerable{Cipher})"/>
     /// <remarks>
     /// This version uses the bulk resource creation service to create the temp table.

src/Infrastructure.Dapper/Vault/Repositories/CipherRepository.cs

@@ -7,6 +7,7 @@ using Bit.Core.Entities;
 using Bit.Core.KeyManagement.UserKey;
 using Bit.Core.Settings;
 using Bit.Core.Tools.Entities;
+using Bit.Core.Utilities;
 using Bit.Core.Vault.Entities;
 using Bit.Core.Vault.Models.Data;
 using Bit.Core.Vault.Repositories;
@@ -867,6 +868,47 @@ public class CipherRepository : Repository<Cipher, Guid>, ICipherRepository
         }
     }
 
+    public async Task<IEnumerable<CipherOrganizationDetailsWithCollections>>
+        GetManyCipherOrganizationDetailsExcludingDefaultCollectionsAsync(Guid orgId)
+    {
+        await using var connection = new SqlConnection(ConnectionString);
+
+        var dict = new Dictionary<Guid, CipherOrganizationDetailsWithCollections>();
+        var tempCollections = new Dictionary<Guid, List<Guid>>();
+
+        await connection.QueryAsync<CipherOrganizationDetailsWithCollections, CollectionCipher, CipherOrganizationDetailsWithCollections>(
+            $"[{Schema}].[CipherOrganizationDetails_ReadByOrganizationIdExcludingDefaultCollections]",
+            (cipher, cc) =>
+            {
+                if (!dict.TryGetValue(cipher.Id, out var details))
+                {
+                    details = new CipherOrganizationDetailsWithCollections(cipher, /*dummy*/null);
+                    dict.Add(cipher.Id, details);
+                    tempCollections[cipher.Id] = new List<Guid>();
+                }
+
+                if (cc?.CollectionId != null)
+                {
+                    tempCollections[cipher.Id].AddIfNotExists(cc.CollectionId);
+                }
+
+                return details;
+            },
+            new { OrganizationId = orgId },
+            splitOn: "CollectionId",
+            commandType: CommandType.StoredProcedure
+        );
+
+        // now assign each List<Guid> back to the array property in one shot
+        foreach (var kv in dict)
+        {
+            kv.Value.CollectionIds = tempCollections[kv.Key].ToArray();
+        }
+
+        return dict.Values.ToList();
+    }
+
+
     private DataTable BuildCiphersTable(SqlBulkCopy bulkCopy, IEnumerable<Cipher> ciphers)
     {
         var c = ciphers.FirstOrDefault();

src/Infrastructure.EntityFramework/Vault/Repositories/CipherRepository.cs

@@ -4,6 +4,7 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using AutoMapper;
+using Bit.Core.Enums;
 using Bit.Core.KeyManagement.UserKey;
 using Bit.Core.Utilities;
 using Bit.Core.Vault.Enums;
@@ -1001,6 +1002,55 @@ public class CipherRepository : Repository<Core.Vault.Entities.Cipher, Cipher, G
         };
     }
 
+    public async Task<IEnumerable<CipherOrganizationDetailsWithCollections>>
+        GetManyCipherOrganizationDetailsExcludingDefaultCollectionsAsync(Guid organizationId)
+    {
+        using var scope = ServiceScopeFactory.CreateScope();
+        var dbContext = GetDatabaseContext(scope);
+
+        var defaultTypeInt = (int)CollectionType.DefaultUserCollection;
+
+        //  filter out any cipher that belongs *only* to default collections
+        //    i.e. keep ciphers with no collections, or with ≥1 non-default collection
+        var query = from c in dbContext.Ciphers.AsNoTracking()
+                    where c.UserId == null
+                       && c.OrganizationId == organizationId
+                       && c.Organization.Enabled
+                       && (
+                            c.CollectionCiphers.Count() == 0
+                            || c.CollectionCiphers.Any(cc => (int)cc.Collection.Type != defaultTypeInt)
+                          )
+                    select new CipherOrganizationDetailsWithCollections(
+                        new CipherOrganizationDetails
+                        {
+                            Id = c.Id,
+                            UserId = c.UserId,
+                            OrganizationId = c.OrganizationId,
+                            Type = c.Type,
+                            Data = c.Data,
+                            Favorites = c.Favorites,
+                            Folders = c.Folders,
+                            Attachments = c.Attachments,
+                            CreationDate = c.CreationDate,
+                            RevisionDate = c.RevisionDate,
+                            DeletedDate = c.DeletedDate,
+                            Reprompt = c.Reprompt,
+                            Key = c.Key,
+                            OrganizationUseTotp = c.Organization.UseTotp
+                        },
+                        new Dictionary<Guid, IGrouping<Guid, Bit.Core.Entities.CollectionCipher>>()
+                    )
+                    {
+                        CollectionIds = c.CollectionCiphers
+                            .Where(cc => (int)cc.Collection.Type != defaultTypeInt)
+                            .Select(cc => cc.CollectionId)
+                            .ToArray()
+                    };
+
+        var result = await query.ToListAsync();
+        return result;
+    }
+
     /// <inheritdoc cref="UpdateForKeyRotation(Guid, IEnumerable{Cipher})"/>
     /// <remarks>
     /// EF does not use the bulk resource creation service, so we need to use the regular update method.

src/Sql/dbo/Stored Procedures/CipherOrganizationDetails_ReadByOrganizationIdExcludingDefaultCollections.sql

@@ -0,0 +1,39 @@
+ -- Stored procedure that filters out ciphers that ONLY belong to default collections
+CREATE PROCEDURE
+  [dbo].[CipherOrganizationDetails_ReadByOrganizationIdExcludingDefaultCollections]
+      @OrganizationId UNIQUEIDENTIFIER
+  AS
+  BEGIN
+      SET NOCOUNT ON;
+
+      WITH [NonDefaultCiphers] AS (
+          SELECT DISTINCT [Id]
+          FROM [dbo].[OrganizationCipherDetailsCollectionsView]
+          WHERE [OrganizationId] = @OrganizationId
+              AND ([CollectionId] IS NULL
+              OR [CollectionType] <> 1)
+      )
+
+      SELECT
+          V.[Id],
+          V.[UserId],
+          V.[OrganizationId],
+          V.[Type],
+          V.[Data],
+          V.[Favorites],
+          V.[Folders],
+          V.[Attachments],
+          V.[CreationDate],
+          V.[RevisionDate],
+          V.[DeletedDate],
+          V.[Reprompt],
+          V.[Key],
+          V.[OrganizationUseTotp],
+          V.[CollectionId]  -- For Dapper splitOn parameter
+      FROM [dbo].[OrganizationCipherDetailsCollectionsView] V
+      INNER JOIN [NonDefaultCiphers] NDC ON V.[Id] = NDC.[Id]
+      WHERE V.[OrganizationId] = @OrganizationId
+          AND (V.[CollectionId] IS NULL OR V.[CollectionType] <> 1)
+      ORDER BY V.[RevisionDate] DESC;
+  END;
+  GO

src/Sql/dbo/Vault/Tables/Cipher.sql

@@ -34,3 +34,4 @@ GO
 CREATE NONCLUSTERED INDEX [IX_Cipher_DeletedDate]
     ON [dbo].[Cipher]([DeletedDate] ASC);
 
+GO

src/Sql/dbo/Views/OrganizationCipherDetailsCollectionsView.sql

@@ -0,0 +1,28 @@
+CREATE VIEW [dbo].[OrganizationCipherDetailsCollectionsView]
+AS
+    SELECT
+      C.[Id],
+      C.[UserId],
+      C.[OrganizationId],
+      C.[Type],
+      C.[Data],
+      C.[Attachments],
+      C.[Favorites],
+      C.[Folders],
+      C.[CreationDate],
+      C.[RevisionDate],
+      C.[DeletedDate],
+      C.[Reprompt],
+      C.[Key],
+      CASE
+          WHEN O.[UseTotp] = 1 THEN 1
+          ELSE 0
+      END AS [OrganizationUseTotp],
+      CC.[CollectionId],
+      COL.[Type] AS [CollectionType]
+    FROM [dbo].[Cipher] C
+    INNER JOIN [dbo].[Organization] O ON C.[OrganizationId] = O.[Id]
+    LEFT JOIN [dbo].[CollectionCipher] CC ON CC.[CipherId] = C.[Id]
+    LEFT JOIN [dbo].[Collection] COL ON CC.[CollectionId] = COL.[Id]
+    WHERE C.[UserId] IS NULL -- Organization ciphers only
+      AND O.[Enabled] = 1; -- Only enabled organizations

test/Core.Test/Vault/Queries/OrganizationCiphersQueryTests.cs

@@ -89,4 +89,47 @@ public class OrganizationCiphersQueryTests
             c.CollectionIds.Any(cId => cId == targetCollectionId) &&
             c.CollectionIds.Any(cId => cId == otherCollectionId));
     }
+
+
+    [Theory, BitAutoData]
+    public async Task GetAllOrganizationCiphersExcludingDefaultUserCollections_DelegatesToRepository(
+         Guid organizationId,
+         SutProvider<OrganizationCiphersQuery> sutProvider)
+    {
+        var item1 = new CipherOrganizationDetailsWithCollections(
+            new CipherOrganizationDetails { Id = Guid.NewGuid(), OrganizationId = organizationId },
+            new Dictionary<Guid, IGrouping<Guid, CollectionCipher>>());
+        var item2 = new CipherOrganizationDetailsWithCollections(
+            new CipherOrganizationDetails { Id = Guid.NewGuid(), OrganizationId = organizationId },
+            new Dictionary<Guid, IGrouping<Guid, CollectionCipher>>());
+
+        var repo = sutProvider.GetDependency<ICipherRepository>();
+        repo.GetManyCipherOrganizationDetailsExcludingDefaultCollectionsAsync(organizationId)
+            .Returns(Task.FromResult<IEnumerable<CipherOrganizationDetailsWithCollections>>(
+                new[] { item1, item2 }));
+
+        var actual = (await sutProvider.Sut
+            .GetAllOrganizationCiphersExcludingDefaultUserCollections(organizationId))
+            .ToList();
+
+        Assert.Equal(2, actual.Count);
+        Assert.Same(item1, actual[0]);
+        Assert.Same(item2, actual[1]);
+
+        // and we indeed called the repo once
+        await repo.Received(1)
+            .GetManyCipherOrganizationDetailsExcludingDefaultCollectionsAsync(organizationId);
+    }
+
+    private CipherOrganizationDetailsWithCollections MakeWith(
+        CipherOrganizationDetails baseCipher,
+        params Guid[] cols)
+    {
+        var dict = cols
+          .Select(cid => new CollectionCipher { CipherId = baseCipher.Id, CollectionId = cid })
+          .GroupBy(cc => cc.CipherId)
+          .ToDictionary(g => g.Key, g => g);
+
+        return new CipherOrganizationDetailsWithCollections(baseCipher, dict);
+    }
 }

util/Migrator/DbScripts/2025-09-03_00_CipherOrganizationDetailsExcludingDefaultCollections.sql

@@ -0,0 +1,69 @@
+-- View that provides organization cipher details with their collection associations
+CREATE OR ALTER VIEW [dbo].[OrganizationCipherDetailsCollectionsView]
+AS
+    SELECT
+      C.[Id],
+      C.[UserId],
+      C.[OrganizationId],
+      C.[Type],
+      C.[Data],
+      C.[Attachments],
+      C.[Favorites],
+      C.[Folders],
+      C.[CreationDate],
+      C.[RevisionDate],
+      C.[DeletedDate],
+      C.[Reprompt],
+      C.[Key],
+      CASE
+          WHEN O.[UseTotp] = 1 THEN 1
+          ELSE 0
+      END AS [OrganizationUseTotp],
+      CC.[CollectionId],
+      COL.[Type] AS [CollectionType]
+    FROM [dbo].[Cipher] C
+    INNER JOIN [dbo].[Organization] O ON C.[OrganizationId] = O.[Id]
+    LEFT JOIN [dbo].[CollectionCipher] CC ON CC.[CipherId] = C.[Id]
+    LEFT JOIN [dbo].[Collection] COL ON CC.[CollectionId] = COL.[Id]
+    WHERE C.[UserId] IS NULL -- Organization ciphers only
+        AND O.[Enabled] = 1; -- Only enabled organizations
+GO
+
+ -- Stored procedure that filters out ciphers that ONLY belong to default collections
+CREATE OR ALTER PROCEDURE
+  [dbo].[CipherOrganizationDetails_ReadByOrganizationIdExcludingDefaultCollections]
+      @OrganizationId UNIQUEIDENTIFIER
+  AS
+  BEGIN
+      SET NOCOUNT ON;
+
+      WITH [NonDefaultCiphers] AS (
+          SELECT DISTINCT [Id]
+          FROM [dbo].[OrganizationCipherDetailsCollectionsView]
+          WHERE [OrganizationId] = @OrganizationId
+              AND ([CollectionId] IS NULL OR [CollectionType] <> 1)
+      )
+
+      SELECT
+          V.[Id],
+          V.[UserId],
+          V.[OrganizationId],
+          V.[Type],
+          V.[Data],
+          V.[Favorites],
+          V.[Folders],
+          V.[Attachments],
+          V.[CreationDate],
+          V.[RevisionDate],
+          V.[DeletedDate],
+          V.[Reprompt],
+          V.[Key],
+          V.[OrganizationUseTotp],
+          V.[CollectionId]  -- For Dapper splitOn parameter
+      FROM [dbo].[OrganizationCipherDetailsCollectionsView] V
+      INNER JOIN [NonDefaultCiphers] NDC ON V.[Id] = NDC.[Id]
+      WHERE V.[OrganizationId] = @OrganizationId
+          AND (V.[CollectionId] IS NULL OR V.[CollectionType] <> 1)
+      ORDER BY V.[RevisionDate] DESC;
+  END;
+GO