mirror of
https://github.com/bitwarden/server
synced 2025-12-11 05:43:35 +00:00
[PM-24233] Use BulkResourceCreationService in CipherRepository (#6201)
* Add constant for CipherRepositoryBulkResourceCreation in FeatureFlagKeys * Add bulk creation methods for Ciphers, Folders, and CollectionCiphers in BulkResourceCreationService - Implemented CreateCiphersAsync, CreateFoldersAsync, CreateCollectionCiphersAsync, and CreateTempCiphersAsync methods for bulk insertion. - Added helper methods to build DataTables for Ciphers, Folders, and CollectionCiphers. - Enhanced error handling for empty collections during bulk operations. * Refactor CipherRepository to utilize BulkResourceCreationService - Introduced IFeatureService to manage feature flag checks for bulk operations. - Updated methods to conditionally use BulkResourceCreationService for creating Ciphers, Folders, and CollectionCiphers based on feature flag status. - Enhanced existing bulk copy logic to maintain functionality while integrating feature flag checks. * Add InlineFeatureService to DatabaseDataAttribute for feature flag management - Introduced EnabledFeatureFlags property to DatabaseDataAttribute for configuring feature flags. - Integrated InlineFeatureService to provide feature flag checks within the service collection. - Enhanced GetData method to utilize feature flags for conditional service registration. * Add tests for bulk creation of Ciphers in CipherRepositoryTests - Implemented tests for bulk creation of Ciphers, Folders, and Collections with feature flag checks. - Added test cases for updating multiple Ciphers to validate bulk update functionality. - Enhanced existing test structure to ensure comprehensive coverage of bulk operations in the CipherRepository. * Refactor BulkResourceCreationService to use dynamic types for DataColumns - Updated DataColumn definitions in BulkResourceCreationService to utilize the actual types of properties from the cipher object instead of hardcoded types. - Simplified the assignment of nullable properties to directly use their values, improving code readability and maintainability. * Update BulkResourceCreationService to use specific types for DataColumns - Changed DataColumn definitions to use specific types (short and string) instead of dynamic types based on cipher properties. - Improved handling of nullable properties when assigning values to DataTable rows, ensuring proper handling of DBNull for null values. * Refactor CipherRepositoryTests for improved clarity and consistency - Renamed test methods to better reflect their purpose and improve readability. - Updated test data to use more descriptive names for users, folders, and collections. - Enhanced test structure with clear Arrange, Act, and Assert sections for better understanding of test flow. - Ensured all tests validate the expected outcomes for bulk operations with feature flag checks. * Update CipherRepositoryBulkResourceCreation feature flag key * Refactor DatabaseDataAttribute usage in CipherRepositoryTests to use array syntax for EnabledFeatureFlags * Update CipherRepositoryTests to use GenerateComb for generating unique IDs * Refactor CipherRepository methods to accept a boolean parameter for enabling bulk resource creation based on feature flags. Update tests to verify functionality with and without the feature flag enabled. * Refactor CipherRepository and related services to support new methods for bulk resource creation without boolean parameters.
This commit is contained in:
Rui Tomé
GitHub
@@ -1,5 +1,6 @@
|
||||
using System.Data;
|
||||
using Bit.Core.Entities;
|
||||
using Bit.Core.Vault.Entities;
|
||||
using Microsoft.Data.SqlClient;
|
||||
|
||||
namespace Bit.Infrastructure.Dapper.AdminConsole.Helpers;
|
||||
@@ -15,6 +16,38 @@ public static class BulkResourceCreationService
|
||||
await bulkCopy.WriteToServerAsync(dataTable);
|
||||
}
|
||||
|
||||
public static async Task CreateCiphersAsync(SqlConnection connection, SqlTransaction transaction, IEnumerable<Cipher> ciphers, string errorMessage = _defaultErrorMessage)
|
||||
{
|
||||
using var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.KeepIdentity, transaction);
|
||||
bulkCopy.DestinationTableName = "[dbo].[Cipher]";
|
||||
var dataTable = BuildCiphersTable(bulkCopy, ciphers, errorMessage);
|
||||
await bulkCopy.WriteToServerAsync(dataTable);
|
||||
}
|
||||
|
||||
public static async Task CreateFoldersAsync(SqlConnection connection, SqlTransaction transaction, IEnumerable<Folder> folders, string errorMessage = _defaultErrorMessage)
|
||||
{
|
||||
using var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.KeepIdentity, transaction);
|
||||
bulkCopy.DestinationTableName = "[dbo].[Folder]";
|
||||
var dataTable = BuildFoldersTable(bulkCopy, folders, errorMessage);
|
||||
await bulkCopy.WriteToServerAsync(dataTable);
|
||||
}
|
||||
|
||||
public static async Task CreateCollectionCiphersAsync(SqlConnection connection, SqlTransaction transaction, IEnumerable<CollectionCipher> collectionCiphers, string errorMessage = _defaultErrorMessage)
|
||||
{
|
||||
using var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.KeepIdentity, transaction);
|
||||
bulkCopy.DestinationTableName = "[dbo].[CollectionCipher]";
|
||||
var dataTable = BuildCollectionCiphersTable(bulkCopy, collectionCiphers, errorMessage);
|
||||
await bulkCopy.WriteToServerAsync(dataTable);
|
||||
}
|
||||
|
||||
public static async Task CreateTempCiphersAsync(SqlConnection connection, SqlTransaction transaction, IEnumerable<Cipher> ciphers, string errorMessage = _defaultErrorMessage)
|
||||
{
|
||||
using var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.KeepIdentity, transaction);
|
||||
bulkCopy.DestinationTableName = "#TempCipher";
|
||||
var dataTable = BuildCiphersTable(bulkCopy, ciphers, errorMessage);
|
||||
await bulkCopy.WriteToServerAsync(dataTable);
|
||||
}
|
||||
|
||||
private static DataTable BuildCollectionsUsersTable(SqlBulkCopy bulkCopy, IEnumerable<CollectionUser> collectionUsers, string errorMessage)
|
||||
{
|
||||
var collectionUser = collectionUsers.FirstOrDefault();
|
||||
@@ -126,4 +159,161 @@ public static class BulkResourceCreationService
|
||||
|
||||
return collectionsTable;
|
||||
}
|
||||
|
||||
private static DataTable BuildCiphersTable(SqlBulkCopy bulkCopy, IEnumerable<Cipher> ciphers, string errorMessage)
|
||||
{
|
||||
var c = ciphers.FirstOrDefault();
|
||||
|
||||
if (c == null)
|
||||
{
|
||||
throw new ApplicationException(errorMessage);
|
||||
}
|
||||
|
||||
var ciphersTable = new DataTable("CipherDataTable");
|
||||
|
||||
var idColumn = new DataColumn(nameof(c.Id), c.Id.GetType());
|
||||
ciphersTable.Columns.Add(idColumn);
|
||||
var userIdColumn = new DataColumn(nameof(c.UserId), typeof(Guid));
|
||||
ciphersTable.Columns.Add(userIdColumn);
|
||||
var organizationId = new DataColumn(nameof(c.OrganizationId), typeof(Guid));
|
||||
ciphersTable.Columns.Add(organizationId);
|
||||
var typeColumn = new DataColumn(nameof(c.Type), typeof(short));
|
||||
ciphersTable.Columns.Add(typeColumn);
|
||||
var dataColumn = new DataColumn(nameof(c.Data), typeof(string));
|
||||
ciphersTable.Columns.Add(dataColumn);
|
||||
var favoritesColumn = new DataColumn(nameof(c.Favorites), typeof(string));
|
||||
ciphersTable.Columns.Add(favoritesColumn);
|
||||
var foldersColumn = new DataColumn(nameof(c.Folders), typeof(string));
|
||||
ciphersTable.Columns.Add(foldersColumn);
|
||||
var attachmentsColumn = new DataColumn(nameof(c.Attachments), typeof(string));
|
||||
ciphersTable.Columns.Add(attachmentsColumn);
|
||||
var creationDateColumn = new DataColumn(nameof(c.CreationDate), c.CreationDate.GetType());
|
||||
ciphersTable.Columns.Add(creationDateColumn);
|
||||
var revisionDateColumn = new DataColumn(nameof(c.RevisionDate), c.RevisionDate.GetType());
|
||||
ciphersTable.Columns.Add(revisionDateColumn);
|
||||
var deletedDateColumn = new DataColumn(nameof(c.DeletedDate), typeof(DateTime));
|
||||
ciphersTable.Columns.Add(deletedDateColumn);
|
||||
var repromptColumn = new DataColumn(nameof(c.Reprompt), typeof(short));
|
||||
ciphersTable.Columns.Add(repromptColumn);
|
||||
var keyColummn = new DataColumn(nameof(c.Key), typeof(string));
|
||||
ciphersTable.Columns.Add(keyColummn);
|
||||
|
||||
foreach (DataColumn col in ciphersTable.Columns)
|
||||
{
|
||||
bulkCopy.ColumnMappings.Add(col.ColumnName, col.ColumnName);
|
||||
}
|
||||
|
||||
var keys = new DataColumn[1];
|
||||
keys[0] = idColumn;
|
||||
ciphersTable.PrimaryKey = keys;
|
||||
|
||||
foreach (var cipher in ciphers)
|
||||
{
|
||||
var row = ciphersTable.NewRow();
|
||||
|
||||
row[idColumn] = cipher.Id;
|
||||
row[userIdColumn] = cipher.UserId.HasValue ? (object)cipher.UserId.Value : DBNull.Value;
|
||||
row[organizationId] = cipher.OrganizationId.HasValue ? (object)cipher.OrganizationId.Value : DBNull.Value;
|
||||
row[typeColumn] = (short)cipher.Type;
|
||||
row[dataColumn] = cipher.Data;
|
||||
row[favoritesColumn] = cipher.Favorites;
|
||||
row[foldersColumn] = cipher.Folders;
|
||||
row[attachmentsColumn] = cipher.Attachments;
|
||||
row[creationDateColumn] = cipher.CreationDate;
|
||||
row[revisionDateColumn] = cipher.RevisionDate;
|
||||
row[deletedDateColumn] = cipher.DeletedDate.HasValue ? (object)cipher.DeletedDate : DBNull.Value;
|
||||
row[repromptColumn] = cipher.Reprompt.HasValue ? cipher.Reprompt.Value : DBNull.Value;
|
||||
row[keyColummn] = cipher.Key;
|
||||
|
||||
ciphersTable.Rows.Add(row);
|
||||
}
|
||||
|
||||
return ciphersTable;
|
||||
}
|
||||
|
||||
private static DataTable BuildFoldersTable(SqlBulkCopy bulkCopy, IEnumerable<Folder> folders, string errorMessage)
|
||||
{
|
||||
var f = folders.FirstOrDefault();
|
||||
|
||||
if (f == null)
|
||||
{
|
||||
throw new ApplicationException(errorMessage);
|
||||
}
|
||||
|
||||
var foldersTable = new DataTable("FolderDataTable");
|
||||
|
||||
var idColumn = new DataColumn(nameof(f.Id), f.Id.GetType());
|
||||
foldersTable.Columns.Add(idColumn);
|
||||
var userIdColumn = new DataColumn(nameof(f.UserId), f.UserId.GetType());
|
||||
foldersTable.Columns.Add(userIdColumn);
|
||||
var nameColumn = new DataColumn(nameof(f.Name), typeof(string));
|
||||
foldersTable.Columns.Add(nameColumn);
|
||||
var creationDateColumn = new DataColumn(nameof(f.CreationDate), f.CreationDate.GetType());
|
||||
foldersTable.Columns.Add(creationDateColumn);
|
||||
var revisionDateColumn = new DataColumn(nameof(f.RevisionDate), f.RevisionDate.GetType());
|
||||
foldersTable.Columns.Add(revisionDateColumn);
|
||||
|
||||
foreach (DataColumn col in foldersTable.Columns)
|
||||
{
|
||||
bulkCopy.ColumnMappings.Add(col.ColumnName, col.ColumnName);
|
||||
}
|
||||
|
||||
var keys = new DataColumn[1];
|
||||
keys[0] = idColumn;
|
||||
foldersTable.PrimaryKey = keys;
|
||||
|
||||
foreach (var folder in folders)
|
||||
{
|
||||
var row = foldersTable.NewRow();
|
||||
|
||||
row[idColumn] = folder.Id;
|
||||
row[userIdColumn] = folder.UserId;
|
||||
row[nameColumn] = folder.Name;
|
||||
row[creationDateColumn] = folder.CreationDate;
|
||||
row[revisionDateColumn] = folder.RevisionDate;
|
||||
|
||||
foldersTable.Rows.Add(row);
|
||||
}
|
||||
|
||||
return foldersTable;
|
||||
}
|
||||
|
||||
private static DataTable BuildCollectionCiphersTable(SqlBulkCopy bulkCopy, IEnumerable<CollectionCipher> collectionCiphers, string errorMessage)
|
||||
{
|
||||
var cc = collectionCiphers.FirstOrDefault();
|
||||
|
||||
if (cc == null)
|
||||
{
|
||||
throw new ApplicationException(errorMessage);
|
||||
}
|
||||
|
||||
var collectionCiphersTable = new DataTable("CollectionCipherDataTable");
|
||||
|
||||
var collectionIdColumn = new DataColumn(nameof(cc.CollectionId), cc.CollectionId.GetType());
|
||||
collectionCiphersTable.Columns.Add(collectionIdColumn);
|
||||
var cipherIdColumn = new DataColumn(nameof(cc.CipherId), cc.CipherId.GetType());
|
||||
collectionCiphersTable.Columns.Add(cipherIdColumn);
|
||||
|
||||
foreach (DataColumn col in collectionCiphersTable.Columns)
|
||||
{
|
||||
bulkCopy.ColumnMappings.Add(col.ColumnName, col.ColumnName);
|
||||
}
|
||||
|
||||
var keys = new DataColumn[2];
|
||||
keys[0] = collectionIdColumn;
|
||||
keys[1] = cipherIdColumn;
|
||||
collectionCiphersTable.PrimaryKey = keys;
|
||||
|
||||
foreach (var collectionCipher in collectionCiphers)
|
||||
{
|
||||
var row = collectionCiphersTable.NewRow();
|
||||
|
||||
row[collectionIdColumn] = collectionCipher.CollectionId;
|
||||
row[cipherIdColumn] = collectionCipher.CipherId;
|
||||
|
||||
collectionCiphersTable.Rows.Add(row);
|
||||
}
|
||||
|
||||
return collectionCiphersTable;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ using Bit.Core.Tools.Entities;
|
||||
using Bit.Core.Vault.Entities;
|
||||
using Bit.Core.Vault.Models.Data;
|
||||
using Bit.Core.Vault.Repositories;
|
||||
using Bit.Infrastructure.Dapper.AdminConsole.Helpers;
|
||||
using Bit.Infrastructure.Dapper.Repositories;
|
||||
using Bit.Infrastructure.Dapper.Vault.Helpers;
|
||||
using Dapper;
|
||||
@@ -408,6 +409,52 @@ public class CipherRepository : Repository<Cipher, Guid>, ICipherRepository
|
||||
};
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public UpdateEncryptedDataForKeyRotation UpdateForKeyRotation_vNext(
|
||||
Guid userId, IEnumerable<Cipher> ciphers)
|
||||
{
|
||||
return async (SqlConnection connection, SqlTransaction transaction) =>
|
||||
{
|
||||
// Create temp table
|
||||
var sqlCreateTemp = @"
|
||||
SELECT TOP 0 *
|
||||
INTO #TempCipher
|
||||
FROM [dbo].[Cipher]";
|
||||
|
||||
await using (var cmd = new SqlCommand(sqlCreateTemp, connection, transaction))
|
||||
{
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
// Bulk copy data into temp table
|
||||
await BulkResourceCreationService.CreateTempCiphersAsync(connection, transaction, ciphers);
|
||||
|
||||
// Update cipher table from temp table
|
||||
var sql = @"
|
||||
UPDATE
|
||||
[dbo].[Cipher]
|
||||
SET
|
||||
[Data] = TC.[Data],
|
||||
[Attachments] = TC.[Attachments],
|
||||
[RevisionDate] = TC.[RevisionDate],
|
||||
[Key] = TC.[Key]
|
||||
FROM
|
||||
[dbo].[Cipher] C
|
||||
INNER JOIN
|
||||
#TempCipher TC ON C.Id = TC.Id
|
||||
WHERE
|
||||
C.[UserId] = @UserId
|
||||
|
||||
DROP TABLE #TempCipher";
|
||||
|
||||
await using (var cmd = new SqlCommand(sql, connection, transaction))
|
||||
{
|
||||
cmd.Parameters.Add("@UserId", SqlDbType.UniqueIdentifier).Value = userId;
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public async Task UpdateCiphersAsync(Guid userId, IEnumerable<Cipher> ciphers)
|
||||
{
|
||||
if (!ciphers.Any())
|
||||
@@ -490,6 +537,83 @@ public class CipherRepository : Repository<Cipher, Guid>, ICipherRepository
|
||||
}
|
||||
}
|
||||
|
||||
public async Task UpdateCiphersAsync_vNext(Guid userId, IEnumerable<Cipher> ciphers)
|
||||
{
|
||||
if (!ciphers.Any())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using (var connection = new SqlConnection(ConnectionString))
|
||||
{
|
||||
connection.Open();
|
||||
|
||||
using (var transaction = connection.BeginTransaction())
|
||||
{
|
||||
try
|
||||
{
|
||||
// 1. Create temp tables to bulk copy into.
|
||||
|
||||
var sqlCreateTemp = @"
|
||||
SELECT TOP 0 *
|
||||
INTO #TempCipher
|
||||
FROM [dbo].[Cipher]";
|
||||
|
||||
using (var cmd = new SqlCommand(sqlCreateTemp, connection, transaction))
|
||||
{
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
// 2. Bulk copy into temp tables.
|
||||
await BulkResourceCreationService.CreateTempCiphersAsync(connection, transaction, ciphers);
|
||||
|
||||
// 3. Insert into real tables from temp tables and clean up.
|
||||
|
||||
// Intentionally not including Favorites, Folders, and CreationDate
|
||||
// since those are not meant to be bulk updated at this time
|
||||
var sql = @"
|
||||
UPDATE
|
||||
[dbo].[Cipher]
|
||||
SET
|
||||
[UserId] = TC.[UserId],
|
||||
[OrganizationId] = TC.[OrganizationId],
|
||||
[Type] = TC.[Type],
|
||||
[Data] = TC.[Data],
|
||||
[Attachments] = TC.[Attachments],
|
||||
[RevisionDate] = TC.[RevisionDate],
|
||||
[DeletedDate] = TC.[DeletedDate],
|
||||
[Key] = TC.[Key]
|
||||
FROM
|
||||
[dbo].[Cipher] C
|
||||
INNER JOIN
|
||||
#TempCipher TC ON C.Id = TC.Id
|
||||
WHERE
|
||||
C.[UserId] = @UserId
|
||||
|
||||
DROP TABLE #TempCipher";
|
||||
|
||||
using (var cmd = new SqlCommand(sql, connection, transaction))
|
||||
{
|
||||
cmd.Parameters.Add("@UserId", SqlDbType.UniqueIdentifier).Value = userId;
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
await connection.ExecuteAsync(
|
||||
$"[{Schema}].[User_BumpAccountRevisionDate]",
|
||||
new { Id = userId },
|
||||
commandType: CommandType.StoredProcedure, transaction: transaction);
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
catch
|
||||
{
|
||||
transaction.Rollback();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CreateAsync(Guid userId, IEnumerable<Cipher> ciphers, IEnumerable<Folder> folders)
|
||||
{
|
||||
if (!ciphers.Any())
|
||||
@@ -538,6 +662,44 @@ public class CipherRepository : Repository<Cipher, Guid>, ICipherRepository
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CreateAsync_vNext(Guid userId, IEnumerable<Cipher> ciphers, IEnumerable<Folder> folders)
|
||||
{
|
||||
if (!ciphers.Any())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using (var connection = new SqlConnection(ConnectionString))
|
||||
{
|
||||
connection.Open();
|
||||
|
||||
using (var transaction = connection.BeginTransaction())
|
||||
{
|
||||
try
|
||||
{
|
||||
if (folders.Any())
|
||||
{
|
||||
await BulkResourceCreationService.CreateFoldersAsync(connection, transaction, folders);
|
||||
}
|
||||
|
||||
await BulkResourceCreationService.CreateCiphersAsync(connection, transaction, ciphers);
|
||||
|
||||
await connection.ExecuteAsync(
|
||||
$"[{Schema}].[User_BumpAccountRevisionDate]",
|
||||
new { Id = userId },
|
||||
commandType: CommandType.StoredProcedure, transaction: transaction);
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
catch
|
||||
{
|
||||
transaction.Rollback();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CreateAsync(IEnumerable<Cipher> ciphers, IEnumerable<Collection> collections,
|
||||
IEnumerable<CollectionCipher> collectionCiphers, IEnumerable<CollectionUser> collectionUsers)
|
||||
{
|
||||
@@ -607,6 +769,55 @@ public class CipherRepository : Repository<Cipher, Guid>, ICipherRepository
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CreateAsync_vNext(IEnumerable<Cipher> ciphers, IEnumerable<Collection> collections,
|
||||
IEnumerable<CollectionCipher> collectionCiphers, IEnumerable<CollectionUser> collectionUsers)
|
||||
{
|
||||
if (!ciphers.Any())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using (var connection = new SqlConnection(ConnectionString))
|
||||
{
|
||||
connection.Open();
|
||||
|
||||
using (var transaction = connection.BeginTransaction())
|
||||
{
|
||||
try
|
||||
{
|
||||
await BulkResourceCreationService.CreateCiphersAsync(connection, transaction, ciphers);
|
||||
|
||||
if (collections.Any())
|
||||
{
|
||||
await BulkResourceCreationService.CreateCollectionsAsync(connection, transaction, collections);
|
||||
}
|
||||
|
||||
if (collectionCiphers.Any())
|
||||
{
|
||||
await BulkResourceCreationService.CreateCollectionCiphersAsync(connection, transaction, collectionCiphers);
|
||||
}
|
||||
|
||||
if (collectionUsers.Any())
|
||||
{
|
||||
await BulkResourceCreationService.CreateCollectionsUsersAsync(connection, transaction, collectionUsers);
|
||||
}
|
||||
|
||||
await connection.ExecuteAsync(
|
||||
$"[{Schema}].[User_BumpAccountRevisionDateByOrganizationId]",
|
||||
new { OrganizationId = ciphers.First().OrganizationId },
|
||||
commandType: CommandType.StoredProcedure, transaction: transaction);
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
catch
|
||||
{
|
||||
transaction.Rollback();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task SoftDeleteAsync(IEnumerable<Guid> ids, Guid userId)
|
||||
{
|
||||
using (var connection = new SqlConnection(ConnectionString))
|
||||
|
||||
Reference in New Issue
Block a user