bitwarden/server
1
0
Fork 0
mirror of https://github.com/bitwarden/server synced 2025-12-23 11:43:23 +00:00
Code Issues Releases Wiki Activity

[PM-6762] Move to Azure.Data.Tables (#3888)

Browse Source 
* Move to Azure.Data.Tables

* Reorder usings

* Add new package to Renovate

* Add manual serialization and deserialization due to enums

* Properly retrieve just the next page
This commit is contained in:
Matt Bishop
2024-03-28 16:36:24 -04:00
committed by GitHub
parent ffd988eeda
commit c53e5eeab3
7 changed files with 144 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
src/Core/Repositories/TableStorage/EventRepository.cs
View File

@@ -1,14 +1,14 @@
using Bit.Core.Models.Data;
using Azure.Data.Tables;
using Bit.Core.Models.Data;
using Bit.Core.Settings;
using Bit.Core.Utilities;
using Bit.Core.Vault.Entities;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Repositories.TableStorage;
public class EventRepository : IEventRepository
{
private readonly CloudTable _table;
private readonly TableClient _tableClient;
public EventRepository(GlobalSettings globalSettings)
: this(globalSettings.Events.ConnectionString)
@@ -16,9 +16,8 @@ public class EventRepository : IEventRepository
public EventRepository(string storageConnectionString)
{
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var tableClient = storageAccount.CreateCloudTableClient();
_table = tableClient.GetTableReference("event");
var tableClient = new TableServiceClient(storageConnectionString);
_tableClient = tableClient.GetTableClient("event");
}
public async Task<PagedResult<IEvent>> GetManyByUserAsync(Guid userId, DateTime startDate, DateTime endDate,
@@ -76,7 +75,7 @@ public class EventRepository : IEventRepository
throw new ArgumentException(nameof(e));
}
await CreateEntityAsync(entity);
await CreateEventAsync(entity);
}
public async Task CreateManyAsync(IEnumerable<IEvent> e)
@@ -99,7 +98,7 @@ public class EventRepository : IEventRepository
var groupEntities = group.ToList();
if (groupEntities.Count == 1)
{
await CreateEntityAsync(groupEntities.First());
await CreateEventAsync(groupEntities.First());
continue;
}
@@ -107,7 +106,7 @@ public class EventRepository : IEventRepository
var iterations = groupEntities.Count / 100;
for (var i = 0; i <= iterations; i++)
{
var batch = new TableBatchOperation();
var batch = new List<TableTransactionAction>();
var batchEntities = groupEntities.Skip(i * 100).Take(100);
if (!batchEntities.Any())
{
@@ -116,19 +115,15 @@ public class EventRepository : IEventRepository
foreach (var entity in batchEntities)
{
batch.InsertOrReplace(entity);
batch.Add(new TableTransactionAction(TableTransactionActionType.Add,
entity.ToAzureEvent()));
}
await _table.ExecuteBatchAsync(batch);
await _tableClient.SubmitTransactionAsync(batch);
}
}
}
public async Task CreateEntityAsync(ITableEntity entity)
{
await _table.ExecuteAsync(TableOperation.InsertOrReplace(entity));
}
public async Task<PagedResult<IEvent>> GetManyAsync(string partitionKey, string rowKey,
DateTime startDate, DateTime endDate, PageOptions pageOptions)
{
@@ -136,60 +131,28 @@ public class EventRepository : IEventRepository
var end = CoreHelpers.DateTimeToTableStorageKey(endDate);
var filter = MakeFilter(partitionKey, string.Format(rowKey, start), string.Format(rowKey, end));
var query = new TableQuery<EventTableEntity>().Where(filter).Take(pageOptions.PageSize);
var result = new PagedResult<IEvent>();
var continuationToken = DeserializeContinuationToken(pageOptions?.ContinuationToken);
var query = _tableClient.QueryAsync<AzureEvent>(filter, pageOptions.PageSize);
var queryResults = await _table.ExecuteQuerySegmentedAsync(query, continuationToken);
result.ContinuationToken = SerializeContinuationToken(queryResults.ContinuationToken);
result.Data.AddRange(queryResults.Results);
await using (var enumerator = query.AsPages(pageOptions?.ContinuationToken,
pageOptions.PageSize).GetAsyncEnumerator())
{
await enumerator.MoveNextAsync();
result.ContinuationToken = enumerator.Current.ContinuationToken;
result.Data.AddRange(enumerator.Current.Values.Select(e => e.ToEventTableEntity()));
}
return result;
}
private async Task CreateEventAsync(EventTableEntity entity)
{
await _tableClient.UpsertEntityAsync(entity.ToAzureEvent());
}
private string MakeFilter(string partitionKey, string rowStart, string rowEnd)
{
var rowFilter = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, $"{rowStart}`"),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, $"{rowEnd}_"));
return TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey),
TableOperators.And,
rowFilter);
}
private string SerializeContinuationToken(TableContinuationToken token)
{
if (token == null)
{
return null;
}
return string.Format("{0}__{1}__{2}__{3}", (int)token.TargetLocation, token.NextTableName,
token.NextPartitionKey, token.NextRowKey);
}
private TableContinuationToken DeserializeContinuationToken(string token)
{
if (string.IsNullOrWhiteSpace(token))
{
return null;
}
var tokenParts = token.Split(new string[] { "__" }, StringSplitOptions.None);
if (tokenParts.Length < 4 || !Enum.TryParse(tokenParts[0], out StorageLocation tLoc))
{
return null;
}
return new TableContinuationToken
{
TargetLocation = tLoc,
NextTableName = string.IsNullOrWhiteSpace(tokenParts[1]) ? null : tokenParts[1],
NextPartitionKey = string.IsNullOrWhiteSpace(tokenParts[2]) ? null : tokenParts[2],
NextRowKey = string.IsNullOrWhiteSpace(tokenParts[3]) ? null : tokenParts[3]
};
return $"PartitionKey eq '{partitionKey}' and RowKey le '{rowStart}' and RowKey ge '{rowEnd}'";
}
}

28
src/Core/Repositories/TableStorage/InstallationDeviceRepository.cs
View File

@@ -1,13 +1,12 @@
using System.Net;
using Azure.Data.Tables;
using Bit.Core.Models.Data;
using Bit.Core.Settings;
using Microsoft.Azure.Cosmos.Table;
namespace Bit.Core.Repositories.TableStorage;
public class InstallationDeviceRepository : IInstallationDeviceRepository
{
private readonly CloudTable _table;
private readonly TableClient _tableClient;
public InstallationDeviceRepository(GlobalSettings globalSettings)
: this(globalSettings.Events.ConnectionString)
@@ -15,14 +14,13 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository
public InstallationDeviceRepository(string storageConnectionString)
{
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var tableClient = storageAccount.CreateCloudTableClient();
_table = tableClient.GetTableReference("installationdevice");
var tableClient = new TableServiceClient(storageConnectionString);
_tableClient = tableClient.GetTableClient("installationdevice");
}
public async Task UpsertAsync(InstallationDeviceEntity entity)
{
await _table.ExecuteAsync(TableOperation.InsertOrReplace(entity));
await _tableClient.UpsertEntityAsync(entity);
}
public async Task UpsertManyAsync(IList<InstallationDeviceEntity> entities)
@@ -52,7 +50,7 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository
var iterations = groupEntities.Count / 100;
for (var i = 0; i <= iterations; i++)
{
var batch = new TableBatchOperation();
var batch = new List<TableTransactionAction>();
var batchEntities = groupEntities.Skip(i * 100).Take(100);
if (!batchEntities.Any())
{
@@ -61,24 +59,16 @@ public class InstallationDeviceRepository : IInstallationDeviceRepository
foreach (var entity in batchEntities)
{
batch.InsertOrReplace(entity);
batch.Add(new TableTransactionAction(TableTransactionActionType.UpsertReplace, entity));
}
await _table.ExecuteBatchAsync(batch);
await _tableClient.SubmitTransactionAsync(batch);
}
}
}
public async Task DeleteAsync(InstallationDeviceEntity entity)
{
try
{
entity.ETag = "*";
await _table.ExecuteAsync(TableOperation.Delete(entity));
}
catch (StorageException e) when (e.RequestInformation.HttpStatusCode != (int)HttpStatusCode.NotFound)
{
throw;
}
await _tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
}
}