1
0
mirror of https://github.com/bitwarden/server synced 2026-01-04 17:43:53 +00:00

Use extended cache for caching integration configuration details (#6650)

* Use extended cache for caching integration configuration details

* Alter strategy to use one cache / database call to retrieve all configurations for an event (including wildcards)

* Renamed migration per @withinfocus suggestion
This commit is contained in:
Brant DeBow
2025-12-05 13:12:27 -05:00
committed by GitHub
parent 2f893768f5
commit 813fad8021
16 changed files with 489 additions and 360 deletions

View File

@@ -2,8 +2,6 @@
using Bit.Core.Enums;
using Bit.Core.Utilities;
#nullable enable
namespace Bit.Core.AdminConsole.Entities;
public class OrganizationIntegration : ITableObject<Guid>

View File

@@ -2,8 +2,6 @@
using Bit.Core.Enums;
using Bit.Core.Utilities;
#nullable enable
namespace Bit.Core.AdminConsole.Entities;
public class OrganizationIntegrationConfiguration : ITableObject<Guid>

View File

@@ -6,10 +6,23 @@ namespace Bit.Core.Repositories;
public interface IOrganizationIntegrationConfigurationRepository : IRepository<OrganizationIntegrationConfiguration, Guid>
{
Task<List<OrganizationIntegrationConfigurationDetails>> GetConfigurationDetailsAsync(
/// <summary>
/// Retrieve the list of available configuration details for a specific event for the organization and
/// integration type.<br/>
/// <br/>
/// <b>Note:</b> This returns all configurations that match the event type explicitly <b>and</b>
/// all the configurations that have a null event type - null event type is considered a
/// wildcard that matches all events.
///
/// </summary>
/// <param name="eventType">The specific event type</param>
/// <param name="organizationId">The id of the organization</param>
/// <param name="integrationType">The integration type</param>
/// <returns>A List of <see cref="OrganizationIntegrationConfigurationDetails"/> that match</returns>
Task<List<OrganizationIntegrationConfigurationDetails>> GetManyByEventTypeOrganizationIdIntegrationType(
EventType eventType,
Guid organizationId,
IntegrationType integrationType,
EventType eventType);
IntegrationType integrationType);
Task<List<OrganizationIntegrationConfigurationDetails>> GetAllConfigurationDetailsAsync();

View File

@@ -5,6 +5,7 @@ using Bit.Core.AdminConsole.Repositories;
using Bit.Core.AdminConsole.Utilities;
using Bit.Core.Enums;
using Bit.Core.Models.Data;
using Bit.Core.Models.Data.Organizations;
using Bit.Core.Models.Data.Organizations.OrganizationUsers;
using Bit.Core.Repositories;
using Bit.Core.Utilities;
@@ -17,8 +18,8 @@ public class EventIntegrationHandler<T>(
IntegrationType integrationType,
IEventIntegrationPublisher eventIntegrationPublisher,
IIntegrationFilterService integrationFilterService,
IIntegrationConfigurationDetailsCache configurationCache,
IFusionCache cache,
IOrganizationIntegrationConfigurationRepository configurationRepository,
IGroupRepository groupRepository,
IOrganizationRepository organizationRepository,
IOrganizationUserRepository organizationUserRepository,
@@ -27,17 +28,7 @@ public class EventIntegrationHandler<T>(
{
public async Task HandleEventAsync(EventMessage eventMessage)
{
if (eventMessage.OrganizationId is not Guid organizationId)
{
return;
}
var configurations = configurationCache.GetConfigurationDetails(
organizationId,
integrationType,
eventMessage.Type);
foreach (var configuration in configurations)
foreach (var configuration in await GetConfigurationDetailsListAsync(eventMessage))
{
try
{
@@ -64,7 +55,7 @@ public class EventIntegrationHandler<T>(
{
IntegrationType = integrationType,
MessageId = messageId.ToString(),
OrganizationId = organizationId.ToString(),
OrganizationId = eventMessage.OrganizationId?.ToString(),
Configuration = config,
RenderedTemplate = renderedTemplate,
RetryCount = 0,
@@ -132,6 +123,37 @@ public class EventIntegrationHandler<T>(
return context;
}
private async Task<List<OrganizationIntegrationConfigurationDetails>> GetConfigurationDetailsListAsync(EventMessage eventMessage)
{
if (eventMessage.OrganizationId is not Guid organizationId)
{
return [];
}
List<OrganizationIntegrationConfigurationDetails> configurations = [];
var integrationTag = EventIntegrationsCacheConstants.BuildCacheTagForOrganizationIntegration(
organizationId,
integrationType
);
configurations.AddRange(await cache.GetOrSetAsync<List<OrganizationIntegrationConfigurationDetails>>(
key: EventIntegrationsCacheConstants.BuildCacheKeyForOrganizationIntegrationConfigurationDetails(
organizationId: organizationId,
integrationType: integrationType,
eventType: eventMessage.Type),
factory: async _ => await configurationRepository.GetManyByEventTypeOrganizationIdIntegrationType(
eventType: eventMessage.Type,
organizationId: organizationId,
integrationType: integrationType),
options: new FusionCacheEntryOptions(
duration: EventIntegrationsCacheConstants.DurationForOrganizationIntegrationConfigurationDetails),
tags: [integrationTag]
));
return configurations;
}
private async Task<OrganizationUserUserDetails?> GetUserFromCacheAsync(Guid organizationId, Guid userId) =>
await cache.GetOrSetAsync<OrganizationUserUserDetails?>(
key: EventIntegrationsCacheConstants.BuildCacheKeyForOrganizationUser(organizationId, userId),

View File

@@ -1,83 +0,0 @@
using System.Diagnostics;
using Bit.Core.Enums;
using Bit.Core.Models.Data.Organizations;
using Bit.Core.Repositories;
using Bit.Core.Settings;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace Bit.Core.Services;
public class IntegrationConfigurationDetailsCacheService : BackgroundService, IIntegrationConfigurationDetailsCache
{
private readonly record struct IntegrationCacheKey(Guid OrganizationId, IntegrationType IntegrationType, EventType? EventType);
private readonly IOrganizationIntegrationConfigurationRepository _repository;
private readonly ILogger<IntegrationConfigurationDetailsCacheService> _logger;
private readonly TimeSpan _refreshInterval;
private Dictionary<IntegrationCacheKey, List<OrganizationIntegrationConfigurationDetails>> _cache = new();
public IntegrationConfigurationDetailsCacheService(
IOrganizationIntegrationConfigurationRepository repository,
GlobalSettings globalSettings,
ILogger<IntegrationConfigurationDetailsCacheService> logger)
{
_repository = repository;
_logger = logger;
_refreshInterval = TimeSpan.FromMinutes(globalSettings.EventLogging.IntegrationCacheRefreshIntervalMinutes);
}
public List<OrganizationIntegrationConfigurationDetails> GetConfigurationDetails(
Guid organizationId,
IntegrationType integrationType,
EventType eventType)
{
var specificKey = new IntegrationCacheKey(organizationId, integrationType, eventType);
var allEventsKey = new IntegrationCacheKey(organizationId, integrationType, null);
var results = new List<OrganizationIntegrationConfigurationDetails>();
if (_cache.TryGetValue(specificKey, out var specificConfigs))
{
results.AddRange(specificConfigs);
}
if (_cache.TryGetValue(allEventsKey, out var fallbackConfigs))
{
results.AddRange(fallbackConfigs);
}
return results;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await RefreshAsync();
var timer = new PeriodicTimer(_refreshInterval);
while (await timer.WaitForNextTickAsync(stoppingToken))
{
await RefreshAsync();
}
}
internal async Task RefreshAsync()
{
var stopwatch = Stopwatch.StartNew();
try
{
var newCache = (await _repository.GetAllConfigurationDetailsAsync())
.GroupBy(x => new IntegrationCacheKey(x.OrganizationId, x.IntegrationType, x.EventType))
.ToDictionary(g => g.Key, g => g.ToList());
_cache = newCache;
stopwatch.Stop();
_logger.LogInformation(
"[IntegrationConfigurationDetailsCacheService] Refreshed successfully: {Count} entries in {Duration}ms",
newCache.Count,
stopwatch.Elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
_logger.LogError("[IntegrationConfigurationDetailsCacheService] Refresh failed: {ex}", ex);
}
}
}

View File

@@ -295,33 +295,59 @@ graph TD
```
## Caching
To reduce database load and improve performance, integration configurations are cached in-memory as a Dictionary
with a periodic load of all configurations. Without caching, each incoming `EventMessage` would trigger a database
To reduce database load and improve performance, event integrations uses its own named extended cache (see
the [README in Utilities](https://github.com/bitwarden/server/blob/main/src/Core/Utilities/README.md#extended-cache)
for more information). Without caching, for instance, each incoming `EventMessage` would trigger a database
query to retrieve the relevant `OrganizationIntegrationConfigurationDetails`.
By loading all configurations into memory on a fixed interval, we ensure:
### `EventIntegrationsCacheConstants`
- Consistent performance for reads.
- Reduced database pressure.
- Predictable refresh timing, independent of event activity.
`EventIntegrationsCacheConstants` allows the code to have strongly typed references to a number of cache-related
details when working with the extended cache. The cache name and all cache keys and tags are programmatically accessed
from `EventIntegrationsCacheConstants` rather than simple strings. For instance,
`EventIntegrationsCacheConstants.CacheName` is used in the cache setup, keyed services, dependency injection, etc.,
rather than using a string literal (i.e. "EventIntegrations") in code.
### Architecture / Design
### `OrganizationIntegrationConfigurationDetails`
- The cache is read-only for consumers. It is only updated in bulk by a background refresh process.
- The cache is fully replaced on each refresh to avoid locking or partial state.
- This is one of the most actively used portions of the architecture because any event that has an associated
organization requires a check of the configurations to determine if we need to fire off an integration.
- By using the extended cache, all reads are hitting the L1 or L2 cache before needing to access the database.
- Reads return a `List<OrganizationIntegrationConfigurationDetails>` for a given key or an empty list if no
match exists.
- Failures or delays in the loading process do not affect the existing cache state. The cache will continue serving
the last known good state until the update replaces the whole cache.
- The TTL is set very high on these records (1 day). This is because when the admin API makes any changes, it
tells the cache to remove that key. This propagates to the event listening code via the extended cache backplane,
which means that the cache is then expired and the next read will fetch the new values. This allows us to have
a high TTL and avoid needing to refresh values except when necessary.
### Background Refresh
#### Tagging per integration
A hosted service (`IntegrationConfigurationDetailsCacheService`) runs in the background and:
- Each entry in the cache (which again, returns `List<OrganizationIntegrationConfigurationDetails>`) is tagged with
the organization id and the integration type.
- This allows us to remove all of a given organization's configuration details for an integration when the admin
makes changes at the integration level.
- For instance, if there were 5 events configured for a given organization's webhook and the admin changed the URL
at the integration level, the updates would need to be propagated or else the cache will continue returning the
stale URL.
- By tagging each of the entries, the API can ask the extended cache to remove all the entries for a given
organization integration in one call. The cache will handle dropping / refreshing these entries in a
performant way.
- There are two places in the code that are both aware of the tagging functionality
- The `EventIntegrationHandler` must use the tag when fetching relevant configuration details. This tells the cache
to store the entry with the tag when it successfully loads from the repository.
- The `OrganizationIntegrationController` needs to use the tag to remove all the tagged entries when and admin
creates, updates, or deletes an integration.
- To ensure both places are synchronized on how to tag entries, they both use
`EventIntegrationsCacheConstants.BuildCacheTagForOrganizationIntegration` to build the tag.
- Loads all configuration records at application startup.
- Refreshes the cache on a configurable interval.
- Logs timing and entry count on success.
- Logs exceptions on failure without disrupting application flow.
### Template Properties
- The `IntegrationTemplateProcessor` supports some properties that require an additional lookup. For instance,
the `UserId` is provided as part of the `EventMessage`, but `UserName` means an additional lookup to map the user
id to the actual name.
- The properties for a `User` (which includes `ActingUser`), `Group`, and `Organization` are cached via the
extended cache with a default TTL of 30 minutes.
- This is cached in both the L1 (Memory) and L2 (Redis) and will be automatically refreshed as needed.
# Building a new integration

View File

@@ -1,4 +1,6 @@
using Bit.Core.AdminConsole.Entities;
using Bit.Core.Enums;
using Bit.Core.Models.Data.Organizations;
using Bit.Core.Models.Data.Organizations.OrganizationUsers;
namespace Bit.Core.Utilities;
@@ -11,7 +13,12 @@ public static class EventIntegrationsCacheConstants
/// <summary>
/// The base cache name used for storing event integration data.
/// </summary>
public static readonly string CacheName = "EventIntegrations";
public const string CacheName = "EventIntegrations";
/// <summary>
/// Duration TimeSpan for adding OrganizationIntegrationConfigurationDetails to the cache.
/// </summary>
public static readonly TimeSpan DurationForOrganizationIntegrationConfigurationDetails = TimeSpan.FromDays(1);
/// <summary>
/// Builds a deterministic cache key for a <see cref="Group"/>.
@@ -20,10 +27,8 @@ public static class EventIntegrationsCacheConstants
/// <returns>
/// A cache key for this Group.
/// </returns>
public static string BuildCacheKeyForGroup(Guid groupId)
{
return $"Group:{groupId:N}";
}
public static string BuildCacheKeyForGroup(Guid groupId) =>
$"Group:{groupId:N}";
/// <summary>
/// Builds a deterministic cache key for an <see cref="Organization"/>.
@@ -32,10 +37,8 @@ public static class EventIntegrationsCacheConstants
/// <returns>
/// A cache key for the Organization.
/// </returns>
public static string BuildCacheKeyForOrganization(Guid organizationId)
{
return $"Organization:{organizationId:N}";
}
public static string BuildCacheKeyForOrganization(Guid organizationId) =>
$"Organization:{organizationId:N}";
/// <summary>
/// Builds a deterministic cache key for an organization user <see cref="OrganizationUserUserDetails"/>.
@@ -45,8 +48,37 @@ public static class EventIntegrationsCacheConstants
/// <returns>
/// A cache key for the user.
/// </returns>
public static string BuildCacheKeyForOrganizationUser(Guid organizationId, Guid userId)
{
return $"OrganizationUserUserDetails:{organizationId:N}:{userId:N}";
}
public static string BuildCacheKeyForOrganizationUser(Guid organizationId, Guid userId) =>
$"OrganizationUserUserDetails:{organizationId:N}:{userId:N}";
/// <summary>
/// Builds a deterministic cache key for an organization's integration configuration details
/// <see cref="OrganizationIntegrationConfigurationDetails"/>.
/// </summary>
/// <param name="organizationId">The unique identifier of the organization to which the user belongs.</param>
/// <param name="integrationType">The <see cref="IntegrationType"/> of the integration.</param>
/// <param name="eventType">The <see cref="EventType"/> of the event configured. Can be null to apply to all events.</param>
/// <returns>
/// A cache key for the configuration details.
/// </returns>
public static string BuildCacheKeyForOrganizationIntegrationConfigurationDetails(
Guid organizationId,
IntegrationType integrationType,
EventType? eventType
) => $"OrganizationIntegrationConfigurationDetails:{organizationId:N}:{integrationType}:{eventType}";
/// <summary>
/// Builds a deterministic tag for tagging an organization's integration configuration details. This tag is then
/// used to tag all of the <see cref="OrganizationIntegrationConfigurationDetails"/> that result from this
/// integration, which allows us to remove all relevant entries when an integration is changed or removed.
/// </summary>
/// <param name="organizationId">The unique identifier of the organization to which the user belongs.</param>
/// <param name="integrationType">The <see cref="IntegrationType"/> of the integration.</param>
/// <returns>
/// A cache tag to use for the configuration details.
/// </returns>
public static string BuildCacheTagForOrganizationIntegration(
Guid organizationId,
IntegrationType integrationType
) => $"OrganizationIntegration:{organizationId:N}:{integrationType}";
}

View File

@@ -20,10 +20,9 @@ public class OrganizationIntegrationConfigurationRepository : Repository<Organiz
: base(connectionString, readOnlyConnectionString)
{ }
public async Task<List<OrganizationIntegrationConfigurationDetails>> GetConfigurationDetailsAsync(
Guid organizationId,
IntegrationType integrationType,
EventType eventType)
public async Task<List<OrganizationIntegrationConfigurationDetails>>
GetManyByEventTypeOrganizationIdIntegrationType(EventType eventType, Guid organizationId,
IntegrationType integrationType)
{
using (var connection = new SqlConnection(ConnectionString))
{

View File

@@ -17,16 +17,17 @@ public class OrganizationIntegrationConfigurationRepository : Repository<Core.Ad
: base(serviceScopeFactory, mapper, context => context.OrganizationIntegrationConfigurations)
{ }
public async Task<List<OrganizationIntegrationConfigurationDetails>> GetConfigurationDetailsAsync(
Guid organizationId,
IntegrationType integrationType,
EventType eventType)
public async Task<List<OrganizationIntegrationConfigurationDetails>>
GetManyByEventTypeOrganizationIdIntegrationType(EventType eventType, Guid organizationId,
IntegrationType integrationType)
{
using (var scope = ServiceScopeFactory.CreateScope())
{
var dbContext = GetDatabaseContext(scope);
var query = new OrganizationIntegrationConfigurationDetailsReadManyByEventTypeOrganizationIdIntegrationTypeQuery(
organizationId, eventType, integrationType
organizationId,
eventType,
integrationType
);
return await query.Run(dbContext).ToListAsync();
}

View File

@@ -1,31 +1,21 @@
#nullable enable
using Bit.Core.Enums;
using Bit.Core.Enums;
using Bit.Core.Models.Data.Organizations;
namespace Bit.Infrastructure.EntityFramework.Repositories.Queries;
public class OrganizationIntegrationConfigurationDetailsReadManyByEventTypeOrganizationIdIntegrationTypeQuery : IQuery<OrganizationIntegrationConfigurationDetails>
public class OrganizationIntegrationConfigurationDetailsReadManyByEventTypeOrganizationIdIntegrationTypeQuery(
Guid organizationId,
EventType eventType,
IntegrationType integrationType)
: IQuery<OrganizationIntegrationConfigurationDetails>
{
private readonly Guid _organizationId;
private readonly EventType _eventType;
private readonly IntegrationType _integrationType;
public OrganizationIntegrationConfigurationDetailsReadManyByEventTypeOrganizationIdIntegrationTypeQuery(Guid organizationId, EventType eventType, IntegrationType integrationType)
{
_organizationId = organizationId;
_eventType = eventType;
_integrationType = integrationType;
}
public IQueryable<OrganizationIntegrationConfigurationDetails> Run(DatabaseContext dbContext)
{
var query = from oic in dbContext.OrganizationIntegrationConfigurations
join oi in dbContext.OrganizationIntegrations on oic.OrganizationIntegrationId equals oi.Id into oioic
from oi in dbContext.OrganizationIntegrations
where oi.OrganizationId == _organizationId &&
oi.Type == _integrationType &&
oic.EventType == _eventType
join oi in dbContext.OrganizationIntegrations on oic.OrganizationIntegrationId equals oi.Id
where oi.OrganizationId == organizationId &&
oi.Type == integrationType &&
(oic.EventType == eventType || oic.EventType == null)
select new OrganizationIntegrationConfigurationDetails()
{
Id = oic.Id,

View File

@@ -893,13 +893,11 @@ public static class ServiceCollectionExtensions
integrationType: listenerConfiguration.IntegrationType,
eventIntegrationPublisher: provider.GetRequiredService<IEventIntegrationPublisher>(),
integrationFilterService: provider.GetRequiredService<IIntegrationFilterService>(),
configurationCache: provider.GetRequiredService<IIntegrationConfigurationDetailsCache>(),
cache: provider.GetRequiredKeyedService<IFusionCache>(EventIntegrationsCacheConstants.CacheName),
configurationRepository: provider.GetRequiredService<IOrganizationIntegrationConfigurationRepository>(),
groupRepository: provider.GetRequiredService<IGroupRepository>(),
organizationRepository: provider.GetRequiredService<IOrganizationRepository>(),
organizationUserRepository: provider.GetRequiredService<IOrganizationUserRepository>(),
logger: provider.GetRequiredService<ILogger<EventIntegrationHandler<TConfig>>>()
)
organizationUserRepository: provider.GetRequiredService<IOrganizationUserRepository>(), logger: provider.GetRequiredService<ILogger<EventIntegrationHandler<TConfig>>>())
);
services.TryAddEnumerable(ServiceDescriptor.Singleton<IHostedService,
AzureServiceBusEventListenerService<TListenerConfig>>(provider =>
@@ -941,10 +939,6 @@ public static class ServiceCollectionExtensions
// Add common services
services.AddDistributedCache(globalSettings);
services.AddExtendedCache(EventIntegrationsCacheConstants.CacheName, globalSettings);
services.TryAddSingleton<IntegrationConfigurationDetailsCacheService>();
services.TryAddSingleton<IIntegrationConfigurationDetailsCache>(provider =>
provider.GetRequiredService<IntegrationConfigurationDetailsCacheService>());
services.AddHostedService(provider => provider.GetRequiredService<IntegrationConfigurationDetailsCacheService>());
services.TryAddSingleton<IIntegrationFilterService, IntegrationFilterService>();
services.TryAddKeyedSingleton<IEventWriteService, RepositoryEventWriteService>("persistent");
@@ -1024,13 +1018,11 @@ public static class ServiceCollectionExtensions
integrationType: listenerConfiguration.IntegrationType,
eventIntegrationPublisher: provider.GetRequiredService<IEventIntegrationPublisher>(),
integrationFilterService: provider.GetRequiredService<IIntegrationFilterService>(),
configurationCache: provider.GetRequiredService<IIntegrationConfigurationDetailsCache>(),
cache: provider.GetRequiredKeyedService<IFusionCache>(EventIntegrationsCacheConstants.CacheName),
configurationRepository: provider.GetRequiredService<IOrganizationIntegrationConfigurationRepository>(),
groupRepository: provider.GetRequiredService<IGroupRepository>(),
organizationRepository: provider.GetRequiredService<IOrganizationRepository>(),
organizationUserRepository: provider.GetRequiredService<IOrganizationUserRepository>(),
logger: provider.GetRequiredService<ILogger<EventIntegrationHandler<TConfig>>>()
)
organizationUserRepository: provider.GetRequiredService<IOrganizationUserRepository>(), logger: provider.GetRequiredService<ILogger<EventIntegrationHandler<TConfig>>>())
);
services.TryAddEnumerable(ServiceDescriptor.Singleton<IHostedService,
RabbitMqEventListenerService<TListenerConfig>>(provider =>

View File

@@ -11,7 +11,7 @@ BEGIN
FROM
[dbo].[OrganizationIntegrationConfigurationDetailsView] oic
WHERE
oic.[EventType] = @EventType
(oic.[EventType] = @EventType OR oic.[EventType] IS NULL)
AND
oic.[OrganizationId] = @OrganizationId
AND