From 747e212b1ba374e4c1be9cfe255bb40c002d0ceb Mon Sep 17 00:00:00 2001 From: Brant DeBow <125889545+brant-livefront@users.noreply.github.com> Date: Mon, 8 Sep 2025 12:39:59 -0400 Subject: [PATCH] Add Datadog integration (#6289) * Event integration updates and cleanups * Add Datadog integration * Update README to include link to Datadog PR * Move doc update into the Datadog PR; Fix empty message on ArgumentException * Adjust exception message Co-authored-by: Matt Bishop * Removed unnecessary nullable enable; Moved Docs link to PR into this PR * Remove unnecessary nullable enable calls --------- Co-authored-by: Matt Bishop --- dev/servicebusemulator_config.json | 17 ++ ...ionIntegrationConfigurationRequestModel.cs | 8 +- .../OrgnizationIntegrationRequestModel.cs | 10 +- .../AdminConsole/Enums/IntegrationType.cs | 5 +- .../EventIntegrations/DatadogIntegration.cs | 3 + .../DatadogIntegrationConfigurationDetails.cs | 3 + .../DatadogListenerConfiguration.cs | 38 +++++ .../DatadogIntegrationHandler.cs | 25 +++ .../EventIntegrations/README.md | 3 +- src/Core/Settings/GlobalSettings.cs | 5 + .../Utilities/ServiceCollectionExtensions.cs | 5 + ...tegrationConfigurationRequestModelTests.cs | 26 +++ ...rganizationIntegrationRequestModelTests.cs | 48 ++++++ .../DatadogIntegrationHandlerTests.cs | 157 ++++++++++++++++++ 14 files changed, 346 insertions(+), 7 deletions(-) create mode 100644 src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogIntegration.cs create mode 100644 src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogIntegrationConfigurationDetails.cs create mode 100644 src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogListenerConfiguration.cs create mode 100644 src/Core/AdminConsole/Services/Implementations/EventIntegrations/DatadogIntegrationHandler.cs create mode 100644 test/Core.Test/AdminConsole/Services/DatadogIntegrationHandlerTests.cs diff --git a/dev/servicebusemulator_config.json b/dev/servicebusemulator_config.json index dcf48b7a8c..294efc1897 100644 --- a/dev/servicebusemulator_config.json +++ b/dev/servicebusemulator_config.json @@ -34,6 +34,9 @@ }, { "Name": "events-hec-subscription" + }, + { + "Name": "events-datadog-subscription" } ] }, @@ -81,6 +84,20 @@ } } ] + }, + { + "Name": "integration-datadog-subscription", + "Rules": [ + { + "Name": "datadog-integration-filter", + "Properties": { + "FilterType": "Correlation", + "CorrelationFilter": { + "Label": "datadog" + } + } + } + ] } ] } diff --git a/src/Api/AdminConsole/Models/Request/Organizations/OrganizationIntegrationConfigurationRequestModel.cs b/src/Api/AdminConsole/Models/Request/Organizations/OrganizationIntegrationConfigurationRequestModel.cs index 17e116b8d1..7d1efe2315 100644 --- a/src/Api/AdminConsole/Models/Request/Organizations/OrganizationIntegrationConfigurationRequestModel.cs +++ b/src/Api/AdminConsole/Models/Request/Organizations/OrganizationIntegrationConfigurationRequestModel.cs @@ -1,6 +1,4 @@ -#nullable enable - -using System.Text.Json; +using System.Text.Json; using Bit.Core.AdminConsole.Entities; using Bit.Core.AdminConsole.Models.Data.EventIntegrations; using Bit.Core.Enums; @@ -36,6 +34,10 @@ public class OrganizationIntegrationConfigurationRequestModel return !string.IsNullOrWhiteSpace(Template) && Configuration is null && IsFiltersValid(); + case IntegrationType.Datadog: + return !string.IsNullOrWhiteSpace(Template) && + Configuration is null && + IsFiltersValid(); default: return false; diff --git a/src/Api/AdminConsole/Models/Request/Organizations/OrgnizationIntegrationRequestModel.cs b/src/Api/AdminConsole/Models/Request/Organizations/OrgnizationIntegrationRequestModel.cs index edae0719e3..5fa2e86a90 100644 --- a/src/Api/AdminConsole/Models/Request/Organizations/OrgnizationIntegrationRequestModel.cs +++ b/src/Api/AdminConsole/Models/Request/Organizations/OrgnizationIntegrationRequestModel.cs @@ -4,8 +4,6 @@ using Bit.Core.AdminConsole.Entities; using Bit.Core.AdminConsole.Models.Data.EventIntegrations; using Bit.Core.Enums; -#nullable enable - namespace Bit.Api.AdminConsole.Models.Request.Organizations; public class OrganizationIntegrationRequestModel : IValidatableObject @@ -60,6 +58,14 @@ public class OrganizationIntegrationRequestModel : IValidatableObject new[] { nameof(Configuration) }); } break; + case IntegrationType.Datadog: + if (!IsIntegrationValid()) + { + yield return new ValidationResult( + "Datadog integrations must include valid configuration.", + new[] { nameof(Configuration) }); + } + break; default: yield return new ValidationResult( $"Integration type '{Type}' is not recognized.", diff --git a/src/Core/AdminConsole/Enums/IntegrationType.cs b/src/Core/AdminConsole/Enums/IntegrationType.cs index 58e55193dc..34edc71fbe 100644 --- a/src/Core/AdminConsole/Enums/IntegrationType.cs +++ b/src/Core/AdminConsole/Enums/IntegrationType.cs @@ -6,7 +6,8 @@ public enum IntegrationType : int Scim = 2, Slack = 3, Webhook = 4, - Hec = 5 + Hec = 5, + Datadog = 6 } public static class IntegrationTypeExtensions @@ -21,6 +22,8 @@ public static class IntegrationTypeExtensions return "webhook"; case IntegrationType.Hec: return "hec"; + case IntegrationType.Datadog: + return "datadog"; default: throw new ArgumentOutOfRangeException(nameof(type), $"Unsupported integration type: {type}"); } diff --git a/src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogIntegration.cs b/src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogIntegration.cs new file mode 100644 index 0000000000..8785a74896 --- /dev/null +++ b/src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogIntegration.cs @@ -0,0 +1,3 @@ +namespace Bit.Core.AdminConsole.Models.Data.EventIntegrations; + +public record DatadogIntegration(string ApiKey, Uri Uri); diff --git a/src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogIntegrationConfigurationDetails.cs b/src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogIntegrationConfigurationDetails.cs new file mode 100644 index 0000000000..07aafa4bd8 --- /dev/null +++ b/src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogIntegrationConfigurationDetails.cs @@ -0,0 +1,3 @@ +namespace Bit.Core.AdminConsole.Models.Data.EventIntegrations; + +public record DatadogIntegrationConfigurationDetails(string ApiKey, Uri Uri); diff --git a/src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogListenerConfiguration.cs b/src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogListenerConfiguration.cs new file mode 100644 index 0000000000..1c74826791 --- /dev/null +++ b/src/Core/AdminConsole/Models/Data/EventIntegrations/DatadogListenerConfiguration.cs @@ -0,0 +1,38 @@ +using Bit.Core.Enums; +using Bit.Core.Settings; + +namespace Bit.Core.AdminConsole.Models.Data.EventIntegrations; + +public class DatadogListenerConfiguration(GlobalSettings globalSettings) + : ListenerConfiguration(globalSettings), IIntegrationListenerConfiguration +{ + public IntegrationType IntegrationType + { + get => IntegrationType.Datadog; + } + + public string EventQueueName + { + get => _globalSettings.EventLogging.RabbitMq.DatadogEventsQueueName; + } + + public string IntegrationQueueName + { + get => _globalSettings.EventLogging.RabbitMq.DatadogIntegrationQueueName; + } + + public string IntegrationRetryQueueName + { + get => _globalSettings.EventLogging.RabbitMq.DatadogIntegrationRetryQueueName; + } + + public string EventSubscriptionName + { + get => _globalSettings.EventLogging.AzureServiceBus.DatadogEventSubscriptionName; + } + + public string IntegrationSubscriptionName + { + get => _globalSettings.EventLogging.AzureServiceBus.DatadogIntegrationSubscriptionName; + } +} diff --git a/src/Core/AdminConsole/Services/Implementations/EventIntegrations/DatadogIntegrationHandler.cs b/src/Core/AdminConsole/Services/Implementations/EventIntegrations/DatadogIntegrationHandler.cs new file mode 100644 index 0000000000..45bb5b6d7d --- /dev/null +++ b/src/Core/AdminConsole/Services/Implementations/EventIntegrations/DatadogIntegrationHandler.cs @@ -0,0 +1,25 @@ +using System.Text; +using Bit.Core.AdminConsole.Models.Data.EventIntegrations; + +namespace Bit.Core.Services; + +public class DatadogIntegrationHandler( + IHttpClientFactory httpClientFactory, + TimeProvider timeProvider) + : IntegrationHandlerBase +{ + private readonly HttpClient _httpClient = httpClientFactory.CreateClient(HttpClientName); + + public const string HttpClientName = "DatadogIntegrationHandlerHttpClient"; + + public override async Task HandleAsync(IntegrationMessage message) + { + var request = new HttpRequestMessage(HttpMethod.Post, message.Configuration.Uri); + request.Content = new StringContent(message.RenderedTemplate, Encoding.UTF8, "application/json"); + request.Headers.Add("DD-API-KEY", message.Configuration.ApiKey); + + var response = await _httpClient.SendAsync(request); + + return ResultFromHttpResponse(response, message, timeProvider); + } +} diff --git a/src/Core/AdminConsole/Services/Implementations/EventIntegrations/README.md b/src/Core/AdminConsole/Services/Implementations/EventIntegrations/README.md index 4092cc20ad..de7ce3f7fd 100644 --- a/src/Core/AdminConsole/Services/Implementations/EventIntegrations/README.md +++ b/src/Core/AdminConsole/Services/Implementations/EventIntegrations/README.md @@ -323,7 +323,8 @@ A hosted service (`IntegrationConfigurationDetailsCacheService`) runs in the bac # Building a new integration These are all the pieces required in the process of building out a new integration. For -clarity in naming, these assume a new integration called "Example". +clarity in naming, these assume a new integration called "Example". To see a complete example +in context, view [the PR for adding the Datadog integration](https://github.com/bitwarden/server/pull/6289). ## IntegrationType diff --git a/src/Core/Settings/GlobalSettings.cs b/src/Core/Settings/GlobalSettings.cs index d6e18a4c81..638e1477c1 100644 --- a/src/Core/Settings/GlobalSettings.cs +++ b/src/Core/Settings/GlobalSettings.cs @@ -304,6 +304,8 @@ public class GlobalSettings : IGlobalSettings public virtual string WebhookIntegrationSubscriptionName { get; set; } = "integration-webhook-subscription"; public virtual string HecEventSubscriptionName { get; set; } = "events-hec-subscription"; public virtual string HecIntegrationSubscriptionName { get; set; } = "integration-hec-subscription"; + public virtual string DatadogEventSubscriptionName { get; set; } = "events-datadog-subscription"; + public virtual string DatadogIntegrationSubscriptionName { get; set; } = "integration-datadog-subscription"; public string ConnectionString { @@ -345,6 +347,9 @@ public class GlobalSettings : IGlobalSettings public virtual string HecEventsQueueName { get; set; } = "events-hec-queue"; public virtual string HecIntegrationQueueName { get; set; } = "integration-hec-queue"; public virtual string HecIntegrationRetryQueueName { get; set; } = "integration-hec-retry-queue"; + public virtual string DatadogEventsQueueName { get; set; } = "events-datadog-queue"; + public virtual string DatadogIntegrationQueueName { get; set; } = "integration-datadog-queue"; + public virtual string DatadogIntegrationRetryQueueName { get; set; } = "integration-datadog-retry-queue"; public string HostName { diff --git a/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs b/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs index 4f0d0d4397..592f7c84c3 100644 --- a/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs +++ b/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs @@ -881,15 +881,18 @@ public static class ServiceCollectionExtensions services.AddSlackService(globalSettings); services.TryAddSingleton(TimeProvider.System); services.AddHttpClient(WebhookIntegrationHandler.HttpClientName); + services.AddHttpClient(DatadogIntegrationHandler.HttpClientName); // Add integration handlers services.TryAddSingleton, SlackIntegrationHandler>(); services.TryAddSingleton, WebhookIntegrationHandler>(); + services.TryAddSingleton, DatadogIntegrationHandler>(); var repositoryConfiguration = new RepositoryListenerConfiguration(globalSettings); var slackConfiguration = new SlackListenerConfiguration(globalSettings); var webhookConfiguration = new WebhookListenerConfiguration(globalSettings); var hecConfiguration = new HecListenerConfiguration(globalSettings); + var datadogConfiguration = new DatadogListenerConfiguration(globalSettings); if (IsRabbitMqEnabled(globalSettings)) { @@ -906,6 +909,7 @@ public static class ServiceCollectionExtensions services.AddRabbitMqIntegration(slackConfiguration); services.AddRabbitMqIntegration(webhookConfiguration); services.AddRabbitMqIntegration(hecConfiguration); + services.AddRabbitMqIntegration(datadogConfiguration); } if (IsAzureServiceBusEnabled(globalSettings)) @@ -923,6 +927,7 @@ public static class ServiceCollectionExtensions services.AddAzureServiceBusIntegration(slackConfiguration); services.AddAzureServiceBusIntegration(webhookConfiguration); services.AddAzureServiceBusIntegration(hecConfiguration); + services.AddAzureServiceBusIntegration(datadogConfiguration); } return services; diff --git a/test/Api.Test/AdminConsole/Models/Request/Organizations/OrganizationIntegrationConfigurationRequestModelTests.cs b/test/Api.Test/AdminConsole/Models/Request/Organizations/OrganizationIntegrationConfigurationRequestModelTests.cs index 6af5b8039b..74fe75a9d7 100644 --- a/test/Api.Test/AdminConsole/Models/Request/Organizations/OrganizationIntegrationConfigurationRequestModelTests.cs +++ b/test/Api.Test/AdminConsole/Models/Request/Organizations/OrganizationIntegrationConfigurationRequestModelTests.cs @@ -62,6 +62,32 @@ public class OrganizationIntegrationConfigurationRequestModelTests Assert.True(condition: model.IsValidForType(IntegrationType.Hec)); } + [Theory] + [InlineData(data: "")] + [InlineData(data: " ")] + public void IsValidForType_EmptyNonNullDatadogConfiguration_ReturnsFalse(string? config) + { + var model = new OrganizationIntegrationConfigurationRequestModel + { + Configuration = config, + Template = "template" + }; + + Assert.False(condition: model.IsValidForType(IntegrationType.Datadog)); + } + + [Fact] + public void IsValidForType_NullDatadogConfiguration_ReturnsTrue() + { + var model = new OrganizationIntegrationConfigurationRequestModel + { + Configuration = null, + Template = "template" + }; + + Assert.True(condition: model.IsValidForType(IntegrationType.Datadog)); + } + [Theory] [InlineData(data: null)] [InlineData(data: "")] diff --git a/test/Api.Test/AdminConsole/Models/Request/Organizations/OrganizationIntegrationRequestModelTests.cs b/test/Api.Test/AdminConsole/Models/Request/Organizations/OrganizationIntegrationRequestModelTests.cs index 147564dd94..9565a76822 100644 --- a/test/Api.Test/AdminConsole/Models/Request/Organizations/OrganizationIntegrationRequestModelTests.cs +++ b/test/Api.Test/AdminConsole/Models/Request/Organizations/OrganizationIntegrationRequestModelTests.cs @@ -147,6 +147,54 @@ public class OrganizationIntegrationRequestModelTests Assert.Empty(results); } + [Fact] + public void Validate_Datadog_WithNullConfiguration_ReturnsError() + { + var model = new OrganizationIntegrationRequestModel + { + Type = IntegrationType.Datadog, + Configuration = null + }; + + var results = model.Validate(new ValidationContext(model)).ToList(); + + Assert.Single(results); + Assert.Contains(nameof(model.Configuration), results[0].MemberNames); + Assert.Contains("must include valid configuration", results[0].ErrorMessage); + } + + [Fact] + public void Validate_Datadog_WithInvalidConfiguration_ReturnsError() + { + var model = new OrganizationIntegrationRequestModel + { + Type = IntegrationType.Datadog, + Configuration = "Not valid" + }; + + var results = model.Validate(new ValidationContext(model)).ToList(); + + Assert.Single(results); + Assert.Contains(nameof(model.Configuration), results[0].MemberNames); + Assert.Contains("must include valid configuration", results[0].ErrorMessage); + } + + [Fact] + public void Validate_Datadog_WithValidConfiguration_ReturnsNoErrors() + { + var model = new OrganizationIntegrationRequestModel + { + Type = IntegrationType.Datadog, + Configuration = JsonSerializer.Serialize( + new DatadogIntegration(ApiKey: "API1234", Uri: new Uri("http://localhost")) + ) + }; + + var results = model.Validate(new ValidationContext(model)).ToList(); + + Assert.Empty(results); + } + [Fact] public void Validate_UnknownIntegrationType_ReturnsUnrecognizedError() { diff --git a/test/Core.Test/AdminConsole/Services/DatadogIntegrationHandlerTests.cs b/test/Core.Test/AdminConsole/Services/DatadogIntegrationHandlerTests.cs new file mode 100644 index 0000000000..5f0a9915bf --- /dev/null +++ b/test/Core.Test/AdminConsole/Services/DatadogIntegrationHandlerTests.cs @@ -0,0 +1,157 @@ +#nullable enable + +using System.Net; +using Bit.Core.AdminConsole.Models.Data.EventIntegrations; +using Bit.Core.Services; +using Bit.Test.Common.AutoFixture; +using Bit.Test.Common.AutoFixture.Attributes; +using Bit.Test.Common.Helpers; +using Bit.Test.Common.MockedHttpClient; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Xunit; + +namespace Bit.Core.Test.Services; + +[SutProviderCustomize] +public class DatadogIntegrationHandlerTests +{ + private readonly MockedHttpMessageHandler _handler; + private readonly HttpClient _httpClient; + private const string _apiKey = "AUTH_TOKEN"; + private static readonly Uri _datadogUri = new Uri("https://localhost"); + + public DatadogIntegrationHandlerTests() + { + _handler = new MockedHttpMessageHandler(); + _handler.Fallback + .WithStatusCode(HttpStatusCode.OK) + .WithContent(new StringContent("testtest")); + _httpClient = _handler.ToHttpClient(); + } + + private SutProvider GetSutProvider() + { + var clientFactory = Substitute.For(); + clientFactory.CreateClient(DatadogIntegrationHandler.HttpClientName).Returns(_httpClient); + + return new SutProvider() + .SetDependency(clientFactory) + .WithFakeTimeProvider() + .Create(); + } + + [Theory, BitAutoData] + public async Task HandleAsync_SuccessfulRequest_ReturnsSuccess(IntegrationMessage message) + { + var sutProvider = GetSutProvider(); + message.Configuration = new DatadogIntegrationConfigurationDetails(ApiKey: _apiKey, Uri: _datadogUri); + + var result = await sutProvider.Sut.HandleAsync(message); + + Assert.True(result.Success); + Assert.Equal(result.Message, message); + Assert.Empty(result.FailureReason); + + sutProvider.GetDependency().Received(1).CreateClient( + Arg.Is(AssertHelper.AssertPropertyEqual(DatadogIntegrationHandler.HttpClientName)) + ); + + Assert.Single(_handler.CapturedRequests); + var request = _handler.CapturedRequests[0]; + Assert.NotNull(request); + Assert.NotNull(request.Content); + var returned = await request.Content.ReadAsStringAsync(); + + Assert.Equal(HttpMethod.Post, request.Method); + Assert.Equal(_apiKey, request.Headers.GetValues("DD-API-KEY").Single()); + Assert.Equal(_datadogUri, request.RequestUri); + AssertHelper.AssertPropertyEqual(message.RenderedTemplate, returned); + } + + [Theory, BitAutoData] + public async Task HandleAsync_TooManyRequests_ReturnsFailureSetsDelayUntilDate(IntegrationMessage message) + { + var sutProvider = GetSutProvider(); + var now = new DateTime(2014, 3, 2, 1, 0, 0, DateTimeKind.Utc); + var retryAfter = now.AddSeconds(60); + + sutProvider.GetDependency().SetUtcNow(now); + message.Configuration = new DatadogIntegrationConfigurationDetails(ApiKey: _apiKey, Uri: _datadogUri); + + _handler.Fallback + .WithStatusCode(HttpStatusCode.TooManyRequests) + .WithHeader("Retry-After", "60") + .WithContent(new StringContent("testtest")); + + var result = await sutProvider.Sut.HandleAsync(message); + + Assert.False(result.Success); + Assert.True(result.Retryable); + Assert.Equal(result.Message, message); + Assert.True(result.DelayUntilDate.HasValue); + Assert.Equal(retryAfter, result.DelayUntilDate.Value); + Assert.Equal("Too Many Requests", result.FailureReason); + } + + [Theory, BitAutoData] + public async Task HandleAsync_TooManyRequestsWithDate_ReturnsFailureSetsDelayUntilDate(IntegrationMessage message) + { + var sutProvider = GetSutProvider(); + var now = new DateTime(2014, 3, 2, 1, 0, 0, DateTimeKind.Utc); + var retryAfter = now.AddSeconds(60); + message.Configuration = new DatadogIntegrationConfigurationDetails(ApiKey: _apiKey, Uri: _datadogUri); + + _handler.Fallback + .WithStatusCode(HttpStatusCode.TooManyRequests) + .WithHeader("Retry-After", retryAfter.ToString("r")) + .WithContent(new StringContent("testtest")); + + var result = await sutProvider.Sut.HandleAsync(message); + + Assert.False(result.Success); + Assert.True(result.Retryable); + Assert.Equal(result.Message, message); + Assert.True(result.DelayUntilDate.HasValue); + Assert.Equal(retryAfter, result.DelayUntilDate.Value); + Assert.Equal("Too Many Requests", result.FailureReason); + } + + [Theory, BitAutoData] + public async Task HandleAsync_InternalServerError_ReturnsFailureSetsRetryable(IntegrationMessage message) + { + var sutProvider = GetSutProvider(); + message.Configuration = new DatadogIntegrationConfigurationDetails(ApiKey: _apiKey, Uri: _datadogUri); + + _handler.Fallback + .WithStatusCode(HttpStatusCode.InternalServerError) + .WithContent(new StringContent("testtest")); + + var result = await sutProvider.Sut.HandleAsync(message); + + Assert.False(result.Success); + Assert.True(result.Retryable); + Assert.Equal(result.Message, message); + Assert.False(result.DelayUntilDate.HasValue); + Assert.Equal("Internal Server Error", result.FailureReason); + } + + [Theory, BitAutoData] + public async Task HandleAsync_UnexpectedRedirect_ReturnsFailureNotRetryable(IntegrationMessage message) + { + var sutProvider = GetSutProvider(); + message.Configuration = new DatadogIntegrationConfigurationDetails(ApiKey: _apiKey, Uri: _datadogUri); + + _handler.Fallback + .WithStatusCode(HttpStatusCode.TemporaryRedirect) + .WithContent(new StringContent("testtest")); + + var result = await sutProvider.Sut.HandleAsync(message); + + Assert.False(result.Success); + Assert.False(result.Retryable); + Assert.Equal(result.Message, message); + Assert.Null(result.DelayUntilDate); + Assert.Equal("Temporary Redirect", result.FailureReason); + } +}