diff --git a/docs/diagnostics/metrics.md b/docs/diagnostics/metrics.md index def9fb6d3..d6cc8d988 100644 --- a/docs/diagnostics/metrics.md +++ b/docs/diagnostics/metrics.md @@ -59,6 +59,16 @@ All `trogon.eventstore.*` instruments are development semantic conventions. Attr The failure, unimplemented, and deadline-exceeded counters mirror distinct diagnostic events. They are separate instruments because the source events can overlap for one call and therefore must not be summed as mutually exclusive outcomes. +### Password authentication + +| Instrument | Kind | Unit | Attributes | Description | +| --- | --- | --- | --- | --- | +| `trogon.eventstore.authentication.password.admitted` | Counter | `{attempt}` | None | Password authentication attempts admitted for processing, regardless of authentication outcome | +| `trogon.eventstore.authentication.password.rejected` | Counter | `{attempt}` | `trogon.eventstore.authentication.password.rejection.reason` | Attempts rejected because the `rate` or `concurrency` limit was exhausted | +| `trogon.eventstore.authentication.password.active` | UpDownCounter | `{attempt}` | None | Admitted password authentication attempts still being processed | + +These node-local admission metrics do not identify users, client addresses, or credentials. Rejections describe exhausted capacity, not invalid passwords. + ### Queues | Instrument | Kind | Unit | Attributes | Description | diff --git a/docs/security.md b/docs/security.md index 4518f0fb4..d33e3a7a4 100644 --- a/docs/security.md +++ b/docs/security.md @@ -527,6 +527,42 @@ making the database authentication method explicit, so password and OAuth access Authentication is applied to all HTTP endpoints by default, except `/-/liveness`, `/-/readiness`, static web content, and redirects. +### Password authentication admission limits + +Built-in password authentication shares a node-local admission budget across UI sign-in, HTTP and gRPC +credentials, TCP authentication, and forwarded credentials. Cached credentials still require password +verification and use the same budget. Attempts are admitted before account reads, including requests +for nonexistent accounts, so changing usernames cannot bypass the node-wide limit. + +| Setting | Default | Purpose | +|:--------|--------:|:--------| +| `Auth:Password:MaxConcurrentAttempts` | 4 | Maximum simultaneous account reads and password checks. | +| `Auth:Password:AttemptsPerSecond` | 100 | Attempt tokens replenished each second. | +| `Auth:Password:BurstSize` | 200 | Maximum accumulated attempt tokens. | + +All values must be positive, and `BurstSize` must be at least `AttemptsPerSecond` so the bucket can +hold a full second's replenishment. Invalid limits prevent the password provider from starting. +There is no waiting queue. When either budget is exhausted, requests +receive the existing authentication-not-ready response: HTTP returns `503` with `Retry-After`, gRPC +returns `Unavailable`, TCP returns `NotReady`, and browser sign-in reports that the provider is not +ready. Clients should use bounded retries with backoff and jitter. + +Limits are shared by all password users on a node, not per account or per IP address. They do not lock +accounts or provide distributed brute-force protection. Use ingress abuse controls and network access +restrictions as well. Size the limits under representative load; cached API password checks also count, +and each node has an independent budget. Changing these settings requires a restart. + +Custom authentication plugins own their admission controls. Certificate authentication, OAuth validation, +and validation of established UI sessions do not consume the password budget. Cancelling a caller does not +stop an account read or password hash already in progress. Its concurrency permit remains held until the +operation completes or the account read times +out, so cancellation cannot allow more password work than the configured limit. + +The `EventStore.Core` meter exports `trogon.eventstore.authentication.password.admitted`, +`trogon.eventstore.authentication.password.rejected`, and `trogon.eventstore.authentication.password.active`. +Rejections distinguish rate exhaustion from concurrency exhaustion without recording usernames, +credentials, or client addresses. + ### Management UI sessions Browser sign-in uses ASP.NET Core cookie authentication with a protected session identifier. Passwords are diff --git a/otel/semconv/registry/trogon/eventstore/metrics.yaml b/otel/semconv/registry/trogon/eventstore/metrics.yaml index 13ff2554c..9d0efa421 100644 --- a/otel/semconv/registry/trogon/eventstore/metrics.yaml +++ b/otel/semconv/registry/trogon/eventstore/metrics.yaml @@ -4,6 +4,19 @@ groups: stability: development brief: Attributes used by TrogonEventStore metrics. attributes: + - id: trogon.eventstore.authentication.password.rejection.reason + type: + members: + - id: rate + value: rate + stability: development + brief: The password authentication rate limit was exhausted. + - id: concurrency + value: concurrency + stability: development + brief: All password authentication concurrency permits were in use. + stability: development + brief: The exhausted password authentication admission limit. - id: trogon.eventstore.activity.name type: string stability: development @@ -105,6 +118,30 @@ groups: brief: Load average sampling period. examples: [1m, 5m, 15m] + - id: metric.trogon.eventstore.authentication.password.admitted + type: metric + stability: development + brief: Number of password authentication attempts admitted for processing. + metric_name: trogon.eventstore.authentication.password.admitted + instrument: counter + unit: "{attempt}" + - id: metric.trogon.eventstore.authentication.password.rejected + type: metric + stability: development + brief: Number of password authentication attempts rejected by admission limits. + metric_name: trogon.eventstore.authentication.password.rejected + instrument: counter + unit: "{attempt}" + attributes: + - ref: trogon.eventstore.authentication.password.rejection.reason + requirement_level: required + - id: metric.trogon.eventstore.authentication.password.active + type: metric + stability: development + brief: Number of admitted password authentication attempts still being processed. + metric_name: trogon.eventstore.authentication.password.active + instrument: updowncounter + unit: "{attempt}" - id: metric.trogon.eventstore.component.status type: metric stability: development diff --git a/src/EventStore.ClusterNode/ClusterVNodeHostedService.cs b/src/EventStore.ClusterNode/ClusterVNodeHostedService.cs index 5dcd180e9..38a688d4a 100644 --- a/src/EventStore.ClusterNode/ClusterVNodeHostedService.cs +++ b/src/EventStore.ClusterNode/ClusterVNodeHostedService.cs @@ -251,7 +251,7 @@ AuthenticationProviderFactory GetAuthenticationProviderFactory() var authenticationMethodFactories = new Dictionary { { AuthenticationMethodNames.Password, new AuthenticationProviderFactory(components => - new InternalAuthenticationProviderFactory(components, _options.DefaultUser)) + new InternalAuthenticationProviderFactory(components, _options.DefaultUser, _options.Auth.Password)) }, { AuthenticationMethodNames.OAuth, new AuthenticationProviderFactory(_ => diff --git a/src/EventStore.Core.Tests/Authentication/PasswordAuthenticationAdmissionTests.cs b/src/EventStore.Core.Tests/Authentication/PasswordAuthenticationAdmissionTests.cs new file mode 100644 index 000000000..f18adca41 --- /dev/null +++ b/src/EventStore.Core.Tests/Authentication/PasswordAuthenticationAdmissionTests.cs @@ -0,0 +1,341 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using EventStore.Common.Exceptions; +using EventStore.Core.Authentication.InternalAuthentication; +using EventStore.Core.Messages; +using EventStore.Core.Services.TimerService; +using EventStore.Core.Tests.Helpers; +using EventStore.Plugins.Authentication; +using Microsoft.AspNetCore.Http; +using NUnit.Framework; +using TrogonEventStore.SemanticConventions; + +namespace EventStore.Core.Tests.Authentication; + +[TestFixture(typeof(LogFormat.V2), typeof(string))] +[FixtureLifeCycle(LifeCycle.InstancePerTestCase)] +public class PasswordAuthenticationAdmissionTests : + with_internal_authentication_provider +{ + protected override void Given() => ExistingEvent("$user-user", "$UserCreated", null, + "{LoginName:'user',Salt:'drowssap',Hash:'password',Groups:['reader']}"); + + [SetUp] + public void SetUp() => SetUpProvider(); + + [TestCase(0, 1, 1)] + [TestCase(-1, 1, 1)] + [TestCase(1, 0, 1)] + [TestCase(1, -1, 1)] + [TestCase(1, 1, 0)] + [TestCase(1, 1, -1)] + [TestCase(1, 100, 1)] + [TestCase(1, 2, 1)] + public void invalid_limits_fail_before_serving_password_requests(int concurrent, int rate, int burst) + { + Assert.Throws(() => new InternalAuthenticationProvider( + _bus, _ioDispatcher, new StubPasswordHashAlgorithm(), 1000, false, + DefaultData.DefaultUserOptions, new() + { + MaxConcurrentAttempts = concurrent, + AttemptsPerSecond = rate, + BurstSize = burst + })); + } + + [TestCase(1, 1)] + [TestCase(1, 2)] + public void burst_at_or_above_replenishment_rate_allows_password_authentication(int rate, int burst) + { + var provider = new InternalAuthenticationProvider(_bus, _ioDispatcher, new StubPasswordHashAlgorithm(), 1000, false, + DefaultData.DefaultUserOptions, new() { MaxConcurrentAttempts = 1, AttemptsPerSecond = rate, BurstSize = burst }); + var authenticated = 0; + provider.Authenticate(new TestAuthenticationRequest("user", "password", + () => Assert.Fail("Unauthorized"), _ => authenticated++, () => Assert.Fail("Error"), () => Assert.Fail("Not ready"))); + Assert.That(authenticated, Is.EqualTo(1)); + } + + [Test] + public void admission_metrics_report_bounded_rejection_reasons_and_active_work() + { + var admitted = MetricDefinitions.TrogonEventstoreAuthenticationPasswordAdmitted.Name; + var rejected = MetricDefinitions.TrogonEventstoreAuthenticationPasswordRejected.Name; + var active = MetricDefinitions.TrogonEventstoreAuthenticationPasswordActive.Name; + var values = new Dictionary { [admitted] = 0, [rejected] = 0, [active] = 0 }; + var attributes = new List>(); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, owner) => + { + if (values.ContainsKey(instrument.Name)) + owner.EnableMeasurementEvents(instrument); + }; + listener.SetMeasurementEventCallback((instrument, measurement, tags, _) => + { + values[instrument.Name] += measurement; + foreach (var tag in tags) + attributes.Add(tag); + }); + listener.Start(); + ReadsBackwardQueuesUp(); + var request = new TestAuthenticationRequest("user", "password", () => { }, _ => { }, () => { }, () => { }); + for (var index = 0; index < 5; index++) + _internalAuthenticationProvider.AuthenticateSession(request); + Assert.That(values[admitted], Is.EqualTo(4)); + Assert.That(values[rejected], Is.EqualTo(1)); + Assert.That(values[active], Is.EqualTo(4)); + CompleteOneReadBackwards(); + Assert.That(values[active], Is.EqualTo(3)); + Assert.That(attributes, Is.EqualTo(new[] + { + new KeyValuePair(TrogonAttributeNames.AuthenticationPasswordRejectionReason, "concurrency") + })); + } + + [Test] + public void api_and_browser_password_authentication_share_bounded_pending_reads() + { + ReadsBackwardQueuesUp(); + _consumer.HandledMessages.Clear(); + var rejected = 0; + for (var index = 0; index < 5; index++) + { + var request = new TestAuthenticationRequest("user", "password", () => { }, _ => { }, () => { }, () => rejected++); + if (index % 2 == 0) + _internalAuthenticationProvider.Authenticate(request); + else + _internalAuthenticationProvider.AuthenticateSession(request); + } + + Assert.That(rejected, Is.EqualTo(1)); + Assert.That(_consumer.HandledMessages.OfType().Count(), Is.EqualTo(4)); + CompleteOneReadBackwards(); + _internalAuthenticationProvider.AuthenticateSession(new TestAuthenticationRequest("user", "password", + () => { }, _ => { }, () => { }, () => rejected++)); + Assert.That(rejected, Is.EqualTo(1)); + Assert.That(_consumer.HandledMessages.OfType().Count(), Is.EqualTo(5)); + } + + [Test] + public void cached_password_verification_cannot_bypass_pending_browser_attempts() + { + var authenticated = 0; + var rejected = 0; + var request = new TestAuthenticationRequest("user", "password", () => { }, _ => authenticated++, () => { }, () => rejected++); + _internalAuthenticationProvider.Authenticate(request); + Assert.That(authenticated, Is.EqualTo(1)); + ReadsBackwardQueuesUp(); + for (var index = 0; index < 4; index++) + _internalAuthenticationProvider.AuthenticateSession(request); + _internalAuthenticationProvider.Authenticate(request); + Assert.That(authenticated, Is.EqualTo(1)); + Assert.That(rejected, Is.EqualTo(1)); + CompleteOneReadBackwards(); + _internalAuthenticationProvider.Authenticate(request); + Assert.That(authenticated, Is.EqualTo(3)); + } + + [TestCase("missing")] + [TestCase("disabled")] + [TestCase("malformed")] + [TestCase("not-ready")] + [TestCase("timeout")] + public void failed_account_reads_release_capacity(string failure) + { + switch (failure) + { + case "missing": + NoStream("$user-user"); + break; + case "disabled": + ExistingEvent("$user-user", "$UserUpdated", null, + "{LoginName:'user',Salt:'drowssap',Hash:'password',Groups:[],Disabled:true}"); + break; + case "malformed": + ExistingEvent("$user-user", "$UserUpdated", null, "invalid"); + break; + case "not-ready": + NotReady(); + break; + case "timeout": + AllReadsTimeOut(); + break; + } + var outcomes = 0; + var request = new TestAuthenticationRequest("user", "password", () => outcomes++, _ => outcomes++, () => outcomes++, () => outcomes++); + _consumer.HandledMessages.Clear(); + for (var index = 0; index < 10; index++) + { + _internalAuthenticationProvider.Authenticate(request); + if (failure == "timeout") + _consumer.HandledMessages.OfType().Last().Reply(); + } + Assert.That(outcomes, Is.EqualTo(10)); + Assert.That(_consumer.HandledMessages.OfType().Count(), Is.EqualTo(10)); + } + + [TestCase(false)] + [TestCase(true)] + public async Task capacity_is_held_until_password_verification_finishes(bool browser) + { + using var hashing = new ControlledHash(); + var provider = new InternalAuthenticationProvider(_bus, _ioDispatcher, hashing, 1000, false, + DefaultData.DefaultUserOptions, new() { MaxConcurrentAttempts = 1 }); + var authenticated = 0; + var rejected = 0; + var request = new TestAuthenticationRequest("user", "password", () => { }, _ => Interlocked.Increment(ref authenticated), + () => { }, () => Interlocked.Increment(ref rejected)); + provider.Authenticate(request); + hashing.Block = true; + var running = Task.Run(() => + { + if (browser) + provider.AuthenticateSession(request); + else + provider.Authenticate(request); + }); + try + { + Assert.That(hashing.Entered.Wait(TimeSpan.FromSeconds(5)), Is.True); + provider.Authenticate(request); + Assert.That(rejected, Is.EqualTo(1)); + Assert.That(authenticated, Is.EqualTo(1)); + } + finally + { + hashing.Release.Set(); + await running.WaitAsync(TimeSpan.FromSeconds(5)); + } + provider.Authenticate(request); + Assert.That(authenticated, Is.EqualTo(3)); + } + + [Test] + public async Task burst_exhaustion_limits_missing_users_and_recovers() + { + NoStream("$user-user"); + var provider = new InternalAuthenticationProvider(_bus, _ioDispatcher, new StubPasswordHashAlgorithm(), 1000, false, + DefaultData.DefaultUserOptions, new() { BurstSize = 1, AttemptsPerSecond = 1 }); + var rejected = 0; + var unauthorized = 0; + var request = new TestAuthenticationRequest("user", "password", () => unauthorized++, _ => { }, () => { }, () => rejected++); + provider.Authenticate(request); + provider.AuthenticateSession(request); + Assert.That(unauthorized, Is.EqualTo(1)); + Assert.That(rejected, Is.EqualTo(1)); + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + provider.Authenticate(request); + Assert.That(unauthorized, Is.EqualTo(2)); + } + + [Test] + public void timeout_then_late_read_cannot_release_another_attempts_capacity() + { + var provider = new InternalAuthenticationProvider(_bus, _ioDispatcher, new StubPasswordHashAlgorithm(), 1000, false, + DefaultData.DefaultUserOptions, new() { MaxConcurrentAttempts = 1 }); + ReadsBackwardQueuesUp(); + var rejected = 0; + var authenticated = 0; + var request = new TestAuthenticationRequest("user", "password", () => { }, _ => authenticated++, () => { }, () => rejected++); + provider.AuthenticateSession(request); + var timeout = _consumer.HandledMessages.OfType().Last(); + timeout.Reply(); + provider.AuthenticateSession(request); + timeout.Reply(); + CompleteOneReadBackwards(); + provider.AuthenticateSession(request); + Assert.That(rejected, Is.EqualTo(2)); + Assert.That(authenticated, Is.Zero); + CompleteOneReadBackwards(); + provider.Authenticate(request); + Assert.That(authenticated, Is.EqualTo(2)); + } + + [Test] + public void password_verifier_exception_releases_capacity() + { + using var hashing = new ControlledHash { Throw = true }; + var provider = new InternalAuthenticationProvider(_bus, _ioDispatcher, hashing, 1000, false, + DefaultData.DefaultUserOptions, new() { MaxConcurrentAttempts = 1 }); + var authenticated = 0; + var unauthorized = 0; + var request = new TestAuthenticationRequest("user", "password", () => unauthorized++, _ => authenticated++, + () => Assert.Fail("Error"), () => Assert.Fail("Capacity was not released")); + provider.Authenticate(request); + Assert.That(unauthorized, Is.EqualTo(1)); + hashing.Throw = false; + provider.Authenticate(request); + Assert.That(authenticated, Is.EqualTo(1)); + hashing.Throw = true; + Assert.Throws(() => provider.Authenticate(request)); + hashing.Throw = false; + provider.Authenticate(request); + Assert.That(authenticated, Is.EqualTo(2)); + } + + [Test] + public async Task existing_session_validation_does_not_consume_password_attempts() + { + var provider = new InternalAuthenticationProvider(_bus, _ioDispatcher, new StubPasswordHashAlgorithm(), 1000, false, + DefaultData.DefaultUserOptions, new() { BurstSize = 1, AttemptsPerSecond = 1 }); + ClaimsPrincipal principal = null; + provider.Authenticate(new TestAuthenticationRequest("user", "password", () => { }, value => principal = value, () => { }, () => { })); + Assert.That(await provider.ValidateSessionAsync(principal, CancellationToken.None), Is.Not.Null); + } + + [Test] + public void shutdown_rejects_password_work_without_throwing() + { + _bus.Publish(new SystemMessage.BecomeShutdown(Guid.NewGuid())); + var rejected = 0; + _internalAuthenticationProvider.Authenticate(new TestAuthenticationRequest("user", "password", () => { }, _ => { }, () => { }, () => rejected++)); + Assert.That(rejected, Is.EqualTo(1)); + } + + [Test] + public async Task certificate_authentication_bypasses_exhausted_password_budget() + { + var provider = new InternalAuthenticationProvider(_bus, _ioDispatcher, new StubPasswordHashAlgorithm(), 1000, false, + DefaultData.DefaultUserOptions, new() { BurstSize = 1, AttemptsPerSecond = 1 }); + provider.Authenticate(new TestAuthenticationRequest("user", "wrong-password", () => { }, _ => { }, () => { }, () => { })); + using var rsa = RSA.Create(2048); + var certificateRequest = new CertificateRequest("CN=user", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + using var certificate = certificateRequest.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-1), DateTimeOffset.UtcNow.AddMinutes(1)); + var request = HttpAuthenticationRequest.CreateWithValidCertificate(new DefaultHttpContext(), "user", certificate); + provider.Authenticate(request); + var (status, principal) = await request.AuthenticateAsync(); + Assert.That(status, Is.EqualTo(HttpAuthenticationRequestStatus.Authenticated)); + Assert.That(principal.Identity.Name, Is.EqualTo("user")); + } + + sealed class ControlledHash : StubPasswordHashAlgorithm, IDisposable + { + public bool Block; + public bool Throw; + public readonly ManualResetEventSlim Entered = new(); + public readonly ManualResetEventSlim Release = new(); + public override bool Verify(string password, string hash, string salt) + { + if (Throw) + throw new InvalidOperationException(); + if (Block) + { + Entered.Set(); + if (!Release.Wait(TimeSpan.FromSeconds(10))) + throw new TimeoutException(); + } + return base.Verify(password, hash, salt); + } + public void Dispose() + { + Entered.Dispose(); + Release.Dispose(); + } + } +} diff --git a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsTests.cs b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsTests.cs index d9a3944c6..5aa70e05a 100644 --- a/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsTests.cs +++ b/src/EventStore.Core.XUnit.Tests/Configuration/ClusterVNodeOptionsTests.cs @@ -34,6 +34,28 @@ public void builds_proper() _ = new ClusterVNodeOptions(); } + [Fact] + public void password_authentication_limits_are_recognized_as_nested_options() + { + var configuration = new ConfigurationBuilder() + .AddEventStoreDefaultValues() + .AddSection(EventStoreConfigurationKeys.Prefix, builder => builder + .AddInMemoryCollection(new Dictionary + { + ["Auth:Password:MaxConcurrentAttempts"] = "2", + ["Auth:Password:AttemptsPerSecond"] = "10", + ["Auth:Password:BurstSize"] = "20" + })) + .Build(); + + var options = ClusterVNodeOptions.FromConfiguration(configuration); + + options.UnknownOptionsDetected.Should().BeFalse(); + options.Auth.Password.MaxConcurrentAttempts.Should().Be(2); + options.Auth.Password.AttemptsPerSecond.Should().Be(10); + options.Auth.Password.BurstSize.Should().Be(20); + } + [Fact] public void confirm_suggested_option() { @@ -209,6 +231,7 @@ public void unknown_options_ignores_repeated_keys_from_other_sources() [Theory] [InlineData("Auth:OAuth:IssuerTypo")] [InlineData("Auth:MethodTypo")] + [InlineData("Auth:Password:BurstSizeTypo")] [InlineData("Auth:OAuth:Issuer:Unexpected")] [InlineData("Auth:OAuth:Audiences:0:Unexpected")] [InlineData("Auth:OAuth:Scopes:Unexpected")] diff --git a/src/EventStore.Core.XUnit.Tests/OpenTelemetry/MetricNamesTests.cs b/src/EventStore.Core.XUnit.Tests/OpenTelemetry/MetricNamesTests.cs index 82d10ff68..9c2d15596 100644 --- a/src/EventStore.Core.XUnit.Tests/OpenTelemetry/MetricNamesTests.cs +++ b/src/EventStore.Core.XUnit.Tests/OpenTelemetry/MetricNamesTests.cs @@ -83,6 +83,7 @@ public void trogon_attribute_catalog_contains_all_custom_attributes_in_name_orde { "trogon.eventstore.activity.name", "trogon.eventstore.activity.outcome", + "trogon.eventstore.authentication.password.rejection.reason", "trogon.eventstore.cache.name", "trogon.eventstore.cache.resource", "trogon.eventstore.cache.result", diff --git a/src/EventStore.Core/Authentication/InternalAuthentication/InternalAuthenticationProvider.cs b/src/EventStore.Core/Authentication/InternalAuthentication/InternalAuthenticationProvider.cs index f17c45350..e5189a90d 100644 --- a/src/EventStore.Core/Authentication/InternalAuthentication/InternalAuthenticationProvider.cs +++ b/src/EventStore.Core/Authentication/InternalAuthentication/InternalAuthenticationProvider.cs @@ -24,6 +24,7 @@ public class InternalAuthenticationProvider : AuthenticationProviderBase, IHandl readonly IODispatcher _ioDispatcher; readonly bool _logFailedAuthenticationAttempts; readonly PasswordHashAlgorithm _passwordHashAlgorithm; + readonly PasswordAuthenticationLimiter _passwordAuthenticationLimiter; readonly LRUCache _userPasswordsCache; @@ -33,13 +34,16 @@ public InternalAuthenticationProvider( ISubscriber subscriber, IODispatcher ioDispatcher, PasswordHashAlgorithm passwordHashAlgorithm, int cacheSize, bool logFailedAuthenticationAttempts, - ClusterVNodeOptions.DefaultUserOptions defaultUserOptions + ClusterVNodeOptions.DefaultUserOptions defaultUserOptions, + ClusterVNodeOptions.PasswordAuthenticationOptions passwordAuthenticationOptions = null ) : base(name: "internal", diagnosticsName: "InternalAuthentication") { _ioDispatcher = ioDispatcher; _passwordHashAlgorithm = passwordHashAlgorithm; _userPasswordsCache = new LRUCache("UserPasswords", cacheSize); _logFailedAuthenticationAttempts = logFailedAuthenticationAttempts; + _passwordAuthenticationLimiter = new(passwordAuthenticationOptions ?? new()); + subscriber.Subscribe(new ShutdownHandler(_passwordAuthenticationLimiter)); var userManagement = new UserManagementService( ioDispatcher: ioDispatcher, @@ -66,28 +70,43 @@ ClusterVNodeOptions.DefaultUserOptions defaultUserOptions public void Handle(InternalAuthenticationProviderMessages.ResetPasswordCache message) => _userPasswordsCache.Remove(message.LoginName); - public override void Authenticate(AuthenticationRequest authenticationRequest) + public override void Authenticate(AuthenticationRequest authenticationRequest) => Authenticate(authenticationRequest, useCache: true); + + public void AuthenticateSession(AuthenticationRequest authenticationRequest) => Authenticate(authenticationRequest, useCache: false); + + void Authenticate(AuthenticationRequest authenticationRequest, bool useCache) { - if (_userPasswordsCache.TryGet(authenticationRequest.Name, out var cached)) + var lease = authenticationRequest.HasValidClientCertificate ? null : _passwordAuthenticationLimiter.TryAcquire(); + if (!authenticationRequest.HasValidClientCertificate && lease is null) + { + authenticationRequest.NotReady(); + return; + } + + try { - AuthenticateCached(authenticationRequest, cached.hash, cached.salt, cached.principal); + if (useCache && _userPasswordsCache.TryGet(authenticationRequest.Name, out var cached)) + { + AuthenticateCached(authenticationRequest, cached.hash, cached.salt, cached.principal); + } + else + { + var handler = new AuthReadResponseHandler(this, authenticationRequest, lease); + _ioDispatcher.ReadBackward($"$user-{authenticationRequest.Name}", -1, 1, false, + SystemAccounts.System, handler, Guid.NewGuid()); + lease = null; + } } - else + finally { - AuthenticateSession(authenticationRequest); + lease?.Dispose(); } } - public void AuthenticateSession(AuthenticationRequest authenticationRequest) => - _ioDispatcher.ReadBackward( - streamId: $"$user-{authenticationRequest.Name}", - fromEventNumber: -1, - maxCount: 1, - resolveLinks: false, - principal: SystemAccounts.System, - handler: new AuthReadResponseHandler(self: this, request: authenticationRequest), - corrId: Guid.NewGuid() - ); + sealed class ShutdownHandler(PasswordAuthenticationLimiter limiter) : IHandle + { + public void Handle(SystemMessage.BecomeShutdown message) => limiter.Dispose(); + } public override IReadOnlyList GetSupportedAuthenticationSchemes() => ["Basic", "UserCertificate"]; @@ -210,13 +229,16 @@ bool AuthenticateImpl(AuthenticationRequest authenticationRequest, string passwo public override Task Initialize() => _tcs.Task; - class AuthReadResponseHandler(InternalAuthenticationProvider self, AuthenticationRequest request) : IReadStreamEventsBackwardHandler + class AuthReadResponseHandler(InternalAuthenticationProvider self, AuthenticationRequest request, IDisposable lease) : IReadStreamEventsBackwardHandler { + int _completed; public bool HandlesAlt => true; public bool HandlesTimeout => true; public void Handle(ClientMessage.ReadStreamEventsBackwardCompleted completed) { + if (Interlocked.Exchange(ref _completed, 1) != 0) + return; try { if (completed.Result == ReadStreamResult.StreamDeleted || @@ -268,10 +290,17 @@ public void Handle(ClientMessage.ReadStreamEventsBackwardCompleted completed) { request.Unauthorized(); } + finally + { + lease?.Dispose(); + } } public void Handle(ClientMessage.NotHandled notHandled) { + if (Interlocked.Exchange(ref _completed, 1) != 0) + return; + using var acquired = lease; if (self._logFailedAuthenticationAttempts) { Logger.Warning( @@ -285,6 +314,9 @@ public void Handle(ClientMessage.NotHandled notHandled) public void Timeout() { + if (Interlocked.Exchange(ref _completed, 1) != 0) + return; + using var acquired = lease; if (self._logFailedAuthenticationAttempts) { Logger.Warning("Authentication Failed for {Id}: {Reason}", request.Id, "Timeout."); diff --git a/src/EventStore.Core/Authentication/InternalAuthentication/InternalAuthenticationProviderFactory.cs b/src/EventStore.Core/Authentication/InternalAuthentication/InternalAuthenticationProviderFactory.cs index 7d68b41b3..93d6dbf35 100644 --- a/src/EventStore.Core/Authentication/InternalAuthentication/InternalAuthenticationProviderFactory.cs +++ b/src/EventStore.Core/Authentication/InternalAuthentication/InternalAuthenticationProviderFactory.cs @@ -12,13 +12,17 @@ public class InternalAuthenticationProviderFactory : IAuthenticationProviderFact private readonly IODispatcher _dispatcher; private readonly Rfc2898PasswordHashAlgorithm _passwordHashAlgorithm; private readonly ClusterVNodeOptions.DefaultUserOptions _defaultUserOptions; + private readonly ClusterVNodeOptions.PasswordAuthenticationOptions _passwordAuthenticationOptions; - public InternalAuthenticationProviderFactory(AuthenticationProviderFactoryComponents components, ClusterVNodeOptions.DefaultUserOptions defaultUserOptions) + public InternalAuthenticationProviderFactory(AuthenticationProviderFactoryComponents components, + ClusterVNodeOptions.DefaultUserOptions defaultUserOptions, + ClusterVNodeOptions.PasswordAuthenticationOptions passwordAuthenticationOptions = null) { _components = components; _passwordHashAlgorithm = new(); _dispatcher = new(components.MainQueue, components.WorkersQueue); _defaultUserOptions = defaultUserOptions; + _passwordAuthenticationOptions = passwordAuthenticationOptions ?? new(); foreach (var bus in components.WorkerBuses) { @@ -42,7 +46,8 @@ public IAuthenticationProvider Build(bool logFailedAuthenticationAttempts) passwordHashAlgorithm: _passwordHashAlgorithm, cacheSize: ESConsts.CachedPrincipalCount, logFailedAuthenticationAttempts: logFailedAuthenticationAttempts, - defaultUserOptions: _defaultUserOptions + defaultUserOptions: _defaultUserOptions, + passwordAuthenticationOptions: _passwordAuthenticationOptions ); var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_components.MainQueue, _dispatcher); diff --git a/src/EventStore.Core/Authentication/InternalAuthentication/PasswordAuthenticationLimiter.cs b/src/EventStore.Core/Authentication/InternalAuthentication/PasswordAuthenticationLimiter.cs new file mode 100644 index 000000000..4c3224983 --- /dev/null +++ b/src/EventStore.Core/Authentication/InternalAuthentication/PasswordAuthenticationLimiter.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.RateLimiting; +using EventStore.Common.Exceptions; +using EventStore.Core.Diagnostics; +using TrogonEventStore.SemanticConventions; + +namespace EventStore.Core.Authentication.InternalAuthentication; + +internal sealed class PasswordAuthenticationLimiter : IDisposable +{ + readonly ConcurrencyLimiter _concurrency; + readonly TokenBucketRateLimiter _attempts; + readonly Meter _meter = new(TelemetryMeterInstrumentation.CoreName, TelemetryMeterInstrumentation.ScopeVersion); + readonly Counter _admitted; + readonly Counter _rejected; + readonly UpDownCounter _active; + + public PasswordAuthenticationLimiter(ClusterVNodeOptions.PasswordAuthenticationOptions options) + { + if (options.MaxConcurrentAttempts <= 0 || options.AttemptsPerSecond <= 0 || options.BurstSize <= 0) + throw new InvalidConfigurationException("Auth:Password authentication limits must be positive."); + if (options.BurstSize < options.AttemptsPerSecond) + throw new InvalidConfigurationException("Auth:Password:BurstSize must be greater than or equal to Auth:Password:AttemptsPerSecond."); + _concurrency = new(new ConcurrencyLimiterOptions { PermitLimit = options.MaxConcurrentAttempts, QueueLimit = 0 }); + _attempts = new(new TokenBucketRateLimiterOptions + { + TokenLimit = options.BurstSize, + TokensPerPeriod = options.AttemptsPerSecond, + ReplenishmentPeriod = TimeSpan.FromSeconds(1), + AutoReplenishment = false, + QueueLimit = 0 + }); + var admitted = MetricDefinitions.TrogonEventstoreAuthenticationPasswordAdmitted; + var rejected = MetricDefinitions.TrogonEventstoreAuthenticationPasswordRejected; + var active = MetricDefinitions.TrogonEventstoreAuthenticationPasswordActive; + _admitted = _meter.CreateCounter(admitted.Name, admitted.Unit, admitted.Description); + _rejected = _meter.CreateCounter(rejected.Name, rejected.Unit, rejected.Description); + _active = _meter.CreateUpDownCounter(active.Name, active.Unit, active.Description); + } + + public IDisposable TryAcquire() + { + try + { + return Acquire(); + } + catch (ObjectDisposedException) + { + return null; + } + } + + IDisposable Acquire() + { + _attempts.TryReplenish(); + using var attempt = _attempts.AttemptAcquire(); + if (!attempt.IsAcquired) + { + _rejected.Add(1, new KeyValuePair(TrogonAttributeNames.AuthenticationPasswordRejectionReason, "rate")); + return null; + } + + var concurrency = _concurrency.AttemptAcquire(); + if (!concurrency.IsAcquired) + { + concurrency.Dispose(); + _rejected.Add(1, new KeyValuePair(TrogonAttributeNames.AuthenticationPasswordRejectionReason, "concurrency")); + return null; + } + + _admitted.Add(1); + _active.Add(1); + return new Lease(concurrency, _active); + } + + public void Dispose() + { + _attempts.Dispose(); + _concurrency.Dispose(); + _meter.Dispose(); + } + + sealed class Lease(RateLimitLease lease, UpDownCounter active) : IDisposable + { + RateLimitLease _lease = lease; + public void Dispose() + { + var acquired = Interlocked.Exchange(ref _lease, null); + if (acquired is null) + return; + active.Add(-1); + acquired.Dispose(); + } + } +} diff --git a/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs b/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs index 66047ed25..f4d51719a 100644 --- a/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs +++ b/src/EventStore.Core/Configuration/ClusterVNodeOptions.cs @@ -259,6 +259,21 @@ public record AuthOptions [Description("OAuth authentication options.")] public OAuthOptions OAuth { get; init; } = new(); + [Description("Per-node password authentication admission limits.")] + public PasswordAuthenticationOptions Password { get; init; } = new(); + + } + + public record PasswordAuthenticationOptions + { + [Description("Maximum password authentication attempts in flight per node, including account reads. Excess attempts are rejected without queuing.")] + public int MaxConcurrentAttempts { get; init; } = 4; + + [Description("Password authentication attempt tokens replenished per second per node.")] + public int AttemptsPerSecond { get; init; } = 100; + + [Description("Maximum password authentication burst tokens per node. Must be at least AttemptsPerSecond.")] + public int BurstSize { get; init; } = 200; } public record OAuthOptions diff --git a/src/TrogonEventStore.SemanticConventions/Generated/MetricDefinitions.g.cs b/src/TrogonEventStore.SemanticConventions/Generated/MetricDefinitions.g.cs index 8d9fbcb11..8fe8c6222 100644 --- a/src/TrogonEventStore.SemanticConventions/Generated/MetricDefinitions.g.cs +++ b/src/TrogonEventStore.SemanticConventions/Generated/MetricDefinitions.g.cs @@ -32,6 +32,21 @@ public static class MetricDefinitions "{retry}", "Number of retries initiated by archive operations.", MetricInstrumentKind.Counter); + public static MetricDefinition TrogonEventstoreAuthenticationPasswordActive { get; } = new MetricDefinition( + "trogon.eventstore.authentication.password.active", + "{attempt}", + "Number of admitted password authentication attempts still being processed.", + MetricInstrumentKind.UpDownCounter); + public static MetricDefinition TrogonEventstoreAuthenticationPasswordAdmitted { get; } = new MetricDefinition( + "trogon.eventstore.authentication.password.admitted", + "{attempt}", + "Number of password authentication attempts admitted for processing.", + MetricInstrumentKind.Counter); + public static MetricDefinition TrogonEventstoreAuthenticationPasswordRejected { get; } = new MetricDefinition( + "trogon.eventstore.authentication.password.rejected", + "{attempt}", + "Number of password authentication attempts rejected by admission limits.", + MetricInstrumentKind.Counter); public static MetricDefinition TrogonEventstoreCacheOperationCount { get; } = new MetricDefinition( "trogon.eventstore.cache.operation.count", "{operation}", @@ -250,6 +265,9 @@ public static class MetricDefinitions TrogonEventstoreArchiveFailureCount, TrogonEventstoreArchiveReadDuration, TrogonEventstoreArchiveRetryCount, + TrogonEventstoreAuthenticationPasswordActive, + TrogonEventstoreAuthenticationPasswordAdmitted, + TrogonEventstoreAuthenticationPasswordRejected, TrogonEventstoreCacheOperationCount, TrogonEventstoreCacheResourceCount, TrogonEventstoreCacheResourceSize, diff --git a/src/TrogonEventStore.SemanticConventions/Generated/TrogonAttributeNames.g.cs b/src/TrogonEventStore.SemanticConventions/Generated/TrogonAttributeNames.g.cs index 8fa1b61f4..b1d45a861 100644 --- a/src/TrogonEventStore.SemanticConventions/Generated/TrogonAttributeNames.g.cs +++ b/src/TrogonEventStore.SemanticConventions/Generated/TrogonAttributeNames.g.cs @@ -9,6 +9,7 @@ public static class TrogonAttributeNames { public const string ActivityName = "trogon.eventstore.activity.name"; public const string ActivityOutcome = "trogon.eventstore.activity.outcome"; + public const string AuthenticationPasswordRejectionReason = "trogon.eventstore.authentication.password.rejection.reason"; public const string CacheName = "trogon.eventstore.cache.name"; public const string CacheResource = "trogon.eventstore.cache.resource"; public const string CacheResult = "trogon.eventstore.cache.result"; @@ -32,6 +33,7 @@ public static class TrogonAttributeNames { ActivityName, ActivityOutcome, + AuthenticationPasswordRejectionReason, CacheName, CacheResource, CacheResult,