diff --git a/src/Altinn.App.Api/Extensions/ServiceCollectionExtensions.cs b/src/Altinn.App.Api/Extensions/ServiceCollectionExtensions.cs index 746ef397c..242c9d8f3 100644 --- a/src/Altinn.App.Api/Extensions/ServiceCollectionExtensions.cs +++ b/src/Altinn.App.Api/Extensions/ServiceCollectionExtensions.cs @@ -6,6 +6,7 @@ using Altinn.App.Api.Helpers.Patch; using Altinn.App.Api.Infrastructure.Filters; using Altinn.App.Api.Infrastructure.Health; +using Altinn.App.Api.Infrastructure.Lifetime; using Altinn.App.Api.Infrastructure.Middleware; using Altinn.App.Api.Infrastructure.Telemetry; using Altinn.App.Core.Constants; @@ -124,6 +125,8 @@ IWebHostEnvironment env options.AllowSynchronousIO = true; }); + ConfigureGracefulShutdown(services); + // HttpClients for platform functionality. Registered as HttpClient so default HttpClientFactory is used services.AddHttpClient(); @@ -546,4 +549,34 @@ IHostEnvironment env return $"InstrumentationKey={key}"; } + + private static void ConfigureGracefulShutdown(IServiceCollection services) + { + // Need to coordinate graceful shutdown (let's assume k8s as the scheduler/runtime): + // - deployment is configured with a terminationGracePeriod of 30s (default timeout before SIGKILL) + // - k8s flow of information is eventually consistent. + // it takes time for knowledge of SIGTERM on the worker node to propagate to e.g. networking layers + // (k8s Service -> Endspoints rotation. It takes time to be taken out of Endpoint rotation) + // - we want to gracefully drain ASP.NET core for requests, leaving some time for active requests to complete + // This leaves us with the following sequence of events + // - container receives SIGTERM + // - `AppHostLifetime` intercepts SIGTERM and delays for `shutdownDelay` + // - `AppHostLifetime` calls `IHostApplicationLifetime.StopApplication`, to start ASP.NET Core shutdown process + // - ASP.NET Core will spend a maximum of `shutdownTimeout` trying to drain active requests + // (cancelable requests can combine cancellation tokens with `IHostApplicationLifetime.ApplicationStopping`) + // - If ASP.NET Core completes shutdown within `shutdownTimeout`, everything is fine + // - If ASP.NET Core is stuck or in some way can't terminate, kubelet will eventually SIGKILL + var shutdownDelay = TimeSpan.FromSeconds(5); + var shutdownTimeout = TimeSpan.FromSeconds(20); + + services.AddSingleton(serviceProvider => + { + var logger = serviceProvider.GetRequiredService>(); + var env = serviceProvider.GetRequiredService(); + var lifetime = serviceProvider.GetRequiredService(); + return new AppHostLifetime(logger, env, lifetime, shutdownDelay); + }); + + services.Configure(options => options.ShutdownTimeout = shutdownTimeout); + } } diff --git a/src/Altinn.App.Api/Infrastructure/Lifetime/AppHostLifetime.cs b/src/Altinn.App.Api/Infrastructure/Lifetime/AppHostLifetime.cs new file mode 100644 index 000000000..726aa5fb1 --- /dev/null +++ b/src/Altinn.App.Api/Infrastructure/Lifetime/AppHostLifetime.cs @@ -0,0 +1,49 @@ +using System.Runtime.InteropServices; + +namespace Altinn.App.Api.Infrastructure.Lifetime; + +// Based on guidance in: +// https://github.com/dotnet/dotnet-docker/blob/2a6f35b9361d1aacb664b0ce09e529698b622d2b/samples/kubernetes/graceful-shutdown/graceful-shutdown.md +internal sealed class AppHostLifetime( + ILogger _logger, + IHostEnvironment _environment, + IHostApplicationLifetime _applicationLifetime, + TimeSpan _delay +) : IHostLifetime, IDisposable +{ + private IEnumerable? _disposables; + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public Task WaitForStartAsync(CancellationToken cancellationToken) + { + if (!_environment.IsDevelopment()) + { + _disposables = + [ + PosixSignalRegistration.Create(PosixSignal.SIGINT, HandleSignal), + PosixSignalRegistration.Create(PosixSignal.SIGQUIT, HandleSignal), + PosixSignalRegistration.Create(PosixSignal.SIGTERM, HandleSignal), + ]; + } + return Task.CompletedTask; + } + + private void HandleSignal(PosixSignalContext ctx) + { + _logger.LogInformation("Received shutdown signal: {Signal}, delaying shutdown", ctx.Signal); + ctx.Cancel = true; // Signal intercepted here, we are now responsible for calling `StopApplication` + Task.Delay(_delay) + .ContinueWith(t => + { + _logger.LogInformation("Starting host shutdown..."); + _applicationLifetime.StopApplication(); + }); + } + + public void Dispose() + { + foreach (var disposable in _disposables ?? []) + disposable.Dispose(); + } +}