Gracefully shutdown app

src/Altinn.App.Api/Extensions/ServiceCollectionExtensions.cs

@@ -6,6 +6,7 @@
 using Altinn.App.Api.Helpers.Patch;
 using Altinn.App.Api.Infrastructure.Filters;
 using Altinn.App.Api.Infrastructure.Health;
+using Altinn.App.Api.Infrastructure.Lifetime;
 using Altinn.App.Api.Infrastructure.Middleware;
 using Altinn.App.Api.Infrastructure.Telemetry;
 using Altinn.App.Core.Constants;
@@ -124,6 +125,8 @@ IWebHostEnvironment env
             options.AllowSynchronousIO = true;
         });
 
+        ConfigureGracefulShutdown(services);
+
         // HttpClients for platform functionality. Registered as HttpClient so default HttpClientFactory is used
         services.AddHttpClient<AuthorizationApiClient>();
 
@@ -546,4 +549,34 @@ IHostEnvironment env
 
         return $"InstrumentationKey={key}";
     }
+
+    private static void ConfigureGracefulShutdown(IServiceCollection services)
+    {
+        // Need to coordinate graceful shutdown (let's assume k8s as the scheduler/runtime):
+        // - deployment is configured with a terminationGracePeriod of 30s (default timeout before SIGKILL)
+        // - k8s flow of information is eventually consistent.
+        //   it takes time for knowledge of SIGTERM on the worker node to propagate to e.g. networking layers
+        //   (k8s Service -> Endspoints rotation. It takes time to be taken out of Endpoint rotation)
+        // - we want to gracefully drain ASP.NET core for requests, leaving some time for active requests to complete
+        // This leaves us with the following sequence of events
+        // - container receives SIGTERM
+        // - `AppHostLifetime` intercepts SIGTERM and delays for `shutdownDelay`
+        // - `AppHostLifetime` calls `IHostApplicationLifetime.StopApplication`, to start ASP.NET Core shutdown process
+        // - ASP.NET Core will spend a maximum of `shutdownTimeout` trying to drain active requests
+        //   (cancelable requests can combine cancellation tokens with `IHostApplicationLifetime.ApplicationStopping`)
+        // - If ASP.NET Core completes shutdown within `shutdownTimeout`, everything is fine
+        // - If ASP.NET Core is stuck or in some way can't terminate, kubelet will eventually SIGKILL
+        var shutdownDelay = TimeSpan.FromSeconds(5);
+        var shutdownTimeout = TimeSpan.FromSeconds(20);
+
+        services.AddSingleton<IHostLifetime>(serviceProvider =>
+        {
+            var logger = serviceProvider.GetRequiredService<ILogger<AppHostLifetime>>();
+            var env = serviceProvider.GetRequiredService<IHostEnvironment>();
+            var lifetime = serviceProvider.GetRequiredService<IHostApplicationLifetime>();
+            return new AppHostLifetime(logger, env, lifetime, shutdownDelay);
+        });
+
+        services.Configure<HostOptions>(options => options.ShutdownTimeout = shutdownTimeout);
+    }
 }

src/Altinn.App.Api/Infrastructure/Lifetime/AppHostLifetime.cs

@@ -0,0 +1,49 @@
+using System.Runtime.InteropServices;
+
+namespace Altinn.App.Api.Infrastructure.Lifetime;
+
+// Based on guidance in:
+// https://github.com/dotnet/dotnet-docker/blob/2a6f35b9361d1aacb664b0ce09e529698b622d2b/samples/kubernetes/graceful-shutdown/graceful-shutdown.md
+internal sealed class AppHostLifetime(
+    ILogger<AppHostLifetime> _logger,
+    IHostEnvironment _environment,
+    IHostApplicationLifetime _applicationLifetime,
+    TimeSpan _delay
+) : IHostLifetime, IDisposable
+{
+    private IEnumerable<IDisposable>? _disposables;
+
+    public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+
+    public Task WaitForStartAsync(CancellationToken cancellationToken)
+    {
+        if (!_environment.IsDevelopment())
+        {
+            _disposables =
+            [
+                PosixSignalRegistration.Create(PosixSignal.SIGINT, HandleSignal),
+                PosixSignalRegistration.Create(PosixSignal.SIGQUIT, HandleSignal),
+                PosixSignalRegistration.Create(PosixSignal.SIGTERM, HandleSignal),
+            ];
+        }
+        return Task.CompletedTask;
+    }
+
+    private void HandleSignal(PosixSignalContext ctx)
+    {
+        _logger.LogInformation("Received shutdown signal: {Signal}, delaying shutdown", ctx.Signal);
+        ctx.Cancel = true; // Signal intercepted here, we are now responsible for calling `StopApplication`
+        Task.Delay(_delay)
+            .ContinueWith(t =>
+            {
+                _logger.LogInformation("Starting host shutdown...");
+                _applicationLifetime.StopApplication();
+            });
+    }
+
+    public void Dispose()
+    {
+        foreach (var disposable in _disposables ?? [])
+            disposable.Dispose();
+    }
+}