Enhance GraphQL OTEL instrumentation with custom metrics and traces

src/Core/Services/BuildRequestStateMiddleware.cs

@@ -1,10 +1,18 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Diagnostics;
+using System.Net;
+using System.Net.Mail;
+using Azure.DataApiBuilder.Config.ObjectModel;
 using Azure.DataApiBuilder.Core.Authorization;
+using Azure.DataApiBuilder.Core.Configurations;
+using Azure.DataApiBuilder.Core.Telemetry;
 using HotChocolate.Execution;
+using HotChocolate.Language;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Primitives;
+using Kestral = Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod;
 using RequestDelegate = HotChocolate.Execution.RequestDelegate;
 
 /// <summary>
@@ -13,10 +21,12 @@
 public sealed class BuildRequestStateMiddleware
 {
     private readonly RequestDelegate _next;
+    private readonly RuntimeConfigProvider _runtimeConfigProvider;
 
-    public BuildRequestStateMiddleware(RequestDelegate next)
+    public BuildRequestStateMiddleware(RequestDelegate next, RuntimeConfigProvider runtimeConfigProvider)
     {
         _next = next;
+        _runtimeConfigProvider = runtimeConfigProvider;
     }
 
     /// <summary>
@@ -26,14 +36,97 @@ public BuildRequestStateMiddleware(RequestDelegate next)
     /// <param name="context">HotChocolate execution request context.</param>
     public async ValueTask InvokeAsync(IRequestContext context)
     {
-        if (context.ContextData.TryGetValue(nameof(HttpContext), out object? value) &&
-            value is HttpContext httpContext)
+        bool isIntrospectionQuery = context.Request.OperationName == "IntrospectionQuery";
+        ApiType apiType = ApiType.GraphQL;
+        Kestral method = Kestral.Post;
+        string route = _runtimeConfigProvider.GetConfig().GraphQLPath.Trim('/');
+        DefaultHttpContext httpContext = (DefaultHttpContext)context.ContextData.First(x => x.Key == "HttpContext").Value!;
+        Stopwatch stopwatch = Stopwatch.StartNew();
+
+        using Activity? activity = !isIntrospectionQuery ?
+            TelemetryTracesHelper.DABActivitySource.StartActivity($"{method} /{route}") : null;
+
+        try
         {
-            // Because Request.Headers is a NameValueCollection type, key not found will return StringValues.Empty and not an exception.
-            StringValues clientRoleHeader = httpContext.Request.Headers[AuthorizationResolver.CLIENT_ROLE_HEADER];
-            context.ContextData.TryAdd(key: AuthorizationResolver.CLIENT_ROLE_HEADER, value: clientRoleHeader);
+            // We want to ignore introspection queries DAB uses to check access to GraphQL since they are not sent by the user.
+            if (!isIntrospectionQuery)
+            {
+                TelemetryMetricsHelper.IncrementActiveRequests(apiType);
+                if (activity is not null)
+                {
+                    activity.TrackMainControllerActivityStarted(
+                        httpMethod: method,
+                        userAgent: httpContext.Request.Headers["User-Agent"].ToString(),
+                        actionType: (context.Request.Query!.ToString().Contains("mutation") ? OperationType.Mutation : OperationType.Query).ToString(),
+                        httpURL: string.Empty, // GraphQL has no route
+                        queryString: null, // GraphQL has no query-string
+                        userRole: httpContext.Request.Headers["X-MS-API-ROLE"].FirstOrDefault() ?? httpContext.User.FindFirst("role")?.Value,
+                        apiType: apiType);
+                }
+            }
+
+            await InvokeAsync();
+        }
+        finally
+        {
+            stopwatch.Stop();
+
+            HttpStatusCode statusCode;
+
+            // We want to ignore introspection queries DAB uses to check access to GraphQL since they are not sent by the user.
+            if (!isIntrospectionQuery)
+            {
+                // There is an error in GraphQL when ContextData is not null
+                if (context.Result!.ContextData is not null)
+                {
+                    if (context.Result.ContextData.ContainsKey(WellKnownContextData.ValidationErrors))
+                    {
+                        statusCode = HttpStatusCode.BadRequest;
+                    }
+                    else if (context.Result.ContextData.ContainsKey(WellKnownContextData.OperationNotAllowed))
+                    {
+                        statusCode = HttpStatusCode.MethodNotAllowed;
+                    }
+                    else
+                    {
+                        statusCode = HttpStatusCode.InternalServerError;
+                    }
+
+                    Exception ex = new();
+                    if (context.Result.Errors is not null)
+                    {
+                        string errorMessage = context.Result.Errors[0].Message;
+                        ex = new(errorMessage);
+                    }
+
+                    // Activity will track error
+                    activity?.TrackControllerActivityFinishedWithException(ex, statusCode);
+                    TelemetryMetricsHelper.TrackError(method, statusCode, route, apiType, ex);
+                }
+                else
+                {
+                    statusCode = HttpStatusCode.OK;
+                    activity?.TrackControllerActivityFinished(statusCode);
+                }
+
+                TelemetryMetricsHelper.TrackRequest(method, statusCode, route, apiType);
+                TelemetryMetricsHelper.TrackRequestDuration(method, statusCode, route, apiType, stopwatch.Elapsed);
+                TelemetryMetricsHelper.DecrementActiveRequests(apiType);
+            }
         }
 
-        await _next(context).ConfigureAwait(false);
+        async Task InvokeAsync()
+        {
+            if (context.ContextData.TryGetValue(nameof(HttpContext), out object? value) &&
+                value is HttpContext httpContext)
+            {
+                // Because Request.Headers is a NameValueCollection type, key not found will return StringValues.Empty and not an exception.
+                StringValues clientRoleHeader = httpContext.Request.Headers[AuthorizationResolver.CLIENT_ROLE_HEADER];
+                context.ContextData.TryAdd(key: AuthorizationResolver.CLIENT_ROLE_HEADER, value: clientRoleHeader);
+            }
+
+            await _next(context).ConfigureAwait(false);
+        }
     }
 }
+

src/Core/Services/ExecutionHelper.cs

@@ -1,13 +1,15 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Diagnostics;
 using System.Globalization;
 using System.Text.Json;
 using Azure.DataApiBuilder.Config.ObjectModel;
 using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Core.Resolvers;
 using Azure.DataApiBuilder.Core.Resolvers.Factories;
+using Azure.DataApiBuilder.Core.Telemetry;
 using Azure.DataApiBuilder.Service.GraphQLBuilder;
 using Azure.DataApiBuilder.Service.GraphQLBuilder.CustomScalars;
 using Azure.DataApiBuilder.Service.GraphQLBuilder.GraphQLTypes;
@@ -17,6 +19,7 @@
 using HotChocolate.Resolvers;
 using HotChocolate.Types.NodaTime;
 using NodaTime.Text;
+using Kestral = Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod;
 
 namespace Azure.DataApiBuilder.Service.Services
 {
@@ -50,6 +53,8 @@ public ExecutionHelper(
         /// </param>
         public async ValueTask ExecuteQueryAsync(IMiddlewareContext context)
         {
+            using Activity? activity = StartQueryActivity(context);
+
             string dataSourceName = GraphQLUtils.GetDataSourceNameFromGraphQLContext(context, _runtimeConfigProvider.GetConfig());
             DataSource ds = _runtimeConfigProvider.GetConfig().GetDataSourceFromDataSourceName(dataSourceName);
             IQueryEngine queryEngine = _queryEngineFactory.GetQueryEngine(ds.DatabaseType);
@@ -91,6 +96,8 @@ public async ValueTask ExecuteQueryAsync(IMiddlewareContext context)
         /// </param>
         public async ValueTask ExecuteMutateAsync(IMiddlewareContext context)
         {
+            using Activity? activity = StartQueryActivity(context);
+
             string dataSourceName = GraphQLUtils.GetDataSourceNameFromGraphQLContext(context, _runtimeConfigProvider.GetConfig());
             DataSource ds = _runtimeConfigProvider.GetConfig().GetDataSourceFromDataSourceName(dataSourceName);
             IQueryEngine queryEngine = _queryEngineFactory.GetQueryEngine(ds.DatabaseType);
@@ -127,6 +134,31 @@ public async ValueTask ExecuteMutateAsync(IMiddlewareContext context)
             }
         }
 
+        /// <summary>
+        /// Starts the activity for the query
+        /// </summary>
+        /// <param name="context">
+        /// The middleware context.
+        /// </param>
+        private Activity? StartQueryActivity(IMiddlewareContext context)
+        {
+            string route = _runtimeConfigProvider.GetConfig().GraphQLPath.Trim('/');
+            Kestral method = Kestral.Post;
+
+            Activity? activity = TelemetryTracesHelper.DABActivitySource.StartActivity($"{method} /{route}");
+
+            if (activity is not null)
+            {
+                string dataSourceName = GraphQLUtils.GetDataSourceNameFromGraphQLContext(context, _runtimeConfigProvider.GetConfig());
+                DataSource ds = _runtimeConfigProvider.GetConfig().GetDataSourceFromDataSourceName(dataSourceName);
+                activity.TrackQueryActivityStarted(
+                    databaseType: ds.DatabaseType,
+                    dataSourceName: dataSourceName);
+            }
+
+            return activity;
+        }
+
         /// <summary>
         /// Represents a pure resolver for a leaf field.
         /// This resolver extracts the field value from the json object.
@@ -425,7 +457,7 @@ internal static IType InnerMostType(IType type)
 
         public static InputObjectType InputObjectTypeFromIInputField(IInputField field)
         {
-            return (InputObjectType)(InnerMostType(field.Type));
+            return (InputObjectType)InnerMostType(field.Type);
         }
 
         /// <summary>

src/Service/Telemetry/TelemetryMetricsHelper.cs → src/Core/Telemetry/TelemetryMetricsHelper.cs

@@ -1,14 +1,12 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System;
-using System.Collections.Generic;
 using System.Diagnostics.Metrics;
 using System.Net;
 using Azure.DataApiBuilder.Config.ObjectModel;
 using Kestral = Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod;
 
-namespace Azure.DataApiBuilder.Service.Telemetry
+namespace Azure.DataApiBuilder.Core.Telemetry
 {
     /// <summary>
     /// Helper class for tracking telemetry metrics such as active requests, errors, total requests,

src/Service/Telemetry/TelemetryTracesHelper.cs → src/Core/Telemetry/TelemetryTracesHelper.cs

@@ -1,14 +1,13 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System;
 using System.Diagnostics;
 using System.Net;
 using Azure.DataApiBuilder.Config.ObjectModel;
 using OpenTelemetry.Trace;
 using Kestral = Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod;
 
-namespace Azure.DataApiBuilder.Service.Telemetry
+namespace Azure.DataApiBuilder.Core.Telemetry
 {
     public static class TelemetryTracesHelper
     {
@@ -18,7 +17,7 @@ public static class TelemetryTracesHelper
         public static readonly ActivitySource DABActivitySource = new("DataApiBuilder");
 
         /// <summary>
-        /// Tracks the start of a REST controller activity.
+        /// Tracks the start of a controller activity.
         /// </summary>
         /// <param name="activity">The activity instance.</param>
         /// <param name="httpMethod">The HTTP method of the request (e.g., GET, POST).</param>
@@ -28,11 +27,11 @@ public static class TelemetryTracesHelper
         /// <param name="queryString">The query string of the request, if any.</param>
         /// <param name="userRole">The role of the user making the request.</param>
         /// <param name="apiType">The type of API being used (e.g., REST, GraphQL).</param>
-        public static void TrackRestControllerActivityStarted(
+        public static void TrackMainControllerActivityStarted(
             this Activity activity,
             Kestral httpMethod,
             string userAgent,
-            string actionType,
+            string actionType, // CRUD(EntityActionOperation) for REST, Query|Mutation(OperationType) for GraphQL
             string httpURL,
             string? queryString,
             string? userRole,
@@ -78,11 +77,11 @@ public static void TrackQueryActivityStarted(
         }
 
         /// <summary>
-        /// Tracks the completion of a REST controller activity.
+        /// Tracks the completion of a controller activity without any exceptions.
         /// </summary>
         /// <param name="activity">The activity instance.</param>
         /// <param name="statusCode">The HTTP status code of the response.</param>
-        public static void TrackRestControllerActivityFinished(
+        public static void TrackControllerActivityFinished(
             this Activity activity,
             HttpStatusCode statusCode)
         {
@@ -93,12 +92,12 @@ public static void TrackRestControllerActivityFinished(
         }
 
         /// <summary>
-        /// Tracks the completion of a REST controller activity with an exception.
+        /// Tracks the completion of a controller activity with an exception.
         /// </summary>
         /// <param name="activity">The activity instance.</param>
         /// <param name="ex">The exception that occurred.</param>
         /// <param name="statusCode">The HTTP status code of the response.</param>
-        public static void TrackRestControllerActivityFinishedWithException(
+        public static void TrackControllerActivityFinishedWithException(
             this Activity activity,
             Exception ex,
             HttpStatusCode statusCode)

src/Service/Controllers/RestController.cs

@@ -11,8 +11,8 @@
 using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Core.Services;
+using Azure.DataApiBuilder.Core.Telemetry;
 using Azure.DataApiBuilder.Service.Exceptions;
-using Azure.DataApiBuilder.Service.Telemetry;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
@@ -208,7 +208,7 @@ private async Task<IActionResult> HandleOperation(
 
                 if (activity is not null)
                 {
-                    activity.TrackRestControllerActivityStarted(
+                    activity.TrackMainControllerActivityStarted(
                         Enum.Parse<HttpMethod>(HttpContext.Request.Method, ignoreCase: true),
                         HttpContext.Request.Headers["User-Agent"].ToString(),
                         operationType.ToString(),
@@ -261,7 +261,7 @@ private async Task<IActionResult> HandleOperation(
                 if (activity is not null && activity.IsAllDataRequested)
                 {
                     HttpStatusCode httpStatusCode = Enum.Parse<HttpStatusCode>(statusCode.ToString(), ignoreCase: true);
-                    activity.TrackRestControllerActivityFinished(httpStatusCode);
+                    activity.TrackControllerActivityFinished(httpStatusCode);
                 }
 
                 return result;
@@ -274,7 +274,7 @@ private async Task<IActionResult> HandleOperation(
                     HttpContextExtensions.GetLoggerCorrelationId(HttpContext));
 
                 Response.StatusCode = (int)ex.StatusCode;
-                activity?.TrackRestControllerActivityFinishedWithException(ex, ex.StatusCode);
+                activity?.TrackControllerActivityFinishedWithException(ex, ex.StatusCode);
 
                 HttpMethod method = Enum.Parse<HttpMethod>(HttpContext.Request.Method, ignoreCase: true);
                 TelemetryMetricsHelper.TrackError(method, ex.StatusCode, route, ApiType.REST, ex);
@@ -290,7 +290,7 @@ private async Task<IActionResult> HandleOperation(
                 Response.StatusCode = (int)HttpStatusCode.InternalServerError;
 
                 HttpMethod method = Enum.Parse<HttpMethod>(HttpContext.Request.Method, ignoreCase: true);
-                activity?.TrackRestControllerActivityFinishedWithException(ex, HttpStatusCode.InternalServerError);
+                activity?.TrackControllerActivityFinishedWithException(ex, HttpStatusCode.InternalServerError);
 
                 TelemetryMetricsHelper.TrackError(method, HttpStatusCode.InternalServerError, route, ApiType.REST, ex);
                 return ErrorResponse(

src/Service/Startup.cs

@@ -21,6 +21,7 @@
 using Azure.DataApiBuilder.Core.Services.Cache;
 using Azure.DataApiBuilder.Core.Services.MetadataProviders;
 using Azure.DataApiBuilder.Core.Services.OpenAPI;
+using Azure.DataApiBuilder.Core.Telemetry;
 using Azure.DataApiBuilder.Service.Controllers;
 using Azure.DataApiBuilder.Service.Exceptions;
 using Azure.DataApiBuilder.Service.HealthCheck;