2554 Enhance OTEL instrumentation with custom metrics and traces

src/Core/Resolvers/SqlQueryEngine.cs

@@ -332,6 +332,7 @@ public object ResolveList(JsonElement array, IObjectField fieldSchema, ref IMeta
                     JsonElement result = await _cache.GetOrSetAsync<JsonElement>(queryExecutor, queryMetadata, cacheEntryTtl: runtimeConfig.GetEntityCacheEntryTtl(entityName: structure.EntityName));
                     byte[] jsonBytes = JsonSerializer.SerializeToUtf8Bytes(result);
                     JsonDocument cacheServiceResponse = JsonDocument.Parse(jsonBytes);
+
                     return cacheServiceResponse;
                 }
             }
@@ -348,6 +349,7 @@ public object ResolveList(JsonElement array, IObjectField fieldSchema, ref IMeta
                 httpContext: _httpContextAccessor.HttpContext!,
                 args: null,
                 dataSourceName: dataSourceName);
+
             return response;
         }
 

src/Service.Tests/Configuration/OpenTelemetryTests.cs

@@ -61,7 +61,6 @@ public void CleanUpTelemetryConfig()
             File.Delete(CONFIG_WITHOUT_TELEMETRY);
         }
 
-        Startup.OpenTelemetryOptions = new();
     }
 
     /// <summary>

src/Service/Azure.DataApiBuilder.Service.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
     <PropertyGroup>
         <TargetFrameworks>net8.0</TargetFrameworks>
@@ -77,7 +77,7 @@
         <PackageReference Include="System.CommandLine" />
         <PackageReference Include="System.IO.Abstractions" />
         <PackageReference Include="ZiggyCreatures.FusionCache" />
-        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol"  />
+        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
         <PackageReference Include="OpenTelemetry.Extensions.Hosting" />
         <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
         <PackageReference Include="OpenTelemetry.Instrumentation.Http" />

src/Service/Controllers/RestController.cs

@@ -2,13 +2,16 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Diagnostics;
 using System.Net;
 using System.Net.Mime;
 using System.Threading.Tasks;
 using Azure.DataApiBuilder.Config.ObjectModel;
+using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Core.Services;
 using Azure.DataApiBuilder.Service.Exceptions;
+using Azure.DataApiBuilder.Service.Telemetry;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
@@ -47,11 +50,14 @@ public class RestController : ControllerBase
 
         private readonly ILogger<RestController> _logger;
 
+        private readonly RuntimeConfigProvider _runtimeConfigProvider;
+
         /// <summary>
         /// Constructor.
         /// </summary>
-        public RestController(RestService restService, IOpenApiDocumentor openApiDocumentor, ILogger<RestController> logger)
+        public RestController(RuntimeConfigProvider runtimeConfigProvider, RestService restService, IOpenApiDocumentor openApiDocumentor, ILogger<RestController> logger)
         {
+            _runtimeConfigProvider = runtimeConfigProvider;
             _restService = restService;
             _openApiDocumentor = openApiDocumentor;
             _logger = logger;
@@ -179,14 +185,30 @@ public async Task<IActionResult> UpsertIncremental(
         /// <summary>
         /// Handle the given operation.
         /// </summary>
+        /// <param name="method">The method.</param>
         /// <param name="route">The entire route.</param>
         /// <param name="operationType">The kind of operation to handle.</param>
         private async Task<IActionResult> HandleOperation(
             string route,
             EntityActionOperation operationType)
         {
-            try
+            var stopwatch = Stopwatch.StartNew();
+            using Activity? activity = TelemetryTracesHelper.DABActivitySource.StartActivity($"{HttpContext.Request.Method} {route.Split('/')[1]}");
+            if(activity is not null)
             {
+                activity.TrackRestControllerActivityStarted(HttpContext.Request.Method,
+                    HttpContext.Request.Headers["User-Agent"].ToString(),
+                    operationType.ToString(),
+                    route,
+                    HttpContext.Request.QueryString.ToString(),
+                    HttpContext.User.FindFirst("role")?.Value,
+                    "REST");
+            }
+
+            TelemetryMetricsHelper.IncrementActiveRequests();
+            try
+            { 
+
                 if (route.Equals(REDIRECTED_ROUTE))
                 {
                     throw new DataApiBuilderException(
@@ -211,8 +233,25 @@ private async Task<IActionResult> HandleOperation(
 
                 (string entityName, string primaryKeyRoute) = _restService.GetEntityNameAndPrimaryKeyRouteFromRoute(routeAfterPathBase);
 
+                using Activity? queryActivity = TelemetryTracesHelper.DABActivitySource.StartActivity($"QUERY {entityName}");
                 IActionResult? result = await _restService.ExecuteAsync(entityName, operationType, primaryKeyRoute);
 
+                RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig();
+                string dataSourceName = runtimeConfig.GetDataSourceNameFromEntityName(entityName);
+                DatabaseType databaseType = runtimeConfig.GetDataSourceFromDataSourceName(dataSourceName).DatabaseType;
+
+                if (queryActivity is not null)
+                {
+                    queryActivity.TrackQueryActivityStarted(
+                        databaseType.ToString(),
+                        dataSourceName);
+                }
+
+                if (queryActivity is not null && queryActivity.IsAllDataRequested)
+                {
+                    queryActivity.Dispose();
+                }
+
                 if (result is null)
                 {
                     throw new DataApiBuilderException(
@@ -221,6 +260,13 @@ private async Task<IActionResult> HandleOperation(
                         subStatusCode: DataApiBuilderException.SubStatusCodes.EntityNotFound);
                 }
 
+                int statusCode = (result as ObjectResult)?.StatusCode ?? (result as StatusCodeResult)?.StatusCode ?? (result as JsonResult)?.StatusCode ?? 200;
+                if (activity is not null && activity.IsAllDataRequested)
+                {
+                    activity.TrackRestControllerActivityFinished(statusCode);
+                }
+
+                TelemetryMetricsHelper.TrackRequest(HttpContext.Request.Method, statusCode, route, "REST");
                 return result;
             }
             catch (DataApiBuilderException ex)
@@ -231,6 +277,9 @@ private async Task<IActionResult> HandleOperation(
                     HttpContextExtensions.GetLoggerCorrelationId(HttpContext));
 
                 Response.StatusCode = (int)ex.StatusCode;
+                activity?.TrackRestControllerActivityFinishedWithWithException(ex, Response.StatusCode);
+
+                TelemetryMetricsHelper.TrackError(HttpContext.Request.Method, Response.StatusCode, route, "REST", ex);
                 return ErrorResponse(ex.SubStatusCode.ToString(), ex.Message, ex.StatusCode);
             }
             catch (Exception ex)
@@ -241,11 +290,25 @@ private async Task<IActionResult> HandleOperation(
                     HttpContextExtensions.GetLoggerCorrelationId(HttpContext));
 
                 Response.StatusCode = (int)HttpStatusCode.InternalServerError;
+                activity?.TrackRestControllerActivityFinishedWithWithException(ex, Response.StatusCode);
+
+                TelemetryMetricsHelper.TrackError(HttpContext.Request.Method, Response.StatusCode, route, "REST", ex);
                 return ErrorResponse(
                     DataApiBuilderException.SubStatusCodes.UnexpectedError.ToString(),
                     SERVER_ERROR,
                     HttpStatusCode.InternalServerError);
             }
+            finally
+            {
+                stopwatch.Stop();
+                TelemetryMetricsHelper.TrackRequestDuration(HttpContext.Request.Method, Response.StatusCode, route, "REST", stopwatch.Elapsed.TotalMilliseconds);
+                if (activity is not null && activity.IsAllDataRequested)
+                {
+                    activity.Dispose();
+                }
+
+                TelemetryMetricsHelper.DecrementActiveRequests();
+            }
         }
 
         /// <summary>

src/Service/Program.cs

@@ -13,6 +13,7 @@
 using Microsoft.AspNetCore;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.ApplicationInsights;
@@ -161,22 +162,6 @@ public static ILoggerFactory GetLoggerFactoryForLogLevel(LogLevel logLevel, Tele
                         .AddFilter<ApplicationInsightsLoggerProvider>(category: string.Empty, logLevel);
                     }
 
-                    if (Startup.OpenTelemetryOptions.Enabled && !string.IsNullOrWhiteSpace(Startup.OpenTelemetryOptions.Endpoint))
-                    {
-                        builder.AddOpenTelemetry(logging =>
-                        {
-                            logging.IncludeFormattedMessage = true;
-                            logging.IncludeScopes = true;
-                            logging.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(Startup.OpenTelemetryOptions.ServiceName!));
-                            logging.AddOtlpExporter(configure =>
-                            {
-                                configure.Endpoint = new Uri(Startup.OpenTelemetryOptions.Endpoint);
-                                configure.Headers = Startup.OpenTelemetryOptions.Headers;
-                                configure.Protocol = OtlpExportProtocol.Grpc;
-                            });
-                        });
-                    }
-
                     builder.AddConsole();
                 });
         }

src/Service/Startup.cs

@@ -24,6 +24,7 @@
 using Azure.DataApiBuilder.Service.Controllers;
 using Azure.DataApiBuilder.Service.Exceptions;
 using Azure.DataApiBuilder.Service.HealthCheck;
+using Azure.DataApiBuilder.Service.Telemetry;
 using HotChocolate.AspNetCore;
 using HotChocolate.Execution;
 using HotChocolate.Execution.Configuration;
@@ -44,6 +45,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using OpenTelemetry.Exporter;
+using OpenTelemetry.Logs;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
@@ -59,7 +61,6 @@ public class Startup
         public static LogLevel MinimumLogLevel = LogLevel.Error;
 
         public static bool IsLogLevelOverriddenByCli;
-        public static OpenTelemetryOptions OpenTelemetryOptions = new();
 
         public static ApplicationInsightsOptions AppInsightsOptions = new();
         public const string NO_HTTPS_REDIRECT_FLAG = "--no-https-redirect";
@@ -116,31 +117,46 @@ public void ConfigureServices(IServiceCollection services)
                 && runtimeConfig?.Runtime?.Telemetry?.OpenTelemetry is not null
                 && runtimeConfig.Runtime.Telemetry.OpenTelemetry.Enabled)
             {
+                services.Configure<OpenTelemetryLoggerOptions>(options =>
+                {
+                    options.IncludeScopes = true;
+                    options.ParseStateValues = true;
+                    options.IncludeFormattedMessage = true;
+                });
                 services.AddOpenTelemetry()
+                .WithLogging(logging =>
+                {
+                    logging.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(runtimeConfig.Runtime.Telemetry.OpenTelemetry.ServiceName!))
+                    .AddOtlpExporter(configure =>
+                    {
+                        configure.Endpoint = new Uri(runtimeConfig.Runtime.Telemetry.OpenTelemetry.Endpoint!);
+                        configure.Headers = runtimeConfig.Runtime.Telemetry.OpenTelemetry.Headers;
+                        configure.Protocol = OtlpExportProtocol.Grpc;
+                    });
+
+                })
                 .WithMetrics(metrics =>
                 {
                     metrics.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(runtimeConfig.Runtime.Telemetry.OpenTelemetry.ServiceName!))
-                        .AddAspNetCoreInstrumentation()
-                        .AddHttpClientInstrumentation()
                         .AddOtlpExporter(configure =>
                         {
                             configure.Endpoint = new Uri(runtimeConfig.Runtime.Telemetry.OpenTelemetry.Endpoint!);
                             configure.Headers = runtimeConfig.Runtime.Telemetry.OpenTelemetry.Headers;
                             configure.Protocol = OtlpExportProtocol.Grpc;
                         })
-                        .AddRuntimeInstrumentation();
+                        .AddMeter(TelemetryMetricsHelper.MeterName);
                 })
                 .WithTracing(tracing =>
                 {
                     tracing.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(runtimeConfig.Runtime.Telemetry.OpenTelemetry.ServiceName!))
-                        .AddAspNetCoreInstrumentation()
                         .AddHttpClientInstrumentation()
                         .AddOtlpExporter(configure =>
                         {
                             configure.Endpoint = new Uri(runtimeConfig.Runtime.Telemetry.OpenTelemetry.Endpoint!);
                             configure.Headers = runtimeConfig.Runtime.Telemetry.OpenTelemetry.Headers;
                             configure.Protocol = OtlpExportProtocol.Grpc;
-                        });
+                        })
+                        .AddSource(TelemetryTracesHelper.DABActivitySource.Name);
                 });
             }
 

src/Service/Telemetry/TelemetryMetricsHelper.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Diagnostics.Metrics;
+
+namespace Azure.DataApiBuilder.Service.Telemetry
+{
+    public static class TelemetryMetricsHelper
+    {
+        public static readonly string MeterName = "DataApiBuilder.Metrics";
+        private static readonly Meter _meter = new(MeterName);
+        private static readonly Counter<long> _activeRequests = _meter.CreateCounter<long>("active_requests");
+        private static readonly Counter<long> _errorCounter = _meter.CreateCounter<long>("total_errors");
+        private static readonly Counter<long> _totalRequests = _meter.CreateCounter<long>("total_requests");
+        private static readonly Histogram<double> _requestDuration = _meter.CreateHistogram<double>("request_duration", "ms");
+
+        public static void IncrementActiveRequests() => _activeRequests.Add(1);
+
+        public static void DecrementActiveRequests() => _activeRequests.Add(-1);
+
+        public static void TrackRequest(string method, int statusCode, string endpoint, string apiType)
+        {
+            _totalRequests.Add(1,
+                new("method", method),
+                new("status_code", statusCode),
+                new("endpoint", endpoint),
+                new("api_type", apiType));
+        }
+
+        public static void TrackError(string method, int statusCode, string endpoint, string apiType, Exception ex)
+        {
+            _errorCounter.Add(1,
+                new("method", method),
+                new("status_code", statusCode),
+                new("endpoint", endpoint),
+                new("api_type", apiType),
+                new("error_type", ex.GetType().Name));
+        }
+
+        public static void TrackRequestDuration(string method, int statusCode, string endpoint, string apiType, double duration)
+        {
+            _requestDuration.Record(duration,
+                new("method", method),
+                new("status_code", statusCode),
+                new("endpoint", endpoint),
+                new("api_type", apiType));
+        }
+    }
+}