diff --git a/src/Core/Resolvers/SqlQueryEngine.cs b/src/Core/Resolvers/SqlQueryEngine.cs index 53547a96a3..872b733f60 100644 --- a/src/Core/Resolvers/SqlQueryEngine.cs +++ b/src/Core/Resolvers/SqlQueryEngine.cs @@ -332,6 +332,7 @@ public object ResolveList(JsonElement array, IObjectField fieldSchema, ref IMeta JsonElement result = await _cache.GetOrSetAsync(queryExecutor, queryMetadata, cacheEntryTtl: runtimeConfig.GetEntityCacheEntryTtl(entityName: structure.EntityName)); byte[] jsonBytes = JsonSerializer.SerializeToUtf8Bytes(result); JsonDocument cacheServiceResponse = JsonDocument.Parse(jsonBytes); + return cacheServiceResponse; } } @@ -348,6 +349,7 @@ public object ResolveList(JsonElement array, IObjectField fieldSchema, ref IMeta httpContext: _httpContextAccessor.HttpContext!, args: null, dataSourceName: dataSourceName); + return response; } diff --git a/src/Service.Tests/Configuration/OpenTelemetryTests.cs b/src/Service.Tests/Configuration/OpenTelemetryTests.cs index dc98c58351..166dc7b001 100644 --- a/src/Service.Tests/Configuration/OpenTelemetryTests.cs +++ b/src/Service.Tests/Configuration/OpenTelemetryTests.cs @@ -61,7 +61,6 @@ public void CleanUpTelemetryConfig() File.Delete(CONFIG_WITHOUT_TELEMETRY); } - Startup.OpenTelemetryOptions = new(); } /// diff --git a/src/Service.Tests/dab-config.CosmosDb_NoSql.json b/src/Service.Tests/dab-config.CosmosDb_NoSql.json index c55314e592..5704dc19be 100644 --- a/src/Service.Tests/dab-config.CosmosDb_NoSql.json +++ b/src/Service.Tests/dab-config.CosmosDb_NoSql.json @@ -31,6 +31,12 @@ "provider": "StaticWebApps" }, "mode": "development" + }, + "telemetry": { + "app-insights": { + "enabled": true, + "connection-string": "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://dc.services.visualstudio.com/v2/track" + } } }, "entities": { @@ -794,4 +800,4 @@ ] } } -} \ No newline at end of file +} diff --git a/src/Service/Azure.DataApiBuilder.Service.csproj b/src/Service/Azure.DataApiBuilder.Service.csproj index e4fbddd825..cb7a1a1644 100644 --- a/src/Service/Azure.DataApiBuilder.Service.csproj +++ b/src/Service/Azure.DataApiBuilder.Service.csproj @@ -1,4 +1,4 @@ - + net8.0 @@ -77,7 +77,7 @@ - + diff --git a/src/Service/Controllers/RestController.cs b/src/Service/Controllers/RestController.cs index b570c91406..10e1f4263d 100644 --- a/src/Service/Controllers/RestController.cs +++ b/src/Service/Controllers/RestController.cs @@ -2,15 +2,20 @@ // Licensed under the MIT License. using System; +using System.Diagnostics; +using System.Linq; using System.Net; using System.Net.Mime; using System.Threading.Tasks; using Azure.DataApiBuilder.Config.ObjectModel; +using Azure.DataApiBuilder.Core.Configurations; using Azure.DataApiBuilder.Core.Models; using Azure.DataApiBuilder.Core.Services; using Azure.DataApiBuilder.Service.Exceptions; +using Azure.DataApiBuilder.Service.Telemetry; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; using Microsoft.Extensions.Logging; namespace Azure.DataApiBuilder.Service.Controllers @@ -47,11 +52,14 @@ public class RestController : ControllerBase private readonly ILogger _logger; + private readonly RuntimeConfigProvider _runtimeConfigProvider; + /// /// Constructor. /// - public RestController(RestService restService, IOpenApiDocumentor openApiDocumentor, ILogger logger) + public RestController(RuntimeConfigProvider runtimeConfigProvider, RestService restService, IOpenApiDocumentor openApiDocumentor, ILogger logger) { + _runtimeConfigProvider = runtimeConfigProvider; _restService = restService; _openApiDocumentor = openApiDocumentor; _logger = logger; @@ -185,11 +193,29 @@ private async Task HandleOperation( string route, EntityActionOperation operationType) { + if (route.Equals(REDIRECTED_ROUTE)) + { + return NotFound(); + } + + Stopwatch stopwatch = Stopwatch.StartNew(); + // This activity tracks the entire REST request. + using Activity? activity = TelemetryTracesHelper.DABActivitySource.StartActivity($"{HttpContext.Request.Method} {(route.Split('/').Length > 1 ? route.Split('/')[1] : string.Empty)}"); + try { - if (route.Equals(REDIRECTED_ROUTE)) + TelemetryMetricsHelper.IncrementActiveRequests(ApiType.REST); + + if (activity is not null) { - return NotFound(); + activity.TrackRestControllerActivityStarted( + Enum.Parse(HttpContext.Request.Method, ignoreCase: true), + HttpContext.Request.Headers["User-Agent"].ToString(), + operationType.ToString(), + route, + HttpContext.Request.QueryString.ToString(), + HttpContext.Request.Headers["X-MS-API-ROLE"].FirstOrDefault() ?? HttpContext.User.FindFirst("role")?.Value, + ApiType.REST); } // Validate the PathBase matches the configured REST path. @@ -208,8 +234,21 @@ private async Task HandleOperation( (string entityName, string primaryKeyRoute) = _restService.GetEntityNameAndPrimaryKeyRouteFromRoute(routeAfterPathBase); + // This activity tracks the query execution. This will create a new activity nested under the REST request activity. + using Activity? queryActivity = TelemetryTracesHelper.DABActivitySource.StartActivity($"QUERY {entityName}"); IActionResult? result = await _restService.ExecuteAsync(entityName, operationType, primaryKeyRoute); + RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig(); + string dataSourceName = runtimeConfig.GetDataSourceNameFromEntityName(entityName); + DatabaseType databaseType = runtimeConfig.GetDataSourceFromDataSourceName(dataSourceName).DatabaseType; + + if (queryActivity is not null) + { + queryActivity.TrackQueryActivityStarted( + databaseType, + dataSourceName); + } + if (result is null) { throw new DataApiBuilderException( @@ -218,6 +257,13 @@ private async Task HandleOperation( subStatusCode: DataApiBuilderException.SubStatusCodes.EntityNotFound); } + int statusCode = (result as ObjectResult)?.StatusCode ?? (result as StatusCodeResult)?.StatusCode ?? (result as JsonResult)?.StatusCode ?? 200; + if (activity is not null && activity.IsAllDataRequested) + { + HttpStatusCode httpStatusCode = Enum.Parse(statusCode.ToString(), ignoreCase: true); + activity.TrackRestControllerActivityFinished(httpStatusCode); + } + return result; } catch (DataApiBuilderException ex) @@ -228,6 +274,10 @@ private async Task HandleOperation( HttpContextExtensions.GetLoggerCorrelationId(HttpContext)); Response.StatusCode = (int)ex.StatusCode; + activity?.TrackRestControllerActivityFinishedWithException(ex, ex.StatusCode); + + HttpMethod method = Enum.Parse(HttpContext.Request.Method, ignoreCase: true); + TelemetryMetricsHelper.TrackError(method, ex.StatusCode, route, ApiType.REST, ex); return ErrorResponse(ex.SubStatusCode.ToString(), ex.Message, ex.StatusCode); } catch (Exception ex) @@ -238,11 +288,26 @@ private async Task HandleOperation( HttpContextExtensions.GetLoggerCorrelationId(HttpContext)); Response.StatusCode = (int)HttpStatusCode.InternalServerError; + + HttpMethod method = Enum.Parse(HttpContext.Request.Method, ignoreCase: true); + activity?.TrackRestControllerActivityFinishedWithException(ex, HttpStatusCode.InternalServerError); + + TelemetryMetricsHelper.TrackError(method, HttpStatusCode.InternalServerError, route, ApiType.REST, ex); return ErrorResponse( DataApiBuilderException.SubStatusCodes.UnexpectedError.ToString(), SERVER_ERROR, HttpStatusCode.InternalServerError); } + finally + { + stopwatch.Stop(); + HttpMethod method = Enum.Parse(HttpContext.Request.Method, ignoreCase: true); + HttpStatusCode httpStatusCode = Enum.Parse(Response.StatusCode.ToString(), ignoreCase: true); + TelemetryMetricsHelper.TrackRequest(method, httpStatusCode, route, ApiType.REST); + TelemetryMetricsHelper.TrackRequestDuration(method, httpStatusCode, route, ApiType.REST, stopwatch.Elapsed); + + TelemetryMetricsHelper.DecrementActiveRequests(ApiType.REST); + } } /// diff --git a/src/Service/Startup.cs b/src/Service/Startup.cs index a871d3354f..a0d3ef3393 100644 --- a/src/Service/Startup.cs +++ b/src/Service/Startup.cs @@ -61,9 +61,9 @@ public class Startup public static LogLevel MinimumLogLevel = LogLevel.Error; public static bool IsLogLevelOverriddenByCli; - public static OpenTelemetryOptions OpenTelemetryOptions = new(); public static ApplicationInsightsOptions AppInsightsOptions = new(); + public static OpenTelemetryOptions OpenTelemetryOptions = new(); public const string NO_HTTPS_REDIRECT_FLAG = "--no-https-redirect"; private HotReloadEventHandler _hotReloadEventHandler = new(); private RuntimeConfigProvider? _configProvider; @@ -119,31 +119,46 @@ public void ConfigureServices(IServiceCollection services) && runtimeConfig?.Runtime?.Telemetry?.OpenTelemetry is not null && runtimeConfig.Runtime.Telemetry.OpenTelemetry.Enabled) { + services.Configure(options => + { + options.IncludeScopes = true; + options.ParseStateValues = true; + options.IncludeFormattedMessage = true; + }); services.AddOpenTelemetry() + .WithLogging(logging => + { + logging.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(runtimeConfig.Runtime.Telemetry.OpenTelemetry.ServiceName!)) + .AddOtlpExporter(configure => + { + configure.Endpoint = new Uri(runtimeConfig.Runtime.Telemetry.OpenTelemetry.Endpoint!); + configure.Headers = runtimeConfig.Runtime.Telemetry.OpenTelemetry.Headers; + configure.Protocol = OtlpExportProtocol.Grpc; + }); + + }) .WithMetrics(metrics => { metrics.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(runtimeConfig.Runtime.Telemetry.OpenTelemetry.ServiceName!)) - .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() .AddOtlpExporter(configure => { configure.Endpoint = new Uri(runtimeConfig.Runtime.Telemetry.OpenTelemetry.Endpoint!); configure.Headers = runtimeConfig.Runtime.Telemetry.OpenTelemetry.Headers; configure.Protocol = OtlpExportProtocol.Grpc; }) - .AddRuntimeInstrumentation(); + .AddMeter(TelemetryMetricsHelper.MeterName); }) .WithTracing(tracing => { tracing.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(runtimeConfig.Runtime.Telemetry.OpenTelemetry.ServiceName!)) - .AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation() .AddOtlpExporter(configure => { configure.Endpoint = new Uri(runtimeConfig.Runtime.Telemetry.OpenTelemetry.Endpoint!); configure.Headers = runtimeConfig.Runtime.Telemetry.OpenTelemetry.Headers; configure.Protocol = OtlpExportProtocol.Grpc; - }); + }) + .AddSource(TelemetryTracesHelper.DABActivitySource.Name); }); } @@ -395,6 +410,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, RuntimeC { // Configure Application Insights Telemetry ConfigureApplicationInsightsTelemetry(app, runtimeConfig); + ConfigureOpenTelemetry(runtimeConfig); // Config provided before starting the engine. isRuntimeReady = PerformOnConfigChangeAsync(app).Result; @@ -712,6 +728,37 @@ private void ConfigureApplicationInsightsTelemetry(IApplicationBuilder app, Runt } } + /// + /// Configure Open Telemetry based on the loaded runtime configuration. If Open Telemetry + /// is enabled, we can track different events and metrics. + /// + /// The provider used to load runtime configuration. + /// + private void ConfigureOpenTelemetry(RuntimeConfig runtimeConfig) + { + if (runtimeConfig?.Runtime?.Telemetry is not null + && runtimeConfig.Runtime.Telemetry.OpenTelemetry is not null) + { + OpenTelemetryOptions = runtimeConfig.Runtime.Telemetry.OpenTelemetry; + + if (!OpenTelemetryOptions.Enabled) + { + _logger.LogInformation("Open Telemetry are disabled."); + return; + } + + if (string.IsNullOrWhiteSpace(OpenTelemetryOptions?.Endpoint)) + { + _logger.LogWarning("Logs won't be sent to Open Telemetry because an Open Telemetry connection string is not available in the runtime config."); + return; + } + + // Updating Startup Logger to Log from Startup Class. + ILoggerFactory? loggerFactory = Program.GetLoggerFactoryForLogLevel(MinimumLogLevel); + _logger = loggerFactory.CreateLogger(); + } + } + /// /// Sets Static Web Apps EasyAuth as the authentication scheme for the engine. /// diff --git a/src/Service/Telemetry/TelemetryMetricsHelper.cs b/src/Service/Telemetry/TelemetryMetricsHelper.cs new file mode 100644 index 0000000000..c8e61032b3 --- /dev/null +++ b/src/Service/Telemetry/TelemetryMetricsHelper.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Net; +using Azure.DataApiBuilder.Config.ObjectModel; +using Kestral = Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + /// + /// Helper class for tracking telemetry metrics such as active requests, errors, total requests, + /// and request durations using the .NET Meter and Counter APIs. + /// + public static class TelemetryMetricsHelper + { + public static readonly string MeterName = "DataApiBuilder.Metrics"; + private static readonly Meter _meter = new(MeterName); + private static readonly UpDownCounter _activeRequests = _meter.CreateUpDownCounter("active_requests"); + private static readonly Counter _errorCounter = _meter.CreateCounter("total_errors"); + private static readonly Counter _totalRequests = _meter.CreateCounter("total_requests"); + private static readonly Histogram _requestDuration = _meter.CreateHistogram("request_duration", "ms"); + + public static void IncrementActiveRequests(ApiType kind) => _activeRequests.Add(1, new KeyValuePair("api_type", kind)); + + public static void DecrementActiveRequests(ApiType kind) => _activeRequests.Add(-1, new KeyValuePair("api_type", kind)); + + /// + /// Tracks a request by incrementing the total requests counter and associating it with metadata. + /// + /// The HTTP method of the request (e.g., GET, POST). + /// The HTTP status code of the response. + /// The endpoint being accessed. + /// The type of API being used (e.g., REST, GraphQL). + public static void TrackRequest(Kestral method, HttpStatusCode statusCode, string endpoint, ApiType apiType) + { + _totalRequests.Add(1, + new("method", method), + new("status_code", statusCode), + new("endpoint", endpoint), + new("api_type", apiType)); + } + + /// + /// Tracks an error by incrementing the error counter and associating it with metadata. + /// + /// The HTTP method of the request (e.g., GET, POST). + /// The HTTP status code of the response. + /// The endpoint being accessed. + /// The type of API being used (e.g., REST, GraphQL). + /// The exception that occurred. + public static void TrackError(Kestral method, HttpStatusCode statusCode, string endpoint, ApiType apiType, Exception ex) + { + _errorCounter.Add(1, + new("method", method), + new("status_code", statusCode), + new("endpoint", endpoint), + new("api_type", apiType), + new("error_type", ex.GetType().Name)); + } + + /// + /// Tracks the duration of a request by recording it in a histogram and associating it with metadata. + /// + /// The HTTP method of the request (e.g., GET, POST). + /// The HTTP status code of the response. + /// The endpoint being accessed. + /// The type of API being used (e.g., REST, GraphQL). + /// The duration of the request in milliseconds. + public static void TrackRequestDuration(Kestral method, HttpStatusCode statusCode, string endpoint, ApiType apiType, TimeSpan duration) + { + _requestDuration.Record(duration.TotalMilliseconds, + new("method", method), + new("status_code", statusCode), + new("endpoint", endpoint), + new("api_type", apiType)); + } + } +} diff --git a/src/Service/Telemetry/TelemetryTracesHelper.cs b/src/Service/Telemetry/TelemetryTracesHelper.cs new file mode 100644 index 0000000000..33a6029f43 --- /dev/null +++ b/src/Service/Telemetry/TelemetryTracesHelper.cs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.Net; +using Azure.DataApiBuilder.Config.ObjectModel; +using OpenTelemetry.Trace; +using Kestral = Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod; + +namespace Azure.DataApiBuilder.Service.Telemetry +{ + public static class TelemetryTracesHelper + { + /// + /// Activity source for Data API Builder telemetry. + /// + public static readonly ActivitySource DABActivitySource = new("DataApiBuilder"); + + /// + /// Tracks the start of a REST controller activity. + /// + /// The activity instance. + /// The HTTP method of the request (e.g., GET, POST). + /// The user agent string from the request. + /// The type of action being performed (e.g. Read). + /// The URL of the request. + /// The query string of the request, if any. + /// The role of the user making the request. + /// The type of API being used (e.g., REST, GraphQL). + public static void TrackRestControllerActivityStarted( + this Activity activity, + Kestral httpMethod, + string userAgent, + string actionType, + string httpURL, + string? queryString, + string? userRole, + ApiType apiType) + { + if (activity.IsAllDataRequested) + { + activity.SetTag("http.method", httpMethod); + activity.SetTag("user-agent", userAgent); + activity.SetTag("action.type", actionType); + activity.SetTag("http.url", httpURL); + if (!string.IsNullOrEmpty(queryString)) + { + activity.SetTag("http.querystring", queryString); + } + + if (!string.IsNullOrEmpty(userRole)) + { + activity.SetTag("user.role", userRole); + } + + activity.SetTag("api.type", apiType); + } + } + + /// + /// Tracks the start of a query activity. + /// + /// The activity instance. + /// The type of database being queried. + /// The name of the data source being queried. + public static void TrackQueryActivityStarted( + this Activity activity, + DatabaseType databaseType, + string dataSourceName) + { + if (activity.IsAllDataRequested) + { + activity.SetTag("data-source.type", databaseType); + activity.SetTag("data-source.name", dataSourceName); + } + + } + + /// + /// Tracks the completion of a REST controller activity. + /// + /// The activity instance. + /// The HTTP status code of the response. + public static void TrackRestControllerActivityFinished( + this Activity activity, + HttpStatusCode statusCode) + { + if (activity.IsAllDataRequested) + { + activity.SetTag("status.code", statusCode); + } + } + + /// + /// Tracks the completion of a REST controller activity with an exception. + /// + /// The activity instance. + /// The exception that occurred. + /// The HTTP status code of the response. + public static void TrackRestControllerActivityFinishedWithException( + this Activity activity, + Exception ex, + HttpStatusCode statusCode) + { + if (activity.IsAllDataRequested) + { + activity.SetStatus(Status.Error.WithDescription(ex.Message)); + activity.RecordException(ex); + activity.SetTag("error.type", ex.GetType().Name); + activity.SetTag("error.message", ex.Message); + activity.SetTag("status.code", statusCode); + } + } + } +}