Telemetry: update enablement (experimental source instead of app context switch) and docs improvements

docs/observability.md

@@ -10,48 +10,61 @@ OpenAI .NET instrumentation follows [OpenTelemetry Semantic Conventions for Gene
 
 ### How to enable
 
-The instrumentation is **experimental** - volume and semantics of the telemetry items may change.
+> [!NOTE]
+> The instrumentation is **experimental** - semantics of telemetry items
+> (such as metric or attribute names, types, value range or other properties) may change.
 
-To enable the instrumentation:
+The following code snippet shows how to enable OpenAI traces and metrics:
 
-1. Set instrumentation feature-flag using one of the following options:
+```csharp
+builder.Services.AddOpenTelemetry()
+    .WithTracing(b =>
+    {
+        b.AddSource("Experimental.OpenAI.*", "OpenAI.*")
+            ...
+        .AddOtlpExporter();
+    })
+    .WithMetrics(b =>
+    {
+        b.AddMeter("Experimental.OpenAI.*", "OpenAI.*")
+        ...
+        .AddOtlpExporter();
+    });
+```
 
-   - set the `OPENAI_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY` environment variable to `"true"`
-   - set the `OpenAI.Experimental.EnableOpenTelemetry` context switch to true in your application code when application
-     is starting and before initializing any OpenAI clients. For example:
+Distributed tracing is enabled with `AddSource("Experimental.OpenAI.*", "OpenAI.*")` which tells OpenTelemetry to listen to all [ActivitySources](https://learn.microsoft.com/dotnet/api/system.diagnostics.activitysource) with names starting with `Experimental.OpenAI.` (experimental ones) or sources which names start with `"OpenAI.*"` (stable ones).
 
-     ```csharp
-     AppContext.SetSwitch("OpenAI.Experimental.EnableOpenTelemetry", true);
-     ```
+Similarly, metrics are configured with `AddMeter("Experimental.OpenAI.*", "OpenAI.*")` which enables all OpenAI-related [Meters](https://learn.microsoft.com/dotnet/api/system.diagnostics.metrics.meter).
 
-2. Enable OpenAI telemetry:
+Once experimental telemetry stabilizes, experimental sources and meters will be renamed (for example, `Experimental.OpenAI.ChatClient` will become `OpenAI.ChatClient`), so it's
+recommended to enable both to avoid changing the source code later.
 
-   ```csharp
-   builder.Services.AddOpenTelemetry()
-       .WithTracing(b =>
-       {
-           b.AddSource("OpenAI.*")
-             ...
-            .AddOtlpExporter();
-       })
-       .WithMetrics(b =>
-       {
-           b.AddMeter("OpenAI.*")
-            ...
-            .AddOtlpExporter();
-       });
-   ```
+Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client
+calls made by your application including those done by the OpenAI SDK.
 
-   Distributed tracing is enabled with `AddSource("OpenAI.*")` which tells OpenTelemetry to listen to all [ActivitySources](https://learn.microsoft.com/dotnet/api/system.diagnostics.activitysource) with names starting with `OpenAI.*`.
+Check out [full example](../examples/OpenTelemetryExamples.cs) and [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details.
 
-   Similarly, metrics are configured with `AddMeter("OpenAI.*")` which enables all OpenAI-related [Meters](https://learn.microsoft.com/dotnet/api/system.diagnostics.metrics.meter).
+### How to view telemetry
 
-Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client
-calls made by your application including those done by the OpenAI SDK.
-Check out [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details.
+You can view traces and metrics in any [telemetry system](https://opentelemetry.io/ecosystem/vendors/) compatible with OpenTelemetry.
+
+You may use [Aspire dashboard](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/standalone) to test things out locally.
+You can run it as a docker container with the following command:
+
+```bash
+docker run --rm -it \
+    -p 18888:18888 \
+    -p 4317:18889 -d \
+    --name aspire-dashboard \
+    mcr.microsoft.com/dotnet/aspire-dashboard:latest
+```
+
+Here's a trace produced by [OpenTelemetry sample](../examples/OpenTelemetryExamples.cs):
+
+![](./images/openai-tracing-with-opentelemetry.png)
 
 ### Available sources and meters
 
 The following sources and meters are available:
 
-- `OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet)
+- `Experimental.OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet)

examples/OpenAI.Examples.csproj

@@ -19,5 +19,8 @@
     <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
     <PackageReference Include="Moq" Version="[4.18.2]" />
+    <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
   </ItemGroup>
 </Project>

examples/OpenTelemetryExamples.cs

@@ -0,0 +1,49 @@
+using NUnit.Framework;
+using OpenAI.Chat;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
+using System;
+using System.Threading.Tasks;
+
+namespace OpenAI.Examples;
+
+public partial class ChatExamples
+{
+    [Test]
+    public async Task OpenTelemetryExamples()
+    {
+        // Let's configure OpenTelemetry to collect OpenAI and HTTP client traces and metrics
+        // and export them to console and also to the local OTLP endpoint.
+        //
+        // If you have some local OTLP listener (e.g. Aspire dashboard) running,
+        // you can explore traces and metrics produced by the test there.
+        //
+        // Check out https://opentelemetry.io/docs/languages/net/getting-started/ for more details and
+        // examples on how to set up OpenTelemetry with ASP.NET Core.
+
+        ResourceBuilder resourceBuilder = ResourceBuilder.CreateDefault().AddService("test");
+        using TracerProvider tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
+            .SetResourceBuilder(resourceBuilder)
+            .AddSource("Experimental.OpenAI.*", "OpenAI.*")
+            .AddHttpClientInstrumentation()
+            .AddConsoleExporter()
+            .AddOtlpExporter()
+            .Build();
+
+        using MeterProvider meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder()
+            .SetResourceBuilder(resourceBuilder)
+            .AddView("gen_ai.client.operation.duration", new ExplicitBucketHistogramConfiguration { Boundaries = [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92] })
+            .AddMeter("Experimental.OpenAI.*", "OpenAI.*")
+            .AddHttpClientInstrumentation()
+            .AddConsoleExporter()
+            .AddOtlpExporter()
+            .Build();
+
+        ChatClient client = new("gpt-4o-mini", Environment.GetEnvironmentVariable("OPENAI_API_KEY"));
+
+        ChatCompletion completion = await client.CompleteChatAsync("Say 'this is a test.'");
+
+        Console.WriteLine($"{completion}");
+    }
+}

src/Utility/Telemetry/OpenTelemetryScope.cs

@@ -10,8 +10,8 @@ namespace OpenAI.Telemetry;
 
 internal class OpenTelemetryScope : IDisposable
 {
-    private static readonly ActivitySource s_chatSource = new ActivitySource("OpenAI.ChatClient");
-    private static readonly Meter s_chatMeter = new Meter("OpenAI.ChatClient");
+    private static readonly ActivitySource s_chatSource = new ActivitySource("Experimental.OpenAI.ChatClient");
+    private static readonly Meter s_chatMeter = new Meter("Experimental.OpenAI.ChatClient");
 
     // TODO: add explicit histogram buckets once System.Diagnostics.DiagnosticSource 9.0 is used
     private static readonly Histogram<double> s_duration = s_chatMeter.CreateHistogram<double>(GenAiClientOperationDurationMetricName, "s", "Measures GenAI operation duration.");

src/Utility/Telemetry/OpenTelemetrySource.cs

@@ -6,9 +6,6 @@ namespace OpenAI.Telemetry;
 internal class OpenTelemetrySource
 {
     private const string ChatOperationName = "chat";
-    private readonly bool IsOTelEnabled = AppContextSwitchHelper
-        .GetConfigValue("OpenAI.Experimental.EnableOpenTelemetry", "OPENAI_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY");
-
     private readonly string _serverAddress;
     private readonly int _serverPort;
     private readonly string _model;
@@ -22,9 +19,6 @@ public OpenTelemetrySource(string model, Uri endpoint)
 
     public OpenTelemetryScope StartChatScope(ChatCompletionOptions completionsOptions)
     {
-        return IsOTelEnabled
-            ? OpenTelemetryScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions)
-            : null;
+        return OpenTelemetryScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions);
     }
-
 }

tests/Chat/ChatTests.cs

@@ -754,9 +754,8 @@ void HandleUpdate(StreamingChatCompletionUpdate update)
     [NonParallelizable]
     public async Task HelloWorldChatWithTracingAndMetrics()
     {
-        using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry();
-        using TestActivityListener activityListener = new TestActivityListener("OpenAI.ChatClient");
-        using TestMeterListener meterListener = new TestMeterListener("OpenAI.ChatClient");
+        using TestActivityListener activityListener = new TestActivityListener("Experimental.OpenAI.ChatClient");
+        using TestMeterListener meterListener = new TestMeterListener("Experimental.OpenAI.ChatClient");
 
         ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
         IEnumerable<ChatMessage> messages = [new UserChatMessage("Hello, world!")];

tests/OpenAI.Tests.csproj

@@ -24,6 +24,5 @@
 
   <ItemGroup>
     <Compile Include="..\src\Utility\Telemetry\*.cs" LinkBase="Telemetry\Shared" />
-    <Compile Include="..\src\Utility\AppContextSwitchHelper.cs" LinkBase="Telemetry\Shared" />
   </ItemGroup>
 </Project>

tests/Telemetry/ChatTelemetryTests.cs

@@ -41,24 +41,12 @@ public void AllTelemetryOff()
         Assert.IsNull(Activity.Current);
     }
 
-    [Test]
-    public void SwitchOffAllTelemetryOn()
-    {
-        using var activityListener = new TestActivityListener("OpenAI.ChatClient");
-        using var meterListener = new TestMeterListener("OpenAI.ChatClient");
-        var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint));
-        Assert.IsNull(telemetry.StartChatScope(new ChatCompletionOptions()));
-        Assert.IsNull(Activity.Current);
-    }
-
     [Test]
     public void MetricsOnTracingOff()
     {
-        using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry();
-
         var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint));
 
-        using var meterListener = new TestMeterListener("OpenAI.ChatClient");
+        using var meterListener = new TestMeterListener("Experimental.OpenAI.ChatClient");
 
         var elapsedMax = Stopwatch.StartNew();
         using var scope = telemetry.StartChatScope(new ChatCompletionOptions());
@@ -83,10 +71,8 @@ public void MetricsOnTracingOff()
     [Test]
     public void MetricsOnTracingOffException()
     {
-        using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry();
-
         var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint));
-        using var meterListener = new TestMeterListener("OpenAI.ChatClient");
+        using var meterListener = new TestMeterListener("Experimental.OpenAI.ChatClient");
 
         using (var scope = telemetry.StartChatScope(new ChatCompletionOptions()))
         {
@@ -100,10 +86,8 @@ public void MetricsOnTracingOffException()
     [Test]
     public void TracingOnMetricsOff()
     {
-        using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry();
-
         var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint));
-        using var listener = new TestActivityListener("OpenAI.ChatClient");
+        using var listener = new TestActivityListener("Experimental.OpenAI.ChatClient");
 
         var chatCompletion = CreateChatCompletion();
 
@@ -129,9 +113,8 @@ public void TracingOnMetricsOff()
     [Test]
     public void ChatTracingAllAttributes()
     {
-        using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry();
         var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint));
-        using var listener = new TestActivityListener("OpenAI.ChatClient");
+        using var listener = new TestActivityListener("Experimental.OpenAI.ChatClient");
         var options = new ChatCompletionOptions()
         {
             Temperature = 0.42f,
@@ -157,10 +140,8 @@ public void ChatTracingAllAttributes()
     [Test]
     public void ChatTracingException()
     {
-        using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry();
-
         var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint));
-        using var listener = new TestActivityListener("OpenAI.ChatClient");
+        using var listener = new TestActivityListener("Experimental.OpenAI.ChatClient");
 
         var error = new SocketException(42, "test error");
         using (var scope = telemetry.StartChatScope(new ChatCompletionOptions()))
@@ -176,11 +157,10 @@ public void ChatTracingException()
     [Test]
     public async Task ChatTracingAndMetricsMultiple()
     {
-        using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry();
         var source = new OpenTelemetrySource(RequestModel, new Uri(Endpoint));
 
-        using var activityListener = new TestActivityListener("OpenAI.ChatClient");
-        using var meterListener = new TestMeterListener("OpenAI.ChatClient");
+        using var activityListener = new TestActivityListener("Experimental.OpenAI.ChatClient");
+        using var meterListener = new TestMeterListener("Experimental.OpenAI.ChatClient");
 
         var options = new ChatCompletionOptions();
 

tests/Telemetry/TestMeterListener.cs

@@ -4,14 +4,15 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics.Metrics;
+using System.Linq;
 
 namespace OpenAI.Tests.Telemetry;
 
 internal class TestMeterListener : IDisposable
 {
     public record TestMeasurement(object value, Dictionary<string, object> tags);
 
-    private readonly ConcurrentDictionary<string, List<TestMeasurement>> _measurements = new();
+    private readonly ConcurrentDictionary<string, ConcurrentQueue<TestMeasurement>> _measurements = new();
     private readonly ConcurrentDictionary<string, Instrument> _instruments = new();
     private readonly MeterListener _listener;
     public TestMeterListener(string meterName)
@@ -31,8 +32,8 @@ public TestMeterListener(string meterName)
 
     public List<TestMeasurement> GetMeasurements(string instrumentName)
     {
-        _measurements.TryGetValue(instrumentName, out var list);
-        return list;
+        _measurements.TryGetValue(instrumentName, out var queue);
+        return queue?.ToList();
     }
 
     public Instrument GetInstrument(string instrumentName)
@@ -46,11 +47,11 @@ private void OnMeasurementRecorded<T>(Instrument instrument, T measurement, Read
         _instruments.TryAdd(instrument.Name, instrument);
 
         var testMeasurement = new TestMeasurement(measurement, new Dictionary<string, object>(tags.ToArray()));
-        _measurements.AddOrUpdate(instrument.Name,
-            k => new() { testMeasurement },
+        _measurements.AddOrUpdate(instrument.Name, 
+            k => new ConcurrentQueue<TestMeasurement>([ testMeasurement ]), 
             (k, l) =>
             {
-                l.Add(testMeasurement);
+                l.Enqueue(testMeasurement);
                 return l;
             });
     }