feat: add gap duration

Cargo.toml

@@ -20,6 +20,12 @@ path = "src/integration_benchmark.rs"
 name = "clock"
 harness = false
 
+# copybara:strip_begin(otel)
+[[bench]]
+name = "otel_utils"
+harness = false
+# copybara:strip_end
+
 [features]
 default = []
 explicit-optin = []

benches/otel_utils.rs

@@ -0,0 +1,99 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// copybara:strip_begin(otel)
+use criterion::{criterion_group, criterion_main, Criterion};
+use nccl_profiler::otel_utils::{
+    DurationHistogram, EventStep, GapTracker, HistogramManager, NcclOpKey,
+};
+use opentelemetry_sdk::metrics::{
+    new_view, Aggregation, Instrument, InstrumentKind, ManualReader, SdkMeterProvider, Stream,
+};
+
+// install a real meter provider with the same base2 exponential histogram
+// aggregation as init_meter_provider() so record costs are representative
+fn init_meter_provider() {
+    let mut builder = SdkMeterProvider::builder().with_reader(ManualReader::builder().build());
+    for name in [
+        "*latency",
+        "nccl.collective.duration",
+        "nccl.collective.gap",
+    ] {
+        let mut histogram_instrument = Instrument::new().name(name);
+        histogram_instrument.kind = Some(InstrumentKind::Histogram);
+        let mask = Stream::new().aggregation(Aggregation::Base2ExponentialHistogram {
+            max_size: 160,
+            max_scale: 20,
+            record_min_max: true,
+        });
+        if let Ok(view) = new_view(histogram_instrument, mask) {
+            builder = builder.with_view(view);
+        }
+    }
+    opentelemetry::global::set_meter_provider(builder.build());
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    init_meter_provider();
+
+    let step = EventStep {
+        step: 0,
+        size: 65536,
+        start_time: 1234567,
+        fifo_wait_dur_ns: None,
+        dur_ns: 512,
+    };
+
+    let mut send_manager = HistogramManager::new("nccl.net_send.latency", "ns", 16);
+    let send_histogram = send_manager.get_histogram(NcclOpKey::NetSend(0x123, 0, 1));
+    c.bench_function("net_send latency record", |b| {
+        b.iter(|| send_histogram.record(&step))
+    });
+
+    let mut recv_manager = HistogramManager::new("nccl.net_recv.latency", "ns", 16);
+    let recv_histogram = recv_manager.get_histogram(NcclOpKey::NetRecv(0x123, 0, 1));
+    c.bench_function("net_recv latency record", |b| {
+        b.iter(|| recv_histogram.record(&step))
+    });
+
+    let mut duration_histogram = DurationHistogram::new("nccl.collective.duration", "ns", 16);
+    c.bench_function("collective duration record", |b| {
+        b.iter(|| duration_histogram.record(4096, 0x123, "ncclAllReduce", 0, 1 << 22))
+    });
+
+    // the clock closure mirrors the shipped default path of
+    // Profiler::recent_timer_ns (an Instant read per call)
+    let gap_tracker = GapTracker::new();
+    gap_tracker.init_instrument();
+    let t0 = std::time::Instant::now();
+    let now_ns = move || t0.elapsed().as_nanos() as u64;
+    c.bench_function("gap idle transition", |b| {
+        b.iter(|| {
+            gap_tracker.activity_begin(now_ns);
+            gap_tracker.activity_end(now_ns());
+        })
+    });
+
+    gap_tracker.activity_begin(now_ns);
+    c.bench_function("gap nested activity", |b| {
+        b.iter(|| {
+            gap_tracker.activity_begin(now_ns);
+            gap_tracker.activity_end(now_ns());
+        })
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
+// copybara:strip_end

docs/otel-support.md

@@ -46,7 +46,53 @@ To prevent high cardinality issues, CoMMA uses a "Top K" cardinality management
   - `nccl.metric.aggregated`: Set to `true`.
   - `nccl.hostname`: Hostname of the node.
 
-The "Top K" list is dynamically updated at the interval defined by `NCCL_PROFILER_OTEL_METRICS_CARDINALITY_GROUPING_INTERVAL`.
+The "Top K" list is dynamically updated at the interval defined by `NCCL_PROFILER_OTEL_METRICS_CARDINALITY_GROUPING_INTERVAL`, and keys of closed communicators are evicted immediately.
+
+Note that the OTel SDK exports with cumulative temporality and keeps a stream for every attribute set it has ever recorded until process exit — including streams for keys that were later demoted from the "Top K" set. CoMMA's cardinality limit bounds how many keys record with high-fidelity attributes at any time, but under communicator churn the SDK-side stream count grows with the total number of distinct attribute sets ever promoted. Keep `NCCL_PROFILER_OTEL_METRICS_MAX_CARDINALITY` modest when communicators churn frequently. This applies to every "Top K" managed metric below.
+
+### `nccl.net_recv.latency` (Histogram, Unit: `ns`)
+
+This metric records the latency of network receive operations. Receive steps are only generated when `NCCL_PROFILER_TRACK_RECV_STEPS` is enabled (subject to `NCCL_PROFILER_P2P_RECV_SAMPLE_RATE` for point-to-point operations).
+
+It uses the same "Top K" cardinality management strategy and attributes as `nccl.net_send.latency`. For receive connections, `nccl.source.rank` is the remote sender and `nccl.destination.rank` is the local rank.
+
+A receive step spans buffer post to data arrival, so its duration includes any delay before the remote sender was ready — unlike send steps, which measure post-clearance transfer time. The two metrics are therefore not symmetric; receive latencies are most meaningful compared across edges rather than against send latencies.
+
+### `nccl.collective.duration` (Histogram, Unit: `ns`)
+
+This metric records, for each completed NCCL operation, the span of its network / kernel activity: from the start of its first proxy op (or kernel channel, when `NCCL_PROFILER_TRACK_KERNEL_CH` is enabled) to the end of its last one. This is the same operation lifetime CoMMA computes for traces and summaries. Four classes of operations are not recorded:
+
+- Operations that produce no proxy op or kernel channel activity, e.g. single-node NVLink-only collectives with kernel channel tracking disabled.
+- Point-to-point operations not selected by sampling (`NCCL_PROFILER_P2P_SAMPLE_RATE`, `NCCL_PROFILER_P2P_RECV_SAMPLE_RATE`). With the default recv sample rate of `0.1`, `ncclRecv` durations are a 10% sample: bucket shapes are unbiased, but counts and sums under-report by 10x, and asymmetrically versus `ncclSend`.
+- Operations reclaimed by the hang timeout (`NCCL_PROFILER_NCCLOP_TIMEOUT`) before completing.
+- Operations on the small-message fast paths: point-to-point operations at or below `NCCL_PROFILER_SMALL_MSG_THRESHOLD` (default 64 KiB) are never tracked, and with the default `NCCL_PROFILER_SKIP_SMALL_COLLECTIVE=true` collectives other than AllReduce at or below the threshold are not tracked either. The `lt1m` size class therefore only covers operations outside these fast paths.
+
+Attributes:
+- `nccl.comm.hash`: Hexadecimal string identifying the NCCL communicator.
+- `nccl.collective.name`: Name of the operation (e.g., `ncclAllReduce`, `ncclSend`).
+- `nccl.rank`: Rank of the process within the communicator.
+- `nccl.size.class`: Coarse operation size class: `lt1m` (< 1 MiB), `1m_16m` (1-16 MiB), or `gt16m` (> 16 MiB). This preserves the latency-bound vs bandwidth-bound split with bounded cardinality.
+- `nccl.hostname`: Hostname of the node.
+
+Distinct attribute sets are capped by `NCCL_PROFILER_OTEL_METRICS_MAX_CARDINALITY` with the same "Top K" strategy as the latency metrics: keys are re-ranked by activity at every `NCCL_PROFILER_OTEL_METRICS_CARDINALITY_GROUPING_INTERVAL`, keys of closed communicators are evicted, and operations outside the top K are recorded with `nccl.metric.aggregated` set to `true`.
+
+### `nccl.collective.gap` (Histogram, Unit: `ns`)
+
+This metric records the duration of intervals where the process has no NCCL activity in flight. Activity is the union of operation enqueue windows, proxy op windows (network transfers), and kernel channel windows (when `NCCL_PROFILER_TRACK_KERNEL_CH` is enabled), so an operation keeps the process busy from its API call until its network / kernel work actually completes, not merely until it is enqueued. Each idle interval is recorded once, when the activity that ends it starts. Overlapping activity never contributes to a gap.
+
+Attributes:
+- `nccl.hostname`: Hostname of the node.
+- `nccl.pid`: Process id. Together with the hostname this identifies the process; a comm-local rank would be ambiguous for a metric that spans communicators.
+
+The gap is a per-process property by definition, so it deliberately carries no communicator dimension.
+
+Caveats — what counts as activity depends on the tracking configuration:
+- The metric assumes the default `NCCL_PROFILER_TRACK_NCCLOP=true`. With operation tracking disabled, only proxy op windows (and kernel channel windows, when enabled) count as activity, and everything else is reported as gap.
+- Operations whose execution produces no tracked activity window (see the duration carve-outs above) are only counted while they are enqueued, so their execution time appears as gap. Enable kernel channel tracking for full coverage of NVLink-only collectives.
+- Operations on the small-message fast paths (see the duration carve-outs above) are covered only while enqueued — small point-to-point operations not at all — so their transfer time, possibly tens of microseconds each, is reported as gap. Interpret gap totals with care for workloads dominated by messages below `NCCL_PROFILER_SMALL_MSG_THRESHOLD`.
+- Point-to-point operations not selected by sampling (`NCCL_PROFILER_P2P_SAMPLE_RATE`, `NCCL_PROFILER_P2P_RECV_SAMPLE_RATE`) produce no activity window at all, so their entire enqueue and transfer time is reported as gap. With the default recv sample rate of `0.1`, 90% of receive transfers count as idle time; set the sample rates to `1.0` before interpreting gap totals for p2p-heavy workloads such as pipeline parallelism.
+- Sub-microsecond gaps can appear between an operation's enqueue window and the start of its proxy activity; they land in the lowest buckets and carry negligible weight in gap-time totals.
+- If NCCL aborts a plan launch on an error path it may never stop the events it started; the in-flight count then stays above zero and the metric reports no further gaps for the process lifetime. CoMMA logs a one-time warning when it detects a stuck in-flight count. This only occurs after NCCL errors, which the job surfaces on its own.
 
 ### `nccl.collective.seq_num` (Gauge)