Upgrade opentelemetry deps from 0.26 -> 0.29

Cargo.toml

@@ -12,7 +12,7 @@ derive_more = { version = "2.0", features = ["constructor", "display", "from", "
 thiserror = "2"
 tonic = "0.12"
 tonic-build = "0.12"
-opentelemetry = { version = "0.26", features = ["metrics"] }
+opentelemetry = { version = "0.29", features = ["metrics"] }
 prost = "0.13"
 prost-types = "0.13"
 

README.md

@@ -64,12 +64,12 @@ You can run it using:
 
 ## Debugging
 
-The crate uses [tracing](https://github.com/tokio-rs/tracing) to help with debugging. To enable
-it for a test, insert the below snippet at the start of the test. By default, tracing data is output
-to stdout in a (reasonably) pretty manner.
+The crate uses [tracing](https://github.com/tokio-rs/tracing) to help with debugging. To enable it
+globally for tests, insert the below snippet at the start of the test. By default, tracing data is
+output to stdout in a (reasonably) pretty manner.
 
 ```rust
-crate::telemetry::test_telem_console();
+crate::telemetry::telemetry_init_fallback();
 ```
 
 The passed in options to initialization can be customized to export to an OTel collector, etc.

core-api/src/telemetry.rs

@@ -3,12 +3,12 @@ pub mod metrics;
 use crate::telemetry::metrics::CoreMeter;
 use std::{
     collections::HashMap,
-    fmt::Debug,
+    fmt::{Debug, Formatter},
     net::SocketAddr,
     sync::Arc,
     time::{Duration, SystemTime, UNIX_EPOCH},
 };
-use tracing_core::Level;
+use tracing_core::{Level, Subscriber};
 use url::Url;
 
 pub static METRIC_PREFIX: &str = "temporal_";
@@ -27,7 +27,7 @@ pub trait CoreTelemetry {
 }
 
 /// Telemetry configuration options. Construct with [TelemetryOptionsBuilder]
-#[derive(Debug, Clone, derive_builder::Builder)]
+#[derive(Clone, derive_builder::Builder)]
 #[non_exhaustive]
 pub struct TelemetryOptions {
     /// Optional logger - set as None to disable.
@@ -45,6 +45,39 @@ pub struct TelemetryOptions {
     /// A prefix to be applied to all core-created metrics. Defaults to "temporal_".
     #[builder(default = "METRIC_PREFIX.to_string()")]
     pub metric_prefix: String,
+    /// If provided, logging config will be ignored and this explicit subscriber will be used for
+    /// all logging and traces.
+    #[builder(setter(strip_option), default)]
+    pub subscriber_override: Option<Arc<dyn Subscriber + Send + Sync>>,
+}
+impl Debug for TelemetryOptions {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        #[derive(Debug)]
+        #[allow(dead_code)]
+        struct TelemetryOptions<'a> {
+            logging: &'a Option<Logger>,
+            metrics: &'a Option<Arc<dyn CoreMeter>>,
+            attach_service_name: &'a bool,
+            metric_prefix: &'a str,
+        }
+        let Self {
+            logging,
+            metrics,
+            attach_service_name,
+            metric_prefix,
+            ..
+        } = self;
+
+        Debug::fmt(
+            &TelemetryOptions {
+                logging,
+                metrics,
+                attach_service_name,
+                metric_prefix,
+            },
+            f,
+        )
+    }
 }
 
 /// Options for exporting to an OpenTelemetry Collector

core/Cargo.toml

@@ -43,13 +43,13 @@ itertools = "0.14"
 lru = "0.13"
 mockall = "0.13"
 opentelemetry = { workspace = true, features = ["metrics"], optional = true }
-opentelemetry_sdk = { version = "0.26", features = ["rt-tokio", "metrics"], optional = true }
-opentelemetry-otlp = { version = "0.26", features = ["tokio", "metrics", "tls", "http-proto", "reqwest-client", ], optional = true }
-opentelemetry-prometheus = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "e911383", optional = true }
+opentelemetry_sdk = { version = "0.29", features = ["rt-tokio", "metrics", "spec_unstable_metrics_views"], optional = true }
+opentelemetry-otlp = { version = "0.29", features = ["tokio", "metrics", "tls", "http-proto", "reqwest-client", "grpc-tonic"], optional = true }
+opentelemetry-prometheus = { version = "0.29", optional = true }
 parking_lot = { version = "0.12", features = ["send_guard"] }
 pid = "4.0"
 pin-project = "1.0"
-prometheus = "0.13"
+prometheus = "0.14"
 prost = { workspace = true }
 prost-types = { version = "0.6", package = "prost-wkt-types" }
 rand = "0.9"
@@ -115,6 +115,11 @@ name = "manual_tests"
 path = "../tests/manual_tests.rs"
 test = false
 
+[[test]]
+name = "global_metric_tests"
+path = "../tests/global_metric_tests.rs"
+test = false
+
 [[bench]]
 name = "workflow_replay"
 harness = false

core/src/telemetry/mod.rs

@@ -33,7 +33,7 @@ use std::{
     },
 };
 use temporal_sdk_core_api::telemetry::{
-    CoreLog, CoreTelemetry, Logger, TelemetryOptions,
+    CoreLog, CoreTelemetry, Logger, TelemetryOptions, TelemetryOptionsBuilder,
     metrics::{CoreMeter, MetricKeyValue, NewAttributes, TemporalMeter},
 };
 use tracing::{Level, Subscriber};
@@ -173,54 +173,59 @@ pub fn telemetry_init(opts: TelemetryOptions) -> Result<TelemetryInstance, anyho
     let mut forward_layer = None;
     // ===================================
 
-    let tracing_sub = opts.logging.map(|logger| {
-        match logger {
-            Logger::Console { filter } => {
-                // This is silly dupe but can't be avoided without boxing.
-                if env::var("TEMPORAL_CORE_PRETTY_LOGS").is_ok() {
-                    console_pretty_layer = Some(
-                        tracing_subscriber::fmt::layer()
-                            .with_target(false)
-                            .event_format(
-                                tracing_subscriber::fmt::format()
-                                    .pretty()
-                                    .with_source_location(false),
-                            )
-                            .with_filter(EnvFilter::new(filter)),
-                    )
-                } else {
-                    console_compact_layer = Some(
-                        tracing_subscriber::fmt::layer()
-                            .with_target(false)
-                            .event_format(
-                                tracing_subscriber::fmt::format()
-                                    .compact()
-                                    .with_source_location(false),
-                            )
-                            .with_filter(EnvFilter::new(filter)),
-                    )
+    let tracing_sub = if let Some(ts) = opts.subscriber_override {
+        Some(ts)
+    } else {
+        opts.logging.map(|logger| {
+            match logger {
+                Logger::Console { filter } => {
+                    // This is silly dupe but can't be avoided without boxing.
+                    if env::var("TEMPORAL_CORE_PRETTY_LOGS").is_ok() {
+                        console_pretty_layer = Some(
+                            tracing_subscriber::fmt::layer()
+                                .with_target(false)
+                                .event_format(
+                                    tracing_subscriber::fmt::format()
+                                        .pretty()
+                                        .with_source_location(false),
+                                )
+                                .with_filter(EnvFilter::new(filter)),
+                        )
+                    } else {
+                        console_compact_layer = Some(
+                            tracing_subscriber::fmt::layer()
+                                .with_target(false)
+                                .event_format(
+                                    tracing_subscriber::fmt::format()
+                                        .compact()
+                                        .with_source_location(false),
+                                )
+                                .with_filter(EnvFilter::new(filter)),
+                        )
+                    }
                 }
-            }
-            Logger::Forward { filter } => {
-                let (export_layer, lo) =
-                    CoreLogConsumerLayer::new_buffered(FORWARD_LOG_BUFFER_SIZE);
-                logs_out = Some(Mutex::new(lo));
-                forward_layer = Some(export_layer.with_filter(EnvFilter::new(filter)));
-            }
-            Logger::Push { filter, consumer } => {
-                forward_layer =
-                    Some(CoreLogConsumerLayer::new(consumer).with_filter(EnvFilter::new(filter)));
-            }
-        };
-        let reg = tracing_subscriber::registry()
-            .with(console_pretty_layer)
-            .with(console_compact_layer)
-            .with(forward_layer);
+                Logger::Forward { filter } => {
+                    let (export_layer, lo) =
+                        CoreLogConsumerLayer::new_buffered(FORWARD_LOG_BUFFER_SIZE);
+                    logs_out = Some(Mutex::new(lo));
+                    forward_layer = Some(export_layer.with_filter(EnvFilter::new(filter)));
+                }
+                Logger::Push { filter, consumer } => {
+                    forward_layer = Some(
+                        CoreLogConsumerLayer::new(consumer).with_filter(EnvFilter::new(filter)),
+                    );
+                }
+            };
+            let reg = tracing_subscriber::registry()
+                .with(console_pretty_layer)
+                .with(console_compact_layer)
+                .with(forward_layer);
 
-        #[cfg(feature = "tokio-console")]
-        let reg = reg.with(console_subscriber::spawn());
-        Arc::new(reg) as Arc<dyn Subscriber + Send + Sync>
-    });
+            #[cfg(feature = "tokio-console")]
+            let reg = reg.with(console_subscriber::spawn());
+            Arc::new(reg) as Arc<dyn Subscriber + Send + Sync>
+        })
+    };
 
     Ok(TelemetryInstance::new(
         tracing_sub,
@@ -231,6 +236,9 @@ pub fn telemetry_init(opts: TelemetryOptions) -> Result<TelemetryInstance, anyho
     ))
 }
 
+/// WARNING: Calling can cause panics because of https://github.com/tokio-rs/tracing/issues/1656
+/// Lang must not start using until resolved
+///
 /// Initialize telemetry/tracing globally. Useful for testing. Only takes affect when called
 /// the first time. Subsequent calls are ignored.
 pub fn telemetry_init_global(opts: TelemetryOptions) -> Result<(), anyhow::Error> {
@@ -247,8 +255,23 @@ pub fn telemetry_init_global(opts: TelemetryOptions) -> Result<(), anyhow::Error
     Ok(())
 }
 
-#[cfg(test)]
-pub use test_initters::*;
+/// WARNING: Calling can cause panics because of https://github.com/tokio-rs/tracing/issues/1656
+/// Lang must not start using until resolved
+///
+/// Initialize the fallback global handler. All lang SDKs should call this somewhere, once, at
+/// startup, as it initializes a fallback handler for any dependencies (looking at you, otel) that
+/// don't provide good ways to customize their tracing usage. It sets a WARN-level global filter
+/// that uses the default console logger.
+pub fn telemetry_init_fallback() -> Result<(), anyhow::Error> {
+    telemetry_init_global(
+        TelemetryOptionsBuilder::default()
+            .logging(Logger::Console {
+                filter: construct_filter_string(Level::DEBUG, Level::WARN),
+            })
+            .build()?,
+    )?;
+    Ok(())
+}
 
 /// A trait for using [Display] on the contents of vecs, etc, which don't implement it.
 ///
@@ -275,24 +298,3 @@ where
         format!("[{}]", self.iter().format(","))
     }
 }
-
-/// Helpers for test initialization
-#[cfg(test)]
-pub mod test_initters {
-    use super::*;
-    use temporal_sdk_core_api::telemetry::TelemetryOptionsBuilder;
-
-    /// Turn on logging to the console
-    #[allow(dead_code)] // Not always used, called to enable for debugging when needed
-    pub fn test_telem_console() {
-        telemetry_init_global(
-            TelemetryOptionsBuilder::default()
-                .logging(Logger::Console {
-                    filter: construct_filter_string(Level::DEBUG, Level::WARN),
-                })
-                .build()
-                .unwrap(),
-        )
-        .unwrap();
-    }
-}