feat: add config include_node_id for file logging

src/common/tracing/src/config.rs

@@ -39,6 +39,7 @@ impl Config {
                 format: "text".to_string(),
                 limit: 48,
                 prefix_filter: "databend_,openraft".to_string(),
+                include_node_id: false,
             },
             stderr: StderrConfig {
                 on: true,
@@ -58,6 +59,7 @@ pub struct FileConfig {
     pub format: String,
     pub limit: usize,
     pub prefix_filter: String,
+    pub include_node_id: bool,
 }
 
 impl Display for FileConfig {
@@ -79,6 +81,7 @@ impl Default for FileConfig {
             format: "json".to_string(),
             limit: 48,
             prefix_filter: "databend_,openraft".to_string(),
+            include_node_id: false,
         }
     }
 }

src/common/tracing/src/init.rs

@@ -92,9 +92,15 @@ pub fn init_logging(
             }
         ),
     };
+    let log_file_name = if cfg.file.include_node_id {
+        &trace_name
+    } else {
+        log_name
+    };
 
     // initialize tracing reporter
     if cfg.tracing.on {
+        let name = trace_name.clone();
         let endpoint = cfg.tracing.otlp.endpoint.clone();
         let mut kvs = cfg
             .tracing
@@ -103,10 +109,7 @@ pub fn init_logging(
             .iter()
             .map(|(k, v)| opentelemetry::KeyValue::new(k.to_string(), v.to_string()))
             .collect::<Vec<_>>();
-        kvs.push(opentelemetry::KeyValue::new(
-            "service.name",
-            trace_name.clone(),
-        ));
+        kvs.push(opentelemetry::KeyValue::new("service.name", name.clone()));
         for (k, v) in &labels {
             kvs.push(opentelemetry::KeyValue::new(k.to_string(), v.to_string()));
         }
@@ -130,7 +133,7 @@ pub fn init_logging(
                 .build_span_exporter()
                 .expect("initialize oltp http exporter"),
         };
-        let (reporter_rt, otlp_reporter) = Thread::spawn(|| {
+        let (reporter_rt, otlp_reporter) = Thread::spawn(move || {
             // init runtime with 2 threads
             let rt = tokio::runtime::Builder::new_multi_thread()
                 .worker_threads(2)
@@ -142,7 +145,7 @@ pub fn init_logging(
                     exporter,
                     opentelemetry::trace::SpanKind::Server,
                     Cow::Owned(opentelemetry_sdk::Resource::new(kvs)),
-                    opentelemetry::InstrumentationLibrary::builder(trace_name).build(),
+                    opentelemetry::InstrumentationLibrary::builder(name).build(),
                 )
             });
             (rt, reporter)
@@ -175,7 +178,7 @@ pub fn init_logging(
     // file logger
     if cfg.file.on {
         let (normal_log_file, flush_guard) =
-            new_rolling_file_appender(&cfg.file.dir, log_name, cfg.file.limit);
+            new_rolling_file_appender(&cfg.file.dir, log_file_name, cfg.file.limit);
         _drop_guards.push(flush_guard);
 
         let dispatch = Dispatch::new()
@@ -286,7 +289,7 @@ pub fn init_logging(
     if cfg.query.on {
         if !cfg.query.dir.is_empty() {
             let (query_log_file, flush_guard) =
-                new_rolling_file_appender(&cfg.query.dir, log_name, cfg.file.limit);
+                new_rolling_file_appender(&cfg.query.dir, log_file_name, cfg.file.limit);
             _drop_guards.push(flush_guard);
 
             let dispatch = Dispatch::new()
@@ -328,7 +331,7 @@ pub fn init_logging(
     if cfg.profile.on {
         if !cfg.profile.dir.is_empty() {
             let (profile_log_file, flush_guard) =
-                new_rolling_file_appender(&cfg.profile.dir, log_name, cfg.file.limit);
+                new_rolling_file_appender(&cfg.profile.dir, log_file_name, cfg.file.limit);
             _drop_guards.push(flush_guard);
 
             let dispatch = Dispatch::new()
@@ -369,7 +372,7 @@ pub fn init_logging(
     // structured logger
     if cfg.structlog.on && !cfg.structlog.dir.is_empty() {
         let (structlog_log_file, flush_guard) =
-            new_rolling_file_appender(&cfg.structlog.dir, log_name, cfg.file.limit);
+            new_rolling_file_appender(&cfg.structlog.dir, log_file_name, cfg.file.limit);
         _drop_guards.push(flush_guard);
 
         let dispatch = Dispatch::new()

src/meta/binaries/metabench/main.rs

@@ -93,6 +93,7 @@ async fn main() {
             format: "text".to_string(),
             limit: 48,
             prefix_filter: "databend_".to_string(),
+            include_node_id: false,
         },
         stderr: StderrConfig {
             on: true,

src/meta/binaries/metactl/main.rs

@@ -356,6 +356,7 @@ async fn main() -> anyhow::Result<()> {
             format: "text".to_string(),
             limit: 48,
             prefix_filter: "databend_".to_string(),
+            include_node_id: false,
         },
         ..Default::default()
     };

src/meta/binaries/metaverifier/main.rs

@@ -85,6 +85,7 @@ async fn main() -> Result<()> {
             format: "text".to_string(),
             limit: 48,
             prefix_filter: "databend_".to_string(),
+            include_node_id: false,
         },
         stderr: StderrConfig {
             on: true,

src/meta/service/src/configs/outer_v0.rs

@@ -730,6 +730,7 @@ impl Into<InnerFileLogConfig> for FileLogConfig {
             format: self.file_format,
             limit: self.file_limit,
             prefix_filter: self.file_prefix_filter,
+            include_node_id: false,
         }
     }
 }

src/query/config/src/config.rs

@@ -2094,6 +2094,16 @@ pub struct FileLogConfig {
     )]
     #[serde(rename = "prefix_filter")]
     pub file_prefix_filter: String,
+
+    /// Whether to include node id in the log file name
+    /// If true, the log file name will be like `databend-query-<cluster_id>@<node_id>-<time>.log`
+    #[clap(
+        long = "log-file-include-node-id",
+        value_name = "VALUE",
+        default_value = "false"
+    )]
+    #[serde(rename = "include_node_id")]
+    pub include_node_id: bool,
 }
 
 impl Default for FileLogConfig {
@@ -2113,6 +2123,7 @@ impl TryInto<InnerFileLogConfig> for FileLogConfig {
             format: self.file_format,
             limit: self.file_limit,
             prefix_filter: self.file_prefix_filter,
+            include_node_id: self.include_node_id,
         })
     }
 }
@@ -2126,6 +2137,7 @@ impl From<InnerFileLogConfig> for FileLogConfig {
             file_format: inner.format,
             file_limit: inner.limit,
             file_prefix_filter: inner.prefix_filter,
+            include_node_id: inner.include_node_id,
         }
     }
 }

src/query/service/tests/it/storages/testdata/configs_table_basic.txt

@@ -29,6 +29,7 @@ DB.Table: 'system'.'configs', Table: configs-table_id:1, ver:0, Engine: SystemCo
 | 'log'     | 'dir'                                           | './.databend/logs'                                                                                                                                                                                | ''       |
 | 'log'     | 'file.dir'                                      | './.databend/logs'                                                                                                                                                                                | ''       |
 | 'log'     | 'file.format'                                   | 'text'                                                                                                                                                                                            | ''       |
+| 'log'     | 'file.include_node_id'                          | 'false'                                                                                                                                                                                           | ''       |
 | 'log'     | 'file.level'                                    | 'DEBUG'                                                                                                                                                                                           | ''       |
 | 'log'     | 'file.limit'                                    | '48'                                                                                                                                                                                              | ''       |
 | 'log'     | 'file.on'                                       | 'true'                                                                                                                                                                                            | ''       |