From 5ae7c2250c71c4e05b56310e34d302a2057f739a Mon Sep 17 00:00:00 2001 From: everpcpc Date: Mon, 21 Oct 2024 16:08:02 +0800 Subject: [PATCH 1/5] feat: add config include_node_id for file logging --- src/common/tracing/src/config.rs | 3 +++ src/common/tracing/src/init.rs | 17 ++++++++++------- src/meta/binaries/metabench/main.rs | 1 + src/meta/binaries/metactl/main.rs | 1 + src/meta/binaries/metaverifier/main.rs | 1 + src/meta/service/src/configs/outer_v0.rs | 1 + src/query/config/src/config.rs | 12 ++++++++++++ .../storages/testdata/configs_table_basic.txt | 1 + 8 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/common/tracing/src/config.rs b/src/common/tracing/src/config.rs index 256700ff9baa..95e251359319 100644 --- a/src/common/tracing/src/config.rs +++ b/src/common/tracing/src/config.rs @@ -39,6 +39,7 @@ impl Config { format: "text".to_string(), limit: 48, prefix_filter: "databend_,openraft".to_string(), + include_node_id: false, }, stderr: StderrConfig { on: true, @@ -58,6 +59,7 @@ pub struct FileConfig { pub format: String, pub limit: usize, pub prefix_filter: String, + pub include_node_id: bool, } impl Display for FileConfig { @@ -79,6 +81,7 @@ impl Default for FileConfig { format: "json".to_string(), limit: 48, prefix_filter: "databend_,openraft".to_string(), + include_node_id: false, } } } diff --git a/src/common/tracing/src/init.rs b/src/common/tracing/src/init.rs index 93b5793dfdea..1c11b9344557 100644 --- a/src/common/tracing/src/init.rs +++ b/src/common/tracing/src/init.rs @@ -95,6 +95,7 @@ pub fn init_logging( // initialize tracing reporter if cfg.tracing.on { + let name = trace_name.clone(); let endpoint = cfg.tracing.otlp.endpoint.clone(); let mut kvs = cfg .tracing @@ -103,10 +104,7 @@ pub fn init_logging( .iter() .map(|(k, v)| opentelemetry::KeyValue::new(k.to_string(), v.to_string())) .collect::>(); - kvs.push(opentelemetry::KeyValue::new( - "service.name", - trace_name.clone(), - )); + kvs.push(opentelemetry::KeyValue::new("service.name", name.clone())); for (k, v) in &labels { kvs.push(opentelemetry::KeyValue::new(k.to_string(), v.to_string())); } @@ -130,7 +128,7 @@ pub fn init_logging( .build_span_exporter() .expect("initialize oltp http exporter"), }; - let (reporter_rt, otlp_reporter) = Thread::spawn(|| { + let (reporter_rt, otlp_reporter) = Thread::spawn(move || { // init runtime with 2 threads let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(2) @@ -142,7 +140,7 @@ pub fn init_logging( exporter, opentelemetry::trace::SpanKind::Server, Cow::Owned(opentelemetry_sdk::Resource::new(kvs)), - opentelemetry::InstrumentationLibrary::builder(trace_name).build(), + opentelemetry::InstrumentationLibrary::builder(name).build(), ) }); (rt, reporter) @@ -174,8 +172,13 @@ pub fn init_logging( // file logger if cfg.file.on { + let name = if cfg.file.include_node_id { + &trace_name + } else { + log_name + }; let (normal_log_file, flush_guard) = - new_rolling_file_appender(&cfg.file.dir, log_name, cfg.file.limit); + new_rolling_file_appender(&cfg.file.dir, name, cfg.file.limit); _drop_guards.push(flush_guard); let dispatch = Dispatch::new() diff --git a/src/meta/binaries/metabench/main.rs b/src/meta/binaries/metabench/main.rs index ef3c9808daf1..20ef5f74ae7a 100644 --- a/src/meta/binaries/metabench/main.rs +++ b/src/meta/binaries/metabench/main.rs @@ -93,6 +93,7 @@ async fn main() { format: "text".to_string(), limit: 48, prefix_filter: "databend_".to_string(), + include_node_id: false, }, stderr: StderrConfig { on: true, diff --git a/src/meta/binaries/metactl/main.rs b/src/meta/binaries/metactl/main.rs index 8900ce45ecda..3c5fbef0b6ab 100644 --- a/src/meta/binaries/metactl/main.rs +++ b/src/meta/binaries/metactl/main.rs @@ -356,6 +356,7 @@ async fn main() -> anyhow::Result<()> { format: "text".to_string(), limit: 48, prefix_filter: "databend_".to_string(), + include_node_id: false, }, ..Default::default() }; diff --git a/src/meta/binaries/metaverifier/main.rs b/src/meta/binaries/metaverifier/main.rs index 04a1ee43043a..ac2ad9c8c9b6 100644 --- a/src/meta/binaries/metaverifier/main.rs +++ b/src/meta/binaries/metaverifier/main.rs @@ -85,6 +85,7 @@ async fn main() -> Result<()> { format: "text".to_string(), limit: 48, prefix_filter: "databend_".to_string(), + include_node_id: false, }, stderr: StderrConfig { on: true, diff --git a/src/meta/service/src/configs/outer_v0.rs b/src/meta/service/src/configs/outer_v0.rs index 57eeb0bf047d..4fd81a2e9e0e 100644 --- a/src/meta/service/src/configs/outer_v0.rs +++ b/src/meta/service/src/configs/outer_v0.rs @@ -730,6 +730,7 @@ impl Into for FileLogConfig { format: self.file_format, limit: self.file_limit, prefix_filter: self.file_prefix_filter, + include_node_id: false, } } } diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index 5cedd0c0d59c..9ad694e6afeb 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -2094,6 +2094,16 @@ pub struct FileLogConfig { )] #[serde(rename = "prefix_filter")] pub file_prefix_filter: String, + + /// Whether to include node id in the log file name + /// If true, the log file name will be like `databend-query-@-