From 609a59e1fd020750e0ec7bcb9f6783826371b8c3 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Sun, 12 Jan 2025 22:17:09 +0800 Subject: [PATCH 01/31] ### Add Process List Management - **Error Handling Enhancements**: --- src/common/meta/src/error.rs | 8 ++ src/common/meta/src/key.rs | 10 +++ src/common/meta/src/key/process_list.rs | 108 ++++++++++++++++++++++ src/common/meta/src/key/view_info.rs | 11 +-- src/query/src/error.rs | 16 ++++ src/query/src/lib.rs | 1 + src/query/src/process_list.rs | 114 ++++++++++++++++++++++++ 7 files changed, 263 insertions(+), 5 deletions(-) create mode 100644 src/common/meta/src/key/process_list.rs create mode 100644 src/query/src/process_list.rs diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs index 1bdb3d08577b..8407dc984caf 100644 --- a/src/common/meta/src/error.rs +++ b/src/common/meta/src/error.rs @@ -741,6 +741,13 @@ pub enum Error { location: Location, }, + #[snafu(display("Invalid process list key: {}", key))] + InvalidProcessKey { + key: String, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Invalid topic name prefix: {}", prefix))] InvalidTopicNamePrefix { prefix: String, @@ -898,6 +905,7 @@ impl ErrorExt for Error { #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))] RdsTransactionRetryFailed { .. } => StatusCode::Internal, Error::DatanodeTableInfoNotFound { .. } => StatusCode::Internal, + InvalidProcessKey { .. } => StatusCode::Internal } } diff --git a/src/common/meta/src/key.rs b/src/common/meta/src/key.rs index 72a93a2d94bc..50a7aec640e4 100644 --- a/src/common/meta/src/key.rs +++ b/src/common/meta/src/key.rs @@ -102,6 +102,7 @@ pub mod datanode_table; pub mod flow; pub mod maintenance; pub mod node_address; +pub mod process_list; mod schema_metadata_manager; pub mod schema_name; pub mod table_info; @@ -154,6 +155,7 @@ use self::tombstone::TombstoneManager; use crate::error::{self, Result, SerdeJsonSnafu}; use crate::key::flow::flow_state::FlowStateValue; use crate::key::node_address::NodeAddressValue; +use crate::key::process_list::ProcessValue; use crate::key::table_route::TableRouteKey; use crate::key::txn_helper::TxnOpGetResponseSet; use crate::kv_backend::txn::{Txn, TxnOp}; @@ -174,6 +176,8 @@ pub const CATALOG_NAME_KEY_PREFIX: &str = "__catalog_name"; pub const SCHEMA_NAME_KEY_PREFIX: &str = "__schema_name"; pub const TABLE_ROUTE_PREFIX: &str = "__table_route"; pub const NODE_ADDRESS_PREFIX: &str = "__node_address"; +/// The prefix for process list values. +pub const PROCESS_LIST_PREFIX: &str = "__process"; pub const KAFKA_TOPIC_KEY_PREFIX: &str = "__topic_name/kafka"; // The legacy topic key prefix is used to store the topic name in previous versions. pub const LEGACY_TOPIC_KEY_PREFIX: &str = "__created_wal_topics/kafka"; @@ -259,6 +263,11 @@ lazy_static! { .unwrap(); } +lazy_static! { + static ref PROCESS_LIST_PATTERN: Regex = + Regex::new(&format!("^{PROCESS_LIST_PREFIX}/([0-9.]+)-([0-9]+)$")).unwrap(); +} + /// The key of metadata. pub trait MetadataKey<'a, T> { fn to_bytes(&self) -> Vec; @@ -1337,6 +1346,7 @@ impl_metadata_value! { SchemaNameValue, FlowStateValue, PoisonValue + ProcessValue } impl_optional_metadata_value! { diff --git a/src/common/meta/src/key/process_list.rs b/src/common/meta/src/key/process_list.rs new file mode 100644 index 000000000000..66bdc69419fb --- /dev/null +++ b/src/common/meta/src/key/process_list.rs @@ -0,0 +1,108 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use snafu::OptionExt; + +use crate::error; +use crate::key::{MetadataKey, PROCESS_LIST_PATTERN, PROCESS_LIST_PREFIX}; + +/// Key for running queries tracked in metasrv. +/// Layout: `__process/{frontend ip}-{query id}` +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ProcessKey { + pub frontend_ip: String, + pub id: u64, +} + +impl Display for ProcessKey { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}/{}-{}", + PROCESS_LIST_PREFIX, self.frontend_ip, self.id + ) + } +} + +impl MetadataKey<'_, ProcessKey> for ProcessKey { + fn to_bytes(&self) -> Vec { + self.to_string().into_bytes() + } + + fn from_bytes(bytes: &[u8]) -> crate::error::Result { + let key_str = std::str::from_utf8(bytes).map_err(|_e| { + error::InvalidProcessKeySnafu { + key: String::from_utf8_lossy(bytes).into_owned(), + } + .build() + })?; + let captures = PROCESS_LIST_PATTERN.captures(key_str).with_context(|| { + error::InvalidProcessKeySnafu { + key: String::from_utf8_lossy(bytes).into_owned(), + } + })?; + let id = u64::from_str(&captures[2]).map_err(|_| { + error::InvalidProcessKeySnafu { + key: String::from_utf8_lossy(bytes).into_owned(), + } + .build() + })?; + Ok(Self { + frontend_ip: captures[1].to_string(), + id, + }) + } +} + +/// Detail value of process. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ProcessValue { + /// The running query sql. + pub query: String, + /// Query start timestamp in milliseconds. + pub start_timestamp_ms: i64, +} + +#[cfg(test)] +mod tests { + use crate::key::process_list::ProcessKey; + use crate::key::MetadataKey; + + fn check_serialization(input: ProcessKey) { + let serialized = input.to_bytes(); + let key = ProcessKey::from_bytes(&serialized).unwrap(); + assert_eq!(&input.frontend_ip, &key.frontend_ip); + assert_eq!(input.id, key.id); + } + + #[test] + fn test_capture_process_list() { + check_serialization(ProcessKey { + frontend_ip: "192.168.0.1".to_string(), + id: 1, + }); + check_serialization(ProcessKey { + frontend_ip: "192.168.0.1".to_string(), + id: 0, + }); + check_serialization(ProcessKey { + frontend_ip: "255.255.255.255".to_string(), + id: 0, + }); + } +} diff --git a/src/common/meta/src/key/view_info.rs b/src/common/meta/src/key/view_info.rs index 9f4f574c2708..75ccdb6f291c 100644 --- a/src/common/meta/src/key/view_info.rs +++ b/src/common/meta/src/key/view_info.rs @@ -70,11 +70,12 @@ impl MetadataKey<'_, ViewInfoKey> for ViewInfoKey { } .build() })?; - let captures = VIEW_INFO_KEY_PATTERN - .captures(key) - .context(InvalidViewInfoSnafu { - err_msg: format!("Invalid ViewInfoKey '{key}'"), - })?; + let captures = + VIEW_INFO_KEY_PATTERN + .captures(key) + .with_context(|| InvalidViewInfoSnafu { + err_msg: format!("Invalid ViewInfoKey '{key}'"), + })?; // Safety: pass the regex check above let view_id = captures[1].parse::().unwrap(); Ok(ViewInfoKey { view_id }) diff --git a/src/query/src/error.rs b/src/query/src/error.rs index c2a2e960b0f6..e74895aa346b 100644 --- a/src/query/src/error.rs +++ b/src/query/src/error.rs @@ -323,6 +323,20 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to serialize process value when registering."))] + RegisterProcess { + source: common_meta::error::Error, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to list all running processes."))] + ListProcesses { + source: common_meta::error::Error, + #[snafu(implicit)] + location: Location, + }, } impl ErrorExt for Error { @@ -376,6 +390,8 @@ impl ErrorExt for Error { GetFulltextOptions { source, .. } | GetSkippingIndexOptions { source, .. } => { source.status_code() } + RegisterProcess { .. } => StatusCode::Internal, + ListProcesses { .. } => StatusCode::StorageUnavailable, } } diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs index 6e1fbfae0af8..d4a8d920eea7 100644 --- a/src/query/src/lib.rs +++ b/src/query/src/lib.rs @@ -42,6 +42,7 @@ pub mod sql; pub mod stats; pub(crate) mod window_sort; +mod process_list; #[cfg(test)] pub(crate) mod test_util; #[cfg(test)] diff --git a/src/query/src/process_list.rs b/src/query/src/process_list.rs new file mode 100644 index 000000000000..2aa96258b35c --- /dev/null +++ b/src/query/src/process_list.rs @@ -0,0 +1,114 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use common_meta::key::process_list::{ProcessKey, ProcessValue}; +use common_meta::key::{MetadataKey, MetadataValue, PROCESS_LIST_PREFIX}; +use common_meta::kv_backend::KvBackendRef; +use common_meta::rpc::store::{PutRequest, RangeRequest}; +use common_meta::sequence::{SequenceBuilder, SequenceRef}; +use common_time::util::current_time_millis; +use snafu::ResultExt; + +use crate::error; + +/// Sequence key for process list entries. +pub const PROCESS_ID_SEQ: &str = "process_id_seq"; + +pub struct Process { + key: ProcessKey, + value: ProcessValue, +} + +pub struct ProcessManager { + server_addr: String, + sequencer: SequenceRef, + kv_client: KvBackendRef, +} + +impl ProcessManager { + pub fn new(server_addr: String, kv_client: KvBackendRef) -> Self { + let sequencer = Arc::new( + SequenceBuilder::new(PROCESS_ID_SEQ, kv_client.clone()) + .initial(0) + .step(100) + .build(), + ); + Self { + server_addr, + sequencer, + kv_client, + } + } + + pub async fn register_query(&self, query: String) -> error::Result<()> { + let process_id = self.sequencer.next().await.unwrap(); + let key = ProcessKey { + frontend_ip: self.server_addr.clone(), + id: process_id, + } + .to_bytes(); + let current_time = current_time_millis(); + let value = ProcessValue { + query, + start_timestamp_ms: current_time, + } + .try_as_raw_value() + .context(error::RegisterProcessSnafu)?; + + self.kv_client + .put(PutRequest { + key, + value, + prev_kv: false, + }) + .await + .context(error::RegisterProcessSnafu)?; + Ok(()) + } + + pub async fn list_all_processes(&self) -> error::Result> { + let mut all_processes = vec![]; + let mut has_nex = true; + while has_nex { + let resp = self + .kv_client + .range(RangeRequest { + key: PROCESS_LIST_PREFIX.to_string().into_bytes(), + range_end: vec![], + limit: 0, + keys_only: false, + }) + .await + .context(error::ListProcessesSnafu)?; + + let process = resp + .kvs + .into_iter() + .map(|kv| { + Ok(Process { + key: ProcessKey::from_bytes(&kv.key)?, + value: ProcessValue::try_from_raw_value(&kv.value)?, + }) + }) + .collect::, _>>() + .context(error::ListProcessesSnafu)?; + all_processes.extend(process); + has_nex = resp.more; + } + + Ok(all_processes) + } +} From 35d65a53a3902eab6f18aeac337cff08d032b028 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Mon, 13 Jan 2025 13:06:06 +0800 Subject: [PATCH 02/31] refactor: Update test IP addresses to include ports in ProcessKey --- src/common/meta/src/key/process_list.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/common/meta/src/key/process_list.rs b/src/common/meta/src/key/process_list.rs index 66bdc69419fb..802279923655 100644 --- a/src/common/meta/src/key/process_list.rs +++ b/src/common/meta/src/key/process_list.rs @@ -25,6 +25,7 @@ use crate::key::{MetadataKey, PROCESS_LIST_PATTERN, PROCESS_LIST_PREFIX}; /// Layout: `__process/{frontend ip}-{query id}` #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ProcessKey { + //todo(hl): maybe we don't have to own a string pub frontend_ip: String, pub id: u64, } @@ -93,15 +94,15 @@ mod tests { #[test] fn test_capture_process_list() { check_serialization(ProcessKey { - frontend_ip: "192.168.0.1".to_string(), + frontend_ip: "192.168.0.1:4001".to_string(), id: 1, }); check_serialization(ProcessKey { - frontend_ip: "192.168.0.1".to_string(), + frontend_ip: "192.168.0.1:4002".to_string(), id: 0, }); check_serialization(ProcessKey { - frontend_ip: "255.255.255.255".to_string(), + frontend_ip: "255.255.255.255:80".to_string(), id: 0, }); } From 0d90277dd40436527c171eb252fb9800bb706719 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 15 Jan 2025 15:42:32 +0800 Subject: [PATCH 03/31] feat/show-process-list: Refactor Process Management in Meta Module - Introduced `ProcessManager` for handling process registration and deregistration. - Added methods for managing and querying process states, including `register_query`, `deregister_query`, and `list_all_processes`. - Removed redundant process management code from the query module. - Updated error handling to reflect changes in process management. - Enhanced test coverage for process management functionalities. --- src/common/meta/src/error.rs | 26 +- src/common/meta/src/key.rs | 6 +- src/common/meta/src/key/process_list.rs | 353 +++++++++++++++++++++++- src/query/src/error.rs | 16 -- src/query/src/lib.rs | 1 - src/query/src/process_list.rs | 114 -------- 6 files changed, 378 insertions(+), 138 deletions(-) delete mode 100644 src/query/src/process_list.rs diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs index 8407dc984caf..7cc48eb0d17f 100644 --- a/src/common/meta/src/error.rs +++ b/src/common/meta/src/error.rs @@ -790,6 +790,27 @@ pub enum Error { #[snafu(source)] source: common_procedure::error::Error, }, + // #[snafu(display("Failed to serialize process value when registering."))] + // RegisterProcess { + // source: crate::error::Error, + // #[snafu(implicit)] + // location: Location, + // }, + // + // #[snafu(display("Failed to list all running processes."))] + // ListProcesses { + // source: crate::error::Error, + // #[snafu(implicit)] + // location: Location, + // }, + // + // #[snafu(display("Failed to delete process {}", msg))] + // DeleteProcess { + // msg: String, + // source: crate::error::Error, + // #[snafu(implicit)] + // location: Location, + // }, } pub type Result = std::result::Result; @@ -905,7 +926,10 @@ impl ErrorExt for Error { #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))] RdsTransactionRetryFailed { .. } => StatusCode::Internal, Error::DatanodeTableInfoNotFound { .. } => StatusCode::Internal, - InvalidProcessKey { .. } => StatusCode::Internal + InvalidProcessKey { .. } => StatusCode::Internal, + // RegisterProcess { .. } | ListProcesses { .. } | DeleteProcess { .. } => { + // StatusCode::Internal + // } } } diff --git a/src/common/meta/src/key.rs b/src/common/meta/src/key.rs index 50a7aec640e4..163d5ffcf6ea 100644 --- a/src/common/meta/src/key.rs +++ b/src/common/meta/src/key.rs @@ -264,8 +264,10 @@ lazy_static! { } lazy_static! { - static ref PROCESS_LIST_PATTERN: Regex = - Regex::new(&format!("^{PROCESS_LIST_PREFIX}/([0-9.]+)-([0-9]+)$")).unwrap(); + static ref PROCESS_LIST_PATTERN: Regex = Regex::new(&format!( + "^{PROCESS_LIST_PREFIX}/([0-9.]+:[0-9]+)-([0-9]+)$" + )) + .unwrap(); } /// The key of metadata. diff --git a/src/common/meta/src/key/process_list.rs b/src/common/meta/src/key/process_list.rs index 802279923655..ec2054b093cb 100644 --- a/src/common/meta/src/key/process_list.rs +++ b/src/common/meta/src/key/process_list.rs @@ -14,15 +14,23 @@ use std::fmt::{Display, Formatter}; use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; +use common_telemetry::{debug, info, warn}; +use common_time::util::current_time_millis; use serde::{Deserialize, Serialize}; use snafu::OptionExt; use crate::error; -use crate::key::{MetadataKey, PROCESS_LIST_PATTERN, PROCESS_LIST_PREFIX}; +use crate::key::{MetadataKey, MetadataValue, PROCESS_LIST_PATTERN, PROCESS_LIST_PREFIX}; +use crate::kv_backend::KvBackendRef; +use crate::range_stream::PaginationStream; +use crate::rpc::store::{DeleteRangeRequest, PutRequest, RangeRequest}; +use crate::sequence::{SequenceBuilder, SequenceRef}; /// Key for running queries tracked in metasrv. -/// Layout: `__process/{frontend ip}-{query id}` +/// Layout: `__process/{frontend server addr}-{query id}` #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ProcessKey { //todo(hl): maybe we don't have to own a string @@ -45,7 +53,7 @@ impl MetadataKey<'_, ProcessKey> for ProcessKey { self.to_string().into_bytes() } - fn from_bytes(bytes: &[u8]) -> crate::error::Result { + fn from_bytes(bytes: &[u8]) -> error::Result { let key_str = std::str::from_utf8(bytes).map_err(|_e| { error::InvalidProcessKeySnafu { key: String::from_utf8_lossy(bytes).into_owned(), @@ -79,9 +87,140 @@ pub struct ProcessValue { pub start_timestamp_ms: i64, } +/// Sequence key for process list entries. +pub const PROCESS_ID_SEQ: &str = "process_id_seq"; + +/// Running process instance. +pub struct Process { + key: ProcessKey, + value: ProcessValue, +} + +impl Process { + /// Returns server address of running process. + pub fn server_addr(&self) -> &str { + &self.key.frontend_ip + } + + /// Returns id of query. + pub fn query_id(&self) -> u64 { + self.key.id + } + + /// Returns query string details. + pub fn query_string(&self) -> &str { + &self.value.query + } + + /// Calculates the elapsed time of query. Returns None of system clock jumps backwards. + pub fn query_elapsed(&self) -> Option { + let now = current_time_millis(); + if now < self.value.start_timestamp_ms { + None + } else { + Some(Duration::from_millis( + (now - self.value.start_timestamp_ms) as u64, + )) + } + } +} + +pub struct ProcessManager { + server_addr: String, + sequencer: SequenceRef, + kv_client: KvBackendRef, +} + +impl ProcessManager { + /// Create a [ProcessManager] instance with server address and kv client. + pub fn new(server_addr: String, kv_client: KvBackendRef) -> Self { + let sequencer = Arc::new( + SequenceBuilder::new(PROCESS_ID_SEQ, kv_client.clone()) + .initial(0) + .step(100) + .build(), + ); + Self { + server_addr, + sequencer, + kv_client, + } + } + + /// Registers a submitted query. + pub async fn register_query(&self, query: String) -> error::Result { + let process_id = self.sequencer.next().await?; + let key = ProcessKey { + frontend_ip: self.server_addr.clone(), + id: process_id, + } + .to_bytes(); + let current_time = current_time_millis(); + let value = ProcessValue { + query, + start_timestamp_ms: current_time, + } + .try_as_raw_value()?; + + self.kv_client + .put(PutRequest { + key, + value, + prev_kv: false, + }) + .await?; + Ok(process_id) + } + + /// De-register a query from process list. + pub async fn deregister_query(&self, id: u64) -> error::Result<()> { + let key = ProcessKey { + frontend_ip: self.server_addr.clone(), + id, + } + .to_bytes(); + let prev_kv = self.kv_client.delete(&key, true).await?; + + if let Some(_kv) = prev_kv { + debug!("Successfully deregistered process {}", id); + } else { + warn!("Cannot find process to deregister process: {}", id); + } + Ok(()) + } + + /// De-register all queries running on current frontend. + pub async fn deregister_all_queries(&self) -> error::Result<()> { + let prefix = format!("{}/{}-", PROCESS_LIST_PREFIX, self.server_addr); + let delete_range_request = DeleteRangeRequest::new().with_prefix(prefix.as_bytes()); + self.kv_client.delete_range(delete_range_request).await?; + info!("All queries on {} has been deregistered", self.server_addr); + Ok(()) + } + + /// List all running processes in cluster. + pub fn list_all_processes(&self) -> error::Result> { + let prefix = format!("{}/{}-", PROCESS_LIST_PREFIX, self.server_addr); + let req = RangeRequest::new().with_prefix(prefix.as_bytes()); + let stream = PaginationStream::new(self.kv_client.clone(), req, 100, |kv| { + let key = ProcessKey::from_bytes(&kv.key)?; + let value = ProcessValue::try_from_raw_value(&kv.value)?; + Ok(Process { key, value }) + }); + Ok(stream) + } + + #[cfg(test)] + async fn dump(&self) -> error::Result> { + use futures_util::TryStreamExt; + self.list_all_processes()?.into_stream().try_collect().await + } +} + #[cfg(test)] mod tests { - use crate::key::process_list::ProcessKey; + + use super::*; use crate::key::MetadataKey; fn check_serialization(input: ProcessKey) { @@ -106,4 +245,210 @@ mod tests { id: 0, }); } + + #[test] + fn test_process_key_display() { + let key = ProcessKey { + frontend_ip: "127.0.0.1:3000".to_string(), + id: 42, + }; + assert_eq!(key.to_string(), "__process/127.0.0.1:3000-42"); + } + + #[test] + fn test_process_key_from_bytes_valid() { + let bytes = b"__process/10.0.0.1:8080-123"; + let key = ProcessKey::from_bytes(bytes).unwrap(); + assert_eq!(key.frontend_ip, "10.0.0.1:8080"); + assert_eq!(key.id, 123); + } + + #[test] + fn test_process_key_from_bytes_invalid_format() { + let bytes = b"invalid_format"; + let result = ProcessKey::from_bytes(bytes); + assert!(result.is_err()); + } + + #[test] + fn test_process_key_from_bytes_invalid_id() { + let bytes = b"__process/10.0.0.1:8080-abc"; + let result = ProcessKey::from_bytes(bytes); + assert!(result.is_err()); + } + + #[test] + fn test_process_value_serialization() { + let value = ProcessValue { + query: "SELECT * FROM test".to_string(), + start_timestamp_ms: 1690000000000, + }; + + // Test serialization roundtrip + let serialized = serde_json::to_string(&value).unwrap(); + let deserialized: ProcessValue = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(value.query, deserialized.query); + assert_eq!(value.start_timestamp_ms, deserialized.start_timestamp_ms); + } + + use std::sync::Arc; + use std::time::{SystemTime, UNIX_EPOCH}; + + use crate::kv_backend::memory::MemoryKvBackend; + + #[tokio::test] + async fn test_register_query() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let process_id = process_manager + .register_query("SELECT * FROM table".to_string()) + .await + .unwrap(); + + let running_processes = process_manager.dump().await.unwrap(); + assert_eq!(running_processes.len(), 1); + assert_eq!(running_processes[0].key.frontend_ip, "127.0.0.1:8000"); + assert_eq!(running_processes[0].key.id, process_id); + assert_eq!(running_processes[0].value.query, "SELECT * FROM table"); + + process_manager.deregister_query(process_id).await.unwrap(); + assert_eq!(process_manager.dump().await.unwrap().len(), 0); + } + + #[tokio::test] + async fn test_register_multiple_queries() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + // Register multiple queries + let id1 = process_manager + .register_query("SELECT 1".to_string()) + .await + .unwrap(); + let id2 = process_manager + .register_query("SELECT 2".to_string()) + .await + .unwrap(); + let id3 = process_manager + .register_query("SELECT 3".to_string()) + .await + .unwrap(); + + // Verify all are registered + assert_eq!(process_manager.dump().await.unwrap().len(), 3); + + // Deregister middle one + process_manager.deregister_query(id2).await.unwrap(); + let processes = process_manager.dump().await.unwrap(); + assert_eq!(processes.len(), 2); + assert!(processes.iter().any(|p| p.key.id == id1)); + assert!(processes.iter().any(|p| p.key.id == id3)); + } + + #[tokio::test] + async fn test_deregister_nonexistent_query() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + // Try to deregister non-existent ID + let result = process_manager.deregister_query(999).await; + assert!(result.is_ok()); // Should succeed with warning + } + + #[tokio::test] + async fn test_process_timestamps() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + let before = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + + process_manager + .register_query("SELECT NOW()".to_string()) + .await + .unwrap(); + + let after = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + + let running_processes = process_manager.dump().await.unwrap(); + let process = &running_processes[0]; + + assert!(process.value.start_timestamp_ms >= before); + assert!(process.value.start_timestamp_ms <= after); + } + + #[tokio::test] + async fn test_multiple_frontends() { + let kv_client = Arc::new(MemoryKvBackend::new()); + + // Create two process managers with different frontend addresses + let pm1 = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let pm2 = ProcessManager::new("127.0.0.1:8001".to_string(), kv_client.clone()); + + let id1 = pm1.register_query("SELECT 1".to_string()).await.unwrap(); + let id2 = pm2.register_query("SELECT 2".to_string()).await.unwrap(); + + // Verify both processes are registered with correct frontend IPs + let pm1_processes = pm1.dump().await.unwrap(); + assert_eq!(pm1_processes.len(), 1); + + let p1 = pm1_processes.iter().find(|p| p.key.id == id1).unwrap(); + assert_eq!(p1.key.frontend_ip, "127.0.0.1:8000"); + + let p2_processes = pm2.dump().await.unwrap(); + assert_eq!(p2_processes.len(), 1); + let p2 = p2_processes.iter().find(|p| p.key.id == id2).unwrap(); + assert_eq!(p2.key.frontend_ip, "127.0.0.1:8001"); + + // deregister all queries on instance 1 + pm1.deregister_all_queries().await.unwrap(); + + let processes: Vec<_> = pm1.dump().await.unwrap(); + assert_eq!(processes.len(), 0); + + assert_eq!(pm2.dump().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn test_list_empty_processes() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + assert!(process_manager.dump().await.unwrap().is_empty()); + } + + #[tokio::test] + async fn test_deregister_all_queries() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + // Register multiple queries + process_manager + .register_query("SELECT 1".to_string()) + .await + .unwrap(); + process_manager + .register_query("SELECT 2".to_string()) + .await + .unwrap(); + process_manager + .register_query("SELECT 3".to_string()) + .await + .unwrap(); + + // Verify they exist + assert_eq!(process_manager.dump().await.unwrap().len(), 3); + + // Deregister all + process_manager.deregister_all_queries().await.unwrap(); + + // Verify none remain + assert_eq!(process_manager.dump().await.unwrap().len(), 0); + } } diff --git a/src/query/src/error.rs b/src/query/src/error.rs index e74895aa346b..c2a2e960b0f6 100644 --- a/src/query/src/error.rs +++ b/src/query/src/error.rs @@ -323,20 +323,6 @@ pub enum Error { #[snafu(implicit)] location: Location, }, - - #[snafu(display("Failed to serialize process value when registering."))] - RegisterProcess { - source: common_meta::error::Error, - #[snafu(implicit)] - location: Location, - }, - - #[snafu(display("Failed to list all running processes."))] - ListProcesses { - source: common_meta::error::Error, - #[snafu(implicit)] - location: Location, - }, } impl ErrorExt for Error { @@ -390,8 +376,6 @@ impl ErrorExt for Error { GetFulltextOptions { source, .. } | GetSkippingIndexOptions { source, .. } => { source.status_code() } - RegisterProcess { .. } => StatusCode::Internal, - ListProcesses { .. } => StatusCode::StorageUnavailable, } } diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs index d4a8d920eea7..6e1fbfae0af8 100644 --- a/src/query/src/lib.rs +++ b/src/query/src/lib.rs @@ -42,7 +42,6 @@ pub mod sql; pub mod stats; pub(crate) mod window_sort; -mod process_list; #[cfg(test)] pub(crate) mod test_util; #[cfg(test)] diff --git a/src/query/src/process_list.rs b/src/query/src/process_list.rs deleted file mode 100644 index 2aa96258b35c..000000000000 --- a/src/query/src/process_list.rs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use common_meta::key::process_list::{ProcessKey, ProcessValue}; -use common_meta::key::{MetadataKey, MetadataValue, PROCESS_LIST_PREFIX}; -use common_meta::kv_backend::KvBackendRef; -use common_meta::rpc::store::{PutRequest, RangeRequest}; -use common_meta::sequence::{SequenceBuilder, SequenceRef}; -use common_time::util::current_time_millis; -use snafu::ResultExt; - -use crate::error; - -/// Sequence key for process list entries. -pub const PROCESS_ID_SEQ: &str = "process_id_seq"; - -pub struct Process { - key: ProcessKey, - value: ProcessValue, -} - -pub struct ProcessManager { - server_addr: String, - sequencer: SequenceRef, - kv_client: KvBackendRef, -} - -impl ProcessManager { - pub fn new(server_addr: String, kv_client: KvBackendRef) -> Self { - let sequencer = Arc::new( - SequenceBuilder::new(PROCESS_ID_SEQ, kv_client.clone()) - .initial(0) - .step(100) - .build(), - ); - Self { - server_addr, - sequencer, - kv_client, - } - } - - pub async fn register_query(&self, query: String) -> error::Result<()> { - let process_id = self.sequencer.next().await.unwrap(); - let key = ProcessKey { - frontend_ip: self.server_addr.clone(), - id: process_id, - } - .to_bytes(); - let current_time = current_time_millis(); - let value = ProcessValue { - query, - start_timestamp_ms: current_time, - } - .try_as_raw_value() - .context(error::RegisterProcessSnafu)?; - - self.kv_client - .put(PutRequest { - key, - value, - prev_kv: false, - }) - .await - .context(error::RegisterProcessSnafu)?; - Ok(()) - } - - pub async fn list_all_processes(&self) -> error::Result> { - let mut all_processes = vec![]; - let mut has_nex = true; - while has_nex { - let resp = self - .kv_client - .range(RangeRequest { - key: PROCESS_LIST_PREFIX.to_string().into_bytes(), - range_end: vec![], - limit: 0, - keys_only: false, - }) - .await - .context(error::ListProcessesSnafu)?; - - let process = resp - .kvs - .into_iter() - .map(|kv| { - Ok(Process { - key: ProcessKey::from_bytes(&kv.key)?, - value: ProcessValue::try_from_raw_value(&kv.value)?, - }) - }) - .collect::, _>>() - .context(error::ListProcessesSnafu)?; - all_processes.extend(process); - has_nex = resp.more; - } - - Ok(all_processes) - } -} From 8b570bf34bab39444cc518f1541c59938e078d44 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 9 Apr 2025 07:42:15 +0000 Subject: [PATCH 04/31] chore: rebase main --- src/common/meta/src/key.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/meta/src/key.rs b/src/common/meta/src/key.rs index 163d5ffcf6ea..406e1a5a0bdd 100644 --- a/src/common/meta/src/key.rs +++ b/src/common/meta/src/key.rs @@ -1347,7 +1347,7 @@ impl_metadata_value! { NodeAddressValue, SchemaNameValue, FlowStateValue, - PoisonValue + PoisonValue, ProcessValue } From 3ab8e21adb08ae7727c45cfdc01326f1e4b61c9f Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 9 Apr 2025 08:59:59 +0000 Subject: [PATCH 05/31] add information schema process list table --- .../src/system_schema/information_schema.rs | 1 + .../information_schema/cluster_info.rs | 10 +- .../information_schema/columns.rs | 16 +- .../system_schema/information_schema/flows.rs | 6 +- .../information_schema/key_column_usage.rs | 14 +- .../information_schema/partitions.rs | 6 +- .../information_schema/procedure_info.rs | 12 +- .../information_schema/process_list.rs | 183 ++++++++++++++++++ .../information_schema/region_peers.rs | 8 +- .../information_schema/region_statistics.rs | 22 +-- .../information_schema/schemata.rs | 10 +- .../information_schema/table_constraints.rs | 10 +- .../information_schema/tables.rs | 12 +- .../system_schema/information_schema/views.rs | 6 +- .../src/system_schema/pg_catalog/pg_class.rs | 10 +- .../system_schema/pg_catalog/pg_database.rs | 6 +- .../system_schema/pg_catalog/pg_namespace.rs | 4 +- src/catalog/src/system_schema/predicate.rs | 80 +++++--- src/common/catalog/src/consts.rs | 2 + src/common/meta/src/key/process_list.rs | 45 +++-- src/datatypes/src/value.rs | 8 + 21 files changed, 352 insertions(+), 119 deletions(-) create mode 100644 src/catalog/src/system_schema/information_schema/process_list.rs diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs index 9ae90b336094..f71d693eaaf0 100644 --- a/src/catalog/src/system_schema/information_schema.rs +++ b/src/catalog/src/system_schema/information_schema.rs @@ -19,6 +19,7 @@ mod information_memory_table; pub mod key_column_usage; mod partitions; mod procedure_info; +mod process_list; pub mod region_peers; mod region_statistics; mod runtime_metrics; diff --git a/src/catalog/src/system_schema/information_schema/cluster_info.rs b/src/catalog/src/system_schema/information_schema/cluster_info.rs index 7672791df68f..2f65610e0f84 100644 --- a/src/catalog/src/system_schema/information_schema/cluster_info.rs +++ b/src/catalog/src/system_schema/information_schema/cluster_info.rs @@ -178,11 +178,11 @@ impl InformationSchemaClusterInfoBuilder { let peer_type = node_info.status.role_name(); let row = [ - (PEER_ID, &Value::from(node_info.peer.id)), - (PEER_TYPE, &Value::from(peer_type)), - (PEER_ADDR, &Value::from(node_info.peer.addr.as_str())), - (VERSION, &Value::from(node_info.version.as_str())), - (GIT_COMMIT, &Value::from(node_info.git_commit.as_str())), + (PEER_ID, Value::from(node_info.peer.id)), + (PEER_TYPE, Value::from(peer_type)), + (PEER_ADDR, Value::from(node_info.peer.addr.as_str())), + (VERSION, Value::from(node_info.version.as_str())), + (GIT_COMMIT, Value::from(node_info.git_commit.as_str())), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/columns.rs b/src/catalog/src/system_schema/information_schema/columns.rs index 0c866f4841d5..523784477cc0 100644 --- a/src/catalog/src/system_schema/information_schema/columns.rs +++ b/src/catalog/src/system_schema/information_schema/columns.rs @@ -315,14 +315,14 @@ impl InformationSchemaColumnsBuilder { }; let row = [ - (TABLE_CATALOG, &Value::from(catalog_name)), - (TABLE_SCHEMA, &Value::from(schema_name)), - (TABLE_NAME, &Value::from(table_name)), - (COLUMN_NAME, &Value::from(column_schema.name.as_str())), - (DATA_TYPE, &Value::from(data_type.as_str())), - (SEMANTIC_TYPE, &Value::from(semantic_type)), - (ORDINAL_POSITION, &Value::from((index + 1) as i64)), - (COLUMN_KEY, &Value::from(column_key)), + (TABLE_CATALOG, Value::from(catalog_name)), + (TABLE_SCHEMA, Value::from(schema_name)), + (TABLE_NAME, Value::from(table_name)), + (COLUMN_NAME, Value::from(column_schema.name.as_str())), + (DATA_TYPE, Value::from(data_type.as_str())), + (SEMANTIC_TYPE, Value::from(semantic_type)), + (ORDINAL_POSITION, Value::from((index + 1) as i64)), + (COLUMN_KEY, Value::from(column_key)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/flows.rs b/src/catalog/src/system_schema/information_schema/flows.rs index 24b75e62e500..c2fdc5788fc0 100644 --- a/src/catalog/src/system_schema/information_schema/flows.rs +++ b/src/catalog/src/system_schema/information_schema/flows.rs @@ -273,11 +273,11 @@ impl InformationSchemaFlowsBuilder { flow_stat: &Option, ) -> Result<()> { let row = [ - (FLOW_NAME, &Value::from(flow_info.flow_name().to_string())), - (FLOW_ID, &Value::from(flow_id)), + (FLOW_NAME, Value::from(flow_info.flow_name().to_string())), + (FLOW_ID, Value::from(flow_id)), ( TABLE_CATALOG, - &Value::from(flow_info.catalog_name().to_string()), + Value::from(flow_info.catalog_name().to_string()), ), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/key_column_usage.rs b/src/catalog/src/system_schema/information_schema/key_column_usage.rs index 9f0883930373..28530e44f213 100644 --- a/src/catalog/src/system_schema/information_schema/key_column_usage.rs +++ b/src/catalog/src/system_schema/information_schema/key_column_usage.rs @@ -290,13 +290,13 @@ impl InformationSchemaKeyColumnUsageBuilder { ordinal_position: u32, ) { let row = [ - (CONSTRAINT_SCHEMA, &Value::from(constraint_schema)), - (CONSTRAINT_NAME, &Value::from(constraint_name)), - (REAL_TABLE_CATALOG, &Value::from(table_catalog)), - (TABLE_SCHEMA, &Value::from(table_schema)), - (TABLE_NAME, &Value::from(table_name)), - (COLUMN_NAME, &Value::from(column_name)), - (ORDINAL_POSITION, &Value::from(ordinal_position)), + (CONSTRAINT_SCHEMA, Value::from(constraint_schema)), + (CONSTRAINT_NAME, Value::from(constraint_name)), + (REAL_TABLE_CATALOG, Value::from(table_catalog)), + (TABLE_SCHEMA, Value::from(table_schema)), + (TABLE_NAME, Value::from(table_name)), + (COLUMN_NAME, Value::from(column_name)), + (ORDINAL_POSITION, Value::from(ordinal_position)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/partitions.rs b/src/catalog/src/system_schema/information_schema/partitions.rs index 9b9634cb44ca..0b8a9ae8b6bc 100644 --- a/src/catalog/src/system_schema/information_schema/partitions.rs +++ b/src/catalog/src/system_schema/information_schema/partitions.rs @@ -311,9 +311,9 @@ impl InformationSchemaPartitionsBuilder { partitions: &[PartitionInfo], ) { let row = [ - (TABLE_CATALOG, &Value::from(catalog_name)), - (TABLE_SCHEMA, &Value::from(schema_name)), - (TABLE_NAME, &Value::from(table_name)), + (TABLE_CATALOG, Value::from(catalog_name)), + (TABLE_SCHEMA, Value::from(schema_name)), + (TABLE_NAME, Value::from(table_name)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/procedure_info.rs b/src/catalog/src/system_schema/information_schema/procedure_info.rs index 3ae237fdf144..23723d2e7df3 100644 --- a/src/catalog/src/system_schema/information_schema/procedure_info.rs +++ b/src/catalog/src/system_schema/information_schema/procedure_info.rs @@ -187,12 +187,12 @@ impl InformationSchemaProcedureInfoBuilder { let lock_keys = lock_keys.join(","); let row = [ - (PROCEDURE_ID, &Value::from(pid.clone())), - (PROCEDURE_TYPE, &Value::from(type_name.clone())), - (START_TIME, &Value::from(start_time)), - (END_TIME, &Value::from(end_time)), - (STATUS, &Value::from(status.clone())), - (LOCK_KEYS, &Value::from(lock_keys.clone())), + (PROCEDURE_ID, Value::from(pid.clone())), + (PROCEDURE_TYPE, Value::from(type_name.clone())), + (START_TIME, Value::from(start_time)), + (END_TIME, Value::from(end_time)), + (STATUS, Value::from(status.clone())), + (LOCK_KEYS, Value::from(lock_keys.clone())), ]; if !predicates.eval(&row) { return; diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs new file mode 100644 index 000000000000..8e382d789b2c --- /dev/null +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -0,0 +1,183 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use common_catalog::consts::INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID; +use common_error::ext::BoxedError; +use common_meta::key::process_list::{Process, ProcessManager}; +use common_recordbatch::adapter::RecordBatchStreamAdapter; +use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; +use common_time::util::current_time_millis; +use common_time::{Duration, Timestamp}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter; +use datatypes::prelude::ConcreteDataType; +use datatypes::scalars::ScalarVectorBuilder; +use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; +use datatypes::value::Value; +use datatypes::vectors::{ + DurationMillisecondVectorBuilder, StringVectorBuilder, TimestampMillisecondVectorBuilder, + UInt64VectorBuilder, VectorRef, +}; +use futures::StreamExt; +use snafu::ResultExt; +use store_api::storage::{ScanRequest, TableId}; + +use super::{InformationTable, Predicates}; +use crate::error::{self, InternalSnafu}; + +pub struct InformationSchemaProcessList { + schema: SchemaRef, + process_manager: Arc, +} + +impl InformationSchemaProcessList { + #[allow(dead_code)] + pub fn new(process_manager: Arc) -> Self { + Self { + schema: Self::schema(), + process_manager, + } + } + + fn schema() -> SchemaRef { + Arc::new(Schema::new( + vec![ + ("id", ConcreteDataType::uint64_datatype(), false), + ("database", ConcreteDataType::string_datatype(), false), + ("query", ConcreteDataType::string_datatype(), false), + ( + "start_timestamp_ms", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + ( + "elapsed_time", + ConcreteDataType::duration_millisecond_datatype(), + false, + ), + ] + .into_iter() + .map(|(name, ty, nullable)| ColumnSchema::new(name, ty, nullable)) + .collect(), + )) + } +} + +impl InformationTable for InformationSchemaProcessList { + fn table_id(&self) -> TableId { + INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID + } + + fn table_name(&self) -> &'static str { + "process_list" + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn to_stream( + &self, + request: store_api::storage::ScanRequest, + ) -> error::Result { + let schema = self.schema.arrow_schema().clone(); + let process_manager = self.process_manager.clone(); + let stream = Box::pin(DfRecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { + make_process_list(process_manager, request) + .await + .map(RecordBatch::into_df_record_batch) + .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e))) + }), + )); + + Ok(Box::pin( + RecordBatchStreamAdapter::try_new(stream) + .map_err(BoxedError::new) + .context(InternalSnafu)?, + )) + } +} + +async fn make_process_list( + process_manager: Arc, + request: ScanRequest, +) -> error::Result { + let predicates = Predicates::from_scan_request(&Some(request)); + let current_time = current_time_millis(); + let mut stream = Box::pin(process_manager.list_all_processes().unwrap().into_stream()); + + let mut rows = Vec::new(); + while let Some(process) = stream.next().await.transpose().unwrap() { + let row = process_to_row(process, current_time); + if predicates.eval(&row) { + rows.push(row); + } + } + Ok(rows_to_record_batch(rows).unwrap()) +} + +fn process_to_row(process: Process, current_time: i64) -> Vec<(&'static str, Value)> { + vec![ + ("id", Value::UInt64(process.query_id())), + ( + "database", + Value::String(process.database().to_string().into()), + ), + ( + "query", + Value::String(process.query_string().to_string().into()), + ), + ( + "start_timestamp_ms", + Value::Timestamp(Timestamp::new_millisecond( + process.query_start_timestamp_ms(), + )), + ), + ( + "elapsed_time", + Value::Duration(Duration::new_millisecond( + current_time - process.query_start_timestamp_ms(), + )), + ), + ] +} + +fn rows_to_record_batch(rows: Vec>) -> error::Result { + let mut id_builder = UInt64VectorBuilder::with_capacity(rows.len()); + let mut database_builder = StringVectorBuilder::with_capacity(rows.len()); + let mut query_builder = StringVectorBuilder::with_capacity(rows.len()); + let mut start_time_builder = TimestampMillisecondVectorBuilder::with_capacity(rows.len()); + let mut elapsed_time_builder = DurationMillisecondVectorBuilder::with_capacity(rows.len()); + + for row in rows { + id_builder.push(row[0].1.as_u64()); + database_builder.push(row[1].1.as_string().as_deref()); + query_builder.push(row[2].1.as_string().as_deref()); + start_time_builder.push(row[3].1.as_timestamp().map(|t| t.value().into())); + elapsed_time_builder.push(row[4].1.as_duration().map(|d| d.value().into())); + } + + let columns: Vec = vec![ + Arc::new(id_builder.finish()), + Arc::new(database_builder.finish()), + Arc::new(query_builder.finish()), + Arc::new(start_time_builder.finish()), + Arc::new(elapsed_time_builder.finish()), + ]; + + Ok(RecordBatch::new(InformationSchemaProcessList::schema(), columns).unwrap()) +} diff --git a/src/catalog/src/system_schema/information_schema/region_peers.rs b/src/catalog/src/system_schema/information_schema/region_peers.rs index 1d46a9db0109..7122a2cef117 100644 --- a/src/catalog/src/system_schema/information_schema/region_peers.rs +++ b/src/catalog/src/system_schema/information_schema/region_peers.rs @@ -263,10 +263,10 @@ impl InformationSchemaRegionPeersBuilder { }; let row = [ - (TABLE_CATALOG, &Value::from(table_catalog)), - (TABLE_SCHEMA, &Value::from(table_schema)), - (TABLE_NAME, &Value::from(table_name)), - (REGION_ID, &Value::from(region_id)), + (TABLE_CATALOG, Value::from(table_catalog)), + (TABLE_SCHEMA, Value::from(table_schema)), + (TABLE_NAME, Value::from(table_name)), + (REGION_ID, Value::from(region_id)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/region_statistics.rs b/src/catalog/src/system_schema/information_schema/region_statistics.rs index 8db6b860bfb6..8980dc7cbe02 100644 --- a/src/catalog/src/system_schema/information_schema/region_statistics.rs +++ b/src/catalog/src/system_schema/information_schema/region_statistics.rs @@ -189,17 +189,17 @@ impl InformationSchemaRegionStatisticsBuilder { fn add_region_statistic(&mut self, predicate: &Predicates, region_stat: RegionStat) { let row = [ - (REGION_ID, &Value::from(region_stat.id.as_u64())), - (TABLE_ID, &Value::from(region_stat.id.table_id())), - (REGION_NUMBER, &Value::from(region_stat.id.region_number())), - (REGION_ROWS, &Value::from(region_stat.num_rows)), - (DISK_SIZE, &Value::from(region_stat.approximate_bytes)), - (MEMTABLE_SIZE, &Value::from(region_stat.memtable_size)), - (MANIFEST_SIZE, &Value::from(region_stat.manifest_size)), - (SST_SIZE, &Value::from(region_stat.sst_size)), - (INDEX_SIZE, &Value::from(region_stat.index_size)), - (ENGINE, &Value::from(region_stat.engine.as_str())), - (REGION_ROLE, &Value::from(region_stat.role.to_string())), + (REGION_ID, Value::from(region_stat.id.as_u64())), + (TABLE_ID, Value::from(region_stat.id.table_id())), + (REGION_NUMBER, Value::from(region_stat.id.region_number())), + (REGION_ROWS, Value::from(region_stat.num_rows)), + (DISK_SIZE, Value::from(region_stat.approximate_bytes)), + (MEMTABLE_SIZE, Value::from(region_stat.memtable_size)), + (MANIFEST_SIZE, Value::from(region_stat.manifest_size)), + (SST_SIZE, Value::from(region_stat.sst_size)), + (INDEX_SIZE, Value::from(region_stat.index_size)), + (ENGINE, Value::from(region_stat.engine.as_str())), + (REGION_ROLE, Value::from(region_stat.role.to_string())), ]; if !predicate.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/schemata.rs b/src/catalog/src/system_schema/information_schema/schemata.rs index 6738267d3fb9..c333f1297ed4 100644 --- a/src/catalog/src/system_schema/information_schema/schemata.rs +++ b/src/catalog/src/system_schema/information_schema/schemata.rs @@ -204,11 +204,11 @@ impl InformationSchemaSchemataBuilder { schema_options: &str, ) { let row = [ - (CATALOG_NAME, &Value::from(catalog_name)), - (SCHEMA_NAME, &Value::from(schema_name)), - (DEFAULT_CHARACTER_SET_NAME, &Value::from("utf8")), - (DEFAULT_COLLATION_NAME, &Value::from("utf8_bin")), - (SCHEMA_OPTS, &Value::from(schema_options)), + (CATALOG_NAME, Value::from(catalog_name)), + (SCHEMA_NAME, Value::from(schema_name)), + (DEFAULT_CHARACTER_SET_NAME, Value::from("utf8")), + (DEFAULT_COLLATION_NAME, Value::from("utf8_bin")), + (SCHEMA_OPTS, Value::from(schema_options)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/table_constraints.rs b/src/catalog/src/system_schema/information_schema/table_constraints.rs index a1f9d899f4b1..1c920bc38285 100644 --- a/src/catalog/src/system_schema/information_schema/table_constraints.rs +++ b/src/catalog/src/system_schema/information_schema/table_constraints.rs @@ -221,11 +221,11 @@ impl InformationSchemaTableConstraintsBuilder { constraint_type: &str, ) { let row = [ - (CONSTRAINT_SCHEMA, &Value::from(constraint_schema)), - (CONSTRAINT_NAME, &Value::from(constraint_name)), - (TABLE_SCHEMA, &Value::from(table_schema)), - (TABLE_NAME, &Value::from(table_name)), - (CONSTRAINT_TYPE, &Value::from(constraint_type)), + (CONSTRAINT_SCHEMA, Value::from(constraint_schema)), + (CONSTRAINT_NAME, Value::from(constraint_name)), + (TABLE_SCHEMA, Value::from(table_schema)), + (TABLE_NAME, Value::from(table_name)), + (CONSTRAINT_TYPE, Value::from(constraint_type)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/tables.rs b/src/catalog/src/system_schema/information_schema/tables.rs index 8c288b8e575d..433e1769bec3 100644 --- a/src/catalog/src/system_schema/information_schema/tables.rs +++ b/src/catalog/src/system_schema/information_schema/tables.rs @@ -324,12 +324,12 @@ impl InformationSchemaTablesBuilder { }; let row = [ - (TABLE_CATALOG, &Value::from(catalog_name)), - (TABLE_ID, &Value::from(table_id)), - (TABLE_SCHEMA, &Value::from(schema_name)), - (ENGINE, &Value::from(engine)), - (TABLE_NAME, &Value::from(table_name)), - (TABLE_TYPE, &Value::from(table_type_text)), + (TABLE_CATALOG, Value::from(catalog_name)), + (TABLE_ID, Value::from(table_id)), + (TABLE_SCHEMA, Value::from(schema_name)), + (ENGINE, Value::from(engine)), + (TABLE_NAME, Value::from(table_name)), + (TABLE_TYPE, Value::from(table_type_text)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/views.rs b/src/catalog/src/system_schema/information_schema/views.rs index d0424f528de8..6cb21c428570 100644 --- a/src/catalog/src/system_schema/information_schema/views.rs +++ b/src/catalog/src/system_schema/information_schema/views.rs @@ -228,9 +228,9 @@ impl InformationSchemaViewsBuilder { definition: &str, ) { let row = [ - (TABLE_CATALOG, &Value::from(catalog_name)), - (TABLE_SCHEMA, &Value::from(schema_name)), - (TABLE_NAME, &Value::from(table_name)), + (TABLE_CATALOG, Value::from(catalog_name)), + (TABLE_SCHEMA, Value::from(schema_name)), + (TABLE_NAME, Value::from(table_name)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/pg_catalog/pg_class.rs b/src/catalog/src/system_schema/pg_catalog/pg_class.rs index 57f8db882c6e..2562db108cee 100644 --- a/src/catalog/src/system_schema/pg_catalog/pg_class.rs +++ b/src/catalog/src/system_schema/pg_catalog/pg_class.rs @@ -245,11 +245,11 @@ impl PGClassBuilder { ) { let namespace_oid = self.namespace_oid_map.get_oid(schema); let row = [ - (OID_COLUMN_NAME, &Value::from(oid)), - (RELNAMESPACE, &Value::from(schema)), - (RELNAME, &Value::from(table)), - (RELKIND, &Value::from(kind)), - (RELOWNER, &Value::from(DUMMY_OWNER_ID)), + (OID_COLUMN_NAME, Value::from(oid)), + (RELNAMESPACE, Value::from(schema)), + (RELNAME, Value::from(table)), + (RELKIND, Value::from(kind)), + (RELOWNER, Value::from(DUMMY_OWNER_ID)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/pg_catalog/pg_database.rs b/src/catalog/src/system_schema/pg_catalog/pg_database.rs index 8a788a25daa4..b83b66f7ba9d 100644 --- a/src/catalog/src/system_schema/pg_catalog/pg_database.rs +++ b/src/catalog/src/system_schema/pg_catalog/pg_database.rs @@ -202,9 +202,9 @@ impl PGCDatabaseBuilder { fn add_database(&mut self, predicates: &Predicates, schema_name: &str) { let oid = self.namespace_oid_map.get_oid(schema_name); - let row: [(&str, &Value); 2] = [ - (OID_COLUMN_NAME, &Value::from(oid)), - (DATNAME, &Value::from(schema_name)), + let row: [(&str, Value); 2] = [ + (OID_COLUMN_NAME, Value::from(oid)), + (DATNAME, Value::from(schema_name)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs b/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs index f8fc3e09c482..121cfda60d71 100644 --- a/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs +++ b/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs @@ -209,8 +209,8 @@ impl PGNamespaceBuilder { fn add_namespace(&mut self, predicates: &Predicates, schema_name: &str) { let oid = self.namespace_oid_map.get_oid(schema_name); let row = [ - (OID_COLUMN_NAME, &Value::from(oid)), - (NSPNAME, &Value::from(schema_name)), + (OID_COLUMN_NAME, Value::from(oid)), + (NSPNAME, Value::from(schema_name)), ]; if !predicates.eval(&row) { return; diff --git a/src/catalog/src/system_schema/predicate.rs b/src/catalog/src/system_schema/predicate.rs index c94141a94778..18f0b3eacf3f 100644 --- a/src/catalog/src/system_schema/predicate.rs +++ b/src/catalog/src/system_schema/predicate.rs @@ -40,14 +40,14 @@ impl Predicate { /// - `None` when the predicate can't evaluate with the row. /// - `Some(true)` when the predicate is satisfied, /// - `Some(false)` when the predicate is not satisfied, - fn eval(&self, row: &[(&str, &Value)]) -> Option { + fn eval(&self, row: &[(&str, Value)]) -> Option { match self { Predicate::Eq(c, v) => { for (column, value) in row { if c != column { continue; } - return Some(v == *value); + return Some(v == value); } } Predicate::Like(c, pattern, case_insensitive) => { @@ -68,7 +68,7 @@ impl Predicate { if c != column { continue; } - return Some(v != *value); + return Some(v != value); } } Predicate::InList(c, values) => { @@ -76,7 +76,7 @@ impl Predicate { if c != column { continue; } - return Some(values.iter().any(|v| v == *value)); + return Some(values.iter().any(|v| v == value)); } } Predicate::And(left, right) => { @@ -271,7 +271,7 @@ impl Predicates { /// Evaluate the predicates with the row. /// returns true when all the predicates are satisfied or can't be evaluated. - pub fn eval(&self, row: &[(&str, &Value)]) -> bool { + pub fn eval(&self, row: &[(&str, Value)]) -> bool { // fast path if self.predicates.is_empty() { return true; @@ -305,9 +305,9 @@ mod tests { let b_value = Value::from("b_value"); let wrong_value = Value::from("wrong_value"); - let a_row = [(a_col.as_str(), &a_value)]; - let b_row = [("b", &wrong_value)]; - let wrong_row = [(a_col.as_str(), &wrong_value)]; + let a_row = [(a_col.as_str(), a_value.clone())]; + let b_row = [("b", wrong_value.clone())]; + let wrong_row = [(a_col.as_str(), wrong_value.clone())]; // Predicate::Eq let p = Predicate::Eq(a_col.clone(), a_value.clone()); @@ -326,30 +326,42 @@ mod tests { assert!(p.eval(&a_row).unwrap()); assert!(p.eval(&b_row).is_none()); assert!(!p.eval(&wrong_row).unwrap()); - assert!(p.eval(&[(&a_col, &b_value)]).unwrap()); + assert!(p.eval(&[(&a_col, b_value.clone())]).unwrap()); let p1 = Predicate::Eq(a_col.clone(), a_value.clone()); let p2 = Predicate::Eq(b_col.clone(), b_value.clone()); - let row = [(a_col.as_str(), &a_value), (b_col.as_str(), &b_value)]; - let wrong_row = [(a_col.as_str(), &a_value), (b_col.as_str(), &wrong_value)]; + let row = [ + (a_col.as_str(), a_value.clone()), + (b_col.as_str(), b_value.clone()), + ]; + let wrong_row = [ + (a_col.as_str(), a_value.clone()), + (b_col.as_str(), wrong_value.clone()), + ]; //Predicate::And let p = Predicate::And(Box::new(p1.clone()), Box::new(p2.clone())); assert!(p.eval(&row).unwrap()); assert!(!p.eval(&wrong_row).unwrap()); assert!(p.eval(&[]).is_none()); - assert!(p.eval(&[("c", &a_value)]).is_none()); + assert!(p.eval(&[("c", a_value.clone())]).is_none()); assert!(!p - .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &a_value)]) + .eval(&[ + (a_col.as_str(), b_value.clone()), + (b_col.as_str(), a_value.clone()) + ]) .unwrap()); assert!(!p - .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &b_value)]) + .eval(&[ + (a_col.as_str(), b_value.clone()), + (b_col.as_str(), b_value.clone()) + ]) .unwrap()); assert!(p - .eval(&[(a_col.as_ref(), &a_value), ("c", &a_value)]) + .eval(&[(a_col.as_ref(), a_value.clone()), ("c", a_value.clone())]) .is_none()); assert!(!p - .eval(&[(a_col.as_ref(), &b_value), ("c", &a_value)]) + .eval(&[(a_col.as_ref(), b_value.clone()), ("c", a_value.clone())]) .unwrap()); //Predicate::Or @@ -357,18 +369,24 @@ mod tests { assert!(p.eval(&row).unwrap()); assert!(p.eval(&wrong_row).unwrap()); assert!(p.eval(&[]).is_none()); - assert!(p.eval(&[("c", &a_value)]).is_none()); + assert!(p.eval(&[("c", a_value.clone())]).is_none()); assert!(!p - .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &a_value)]) + .eval(&[ + (a_col.as_str(), b_value.clone()), + (b_col.as_str(), a_value.clone()) + ]) .unwrap()); assert!(p - .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &b_value)]) + .eval(&[ + (a_col.as_str(), b_value.clone()), + (b_col.as_str(), b_value.clone()) + ]) .unwrap()); assert!(p - .eval(&[(a_col.as_ref(), &a_value), ("c", &a_value)]) + .eval(&[(a_col.as_ref(), a_value.clone()), ("c", a_value.clone())]) .unwrap()); assert!(p - .eval(&[(a_col.as_ref(), &b_value), ("c", &a_value)]) + .eval(&[(a_col.as_ref(), b_value.clone()), ("c", a_value.clone())]) .is_none()); } @@ -392,10 +410,10 @@ mod tests { ); let match_row = [ - ("a", &Value::from("hello AbC")), - ("b", &Value::from("b value")), + ("a", Value::from("hello AbC")), + ("b", Value::from("b value")), ]; - let unmatch_row = [("a", &Value::from("bca")), ("b", &Value::from("b value"))]; + let unmatch_row = [("a", Value::from("bca")), ("b", Value::from("b value"))]; assert!(p.eval(&match_row).unwrap()); assert!(!p.eval(&unmatch_row).unwrap()); @@ -555,16 +573,16 @@ mod tests { #[test] fn test_predicates_eval_row() { let wrong_row = [ - ("a", &Value::from("a_value")), - ("b", &Value::from("b_value")), - ("c", &Value::from("c_value")), + ("a", Value::from("a_value")), + ("b", Value::from("b_value")), + ("c", Value::from("c_value")), ]; let row = [ - ("a", &Value::from("a_value")), - ("b", &Value::from("not_b_value")), - ("c", &Value::from("c_value")), + ("a", Value::from("a_value")), + ("b", Value::from("not_b_value")), + ("c", Value::from("c_value")), ]; - let c_row = [("c", &Value::from("c_value"))]; + let c_row = [("c", Value::from("c_value"))]; // test empty predicates, always returns true let predicates = Predicates::from_scan_request(&None); diff --git a/src/common/catalog/src/consts.rs b/src/common/catalog/src/consts.rs index 62308112c65e..90bc347c1aa1 100644 --- a/src/common/catalog/src/consts.rs +++ b/src/common/catalog/src/consts.rs @@ -102,6 +102,8 @@ pub const INFORMATION_SCHEMA_FLOW_TABLE_ID: u32 = 33; pub const INFORMATION_SCHEMA_PROCEDURE_INFO_TABLE_ID: u32 = 34; /// id for information_schema.region_statistics pub const INFORMATION_SCHEMA_REGION_STATISTICS_TABLE_ID: u32 = 35; +/// id for information_schema.process_list +pub const INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID: u32 = 36; // ----- End of information_schema tables ----- diff --git a/src/common/meta/src/key/process_list.rs b/src/common/meta/src/key/process_list.rs index ec2054b093cb..0c3adea20451 100644 --- a/src/common/meta/src/key/process_list.rs +++ b/src/common/meta/src/key/process_list.rs @@ -81,6 +81,8 @@ impl MetadataKey<'_, ProcessKey> for ProcessKey { /// Detail value of process. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ProcessValue { + /// Database name. + pub database: String, /// The running query sql. pub query: String, /// Query start timestamp in milliseconds. @@ -102,6 +104,11 @@ impl Process { &self.key.frontend_ip } + /// Returns database name of running process. + pub fn database(&self) -> &str { + &self.value.database + } + /// Returns id of query. pub fn query_id(&self) -> u64 { self.key.id @@ -112,6 +119,11 @@ impl Process { &self.value.query } + /// Returns query start timestamp in milliseconds. + pub fn query_start_timestamp_ms(&self) -> i64 { + self.value.start_timestamp_ms + } + /// Calculates the elapsed time of query. Returns None of system clock jumps backwards. pub fn query_elapsed(&self) -> Option { let now = current_time_millis(); @@ -148,7 +160,7 @@ impl ProcessManager { } /// Registers a submitted query. - pub async fn register_query(&self, query: String) -> error::Result { + pub async fn register_query(&self, database: String, query: String) -> error::Result { let process_id = self.sequencer.next().await?; let key = ProcessKey { frontend_ip: self.server_addr.clone(), @@ -157,6 +169,7 @@ impl ProcessManager { .to_bytes(); let current_time = current_time_millis(); let value = ProcessValue { + database, query, start_timestamp_ms: current_time, } @@ -282,6 +295,7 @@ mod tests { let value = ProcessValue { query: "SELECT * FROM test".to_string(), start_timestamp_ms: 1690000000000, + database: "public".to_string(), }; // Test serialization roundtrip @@ -290,6 +304,7 @@ mod tests { assert_eq!(value.query, deserialized.query); assert_eq!(value.start_timestamp_ms, deserialized.start_timestamp_ms); + assert_eq!(value.database, deserialized.database); } use std::sync::Arc; @@ -302,7 +317,7 @@ mod tests { let kv_client = Arc::new(MemoryKvBackend::new()); let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); let process_id = process_manager - .register_query("SELECT * FROM table".to_string()) + .register_query("public".to_string(), "SELECT * FROM table".to_string()) .await .unwrap(); @@ -323,15 +338,15 @@ mod tests { // Register multiple queries let id1 = process_manager - .register_query("SELECT 1".to_string()) + .register_query("public".to_string(), "SELECT 1".to_string()) .await .unwrap(); let id2 = process_manager - .register_query("SELECT 2".to_string()) + .register_query("public".to_string(), "SELECT 2".to_string()) .await .unwrap(); let id3 = process_manager - .register_query("SELECT 3".to_string()) + .register_query("public".to_string(), "SELECT 3".to_string()) .await .unwrap(); @@ -367,7 +382,7 @@ mod tests { .as_millis() as i64; process_manager - .register_query("SELECT NOW()".to_string()) + .register_query("public".to_string(), "SELECT NOW()".to_string()) .await .unwrap(); @@ -391,8 +406,14 @@ mod tests { let pm1 = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); let pm2 = ProcessManager::new("127.0.0.1:8001".to_string(), kv_client.clone()); - let id1 = pm1.register_query("SELECT 1".to_string()).await.unwrap(); - let id2 = pm2.register_query("SELECT 2".to_string()).await.unwrap(); + let id1 = pm1 + .register_query("public".to_string(), "SELECT 1".to_string()) + .await + .unwrap(); + let id2 = pm2 + .register_query("public".to_string(), "SELECT 2".to_string()) + .await + .unwrap(); // Verify both processes are registered with correct frontend IPs let pm1_processes = pm1.dump().await.unwrap(); @@ -405,7 +426,7 @@ mod tests { assert_eq!(p2_processes.len(), 1); let p2 = p2_processes.iter().find(|p| p.key.id == id2).unwrap(); assert_eq!(p2.key.frontend_ip, "127.0.0.1:8001"); - + assert_eq!(p2.value.database, "public"); // deregister all queries on instance 1 pm1.deregister_all_queries().await.unwrap(); @@ -430,15 +451,15 @@ mod tests { // Register multiple queries process_manager - .register_query("SELECT 1".to_string()) + .register_query("public".to_string(), "SELECT 1".to_string()) .await .unwrap(); process_manager - .register_query("SELECT 2".to_string()) + .register_query("public".to_string(), "SELECT 2".to_string()) .await .unwrap(); process_manager - .register_query("SELECT 3".to_string()) + .register_query("public".to_string(), "SELECT 3".to_string()) .await .unwrap(); diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs index aa670d2fe840..ba630d4e64df 100644 --- a/src/datatypes/src/value.rs +++ b/src/datatypes/src/value.rs @@ -326,6 +326,14 @@ impl Value { } } + /// Cast Value to [Duration]. Return None if value is not a valid duration data type. + pub fn as_duration(&self) -> Option { + match self { + Value::Duration(d) => Some(*d), + _ => None, + } + } + /// Returns the logical type of the value. pub fn logical_type_id(&self) -> LogicalTypeId { match self { From 4c65aa8e4f9806434bc0cf10bc1a5c04fc412374 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 9 Apr 2025 09:22:00 +0000 Subject: [PATCH 06/31] integrate process list table to system catalog --- src/catalog/src/kvbackend/manager.rs | 4 ++++ src/catalog/src/memory/manager.rs | 1 + src/catalog/src/system_schema/information_schema.rs | 13 +++++++++++++ .../system_schema/information_schema/table_names.rs | 1 + src/catalog/src/table_source.rs | 1 + src/cli/src/repl.rs | 0 src/cmd/src/flownode.rs | 1 + src/cmd/src/frontend.rs | 1 + src/cmd/src/standalone.rs | 1 + tests-integration/src/cluster.rs | 1 + tests-integration/src/standalone.rs | 1 + 11 files changed, 25 insertions(+) create mode 100644 src/cli/src/repl.rs diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs index 82e1c8f87626..8f30f9f23dc8 100644 --- a/src/catalog/src/kvbackend/manager.rs +++ b/src/catalog/src/kvbackend/manager.rs @@ -25,6 +25,7 @@ use common_error::ext::BoxedError; use common_meta::cache::{LayeredCacheRegistryRef, ViewInfoCacheRef}; use common_meta::key::catalog_name::CatalogNameKey; use common_meta::key::flow::FlowMetadataManager; +use common_meta::key::process_list::ProcessManager; use common_meta::key::schema_name::SchemaNameKey; use common_meta::key::table_info::TableInfoValue; use common_meta::key::table_name::TableNameKey; @@ -84,6 +85,7 @@ impl KvBackendCatalogManager { backend: KvBackendRef, cache_registry: LayeredCacheRegistryRef, procedure_manager: Option, + process_manager: Option>, ) -> Arc { Arc::new_cyclic(|me| Self { information_extension, @@ -102,6 +104,7 @@ impl KvBackendCatalogManager { DEFAULT_CATALOG_NAME.to_string(), me.clone(), Arc::new(FlowMetadataManager::new(backend.clone())), + process_manager, )), pg_catalog_provider: Arc::new(PGCatalogProvider::new( DEFAULT_CATALOG_NAME.to_string(), @@ -486,6 +489,7 @@ impl SystemCatalog { catalog.to_string(), self.catalog_manager.clone(), Arc::new(FlowMetadataManager::new(self.backend.clone())), + None, //todo: we should pass a meaningful process manager here. )) }); information_schema_provider.table(table_name) diff --git a/src/catalog/src/memory/manager.rs b/src/catalog/src/memory/manager.rs index 9b53a20e3d9b..9e5d18141daf 100644 --- a/src/catalog/src/memory/manager.rs +++ b/src/catalog/src/memory/manager.rs @@ -356,6 +356,7 @@ impl MemoryCatalogManager { catalog, Arc::downgrade(self) as Weak, Arc::new(FlowMetadataManager::new(Arc::new(MemoryKvBackend::new()))), + None, // we don't need ProcessManager on regions server. ); let information_schema = information_schema_provider.tables().clone(); diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs index f71d693eaaf0..e4fb1c65af3b 100644 --- a/src/catalog/src/system_schema/information_schema.rs +++ b/src/catalog/src/system_schema/information_schema.rs @@ -38,11 +38,13 @@ use common_meta::cluster::NodeInfo; use common_meta::datanode::RegionStat; use common_meta::key::flow::flow_state::FlowStat; use common_meta::key::flow::FlowMetadataManager; +use common_meta::key::process_list::ProcessManager; use common_procedure::ProcedureInfo; use common_recordbatch::SendableRecordBatchStream; use datatypes::schema::SchemaRef; use lazy_static::lazy_static; use paste::paste; +use process_list::InformationSchemaProcessList; use store_api::storage::{ScanRequest, TableId}; use table::metadata::TableType; use table::TableRef; @@ -114,6 +116,7 @@ macro_rules! setup_memory_table { pub struct InformationSchemaProvider { catalog_name: String, catalog_manager: Weak, + process_manager: Option>, flow_metadata_manager: Arc, tables: HashMap, } @@ -208,6 +211,10 @@ impl SystemSchemaProviderInner for InformationSchemaProvider { self.catalog_manager.clone(), ), ) as _), + PROCESS_LIST => self + .process_manager + .as_ref() + .map(|p| Arc::new(InformationSchemaProcessList::new(p.clone())) as _), _ => None, } } @@ -218,11 +225,13 @@ impl InformationSchemaProvider { catalog_name: String, catalog_manager: Weak, flow_metadata_manager: Arc, + process_manager: Option>, ) -> Self { let mut provider = Self { catalog_name, catalog_manager, flow_metadata_manager, + process_manager, tables: HashMap::new(), }; @@ -278,6 +287,10 @@ impl InformationSchemaProvider { self.build_table(TABLE_CONSTRAINTS).unwrap(), ); tables.insert(FLOWS.to_string(), self.build_table(FLOWS).unwrap()); + tables.insert( + PROCESS_LIST.to_string(), + self.build_table(PROCESS_LIST).unwrap(), + ); // Add memory tables for name in MEMORY_TABLES.iter() { tables.insert((*name).to_string(), self.build_table(name).expect(name)); diff --git a/src/catalog/src/system_schema/information_schema/table_names.rs b/src/catalog/src/system_schema/information_schema/table_names.rs index cdff5ebb7780..95e8fe74f44c 100644 --- a/src/catalog/src/system_schema/information_schema/table_names.rs +++ b/src/catalog/src/system_schema/information_schema/table_names.rs @@ -47,3 +47,4 @@ pub const VIEWS: &str = "views"; pub const FLOWS: &str = "flows"; pub const PROCEDURE_INFO: &str = "procedure_info"; pub const REGION_STATISTICS: &str = "region_statistics"; +pub const PROCESS_LIST: &str = "process_list"; diff --git a/src/catalog/src/table_source.rs b/src/catalog/src/table_source.rs index 3cb3b5087d08..de68f4b9e326 100644 --- a/src/catalog/src/table_source.rs +++ b/src/catalog/src/table_source.rs @@ -328,6 +328,7 @@ mod tests { backend.clone(), layered_cache_registry, None, + None, ); let table_metadata_manager = TableMetadataManager::new(backend); let mut view_info = common_meta::key::test_utils::new_test_table_info(1024, vec![]); diff --git a/src/cli/src/repl.rs b/src/cli/src/repl.rs new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/cmd/src/flownode.rs b/src/cmd/src/flownode.rs index fc23d37c23e8..af38941abaee 100644 --- a/src/cmd/src/flownode.rs +++ b/src/cmd/src/flownode.rs @@ -291,6 +291,7 @@ impl StartCommand { cached_meta_backend.clone(), layered_cache_registry.clone(), None, + None, ); let table_metadata_manager = diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index 256c565eee8c..a85a8e27d4b0 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -336,6 +336,7 @@ impl StartCommand { cached_meta_backend.clone(), layered_cache_registry.clone(), None, + None, ); let executor = HandlerGroupExecutor::new(vec![ diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 4504927cc893..9d64cbb0a804 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -513,6 +513,7 @@ impl StartCommand { kv_backend.clone(), layered_cache_registry.clone(), Some(procedure_manager.clone()), + None, ); let table_metadata_manager = diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs index c1159e18d52c..28fc33339f76 100644 --- a/tests-integration/src/cluster.rs +++ b/tests-integration/src/cluster.rs @@ -393,6 +393,7 @@ impl GreptimeDbClusterBuilder { cached_meta_backend.clone(), cache_registry.clone(), None, + None, ); let handlers_executor = HandlerGroupExecutor::new(vec![ diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs index 2d6c9bcf978a..85db0b9d16c6 100644 --- a/tests-integration/src/standalone.rs +++ b/tests-integration/src/standalone.rs @@ -172,6 +172,7 @@ impl GreptimeDbStandaloneBuilder { kv_backend.clone(), cache_registry.clone(), Some(procedure_manager.clone()), + None, ); let flow_builder = FlownodeBuilder::new( From 9b38bdbc745b1d2e20ca20f8268913e1cc35859c Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 9 Apr 2025 09:26:41 +0000 Subject: [PATCH 07/31] build ProcessManager on frontend and standalone mode --- src/cmd/src/frontend.rs | 8 +++++++- src/cmd/src/standalone.rs | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index a85a8e27d4b0..c486cebc6a63 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -28,6 +28,7 @@ use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder}; use common_meta::heartbeat::handler::invalidate_table_cache::InvalidateCacheHandler; use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler; use common_meta::heartbeat::handler::HandlerGroupExecutor; +use common_meta::key::process_list::ProcessManager; use common_telemetry::info; use common_telemetry::logging::TracingOptions; use common_time::timezone::set_default_timezone; @@ -331,12 +332,17 @@ impl StartCommand { let information_extension = Arc::new(DistributedInformationExtension::new(meta_client.clone())); + + let process_manager = Arc::new(ProcessManager::new( + opts.grpc.server_addr.clone(), + cached_meta_backend.clone(), + )); let catalog_manager = KvBackendCatalogManager::new( information_extension, cached_meta_backend.clone(), layered_cache_registry.clone(), None, - None, + Some(process_manager), ); let executor = HandlerGroupExecutor::new(vec![ diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 9d64cbb0a804..7d486a66801c 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -37,6 +37,7 @@ use common_meta::ddl::{DdlContext, NoopRegionFailureDetectorControl, ProcedureEx use common_meta::ddl_manager::DdlManager; use common_meta::key::flow::flow_state::FlowStat; use common_meta::key::flow::{FlowMetadataManager, FlowMetadataManagerRef}; +use common_meta::key::process_list::ProcessManager; use common_meta::key::{TableMetadataManager, TableMetadataManagerRef}; use common_meta::kv_backend::KvBackendRef; use common_meta::node_manager::NodeManagerRef; @@ -508,12 +509,17 @@ impl StartCommand { datanode.region_server(), procedure_manager.clone(), )); + + let process_manager = Arc::new(ProcessManager::new( + opts.grpc.server_addr.clone(), + kv_backend.clone(), + )); let catalog_manager = KvBackendCatalogManager::new( information_extension.clone(), kv_backend.clone(), layered_cache_registry.clone(), Some(procedure_manager.clone()), - None, + Some(process_manager), ); let table_metadata_manager = From e1f46a30b45ef8da91211b0c75bcb18cd4fede0b Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 9 Apr 2025 12:01:06 +0000 Subject: [PATCH 08/31] feat/show-process-list: **Add Process Management Enhancements** - **`manager.rs`**: Introduced `process_manager` to `SystemCatalog` and `KvBackendCatalogManager` for improved process handling. - **`information_schema.rs`**: Updated table insertion logic to conditionally include `PROCESS_LIST`. - **`frontend.rs`, `standalone.rs`**: Enhanced `StartCommand` to clone `process_manager` for better resource management. - **`instance.rs`, `builder.rs`**: Integrated `ProcessManager` into `Instance` and `FrontendBuilder` to manage query --- src/catalog/src/kvbackend/manager.rs | 6 +++-- .../src/system_schema/information_schema.rs | 7 +++-- src/cmd/src/frontend.rs | 3 ++- src/cmd/src/standalone.rs | 3 ++- src/frontend/src/instance.rs | 27 +++++++++++++++++++ src/frontend/src/instance/builder.rs | 7 ++++- tests-integration/src/cluster.rs | 1 + tests-integration/src/standalone.rs | 1 + 8 files changed, 46 insertions(+), 9 deletions(-) diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs index 8f30f9f23dc8..4321881af0a4 100644 --- a/src/catalog/src/kvbackend/manager.rs +++ b/src/catalog/src/kvbackend/manager.rs @@ -104,13 +104,14 @@ impl KvBackendCatalogManager { DEFAULT_CATALOG_NAME.to_string(), me.clone(), Arc::new(FlowMetadataManager::new(backend.clone())), - process_manager, + process_manager.clone(), )), pg_catalog_provider: Arc::new(PGCatalogProvider::new( DEFAULT_CATALOG_NAME.to_string(), me.clone(), )), backend, + process_manager, }, cache_registry, procedure_manager, @@ -422,6 +423,7 @@ struct SystemCatalog { information_schema_provider: Arc, pg_catalog_provider: Arc, backend: KvBackendRef, + process_manager: Option>, } impl SystemCatalog { @@ -489,7 +491,7 @@ impl SystemCatalog { catalog.to_string(), self.catalog_manager.clone(), Arc::new(FlowMetadataManager::new(self.backend.clone())), - None, //todo: we should pass a meaningful process manager here. + self.process_manager.clone(), )) }); information_schema_provider.table(table_name) diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs index e4fb1c65af3b..cc57f68f1dd4 100644 --- a/src/catalog/src/system_schema/information_schema.rs +++ b/src/catalog/src/system_schema/information_schema.rs @@ -287,10 +287,9 @@ impl InformationSchemaProvider { self.build_table(TABLE_CONSTRAINTS).unwrap(), ); tables.insert(FLOWS.to_string(), self.build_table(FLOWS).unwrap()); - tables.insert( - PROCESS_LIST.to_string(), - self.build_table(PROCESS_LIST).unwrap(), - ); + if let Some(process_list) = self.build_table(PROCESS_LIST) { + tables.insert(PROCESS_LIST.to_string(), process_list); + } // Add memory tables for name in MEMORY_TABLES.iter() { tables.insert((*name).to_string(), self.build_table(name).expect(name)); diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index c486cebc6a63..b6e5e335909c 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -342,7 +342,7 @@ impl StartCommand { cached_meta_backend.clone(), layered_cache_registry.clone(), None, - Some(process_manager), + Some(process_manager.clone()), ); let executor = HandlerGroupExecutor::new(vec![ @@ -376,6 +376,7 @@ impl StartCommand { Arc::new(client), meta_client, StatementStatistics::new(opts.logging.slow_query.clone()), + Some(process_manager), ) .with_plugin(plugins.clone()) .with_local_cache_invalidator(layered_cache_registry) diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 7d486a66801c..30de4dee79cf 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -519,7 +519,7 @@ impl StartCommand { kv_backend.clone(), layered_cache_registry.clone(), Some(procedure_manager.clone()), - Some(process_manager), + Some(process_manager.clone()), ); let table_metadata_manager = @@ -600,6 +600,7 @@ impl StartCommand { node_manager.clone(), ddl_task_executor.clone(), StatementStatistics::new(opts.logging.slow_query.clone()), + Some(process_manager), ) .with_plugin(plugins.clone()) .try_build() diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index 9c94ec326afe..6db06f735d43 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -35,6 +35,7 @@ use client::OutputData; use common_base::Plugins; use common_config::KvBackendConfig; use common_error::ext::{BoxedError, ErrorExt}; +use common_meta::key::process_list::ProcessManager; use common_meta::key::TableMetadataManagerRef; use common_meta::kv_backend::KvBackendRef; use common_meta::state_store::KvStateStore; @@ -96,6 +97,7 @@ pub struct Instance { table_metadata_manager: TableMetadataManagerRef, stats: StatementStatistics, limiter: Option, + process_manager: Option>, } impl Instance { @@ -170,6 +172,27 @@ impl Instance { .stats .start_slow_query_timer(QueryStatement::Sql(stmt.clone())); + let query_finish_notifier = if let Some(process_manager) = &self.process_manager { + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + let process_manager = process_manager.clone(); + let database = query_ctx.current_schema(); + let query = stmt.to_string(); + common_runtime::spawn_global(async move { + match process_manager.register_query(database, query).await { + Ok(id) => { + let _ = rx.await; + process_manager.deregister_query(id).await.unwrap(); + } + Err(e) => { + error!(e; "Failed to register query id"); + } + } + }); + Some(tx) + } else { + None + }; + let output = match stmt { Statement::Query(_) | Statement::Explain(_) | Statement::Delete(_) => { // TODO: remove this when format is supported in datafusion @@ -212,6 +235,10 @@ impl Instance { self.statement_executor.execute_sql(stmt, query_ctx).await } }; + + if let Some(query_finish_notifier) = query_finish_notifier { + let _ = query_finish_notifier.send(()); + } output.context(TableOperationSnafu) } } diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index 8503999b2c06..e175c67c8207 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -21,6 +21,7 @@ use common_meta::cache::{LayeredCacheRegistryRef, TableRouteCacheRef}; use common_meta::cache_invalidator::{CacheInvalidatorRef, DummyCacheInvalidator}; use common_meta::ddl::ProcedureExecutorRef; use common_meta::key::flow::FlowMetadataManager; +use common_meta::key::process_list::ProcessManager; use common_meta::key::TableMetadataManager; use common_meta::kv_backend::KvBackendRef; use common_meta::node_manager::NodeManagerRef; @@ -55,6 +56,7 @@ pub struct FrontendBuilder { plugins: Option, procedure_executor: ProcedureExecutorRef, stats: StatementStatistics, + process_manager: Option>, } impl FrontendBuilder { @@ -66,6 +68,7 @@ impl FrontendBuilder { node_manager: NodeManagerRef, procedure_executor: ProcedureExecutorRef, stats: StatementStatistics, + process_manager: Option>, ) -> Self { Self { options, @@ -77,6 +80,7 @@ impl FrontendBuilder { plugins: None, procedure_executor, stats, + process_manager, } } @@ -98,7 +102,7 @@ impl FrontendBuilder { let kv_backend = self.kv_backend; let node_manager = self.node_manager; let plugins = self.plugins.unwrap_or_default(); - + let process_manager = self.process_manager; let table_route_cache: TableRouteCacheRef = self.layered_cache_registry .get() @@ -207,6 +211,7 @@ impl FrontendBuilder { table_metadata_manager: Arc::new(TableMetadataManager::new(kv_backend)), stats: self.stats, limiter, + process_manager, }) } } diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs index 28fc33339f76..1ec7dd8931ec 100644 --- a/tests-integration/src/cluster.rs +++ b/tests-integration/src/cluster.rs @@ -417,6 +417,7 @@ impl GreptimeDbClusterBuilder { datanode_clients, meta_client, StatementStatistics::default(), + None, ) .with_local_cache_invalidator(cache_registry) .try_build() diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs index 85db0b9d16c6..6dcb5f3dd70c 100644 --- a/tests-integration/src/standalone.rs +++ b/tests-integration/src/standalone.rs @@ -241,6 +241,7 @@ impl GreptimeDbStandaloneBuilder { node_manager.clone(), ddl_task_executor.clone(), StatementStatistics::default(), + None, ) .with_plugin(plugins) .try_build() From d20ed955b4e31262ca5029b50ecd229e4ccc72a7 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 9 Apr 2025 13:01:14 +0000 Subject: [PATCH 09/31] feat/show-process-list: ### Add Process Listing and Error Handling Enhancements - **Error Handling**: Introduced a new error variant `ListProcess` in `error.rs` to handle failures when listing running processes. - **Process List Implementation**: Enhanced `InformationSchemaProcessList` in `process_list.rs` to track running queries, including defining column names and implementing the `make_process_list` function to build the process list. - **Frontend Builder**: Added a `#[allow(clippy::too_many_arguments)]` attribute in `builder.rs` to suppress Clippy warnings for the `FrontendBuilder::new` function. These changes improve error handling and process tracking capabilities within the system. --- src/catalog/src/error.rs | 13 ++- .../information_schema/process_list.rs | 82 +++++++++++-------- src/frontend/src/instance/builder.rs | 1 + 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs index c4fdf1cdc1ec..a92bd5d849c9 100644 --- a/src/catalog/src/error.rs +++ b/src/catalog/src/error.rs @@ -277,6 +277,13 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to list running processes"))] + ListProcess { + source: common_meta::error::Error, + #[snafu(implicit)] + location: Location, + }, } impl Error { @@ -342,9 +349,9 @@ impl ErrorExt for Error { Error::Datafusion { error, .. } => datafusion_status_code::(error, None), Error::ProjectViewColumns { .. } => StatusCode::EngineExecuteQuery, Error::TableMetadataManager { source, .. } => source.status_code(), - Error::GetViewCache { source, .. } | Error::GetTableCache { source, .. } => { - source.status_code() - } + Error::GetViewCache { source, .. } + | Error::GetTableCache { source, .. } + | Error::ListProcess { source, .. } => source.status_code(), } } diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 8e382d789b2c..d511a52d351d 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -37,13 +37,21 @@ use store_api::storage::{ScanRequest, TableId}; use super::{InformationTable, Predicates}; use crate::error::{self, InternalSnafu}; +/// Column names of `information_schema.process_list` +const ID: &str = "id"; +const DATABASE: &str = "database"; +const QUERY: &str = "query"; +const START_TIMESTAMP: &str = "start_timestamp"; +const ELAPSED_TIME: &str = "elapsed_time"; + +/// `information_schema.process_list` table implementation that tracks running +/// queries in current cluster. pub struct InformationSchemaProcessList { schema: SchemaRef, process_manager: Arc, } impl InformationSchemaProcessList { - #[allow(dead_code)] pub fn new(process_manager: Arc) -> Self { Self { schema: Self::schema(), @@ -52,26 +60,21 @@ impl InformationSchemaProcessList { } fn schema() -> SchemaRef { - Arc::new(Schema::new( - vec![ - ("id", ConcreteDataType::uint64_datatype(), false), - ("database", ConcreteDataType::string_datatype(), false), - ("query", ConcreteDataType::string_datatype(), false), - ( - "start_timestamp_ms", - ConcreteDataType::timestamp_millisecond_datatype(), - false, - ), - ( - "elapsed_time", - ConcreteDataType::duration_millisecond_datatype(), - false, - ), - ] - .into_iter() - .map(|(name, ty, nullable)| ColumnSchema::new(name, ty, nullable)) - .collect(), - )) + Arc::new(Schema::new(vec![ + ColumnSchema::new(ID, ConcreteDataType::uint64_datatype(), false), + ColumnSchema::new(DATABASE, ConcreteDataType::string_datatype(), false), + ColumnSchema::new(QUERY, ConcreteDataType::string_datatype(), false), + ColumnSchema::new( + START_TIMESTAMP, + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + ColumnSchema::new( + ELAPSED_TIME, + ConcreteDataType::duration_millisecond_datatype(), + false, + ), + ])) } } @@ -88,10 +91,7 @@ impl InformationTable for InformationSchemaProcessList { self.schema.clone() } - fn to_stream( - &self, - request: store_api::storage::ScanRequest, - ) -> error::Result { + fn to_stream(&self, request: ScanRequest) -> error::Result { let schema = self.schema.arrow_schema().clone(); let process_manager = self.process_manager.clone(); let stream = Box::pin(DfRecordBatchStreamAdapter::new( @@ -112,43 +112,55 @@ impl InformationTable for InformationSchemaProcessList { } } +/// Build running process list. async fn make_process_list( process_manager: Arc, request: ScanRequest, ) -> error::Result { let predicates = Predicates::from_scan_request(&Some(request)); let current_time = current_time_millis(); - let mut stream = Box::pin(process_manager.list_all_processes().unwrap().into_stream()); + let mut stream = Box::pin( + process_manager + .list_all_processes() + .context(error::ListProcessSnafu)? + .into_stream(), + ); let mut rows = Vec::new(); - while let Some(process) = stream.next().await.transpose().unwrap() { + while let Some(process) = stream + .next() + .await + .transpose() + .context(error::ListProcessSnafu)? + { let row = process_to_row(process, current_time); if predicates.eval(&row) { rows.push(row); } } - Ok(rows_to_record_batch(rows).unwrap()) + rows_to_record_batch(rows) } +/// Convert [Process] structs to rows. fn process_to_row(process: Process, current_time: i64) -> Vec<(&'static str, Value)> { vec![ - ("id", Value::UInt64(process.query_id())), + (ID, Value::UInt64(process.query_id())), ( - "database", + DATABASE, Value::String(process.database().to_string().into()), ), ( - "query", + QUERY, Value::String(process.query_string().to_string().into()), ), ( - "start_timestamp_ms", + START_TIMESTAMP, Value::Timestamp(Timestamp::new_millisecond( process.query_start_timestamp_ms(), )), ), ( - "elapsed_time", + ELAPSED_TIME, Value::Duration(Duration::new_millisecond( current_time - process.query_start_timestamp_ms(), )), @@ -156,6 +168,7 @@ fn process_to_row(process: Process, current_time: i64) -> Vec<(&'static str, Val ] } +/// Convert rows to [RecordBatch]. fn rows_to_record_batch(rows: Vec>) -> error::Result { let mut id_builder = UInt64VectorBuilder::with_capacity(rows.len()); let mut database_builder = StringVectorBuilder::with_capacity(rows.len()); @@ -179,5 +192,6 @@ fn rows_to_record_batch(rows: Vec>) -> error::Result< Arc::new(elapsed_time_builder.finish()), ]; - Ok(RecordBatch::new(InformationSchemaProcessList::schema(), columns).unwrap()) + RecordBatch::new(InformationSchemaProcessList::schema(), columns) + .context(error::CreateRecordBatchSnafu) } diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index e175c67c8207..bf1ae3920a19 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -60,6 +60,7 @@ pub struct FrontendBuilder { } impl FrontendBuilder { + #[allow(clippy::too_many_arguments)] pub fn new( options: FrontendOptions, kv_backend: KvBackendRef, From 5c5211b2eafd965018e048e1cef4329aa94adf4e Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 9 Apr 2025 13:04:33 +0000 Subject: [PATCH 10/31] feat/show-process-list: Refactor imports in `process_list.rs` - Updated import paths for `Predicates` and `InformationTable` in `process_list.rs` to align with the new module structure. --- .../src/system_schema/information_schema/process_list.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index d511a52d351d..326b75dbbf8c 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -34,8 +34,9 @@ use futures::StreamExt; use snafu::ResultExt; use store_api::storage::{ScanRequest, TableId}; -use super::{InformationTable, Predicates}; use crate::error::{self, InternalSnafu}; +use crate::information_schema::Predicates; +use crate::system_schema::information_schema::InformationTable; /// Column names of `information_schema.process_list` const ID: &str = "id"; From 9f957b4aeeb997d7b6335b743fc4d5957bdf6a51 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Thu, 10 Apr 2025 02:41:01 +0000 Subject: [PATCH 11/31] feat/show-process-list: Refactor process list generation in `process_list.rs` - Simplified the process list generation by removing intermediate row storage and directly building vectors. - Updated `process_to_row` function to use a mutable vector for current row data, improving memory efficiency. - Removed `rows_to_record_batch` function, integrating its logic directly into the main loop for streamlined processing. --- .../information_schema/process_list.rs | 94 +++++++++---------- 1 file changed, 45 insertions(+), 49 deletions(-) diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 326b75dbbf8c..8acedce7ea06 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -127,63 +127,28 @@ async fn make_process_list( .into_stream(), ); - let mut rows = Vec::new(); + let mut id_builder = UInt64VectorBuilder::with_capacity(8); + let mut database_builder = StringVectorBuilder::with_capacity(8); + let mut query_builder = StringVectorBuilder::with_capacity(8); + let mut start_time_builder = TimestampMillisecondVectorBuilder::with_capacity(8); + let mut elapsed_time_builder = DurationMillisecondVectorBuilder::with_capacity(8); + + let mut current_row = Vec::with_capacity(5); while let Some(process) = stream .next() .await .transpose() .context(error::ListProcessSnafu)? { - let row = process_to_row(process, current_time); - if predicates.eval(&row) { - rows.push(row); + process_to_row(process, current_time, &mut current_row); + if predicates.eval(¤t_row) { + id_builder.push(current_row[0].1.as_u64()); + database_builder.push(current_row[1].1.as_string().as_deref()); + query_builder.push(current_row[2].1.as_string().as_deref()); + start_time_builder.push(current_row[3].1.as_timestamp().map(|t| t.value().into())); + elapsed_time_builder.push(current_row[4].1.as_duration().map(|d| d.value().into())); } } - rows_to_record_batch(rows) -} - -/// Convert [Process] structs to rows. -fn process_to_row(process: Process, current_time: i64) -> Vec<(&'static str, Value)> { - vec![ - (ID, Value::UInt64(process.query_id())), - ( - DATABASE, - Value::String(process.database().to_string().into()), - ), - ( - QUERY, - Value::String(process.query_string().to_string().into()), - ), - ( - START_TIMESTAMP, - Value::Timestamp(Timestamp::new_millisecond( - process.query_start_timestamp_ms(), - )), - ), - ( - ELAPSED_TIME, - Value::Duration(Duration::new_millisecond( - current_time - process.query_start_timestamp_ms(), - )), - ), - ] -} - -/// Convert rows to [RecordBatch]. -fn rows_to_record_batch(rows: Vec>) -> error::Result { - let mut id_builder = UInt64VectorBuilder::with_capacity(rows.len()); - let mut database_builder = StringVectorBuilder::with_capacity(rows.len()); - let mut query_builder = StringVectorBuilder::with_capacity(rows.len()); - let mut start_time_builder = TimestampMillisecondVectorBuilder::with_capacity(rows.len()); - let mut elapsed_time_builder = DurationMillisecondVectorBuilder::with_capacity(rows.len()); - - for row in rows { - id_builder.push(row[0].1.as_u64()); - database_builder.push(row[1].1.as_string().as_deref()); - query_builder.push(row[2].1.as_string().as_deref()); - start_time_builder.push(row[3].1.as_timestamp().map(|t| t.value().into())); - elapsed_time_builder.push(row[4].1.as_duration().map(|d| d.value().into())); - } let columns: Vec = vec![ Arc::new(id_builder.finish()), @@ -196,3 +161,34 @@ fn rows_to_record_batch(rows: Vec>) -> error::Result< RecordBatch::new(InformationSchemaProcessList::schema(), columns) .context(error::CreateRecordBatchSnafu) } + +// Convert [Process] structs to rows. +fn process_to_row( + process: Process, + current_time_ms: i64, + current_row: &mut Vec<(&'static str, Value)>, +) { + current_row.clear(); + current_row.push((ID, Value::UInt64(process.query_id()))); + current_row.push(( + DATABASE, + Value::String(process.database().to_string().into()), + )); + current_row.push(( + QUERY, + Value::String(process.query_string().to_string().into()), + )); + + current_row.push(( + START_TIMESTAMP, + Value::Timestamp(Timestamp::new_millisecond( + process.query_start_timestamp_ms(), + )), + )); + current_row.push(( + ELAPSED_TIME, + Value::Duration(Duration::new_millisecond( + current_time_ms - process.query_start_timestamp_ms(), + )), + )); +} From 2c6254508a64e1a1d0f23b315c738ee75eab66b4 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Fri, 11 Apr 2025 07:03:09 +0000 Subject: [PATCH 12/31] wip: move ProcessManager to catalog crate --- src/catalog/src/kvbackend/manager.rs | 2 +- src/catalog/src/lib.rs | 1 + src/catalog/src/process_manager.rs | 275 ++++++++++++++++++ .../src/system_schema/information_schema.rs | 2 +- .../information_schema/process_list.rs | 3 +- src/cmd/src/frontend.rs | 2 +- src/cmd/src/standalone.rs | 2 +- src/common/meta/src/key/process_list.rs | 272 +---------------- src/frontend/src/instance.rs | 2 +- src/frontend/src/instance/builder.rs | 2 +- 10 files changed, 287 insertions(+), 276 deletions(-) create mode 100644 src/catalog/src/process_manager.rs diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs index 4321881af0a4..a8ac373a4a96 100644 --- a/src/catalog/src/kvbackend/manager.rs +++ b/src/catalog/src/kvbackend/manager.rs @@ -25,7 +25,6 @@ use common_error::ext::BoxedError; use common_meta::cache::{LayeredCacheRegistryRef, ViewInfoCacheRef}; use common_meta::key::catalog_name::CatalogNameKey; use common_meta::key::flow::FlowMetadataManager; -use common_meta::key::process_list::ProcessManager; use common_meta::key::schema_name::SchemaNameKey; use common_meta::key::table_info::TableInfoValue; use common_meta::key::table_name::TableNameKey; @@ -52,6 +51,7 @@ use crate::error::{ }; use crate::information_schema::{InformationExtensionRef, InformationSchemaProvider}; use crate::kvbackend::TableCacheRef; +use crate::process_manager::ProcessManager; use crate::system_schema::pg_catalog::PGCatalogProvider; use crate::system_schema::SystemSchemaProvider; use crate::CatalogManager; diff --git a/src/catalog/src/lib.rs b/src/catalog/src/lib.rs index 34884f1355ba..0e92601eb65c 100644 --- a/src/catalog/src/lib.rs +++ b/src/catalog/src/lib.rs @@ -40,6 +40,7 @@ pub mod information_schema { pub use crate::system_schema::information_schema::*; } +pub mod process_manager; pub mod table_source; #[async_trait::async_trait] diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs new file mode 100644 index 000000000000..8bd6b8f9569a --- /dev/null +++ b/src/catalog/src/process_manager.rs @@ -0,0 +1,275 @@ +use std::sync::Arc; + +use common_meta::error; +use common_meta::key::process_list::{Process, ProcessKey, ProcessValue, PROCESS_ID_SEQ}; +use common_meta::key::{MetadataKey, MetadataValue, PROCESS_LIST_PREFIX}; +use common_meta::kv_backend::KvBackendRef; +use common_meta::range_stream::PaginationStream; +use common_meta::rpc::store::{DeleteRangeRequest, PutRequest, RangeRequest}; +use common_meta::sequence::{SequenceBuilder, SequenceRef}; +use common_telemetry::{debug, info, warn}; +use common_time::util::current_time_millis; + +pub struct ProcessManager { + server_addr: String, + sequencer: SequenceRef, + kv_client: KvBackendRef, +} + +impl ProcessManager { + /// Create a [ProcessManager] instance with server address and kv client. + pub fn new(server_addr: String, kv_client: KvBackendRef) -> Self { + let sequencer = Arc::new( + SequenceBuilder::new(PROCESS_ID_SEQ, kv_client.clone()) + .initial(0) + .step(100) + .build(), + ); + Self { + server_addr, + sequencer, + kv_client, + } + } + + /// Registers a submitted query. + pub async fn register_query(&self, database: String, query: String) -> error::Result { + let process_id = self.sequencer.next().await?; + let key = ProcessKey { + frontend_ip: self.server_addr.clone(), + id: process_id, + } + .to_bytes(); + let current_time = current_time_millis(); + let value = ProcessValue { + database, + query, + start_timestamp_ms: current_time, + } + .try_as_raw_value()?; + + self.kv_client + .put(PutRequest { + key, + value, + prev_kv: false, + }) + .await?; + Ok(process_id) + } + + /// De-register a query from process list. + pub async fn deregister_query(&self, id: u64) -> error::Result<()> { + let key = ProcessKey { + frontend_ip: self.server_addr.clone(), + id, + } + .to_bytes(); + let prev_kv = self.kv_client.delete(&key, true).await?; + + if let Some(_kv) = prev_kv { + debug!("Successfully deregistered process {}", id); + } else { + warn!("Cannot find process to deregister process: {}", id); + } + Ok(()) + } + + /// De-register all queries running on current frontend. + pub async fn deregister_all_queries(&self) -> error::Result<()> { + let prefix = format!("{}/{}-", PROCESS_LIST_PREFIX, self.server_addr); + let delete_range_request = DeleteRangeRequest::new().with_prefix(prefix.as_bytes()); + self.kv_client.delete_range(delete_range_request).await?; + info!("All queries on {} has been deregistered", self.server_addr); + Ok(()) + } + + /// List all running processes in cluster. + pub fn list_all_processes(&self) -> error::Result> { + let prefix = format!("{}/{}-", PROCESS_LIST_PREFIX, self.server_addr); + let req = RangeRequest::new().with_prefix(prefix.as_bytes()); + let stream = PaginationStream::new(self.kv_client.clone(), req, 100, |kv| { + let key = ProcessKey::from_bytes(&kv.key)?; + let value = ProcessValue::try_from_raw_value(&kv.value)?; + Ok(Process { key, value }) + }); + Ok(stream) + } + + #[cfg(test)] + async fn dump(&self) -> error::Result> { + use futures_util::TryStreamExt; + self.list_all_processes()?.into_stream().try_collect().await + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::time::{SystemTime, UNIX_EPOCH}; + + use common_meta::kv_backend::memory::MemoryKvBackend; + + use crate::process_manager::ProcessManager; + + #[tokio::test] + async fn test_register_query() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let process_id = process_manager + .register_query("public".to_string(), "SELECT * FROM table".to_string()) + .await + .unwrap(); + + let running_processes = process_manager.dump().await.unwrap(); + assert_eq!(running_processes.len(), 1); + assert_eq!(running_processes[0].key.frontend_ip, "127.0.0.1:8000"); + assert_eq!(running_processes[0].key.id, process_id); + assert_eq!(running_processes[0].value.query, "SELECT * FROM table"); + + process_manager.deregister_query(process_id).await.unwrap(); + assert_eq!(process_manager.dump().await.unwrap().len(), 0); + } + + #[tokio::test] + async fn test_register_multiple_queries() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + // Register multiple queries + let id1 = process_manager + .register_query("public".to_string(), "SELECT 1".to_string()) + .await + .unwrap(); + let id2 = process_manager + .register_query("public".to_string(), "SELECT 2".to_string()) + .await + .unwrap(); + let id3 = process_manager + .register_query("public".to_string(), "SELECT 3".to_string()) + .await + .unwrap(); + + // Verify all are registered + assert_eq!(process_manager.dump().await.unwrap().len(), 3); + + // Deregister middle one + process_manager.deregister_query(id2).await.unwrap(); + let processes = process_manager.dump().await.unwrap(); + assert_eq!(processes.len(), 2); + assert!(processes.iter().any(|p| p.key.id == id1)); + assert!(processes.iter().any(|p| p.key.id == id3)); + } + + #[tokio::test] + async fn test_deregister_nonexistent_query() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + // Try to deregister non-existent ID + let result = process_manager.deregister_query(999).await; + assert!(result.is_ok()); // Should succeed with warning + } + + #[tokio::test] + async fn test_process_timestamps() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + let before = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + + process_manager + .register_query("public".to_string(), "SELECT NOW()".to_string()) + .await + .unwrap(); + + let after = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + + let running_processes = process_manager.dump().await.unwrap(); + let process = &running_processes[0]; + + assert!(process.value.start_timestamp_ms >= before); + assert!(process.value.start_timestamp_ms <= after); + } + + #[tokio::test] + async fn test_multiple_frontends() { + let kv_client = Arc::new(MemoryKvBackend::new()); + + // Create two process managers with different frontend addresses + let pm1 = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let pm2 = ProcessManager::new("127.0.0.1:8001".to_string(), kv_client.clone()); + + let id1 = pm1 + .register_query("public".to_string(), "SELECT 1".to_string()) + .await + .unwrap(); + let id2 = pm2 + .register_query("public".to_string(), "SELECT 2".to_string()) + .await + .unwrap(); + + // Verify both processes are registered with correct frontend IPs + let pm1_processes = pm1.dump().await.unwrap(); + assert_eq!(pm1_processes.len(), 1); + + let p1 = pm1_processes.iter().find(|p| p.key.id == id1).unwrap(); + assert_eq!(p1.key.frontend_ip, "127.0.0.1:8000"); + + let p2_processes = pm2.dump().await.unwrap(); + assert_eq!(p2_processes.len(), 1); + let p2 = p2_processes.iter().find(|p| p.key.id == id2).unwrap(); + assert_eq!(p2.key.frontend_ip, "127.0.0.1:8001"); + assert_eq!(p2.value.database, "public"); + // deregister all queries on instance 1 + pm1.deregister_all_queries().await.unwrap(); + + let processes: Vec<_> = pm1.dump().await.unwrap(); + assert_eq!(processes.len(), 0); + + assert_eq!(pm2.dump().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn test_list_empty_processes() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + assert!(process_manager.dump().await.unwrap().is_empty()); + } + + #[tokio::test] + async fn test_deregister_all_queries() { + let kv_client = Arc::new(MemoryKvBackend::new()); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + + // Register multiple queries + process_manager + .register_query("public".to_string(), "SELECT 1".to_string()) + .await + .unwrap(); + process_manager + .register_query("public".to_string(), "SELECT 2".to_string()) + .await + .unwrap(); + process_manager + .register_query("public".to_string(), "SELECT 3".to_string()) + .await + .unwrap(); + + // Verify they exist + assert_eq!(process_manager.dump().await.unwrap().len(), 3); + + // Deregister all + process_manager.deregister_all_queries().await.unwrap(); + + // Verify none remain + assert_eq!(process_manager.dump().await.unwrap().len(), 0); + } +} diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs index cc57f68f1dd4..8bad1a68ea4b 100644 --- a/src/catalog/src/system_schema/information_schema.rs +++ b/src/catalog/src/system_schema/information_schema.rs @@ -38,7 +38,6 @@ use common_meta::cluster::NodeInfo; use common_meta::datanode::RegionStat; use common_meta::key::flow::flow_state::FlowStat; use common_meta::key::flow::FlowMetadataManager; -use common_meta::key::process_list::ProcessManager; use common_procedure::ProcedureInfo; use common_recordbatch::SendableRecordBatchStream; use datatypes::schema::SchemaRef; @@ -53,6 +52,7 @@ use views::InformationSchemaViews; use self::columns::InformationSchemaColumns; use crate::error::{Error, Result}; +use crate::process_manager::ProcessManager; use crate::system_schema::information_schema::cluster_info::InformationSchemaClusterInfo; use crate::system_schema::information_schema::flows::InformationSchemaFlows; use crate::system_schema::information_schema::information_memory_table::get_schema_columns; diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 8acedce7ea06..dd321f33d677 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use common_catalog::consts::INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID; use common_error::ext::BoxedError; -use common_meta::key::process_list::{Process, ProcessManager}; +use common_meta::key::process_list::Process; use common_recordbatch::adapter::RecordBatchStreamAdapter; use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; use common_time::util::current_time_millis; @@ -36,6 +36,7 @@ use store_api::storage::{ScanRequest, TableId}; use crate::error::{self, InternalSnafu}; use crate::information_schema::Predicates; +use crate::process_manager::ProcessManager; use crate::system_schema::information_schema::InformationTable; /// Column names of `information_schema.process_list` diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index b6e5e335909c..38b26c5da882 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -19,6 +19,7 @@ use async_trait::async_trait; use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; use catalog::information_extension::DistributedInformationExtension; use catalog::kvbackend::{CachedKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend}; +use catalog::process_manager::ProcessManager; use clap::Parser; use client::client_manager::NodeClients; use common_base::Plugins; @@ -28,7 +29,6 @@ use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder}; use common_meta::heartbeat::handler::invalidate_table_cache::InvalidateCacheHandler; use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler; use common_meta::heartbeat::handler::HandlerGroupExecutor; -use common_meta::key::process_list::ProcessManager; use common_telemetry::info; use common_telemetry::logging::TracingOptions; use common_time::timezone::set_default_timezone; diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 30de4dee79cf..969ab1811c33 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -20,6 +20,7 @@ use async_trait::async_trait; use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; use catalog::information_schema::InformationExtension; use catalog::kvbackend::KvBackendCatalogManager; +use catalog::process_manager::ProcessManager; use clap::Parser; use client::api::v1::meta::RegionRole; use common_base::readable_size::ReadableSize; @@ -37,7 +38,6 @@ use common_meta::ddl::{DdlContext, NoopRegionFailureDetectorControl, ProcedureEx use common_meta::ddl_manager::DdlManager; use common_meta::key::flow::flow_state::FlowStat; use common_meta::key::flow::{FlowMetadataManager, FlowMetadataManagerRef}; -use common_meta::key::process_list::ProcessManager; use common_meta::key::{TableMetadataManager, TableMetadataManagerRef}; use common_meta::kv_backend::KvBackendRef; use common_meta::node_manager::NodeManagerRef; diff --git a/src/common/meta/src/key/process_list.rs b/src/common/meta/src/key/process_list.rs index 0c3adea20451..6ef0368502f4 100644 --- a/src/common/meta/src/key/process_list.rs +++ b/src/common/meta/src/key/process_list.rs @@ -14,20 +14,14 @@ use std::fmt::{Display, Formatter}; use std::str::FromStr; -use std::sync::Arc; use std::time::Duration; -use common_telemetry::{debug, info, warn}; use common_time::util::current_time_millis; use serde::{Deserialize, Serialize}; use snafu::OptionExt; use crate::error; -use crate::key::{MetadataKey, MetadataValue, PROCESS_LIST_PATTERN, PROCESS_LIST_PREFIX}; -use crate::kv_backend::KvBackendRef; -use crate::range_stream::PaginationStream; -use crate::rpc::store::{DeleteRangeRequest, PutRequest, RangeRequest}; -use crate::sequence::{SequenceBuilder, SequenceRef}; +use crate::key::{MetadataKey, PROCESS_LIST_PATTERN, PROCESS_LIST_PREFIX}; /// Key for running queries tracked in metasrv. /// Layout: `__process/{frontend server addr}-{query id}` @@ -94,8 +88,8 @@ pub const PROCESS_ID_SEQ: &str = "process_id_seq"; /// Running process instance. pub struct Process { - key: ProcessKey, - value: ProcessValue, + pub key: ProcessKey, + pub value: ProcessValue, } impl Process { @@ -137,102 +131,8 @@ impl Process { } } -pub struct ProcessManager { - server_addr: String, - sequencer: SequenceRef, - kv_client: KvBackendRef, -} - -impl ProcessManager { - /// Create a [ProcessManager] instance with server address and kv client. - pub fn new(server_addr: String, kv_client: KvBackendRef) -> Self { - let sequencer = Arc::new( - SequenceBuilder::new(PROCESS_ID_SEQ, kv_client.clone()) - .initial(0) - .step(100) - .build(), - ); - Self { - server_addr, - sequencer, - kv_client, - } - } - - /// Registers a submitted query. - pub async fn register_query(&self, database: String, query: String) -> error::Result { - let process_id = self.sequencer.next().await?; - let key = ProcessKey { - frontend_ip: self.server_addr.clone(), - id: process_id, - } - .to_bytes(); - let current_time = current_time_millis(); - let value = ProcessValue { - database, - query, - start_timestamp_ms: current_time, - } - .try_as_raw_value()?; - - self.kv_client - .put(PutRequest { - key, - value, - prev_kv: false, - }) - .await?; - Ok(process_id) - } - - /// De-register a query from process list. - pub async fn deregister_query(&self, id: u64) -> error::Result<()> { - let key = ProcessKey { - frontend_ip: self.server_addr.clone(), - id, - } - .to_bytes(); - let prev_kv = self.kv_client.delete(&key, true).await?; - - if let Some(_kv) = prev_kv { - debug!("Successfully deregistered process {}", id); - } else { - warn!("Cannot find process to deregister process: {}", id); - } - Ok(()) - } - - /// De-register all queries running on current frontend. - pub async fn deregister_all_queries(&self) -> error::Result<()> { - let prefix = format!("{}/{}-", PROCESS_LIST_PREFIX, self.server_addr); - let delete_range_request = DeleteRangeRequest::new().with_prefix(prefix.as_bytes()); - self.kv_client.delete_range(delete_range_request).await?; - info!("All queries on {} has been deregistered", self.server_addr); - Ok(()) - } - - /// List all running processes in cluster. - pub fn list_all_processes(&self) -> error::Result> { - let prefix = format!("{}/{}-", PROCESS_LIST_PREFIX, self.server_addr); - let req = RangeRequest::new().with_prefix(prefix.as_bytes()); - let stream = PaginationStream::new(self.kv_client.clone(), req, 100, |kv| { - let key = ProcessKey::from_bytes(&kv.key)?; - let value = ProcessValue::try_from_raw_value(&kv.value)?; - Ok(Process { key, value }) - }); - Ok(stream) - } - - #[cfg(test)] - async fn dump(&self) -> error::Result> { - use futures_util::TryStreamExt; - self.list_all_processes()?.into_stream().try_collect().await - } -} - #[cfg(test)] mod tests { - use super::*; use crate::key::MetadataKey; @@ -306,170 +206,4 @@ mod tests { assert_eq!(value.start_timestamp_ms, deserialized.start_timestamp_ms); assert_eq!(value.database, deserialized.database); } - - use std::sync::Arc; - use std::time::{SystemTime, UNIX_EPOCH}; - - use crate::kv_backend::memory::MemoryKvBackend; - - #[tokio::test] - async fn test_register_query() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); - let process_id = process_manager - .register_query("public".to_string(), "SELECT * FROM table".to_string()) - .await - .unwrap(); - - let running_processes = process_manager.dump().await.unwrap(); - assert_eq!(running_processes.len(), 1); - assert_eq!(running_processes[0].key.frontend_ip, "127.0.0.1:8000"); - assert_eq!(running_processes[0].key.id, process_id); - assert_eq!(running_processes[0].value.query, "SELECT * FROM table"); - - process_manager.deregister_query(process_id).await.unwrap(); - assert_eq!(process_manager.dump().await.unwrap().len(), 0); - } - - #[tokio::test] - async fn test_register_multiple_queries() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); - - // Register multiple queries - let id1 = process_manager - .register_query("public".to_string(), "SELECT 1".to_string()) - .await - .unwrap(); - let id2 = process_manager - .register_query("public".to_string(), "SELECT 2".to_string()) - .await - .unwrap(); - let id3 = process_manager - .register_query("public".to_string(), "SELECT 3".to_string()) - .await - .unwrap(); - - // Verify all are registered - assert_eq!(process_manager.dump().await.unwrap().len(), 3); - - // Deregister middle one - process_manager.deregister_query(id2).await.unwrap(); - let processes = process_manager.dump().await.unwrap(); - assert_eq!(processes.len(), 2); - assert!(processes.iter().any(|p| p.key.id == id1)); - assert!(processes.iter().any(|p| p.key.id == id3)); - } - - #[tokio::test] - async fn test_deregister_nonexistent_query() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); - - // Try to deregister non-existent ID - let result = process_manager.deregister_query(999).await; - assert!(result.is_ok()); // Should succeed with warning - } - - #[tokio::test] - async fn test_process_timestamps() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); - - let before = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis() as i64; - - process_manager - .register_query("public".to_string(), "SELECT NOW()".to_string()) - .await - .unwrap(); - - let after = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis() as i64; - - let running_processes = process_manager.dump().await.unwrap(); - let process = &running_processes[0]; - - assert!(process.value.start_timestamp_ms >= before); - assert!(process.value.start_timestamp_ms <= after); - } - - #[tokio::test] - async fn test_multiple_frontends() { - let kv_client = Arc::new(MemoryKvBackend::new()); - - // Create two process managers with different frontend addresses - let pm1 = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); - let pm2 = ProcessManager::new("127.0.0.1:8001".to_string(), kv_client.clone()); - - let id1 = pm1 - .register_query("public".to_string(), "SELECT 1".to_string()) - .await - .unwrap(); - let id2 = pm2 - .register_query("public".to_string(), "SELECT 2".to_string()) - .await - .unwrap(); - - // Verify both processes are registered with correct frontend IPs - let pm1_processes = pm1.dump().await.unwrap(); - assert_eq!(pm1_processes.len(), 1); - - let p1 = pm1_processes.iter().find(|p| p.key.id == id1).unwrap(); - assert_eq!(p1.key.frontend_ip, "127.0.0.1:8000"); - - let p2_processes = pm2.dump().await.unwrap(); - assert_eq!(p2_processes.len(), 1); - let p2 = p2_processes.iter().find(|p| p.key.id == id2).unwrap(); - assert_eq!(p2.key.frontend_ip, "127.0.0.1:8001"); - assert_eq!(p2.value.database, "public"); - // deregister all queries on instance 1 - pm1.deregister_all_queries().await.unwrap(); - - let processes: Vec<_> = pm1.dump().await.unwrap(); - assert_eq!(processes.len(), 0); - - assert_eq!(pm2.dump().await.unwrap().len(), 1); - } - - #[tokio::test] - async fn test_list_empty_processes() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); - - assert!(process_manager.dump().await.unwrap().is_empty()); - } - - #[tokio::test] - async fn test_deregister_all_queries() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); - - // Register multiple queries - process_manager - .register_query("public".to_string(), "SELECT 1".to_string()) - .await - .unwrap(); - process_manager - .register_query("public".to_string(), "SELECT 2".to_string()) - .await - .unwrap(); - process_manager - .register_query("public".to_string(), "SELECT 3".to_string()) - .await - .unwrap(); - - // Verify they exist - assert_eq!(process_manager.dump().await.unwrap().len(), 3); - - // Deregister all - process_manager.deregister_all_queries().await.unwrap(); - - // Verify none remain - assert_eq!(process_manager.dump().await.unwrap().len(), 0); - } } diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index 6db06f735d43..c5741b6a7d80 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -30,12 +30,12 @@ use std::time::SystemTime; use async_trait::async_trait; use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq}; +use catalog::process_manager::ProcessManager; use catalog::CatalogManagerRef; use client::OutputData; use common_base::Plugins; use common_config::KvBackendConfig; use common_error::ext::{BoxedError, ErrorExt}; -use common_meta::key::process_list::ProcessManager; use common_meta::key::TableMetadataManagerRef; use common_meta::kv_backend::KvBackendRef; use common_meta::state_store::KvStateStore; diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index bf1ae3920a19..f66ec27ebb6f 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -15,13 +15,13 @@ use std::sync::Arc; use cache::{TABLE_FLOWNODE_SET_CACHE_NAME, TABLE_ROUTE_CACHE_NAME}; +use catalog::process_manager::ProcessManager; use catalog::CatalogManagerRef; use common_base::Plugins; use common_meta::cache::{LayeredCacheRegistryRef, TableRouteCacheRef}; use common_meta::cache_invalidator::{CacheInvalidatorRef, DummyCacheInvalidator}; use common_meta::ddl::ProcedureExecutorRef; use common_meta::key::flow::FlowMetadataManager; -use common_meta::key::process_list::ProcessManager; use common_meta::key::TableMetadataManager; use common_meta::kv_backend::KvBackendRef; use common_meta::node_manager::NodeManagerRef; From 66535fcf558cceed0cfda5997f1f2ce5f26b045f Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Fri, 11 Apr 2025 07:21:20 +0000 Subject: [PATCH 13/31] feat/show-process-list: - **Refactor Row Construction**: Updated row construction in multiple files to use references for `Value` objects, improving memory efficiency. Affected files include: - `cluster_info.rs` - `columns.rs` - `flows.rs` - `key_column_usage.rs` - `partitions.rs` - `procedure_info.rs` - `process_list.rs` - `region_peers.rs` - `region_statistics.rs` - `schemata.rs` - `table_constraints.rs` - `tables.rs` - `views.rs` - `pg_class.rs` - `pg_database.rs` - `pg_namespace.rs` - **Remove Unused Code**: Deleted unused functions and error variants related to process management in `process_list.rs` and `error.rs`. - **Predicate Evaluation Update**: Modified predicate evaluation functions in `predicate.rs` to work with references, enhancing performance. --- .../information_schema/cluster_info.rs | 10 +-- .../information_schema/columns.rs | 16 ++-- .../system_schema/information_schema/flows.rs | 6 +- .../information_schema/key_column_usage.rs | 14 ++-- .../information_schema/partitions.rs | 6 +- .../information_schema/procedure_info.rs | 12 +-- .../information_schema/process_list.rs | 63 ++++++--------- .../information_schema/region_peers.rs | 8 +- .../information_schema/region_statistics.rs | 22 ++--- .../information_schema/schemata.rs | 10 +-- .../information_schema/table_constraints.rs | 10 +-- .../information_schema/tables.rs | 12 +-- .../system_schema/information_schema/views.rs | 6 +- .../src/system_schema/pg_catalog/pg_class.rs | 10 +-- .../system_schema/pg_catalog/pg_database.rs | 6 +- .../system_schema/pg_catalog/pg_namespace.rs | 4 +- src/catalog/src/system_schema/predicate.rs | 80 +++++++------------ src/common/meta/src/error.rs | 24 ------ 18 files changed, 130 insertions(+), 189 deletions(-) diff --git a/src/catalog/src/system_schema/information_schema/cluster_info.rs b/src/catalog/src/system_schema/information_schema/cluster_info.rs index 2f65610e0f84..7672791df68f 100644 --- a/src/catalog/src/system_schema/information_schema/cluster_info.rs +++ b/src/catalog/src/system_schema/information_schema/cluster_info.rs @@ -178,11 +178,11 @@ impl InformationSchemaClusterInfoBuilder { let peer_type = node_info.status.role_name(); let row = [ - (PEER_ID, Value::from(node_info.peer.id)), - (PEER_TYPE, Value::from(peer_type)), - (PEER_ADDR, Value::from(node_info.peer.addr.as_str())), - (VERSION, Value::from(node_info.version.as_str())), - (GIT_COMMIT, Value::from(node_info.git_commit.as_str())), + (PEER_ID, &Value::from(node_info.peer.id)), + (PEER_TYPE, &Value::from(peer_type)), + (PEER_ADDR, &Value::from(node_info.peer.addr.as_str())), + (VERSION, &Value::from(node_info.version.as_str())), + (GIT_COMMIT, &Value::from(node_info.git_commit.as_str())), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/columns.rs b/src/catalog/src/system_schema/information_schema/columns.rs index 523784477cc0..0c866f4841d5 100644 --- a/src/catalog/src/system_schema/information_schema/columns.rs +++ b/src/catalog/src/system_schema/information_schema/columns.rs @@ -315,14 +315,14 @@ impl InformationSchemaColumnsBuilder { }; let row = [ - (TABLE_CATALOG, Value::from(catalog_name)), - (TABLE_SCHEMA, Value::from(schema_name)), - (TABLE_NAME, Value::from(table_name)), - (COLUMN_NAME, Value::from(column_schema.name.as_str())), - (DATA_TYPE, Value::from(data_type.as_str())), - (SEMANTIC_TYPE, Value::from(semantic_type)), - (ORDINAL_POSITION, Value::from((index + 1) as i64)), - (COLUMN_KEY, Value::from(column_key)), + (TABLE_CATALOG, &Value::from(catalog_name)), + (TABLE_SCHEMA, &Value::from(schema_name)), + (TABLE_NAME, &Value::from(table_name)), + (COLUMN_NAME, &Value::from(column_schema.name.as_str())), + (DATA_TYPE, &Value::from(data_type.as_str())), + (SEMANTIC_TYPE, &Value::from(semantic_type)), + (ORDINAL_POSITION, &Value::from((index + 1) as i64)), + (COLUMN_KEY, &Value::from(column_key)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/flows.rs b/src/catalog/src/system_schema/information_schema/flows.rs index c2fdc5788fc0..24b75e62e500 100644 --- a/src/catalog/src/system_schema/information_schema/flows.rs +++ b/src/catalog/src/system_schema/information_schema/flows.rs @@ -273,11 +273,11 @@ impl InformationSchemaFlowsBuilder { flow_stat: &Option, ) -> Result<()> { let row = [ - (FLOW_NAME, Value::from(flow_info.flow_name().to_string())), - (FLOW_ID, Value::from(flow_id)), + (FLOW_NAME, &Value::from(flow_info.flow_name().to_string())), + (FLOW_ID, &Value::from(flow_id)), ( TABLE_CATALOG, - Value::from(flow_info.catalog_name().to_string()), + &Value::from(flow_info.catalog_name().to_string()), ), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/key_column_usage.rs b/src/catalog/src/system_schema/information_schema/key_column_usage.rs index 28530e44f213..9f0883930373 100644 --- a/src/catalog/src/system_schema/information_schema/key_column_usage.rs +++ b/src/catalog/src/system_schema/information_schema/key_column_usage.rs @@ -290,13 +290,13 @@ impl InformationSchemaKeyColumnUsageBuilder { ordinal_position: u32, ) { let row = [ - (CONSTRAINT_SCHEMA, Value::from(constraint_schema)), - (CONSTRAINT_NAME, Value::from(constraint_name)), - (REAL_TABLE_CATALOG, Value::from(table_catalog)), - (TABLE_SCHEMA, Value::from(table_schema)), - (TABLE_NAME, Value::from(table_name)), - (COLUMN_NAME, Value::from(column_name)), - (ORDINAL_POSITION, Value::from(ordinal_position)), + (CONSTRAINT_SCHEMA, &Value::from(constraint_schema)), + (CONSTRAINT_NAME, &Value::from(constraint_name)), + (REAL_TABLE_CATALOG, &Value::from(table_catalog)), + (TABLE_SCHEMA, &Value::from(table_schema)), + (TABLE_NAME, &Value::from(table_name)), + (COLUMN_NAME, &Value::from(column_name)), + (ORDINAL_POSITION, &Value::from(ordinal_position)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/partitions.rs b/src/catalog/src/system_schema/information_schema/partitions.rs index 0b8a9ae8b6bc..9b9634cb44ca 100644 --- a/src/catalog/src/system_schema/information_schema/partitions.rs +++ b/src/catalog/src/system_schema/information_schema/partitions.rs @@ -311,9 +311,9 @@ impl InformationSchemaPartitionsBuilder { partitions: &[PartitionInfo], ) { let row = [ - (TABLE_CATALOG, Value::from(catalog_name)), - (TABLE_SCHEMA, Value::from(schema_name)), - (TABLE_NAME, Value::from(table_name)), + (TABLE_CATALOG, &Value::from(catalog_name)), + (TABLE_SCHEMA, &Value::from(schema_name)), + (TABLE_NAME, &Value::from(table_name)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/procedure_info.rs b/src/catalog/src/system_schema/information_schema/procedure_info.rs index 23723d2e7df3..3ae237fdf144 100644 --- a/src/catalog/src/system_schema/information_schema/procedure_info.rs +++ b/src/catalog/src/system_schema/information_schema/procedure_info.rs @@ -187,12 +187,12 @@ impl InformationSchemaProcedureInfoBuilder { let lock_keys = lock_keys.join(","); let row = [ - (PROCEDURE_ID, Value::from(pid.clone())), - (PROCEDURE_TYPE, Value::from(type_name.clone())), - (START_TIME, Value::from(start_time)), - (END_TIME, Value::from(end_time)), - (STATUS, Value::from(status.clone())), - (LOCK_KEYS, Value::from(lock_keys.clone())), + (PROCEDURE_ID, &Value::from(pid.clone())), + (PROCEDURE_TYPE, &Value::from(type_name.clone())), + (START_TIME, &Value::from(start_time)), + (END_TIME, &Value::from(end_time)), + (STATUS, &Value::from(status.clone())), + (LOCK_KEYS, &Value::from(lock_keys.clone())), ]; if !predicates.eval(&row) { return; diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index dd321f33d677..fd113d5999b2 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -16,7 +16,6 @@ use std::sync::Arc; use common_catalog::consts::INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID; use common_error::ext::BoxedError; -use common_meta::key::process_list::Process; use common_recordbatch::adapter::RecordBatchStreamAdapter; use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; use common_time::util::current_time_millis; @@ -134,20 +133,35 @@ async fn make_process_list( let mut start_time_builder = TimestampMillisecondVectorBuilder::with_capacity(8); let mut elapsed_time_builder = DurationMillisecondVectorBuilder::with_capacity(8); - let mut current_row = Vec::with_capacity(5); while let Some(process) = stream .next() .await .transpose() .context(error::ListProcessSnafu)? { - process_to_row(process, current_time, &mut current_row); - if predicates.eval(¤t_row) { - id_builder.push(current_row[0].1.as_u64()); - database_builder.push(current_row[1].1.as_string().as_deref()); - query_builder.push(current_row[2].1.as_string().as_deref()); - start_time_builder.push(current_row[3].1.as_timestamp().map(|t| t.value().into())); - elapsed_time_builder.push(current_row[4].1.as_duration().map(|d| d.value().into())); + let row = [ + (ID, &Value::from(process.query_id())), + (DATABASE, &Value::from(process.database())), + (QUERY, &Value::from(process.query_string())), + ( + START_TIMESTAMP, + &Value::from(Timestamp::new_millisecond( + process.query_start_timestamp_ms(), + )), + ), + ( + ELAPSED_TIME, + &Value::from(Duration::new_millisecond( + current_time - process.query_start_timestamp_ms(), + )), + ), + ]; + if predicates.eval(&row) { + id_builder.push(row[0].1.as_u64()); + database_builder.push(row[1].1.as_string().as_deref()); + query_builder.push(row[2].1.as_string().as_deref()); + start_time_builder.push(row[3].1.as_timestamp().map(|t| t.value().into())); + elapsed_time_builder.push(row[4].1.as_duration().map(|d| d.value().into())); } } @@ -162,34 +176,3 @@ async fn make_process_list( RecordBatch::new(InformationSchemaProcessList::schema(), columns) .context(error::CreateRecordBatchSnafu) } - -// Convert [Process] structs to rows. -fn process_to_row( - process: Process, - current_time_ms: i64, - current_row: &mut Vec<(&'static str, Value)>, -) { - current_row.clear(); - current_row.push((ID, Value::UInt64(process.query_id()))); - current_row.push(( - DATABASE, - Value::String(process.database().to_string().into()), - )); - current_row.push(( - QUERY, - Value::String(process.query_string().to_string().into()), - )); - - current_row.push(( - START_TIMESTAMP, - Value::Timestamp(Timestamp::new_millisecond( - process.query_start_timestamp_ms(), - )), - )); - current_row.push(( - ELAPSED_TIME, - Value::Duration(Duration::new_millisecond( - current_time_ms - process.query_start_timestamp_ms(), - )), - )); -} diff --git a/src/catalog/src/system_schema/information_schema/region_peers.rs b/src/catalog/src/system_schema/information_schema/region_peers.rs index 7122a2cef117..1d46a9db0109 100644 --- a/src/catalog/src/system_schema/information_schema/region_peers.rs +++ b/src/catalog/src/system_schema/information_schema/region_peers.rs @@ -263,10 +263,10 @@ impl InformationSchemaRegionPeersBuilder { }; let row = [ - (TABLE_CATALOG, Value::from(table_catalog)), - (TABLE_SCHEMA, Value::from(table_schema)), - (TABLE_NAME, Value::from(table_name)), - (REGION_ID, Value::from(region_id)), + (TABLE_CATALOG, &Value::from(table_catalog)), + (TABLE_SCHEMA, &Value::from(table_schema)), + (TABLE_NAME, &Value::from(table_name)), + (REGION_ID, &Value::from(region_id)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/region_statistics.rs b/src/catalog/src/system_schema/information_schema/region_statistics.rs index 8980dc7cbe02..8db6b860bfb6 100644 --- a/src/catalog/src/system_schema/information_schema/region_statistics.rs +++ b/src/catalog/src/system_schema/information_schema/region_statistics.rs @@ -189,17 +189,17 @@ impl InformationSchemaRegionStatisticsBuilder { fn add_region_statistic(&mut self, predicate: &Predicates, region_stat: RegionStat) { let row = [ - (REGION_ID, Value::from(region_stat.id.as_u64())), - (TABLE_ID, Value::from(region_stat.id.table_id())), - (REGION_NUMBER, Value::from(region_stat.id.region_number())), - (REGION_ROWS, Value::from(region_stat.num_rows)), - (DISK_SIZE, Value::from(region_stat.approximate_bytes)), - (MEMTABLE_SIZE, Value::from(region_stat.memtable_size)), - (MANIFEST_SIZE, Value::from(region_stat.manifest_size)), - (SST_SIZE, Value::from(region_stat.sst_size)), - (INDEX_SIZE, Value::from(region_stat.index_size)), - (ENGINE, Value::from(region_stat.engine.as_str())), - (REGION_ROLE, Value::from(region_stat.role.to_string())), + (REGION_ID, &Value::from(region_stat.id.as_u64())), + (TABLE_ID, &Value::from(region_stat.id.table_id())), + (REGION_NUMBER, &Value::from(region_stat.id.region_number())), + (REGION_ROWS, &Value::from(region_stat.num_rows)), + (DISK_SIZE, &Value::from(region_stat.approximate_bytes)), + (MEMTABLE_SIZE, &Value::from(region_stat.memtable_size)), + (MANIFEST_SIZE, &Value::from(region_stat.manifest_size)), + (SST_SIZE, &Value::from(region_stat.sst_size)), + (INDEX_SIZE, &Value::from(region_stat.index_size)), + (ENGINE, &Value::from(region_stat.engine.as_str())), + (REGION_ROLE, &Value::from(region_stat.role.to_string())), ]; if !predicate.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/schemata.rs b/src/catalog/src/system_schema/information_schema/schemata.rs index c333f1297ed4..6738267d3fb9 100644 --- a/src/catalog/src/system_schema/information_schema/schemata.rs +++ b/src/catalog/src/system_schema/information_schema/schemata.rs @@ -204,11 +204,11 @@ impl InformationSchemaSchemataBuilder { schema_options: &str, ) { let row = [ - (CATALOG_NAME, Value::from(catalog_name)), - (SCHEMA_NAME, Value::from(schema_name)), - (DEFAULT_CHARACTER_SET_NAME, Value::from("utf8")), - (DEFAULT_COLLATION_NAME, Value::from("utf8_bin")), - (SCHEMA_OPTS, Value::from(schema_options)), + (CATALOG_NAME, &Value::from(catalog_name)), + (SCHEMA_NAME, &Value::from(schema_name)), + (DEFAULT_CHARACTER_SET_NAME, &Value::from("utf8")), + (DEFAULT_COLLATION_NAME, &Value::from("utf8_bin")), + (SCHEMA_OPTS, &Value::from(schema_options)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/table_constraints.rs b/src/catalog/src/system_schema/information_schema/table_constraints.rs index 1c920bc38285..a1f9d899f4b1 100644 --- a/src/catalog/src/system_schema/information_schema/table_constraints.rs +++ b/src/catalog/src/system_schema/information_schema/table_constraints.rs @@ -221,11 +221,11 @@ impl InformationSchemaTableConstraintsBuilder { constraint_type: &str, ) { let row = [ - (CONSTRAINT_SCHEMA, Value::from(constraint_schema)), - (CONSTRAINT_NAME, Value::from(constraint_name)), - (TABLE_SCHEMA, Value::from(table_schema)), - (TABLE_NAME, Value::from(table_name)), - (CONSTRAINT_TYPE, Value::from(constraint_type)), + (CONSTRAINT_SCHEMA, &Value::from(constraint_schema)), + (CONSTRAINT_NAME, &Value::from(constraint_name)), + (TABLE_SCHEMA, &Value::from(table_schema)), + (TABLE_NAME, &Value::from(table_name)), + (CONSTRAINT_TYPE, &Value::from(constraint_type)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/tables.rs b/src/catalog/src/system_schema/information_schema/tables.rs index 433e1769bec3..8c288b8e575d 100644 --- a/src/catalog/src/system_schema/information_schema/tables.rs +++ b/src/catalog/src/system_schema/information_schema/tables.rs @@ -324,12 +324,12 @@ impl InformationSchemaTablesBuilder { }; let row = [ - (TABLE_CATALOG, Value::from(catalog_name)), - (TABLE_ID, Value::from(table_id)), - (TABLE_SCHEMA, Value::from(schema_name)), - (ENGINE, Value::from(engine)), - (TABLE_NAME, Value::from(table_name)), - (TABLE_TYPE, Value::from(table_type_text)), + (TABLE_CATALOG, &Value::from(catalog_name)), + (TABLE_ID, &Value::from(table_id)), + (TABLE_SCHEMA, &Value::from(schema_name)), + (ENGINE, &Value::from(engine)), + (TABLE_NAME, &Value::from(table_name)), + (TABLE_TYPE, &Value::from(table_type_text)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/information_schema/views.rs b/src/catalog/src/system_schema/information_schema/views.rs index 6cb21c428570..d0424f528de8 100644 --- a/src/catalog/src/system_schema/information_schema/views.rs +++ b/src/catalog/src/system_schema/information_schema/views.rs @@ -228,9 +228,9 @@ impl InformationSchemaViewsBuilder { definition: &str, ) { let row = [ - (TABLE_CATALOG, Value::from(catalog_name)), - (TABLE_SCHEMA, Value::from(schema_name)), - (TABLE_NAME, Value::from(table_name)), + (TABLE_CATALOG, &Value::from(catalog_name)), + (TABLE_SCHEMA, &Value::from(schema_name)), + (TABLE_NAME, &Value::from(table_name)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/pg_catalog/pg_class.rs b/src/catalog/src/system_schema/pg_catalog/pg_class.rs index 2562db108cee..57f8db882c6e 100644 --- a/src/catalog/src/system_schema/pg_catalog/pg_class.rs +++ b/src/catalog/src/system_schema/pg_catalog/pg_class.rs @@ -245,11 +245,11 @@ impl PGClassBuilder { ) { let namespace_oid = self.namespace_oid_map.get_oid(schema); let row = [ - (OID_COLUMN_NAME, Value::from(oid)), - (RELNAMESPACE, Value::from(schema)), - (RELNAME, Value::from(table)), - (RELKIND, Value::from(kind)), - (RELOWNER, Value::from(DUMMY_OWNER_ID)), + (OID_COLUMN_NAME, &Value::from(oid)), + (RELNAMESPACE, &Value::from(schema)), + (RELNAME, &Value::from(table)), + (RELKIND, &Value::from(kind)), + (RELOWNER, &Value::from(DUMMY_OWNER_ID)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/pg_catalog/pg_database.rs b/src/catalog/src/system_schema/pg_catalog/pg_database.rs index b83b66f7ba9d..8a788a25daa4 100644 --- a/src/catalog/src/system_schema/pg_catalog/pg_database.rs +++ b/src/catalog/src/system_schema/pg_catalog/pg_database.rs @@ -202,9 +202,9 @@ impl PGCDatabaseBuilder { fn add_database(&mut self, predicates: &Predicates, schema_name: &str) { let oid = self.namespace_oid_map.get_oid(schema_name); - let row: [(&str, Value); 2] = [ - (OID_COLUMN_NAME, Value::from(oid)), - (DATNAME, Value::from(schema_name)), + let row: [(&str, &Value); 2] = [ + (OID_COLUMN_NAME, &Value::from(oid)), + (DATNAME, &Value::from(schema_name)), ]; if !predicates.eval(&row) { diff --git a/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs b/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs index 121cfda60d71..f8fc3e09c482 100644 --- a/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs +++ b/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs @@ -209,8 +209,8 @@ impl PGNamespaceBuilder { fn add_namespace(&mut self, predicates: &Predicates, schema_name: &str) { let oid = self.namespace_oid_map.get_oid(schema_name); let row = [ - (OID_COLUMN_NAME, Value::from(oid)), - (NSPNAME, Value::from(schema_name)), + (OID_COLUMN_NAME, &Value::from(oid)), + (NSPNAME, &Value::from(schema_name)), ]; if !predicates.eval(&row) { return; diff --git a/src/catalog/src/system_schema/predicate.rs b/src/catalog/src/system_schema/predicate.rs index 18f0b3eacf3f..c94141a94778 100644 --- a/src/catalog/src/system_schema/predicate.rs +++ b/src/catalog/src/system_schema/predicate.rs @@ -40,14 +40,14 @@ impl Predicate { /// - `None` when the predicate can't evaluate with the row. /// - `Some(true)` when the predicate is satisfied, /// - `Some(false)` when the predicate is not satisfied, - fn eval(&self, row: &[(&str, Value)]) -> Option { + fn eval(&self, row: &[(&str, &Value)]) -> Option { match self { Predicate::Eq(c, v) => { for (column, value) in row { if c != column { continue; } - return Some(v == value); + return Some(v == *value); } } Predicate::Like(c, pattern, case_insensitive) => { @@ -68,7 +68,7 @@ impl Predicate { if c != column { continue; } - return Some(v != value); + return Some(v != *value); } } Predicate::InList(c, values) => { @@ -76,7 +76,7 @@ impl Predicate { if c != column { continue; } - return Some(values.iter().any(|v| v == value)); + return Some(values.iter().any(|v| v == *value)); } } Predicate::And(left, right) => { @@ -271,7 +271,7 @@ impl Predicates { /// Evaluate the predicates with the row. /// returns true when all the predicates are satisfied or can't be evaluated. - pub fn eval(&self, row: &[(&str, Value)]) -> bool { + pub fn eval(&self, row: &[(&str, &Value)]) -> bool { // fast path if self.predicates.is_empty() { return true; @@ -305,9 +305,9 @@ mod tests { let b_value = Value::from("b_value"); let wrong_value = Value::from("wrong_value"); - let a_row = [(a_col.as_str(), a_value.clone())]; - let b_row = [("b", wrong_value.clone())]; - let wrong_row = [(a_col.as_str(), wrong_value.clone())]; + let a_row = [(a_col.as_str(), &a_value)]; + let b_row = [("b", &wrong_value)]; + let wrong_row = [(a_col.as_str(), &wrong_value)]; // Predicate::Eq let p = Predicate::Eq(a_col.clone(), a_value.clone()); @@ -326,42 +326,30 @@ mod tests { assert!(p.eval(&a_row).unwrap()); assert!(p.eval(&b_row).is_none()); assert!(!p.eval(&wrong_row).unwrap()); - assert!(p.eval(&[(&a_col, b_value.clone())]).unwrap()); + assert!(p.eval(&[(&a_col, &b_value)]).unwrap()); let p1 = Predicate::Eq(a_col.clone(), a_value.clone()); let p2 = Predicate::Eq(b_col.clone(), b_value.clone()); - let row = [ - (a_col.as_str(), a_value.clone()), - (b_col.as_str(), b_value.clone()), - ]; - let wrong_row = [ - (a_col.as_str(), a_value.clone()), - (b_col.as_str(), wrong_value.clone()), - ]; + let row = [(a_col.as_str(), &a_value), (b_col.as_str(), &b_value)]; + let wrong_row = [(a_col.as_str(), &a_value), (b_col.as_str(), &wrong_value)]; //Predicate::And let p = Predicate::And(Box::new(p1.clone()), Box::new(p2.clone())); assert!(p.eval(&row).unwrap()); assert!(!p.eval(&wrong_row).unwrap()); assert!(p.eval(&[]).is_none()); - assert!(p.eval(&[("c", a_value.clone())]).is_none()); + assert!(p.eval(&[("c", &a_value)]).is_none()); assert!(!p - .eval(&[ - (a_col.as_str(), b_value.clone()), - (b_col.as_str(), a_value.clone()) - ]) + .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &a_value)]) .unwrap()); assert!(!p - .eval(&[ - (a_col.as_str(), b_value.clone()), - (b_col.as_str(), b_value.clone()) - ]) + .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &b_value)]) .unwrap()); assert!(p - .eval(&[(a_col.as_ref(), a_value.clone()), ("c", a_value.clone())]) + .eval(&[(a_col.as_ref(), &a_value), ("c", &a_value)]) .is_none()); assert!(!p - .eval(&[(a_col.as_ref(), b_value.clone()), ("c", a_value.clone())]) + .eval(&[(a_col.as_ref(), &b_value), ("c", &a_value)]) .unwrap()); //Predicate::Or @@ -369,24 +357,18 @@ mod tests { assert!(p.eval(&row).unwrap()); assert!(p.eval(&wrong_row).unwrap()); assert!(p.eval(&[]).is_none()); - assert!(p.eval(&[("c", a_value.clone())]).is_none()); + assert!(p.eval(&[("c", &a_value)]).is_none()); assert!(!p - .eval(&[ - (a_col.as_str(), b_value.clone()), - (b_col.as_str(), a_value.clone()) - ]) + .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &a_value)]) .unwrap()); assert!(p - .eval(&[ - (a_col.as_str(), b_value.clone()), - (b_col.as_str(), b_value.clone()) - ]) + .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &b_value)]) .unwrap()); assert!(p - .eval(&[(a_col.as_ref(), a_value.clone()), ("c", a_value.clone())]) + .eval(&[(a_col.as_ref(), &a_value), ("c", &a_value)]) .unwrap()); assert!(p - .eval(&[(a_col.as_ref(), b_value.clone()), ("c", a_value.clone())]) + .eval(&[(a_col.as_ref(), &b_value), ("c", &a_value)]) .is_none()); } @@ -410,10 +392,10 @@ mod tests { ); let match_row = [ - ("a", Value::from("hello AbC")), - ("b", Value::from("b value")), + ("a", &Value::from("hello AbC")), + ("b", &Value::from("b value")), ]; - let unmatch_row = [("a", Value::from("bca")), ("b", Value::from("b value"))]; + let unmatch_row = [("a", &Value::from("bca")), ("b", &Value::from("b value"))]; assert!(p.eval(&match_row).unwrap()); assert!(!p.eval(&unmatch_row).unwrap()); @@ -573,16 +555,16 @@ mod tests { #[test] fn test_predicates_eval_row() { let wrong_row = [ - ("a", Value::from("a_value")), - ("b", Value::from("b_value")), - ("c", Value::from("c_value")), + ("a", &Value::from("a_value")), + ("b", &Value::from("b_value")), + ("c", &Value::from("c_value")), ]; let row = [ - ("a", Value::from("a_value")), - ("b", Value::from("not_b_value")), - ("c", Value::from("c_value")), + ("a", &Value::from("a_value")), + ("b", &Value::from("not_b_value")), + ("c", &Value::from("c_value")), ]; - let c_row = [("c", Value::from("c_value"))]; + let c_row = [("c", &Value::from("c_value"))]; // test empty predicates, always returns true let predicates = Predicates::from_scan_request(&None); diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs index 7cc48eb0d17f..ce84003cfd9b 100644 --- a/src/common/meta/src/error.rs +++ b/src/common/meta/src/error.rs @@ -790,27 +790,6 @@ pub enum Error { #[snafu(source)] source: common_procedure::error::Error, }, - // #[snafu(display("Failed to serialize process value when registering."))] - // RegisterProcess { - // source: crate::error::Error, - // #[snafu(implicit)] - // location: Location, - // }, - // - // #[snafu(display("Failed to list all running processes."))] - // ListProcesses { - // source: crate::error::Error, - // #[snafu(implicit)] - // location: Location, - // }, - // - // #[snafu(display("Failed to delete process {}", msg))] - // DeleteProcess { - // msg: String, - // source: crate::error::Error, - // #[snafu(implicit)] - // location: Location, - // }, } pub type Result = std::result::Result; @@ -927,9 +906,6 @@ impl ErrorExt for Error { RdsTransactionRetryFailed { .. } => StatusCode::Internal, Error::DatanodeTableInfoNotFound { .. } => StatusCode::Internal, InvalidProcessKey { .. } => StatusCode::Internal, - // RegisterProcess { .. } | ListProcesses { .. } | DeleteProcess { .. } => { - // StatusCode::Internal - // } } } From a38c12c9a64de83665234026dcde423c3433067b Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Sun, 13 Apr 2025 16:44:19 +0000 Subject: [PATCH 14/31] feat/show-process-list: ### Implement Process Management Enhancements - **Error Handling Enhancements**: - Added new error variants `BumpSequence`, `StartReportTask`, `ReportProcess`, and `BuildProcessManager` in `error.rs` to improve error handling for process management tasks. - Updated `ErrorExt` implementations to handle new error types. - **Process Manager Improvements**: - Introduced `ProcessManager` enhancements in `process_manager.rs` to manage process states using `ProcessWithState` and `ProcessState` enums. - Implemented periodic task `ReportTask` to report running queries to the KV backend. - Modified `register_query` and `deregister_query` methods to use the new state management system. - **Testing and Validation**: - Updated tests in `process_manager.rs` to validate new process management logic. - Replaced `dump` method with `list_all_processes` for listing processes. - **Integration with Frontend and Standalone**: - Updated `frontend.rs` and `standalone.rs` to handle `ProcessManager` initialization errors using `BuildProcessManager` error variant. - **Schema Adjustments**: - Modified `process_list.rs` in `system_schema/information_schema` to use the updated process listing method. - **Key-Value Conversion**: - Added `TryFrom` implementation for converting `Process` to `KeyValue` in `process_list.rs`. --- src/catalog/src/error.rs | 26 +- src/catalog/src/process_manager.rs | 271 +++++++++++++----- .../information_schema/process_list.rs | 69 ++--- src/cmd/src/error.rs | 8 + src/cmd/src/frontend.rs | 8 +- src/cmd/src/standalone.rs | 8 +- src/common/meta/src/key/process_list.rs | 15 +- 7 files changed, 283 insertions(+), 122 deletions(-) diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs index a92bd5d849c9..ae005d960db5 100644 --- a/src/catalog/src/error.rs +++ b/src/catalog/src/error.rs @@ -278,12 +278,33 @@ pub enum Error { location: Location, }, + #[snafu(display("Failed to bump sequence for process manager"))] + BumpSequence { + source: common_meta::error::Error, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Failed to list running processes"))] ListProcess { source: common_meta::error::Error, #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to start process report task"))] + StartReportTask { + source: common_runtime::error::Error, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to report process state"))] + ReportProcess { + source: common_meta::error::Error, + #[snafu(implicit)] + location: Location, + }, } impl Error { @@ -351,7 +372,10 @@ impl ErrorExt for Error { Error::TableMetadataManager { source, .. } => source.status_code(), Error::GetViewCache { source, .. } | Error::GetTableCache { source, .. } - | Error::ListProcess { source, .. } => source.status_code(), + | Error::BumpSequence { source, .. } + | Error::ListProcess { source, .. } + | Error::ReportProcess { source, .. } => source.status_code(), + Error::StartReportTask { .. } => StatusCode::Internal, } } diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index 8bd6b8f9569a..2ffcd1c2c14b 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -1,76 +1,111 @@ -use std::sync::Arc; +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; -use common_meta::error; use common_meta::key::process_list::{Process, ProcessKey, ProcessValue, PROCESS_ID_SEQ}; -use common_meta::key::{MetadataKey, MetadataValue, PROCESS_LIST_PREFIX}; +use common_meta::key::{MetadataKey, PROCESS_LIST_PREFIX}; use common_meta::kv_backend::KvBackendRef; -use common_meta::range_stream::PaginationStream; -use common_meta::rpc::store::{DeleteRangeRequest, PutRequest, RangeRequest}; +use common_meta::rpc::store::{BatchDeleteRequest, BatchPutRequest, DeleteRangeRequest}; +use common_meta::rpc::KeyValue; use common_meta::sequence::{SequenceBuilder, SequenceRef}; -use common_telemetry::{debug, info, warn}; +use common_runtime::{RepeatedTask, TaskFunction}; +use common_telemetry::{debug, info}; use common_time::util::current_time_millis; +use snafu::ResultExt; + +use crate::error; pub struct ProcessManager { server_addr: String, sequencer: SequenceRef, kv_client: KvBackendRef, + _report_task: RepeatedTask, + pending_queries: Arc>>, } impl ProcessManager { /// Create a [ProcessManager] instance with server address and kv client. - pub fn new(server_addr: String, kv_client: KvBackendRef) -> Self { + pub fn new(server_addr: String, kv_client: KvBackendRef) -> error::Result { let sequencer = Arc::new( SequenceBuilder::new(PROCESS_ID_SEQ, kv_client.clone()) .initial(0) - .step(100) + .step(10) .build(), ); - Self { + + let pending_queries = Arc::new(Mutex::new(HashMap::new())); + let pending_queries_clone = pending_queries.clone(); + let report_task = RepeatedTask::new( + Duration::from_secs(5), + Box::new(ReportTask { + queries: pending_queries_clone, + kv_backend: kv_client.clone(), + }), + ); + report_task + .start(common_runtime::global_runtime()) + .context(error::StartReportTaskSnafu)?; + Ok(Self { server_addr, sequencer, kv_client, - } + _report_task: report_task, + pending_queries, + }) } /// Registers a submitted query. pub async fn register_query(&self, database: String, query: String) -> error::Result { - let process_id = self.sequencer.next().await?; + let process_id = self + .sequencer + .next() + .await + .context(error::BumpSequenceSnafu)?; let key = ProcessKey { frontend_ip: self.server_addr.clone(), id: process_id, - } - .to_bytes(); - let current_time = current_time_millis(); + }; let value = ProcessValue { database, query, - start_timestamp_ms: current_time, - } - .try_as_raw_value()?; - - self.kv_client - .put(PutRequest { - key, - value, - prev_kv: false, - }) - .await?; + start_timestamp_ms: current_time_millis(), + }; + let process = Process { key, value }; + self.pending_queries.lock().unwrap().insert( + process_id, + ProcessWithState { + process, + state: ProcessState::Running, + }, + ); Ok(process_id) } /// De-register a query from process list. pub async fn deregister_query(&self, id: u64) -> error::Result<()> { - let key = ProcessKey { - frontend_ip: self.server_addr.clone(), - id, - } - .to_bytes(); - let prev_kv = self.kv_client.delete(&key, true).await?; - - if let Some(_kv) = prev_kv { - debug!("Successfully deregistered process {}", id); - } else { - warn!("Cannot find process to deregister process: {}", id); + if let Entry::Occupied(mut e) = self.pending_queries.lock().unwrap().entry(id) { + let process = e.get_mut(); + if process.state == ProcessState::Running { + e.remove(); + } else { + debug_assert_eq!(process.state, ProcessState::Registered); + process.state = ProcessState::Finished; + } } Ok(()) } @@ -79,27 +114,117 @@ impl ProcessManager { pub async fn deregister_all_queries(&self) -> error::Result<()> { let prefix = format!("{}/{}-", PROCESS_LIST_PREFIX, self.server_addr); let delete_range_request = DeleteRangeRequest::new().with_prefix(prefix.as_bytes()); - self.kv_client.delete_range(delete_range_request).await?; + self.pending_queries.lock().unwrap().clear(); + self.kv_client + .delete_range(delete_range_request) + .await + .context(error::ReportProcessSnafu)?; info!("All queries on {} has been deregistered", self.server_addr); Ok(()) } /// List all running processes in cluster. - pub fn list_all_processes(&self) -> error::Result> { - let prefix = format!("{}/{}-", PROCESS_LIST_PREFIX, self.server_addr); - let req = RangeRequest::new().with_prefix(prefix.as_bytes()); - let stream = PaginationStream::new(self.kv_client.clone(), req, 100, |kv| { - let key = ProcessKey::from_bytes(&kv.key)?; - let value = ProcessValue::try_from_raw_value(&kv.value)?; - Ok(Process { key, value }) - }); - Ok(stream) + pub fn list_all_processes(&self) -> error::Result> { + let queries = self.pending_queries.lock().unwrap().clone(); + Ok(queries.into_values().map(|p| p.process).collect()) + } +} + +#[derive(Debug, Clone)] +struct ProcessWithState { + process: Process, + state: ProcessState, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +enum ProcessState { + /// Running but not registered to meta + Running, + /// Running and registered to meta + Registered, + /// Finished + Finished, +} + +/// Periodical task to report running queries to kv backend. +struct ReportTask { + queries: Arc>>, + kv_backend: KvBackendRef, +} + +#[async_trait::async_trait] +impl TaskFunction for ReportTask { + async fn call(&mut self) -> error::Result<()> { + let pending_queries = self.queries.lock().unwrap().clone(); + let mut queries_to_register = vec![]; + let mut queries_to_deregister = vec![]; + + for (_, process) in pending_queries { + match process.state { + ProcessState::Running => { + queries_to_register.push(process); + } + ProcessState::Finished => { + queries_to_deregister.push(process); + } + _ => {} + } + } + + self.kv_backend + .batch_put(BatchPutRequest { + kvs: queries_to_register + .iter() + .map(|p| KeyValue::try_from(&p.process)) + .collect::>() + .context(error::ReportProcessSnafu)?, + prev_kv: false, + }) + .await + .context(error::ReportProcessSnafu)?; + + for registered in queries_to_register { + // for running queries, it can either be still in running state or removed. + if let Some(process) = self + .queries + .lock() + .unwrap() + .get_mut(®istered.process.key.id) + { + debug_assert_eq!(process.state, ProcessState::Running); + process.state = ProcessState::Registered; + } else { + // Process can also be removed by `deregister_query` + queries_to_deregister.push(registered); + } + } + + self.kv_backend + .batch_delete(BatchDeleteRequest { + keys: queries_to_deregister + .iter() + .map(|p| p.process.key.to_bytes()) + .collect(), + prev_kv: false, + }) + .await + .context(error::ReportProcessSnafu)?; + for deregistered in queries_to_deregister { + if let Some(process) = self + .queries + .lock() + .unwrap() + .remove(&deregistered.process.key.id) + { + debug!("Deregistered process {}", process.process.key); + } + } + + Ok(()) } - #[cfg(test)] - async fn dump(&self) -> error::Result> { - use futures_util::TryStreamExt; - self.list_all_processes()?.into_stream().try_collect().await + fn name(&self) -> &str { + "ProcessManagerReportTask" } } @@ -115,26 +240,28 @@ mod tests { #[tokio::test] async fn test_register_query() { let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let process_manager = + ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); let process_id = process_manager .register_query("public".to_string(), "SELECT * FROM table".to_string()) .await .unwrap(); - let running_processes = process_manager.dump().await.unwrap(); + let running_processes = process_manager.list_all_processes().unwrap(); assert_eq!(running_processes.len(), 1); assert_eq!(running_processes[0].key.frontend_ip, "127.0.0.1:8000"); assert_eq!(running_processes[0].key.id, process_id); assert_eq!(running_processes[0].value.query, "SELECT * FROM table"); process_manager.deregister_query(process_id).await.unwrap(); - assert_eq!(process_manager.dump().await.unwrap().len(), 0); + assert_eq!(process_manager.list_all_processes().unwrap().len(), 0); } #[tokio::test] async fn test_register_multiple_queries() { let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let process_manager = + ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); // Register multiple queries let id1 = process_manager @@ -151,11 +278,11 @@ mod tests { .unwrap(); // Verify all are registered - assert_eq!(process_manager.dump().await.unwrap().len(), 3); + assert_eq!(process_manager.list_all_processes().unwrap().len(), 3); // Deregister middle one process_manager.deregister_query(id2).await.unwrap(); - let processes = process_manager.dump().await.unwrap(); + let processes = process_manager.list_all_processes().unwrap(); assert_eq!(processes.len(), 2); assert!(processes.iter().any(|p| p.key.id == id1)); assert!(processes.iter().any(|p| p.key.id == id3)); @@ -164,7 +291,8 @@ mod tests { #[tokio::test] async fn test_deregister_nonexistent_query() { let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let process_manager = + ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); // Try to deregister non-existent ID let result = process_manager.deregister_query(999).await; @@ -174,7 +302,8 @@ mod tests { #[tokio::test] async fn test_process_timestamps() { let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let process_manager = + ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); let before = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -191,7 +320,7 @@ mod tests { .unwrap() .as_millis() as i64; - let running_processes = process_manager.dump().await.unwrap(); + let running_processes = process_manager.list_all_processes().unwrap(); let process = &running_processes[0]; assert!(process.value.start_timestamp_ms >= before); @@ -203,8 +332,8 @@ mod tests { let kv_client = Arc::new(MemoryKvBackend::new()); // Create two process managers with different frontend addresses - let pm1 = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); - let pm2 = ProcessManager::new("127.0.0.1:8001".to_string(), kv_client.clone()); + let pm1 = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); + let pm2 = ProcessManager::new("127.0.0.1:8001".to_string(), kv_client.clone()).unwrap(); let id1 = pm1 .register_query("public".to_string(), "SELECT 1".to_string()) @@ -216,13 +345,13 @@ mod tests { .unwrap(); // Verify both processes are registered with correct frontend IPs - let pm1_processes = pm1.dump().await.unwrap(); + let pm1_processes = pm1.list_all_processes().unwrap(); assert_eq!(pm1_processes.len(), 1); let p1 = pm1_processes.iter().find(|p| p.key.id == id1).unwrap(); assert_eq!(p1.key.frontend_ip, "127.0.0.1:8000"); - let p2_processes = pm2.dump().await.unwrap(); + let p2_processes = pm2.list_all_processes().unwrap(); assert_eq!(p2_processes.len(), 1); let p2 = p2_processes.iter().find(|p| p.key.id == id2).unwrap(); assert_eq!(p2.key.frontend_ip, "127.0.0.1:8001"); @@ -230,24 +359,26 @@ mod tests { // deregister all queries on instance 1 pm1.deregister_all_queries().await.unwrap(); - let processes: Vec<_> = pm1.dump().await.unwrap(); + let processes: Vec<_> = pm1.list_all_processes().unwrap(); assert_eq!(processes.len(), 0); - assert_eq!(pm2.dump().await.unwrap().len(), 1); + assert_eq!(pm2.list_all_processes().unwrap().len(), 1); } #[tokio::test] async fn test_list_empty_processes() { let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let process_manager = + ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); - assert!(process_manager.dump().await.unwrap().is_empty()); + assert!(process_manager.list_all_processes().unwrap().is_empty()); } #[tokio::test] async fn test_deregister_all_queries() { let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()); + let process_manager = + ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); // Register multiple queries process_manager @@ -264,12 +395,12 @@ mod tests { .unwrap(); // Verify they exist - assert_eq!(process_manager.dump().await.unwrap().len(), 3); + assert_eq!(process_manager.list_all_processes().unwrap().len(), 3); // Deregister all process_manager.deregister_all_queries().await.unwrap(); // Verify none remain - assert_eq!(process_manager.dump().await.unwrap().len(), 0); + assert_eq!(process_manager.list_all_processes().unwrap().len(), 0); } } diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index fd113d5999b2..865c8c5c984e 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -21,7 +21,7 @@ use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; use common_time::util::current_time_millis; use common_time::{Duration, Timestamp}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter; -use datatypes::prelude::ConcreteDataType; +use datatypes::prelude::ConcreteDataType as CDT; use datatypes::scalars::ScalarVectorBuilder; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::value::Value; @@ -29,7 +29,6 @@ use datatypes::vectors::{ DurationMillisecondVectorBuilder, StringVectorBuilder, TimestampMillisecondVectorBuilder, UInt64VectorBuilder, VectorRef, }; -use futures::StreamExt; use snafu::ResultExt; use store_api::storage::{ScanRequest, TableId}; @@ -62,19 +61,15 @@ impl InformationSchemaProcessList { fn schema() -> SchemaRef { Arc::new(Schema::new(vec![ - ColumnSchema::new(ID, ConcreteDataType::uint64_datatype(), false), - ColumnSchema::new(DATABASE, ConcreteDataType::string_datatype(), false), - ColumnSchema::new(QUERY, ConcreteDataType::string_datatype(), false), + ColumnSchema::new(ID, CDT::uint64_datatype(), false), + ColumnSchema::new(DATABASE, CDT::string_datatype(), false), + ColumnSchema::new(QUERY, CDT::string_datatype(), false), ColumnSchema::new( START_TIMESTAMP, - ConcreteDataType::timestamp_millisecond_datatype(), - false, - ), - ColumnSchema::new( - ELAPSED_TIME, - ConcreteDataType::duration_millisecond_datatype(), + CDT::timestamp_millisecond_datatype(), false, ), + ColumnSchema::new(ELAPSED_TIME, CDT::duration_millisecond_datatype(), false), ])) } } @@ -93,10 +88,9 @@ impl InformationTable for InformationSchemaProcessList { } fn to_stream(&self, request: ScanRequest) -> error::Result { - let schema = self.schema.arrow_schema().clone(); let process_manager = self.process_manager.clone(); let stream = Box::pin(DfRecordBatchStreamAdapter::new( - schema, + self.schema.arrow_schema().clone(), futures::stream::once(async move { make_process_list(process_manager, request) .await @@ -120,25 +114,15 @@ async fn make_process_list( ) -> error::Result { let predicates = Predicates::from_scan_request(&Some(request)); let current_time = current_time_millis(); - let mut stream = Box::pin( - process_manager - .list_all_processes() - .context(error::ListProcessSnafu)? - .into_stream(), - ); - - let mut id_builder = UInt64VectorBuilder::with_capacity(8); - let mut database_builder = StringVectorBuilder::with_capacity(8); - let mut query_builder = StringVectorBuilder::with_capacity(8); - let mut start_time_builder = TimestampMillisecondVectorBuilder::with_capacity(8); - let mut elapsed_time_builder = DurationMillisecondVectorBuilder::with_capacity(8); - - while let Some(process) = stream - .next() - .await - .transpose() - .context(error::ListProcessSnafu)? - { + let queries = process_manager.list_all_processes()?; + + let mut id_builder = UInt64VectorBuilder::with_capacity(queries.len()); + let mut database_builder = StringVectorBuilder::with_capacity(queries.len()); + let mut query_builder = StringVectorBuilder::with_capacity(queries.len()); + let mut start_time_builder = TimestampMillisecondVectorBuilder::with_capacity(queries.len()); + let mut elapsed_time_builder = DurationMillisecondVectorBuilder::with_capacity(queries.len()); + + for process in queries { let row = [ (ID, &Value::from(process.query_id())), (DATABASE, &Value::from(process.database())), @@ -165,14 +149,15 @@ async fn make_process_list( } } - let columns: Vec = vec![ - Arc::new(id_builder.finish()), - Arc::new(database_builder.finish()), - Arc::new(query_builder.finish()), - Arc::new(start_time_builder.finish()), - Arc::new(elapsed_time_builder.finish()), - ]; - - RecordBatch::new(InformationSchemaProcessList::schema(), columns) - .context(error::CreateRecordBatchSnafu) + RecordBatch::new( + InformationSchemaProcessList::schema(), + vec![ + Arc::new(id_builder.finish()), + Arc::new(database_builder.finish()), + Arc::new(query_builder.finish()), + Arc::new(start_time_builder.finish()), + Arc::new(elapsed_time_builder.finish()), + ], + ) + .context(error::CreateRecordBatchSnafu) } diff --git a/src/cmd/src/error.rs b/src/cmd/src/error.rs index a671290503e7..6db5942cdbb7 100644 --- a/src/cmd/src/error.rs +++ b/src/cmd/src/error.rs @@ -305,6 +305,13 @@ pub enum Error { location: Location, source: common_meta::error::Error, }, + + #[snafu(display("Failed to build ProcessManager"))] + BuildProcessManager { + #[snafu(implicit)] + location: Location, + source: catalog::error::Error, + }, } pub type Result = std::result::Result; @@ -361,6 +368,7 @@ impl ErrorExt for Error { } Error::MetaClientInit { source, .. } => source.status_code(), Error::SchemaNotFound { .. } => StatusCode::DatabaseNotFound, + Error::BuildProcessManager { source, .. } => source.status_code(), } } diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index 38b26c5da882..478b9f1a3e6d 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -333,10 +333,10 @@ impl StartCommand { let information_extension = Arc::new(DistributedInformationExtension::new(meta_client.clone())); - let process_manager = Arc::new(ProcessManager::new( - opts.grpc.server_addr.clone(), - cached_meta_backend.clone(), - )); + let process_manager = Arc::new( + ProcessManager::new(opts.grpc.server_addr.clone(), cached_meta_backend.clone()) + .context(error::BuildProcessManagerSnafu)?, + ); let catalog_manager = KvBackendCatalogManager::new( information_extension, cached_meta_backend.clone(), diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 969ab1811c33..b184a4d916e8 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -510,10 +510,10 @@ impl StartCommand { procedure_manager.clone(), )); - let process_manager = Arc::new(ProcessManager::new( - opts.grpc.server_addr.clone(), - kv_backend.clone(), - )); + let process_manager = Arc::new( + ProcessManager::new(opts.grpc.server_addr.clone(), kv_backend.clone()) + .context(error::BuildProcessManagerSnafu)?, + ); let catalog_manager = KvBackendCatalogManager::new( information_extension.clone(), kv_backend.clone(), diff --git a/src/common/meta/src/key/process_list.rs b/src/common/meta/src/key/process_list.rs index 6ef0368502f4..d13c9c0c6ed4 100644 --- a/src/common/meta/src/key/process_list.rs +++ b/src/common/meta/src/key/process_list.rs @@ -21,7 +21,8 @@ use serde::{Deserialize, Serialize}; use snafu::OptionExt; use crate::error; -use crate::key::{MetadataKey, PROCESS_LIST_PATTERN, PROCESS_LIST_PREFIX}; +use crate::key::{MetadataKey, MetadataValue, PROCESS_LIST_PATTERN, PROCESS_LIST_PREFIX}; +use crate::rpc::KeyValue; /// Key for running queries tracked in metasrv. /// Layout: `__process/{frontend server addr}-{query id}` @@ -87,11 +88,23 @@ pub struct ProcessValue { pub const PROCESS_ID_SEQ: &str = "process_id_seq"; /// Running process instance. +#[derive(Clone, Debug)] pub struct Process { pub key: ProcessKey, pub value: ProcessValue, } +impl TryFrom<&Process> for KeyValue { + type Error = error::Error; + + fn try_from(process: &Process) -> Result { + Ok(KeyValue { + key: process.key.to_bytes(), + value: process.value.try_as_raw_value()?, + }) + } +} + impl Process { /// Returns server address of running process. pub fn server_addr(&self) -> &str { From caadc7bbf854ff5f00794740153bc27164372663 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Mon, 14 Apr 2025 13:41:18 +0000 Subject: [PATCH 15/31] chore: remove register --- src/catalog/src/error.rs | 8 ------ .../information_schema/process_list.rs | 10 +++---- src/cli/src/repl.rs | 0 src/frontend/src/instance.rs | 26 +------------------ 4 files changed, 6 insertions(+), 38 deletions(-) delete mode 100644 src/cli/src/repl.rs diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs index ae005d960db5..212889ba0890 100644 --- a/src/catalog/src/error.rs +++ b/src/catalog/src/error.rs @@ -285,13 +285,6 @@ pub enum Error { location: Location, }, - #[snafu(display("Failed to list running processes"))] - ListProcess { - source: common_meta::error::Error, - #[snafu(implicit)] - location: Location, - }, - #[snafu(display("Failed to start process report task"))] StartReportTask { source: common_runtime::error::Error, @@ -373,7 +366,6 @@ impl ErrorExt for Error { Error::GetViewCache { source, .. } | Error::GetTableCache { source, .. } | Error::BumpSequence { source, .. } - | Error::ListProcess { source, .. } | Error::ReportProcess { source, .. } => source.status_code(), Error::StartReportTask { .. } => StatusCode::Internal, } diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 865c8c5c984e..f546209dbab9 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -152,11 +152,11 @@ async fn make_process_list( RecordBatch::new( InformationSchemaProcessList::schema(), vec![ - Arc::new(id_builder.finish()), - Arc::new(database_builder.finish()), - Arc::new(query_builder.finish()), - Arc::new(start_time_builder.finish()), - Arc::new(elapsed_time_builder.finish()), + Arc::new(id_builder.finish()) as VectorRef, + Arc::new(database_builder.finish()) as VectorRef, + Arc::new(query_builder.finish()) as VectorRef, + Arc::new(start_time_builder.finish()) as VectorRef, + Arc::new(elapsed_time_builder.finish()) as VectorRef, ], ) .context(error::CreateRecordBatchSnafu) diff --git a/src/cli/src/repl.rs b/src/cli/src/repl.rs deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index c5741b6a7d80..d744e82503e4 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -97,6 +97,7 @@ pub struct Instance { table_metadata_manager: TableMetadataManagerRef, stats: StatementStatistics, limiter: Option, + #[allow(dead_code)] process_manager: Option>, } @@ -172,27 +173,6 @@ impl Instance { .stats .start_slow_query_timer(QueryStatement::Sql(stmt.clone())); - let query_finish_notifier = if let Some(process_manager) = &self.process_manager { - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - let process_manager = process_manager.clone(); - let database = query_ctx.current_schema(); - let query = stmt.to_string(); - common_runtime::spawn_global(async move { - match process_manager.register_query(database, query).await { - Ok(id) => { - let _ = rx.await; - process_manager.deregister_query(id).await.unwrap(); - } - Err(e) => { - error!(e; "Failed to register query id"); - } - } - }); - Some(tx) - } else { - None - }; - let output = match stmt { Statement::Query(_) | Statement::Explain(_) | Statement::Delete(_) => { // TODO: remove this when format is supported in datafusion @@ -235,10 +215,6 @@ impl Instance { self.statement_executor.execute_sql(stmt, query_ctx).await } }; - - if let Some(query_finish_notifier) = query_finish_notifier { - let _ = query_finish_notifier.send(()); - } output.context(TableOperationSnafu) } } From 5c70e104ebf579e0cb6ce920c779284a6a886e89 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Mon, 14 Apr 2025 15:36:13 +0000 Subject: [PATCH 16/31] fix: sqlness tests --- .../common/show/show_databases_tables.result | 3 + .../common/system/information_schema.result | 780 +++++++++--------- .../standalone/common/view/create.result | 1 + 3 files changed, 397 insertions(+), 387 deletions(-) diff --git a/tests/cases/standalone/common/show/show_databases_tables.result b/tests/cases/standalone/common/show/show_databases_tables.result index 6dc267b61bac..b4985aa80ad4 100644 --- a/tests/cases/standalone/common/show/show_databases_tables.result +++ b/tests/cases/standalone/common/show/show_databases_tables.result @@ -46,6 +46,7 @@ SHOW TABLES; | parameters | | partitions | | procedure_info | +| process_list | | profiling | | referential_constraints | | region_peers | @@ -94,6 +95,7 @@ SHOW FULL TABLES; | parameters | LOCAL TEMPORARY | | partitions | LOCAL TEMPORARY | | procedure_info | LOCAL TEMPORARY | +| process_list | LOCAL TEMPORARY | | profiling | LOCAL TEMPORARY | | referential_constraints | LOCAL TEMPORARY | | region_peers | LOCAL TEMPORARY | @@ -136,6 +138,7 @@ SHOW TABLE STATUS; |parameters||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0||| |partitions||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0||| |procedure_info||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0||| +|process_list||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0||| |profiling||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0||| |referential_constraints||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0||| |region_peers||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0||| diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index bfcf39b69e18..becde708a268 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -33,6 +33,7 @@ order by table_schema, table_name; |greptime|information_schema|parameters|LOCALTEMPORARY|18|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y| |greptime|information_schema|partitions|LOCALTEMPORARY|28|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y| |greptime|information_schema|procedure_info|LOCALTEMPORARY|34|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y| +|greptime|information_schema|process_list|LOCALTEMPORARY|36|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y| |greptime|information_schema|profiling|LOCALTEMPORARY|19|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y| |greptime|information_schema|referential_constraints|LOCALTEMPORARY|20|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y| |greptime|information_schema|region_peers|LOCALTEMPORARY|29|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y| @@ -52,393 +53,398 @@ order by table_schema, table_name; select * from information_schema.columns order by table_schema, table_name, column_name; -+---------------+--------------------+---------------------------------------+-----------------------------------+------------------+--------------------------+------------------------+-------------------+---------------+--------------------+--------------------+----------------+------------+-------+---------------+-----------------------+----------------------+-----------------+---------------+----------------+-------------+-----------------+----------------+--------+ -| table_catalog | table_schema | table_name | column_name | ordinal_position | character_maximum_length | character_octet_length | numeric_precision | numeric_scale | datetime_precision | character_set_name | collation_name | column_key | extra | privileges | generation_expression | greptime_data_type | data_type | semantic_type | column_default | is_nullable | column_type | column_comment | srs_id | -+---------------+--------------------+---------------------------------------+-----------------------------------+------------------+--------------------------+------------------------+-------------------+---------------+--------------------+--------------------+----------------+------------+-------+---------------+-----------------------+----------------------+-----------------+---------------+----------------+-------------+-----------------+----------------+--------+ -| greptime | information_schema | build_info | git_branch | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | build_info | git_clean | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | build_info | git_commit | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | build_info | git_commit_short | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | build_info | pkg_version | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | character_sets | character_set_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | character_sets | default_collate_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | character_sets | description | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | character_sets | maxlen | 4 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | check_constraints | check_clause | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | check_constraints | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | check_constraints | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | check_constraints | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | cluster_info | active_time | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | cluster_info | git_commit | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | cluster_info | peer_addr | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | cluster_info | peer_id | 1 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | cluster_info | peer_type | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | cluster_info | start_time | 6 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | -| greptime | information_schema | cluster_info | uptime | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | cluster_info | version | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | collation_character_set_applicability | character_set_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | collation_character_set_applicability | collation_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | collations | character_set_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | collations | collation_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | collations | id | 3 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | collations | is_compiled | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | collations | is_default | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | collations | sortlen | 6 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | column_privileges | column_name | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_privileges | grantee | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_privileges | is_grantable | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_privileges | privilege_type | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_privileges | table_catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_privileges | table_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_privileges | table_schema | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_statistics | column_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_statistics | histogram | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_statistics | schema_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | column_statistics | table_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | character_maximum_length | 6 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | columns | character_octet_length | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | columns | character_set_name | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | columns | collation_name | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | columns | column_comment | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | columns | column_default | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | columns | column_key | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | column_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | column_type | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | data_type | 18 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | datetime_precision | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | columns | extra | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | generation_expression | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | greptime_data_type | 17 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | is_nullable | 21 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | numeric_precision | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | columns | numeric_scale | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | columns | ordinal_position | 5 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | columns | privileges | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | semantic_type | 19 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | srs_id | 24 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | columns | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | columns | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | engines | comment | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | engines | engine | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | engines | savepoints | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | engines | support | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | engines | transactions | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | engines | xa | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | character_set_client | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | collation_connection | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | created | 17 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | events | database_collation | 24 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | definer | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | ends | 14 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | events | event_body | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | event_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | event_comment | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | event_definition | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | event_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | event_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | event_type | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | execute_at | 9 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | events | interval_field | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | interval_value | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | events | last_altered | 18 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | events | last_executed | 19 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | events | on_completion | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | originator | 21 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | events | sql_mode | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | starts | 13 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | events | status | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | events | time_zone | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | autoextend_size | 19 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | avg_row_length | 28 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | check_time | 35 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | files | checksum | 36 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | create_time | 33 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | files | creation_time | 20 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | files | data_free | 32 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | data_length | 29 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | deleted_rows | 12 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | engine | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | extent_size | 16 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | extra | 38 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | file_id | 1 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | file_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | file_type | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | free_extents | 14 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | fulltext_keys | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | index_length | 31 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | initial_size | 17 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | last_access_time | 22 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | files | last_update_time | 21 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | files | logfile_group_name | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | logfile_group_number | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | max_data_length | 30 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | maximum_size | 18 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | recover_time | 23 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | files | row_format | 26 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | status | 37 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | table_catalog | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | table_name | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | table_rows | 27 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | table_schema | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | tablespace_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | files | total_extents | 15 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | transaction_counter | 24 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | update_count | 13 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | files | update_time | 34 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | files | version | 25 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | flows | comment | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | flows | created_time | 12 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | No | timestamp(3) | | | -| greptime | information_schema | flows | expire_after | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | flows | flow_definition | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | flows | flow_id | 2 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | -| greptime | information_schema | flows | flow_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | flows | flownode_ids | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | flows | last_execution_time | 14 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | -| greptime | information_schema | flows | options | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | flows | sink_table_name | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | flows | source_table_ids | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | flows | source_table_names | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | flows | state_size | 3 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | flows | table_catalog | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | flows | updated_time | 13 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | No | timestamp(3) | | | -| greptime | information_schema | global_status | variable_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | global_status | variable_value | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | key_column_usage | column_name | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | key_column_usage | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | key_column_usage | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | key_column_usage | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | key_column_usage | ordinal_position | 9 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | -| greptime | information_schema | key_column_usage | position_in_unique_constraint | 10 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | Yes | int unsigned | | | -| greptime | information_schema | key_column_usage | real_table_catalog | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | key_column_usage | referenced_column_name | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | key_column_usage | referenced_table_name | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | key_column_usage | referenced_table_schema | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | key_column_usage | table_catalog | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | key_column_usage | table_name | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | key_column_usage | table_schema | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | optimizer_trace | insufficient_privileges | 4 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | optimizer_trace | missing_bytes_beyond_max_mem_size | 3 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | optimizer_trace | query | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | optimizer_trace | trace | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | character_maximum_length | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | parameters | character_octet_length | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | parameters | character_set_name | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | collation_name | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | data_type | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | datetime_precision | 12 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | parameters | dtd_identifier | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | numeric_precision | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | parameters | numeric_scale | 11 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | parameters | ordinal_position | 4 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | parameters | parameter_mode | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | parameter_name | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | routine_type | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | specific_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | specific_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | parameters | specific_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | partitions | avg_row_length | 14 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | partitions | check_time | 21 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | -| greptime | information_schema | partitions | checksum | 22 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | partitions | create_time | 19 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | -| greptime | information_schema | partitions | data_free | 18 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | partitions | data_length | 15 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | partitions | greptime_partition_id | 26 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | partitions | index_length | 17 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | partitions | max_data_length | 16 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | partitions | nodegroup | 24 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | partitions | partition_comment | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | partitions | partition_description | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | partitions | partition_expression | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | partitions | partition_method | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | partitions | partition_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | partitions | partition_ordinal_position | 6 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | partitions | subpartition_expression | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | partitions | subpartition_method | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | partitions | subpartition_name | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | partitions | subpartition_ordinal_position | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | partitions | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | partitions | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | partitions | table_rows | 13 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | partitions | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | partitions | tablespace_name | 25 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | partitions | update_time | 20 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | -| greptime | information_schema | procedure_info | end_time | 4 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | -| greptime | information_schema | procedure_info | lock_keys | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | procedure_info | procedure_id | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | procedure_info | procedure_type | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | procedure_info | start_time | 3 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | -| greptime | information_schema | procedure_info | status | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | profiling | block_ops_in | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | block_ops_out | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | context_involuntary | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | context_voluntary | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | cpu_system | 6 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | cpu_user | 5 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | duration | 4 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | messages_received | 12 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | messages_sent | 11 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | page_faults_major | 13 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | page_faults_minor | 14 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | query_id | 1 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | seq | 2 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | source_file | 17 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | profiling | source_function | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | profiling | source_line | 18 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | profiling | state | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | profiling | swaps | 15 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | referential_constraints | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | delete_rule | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | match_option | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | referenced_table_name | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | table_name | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | unique_constraint_catalog | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | unique_constraint_name | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | unique_constraint_schema | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | referential_constraints | update_rule | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | region_peers | down_seconds | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | -| greptime | information_schema | region_peers | is_leader | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | region_peers | peer_addr | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | region_peers | peer_id | 5 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | region_peers | region_id | 4 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | -| greptime | information_schema | region_peers | status | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | region_peers | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | region_peers | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | region_peers | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | region_statistics | disk_size | 5 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | region_statistics | engine | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | region_statistics | index_size | 9 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | region_statistics | manifest_size | 7 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | region_statistics | memtable_size | 6 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | region_statistics | region_id | 1 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | -| greptime | information_schema | region_statistics | region_number | 3 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | -| greptime | information_schema | region_statistics | region_role | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | region_statistics | region_rows | 4 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | region_statistics | sst_size | 8 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | region_statistics | table_id | 2 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | -| greptime | information_schema | routines | character_maximum_length | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | routines | character_octet_length | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | routines | character_set_client | 29 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | character_set_name | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | collation_connection | 30 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | collation_name | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | created | 24 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | routines | data_type | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | database_collation | 31 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | datetime_precision | 11 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | routines | definer | 28 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | dtd_identifier | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | external_language | 18 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | external_name | 17 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | is_deterministic | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | last_altered | 25 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | routines | numeric_precision | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | routines | numeric_scale | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | routines | parameter_style | 19 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | routine_body | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | routine_catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | routine_comment | 27 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | routine_definition | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | routine_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | routine_schema | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | routine_type | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | security_type | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | specific_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | sql_data_access | 21 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | sql_mode | 26 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | routines | sql_path | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | runtime_metrics | labels | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | runtime_metrics | metric_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | runtime_metrics | peer_addr | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | runtime_metrics | peer_type | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | runtime_metrics | timestamp | 6 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | No | timestamp(3) | | | -| greptime | information_schema | runtime_metrics | value | 2 | | | 22 | | | | | | | select,insert | | Float64 | double | FIELD | | No | double | | | -| greptime | information_schema | schema_privileges | grantee | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | schema_privileges | is_grantable | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | schema_privileges | privilege_type | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | schema_privileges | table_catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | schema_privileges | table_schema | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | schemata | catalog_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | schemata | default_character_set_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | schemata | default_collation_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | schemata | options | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | schemata | schema_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | schemata | sql_path | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | session_status | variable_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | session_status | variable_value | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_constraints | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_constraints | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_constraints | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_constraints | constraint_type | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_constraints | enforced | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_constraints | table_name | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_constraints | table_schema | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_privileges | grantee | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_privileges | is_grantable | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_privileges | privilege_type | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_privileges | table_catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_privileges | table_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | table_privileges | table_schema | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | tables | auto_increment | 16 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | tables | avg_row_length | 10 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | tables | check_time | 19 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | -| greptime | information_schema | tables | checksum | 21 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | tables | create_options | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | tables | create_time | 17 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | -| greptime | information_schema | tables | data_free | 15 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | tables | data_length | 6 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | tables | engine | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | tables | index_length | 8 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | tables | max_data_length | 7 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | tables | max_index_length | 9 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | tables | row_format | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | tables | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | tables | table_collation | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | tables | table_comment | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | tables | table_id | 5 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | Yes | int unsigned | | | -| greptime | information_schema | tables | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | tables | table_rows | 14 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | tables | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | tables | table_type | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | tables | temporary | 24 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | tables | update_time | 18 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | -| greptime | information_schema | tables | version | 12 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | -| greptime | information_schema | triggers | action_condition | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | action_order | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | -| greptime | information_schema | triggers | action_orientation | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | action_reference_new_row | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | action_reference_new_table | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | action_reference_old_row | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | action_reference_old_table | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | action_statement | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | action_timing | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | character_set_client | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | collation_connection | 21 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | created | 17 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | -| greptime | information_schema | triggers | database_collation | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | definer | 19 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | event_manipulation | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | event_object_catalog | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | event_object_schema | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | event_object_table | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | sql_mode | 18 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | trigger_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | trigger_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | triggers | trigger_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | views | character_set_client | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | views | check_option | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | views | collation_connection | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | views | definer | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | views | is_updatable | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | views | security_type | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | views | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | views | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | views | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | views | view_definition | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | public | numbers | number | 1 | | | 10 | 0 | | | | PRI | | select,insert | | UInt32 | int unsigned | TAG | | No | int unsigned | | | -+---------------+--------------------+---------------------------------------+-----------------------------------+------------------+--------------------------+------------------------+-------------------+---------------+--------------------+--------------------+----------------+------------+-------+---------------+-----------------------+----------------------+-----------------+---------------+----------------+-------------+-----------------+----------------+--------+ ++---------------+--------------------+---------------------------------------+-----------------------------------+------------------+--------------------------+------------------------+-------------------+---------------+--------------------+--------------------+----------------+------------+-------+---------------+-----------------------+----------------------+---------------------+---------------+----------------+-------------+---------------------+----------------+--------+ +| table_catalog | table_schema | table_name | column_name | ordinal_position | character_maximum_length | character_octet_length | numeric_precision | numeric_scale | datetime_precision | character_set_name | collation_name | column_key | extra | privileges | generation_expression | greptime_data_type | data_type | semantic_type | column_default | is_nullable | column_type | column_comment | srs_id | ++---------------+--------------------+---------------------------------------+-----------------------------------+------------------+--------------------------+------------------------+-------------------+---------------+--------------------+--------------------+----------------+------------+-------+---------------+-----------------------+----------------------+---------------------+---------------+----------------+-------------+---------------------+----------------+--------+ +| greptime | information_schema | build_info | git_branch | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | build_info | git_clean | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | build_info | git_commit | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | build_info | git_commit_short | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | build_info | pkg_version | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | character_sets | character_set_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | character_sets | default_collate_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | character_sets | description | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | character_sets | maxlen | 4 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | check_constraints | check_clause | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | check_constraints | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | check_constraints | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | check_constraints | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | cluster_info | active_time | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | cluster_info | git_commit | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | cluster_info | peer_addr | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | cluster_info | peer_id | 1 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | cluster_info | peer_type | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | cluster_info | start_time | 6 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | +| greptime | information_schema | cluster_info | uptime | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | cluster_info | version | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | collation_character_set_applicability | character_set_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | collation_character_set_applicability | collation_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | collations | character_set_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | collations | collation_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | collations | id | 3 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | collations | is_compiled | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | collations | is_default | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | collations | sortlen | 6 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | column_privileges | column_name | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_privileges | grantee | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_privileges | is_grantable | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_privileges | privilege_type | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_privileges | table_catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_privileges | table_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_privileges | table_schema | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_statistics | column_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_statistics | histogram | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_statistics | schema_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | column_statistics | table_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | character_maximum_length | 6 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | columns | character_octet_length | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | columns | character_set_name | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | columns | collation_name | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | columns | column_comment | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | columns | column_default | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | columns | column_key | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | column_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | column_type | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | data_type | 18 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | datetime_precision | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | columns | extra | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | generation_expression | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | greptime_data_type | 17 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | is_nullable | 21 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | numeric_precision | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | columns | numeric_scale | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | columns | ordinal_position | 5 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | columns | privileges | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | semantic_type | 19 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | srs_id | 24 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | columns | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | columns | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | engines | comment | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | engines | engine | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | engines | savepoints | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | engines | support | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | engines | transactions | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | engines | xa | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | character_set_client | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | collation_connection | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | created | 17 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | events | database_collation | 24 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | definer | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | ends | 14 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | events | event_body | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | event_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | event_comment | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | event_definition | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | event_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | event_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | event_type | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | execute_at | 9 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | events | interval_field | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | interval_value | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | events | last_altered | 18 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | events | last_executed | 19 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | events | on_completion | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | originator | 21 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | events | sql_mode | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | starts | 13 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | events | status | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | events | time_zone | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | autoextend_size | 19 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | avg_row_length | 28 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | check_time | 35 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | files | checksum | 36 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | create_time | 33 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | files | creation_time | 20 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | files | data_free | 32 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | data_length | 29 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | deleted_rows | 12 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | engine | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | extent_size | 16 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | extra | 38 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | file_id | 1 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | file_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | file_type | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | free_extents | 14 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | fulltext_keys | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | index_length | 31 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | initial_size | 17 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | last_access_time | 22 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | files | last_update_time | 21 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | files | logfile_group_name | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | logfile_group_number | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | max_data_length | 30 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | maximum_size | 18 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | recover_time | 23 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | files | row_format | 26 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | status | 37 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | table_catalog | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | table_name | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | table_rows | 27 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | table_schema | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | tablespace_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | files | total_extents | 15 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | transaction_counter | 24 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | update_count | 13 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | files | update_time | 34 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | files | version | 25 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | flows | comment | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | flows | created_time | 12 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | No | timestamp(3) | | | +| greptime | information_schema | flows | expire_after | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | flows | flow_definition | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | flows | flow_id | 2 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | +| greptime | information_schema | flows | flow_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | flows | flownode_ids | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | flows | last_execution_time | 14 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | +| greptime | information_schema | flows | options | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | flows | sink_table_name | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | flows | source_table_ids | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | flows | source_table_names | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | flows | state_size | 3 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | flows | table_catalog | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | flows | updated_time | 13 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | No | timestamp(3) | | | +| greptime | information_schema | global_status | variable_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | global_status | variable_value | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | key_column_usage | column_name | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | key_column_usage | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | key_column_usage | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | key_column_usage | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | key_column_usage | ordinal_position | 9 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | +| greptime | information_schema | key_column_usage | position_in_unique_constraint | 10 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | Yes | int unsigned | | | +| greptime | information_schema | key_column_usage | real_table_catalog | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | key_column_usage | referenced_column_name | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | key_column_usage | referenced_table_name | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | key_column_usage | referenced_table_schema | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | key_column_usage | table_catalog | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | key_column_usage | table_name | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | key_column_usage | table_schema | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | optimizer_trace | insufficient_privileges | 4 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | optimizer_trace | missing_bytes_beyond_max_mem_size | 3 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | optimizer_trace | query | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | optimizer_trace | trace | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | character_maximum_length | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | parameters | character_octet_length | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | parameters | character_set_name | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | collation_name | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | data_type | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | datetime_precision | 12 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | parameters | dtd_identifier | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | numeric_precision | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | parameters | numeric_scale | 11 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | parameters | ordinal_position | 4 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | parameters | parameter_mode | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | parameter_name | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | routine_type | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | specific_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | specific_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | parameters | specific_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | partitions | avg_row_length | 14 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | partitions | check_time | 21 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | +| greptime | information_schema | partitions | checksum | 22 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | partitions | create_time | 19 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | +| greptime | information_schema | partitions | data_free | 18 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | partitions | data_length | 15 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | partitions | greptime_partition_id | 26 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | partitions | index_length | 17 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | partitions | max_data_length | 16 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | partitions | nodegroup | 24 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | partitions | partition_comment | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | partitions | partition_description | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | partitions | partition_expression | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | partitions | partition_method | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | partitions | partition_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | partitions | partition_ordinal_position | 6 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | partitions | subpartition_expression | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | partitions | subpartition_method | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | partitions | subpartition_name | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | partitions | subpartition_ordinal_position | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | partitions | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | partitions | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | partitions | table_rows | 13 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | partitions | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | partitions | tablespace_name | 25 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | partitions | update_time | 20 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | +| greptime | information_schema | procedure_info | end_time | 4 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | +| greptime | information_schema | procedure_info | lock_keys | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | procedure_info | procedure_id | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | procedure_info | procedure_type | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | procedure_info | start_time | 3 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | +| greptime | information_schema | procedure_info | status | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | process_list | database | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | process_list | elapsed_time | 5 | | | | | | | | | | select,insert | | DurationMillisecond | DurationMillisecond | FIELD | | No | DurationMillisecond | | | +| greptime | information_schema | process_list | id | 1 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | +| greptime | information_schema | process_list | query | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | process_list | start_timestamp | 4 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | No | timestamp(3) | | | +| greptime | information_schema | profiling | block_ops_in | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | block_ops_out | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | context_involuntary | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | context_voluntary | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | cpu_system | 6 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | cpu_user | 5 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | duration | 4 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | messages_received | 12 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | messages_sent | 11 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | page_faults_major | 13 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | page_faults_minor | 14 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | query_id | 1 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | seq | 2 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | source_file | 17 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | profiling | source_function | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | profiling | source_line | 18 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | profiling | state | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | profiling | swaps | 15 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | referential_constraints | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | delete_rule | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | match_option | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | referenced_table_name | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | table_name | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | unique_constraint_catalog | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | unique_constraint_name | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | unique_constraint_schema | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | referential_constraints | update_rule | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | region_peers | down_seconds | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | Yes | bigint | | | +| greptime | information_schema | region_peers | is_leader | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | region_peers | peer_addr | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | region_peers | peer_id | 5 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | region_peers | region_id | 4 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | +| greptime | information_schema | region_peers | status | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | region_peers | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | region_peers | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | region_peers | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | region_statistics | disk_size | 5 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | region_statistics | engine | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | region_statistics | index_size | 9 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | region_statistics | manifest_size | 7 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | region_statistics | memtable_size | 6 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | region_statistics | region_id | 1 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | +| greptime | information_schema | region_statistics | region_number | 3 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | +| greptime | information_schema | region_statistics | region_role | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | region_statistics | region_rows | 4 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | region_statistics | sst_size | 8 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | region_statistics | table_id | 2 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | +| greptime | information_schema | routines | character_maximum_length | 7 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | routines | character_octet_length | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | routines | character_set_client | 29 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | character_set_name | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | collation_connection | 30 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | collation_name | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | created | 24 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | routines | data_type | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | database_collation | 31 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | datetime_precision | 11 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | routines | definer | 28 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | dtd_identifier | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | external_language | 18 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | external_name | 17 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | is_deterministic | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | last_altered | 25 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | routines | numeric_precision | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | routines | numeric_scale | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | routines | parameter_style | 19 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | routine_body | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | routine_catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | routine_comment | 27 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | routine_definition | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | routine_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | routine_schema | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | routine_type | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | security_type | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | specific_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | sql_data_access | 21 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | sql_mode | 26 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | routines | sql_path | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | runtime_metrics | labels | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | runtime_metrics | metric_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | runtime_metrics | peer_addr | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | runtime_metrics | peer_type | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | runtime_metrics | timestamp | 6 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | No | timestamp(3) | | | +| greptime | information_schema | runtime_metrics | value | 2 | | | 22 | | | | | | | select,insert | | Float64 | double | FIELD | | No | double | | | +| greptime | information_schema | schema_privileges | grantee | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schema_privileges | is_grantable | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schema_privileges | privilege_type | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schema_privileges | table_catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schema_privileges | table_schema | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schemata | catalog_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schemata | default_character_set_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schemata | default_collation_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schemata | options | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | schemata | schema_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schemata | sql_path | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | session_status | variable_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | session_status | variable_value | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_constraints | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_constraints | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_constraints | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_constraints | constraint_type | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_constraints | enforced | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_constraints | table_name | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_constraints | table_schema | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_privileges | grantee | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_privileges | is_grantable | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_privileges | privilege_type | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_privileges | table_catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_privileges | table_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | table_privileges | table_schema | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | tables | auto_increment | 16 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | tables | avg_row_length | 10 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | tables | check_time | 19 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | +| greptime | information_schema | tables | checksum | 21 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | tables | create_options | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | tables | create_time | 17 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | +| greptime | information_schema | tables | data_free | 15 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | tables | data_length | 6 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | tables | engine | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | tables | index_length | 8 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | tables | max_data_length | 7 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | tables | max_index_length | 9 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | tables | row_format | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | tables | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | tables | table_collation | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | tables | table_comment | 23 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | tables | table_id | 5 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | Yes | int unsigned | | | +| greptime | information_schema | tables | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | tables | table_rows | 14 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | tables | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | tables | table_type | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | tables | temporary | 24 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | tables | update_time | 18 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | Yes | timestamp(6) | | | +| greptime | information_schema | tables | version | 12 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | Yes | bigint unsigned | | | +| greptime | information_schema | triggers | action_condition | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | action_order | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | +| greptime | information_schema | triggers | action_orientation | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | action_reference_new_row | 16 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | action_reference_new_table | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | action_reference_old_row | 15 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | action_reference_old_table | 13 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | action_statement | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | action_timing | 12 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | character_set_client | 20 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | collation_connection | 21 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | created | 17 | | | | | 6 | | | | | select,insert | | TimestampMicrosecond | timestamp(6) | FIELD | | No | timestamp(6) | | | +| greptime | information_schema | triggers | database_collation | 22 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | definer | 19 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | event_manipulation | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | event_object_catalog | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | event_object_schema | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | event_object_table | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | sql_mode | 18 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | trigger_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | trigger_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | triggers | trigger_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | views | character_set_client | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | views | check_option | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | views | collation_connection | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | views | definer | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | views | is_updatable | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | views | security_type | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | views | table_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | views | table_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | views | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | views | view_definition | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | public | numbers | number | 1 | | | 10 | 0 | | | | PRI | | select,insert | | UInt32 | int unsigned | TAG | | No | int unsigned | | | ++---------------+--------------------+---------------------------------------+-----------------------------------+------------------+--------------------------+------------------------+-------------------+---------------+--------------------+--------------------+----------------+------------+-------+---------------+-----------------------+----------------------+---------------------+---------------+----------------+-------------+---------------------+----------------+--------+ select table_schema, table_name from information_schema.tables order by table_name limit 5; diff --git a/tests/cases/standalone/common/view/create.result b/tests/cases/standalone/common/view/create.result index e7651bb8adbc..fd26c6a63a46 100644 --- a/tests/cases/standalone/common/view/create.result +++ b/tests/cases/standalone/common/view/create.result @@ -107,6 +107,7 @@ SELECT * FROM INFORMATION_SCHEMA.TABLES ORDER BY TABLE_NAME, TABLE_TYPE; |greptime|information_schema|parameters|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y| |greptime|information_schema|partitions|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y| |greptime|information_schema|procedure_info|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y| +|greptime|information_schema|process_list|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y| |greptime|information_schema|profiling|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y| |greptime|information_schema|referential_constraints|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y| |greptime|information_schema|region_peers|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y| From 0b3fd381490f389531d739795740294b560e264e Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Mon, 9 Jun 2025 08:41:52 +0000 Subject: [PATCH 17/31] merge main Signed-off-by: Lei, HUANG --- Cargo.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31c54355486e..0b79e465be92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4511,9 +4511,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", @@ -5146,9 +5146,9 @@ dependencies = [ [[package]] name = "grok" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "273797968160270573071022613fc4aa28b91fe68f3eef6c96a1b2a1947ddfbd" +checksum = "6c52724b609896f661a3f4641dd3a44dc602958ef615857c12d00756b4e9355b" dependencies = [ "glob", "onig", @@ -6716,9 +6716,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" dependencies = [ "zlib-rs", ] @@ -8885,9 +8885,9 @@ dependencies = [ [[package]] name = "pgwire" -version = "0.30.1" +version = "0.30.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec79ee18e6cafde8698885646780b967ecc905120798b8359dd0da64f9688e89" +checksum = "4ca6c26b25be998208a13ff2f0c55b567363f34675410e6d6f1c513a150583fd" dependencies = [ "async-trait", "bytes", @@ -9664,9 +9664,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.112" +version = "2.1.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c6b4c497a0c6bfb466f75167c728b1a861b0cdc39de9c35b877208a270a9590" +checksum = "d0e49aa528239f2ca13ad87387977c208e59c3fb8c437609f95f1b3898ec6ef1" dependencies = [ "psl-types", ] @@ -14701,9 +14701,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" [[package]] name = "zstd" From 75a15aa503d797952662f7659978f93523c47daf Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Tue, 10 Jun 2025 07:54:11 +0000 Subject: [PATCH 18/31] feat/show-process-list: - **Update `greptime-proto` Dependency**: Updated the `greptime-proto` dependency in `Cargo.lock` and `Cargo.toml` to a new revision. - **Refactor `ProcessManager`**: Simplified the `ProcessManager` implementation by removing the use of `KvBackendRef` and `SequenceRef`, and replaced them with `AtomicU64` and `RwLock` for managing process IDs and catalogs in `process_manager.rs`. - **Remove Process List Metadata**: Deleted the `process_list.rs` file and removed related metadata key definitions in `key.rs`. - **Update Process List Logic**: Modified the process list logic in `process_list.rs` to use the new `ProcessManager` structure. - **Adjust Frontend and Standalone Start Commands**: Updated `frontend.rs` and `standalone.rs` to use the new `ProcessManager` constructor. Signed-off-by: Lei, HUANG --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/catalog/src/process_manager.rs | 397 +++--------------- .../information_schema/process_list.rs | 55 +-- src/cmd/src/frontend.rs | 8 +- src/cmd/src/standalone.rs | 4 +- src/common/meta/src/key.rs | 14 +- src/common/meta/src/key/process_list.rs | 222 ---------- 8 files changed, 97 insertions(+), 607 deletions(-) delete mode 100644 src/common/meta/src/key/process_list.rs diff --git a/Cargo.lock b/Cargo.lock index 0b79e465be92..10f9cd190918 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5133,7 +5133,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=454c52634c3bac27de10bf0d85d5533eed1cf03f#454c52634c3bac27de10bf0d85d5533eed1cf03f" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=d05ff47433bf5249ae7eda1fb5c591dc8d2c75b1#d05ff47433bf5249ae7eda1fb5c591dc8d2c75b1" dependencies = [ "prost 0.13.5", "serde", diff --git a/Cargo.toml b/Cargo.toml index 500f7e6eed00..2d32c1661ed0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,7 +133,7 @@ etcd-client = "0.14" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "454c52634c3bac27de10bf0d85d5533eed1cf03f" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "d05ff47433bf5249ae7eda1fb5c591dc8d2c75b1" } hex = "0.4" http = "1" humantime = "2.1" diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index 2ffcd1c2c14b..c9658cc3ff73 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -14,15 +14,13 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::RwLock; -use common_meta::key::process_list::{Process, ProcessKey, ProcessValue, PROCESS_ID_SEQ}; -use common_meta::key::{MetadataKey, PROCESS_LIST_PREFIX}; +use api::v1::frontend::ProcessInfo; use common_meta::kv_backend::KvBackendRef; use common_meta::rpc::store::{BatchDeleteRequest, BatchPutRequest, DeleteRangeRequest}; use common_meta::rpc::KeyValue; -use common_meta::sequence::{SequenceBuilder, SequenceRef}; use common_runtime::{RepeatedTask, TaskFunction}; use common_telemetry::{debug, info}; use common_time::util::current_time_millis; @@ -32,375 +30,96 @@ use crate::error; pub struct ProcessManager { server_addr: String, - sequencer: SequenceRef, - kv_client: KvBackendRef, - _report_task: RepeatedTask, - pending_queries: Arc>>, + next_id: AtomicU64, + catalogs: RwLock>>, } impl ProcessManager { /// Create a [ProcessManager] instance with server address and kv client. - pub fn new(server_addr: String, kv_client: KvBackendRef) -> error::Result { - let sequencer = Arc::new( - SequenceBuilder::new(PROCESS_ID_SEQ, kv_client.clone()) - .initial(0) - .step(10) - .build(), - ); - - let pending_queries = Arc::new(Mutex::new(HashMap::new())); - let pending_queries_clone = pending_queries.clone(); - let report_task = RepeatedTask::new( - Duration::from_secs(5), - Box::new(ReportTask { - queries: pending_queries_clone, - kv_backend: kv_client.clone(), - }), - ); - report_task - .start(common_runtime::global_runtime()) - .context(error::StartReportTaskSnafu)?; + pub fn new(server_addr: String) -> error::Result { Ok(Self { server_addr, - sequencer, - kv_client, - _report_task: report_task, - pending_queries, + next_id: Default::default(), + catalogs: Default::default(), }) } /// Registers a submitted query. - pub async fn register_query(&self, database: String, query: String) -> error::Result { - let process_id = self - .sequencer - .next() - .await - .context(error::BumpSequenceSnafu)?; - let key = ProcessKey { - frontend_ip: self.server_addr.clone(), - id: process_id, - }; - let value = ProcessValue { - database, + pub fn register_query( + &self, + catalog: String, + schema: Vec, + query: String, + client: String, + ) -> u64 { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let process = ProcessInfo { + id, + catalog: catalog.clone(), + schema, query, - start_timestamp_ms: current_time_millis(), + start_timestamp: current_time_millis(), + client, + frontend: self.server_addr.clone(), }; - let process = Process { key, value }; - self.pending_queries.lock().unwrap().insert( - process_id, - ProcessWithState { - process, - state: ProcessState::Running, - }, - ); - Ok(process_id) + self.catalogs + .write() + .unwrap() + .entry(catalog) + .or_default() + .insert(id, process); + id } /// De-register a query from process list. - pub async fn deregister_query(&self, id: u64) -> error::Result<()> { - if let Entry::Occupied(mut e) = self.pending_queries.lock().unwrap().entry(id) { - let process = e.get_mut(); - if process.state == ProcessState::Running { - e.remove(); - } else { - debug_assert_eq!(process.state, ProcessState::Registered); - process.state = ProcessState::Finished; + pub fn deregister_query(&self, catalog: String, id: u64) { + if let Entry::Occupied(mut o) = self.catalogs.write().unwrap().entry(catalog) { + let process = o.get_mut().remove(&id); + debug!("Deregister process: {:?}", process); + if o.get_mut().is_empty() { + o.remove(); } } - Ok(()) } /// De-register all queries running on current frontend. - pub async fn deregister_all_queries(&self) -> error::Result<()> { - let prefix = format!("{}/{}-", PROCESS_LIST_PREFIX, self.server_addr); - let delete_range_request = DeleteRangeRequest::new().with_prefix(prefix.as_bytes()); - self.pending_queries.lock().unwrap().clear(); - self.kv_client - .delete_range(delete_range_request) - .await - .context(error::ReportProcessSnafu)?; + pub fn deregister_all_queries(&self) { + self.catalogs.write().unwrap().clear(); info!("All queries on {} has been deregistered", self.server_addr); - Ok(()) } /// List all running processes in cluster. - pub fn list_all_processes(&self) -> error::Result> { - let queries = self.pending_queries.lock().unwrap().clone(); - Ok(queries.into_values().map(|p| p.process).collect()) - } -} - -#[derive(Debug, Clone)] -struct ProcessWithState { - process: Process, - state: ProcessState, -} - -#[derive(Debug, Clone, Eq, PartialEq)] -enum ProcessState { - /// Running but not registered to meta - Running, - /// Running and registered to meta - Registered, - /// Finished - Finished, -} - -/// Periodical task to report running queries to kv backend. -struct ReportTask { - queries: Arc>>, - kv_backend: KvBackendRef, -} - -#[async_trait::async_trait] -impl TaskFunction for ReportTask { - async fn call(&mut self) -> error::Result<()> { - let pending_queries = self.queries.lock().unwrap().clone(); - let mut queries_to_register = vec![]; - let mut queries_to_deregister = vec![]; - - for (_, process) in pending_queries { - match process.state { - ProcessState::Running => { - queries_to_register.push(process); - } - ProcessState::Finished => { - queries_to_deregister.push(process); - } - _ => {} - } - } - - self.kv_backend - .batch_put(BatchPutRequest { - kvs: queries_to_register - .iter() - .map(|p| KeyValue::try_from(&p.process)) - .collect::>() - .context(error::ReportProcessSnafu)?, - prev_kv: false, - }) - .await - .context(error::ReportProcessSnafu)?; - - for registered in queries_to_register { - // for running queries, it can either be still in running state or removed. - if let Some(process) = self - .queries - .lock() - .unwrap() - .get_mut(®istered.process.key.id) - { - debug_assert_eq!(process.state, ProcessState::Running); - process.state = ProcessState::Registered; - } else { - // Process can also be removed by `deregister_query` - queries_to_deregister.push(registered); - } - } - - self.kv_backend - .batch_delete(BatchDeleteRequest { - keys: queries_to_deregister - .iter() - .map(|p| p.process.key.to_bytes()) - .collect(), - prev_kv: false, - }) - .await - .context(error::ReportProcessSnafu)?; - for deregistered in queries_to_deregister { - if let Some(process) = self - .queries - .lock() - .unwrap() - .remove(&deregistered.process.key.id) - { - debug!("Deregistered process {}", process.process.key); - } - } - - Ok(()) - } - - fn name(&self) -> &str { - "ProcessManagerReportTask" + pub fn list_all_processes(&self) -> Vec { + self.catalogs + .read() + .unwrap() + .values() + .flat_map(|v| v.values().cloned()) + .collect() } } #[cfg(test)] mod tests { - use std::sync::Arc; - use std::time::{SystemTime, UNIX_EPOCH}; - - use common_meta::kv_backend::memory::MemoryKvBackend; - use crate::process_manager::ProcessManager; #[tokio::test] async fn test_register_query() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = - ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); - let process_id = process_manager - .register_query("public".to_string(), "SELECT * FROM table".to_string()) - .await - .unwrap(); + let process_manager = ProcessManager::new("127.0.0.1:8000".to_string()).unwrap(); + let process_id = process_manager.register_query( + "public".to_string(), + vec!["test".to_string()], + "SELECT * FROM table".to_string(), + "".to_string(), + ); - let running_processes = process_manager.list_all_processes().unwrap(); + let running_processes = process_manager.list_all_processes(); assert_eq!(running_processes.len(), 1); - assert_eq!(running_processes[0].key.frontend_ip, "127.0.0.1:8000"); - assert_eq!(running_processes[0].key.id, process_id); - assert_eq!(running_processes[0].value.query, "SELECT * FROM table"); - - process_manager.deregister_query(process_id).await.unwrap(); - assert_eq!(process_manager.list_all_processes().unwrap().len(), 0); - } - - #[tokio::test] - async fn test_register_multiple_queries() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = - ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); - - // Register multiple queries - let id1 = process_manager - .register_query("public".to_string(), "SELECT 1".to_string()) - .await - .unwrap(); - let id2 = process_manager - .register_query("public".to_string(), "SELECT 2".to_string()) - .await - .unwrap(); - let id3 = process_manager - .register_query("public".to_string(), "SELECT 3".to_string()) - .await - .unwrap(); - - // Verify all are registered - assert_eq!(process_manager.list_all_processes().unwrap().len(), 3); - - // Deregister middle one - process_manager.deregister_query(id2).await.unwrap(); - let processes = process_manager.list_all_processes().unwrap(); - assert_eq!(processes.len(), 2); - assert!(processes.iter().any(|p| p.key.id == id1)); - assert!(processes.iter().any(|p| p.key.id == id3)); - } - - #[tokio::test] - async fn test_deregister_nonexistent_query() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = - ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); - - // Try to deregister non-existent ID - let result = process_manager.deregister_query(999).await; - assert!(result.is_ok()); // Should succeed with warning - } - - #[tokio::test] - async fn test_process_timestamps() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = - ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); - - let before = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis() as i64; - - process_manager - .register_query("public".to_string(), "SELECT NOW()".to_string()) - .await - .unwrap(); - - let after = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis() as i64; - - let running_processes = process_manager.list_all_processes().unwrap(); - let process = &running_processes[0]; - - assert!(process.value.start_timestamp_ms >= before); - assert!(process.value.start_timestamp_ms <= after); - } - - #[tokio::test] - async fn test_multiple_frontends() { - let kv_client = Arc::new(MemoryKvBackend::new()); - - // Create two process managers with different frontend addresses - let pm1 = ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); - let pm2 = ProcessManager::new("127.0.0.1:8001".to_string(), kv_client.clone()).unwrap(); - - let id1 = pm1 - .register_query("public".to_string(), "SELECT 1".to_string()) - .await - .unwrap(); - let id2 = pm2 - .register_query("public".to_string(), "SELECT 2".to_string()) - .await - .unwrap(); - - // Verify both processes are registered with correct frontend IPs - let pm1_processes = pm1.list_all_processes().unwrap(); - assert_eq!(pm1_processes.len(), 1); - - let p1 = pm1_processes.iter().find(|p| p.key.id == id1).unwrap(); - assert_eq!(p1.key.frontend_ip, "127.0.0.1:8000"); - - let p2_processes = pm2.list_all_processes().unwrap(); - assert_eq!(p2_processes.len(), 1); - let p2 = p2_processes.iter().find(|p| p.key.id == id2).unwrap(); - assert_eq!(p2.key.frontend_ip, "127.0.0.1:8001"); - assert_eq!(p2.value.database, "public"); - // deregister all queries on instance 1 - pm1.deregister_all_queries().await.unwrap(); - - let processes: Vec<_> = pm1.list_all_processes().unwrap(); - assert_eq!(processes.len(), 0); - - assert_eq!(pm2.list_all_processes().unwrap().len(), 1); - } - - #[tokio::test] - async fn test_list_empty_processes() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = - ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); - - assert!(process_manager.list_all_processes().unwrap().is_empty()); - } - - #[tokio::test] - async fn test_deregister_all_queries() { - let kv_client = Arc::new(MemoryKvBackend::new()); - let process_manager = - ProcessManager::new("127.0.0.1:8000".to_string(), kv_client.clone()).unwrap(); - - // Register multiple queries - process_manager - .register_query("public".to_string(), "SELECT 1".to_string()) - .await - .unwrap(); - process_manager - .register_query("public".to_string(), "SELECT 2".to_string()) - .await - .unwrap(); - process_manager - .register_query("public".to_string(), "SELECT 3".to_string()) - .await - .unwrap(); - - // Verify they exist - assert_eq!(process_manager.list_all_processes().unwrap().len(), 3); - - // Deregister all - process_manager.deregister_all_queries().await.unwrap(); + assert_eq!(&running_processes[0].frontend, "127.0.0.1:8000"); + assert_eq!(running_processes[0].id, process_id); + assert_eq!(&running_processes[0].query, "SELECT * FROM table"); - // Verify none remain - assert_eq!(process_manager.list_all_processes().unwrap().len(), 0); + process_manager.deregister_query("public".to_string(), process_id); + assert_eq!(process_manager.list_all_processes().len(), 0); } } diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index f546209dbab9..6b042c5e1bb4 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -20,6 +20,7 @@ use common_recordbatch::adapter::RecordBatchStreamAdapter; use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; use common_time::util::current_time_millis; use common_time::{Duration, Timestamp}; +use datafusion::logical_expr::UserDefinedLogicalNode; use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter; use datatypes::prelude::ConcreteDataType as CDT; use datatypes::scalars::ScalarVectorBuilder; @@ -114,7 +115,7 @@ async fn make_process_list( ) -> error::Result { let predicates = Predicates::from_scan_request(&Some(request)); let current_time = current_time_millis(); - let queries = process_manager.list_all_processes()?; + let queries = process_manager.list_all_processes(); let mut id_builder = UInt64VectorBuilder::with_capacity(queries.len()); let mut database_builder = StringVectorBuilder::with_capacity(queries.len()); @@ -122,32 +123,32 @@ async fn make_process_list( let mut start_time_builder = TimestampMillisecondVectorBuilder::with_capacity(queries.len()); let mut elapsed_time_builder = DurationMillisecondVectorBuilder::with_capacity(queries.len()); - for process in queries { - let row = [ - (ID, &Value::from(process.query_id())), - (DATABASE, &Value::from(process.database())), - (QUERY, &Value::from(process.query_string())), - ( - START_TIMESTAMP, - &Value::from(Timestamp::new_millisecond( - process.query_start_timestamp_ms(), - )), - ), - ( - ELAPSED_TIME, - &Value::from(Duration::new_millisecond( - current_time - process.query_start_timestamp_ms(), - )), - ), - ]; - if predicates.eval(&row) { - id_builder.push(row[0].1.as_u64()); - database_builder.push(row[1].1.as_string().as_deref()); - query_builder.push(row[2].1.as_string().as_deref()); - start_time_builder.push(row[3].1.as_timestamp().map(|t| t.value().into())); - elapsed_time_builder.push(row[4].1.as_duration().map(|d| d.value().into())); - } - } + // for process in queries { + // let row = [ + // (ID, &Value::from(process.id)), + // (DATABASE, &Value::from(process.catalog)), + // (QUERY, &Value::from(process.query_string())), + // ( + // START_TIMESTAMP, + // &Value::from(Timestamp::new_millisecond( + // process.query_start_timestamp_ms(), + // )), + // ), + // ( + // ELAPSED_TIME, + // &Value::from(Duration::new_millisecond( + // current_time - process.query_start_timestamp_ms(), + // )), + // ), + // ]; + // if predicates.eval(&row) { + // id_builder.push(row[0].1.as_u64()); + // database_builder.push(row[1].1.as_string().as_deref()); + // query_builder.push(row[2].1.as_string().as_deref()); + // start_time_builder.push(row[3].1.as_timestamp().map(|t| t.value().into())); + // elapsed_time_builder.push(row[4].1.as_duration().map(|d| d.value().into())); + // } + // } RecordBatch::new( InformationSchemaProcessList::schema(), diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index 5a58d1c75d88..8f8d410c01c8 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -39,6 +39,7 @@ use frontend::heartbeat::HeartbeatTask; use frontend::instance::builder::FrontendBuilder; use frontend::server::Services; use meta_client::{MetaClientOptions, MetaClientType}; +use servers::addrs; use servers::export_metrics::ExportMetricsTask; use servers::tls::{TlsMode, TlsOption}; use snafu::{OptionExt, ResultExt}; @@ -345,8 +346,11 @@ impl StartCommand { Arc::new(DistributedInformationExtension::new(meta_client.clone())); let process_manager = Arc::new( - ProcessManager::new(opts.grpc.server_addr.clone(), cached_meta_backend.clone()) - .context(error::BuildProcessManagerSnafu)?, + ProcessManager::new(addrs::resolve_addr( + &opts.grpc.bind_addr, + Some(&opts.grpc.server_addr), + )) + .context(error::BuildProcessManagerSnafu)?, ); let catalog_manager = KvBackendCatalogManager::new( information_extension, diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 1d82bcb954e4..71301ea5356d 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -529,7 +529,7 @@ impl StartCommand { )); let process_manager = Arc::new( - ProcessManager::new(opts.grpc.server_addr.clone(), kv_backend.clone()) + ProcessManager::new(opts.grpc.server_addr.clone()) .context(error::BuildProcessManagerSnafu)?, ); let catalog_manager = KvBackendCatalogManager::new( @@ -655,7 +655,7 @@ impl StartCommand { node_manager, ) .await - .context(error::StartFlownodeSnafu)?; + .context(StartFlownodeSnafu)?; flow_streaming_engine.set_frontend_invoker(invoker).await; let export_metrics_task = ExportMetricsTask::try_new(&opts.export_metrics, Some(&plugins)) diff --git a/src/common/meta/src/key.rs b/src/common/meta/src/key.rs index 1ed84ecb0837..fdc2de38b68c 100644 --- a/src/common/meta/src/key.rs +++ b/src/common/meta/src/key.rs @@ -102,7 +102,6 @@ pub mod datanode_table; pub mod flow; pub mod maintenance; pub mod node_address; -pub mod process_list; mod schema_metadata_manager; pub mod schema_name; pub mod table_info; @@ -155,7 +154,6 @@ use self::tombstone::TombstoneManager; use crate::error::{self, Result, SerdeJsonSnafu}; use crate::key::flow::flow_state::FlowStateValue; use crate::key::node_address::NodeAddressValue; -use crate::key::process_list::ProcessValue; use crate::key::table_route::TableRouteKey; use crate::key::txn_helper::TxnOpGetResponseSet; use crate::kv_backend::txn::{Txn, TxnOp}; @@ -176,8 +174,6 @@ pub const CATALOG_NAME_KEY_PREFIX: &str = "__catalog_name"; pub const SCHEMA_NAME_KEY_PREFIX: &str = "__schema_name"; pub const TABLE_ROUTE_PREFIX: &str = "__table_route"; pub const NODE_ADDRESS_PREFIX: &str = "__node_address"; -/// The prefix for process list values. -pub const PROCESS_LIST_PREFIX: &str = "__process"; pub const KAFKA_TOPIC_KEY_PREFIX: &str = "__topic_name/kafka"; // The legacy topic key prefix is used to store the topic name in previous versions. pub const LEGACY_TOPIC_KEY_PREFIX: &str = "__created_wal_topics/kafka"; @@ -327,13 +323,6 @@ lazy_static! { .unwrap(); } -lazy_static! { - static ref PROCESS_LIST_PATTERN: Regex = Regex::new(&format!( - "^{PROCESS_LIST_PREFIX}/([0-9.]+:[0-9]+)-([0-9]+)$" - )) - .unwrap(); -} - /// The key of metadata. pub trait MetadataKey<'a, T> { fn to_bytes(&self) -> Vec; @@ -1411,8 +1400,7 @@ impl_metadata_value! { NodeAddressValue, SchemaNameValue, FlowStateValue, - PoisonValue, - ProcessValue + PoisonValue } impl_optional_metadata_value! { diff --git a/src/common/meta/src/key/process_list.rs b/src/common/meta/src/key/process_list.rs deleted file mode 100644 index d13c9c0c6ed4..000000000000 --- a/src/common/meta/src/key/process_list.rs +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt::{Display, Formatter}; -use std::str::FromStr; -use std::time::Duration; - -use common_time::util::current_time_millis; -use serde::{Deserialize, Serialize}; -use snafu::OptionExt; - -use crate::error; -use crate::key::{MetadataKey, MetadataValue, PROCESS_LIST_PATTERN, PROCESS_LIST_PREFIX}; -use crate::rpc::KeyValue; - -/// Key for running queries tracked in metasrv. -/// Layout: `__process/{frontend server addr}-{query id}` -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct ProcessKey { - //todo(hl): maybe we don't have to own a string - pub frontend_ip: String, - pub id: u64, -} - -impl Display for ProcessKey { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}/{}-{}", - PROCESS_LIST_PREFIX, self.frontend_ip, self.id - ) - } -} - -impl MetadataKey<'_, ProcessKey> for ProcessKey { - fn to_bytes(&self) -> Vec { - self.to_string().into_bytes() - } - - fn from_bytes(bytes: &[u8]) -> error::Result { - let key_str = std::str::from_utf8(bytes).map_err(|_e| { - error::InvalidProcessKeySnafu { - key: String::from_utf8_lossy(bytes).into_owned(), - } - .build() - })?; - let captures = PROCESS_LIST_PATTERN.captures(key_str).with_context(|| { - error::InvalidProcessKeySnafu { - key: String::from_utf8_lossy(bytes).into_owned(), - } - })?; - let id = u64::from_str(&captures[2]).map_err(|_| { - error::InvalidProcessKeySnafu { - key: String::from_utf8_lossy(bytes).into_owned(), - } - .build() - })?; - Ok(Self { - frontend_ip: captures[1].to_string(), - id, - }) - } -} - -/// Detail value of process. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct ProcessValue { - /// Database name. - pub database: String, - /// The running query sql. - pub query: String, - /// Query start timestamp in milliseconds. - pub start_timestamp_ms: i64, -} - -/// Sequence key for process list entries. -pub const PROCESS_ID_SEQ: &str = "process_id_seq"; - -/// Running process instance. -#[derive(Clone, Debug)] -pub struct Process { - pub key: ProcessKey, - pub value: ProcessValue, -} - -impl TryFrom<&Process> for KeyValue { - type Error = error::Error; - - fn try_from(process: &Process) -> Result { - Ok(KeyValue { - key: process.key.to_bytes(), - value: process.value.try_as_raw_value()?, - }) - } -} - -impl Process { - /// Returns server address of running process. - pub fn server_addr(&self) -> &str { - &self.key.frontend_ip - } - - /// Returns database name of running process. - pub fn database(&self) -> &str { - &self.value.database - } - - /// Returns id of query. - pub fn query_id(&self) -> u64 { - self.key.id - } - - /// Returns query string details. - pub fn query_string(&self) -> &str { - &self.value.query - } - - /// Returns query start timestamp in milliseconds. - pub fn query_start_timestamp_ms(&self) -> i64 { - self.value.start_timestamp_ms - } - - /// Calculates the elapsed time of query. Returns None of system clock jumps backwards. - pub fn query_elapsed(&self) -> Option { - let now = current_time_millis(); - if now < self.value.start_timestamp_ms { - None - } else { - Some(Duration::from_millis( - (now - self.value.start_timestamp_ms) as u64, - )) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::key::MetadataKey; - - fn check_serialization(input: ProcessKey) { - let serialized = input.to_bytes(); - let key = ProcessKey::from_bytes(&serialized).unwrap(); - assert_eq!(&input.frontend_ip, &key.frontend_ip); - assert_eq!(input.id, key.id); - } - - #[test] - fn test_capture_process_list() { - check_serialization(ProcessKey { - frontend_ip: "192.168.0.1:4001".to_string(), - id: 1, - }); - check_serialization(ProcessKey { - frontend_ip: "192.168.0.1:4002".to_string(), - id: 0, - }); - check_serialization(ProcessKey { - frontend_ip: "255.255.255.255:80".to_string(), - id: 0, - }); - } - - #[test] - fn test_process_key_display() { - let key = ProcessKey { - frontend_ip: "127.0.0.1:3000".to_string(), - id: 42, - }; - assert_eq!(key.to_string(), "__process/127.0.0.1:3000-42"); - } - - #[test] - fn test_process_key_from_bytes_valid() { - let bytes = b"__process/10.0.0.1:8080-123"; - let key = ProcessKey::from_bytes(bytes).unwrap(); - assert_eq!(key.frontend_ip, "10.0.0.1:8080"); - assert_eq!(key.id, 123); - } - - #[test] - fn test_process_key_from_bytes_invalid_format() { - let bytes = b"invalid_format"; - let result = ProcessKey::from_bytes(bytes); - assert!(result.is_err()); - } - - #[test] - fn test_process_key_from_bytes_invalid_id() { - let bytes = b"__process/10.0.0.1:8080-abc"; - let result = ProcessKey::from_bytes(bytes); - assert!(result.is_err()); - } - - #[test] - fn test_process_value_serialization() { - let value = ProcessValue { - query: "SELECT * FROM test".to_string(), - start_timestamp_ms: 1690000000000, - database: "public".to_string(), - }; - - // Test serialization roundtrip - let serialized = serde_json::to_string(&value).unwrap(); - let deserialized: ProcessValue = serde_json::from_str(&serialized).unwrap(); - - assert_eq!(value.query, deserialized.query); - assert_eq!(value.start_timestamp_ms, deserialized.start_timestamp_ms); - assert_eq!(value.database, deserialized.database); - } -} From 2ae4c6d734b71c095d523ddf8571a6570c677f2c Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Tue, 10 Jun 2025 09:05:23 +0000 Subject: [PATCH 19/31] feat/show-process-list: - **Update `greptime-proto` Dependency**: Updated the `greptime-proto` dependency version in `Cargo.lock` and `Cargo.toml` to a new commit hash. - **Refactor Error Handling**: Removed unused error variants and added a new `ParseProcessId` error in `src/catalog/src/error.rs`. - **Enhance Process Management**: Introduced `DisplayProcessId` struct for better process ID representation and parsing in `src/catalog/src/process_manager.rs`. - **Revise Process List Schema**: Updated the schema and logic for process listing in `src/catalog/src/system_schema/information_schema/process_list.rs` to include new fields like `client` and `frontend`. Signed-off-by: Lei, HUANG --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/catalog/src/error.rs | 29 ++---- src/catalog/src/process_manager.rs | 56 +++++++++-- .../information_schema/process_list.rs | 93 ++++++++++++------- 5 files changed, 115 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 10f9cd190918..07978749463b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5133,7 +5133,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=d05ff47433bf5249ae7eda1fb5c591dc8d2c75b1#d05ff47433bf5249ae7eda1fb5c591dc8d2c75b1" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=0807d6dc1069bfc33158b78b095a180ddf120084#0807d6dc1069bfc33158b78b095a180ddf120084" dependencies = [ "prost 0.13.5", "serde", diff --git a/Cargo.toml b/Cargo.toml index 2d32c1661ed0..72701c0547f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,7 +133,7 @@ etcd-client = "0.14" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "d05ff47433bf5249ae7eda1fb5c591dc8d2c75b1" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0807d6dc1069bfc33158b78b095a180ddf120084" } hex = "0.4" http = "1" humantime = "2.1" diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs index 212889ba0890..d7e29b4341b7 100644 --- a/src/catalog/src/error.rs +++ b/src/catalog/src/error.rs @@ -278,23 +278,9 @@ pub enum Error { location: Location, }, - #[snafu(display("Failed to bump sequence for process manager"))] - BumpSequence { - source: common_meta::error::Error, - #[snafu(implicit)] - location: Location, - }, - - #[snafu(display("Failed to start process report task"))] - StartReportTask { - source: common_runtime::error::Error, - #[snafu(implicit)] - location: Location, - }, - - #[snafu(display("Failed to report process state"))] - ReportProcess { - source: common_meta::error::Error, + #[snafu(display("Failed to parse process id: {}", s))] + ParseProcessId { + s: String, #[snafu(implicit)] location: Location, }, @@ -363,11 +349,10 @@ impl ErrorExt for Error { Error::Datafusion { error, .. } => datafusion_status_code::(error, None), Error::ProjectViewColumns { .. } => StatusCode::EngineExecuteQuery, Error::TableMetadataManager { source, .. } => source.status_code(), - Error::GetViewCache { source, .. } - | Error::GetTableCache { source, .. } - | Error::BumpSequence { source, .. } - | Error::ReportProcess { source, .. } => source.status_code(), - Error::StartReportTask { .. } => StatusCode::Internal, + Error::GetViewCache { source, .. } | Error::GetTableCache { source, .. } => { + source.status_code() + } + Error::ParseProcessId { .. } => StatusCode::Internal, } } diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index c9658cc3ff73..9fb1a51b6d13 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -14,17 +14,15 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::fmt::{Display, Formatter}; +use std::str::FromStr; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::RwLock; use api::v1::frontend::ProcessInfo; -use common_meta::kv_backend::KvBackendRef; -use common_meta::rpc::store::{BatchDeleteRequest, BatchPutRequest, DeleteRangeRequest}; -use common_meta::rpc::KeyValue; -use common_runtime::{RepeatedTask, TaskFunction}; use common_telemetry::{debug, info}; use common_time::util::current_time_millis; -use snafu::ResultExt; +use snafu::OptionExt; use crate::error; @@ -48,7 +46,7 @@ impl ProcessManager { pub fn register_query( &self, catalog: String, - schema: Vec, + schemas: Vec, query: String, client: String, ) -> u64 { @@ -56,7 +54,7 @@ impl ProcessManager { let process = ProcessInfo { id, catalog: catalog.clone(), - schema, + schemas, query, start_timestamp: current_time_millis(), client, @@ -99,9 +97,40 @@ impl ProcessManager { } } +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct DisplayProcessId { + pub server_addr: String, + pub id: u64, +} + +impl Display for DisplayProcessId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}/{}", self.server_addr, self.id) + } +} + +impl TryFrom<&str> for DisplayProcessId { + type Error = error::Error; + + fn try_from(value: &str) -> Result { + let mut split = value.split('/'); + let server_addr = split + .next() + .context(error::ParseProcessIdSnafu { s: value })? + .to_string(); + let id = split + .next() + .context(error::ParseProcessIdSnafu { s: value })?; + let id = u64::from_str(id) + .ok() + .context(error::ParseProcessIdSnafu { s: value })?; + Ok(DisplayProcessId { server_addr, id }) + } +} + #[cfg(test)] mod tests { - use crate::process_manager::ProcessManager; + use crate::process_manager::{DisplayProcessId, ProcessManager}; #[tokio::test] async fn test_register_query() { @@ -122,4 +151,15 @@ mod tests { process_manager.deregister_query("public".to_string(), process_id); assert_eq!(process_manager.list_all_processes().len(), 0); } + + #[test] + fn test_display_process_id() { + assert_eq!( + DisplayProcessId::try_from("1.1.1.1:3000/123").unwrap(), + DisplayProcessId { + server_addr: "1.1.1.1:3000".to_string(), + id: 123, + } + ); + } } diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 6b042c5e1bb4..109daf9c0a95 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -20,7 +20,6 @@ use common_recordbatch::adapter::RecordBatchStreamAdapter; use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; use common_time::util::current_time_millis; use common_time::{Duration, Timestamp}; -use datafusion::logical_expr::UserDefinedLogicalNode; use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter; use datatypes::prelude::ConcreteDataType as CDT; use datatypes::scalars::ScalarVectorBuilder; @@ -28,20 +27,23 @@ use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::value::Value; use datatypes::vectors::{ DurationMillisecondVectorBuilder, StringVectorBuilder, TimestampMillisecondVectorBuilder, - UInt64VectorBuilder, VectorRef, + VectorRef, }; use snafu::ResultExt; use store_api::storage::{ScanRequest, TableId}; use crate::error::{self, InternalSnafu}; use crate::information_schema::Predicates; -use crate::process_manager::ProcessManager; +use crate::process_manager::{DisplayProcessId, ProcessManager}; use crate::system_schema::information_schema::InformationTable; /// Column names of `information_schema.process_list` const ID: &str = "id"; -const DATABASE: &str = "database"; +const CATALOG: &str = "catalog"; +const SCHEMAS: &str = "schemas"; const QUERY: &str = "query"; +const CLIENT: &str = "client"; +const FRONTEND: &str = "frontend"; const START_TIMESTAMP: &str = "start_timestamp"; const ELAPSED_TIME: &str = "elapsed_time"; @@ -62,9 +64,12 @@ impl InformationSchemaProcessList { fn schema() -> SchemaRef { Arc::new(Schema::new(vec![ - ColumnSchema::new(ID, CDT::uint64_datatype(), false), - ColumnSchema::new(DATABASE, CDT::string_datatype(), false), + ColumnSchema::new(ID, CDT::string_datatype(), false), + ColumnSchema::new(CATALOG, CDT::string_datatype(), false), + ColumnSchema::new(SCHEMAS, CDT::string_datatype(), false), ColumnSchema::new(QUERY, CDT::string_datatype(), false), + ColumnSchema::new(CLIENT, CDT::string_datatype(), false), + ColumnSchema::new(FRONTEND, CDT::string_datatype(), false), ColumnSchema::new( START_TIMESTAMP, CDT::timestamp_millisecond_datatype(), @@ -117,45 +122,63 @@ async fn make_process_list( let current_time = current_time_millis(); let queries = process_manager.list_all_processes(); - let mut id_builder = UInt64VectorBuilder::with_capacity(queries.len()); - let mut database_builder = StringVectorBuilder::with_capacity(queries.len()); + let mut id_builder = StringVectorBuilder::with_capacity(queries.len()); + let mut catalog_builder = StringVectorBuilder::with_capacity(queries.len()); + let mut schemas_builder = StringVectorBuilder::with_capacity(queries.len()); let mut query_builder = StringVectorBuilder::with_capacity(queries.len()); + let mut client_builder = StringVectorBuilder::with_capacity(queries.len()); + let mut frontend_builder = StringVectorBuilder::with_capacity(queries.len()); let mut start_time_builder = TimestampMillisecondVectorBuilder::with_capacity(queries.len()); let mut elapsed_time_builder = DurationMillisecondVectorBuilder::with_capacity(queries.len()); - // for process in queries { - // let row = [ - // (ID, &Value::from(process.id)), - // (DATABASE, &Value::from(process.catalog)), - // (QUERY, &Value::from(process.query_string())), - // ( - // START_TIMESTAMP, - // &Value::from(Timestamp::new_millisecond( - // process.query_start_timestamp_ms(), - // )), - // ), - // ( - // ELAPSED_TIME, - // &Value::from(Duration::new_millisecond( - // current_time - process.query_start_timestamp_ms(), - // )), - // ), - // ]; - // if predicates.eval(&row) { - // id_builder.push(row[0].1.as_u64()); - // database_builder.push(row[1].1.as_string().as_deref()); - // query_builder.push(row[2].1.as_string().as_deref()); - // start_time_builder.push(row[3].1.as_timestamp().map(|t| t.value().into())); - // elapsed_time_builder.push(row[4].1.as_duration().map(|d| d.value().into())); - // } - // } + for process in queries { + let display_id = DisplayProcessId { + server_addr: process.frontend.to_string(), + id: process.id, + } + .to_string(); + let schemas = process.schemas.join(","); + let id = Value::from(display_id); + let catalog = Value::from(process.catalog); + let schemas = Value::from(schemas); + let query = Value::from(process.query); + let client = Value::from(process.client); + let frontend = Value::from(process.frontend); + let start_timestamp = Value::from(Timestamp::new_millisecond(process.start_timestamp)); + let elapsed_time = Value::from(Duration::new_millisecond( + current_time - process.start_timestamp, + )); + let row = [ + (ID, &id), + (CATALOG, &catalog), + (SCHEMAS, &schemas), + (QUERY, &query), + (CLIENT, &client), + (FRONTEND, &frontend), + (START_TIMESTAMP, &start_timestamp), + (ELAPSED_TIME, &elapsed_time), + ]; + if predicates.eval(&row) { + id_builder.push(id.as_string().as_deref()); + catalog_builder.push(catalog.as_string().as_deref()); + schemas_builder.push(schemas.as_string().as_deref()); + query_builder.push(query.as_string().as_deref()); + client_builder.push(client.as_string().as_deref()); + frontend_builder.push(frontend.as_string().as_deref()); + start_time_builder.push(start_timestamp.as_timestamp().map(|t| t.value().into())); + elapsed_time_builder.push(elapsed_time.as_duration().map(|d| d.value().into())); + } + } RecordBatch::new( InformationSchemaProcessList::schema(), vec![ Arc::new(id_builder.finish()) as VectorRef, - Arc::new(database_builder.finish()) as VectorRef, + Arc::new(catalog_builder.finish()) as VectorRef, + Arc::new(schemas_builder.finish()) as VectorRef, Arc::new(query_builder.finish()) as VectorRef, + Arc::new(client_builder.finish()) as VectorRef, + Arc::new(frontend_builder.finish()) as VectorRef, Arc::new(start_time_builder.finish()) as VectorRef, Arc::new(elapsed_time_builder.finish()) as VectorRef, ], From f06ce5902e428f8f0c242bf6dbe8773cada9f113 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 11 Jun 2025 11:58:10 +0000 Subject: [PATCH 20/31] feat/show-process-list: ### Commit Message **Enhancements and Refactoring** - **Process Management:** - Refactored `ProcessManager` to list local processes with an optional catalog filter in `process_manager.rs`. - Updated related tests in `process_manager.rs` and `process_list.rs`. - **Client Enhancements:** - Added `frontend_client` method in `client.rs` to support gRPC communication with the frontend. - **Error Handling:** - Extended error handling in `error.rs` to include gRPC and Meta errors. - **Frontend Module:** - Introduced `selector.rs` for frontend client selection and process listing. - Updated `Cargo.toml` to include new dependencies and dev-dependencies. - **gRPC Server:** - Integrated `FrontendServer` in `builder.rs` for enhanced gRPC server capabilities. Signed-off-by: Lei, HUANG --- Cargo.lock | 8 ++ src/catalog/src/process_manager.rs | 27 ++-- .../information_schema/process_list.rs | 2 +- src/client/src/client.rs | 11 ++ src/common/frontend/Cargo.toml | 10 ++ src/common/frontend/src/error.rs | 26 ++++ src/common/frontend/src/lib.rs | 1 + src/common/frontend/src/selector.rs | 116 ++++++++++++++++++ src/frontend/Cargo.toml | 1 + src/servers/src/grpc/builder.rs | 1 + 10 files changed, 192 insertions(+), 11 deletions(-) create mode 100644 src/common/frontend/src/selector.rs diff --git a/Cargo.lock b/Cargo.lock index 07978749463b..2c5860f88468 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2292,9 +2292,16 @@ name = "common-frontend" version = "0.15.0" dependencies = [ "async-trait", + "client", "common-error", + "common-grpc", "common-macro", + "common-meta", + "greptime-proto", + "meta-client", "snafu 0.8.5", + "tokio", + "tonic 0.12.3", ] [[package]] @@ -4701,6 +4708,7 @@ dependencies = [ "common-config", "common-datasource", "common-error", + "common-frontend", "common-function", "common-grpc", "common-macro", diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index 9fb1a51b6d13..1be462e2dc4b 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -86,14 +86,21 @@ impl ProcessManager { info!("All queries on {} has been deregistered", self.server_addr); } - /// List all running processes in cluster. - pub fn list_all_processes(&self) -> Vec { - self.catalogs - .read() - .unwrap() - .values() - .flat_map(|v| v.values().cloned()) - .collect() + /// List local running processes in given catalog. + pub fn local_processes(&self, catalog: Option<&str>) -> Vec { + let catalogs = self.catalogs.read().unwrap(); + if let Some(catalog) = catalog { + if let Some(catalogs) = catalogs.get(catalog) { + catalogs.values().cloned().collect() + } else { + vec![] + } + } else { + catalogs + .values() + .flat_map(|v| v.values().cloned()) + .collect() + } } } @@ -142,14 +149,14 @@ mod tests { "".to_string(), ); - let running_processes = process_manager.list_all_processes(); + let running_processes = process_manager.local_processes(None); assert_eq!(running_processes.len(), 1); assert_eq!(&running_processes[0].frontend, "127.0.0.1:8000"); assert_eq!(running_processes[0].id, process_id); assert_eq!(&running_processes[0].query, "SELECT * FROM table"); process_manager.deregister_query("public".to_string(), process_id); - assert_eq!(process_manager.list_all_processes().len(), 0); + assert_eq!(process_manager.local_processes(None).len(), 0); } #[test] diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 109daf9c0a95..24f2e9dde726 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -120,7 +120,7 @@ async fn make_process_list( ) -> error::Result { let predicates = Predicates::from_scan_request(&Some(request)); let current_time = current_time_millis(); - let queries = process_manager.list_all_processes(); + let queries = process_manager.local_processes(None); let mut id_builder = StringVectorBuilder::with_capacity(queries.len()); let mut catalog_builder = StringVectorBuilder::with_capacity(queries.len()); diff --git a/src/client/src/client.rs b/src/client/src/client.rs index a604bf5124ad..06047b0f4022 100644 --- a/src/client/src/client.rs +++ b/src/client/src/client.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use api::v1::flow::flow_client::FlowClient as PbFlowClient; +use api::v1::frontend::frontend_client::FrontendClient; use api::v1::health_check_client::HealthCheckClient; use api::v1::prometheus_gateway_client::PrometheusGatewayClient; use api::v1::region::region_client::RegionClient as PbRegionClient; @@ -190,6 +191,16 @@ impl Client { Ok((addr, client)) } + pub fn frontend_client(&self) -> Result> { + let (_, channel) = self.find_channel()?; + let client = FrontendClient::new(channel) + .max_decoding_message_size(self.max_grpc_recv_message_size()) + .max_encoding_message_size(self.max_grpc_send_message_size()) + .accept_compressed(CompressionEncoding::Zstd) + .send_compressed(CompressionEncoding::Zstd); + Ok(client) + } + pub fn make_prometheus_gateway_client(&self) -> Result> { let (_, channel) = self.find_channel()?; let client = PrometheusGatewayClient::new(channel) diff --git a/src/common/frontend/Cargo.toml b/src/common/frontend/Cargo.toml index 7c3b705bddcd..e072cfdfb801 100644 --- a/src/common/frontend/Cargo.toml +++ b/src/common/frontend/Cargo.toml @@ -8,4 +8,14 @@ license.workspace = true async-trait.workspace = true common-error.workspace = true common-macro.workspace = true +common-grpc.workspace = true +client.workspace = true +greptime-proto.workspace = true +common-meta.workspace = true +meta-client.workspace = true snafu.workspace = true +tonic.workspace = true + + +[dev-dependencies] +tokio.workspace = true diff --git a/src/common/frontend/src/error.rs b/src/common/frontend/src/error.rs index 7fc905dc6f3f..8119aad0128b 100644 --- a/src/common/frontend/src/error.rs +++ b/src/common/frontend/src/error.rs @@ -16,6 +16,7 @@ use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; use snafu::{Location, Snafu}; +use tonic::Status; #[derive(Snafu)] #[snafu(visibility(pub))] @@ -27,6 +28,29 @@ pub enum Error { location: Location, source: BoxedError, }, + + #[snafu(display("Failed to send gRPC request"))] + Grpc { + source: client::error::Error, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to list nodes from metasrv"))] + Meta { + source: meta_client::error::Error, + #[snafu(implicit)] + location: Location, + }, +} + +impl From for Error { + fn from(value: Status) -> Self { + Self::Grpc { + source: client::error::Error::from(value), + location: Location::default(), + } + } } pub type Result = std::result::Result; @@ -36,6 +60,8 @@ impl ErrorExt for Error { use Error::*; match self { External { source, .. } => source.status_code(), + Grpc { source, .. } => source.status_code(), + Meta { source, .. } => source.status_code(), } } diff --git a/src/common/frontend/src/lib.rs b/src/common/frontend/src/lib.rs index 3824f3b3772c..f7e1263b1c45 100644 --- a/src/common/frontend/src/lib.rs +++ b/src/common/frontend/src/lib.rs @@ -13,3 +13,4 @@ // limitations under the License. pub mod error; +pub mod selector; diff --git a/src/common/frontend/src/selector.rs b/src/common/frontend/src/selector.rs new file mode 100644 index 000000000000..bbc57e4b0017 --- /dev/null +++ b/src/common/frontend/src/selector.rs @@ -0,0 +1,116 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use client::Client; +use common_grpc::channel_manager::{ChannelConfig, ChannelManager}; +use common_meta::cluster::{ClusterInfo, Role}; +use greptime_proto::v1::frontend::{frontend_client, ListProcessRequest, ListProcessResponse}; +use meta_client::MetaClientRef; +use snafu::ResultExt; + +use crate::error::{GrpcSnafu, MetaSnafu, Result}; + +pub type FrontendClientPtr = Box; + +#[async_trait::async_trait] +pub trait FrontendClient { + async fn list_process(&mut self, req: ListProcessRequest) -> Result; +} + +#[async_trait::async_trait] +impl FrontendClient for frontend_client::FrontendClient { + async fn list_process(&mut self, req: ListProcessRequest) -> Result { + let response: ListProcessResponse = frontend_client::FrontendClient::< + tonic::transport::channel::Channel, + >::list_process(&mut self, req) + .await + .map_err(client::error::Error::from) + .context(GrpcSnafu)? + .into_inner(); + Ok(response) + } +} + +#[async_trait::async_trait] +pub trait FrontendSelector { + async fn select_all(&self) -> Result>; +} + +#[derive(Debug, Clone)] +pub struct MetaClientSelector { + meta_client: MetaClientRef, + channel_manager: ChannelManager, +} + +#[async_trait::async_trait] +impl FrontendSelector for MetaClientSelector { + async fn select_all(&self) -> Result> { + let nodes = self + .meta_client + .list_nodes(Some(Role::Frontend)) + .await + .context(MetaSnafu)?; + + let res = nodes + .into_iter() + .map(|node| { + Client::with_manager_and_urls(self.channel_manager.clone(), vec![node.peer.addr]) + .frontend_client() + .map(|c| Box::new(c) as FrontendClientPtr) + }) + .collect::>>() + .context(GrpcSnafu)?; + Ok(res) + } +} + +impl MetaClientSelector { + pub fn new(meta_client: MetaClientRef) -> Self { + let cfg = ChannelConfig::new() + .connect_timeout(Duration::from_secs(30)) + .timeout(Duration::from_secs(30)); + let channel_manager = ChannelManager::with_config(cfg); + Self { + meta_client, + channel_manager, + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::vec; + + use meta_client::client::MetaClientBuilder; + + use super::*; + + #[tokio::test] + async fn test_meta_client_selector() { + let mut meta_client = MetaClientBuilder::frontend_default_options().build(); + meta_client + .start(vec!["192.168.50.164:3002"]) + .await + .unwrap(); + let selector = MetaClientSelector::new(Arc::new(meta_client)); + let clients = selector.select_all().await.unwrap(); + for mut client in clients { + let resp = client.list_process(ListProcessRequest {}).await; + println!("{:?}", resp); + } + } +} diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml index 47cd0d49750c..9a25b1485fe2 100644 --- a/src/frontend/Cargo.toml +++ b/src/frontend/Cargo.toml @@ -25,6 +25,7 @@ common-catalog.workspace = true common-config.workspace = true common-datasource.workspace = true common-error.workspace = true +common-frontend.workspace = true common-function.workspace = true common-grpc.workspace = true common-macro.workspace = true diff --git a/src/servers/src/grpc/builder.rs b/src/servers/src/grpc/builder.rs index cee125b0c60a..a53a2f3c39da 100644 --- a/src/servers/src/grpc/builder.rs +++ b/src/servers/src/grpc/builder.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use api::v1::frontend::frontend_server::FrontendServer; use api::v1::greptime_database_server::GreptimeDatabaseServer; use api::v1::prometheus_gateway_server::PrometheusGatewayServer; use api::v1::region::region_server::RegionServer; From eb3fd430d46a0f0525e2dbbcbe84ce0f37c386c6 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 11 Jun 2025 12:47:28 +0000 Subject: [PATCH 21/31] feat/show-process-list: ### Commit Message **Refactor Process Management and Frontend Integration** - **Add `common-frontend` Dependency**: - Updated `Cargo.lock`, `Cargo.toml` files to include `common-frontend` as a dependency. - **Refactor Process Management**: - Moved `ProcessManager` trait and `DisplayProcessId` struct to `common-frontend`. - Updated `process_manager.rs` to use `MetaProcessManager` and `ProcessManagerRef`. - Removed `ParseProcessId` error variant from `error.rs` in `catalog` and `frontend`. - **Frontend gRPC Service**: - Added `frontend_grpc_handler.rs` to handle gRPC requests for frontend processes. - Updated `grpc.rs` and `builder.rs` to integrate `FrontendGrpcHandler`. - **Update Tests**: - Modified tests in `process_manager.rs` to align with new `ProcessManager` implementation. - **Remove Unused Code**: - Removed `DisplayProcessId` and related parsing logic from `process_manager.rs`. Signed-off-by: Lei, HUANG --- Cargo.lock | 3 +- src/catalog/Cargo.toml | 1 + src/catalog/src/error.rs | 8 -- src/catalog/src/kvbackend/manager.rs | 7 +- src/catalog/src/process_manager.rs | 100 ++++++++---------- .../src/system_schema/information_schema.rs | 6 +- .../information_schema/process_list.rs | 10 +- src/cmd/src/frontend.rs | 5 +- src/cmd/src/standalone.rs | 5 +- src/common/frontend/Cargo.toml | 1 - src/common/frontend/src/error.rs | 24 ++--- src/common/frontend/src/lib.rs | 59 +++++++++++ src/common/frontend/src/selector.rs | 34 +++--- src/frontend/src/instance.rs | 4 +- src/frontend/src/instance/builder.rs | 7 +- src/servers/Cargo.toml | 1 + src/servers/src/grpc.rs | 1 + src/servers/src/grpc/builder.rs | 7 ++ src/servers/src/grpc/frontend_grpc_handler.rs | 40 +++++++ 19 files changed, 199 insertions(+), 124 deletions(-) create mode 100644 src/servers/src/grpc/frontend_grpc_handler.rs diff --git a/Cargo.lock b/Cargo.lock index 2c5860f88468..57fee5b3831f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1623,6 +1623,7 @@ dependencies = [ "chrono", "common-catalog", "common-error", + "common-frontend", "common-macro", "common-meta", "common-procedure", @@ -2292,7 +2293,6 @@ name = "common-frontend" version = "0.15.0" dependencies = [ "async-trait", - "client", "common-error", "common-grpc", "common-macro", @@ -11143,6 +11143,7 @@ dependencies = [ "common-catalog", "common-config", "common-error", + "common-frontend", "common-grpc", "common-macro", "common-mem-prof", diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml index b425aa02c641..5a1d281270cb 100644 --- a/src/catalog/Cargo.toml +++ b/src/catalog/Cargo.toml @@ -23,6 +23,7 @@ common-macro.workspace = true common-meta.workspace = true common-procedure.workspace = true common-query.workspace = true +common-frontend.workspace = true common-recordbatch.workspace = true common-runtime.workspace = true common-telemetry.workspace = true diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs index d7e29b4341b7..c4fdf1cdc1ec 100644 --- a/src/catalog/src/error.rs +++ b/src/catalog/src/error.rs @@ -277,13 +277,6 @@ pub enum Error { #[snafu(implicit)] location: Location, }, - - #[snafu(display("Failed to parse process id: {}", s))] - ParseProcessId { - s: String, - #[snafu(implicit)] - location: Location, - }, } impl Error { @@ -352,7 +345,6 @@ impl ErrorExt for Error { Error::GetViewCache { source, .. } | Error::GetTableCache { source, .. } => { source.status_code() } - Error::ParseProcessId { .. } => StatusCode::Internal, } } diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs index a8ac373a4a96..b7089bc8ce55 100644 --- a/src/catalog/src/kvbackend/manager.rs +++ b/src/catalog/src/kvbackend/manager.rs @@ -44,14 +44,13 @@ use table::table_name::TableName; use table::TableRef; use tokio::sync::Semaphore; use tokio_stream::wrappers::ReceiverStream; - +use common_frontend::ProcessManagerRef; use crate::error::{ CacheNotFoundSnafu, GetTableCacheSnafu, InvalidTableInfoInCatalogSnafu, ListCatalogsSnafu, ListSchemasSnafu, ListTablesSnafu, Result, TableMetadataManagerSnafu, }; use crate::information_schema::{InformationExtensionRef, InformationSchemaProvider}; use crate::kvbackend::TableCacheRef; -use crate::process_manager::ProcessManager; use crate::system_schema::pg_catalog::PGCatalogProvider; use crate::system_schema::SystemSchemaProvider; use crate::CatalogManager; @@ -85,7 +84,7 @@ impl KvBackendCatalogManager { backend: KvBackendRef, cache_registry: LayeredCacheRegistryRef, procedure_manager: Option, - process_manager: Option>, + process_manager: Option, ) -> Arc { Arc::new_cyclic(|me| Self { information_extension, @@ -423,7 +422,7 @@ struct SystemCatalog { information_schema_provider: Arc, pg_catalog_provider: Arc, backend: KvBackendRef, - process_manager: Option>, + process_manager: Option, } impl SystemCatalog { diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index 1be462e2dc4b..8562de837bc3 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -14,36 +14,40 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; -use std::fmt::{Display, Formatter}; -use std::str::FromStr; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::RwLock; -use api::v1::frontend::ProcessInfo; +use api::v1::frontend::{ListProcessRequest, ProcessInfo}; +use common_frontend::selector::{FrontendSelector, MetaClientSelector}; +use common_frontend::ProcessManager; use common_telemetry::{debug, info}; use common_time::util::current_time_millis; -use snafu::OptionExt; use crate::error; -pub struct ProcessManager { +pub struct MetaProcessManager { server_addr: String, next_id: AtomicU64, catalogs: RwLock>>, + frontend_selector: Option, } -impl ProcessManager { - /// Create a [ProcessManager] instance with server address and kv client. +impl MetaProcessManager { + /// Create a [MetaProcessManager] instance with server address and kv client. pub fn new(server_addr: String) -> error::Result { Ok(Self { server_addr, next_id: Default::default(), catalogs: Default::default(), + frontend_selector: None, }) } +} +#[async_trait::async_trait] +impl ProcessManager for MetaProcessManager { /// Registers a submitted query. - pub fn register_query( + fn register_query( &self, catalog: String, schemas: Vec, @@ -70,7 +74,7 @@ impl ProcessManager { } /// De-register a query from process list. - pub fn deregister_query(&self, catalog: String, id: u64) { + fn deregister_query(&self, catalog: String, id: u64) { if let Entry::Occupied(mut o) = self.catalogs.write().unwrap().entry(catalog) { let process = o.get_mut().remove(&id); debug!("Deregister process: {:?}", process); @@ -80,16 +84,18 @@ impl ProcessManager { } } - /// De-register all queries running on current frontend. - pub fn deregister_all_queries(&self) { + fn deregister_all_queries(&self) { self.catalogs.write().unwrap().clear(); info!("All queries on {} has been deregistered", self.server_addr); } /// List local running processes in given catalog. - pub fn local_processes(&self, catalog: Option<&str>) -> Vec { + fn local_processes( + &self, + catalog: Option<&str>, + ) -> common_frontend::error::Result> { let catalogs = self.catalogs.read().unwrap(); - if let Some(catalog) = catalog { + let result = if let Some(catalog) = catalog { if let Some(catalogs) = catalogs.get(catalog) { catalogs.values().cloned().collect() } else { @@ -100,48 +106,37 @@ impl ProcessManager { .values() .flat_map(|v| v.values().cloned()) .collect() - } - } -} - -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct DisplayProcessId { - pub server_addr: String, - pub id: u64, -} - -impl Display for DisplayProcessId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}/{}", self.server_addr, self.id) + }; + Ok(result) } -} -impl TryFrom<&str> for DisplayProcessId { - type Error = error::Error; - - fn try_from(value: &str) -> Result { - let mut split = value.split('/'); - let server_addr = split - .next() - .context(error::ParseProcessIdSnafu { s: value })? - .to_string(); - let id = split - .next() - .context(error::ParseProcessIdSnafu { s: value })?; - let id = u64::from_str(id) - .ok() - .context(error::ParseProcessIdSnafu { s: value })?; - Ok(DisplayProcessId { server_addr, id }) + async fn list_all_processes( + &self, + catalog: Option<&str>, + ) -> common_frontend::error::Result> { + let mut processes = vec![]; + if let Some(remote_frontend_selector) = self.frontend_selector.as_ref() { + let frontends = remote_frontend_selector + .select(|node| &node.peer.addr != &self.server_addr) + .await?; + for mut f in frontends { + processes.extend(f.list_process(ListProcessRequest {}).await?.processes); + } + } + processes.extend(self.local_processes(catalog)?); + Ok(processes) } } #[cfg(test)] mod tests { - use crate::process_manager::{DisplayProcessId, ProcessManager}; + use common_frontend::ProcessManager; + + use crate::process_manager::MetaProcessManager; #[tokio::test] async fn test_register_query() { - let process_manager = ProcessManager::new("127.0.0.1:8000".to_string()).unwrap(); + let process_manager = MetaProcessManager::new("127.0.0.1:8000".to_string()).unwrap(); let process_id = process_manager.register_query( "public".to_string(), vec!["test".to_string()], @@ -149,24 +144,13 @@ mod tests { "".to_string(), ); - let running_processes = process_manager.local_processes(None); + let running_processes = process_manager.local_processes(None).unwrap(); assert_eq!(running_processes.len(), 1); assert_eq!(&running_processes[0].frontend, "127.0.0.1:8000"); assert_eq!(running_processes[0].id, process_id); assert_eq!(&running_processes[0].query, "SELECT * FROM table"); process_manager.deregister_query("public".to_string(), process_id); - assert_eq!(process_manager.local_processes(None).len(), 0); - } - - #[test] - fn test_display_process_id() { - assert_eq!( - DisplayProcessId::try_from("1.1.1.1:3000/123").unwrap(), - DisplayProcessId { - server_addr: "1.1.1.1:3000".to_string(), - id: 123, - } - ); + assert_eq!(process_manager.local_processes(None).unwrap().len(), 0); } } diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs index 8bad1a68ea4b..125f6e771189 100644 --- a/src/catalog/src/system_schema/information_schema.rs +++ b/src/catalog/src/system_schema/information_schema.rs @@ -43,6 +43,7 @@ use common_recordbatch::SendableRecordBatchStream; use datatypes::schema::SchemaRef; use lazy_static::lazy_static; use paste::paste; +use common_frontend::ProcessManagerRef; use process_list::InformationSchemaProcessList; use store_api::storage::{ScanRequest, TableId}; use table::metadata::TableType; @@ -52,7 +53,6 @@ use views::InformationSchemaViews; use self::columns::InformationSchemaColumns; use crate::error::{Error, Result}; -use crate::process_manager::ProcessManager; use crate::system_schema::information_schema::cluster_info::InformationSchemaClusterInfo; use crate::system_schema::information_schema::flows::InformationSchemaFlows; use crate::system_schema::information_schema::information_memory_table::get_schema_columns; @@ -116,7 +116,7 @@ macro_rules! setup_memory_table { pub struct InformationSchemaProvider { catalog_name: String, catalog_manager: Weak, - process_manager: Option>, + process_manager: Option, flow_metadata_manager: Arc, tables: HashMap, } @@ -225,7 +225,7 @@ impl InformationSchemaProvider { catalog_name: String, catalog_manager: Weak, flow_metadata_manager: Arc, - process_manager: Option>, + process_manager: Option, ) -> Self { let mut provider = Self { catalog_name, diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 24f2e9dde726..ad2f44d67fa3 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -30,11 +30,11 @@ use datatypes::vectors::{ VectorRef, }; use snafu::ResultExt; +use common_frontend::{DisplayProcessId, ProcessManagerRef}; use store_api::storage::{ScanRequest, TableId}; use crate::error::{self, InternalSnafu}; use crate::information_schema::Predicates; -use crate::process_manager::{DisplayProcessId, ProcessManager}; use crate::system_schema::information_schema::InformationTable; /// Column names of `information_schema.process_list` @@ -51,11 +51,11 @@ const ELAPSED_TIME: &str = "elapsed_time"; /// queries in current cluster. pub struct InformationSchemaProcessList { schema: SchemaRef, - process_manager: Arc, + process_manager: ProcessManagerRef, } impl InformationSchemaProcessList { - pub fn new(process_manager: Arc) -> Self { + pub fn new(process_manager: ProcessManagerRef) -> Self { Self { schema: Self::schema(), process_manager, @@ -115,12 +115,12 @@ impl InformationTable for InformationSchemaProcessList { /// Build running process list. async fn make_process_list( - process_manager: Arc, + process_manager: ProcessManagerRef, request: ScanRequest, ) -> error::Result { let predicates = Predicates::from_scan_request(&Some(request)); let current_time = current_time_millis(); - let queries = process_manager.local_processes(None); + let queries = process_manager.local_processes(None).unwrap(); let mut id_builder = StringVectorBuilder::with_capacity(queries.len()); let mut catalog_builder = StringVectorBuilder::with_capacity(queries.len()); diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index 8f8d410c01c8..143645aa1f75 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -20,7 +20,6 @@ use async_trait::async_trait; use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; use catalog::information_extension::DistributedInformationExtension; use catalog::kvbackend::{CachedKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend}; -use catalog::process_manager::ProcessManager; use clap::Parser; use client::client_manager::NodeClients; use common_base::Plugins; @@ -44,7 +43,7 @@ use servers::export_metrics::ExportMetricsTask; use servers::tls::{TlsMode, TlsOption}; use snafu::{OptionExt, ResultExt}; use tracing_appender::non_blocking::WorkerGuard; - +use catalog::process_manager::MetaProcessManager; use crate::error::{self, Result}; use crate::options::{GlobalOptions, GreptimeOptions}; use crate::{create_resource_limit_metrics, log_versions, App}; @@ -346,7 +345,7 @@ impl StartCommand { Arc::new(DistributedInformationExtension::new(meta_client.clone())); let process_manager = Arc::new( - ProcessManager::new(addrs::resolve_addr( + MetaProcessManager::new(addrs::resolve_addr( &opts.grpc.bind_addr, Some(&opts.grpc.server_addr), )) diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 71301ea5356d..24073b1b2da1 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -21,7 +21,6 @@ use async_trait::async_trait; use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; use catalog::information_schema::InformationExtension; use catalog::kvbackend::KvBackendCatalogManager; -use catalog::process_manager::ProcessManager; use clap::Parser; use client::api::v1::meta::RegionRole; use common_base::readable_size::ReadableSize; @@ -84,7 +83,7 @@ use servers::tls::{TlsMode, TlsOption}; use snafu::ResultExt; use tokio::sync::RwLock; use tracing_appender::non_blocking::WorkerGuard; - +use catalog::process_manager::MetaProcessManager; use crate::error::{Result, StartFlownodeSnafu}; use crate::options::{GlobalOptions, GreptimeOptions}; use crate::{create_resource_limit_metrics, error, log_versions, App}; @@ -529,7 +528,7 @@ impl StartCommand { )); let process_manager = Arc::new( - ProcessManager::new(opts.grpc.server_addr.clone()) + MetaProcessManager::new(opts.grpc.server_addr.clone()) .context(error::BuildProcessManagerSnafu)?, ); let catalog_manager = KvBackendCatalogManager::new( diff --git a/src/common/frontend/Cargo.toml b/src/common/frontend/Cargo.toml index e072cfdfb801..471d7bdedd55 100644 --- a/src/common/frontend/Cargo.toml +++ b/src/common/frontend/Cargo.toml @@ -9,7 +9,6 @@ async-trait.workspace = true common-error.workspace = true common-macro.workspace = true common-grpc.workspace = true -client.workspace = true greptime-proto.workspace = true common-meta.workspace = true meta-client.workspace = true diff --git a/src/common/frontend/src/error.rs b/src/common/frontend/src/error.rs index 8119aad0128b..55db692c3cf5 100644 --- a/src/common/frontend/src/error.rs +++ b/src/common/frontend/src/error.rs @@ -16,7 +16,6 @@ use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; use snafu::{Location, Snafu}; -use tonic::Status; #[derive(Snafu)] #[snafu(visibility(pub))] @@ -29,30 +28,21 @@ pub enum Error { source: BoxedError, }, - #[snafu(display("Failed to send gRPC request"))] - Grpc { - source: client::error::Error, + #[snafu(display("Failed to list nodes from metasrv"))] + Meta { + source: Box, #[snafu(implicit)] location: Location, }, - #[snafu(display("Failed to list nodes from metasrv"))] - Meta { - source: meta_client::error::Error, + #[snafu(display("Failed to parse process id: {}", s))] + ParseProcessId { + s: String, #[snafu(implicit)] location: Location, }, } -impl From for Error { - fn from(value: Status) -> Self { - Self::Grpc { - source: client::error::Error::from(value), - location: Location::default(), - } - } -} - pub type Result = std::result::Result; impl ErrorExt for Error { @@ -60,8 +50,8 @@ impl ErrorExt for Error { use Error::*; match self { External { source, .. } => source.status_code(), - Grpc { source, .. } => source.status_code(), Meta { source, .. } => source.status_code(), + Error::ParseProcessId { .. } => StatusCode::InvalidArguments, } } diff --git a/src/common/frontend/src/lib.rs b/src/common/frontend/src/lib.rs index f7e1263b1c45..0d1f6dce08ca 100644 --- a/src/common/frontend/src/lib.rs +++ b/src/common/frontend/src/lib.rs @@ -12,5 +12,64 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::fmt::{Display, Formatter}; +use std::str::FromStr; +use std::sync::Arc; + +use greptime_proto::v1::frontend::ProcessInfo; +use snafu::OptionExt; + pub mod error; pub mod selector; + +pub type ProcessManagerRef = Arc; + +#[async_trait::async_trait] +pub trait ProcessManager: Send+Sync { + fn register_query( + &self, + catalog: String, + schemas: Vec, + query: String, + client: String, + ) -> u64; + + fn deregister_query(&self, catalog: String, id: u64); + + fn deregister_all_queries(&self); + + fn local_processes(&self, catalog: Option<&str>) -> error::Result>; + + async fn list_all_processes(&self, catalog: Option<&str>) -> error::Result>; +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct DisplayProcessId { + pub server_addr: String, + pub id: u64, +} + +impl Display for DisplayProcessId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}/{}", self.server_addr, self.id) + } +} + +impl TryFrom<&str> for DisplayProcessId { + type Error = error::Error; + + fn try_from(value: &str) -> Result { + let mut split = value.split('/'); + let server_addr = split + .next() + .context(error::ParseProcessIdSnafu { s: value })? + .to_string(); + let id = split + .next() + .context(error::ParseProcessIdSnafu { s: value })?; + let id = u64::from_str(id) + .ok() + .context(error::ParseProcessIdSnafu { s: value })?; + Ok(DisplayProcessId { server_addr, id }) + } +} diff --git a/src/common/frontend/src/selector.rs b/src/common/frontend/src/selector.rs index bbc57e4b0017..ded72f30ff7f 100644 --- a/src/common/frontend/src/selector.rs +++ b/src/common/frontend/src/selector.rs @@ -14,19 +14,18 @@ use std::time::Duration; -use client::Client; use common_grpc::channel_manager::{ChannelConfig, ChannelManager}; -use common_meta::cluster::{ClusterInfo, Role}; +use common_meta::cluster::{ClusterInfo, NodeInfo, Role}; use greptime_proto::v1::frontend::{frontend_client, ListProcessRequest, ListProcessResponse}; use meta_client::MetaClientRef; use snafu::ResultExt; -use crate::error::{GrpcSnafu, MetaSnafu, Result}; +use crate::error::{MetaSnafu, Result}; pub type FrontendClientPtr = Box; #[async_trait::async_trait] -pub trait FrontendClient { +pub trait FrontendClient: Send { async fn list_process(&mut self, req: ListProcessRequest) -> Result; } @@ -35,10 +34,9 @@ impl FrontendClient for frontend_client::FrontendClient Result { let response: ListProcessResponse = frontend_client::FrontendClient::< tonic::transport::channel::Channel, - >::list_process(&mut self, req) + >::list_process(self, req) .await - .map_err(client::error::Error::from) - .context(GrpcSnafu)? + .unwrap() .into_inner(); Ok(response) } @@ -46,7 +44,9 @@ impl FrontendClient for frontend_client::FrontendClient Result>; + async fn select(&self, predicate: F) -> Result> + where + F: Fn(&NodeInfo) -> bool + Send; } #[derive(Debug, Clone)] @@ -57,22 +57,26 @@ pub struct MetaClientSelector { #[async_trait::async_trait] impl FrontendSelector for MetaClientSelector { - async fn select_all(&self) -> Result> { + async fn select(&self, predicate: F) -> Result> + where + F: Fn(&NodeInfo) -> bool + Send, + { let nodes = self .meta_client .list_nodes(Some(Role::Frontend)) .await + .map_err(Box::new) .context(MetaSnafu)?; let res = nodes .into_iter() + .filter(predicate) .map(|node| { - Client::with_manager_and_urls(self.channel_manager.clone(), vec![node.peer.addr]) - .frontend_client() - .map(|c| Box::new(c) as FrontendClientPtr) + let channel = self.channel_manager.get(node.peer.addr).unwrap(); + let client = frontend_client::FrontendClient::new(channel); + Box::new(client) as FrontendClientPtr }) - .collect::>>() - .context(GrpcSnafu)?; + .collect::>(); Ok(res) } } @@ -107,7 +111,7 @@ mod tests { .await .unwrap(); let selector = MetaClientSelector::new(Arc::new(meta_client)); - let clients = selector.select_all().await.unwrap(); + let clients = selector.select(|_| true).await.unwrap(); for mut client in clients { let resp = client.list_process(ListProcessRequest {}).await; println!("{:?}", resp); diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index c0990a5ac38e..b40d04c75181 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -30,7 +30,6 @@ use std::time::SystemTime; use async_trait::async_trait; use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq}; -use catalog::process_manager::ProcessManager; use catalog::CatalogManagerRef; use client::OutputData; use common_base::Plugins; @@ -72,6 +71,7 @@ use sql::parser::{ParseOptions, ParserContext}; use sql::statements::copy::{CopyDatabase, CopyTable}; use sql::statements::statement::Statement; use sqlparser::ast::ObjectName; +use common_frontend::ProcessManagerRef; pub use standalone::StandaloneDatanodeManager; use crate::error::{ @@ -98,7 +98,7 @@ pub struct Instance { slow_query_recorder: Option, limiter: Option, #[allow(dead_code)] - process_manager: Option>, + process_manager: Option, } impl Instance { diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index 0a248ad9c1e8..7d445ea1f3f2 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -15,7 +15,6 @@ use std::sync::Arc; use cache::{TABLE_FLOWNODE_SET_CACHE_NAME, TABLE_ROUTE_CACHE_NAME}; -use catalog::process_manager::ProcessManager; use catalog::CatalogManagerRef; use common_base::Plugins; use common_meta::cache::{LayeredCacheRegistryRef, TableRouteCacheRef}; @@ -37,7 +36,7 @@ use pipeline::pipeline_operator::PipelineOperator; use query::region_query::RegionQueryHandlerFactoryRef; use query::QueryEngineFactory; use snafu::OptionExt; - +use common_frontend::ProcessManagerRef; use crate::error::{self, Result}; use crate::frontend::FrontendOptions; use crate::instance::region_query::FrontendRegionQueryHandler; @@ -55,7 +54,7 @@ pub struct FrontendBuilder { node_manager: NodeManagerRef, plugins: Option, procedure_executor: ProcedureExecutorRef, - process_manager: Option>, + process_manager: Option, } impl FrontendBuilder { @@ -67,7 +66,7 @@ impl FrontendBuilder { catalog_manager: CatalogManagerRef, node_manager: NodeManagerRef, procedure_executor: ProcedureExecutorRef, - process_manager: Option>, + process_manager: Option, ) -> Self { Self { options, diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index d3d299a15592..92f10e05ab93 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -38,6 +38,7 @@ common-catalog.workspace = true common-config.workspace = true common-error.workspace = true common-grpc.workspace = true +common-frontend.workspace = true common-macro.workspace = true common-mem-prof = { workspace = true, optional = true } common-meta.workspace = true diff --git a/src/servers/src/grpc.rs b/src/servers/src/grpc.rs index bad5dd9ae746..ac88db634f12 100644 --- a/src/servers/src/grpc.rs +++ b/src/servers/src/grpc.rs @@ -20,6 +20,7 @@ pub mod flight; pub mod greptime_handler; pub mod prom_query_gateway; pub mod region_server; +mod frontend_grpc_handler; use std::net::SocketAddr; diff --git a/src/servers/src/grpc/builder.rs b/src/servers/src/grpc/builder.rs index a53a2f3c39da..29f0ed589470 100644 --- a/src/servers/src/grpc/builder.rs +++ b/src/servers/src/grpc/builder.rs @@ -30,6 +30,7 @@ use tonic::transport::{Identity, ServerTlsConfig}; use crate::grpc::database::DatabaseService; use crate::grpc::flight::{FlightCraftRef, FlightCraftWrapper}; +use crate::grpc::frontend_grpc_handler::FrontendGrpcHandler; use crate::grpc::greptime_handler::GreptimeRequestHandler; use crate::grpc::prom_query_gateway::PrometheusGatewayService; use crate::grpc::region_server::{RegionServerHandlerRef, RegionServerRequestHandler}; @@ -126,6 +127,12 @@ impl GrpcServerBuilder { self } + /// Add handler for Frontend gRPC service. + pub fn frontend_grpc_handler(mut self, handler: FrontendGrpcHandler) -> Self { + add_service!(self, FrontendServer::new(handler)); + self + } + /// Add handler for [OtelArrowService]. pub fn otel_arrow_handler( mut self, diff --git a/src/servers/src/grpc/frontend_grpc_handler.rs b/src/servers/src/grpc/frontend_grpc_handler.rs new file mode 100644 index 000000000000..1596550ed63a --- /dev/null +++ b/src/servers/src/grpc/frontend_grpc_handler.rs @@ -0,0 +1,40 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use api::v1::frontend::frontend_server::Frontend; +use api::v1::frontend::{ListProcessRequest, ListProcessResponse}; +use common_frontend::ProcessManagerRef; +use tonic::{Request, Response, Status}; + +#[derive(Clone)] +pub struct FrontendGrpcHandler { + process_manager: ProcessManagerRef, +} + +impl FrontendGrpcHandler { + pub fn new(process_manager: ProcessManagerRef) -> Self { + Self { process_manager } + } +} + +#[async_trait::async_trait] +impl Frontend for FrontendGrpcHandler { + async fn list_process( + &self, + _request: Request, + ) -> Result, Status> { + let processes = self.process_manager.local_processes(None).unwrap(); + Ok(Response::new(ListProcessResponse { processes })) + } +} From 5f986baa743036e7b04e02b5fb518b97e5cdc534 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 11 Jun 2025 12:51:36 +0000 Subject: [PATCH 22/31] feat/show-process-list: ### Add `MetaClientRef` to `MetaProcessManager` and Update Instantiation - **Files Modified**: - `src/catalog/src/process_manager.rs` - `src/cmd/src/frontend.rs` - `src/cmd/src/standalone.rs` - **Key Changes**: - Added `MetaClientRef` as an optional parameter to the `MetaProcessManager::new` method. - Updated instantiation of `MetaProcessManager` to include `MetaClientRef` where applicable. ### Update `ProcessManagerRef` Usage - **Files Modified**: - `src/catalog/src/kvbackend/manager.rs` - `src/catalog/src/system_schema/information_schema.rs` - `src/catalog/src/system_schema/information_schema/process_list.rs` - `src/frontend/src/instance.rs` - `src/frontend/src/instance/builder.rs` - **Key Changes**: - Ensured consistent usage of `ProcessManagerRef` across various modules. Signed-off-by: Lei, HUANG --- src/catalog/src/kvbackend/manager.rs | 3 ++- src/catalog/src/process_manager.rs | 8 +++++--- src/catalog/src/system_schema/information_schema.rs | 2 +- .../system_schema/information_schema/process_list.rs | 2 +- src/cmd/src/frontend.rs | 11 ++++++----- src/cmd/src/standalone.rs | 5 +++-- src/common/frontend/src/lib.rs | 2 +- src/common/frontend/src/selector.rs | 2 +- src/frontend/src/instance.rs | 2 +- src/frontend/src/instance/builder.rs | 3 ++- src/servers/src/grpc.rs | 2 +- 11 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs index b7089bc8ce55..f0c19964e949 100644 --- a/src/catalog/src/kvbackend/manager.rs +++ b/src/catalog/src/kvbackend/manager.rs @@ -22,6 +22,7 @@ use common_catalog::consts::{ PG_CATALOG_NAME, }; use common_error::ext::BoxedError; +use common_frontend::ProcessManagerRef; use common_meta::cache::{LayeredCacheRegistryRef, ViewInfoCacheRef}; use common_meta::key::catalog_name::CatalogNameKey; use common_meta::key::flow::FlowMetadataManager; @@ -44,7 +45,7 @@ use table::table_name::TableName; use table::TableRef; use tokio::sync::Semaphore; use tokio_stream::wrappers::ReceiverStream; -use common_frontend::ProcessManagerRef; + use crate::error::{ CacheNotFoundSnafu, GetTableCacheSnafu, InvalidTableInfoInCatalogSnafu, ListCatalogsSnafu, ListSchemasSnafu, ListTablesSnafu, Result, TableMetadataManagerSnafu, diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index 8562de837bc3..34930578a7f7 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -22,6 +22,7 @@ use common_frontend::selector::{FrontendSelector, MetaClientSelector}; use common_frontend::ProcessManager; use common_telemetry::{debug, info}; use common_time::util::current_time_millis; +use meta_client::MetaClientRef; use crate::error; @@ -34,12 +35,13 @@ pub struct MetaProcessManager { impl MetaProcessManager { /// Create a [MetaProcessManager] instance with server address and kv client. - pub fn new(server_addr: String) -> error::Result { + pub fn new(server_addr: String, meta_client: Option) -> error::Result { + let frontend_selector = meta_client.map(MetaClientSelector::new); Ok(Self { server_addr, next_id: Default::default(), catalogs: Default::default(), - frontend_selector: None, + frontend_selector, }) } } @@ -136,7 +138,7 @@ mod tests { #[tokio::test] async fn test_register_query() { - let process_manager = MetaProcessManager::new("127.0.0.1:8000".to_string()).unwrap(); + let process_manager = MetaProcessManager::new("127.0.0.1:8000".to_string(), None).unwrap(); let process_id = process_manager.register_query( "public".to_string(), vec!["test".to_string()], diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs index 125f6e771189..8296eb6dc70e 100644 --- a/src/catalog/src/system_schema/information_schema.rs +++ b/src/catalog/src/system_schema/information_schema.rs @@ -34,6 +34,7 @@ use std::sync::{Arc, Weak}; use common_catalog::consts::{self, DEFAULT_CATALOG_NAME, INFORMATION_SCHEMA_NAME}; use common_error::ext::ErrorExt; +use common_frontend::ProcessManagerRef; use common_meta::cluster::NodeInfo; use common_meta::datanode::RegionStat; use common_meta::key::flow::flow_state::FlowStat; @@ -43,7 +44,6 @@ use common_recordbatch::SendableRecordBatchStream; use datatypes::schema::SchemaRef; use lazy_static::lazy_static; use paste::paste; -use common_frontend::ProcessManagerRef; use process_list::InformationSchemaProcessList; use store_api::storage::{ScanRequest, TableId}; use table::metadata::TableType; diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index ad2f44d67fa3..0a072d81b088 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use common_catalog::consts::INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID; use common_error::ext::BoxedError; +use common_frontend::{DisplayProcessId, ProcessManagerRef}; use common_recordbatch::adapter::RecordBatchStreamAdapter; use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; use common_time::util::current_time_millis; @@ -30,7 +31,6 @@ use datatypes::vectors::{ VectorRef, }; use snafu::ResultExt; -use common_frontend::{DisplayProcessId, ProcessManagerRef}; use store_api::storage::{ScanRequest, TableId}; use crate::error::{self, InternalSnafu}; diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index 143645aa1f75..8e82b0544f82 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -20,6 +20,7 @@ use async_trait::async_trait; use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; use catalog::information_extension::DistributedInformationExtension; use catalog::kvbackend::{CachedKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend}; +use catalog::process_manager::MetaProcessManager; use clap::Parser; use client::client_manager::NodeClients; use common_base::Plugins; @@ -43,7 +44,7 @@ use servers::export_metrics::ExportMetricsTask; use servers::tls::{TlsMode, TlsOption}; use snafu::{OptionExt, ResultExt}; use tracing_appender::non_blocking::WorkerGuard; -use catalog::process_manager::MetaProcessManager; + use crate::error::{self, Result}; use crate::options::{GlobalOptions, GreptimeOptions}; use crate::{create_resource_limit_metrics, log_versions, App}; @@ -345,10 +346,10 @@ impl StartCommand { Arc::new(DistributedInformationExtension::new(meta_client.clone())); let process_manager = Arc::new( - MetaProcessManager::new(addrs::resolve_addr( - &opts.grpc.bind_addr, - Some(&opts.grpc.server_addr), - )) + MetaProcessManager::new( + addrs::resolve_addr(&opts.grpc.bind_addr, Some(&opts.grpc.server_addr)), + Some(meta_client.clone()), + ) .context(error::BuildProcessManagerSnafu)?, ); let catalog_manager = KvBackendCatalogManager::new( diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 24073b1b2da1..c2ad96036fa7 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -21,6 +21,7 @@ use async_trait::async_trait; use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; use catalog::information_schema::InformationExtension; use catalog::kvbackend::KvBackendCatalogManager; +use catalog::process_manager::MetaProcessManager; use clap::Parser; use client::api::v1::meta::RegionRole; use common_base::readable_size::ReadableSize; @@ -83,7 +84,7 @@ use servers::tls::{TlsMode, TlsOption}; use snafu::ResultExt; use tokio::sync::RwLock; use tracing_appender::non_blocking::WorkerGuard; -use catalog::process_manager::MetaProcessManager; + use crate::error::{Result, StartFlownodeSnafu}; use crate::options::{GlobalOptions, GreptimeOptions}; use crate::{create_resource_limit_metrics, error, log_versions, App}; @@ -528,7 +529,7 @@ impl StartCommand { )); let process_manager = Arc::new( - MetaProcessManager::new(opts.grpc.server_addr.clone()) + MetaProcessManager::new(opts.grpc.server_addr.clone(), None) .context(error::BuildProcessManagerSnafu)?, ); let catalog_manager = KvBackendCatalogManager::new( diff --git a/src/common/frontend/src/lib.rs b/src/common/frontend/src/lib.rs index 0d1f6dce08ca..3c72c8b1d756 100644 --- a/src/common/frontend/src/lib.rs +++ b/src/common/frontend/src/lib.rs @@ -25,7 +25,7 @@ pub mod selector; pub type ProcessManagerRef = Arc; #[async_trait::async_trait] -pub trait ProcessManager: Send+Sync { +pub trait ProcessManager: Send + Sync { fn register_query( &self, catalog: String, diff --git a/src/common/frontend/src/selector.rs b/src/common/frontend/src/selector.rs index ded72f30ff7f..e5ace8b81fc2 100644 --- a/src/common/frontend/src/selector.rs +++ b/src/common/frontend/src/selector.rs @@ -25,7 +25,7 @@ use crate::error::{MetaSnafu, Result}; pub type FrontendClientPtr = Box; #[async_trait::async_trait] -pub trait FrontendClient: Send { +pub trait FrontendClient: Send { async fn list_process(&mut self, req: ListProcessRequest) -> Result; } diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index b40d04c75181..5e76496ca448 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -35,6 +35,7 @@ use client::OutputData; use common_base::Plugins; use common_config::KvBackendConfig; use common_error::ext::{BoxedError, ErrorExt}; +use common_frontend::ProcessManagerRef; use common_meta::key::TableMetadataManagerRef; use common_meta::kv_backend::KvBackendRef; use common_meta::state_store::KvStateStore; @@ -71,7 +72,6 @@ use sql::parser::{ParseOptions, ParserContext}; use sql::statements::copy::{CopyDatabase, CopyTable}; use sql::statements::statement::Statement; use sqlparser::ast::ObjectName; -use common_frontend::ProcessManagerRef; pub use standalone::StandaloneDatanodeManager; use crate::error::{ diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index 7d445ea1f3f2..9e8909262490 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use cache::{TABLE_FLOWNODE_SET_CACHE_NAME, TABLE_ROUTE_CACHE_NAME}; use catalog::CatalogManagerRef; use common_base::Plugins; +use common_frontend::ProcessManagerRef; use common_meta::cache::{LayeredCacheRegistryRef, TableRouteCacheRef}; use common_meta::cache_invalidator::{CacheInvalidatorRef, DummyCacheInvalidator}; use common_meta::ddl::ProcedureExecutorRef; @@ -36,7 +37,7 @@ use pipeline::pipeline_operator::PipelineOperator; use query::region_query::RegionQueryHandlerFactoryRef; use query::QueryEngineFactory; use snafu::OptionExt; -use common_frontend::ProcessManagerRef; + use crate::error::{self, Result}; use crate::frontend::FrontendOptions; use crate::instance::region_query::FrontendRegionQueryHandler; diff --git a/src/servers/src/grpc.rs b/src/servers/src/grpc.rs index ac88db634f12..60dc175f3b24 100644 --- a/src/servers/src/grpc.rs +++ b/src/servers/src/grpc.rs @@ -17,10 +17,10 @@ pub mod builder; mod cancellation; mod database; pub mod flight; +mod frontend_grpc_handler; pub mod greptime_handler; pub mod prom_query_gateway; pub mod region_server; -mod frontend_grpc_handler; use std::net::SocketAddr; From b375458b73f31b02d31f00ed24486cd30a7cd72a Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 11 Jun 2025 13:50:41 +0000 Subject: [PATCH 23/31] feat/show-process-list: ## Refactor Process Management - **Unified Process Manager**: - Replaced `MetaProcessManager` with `ProcessManager` across the codebase. - Updated `ProcessManager` to use `Arc` for shared references and introduced a `Ticket` struct for query registration and deregistration. - Affected files: `manager.rs`, `process_manager.rs`, `frontend.rs`, `standalone.rs`, `frontend_grpc_handler.rs`, `instance.rs`, `builder.rs`, `cluster.rs`, `standalone.rs`. - **Stream Wrapper Implementation**: - Added `StreamWrapper` to handle record batch streams with process management. - Affected file: `stream_wrapper.rs`. - **Test Adjustments**: - Updated tests to align with the new `ProcessManager` implementation. - Affected file: `tests-integration/src/cluster.rs`, `tests-integration/src/standalone.rs`. Signed-off-by: Lei, HUANG --- src/catalog/src/kvbackend/manager.rs | 2 +- src/catalog/src/process_manager.rs | 67 ++++++++++++------- .../src/system_schema/information_schema.rs | 2 +- .../information_schema/process_list.rs | 3 +- src/cmd/src/frontend.rs | 15 ++--- src/cmd/src/standalone.rs | 9 +-- src/common/frontend/src/lib.rs | 23 ------- src/frontend/src/instance.rs | 28 ++++++-- src/frontend/src/instance/builder.rs | 6 +- src/frontend/src/lib.rs | 1 + src/frontend/src/stream_wrapper.rs | 46 +++++++++++++ src/servers/src/grpc/frontend_grpc_handler.rs | 2 +- tests-integration/src/cluster.rs | 4 +- tests-integration/src/standalone.rs | 5 +- 14 files changed, 136 insertions(+), 77 deletions(-) create mode 100644 src/frontend/src/stream_wrapper.rs diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs index f0c19964e949..f486d35bdd75 100644 --- a/src/catalog/src/kvbackend/manager.rs +++ b/src/catalog/src/kvbackend/manager.rs @@ -22,7 +22,6 @@ use common_catalog::consts::{ PG_CATALOG_NAME, }; use common_error::ext::BoxedError; -use common_frontend::ProcessManagerRef; use common_meta::cache::{LayeredCacheRegistryRef, ViewInfoCacheRef}; use common_meta::key::catalog_name::CatalogNameKey; use common_meta::key::flow::FlowMetadataManager; @@ -52,6 +51,7 @@ use crate::error::{ }; use crate::information_schema::{InformationExtensionRef, InformationSchemaProvider}; use crate::kvbackend::TableCacheRef; +use crate::process_manager::ProcessManagerRef; use crate::system_schema::pg_catalog::PGCatalogProvider; use crate::system_schema::SystemSchemaProvider; use crate::CatalogManager; diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index 34930578a7f7..19d8339a60f1 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -15,47 +15,45 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use api::v1::frontend::{ListProcessRequest, ProcessInfo}; use common_frontend::selector::{FrontendSelector, MetaClientSelector}; -use common_frontend::ProcessManager; use common_telemetry::{debug, info}; use common_time::util::current_time_millis; use meta_client::MetaClientRef; -use crate::error; +pub type ProcessManagerRef = Arc; -pub struct MetaProcessManager { +pub struct ProcessManager { server_addr: String, next_id: AtomicU64, catalogs: RwLock>>, frontend_selector: Option, } -impl MetaProcessManager { - /// Create a [MetaProcessManager] instance with server address and kv client. - pub fn new(server_addr: String, meta_client: Option) -> error::Result { +impl ProcessManager { + /// Create a [ProcessManager] instance with server address and kv client. + pub fn new(server_addr: String, meta_client: Option) -> Self { let frontend_selector = meta_client.map(MetaClientSelector::new); - Ok(Self { + Self { server_addr, next_id: Default::default(), catalogs: Default::default(), frontend_selector, - }) + } } } -#[async_trait::async_trait] -impl ProcessManager for MetaProcessManager { +impl ProcessManager { /// Registers a submitted query. - fn register_query( - &self, + pub fn register_query( + self: &Arc, catalog: String, schemas: Vec, query: String, client: String, - ) -> u64 { + ) -> Ticket { let id = self.next_id.fetch_add(1, Ordering::Relaxed); let process = ProcessInfo { id, @@ -69,14 +67,18 @@ impl ProcessManager for MetaProcessManager { self.catalogs .write() .unwrap() - .entry(catalog) + .entry(catalog.clone()) .or_default() .insert(id, process); - id + Ticket { + catalog, + manager: self.clone(), + id, + } } /// De-register a query from process list. - fn deregister_query(&self, catalog: String, id: u64) { + pub fn deregister_query(&self, catalog: String, id: u64) { if let Entry::Occupied(mut o) = self.catalogs.write().unwrap().entry(catalog) { let process = o.get_mut().remove(&id); debug!("Deregister process: {:?}", process); @@ -86,13 +88,13 @@ impl ProcessManager for MetaProcessManager { } } - fn deregister_all_queries(&self) { + pub fn deregister_all_queries(&self) { self.catalogs.write().unwrap().clear(); info!("All queries on {} has been deregistered", self.server_addr); } /// List local running processes in given catalog. - fn local_processes( + pub fn local_processes( &self, catalog: Option<&str>, ) -> common_frontend::error::Result> { @@ -112,7 +114,7 @@ impl ProcessManager for MetaProcessManager { Ok(result) } - async fn list_all_processes( + pub async fn list_all_processes( &self, catalog: Option<&str>, ) -> common_frontend::error::Result> { @@ -130,16 +132,29 @@ impl ProcessManager for MetaProcessManager { } } +pub struct Ticket { + pub(crate) catalog: String, + pub(crate) manager: ProcessManagerRef, + pub(crate) id: u64, +} + +impl Drop for Ticket { + fn drop(&mut self) { + self.manager + .deregister_query(std::mem::take(&mut self.catalog), self.id); + } +} + #[cfg(test)] mod tests { - use common_frontend::ProcessManager; + use std::sync::Arc; - use crate::process_manager::MetaProcessManager; + use crate::process_manager::ProcessManager; #[tokio::test] async fn test_register_query() { - let process_manager = MetaProcessManager::new("127.0.0.1:8000".to_string(), None).unwrap(); - let process_id = process_manager.register_query( + let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None)); + let ticket = process_manager.clone().register_query( "public".to_string(), vec!["test".to_string()], "SELECT * FROM table".to_string(), @@ -149,10 +164,10 @@ mod tests { let running_processes = process_manager.local_processes(None).unwrap(); assert_eq!(running_processes.len(), 1); assert_eq!(&running_processes[0].frontend, "127.0.0.1:8000"); - assert_eq!(running_processes[0].id, process_id); + assert_eq!(running_processes[0].id, ticket.id); assert_eq!(&running_processes[0].query, "SELECT * FROM table"); - process_manager.deregister_query("public".to_string(), process_id); + drop(ticket); assert_eq!(process_manager.local_processes(None).unwrap().len(), 0); } } diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs index 8296eb6dc70e..5e2480985229 100644 --- a/src/catalog/src/system_schema/information_schema.rs +++ b/src/catalog/src/system_schema/information_schema.rs @@ -34,7 +34,6 @@ use std::sync::{Arc, Weak}; use common_catalog::consts::{self, DEFAULT_CATALOG_NAME, INFORMATION_SCHEMA_NAME}; use common_error::ext::ErrorExt; -use common_frontend::ProcessManagerRef; use common_meta::cluster::NodeInfo; use common_meta::datanode::RegionStat; use common_meta::key::flow::flow_state::FlowStat; @@ -53,6 +52,7 @@ use views::InformationSchemaViews; use self::columns::InformationSchemaColumns; use crate::error::{Error, Result}; +use crate::process_manager::ProcessManagerRef; use crate::system_schema::information_schema::cluster_info::InformationSchemaClusterInfo; use crate::system_schema::information_schema::flows::InformationSchemaFlows; use crate::system_schema::information_schema::information_memory_table::get_schema_columns; diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 0a072d81b088..99de40d3672c 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use common_catalog::consts::INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID; use common_error::ext::BoxedError; -use common_frontend::{DisplayProcessId, ProcessManagerRef}; +use common_frontend::DisplayProcessId; use common_recordbatch::adapter::RecordBatchStreamAdapter; use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; use common_time::util::current_time_millis; @@ -35,6 +35,7 @@ use store_api::storage::{ScanRequest, TableId}; use crate::error::{self, InternalSnafu}; use crate::information_schema::Predicates; +use crate::process_manager::ProcessManagerRef; use crate::system_schema::information_schema::InformationTable; /// Column names of `information_schema.process_list` diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index 8e82b0544f82..76f247796689 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -20,7 +20,7 @@ use async_trait::async_trait; use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; use catalog::information_extension::DistributedInformationExtension; use catalog::kvbackend::{CachedKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend}; -use catalog::process_manager::MetaProcessManager; +use catalog::process_manager::ProcessManager; use clap::Parser; use client::client_manager::NodeClients; use common_base::Plugins; @@ -345,13 +345,10 @@ impl StartCommand { let information_extension = Arc::new(DistributedInformationExtension::new(meta_client.clone())); - let process_manager = Arc::new( - MetaProcessManager::new( - addrs::resolve_addr(&opts.grpc.bind_addr, Some(&opts.grpc.server_addr)), - Some(meta_client.clone()), - ) - .context(error::BuildProcessManagerSnafu)?, - ); + let process_manager = Arc::new(ProcessManager::new( + addrs::resolve_addr(&opts.grpc.bind_addr, Some(&opts.grpc.server_addr)), + Some(meta_client.clone()), + )); let catalog_manager = KvBackendCatalogManager::new( information_extension, cached_meta_backend.clone(), @@ -390,7 +387,7 @@ impl StartCommand { catalog_manager, Arc::new(client), meta_client, - Some(process_manager), + process_manager, ) .with_plugin(plugins.clone()) .with_local_cache_invalidator(layered_cache_registry) diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index c2ad96036fa7..4146e54e566c 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -21,7 +21,7 @@ use async_trait::async_trait; use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry}; use catalog::information_schema::InformationExtension; use catalog::kvbackend::KvBackendCatalogManager; -use catalog::process_manager::MetaProcessManager; +use catalog::process_manager::ProcessManager; use clap::Parser; use client::api::v1::meta::RegionRole; use common_base::readable_size::ReadableSize; @@ -528,10 +528,7 @@ impl StartCommand { procedure_manager.clone(), )); - let process_manager = Arc::new( - MetaProcessManager::new(opts.grpc.server_addr.clone(), None) - .context(error::BuildProcessManagerSnafu)?, - ); + let process_manager = Arc::new(ProcessManager::new(opts.grpc.server_addr.clone(), None)); let catalog_manager = KvBackendCatalogManager::new( information_extension.clone(), kv_backend.clone(), @@ -627,7 +624,7 @@ impl StartCommand { catalog_manager.clone(), node_manager.clone(), ddl_task_executor.clone(), - Some(process_manager), + process_manager, ) .with_plugin(plugins.clone()) .try_build() diff --git a/src/common/frontend/src/lib.rs b/src/common/frontend/src/lib.rs index 3c72c8b1d756..ab07f54168a6 100644 --- a/src/common/frontend/src/lib.rs +++ b/src/common/frontend/src/lib.rs @@ -14,35 +14,12 @@ use std::fmt::{Display, Formatter}; use std::str::FromStr; -use std::sync::Arc; -use greptime_proto::v1::frontend::ProcessInfo; use snafu::OptionExt; pub mod error; pub mod selector; -pub type ProcessManagerRef = Arc; - -#[async_trait::async_trait] -pub trait ProcessManager: Send + Sync { - fn register_query( - &self, - catalog: String, - schemas: Vec, - query: String, - client: String, - ) -> u64; - - fn deregister_query(&self, catalog: String, id: u64); - - fn deregister_all_queries(&self); - - fn local_processes(&self, catalog: Option<&str>) -> error::Result>; - - async fn list_all_processes(&self, catalog: Option<&str>) -> error::Result>; -} - #[derive(Debug, Clone, Eq, PartialEq)] pub struct DisplayProcessId { pub server_addr: String, diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index 5e76496ca448..8527000f24dd 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -30,12 +30,12 @@ use std::time::SystemTime; use async_trait::async_trait; use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq}; +use catalog::process_manager::ProcessManagerRef; use catalog::CatalogManagerRef; use client::OutputData; use common_base::Plugins; use common_config::KvBackendConfig; use common_error::ext::{BoxedError, ErrorExt}; -use common_frontend::ProcessManagerRef; use common_meta::key::TableMetadataManagerRef; use common_meta::kv_backend::KvBackendRef; use common_meta::state_store::KvStateStore; @@ -81,6 +81,7 @@ use crate::error::{ }; use crate::limiter::LimiterRef; use crate::slow_query_recorder::SlowQueryRecorder; +use crate::stream_wrapper::StreamWrapper; /// The frontend instance contains necessary components, and implements many /// traits, like [`servers::query_handler::grpc::GrpcQueryHandler`], @@ -97,8 +98,7 @@ pub struct Instance { table_metadata_manager: TableMetadataManagerRef, slow_query_recorder: Option, limiter: Option, - #[allow(dead_code)] - process_manager: Option, + process_manager: ProcessManagerRef, } impl Instance { @@ -175,6 +175,13 @@ impl Instance { None }; + let ticket = self.process_manager.register_query( + query_ctx.current_catalog().to_string(), + vec![query_ctx.current_schema()], + stmt.to_string(), + "unknown".to_string(), + ); + let output = match stmt { Statement::Query(_) | Statement::Explain(_) | Statement::Delete(_) => { // TODO: remove this when format is supported in datafusion @@ -218,7 +225,20 @@ impl Instance { } }; - output.context(TableOperationSnafu) + match output { + Ok(output) => { + let Output { meta, data } = output; + + let data = match data { + OutputData::Stream(stream) => { + OutputData::Stream(Box::pin(StreamWrapper::new(stream, ticket))) + } + other => other, + }; + Ok(Output { data, meta }) + } + Err(e) => Err(e).context(TableOperationSnafu), + } } } diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index 9e8909262490..3a5dfcca3b3d 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use cache::{TABLE_FLOWNODE_SET_CACHE_NAME, TABLE_ROUTE_CACHE_NAME}; +use catalog::process_manager::ProcessManagerRef; use catalog::CatalogManagerRef; use common_base::Plugins; -use common_frontend::ProcessManagerRef; use common_meta::cache::{LayeredCacheRegistryRef, TableRouteCacheRef}; use common_meta::cache_invalidator::{CacheInvalidatorRef, DummyCacheInvalidator}; use common_meta::ddl::ProcedureExecutorRef; @@ -55,7 +55,7 @@ pub struct FrontendBuilder { node_manager: NodeManagerRef, plugins: Option, procedure_executor: ProcedureExecutorRef, - process_manager: Option, + process_manager: ProcessManagerRef, } impl FrontendBuilder { @@ -67,7 +67,7 @@ impl FrontendBuilder { catalog_manager: CatalogManagerRef, node_manager: NodeManagerRef, procedure_executor: ProcedureExecutorRef, - process_manager: Option, + process_manager: ProcessManagerRef, ) -> Self { Self { options, diff --git a/src/frontend/src/lib.rs b/src/frontend/src/lib.rs index 7672ae26c41d..9fcfe4e87035 100644 --- a/src/frontend/src/lib.rs +++ b/src/frontend/src/lib.rs @@ -23,3 +23,4 @@ pub(crate) mod metrics; pub mod server; pub mod service_config; pub(crate) mod slow_query_recorder; +mod stream_wrapper; diff --git a/src/frontend/src/stream_wrapper.rs b/src/frontend/src/stream_wrapper.rs new file mode 100644 index 000000000000..c851067037ed --- /dev/null +++ b/src/frontend/src/stream_wrapper.rs @@ -0,0 +1,46 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + +use common_recordbatch::adapter::RecordBatchMetrics; +use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream}; +use datatypes::schema::SchemaRef; +use futures::Stream; + +pub struct StreamWrapper { + inner: SendableRecordBatchStream, + _attachment: T, +} + +impl Unpin for StreamWrapper {} + +impl StreamWrapper { + pub fn new(stream: SendableRecordBatchStream, attachment: T) -> Self { + Self { + inner: stream, + _attachment: attachment, + } + } +} + +impl Stream for StreamWrapper { + type Item = common_recordbatch::error::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = &mut *self; + Pin::new(&mut this.inner).poll_next(cx) + } +} + +impl RecordBatchStream for StreamWrapper { + fn schema(&self) -> SchemaRef { + self.inner.schema() + } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + self.inner.output_ordering() + } + + fn metrics(&self) -> Option { + self.inner.metrics() + } +} diff --git a/src/servers/src/grpc/frontend_grpc_handler.rs b/src/servers/src/grpc/frontend_grpc_handler.rs index 1596550ed63a..8b985d311d30 100644 --- a/src/servers/src/grpc/frontend_grpc_handler.rs +++ b/src/servers/src/grpc/frontend_grpc_handler.rs @@ -14,7 +14,7 @@ use api::v1::frontend::frontend_server::Frontend; use api::v1::frontend::{ListProcessRequest, ListProcessResponse}; -use common_frontend::ProcessManagerRef; +use catalog::process_manager::ProcessManagerRef; use tonic::{Request, Response, Status}; #[derive(Clone)] diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs index bf8f351c2a29..a7ec7854e82f 100644 --- a/tests-integration/src/cluster.rs +++ b/tests-integration/src/cluster.rs @@ -25,6 +25,7 @@ use cache::{ }; use catalog::information_extension::DistributedInformationExtension; use catalog::kvbackend::{CachedKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend}; +use catalog::process_manager::ProcessManager; use client::client_manager::NodeClients; use client::Client; use common_base::Plugins; @@ -396,6 +397,7 @@ impl GreptimeDbClusterBuilder { Arc::new(handlers_executor), ); + let server_addr = options.grpc.server_addr.clone(); let instance = FrontendBuilder::new( options, cached_meta_backend.clone(), @@ -403,7 +405,7 @@ impl GreptimeDbClusterBuilder { catalog_manager, datanode_clients, meta_client, - None, + Arc::new(ProcessManager::new(server_addr, None)), ) .with_local_cache_invalidator(cache_registry) .try_build() diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs index 13db0d43d19b..9935bce7167f 100644 --- a/tests-integration/src/standalone.rs +++ b/tests-integration/src/standalone.rs @@ -20,6 +20,7 @@ use cache::{ }; use catalog::information_schema::NoopInformationExtension; use catalog::kvbackend::KvBackendCatalogManager; +use catalog::process_manager::ProcessManager; use cmd::error::StartFlownodeSnafu; use cmd::standalone::StandaloneOptions; use common_base::Plugins; @@ -233,6 +234,8 @@ impl GreptimeDbStandaloneBuilder { .unwrap(), ); + let server_addr = opts.frontend_options().grpc.server_addr.clone(); + let instance = FrontendBuilder::new( opts.frontend_options(), kv_backend.clone(), @@ -240,7 +243,7 @@ impl GreptimeDbStandaloneBuilder { catalog_manager.clone(), node_manager.clone(), ddl_task_executor.clone(), - None, + Arc::new(ProcessManager::new(server_addr, None)), ) .with_plugin(plugins) .try_build() From 78cb9c76f1ed78bd4b145b5d5bc2a455b8978274 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 11 Jun 2025 14:23:12 +0000 Subject: [PATCH 24/31] feat/show-process-list: ### Add Error Handling and Process Management - **Error Handling Enhancements**: - Added new error variants `ListProcess` and `CreateChannel` in `error.rs` to handle specific gRPC service invocation failures. - Updated error handling in `selector.rs` to use the new error variants for better context and error propagation. - **Process Management Integration**: - Introduced `process_manager` method in `instance.rs` to access the process manager. - Integrated `FrontendGrpcHandler` with process management in `server.rs` to handle gRPC requests related to process management. - **gRPC Server Enhancements**: - Made `frontend_grpc_handler` public in `grpc.rs` to allow external access and integration with other modules. Signed-off-by: Lei, HUANG --- src/common/frontend/src/error.rs | 18 ++++++++++++- src/common/frontend/src/selector.rs | 40 +++++++---------------------- src/frontend/src/instance.rs | 4 +++ src/frontend/src/server.rs | 4 +++ src/servers/src/grpc.rs | 2 +- 5 files changed, 35 insertions(+), 33 deletions(-) diff --git a/src/common/frontend/src/error.rs b/src/common/frontend/src/error.rs index 55db692c3cf5..ec3c1158dd1b 100644 --- a/src/common/frontend/src/error.rs +++ b/src/common/frontend/src/error.rs @@ -41,6 +41,21 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to invoke list process service"))] + ListProcess { + #[snafu(source)] + error: tonic::Status, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to invoke list process service"))] + CreateChannel { + source: common_grpc::error::Error, + #[snafu(implicit)] + location: Location, + }, } pub type Result = std::result::Result; @@ -51,7 +66,8 @@ impl ErrorExt for Error { match self { External { source, .. } => source.status_code(), Meta { source, .. } => source.status_code(), - Error::ParseProcessId { .. } => StatusCode::InvalidArguments, + ParseProcessId { .. } => StatusCode::InvalidArguments, + ListProcess { .. } | CreateChannel { .. } => StatusCode::External, } } diff --git a/src/common/frontend/src/selector.rs b/src/common/frontend/src/selector.rs index e5ace8b81fc2..4e36b67b18f9 100644 --- a/src/common/frontend/src/selector.rs +++ b/src/common/frontend/src/selector.rs @@ -20,6 +20,7 @@ use greptime_proto::v1::frontend::{frontend_client, ListProcessRequest, ListProc use meta_client::MetaClientRef; use snafu::ResultExt; +use crate::error; use crate::error::{MetaSnafu, Result}; pub type FrontendClientPtr = Box; @@ -36,7 +37,7 @@ impl FrontendClient for frontend_client::FrontendClient::list_process(self, req) .await - .unwrap() + .context(error::ListProcessSnafu)? .into_inner(); Ok(response) } @@ -68,16 +69,18 @@ impl FrontendSelector for MetaClientSelector { .map_err(Box::new) .context(MetaSnafu)?; - let res = nodes + nodes .into_iter() .filter(predicate) .map(|node| { - let channel = self.channel_manager.get(node.peer.addr).unwrap(); + let channel = self + .channel_manager + .get(node.peer.addr) + .context(error::CreateChannelSnafu)?; let client = frontend_client::FrontendClient::new(channel); - Box::new(client) as FrontendClientPtr + Ok(Box::new(client) as FrontendClientPtr) }) - .collect::>(); - Ok(res) + .collect::>>() } } @@ -93,28 +96,3 @@ impl MetaClientSelector { } } } - -#[cfg(test)] -mod tests { - use std::sync::Arc; - use std::vec; - - use meta_client::client::MetaClientBuilder; - - use super::*; - - #[tokio::test] - async fn test_meta_client_selector() { - let mut meta_client = MetaClientBuilder::frontend_default_options().build(); - meta_client - .start(vec!["192.168.50.164:3002"]) - .await - .unwrap(); - let selector = MetaClientSelector::new(Arc::new(meta_client)); - let clients = selector.select(|_| true).await.unwrap(); - for mut client in clients { - let resp = client.list_process(ListProcessRequest {}).await; - println!("{:?}", resp); - } - } -} diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index 8527000f24dd..e4c5a9add189 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -156,6 +156,10 @@ impl Instance { pub fn inserter(&self) -> &InserterRef { &self.inserter } + + pub fn process_manager(&self) -> &ProcessManagerRef { + &self.process_manager + } } fn parse_stmt(sql: &str, dialect: &(dyn Dialect + Send + Sync)) -> Result> { diff --git a/src/frontend/src/server.rs b/src/frontend/src/server.rs index abd80c940b38..4aeb36b08b27 100644 --- a/src/frontend/src/server.rs +++ b/src/frontend/src/server.rs @@ -20,6 +20,7 @@ use common_base::Plugins; use common_config::Configurable; use servers::error::Error as ServerError; use servers::grpc::builder::GrpcServerBuilder; +use servers::grpc::frontend_grpc_handler::FrontendGrpcHandler; use servers::grpc::greptime_handler::GreptimeRequestHandler; use servers::grpc::{GrpcOptions, GrpcServer, GrpcServerConfig}; use servers::http::event::LogValidatorRef; @@ -156,11 +157,14 @@ where runtime, ); + let frontend_grpc_handler = + FrontendGrpcHandler::new(self.instance.process_manager().clone()); let grpc_server = builder .database_handler(greptime_request_handler.clone()) .prometheus_handler(self.instance.clone(), user_provider.clone()) .otel_arrow_handler(OtelArrowServiceHandler(self.instance.clone())) .flight_handler(Arc::new(greptime_request_handler)) + .frontend_grpc_handler(frontend_grpc_handler) .build(); Ok(grpc_server) } diff --git a/src/servers/src/grpc.rs b/src/servers/src/grpc.rs index 60dc175f3b24..9f1f399302cc 100644 --- a/src/servers/src/grpc.rs +++ b/src/servers/src/grpc.rs @@ -17,7 +17,7 @@ pub mod builder; mod cancellation; mod database; pub mod flight; -mod frontend_grpc_handler; +pub mod frontend_grpc_handler; pub mod greptime_handler; pub mod prom_query_gateway; pub mod region_server; From 01fdcc237d33457b8e5415e25d37c6d5574d0000 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 11 Jun 2025 14:37:22 +0000 Subject: [PATCH 25/31] feat/show-process-list: Update `greptime-proto` dependency and enhance process management - **Dependency Update**: Updated `greptime-proto` in `Cargo.lock` and `Cargo.toml` to a new revision. - **Process Management**: - Modified `process_manager.rs` to include catalog filtering in `list_process`. - Updated `frontend_grpc_handler.rs` to handle catalog filtering in `list_process` requests. - **System Schema**: Added a TODO comment in `process_list.rs` for future user catalog filtering implementation. Signed-off-by: Lei, HUANG --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/catalog/src/process_manager.rs | 8 +++++++- .../system_schema/information_schema/process_list.rs | 1 + src/servers/src/grpc/frontend_grpc_handler.rs | 10 ++++++++-- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57fee5b3831f..c63bf765c3a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5141,7 +5141,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=0807d6dc1069bfc33158b78b095a180ddf120084#0807d6dc1069bfc33158b78b095a180ddf120084" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=58649407b5f502b947705e133b2f66c3a9049caa#58649407b5f502b947705e133b2f66c3a9049caa" dependencies = [ "prost 0.13.5", "serde", diff --git a/Cargo.toml b/Cargo.toml index 72701c0547f2..a4e8b83a5ecc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,7 +133,7 @@ etcd-client = "0.14" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0807d6dc1069bfc33158b78b095a180ddf120084" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "58649407b5f502b947705e133b2f66c3a9049caa" } hex = "0.4" http = "1" humantime = "2.1" diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index 19d8339a60f1..1d7c586acf64 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -124,7 +124,13 @@ impl ProcessManager { .select(|node| &node.peer.addr != &self.server_addr) .await?; for mut f in frontends { - processes.extend(f.list_process(ListProcessRequest {}).await?.processes); + processes.extend( + f.list_process(ListProcessRequest { + catalog: catalog.unwrap_or_default().to_string(), + }) + .await? + .processes, + ); } } processes.extend(self.local_processes(catalog)?); diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 99de40d3672c..050279bb58f5 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -121,6 +121,7 @@ async fn make_process_list( ) -> error::Result { let predicates = Predicates::from_scan_request(&Some(request)); let current_time = current_time_millis(); + // todo(hl): find a way to extract user catalog to filter queries from other users. let queries = process_manager.local_processes(None).unwrap(); let mut id_builder = StringVectorBuilder::with_capacity(queries.len()); diff --git a/src/servers/src/grpc/frontend_grpc_handler.rs b/src/servers/src/grpc/frontend_grpc_handler.rs index 8b985d311d30..83d9ab82a91c 100644 --- a/src/servers/src/grpc/frontend_grpc_handler.rs +++ b/src/servers/src/grpc/frontend_grpc_handler.rs @@ -32,9 +32,15 @@ impl FrontendGrpcHandler { impl Frontend for FrontendGrpcHandler { async fn list_process( &self, - _request: Request, + request: Request, ) -> Result, Status> { - let processes = self.process_manager.local_processes(None).unwrap(); + let list_process_request = request.into_inner(); + let catalog = if list_process_request.catalog.is_empty() { + None + } else { + Some(list_process_request.catalog.as_str()) + }; + let processes = self.process_manager.local_processes(catalog).unwrap(); Ok(Response::new(ListProcessResponse { processes })) } } From d3ed78c9a62b34b7203f2e774d0a775f7611fe99 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 11 Jun 2025 16:07:06 +0000 Subject: [PATCH 26/31] feat/show-process-list: - **Update Workspace Dependencies**: - Modified `Cargo.toml` files in `src/catalog`, `src/common/frontend`, and `src/servers` to adjust workspace dependencies. - **Refactor `ProcessManager` Logic**: - Updated `process_manager.rs` to simplify the condition in the `select` method. - **Remove Unused Error Variants**: - Deleted `BuildProcessManager` error variant from `error.rs` in `src/cmd`. - Removed `InvalidProcessKey` error variant from `error.rs` in `src/common/meta`. - **Add License Header**: - Added Apache License header to `stream_wrapper.rs` in `src/frontend`. - **Update Test Results**: - Adjusted expected results in `information_schema.result` to reflect changes in the schema. Signed-off-by: Lei, HUANG --- src/catalog/Cargo.toml | 2 +- src/catalog/src/process_manager.rs | 2 +- src/cmd/src/error.rs | 8 -------- src/common/frontend/Cargo.toml | 5 ++--- src/common/meta/src/error.rs | 10 +--------- src/frontend/src/stream_wrapper.rs | 14 ++++++++++++++ src/servers/Cargo.toml | 2 +- .../common/system/information_schema.result | 15 +++++++++------ 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml index 5a1d281270cb..e5d888735719 100644 --- a/src/catalog/Cargo.toml +++ b/src/catalog/Cargo.toml @@ -19,11 +19,11 @@ async-trait.workspace = true bytes.workspace = true common-catalog.workspace = true common-error.workspace = true +common-frontend.workspace = true common-macro.workspace = true common-meta.workspace = true common-procedure.workspace = true common-query.workspace = true -common-frontend.workspace = true common-recordbatch.workspace = true common-runtime.workspace = true common-telemetry.workspace = true diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index 1d7c586acf64..c548d48e228b 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -121,7 +121,7 @@ impl ProcessManager { let mut processes = vec![]; if let Some(remote_frontend_selector) = self.frontend_selector.as_ref() { let frontends = remote_frontend_selector - .select(|node| &node.peer.addr != &self.server_addr) + .select(|node| node.peer.addr != self.server_addr) .await?; for mut f in frontends { processes.extend( diff --git a/src/cmd/src/error.rs b/src/cmd/src/error.rs index 382957c5272b..ed9dda22d30b 100644 --- a/src/cmd/src/error.rs +++ b/src/cmd/src/error.rs @@ -302,13 +302,6 @@ pub enum Error { location: Location, source: common_meta::error::Error, }, - - #[snafu(display("Failed to build ProcessManager"))] - BuildProcessManager { - #[snafu(implicit)] - location: Location, - source: catalog::error::Error, - }, } pub type Result = std::result::Result; @@ -364,7 +357,6 @@ impl ErrorExt for Error { } Error::MetaClientInit { source, .. } => source.status_code(), Error::SchemaNotFound { .. } => StatusCode::DatabaseNotFound, - Error::BuildProcessManager { source, .. } => source.status_code(), } } diff --git a/src/common/frontend/Cargo.toml b/src/common/frontend/Cargo.toml index 471d7bdedd55..5058fcc6680c 100644 --- a/src/common/frontend/Cargo.toml +++ b/src/common/frontend/Cargo.toml @@ -7,14 +7,13 @@ license.workspace = true [dependencies] async-trait.workspace = true common-error.workspace = true -common-macro.workspace = true common-grpc.workspace = true -greptime-proto.workspace = true +common-macro.workspace = true common-meta.workspace = true +greptime-proto.workspace = true meta-client.workspace = true snafu.workspace = true tonic.workspace = true - [dev-dependencies] tokio.workspace = true diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs index 1e2e806fd1f7..8abd4982af07 100644 --- a/src/common/meta/src/error.rs +++ b/src/common/meta/src/error.rs @@ -765,13 +765,6 @@ pub enum Error { location: Location, }, - #[snafu(display("Invalid process list key: {}", key))] - InvalidProcessKey { - key: String, - #[snafu(implicit)] - location: Location, - }, - #[snafu(display("Invalid topic name prefix: {}", prefix))] InvalidTopicNamePrefix { prefix: String, @@ -1008,8 +1001,7 @@ impl ErrorExt for Error { } #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))] RdsTransactionRetryFailed { .. } => StatusCode::Internal, - Error::DatanodeTableInfoNotFound { .. } => StatusCode::Internal, - InvalidProcessKey { .. } => StatusCode::Internal, + DatanodeTableInfoNotFound { .. } => StatusCode::Internal, } } diff --git a/src/frontend/src/stream_wrapper.rs b/src/frontend/src/stream_wrapper.rs index c851067037ed..c38996c57c71 100644 --- a/src/frontend/src/stream_wrapper.rs +++ b/src/frontend/src/stream_wrapper.rs @@ -1,3 +1,17 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use std::pin::Pin; use std::task::{Context, Poll}; diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index 92f10e05ab93..f851684447fd 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -37,8 +37,8 @@ common-base.workspace = true common-catalog.workspace = true common-config.workspace = true common-error.workspace = true -common-grpc.workspace = true common-frontend.workspace = true +common-grpc.workspace = true common-macro.workspace = true common-mem-prof = { workspace = true, optional = true } common-meta.workspace = true diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index af30d89f9398..85eb99f2a361 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -209,7 +209,7 @@ select * from information_schema.columns order by table_schema, table_name, colu | greptime | information_schema | key_column_usage | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | key_column_usage | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | key_column_usage | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | key_column_usage | greptime_index_type | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | +| greptime | information_schema | key_column_usage | greptime_index_type | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | | greptime | information_schema | key_column_usage | ordinal_position | 9 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | | greptime | information_schema | key_column_usage | position_in_unique_constraint | 10 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | Yes | int unsigned | | | | greptime | information_schema | key_column_usage | real_table_catalog | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | @@ -271,11 +271,14 @@ select * from information_schema.columns order by table_schema, table_name, colu | greptime | information_schema | procedure_info | procedure_type | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | procedure_info | start_time | 3 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | | greptime | information_schema | procedure_info | status | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | process_list | database | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | process_list | elapsed_time | 5 | | | | | | | | | | select,insert | | DurationMillisecond | DurationMillisecond | FIELD | | No | DurationMillisecond | | | -| greptime | information_schema | process_list | id | 1 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | -| greptime | information_schema | process_list | query | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | process_list | start_timestamp | 4 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | No | timestamp(3) | | | +| greptime | information_schema | process_list | catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | process_list | client | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | process_list | elapsed_time | 8 | | | | | | | | | | select,insert | | DurationMillisecond | DurationMillisecond | FIELD | | No | DurationMillisecond | | | +| greptime | information_schema | process_list | frontend | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | process_list | id | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | process_list | query | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | process_list | schemas | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | process_list | start_timestamp | 7 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | No | timestamp(3) | | | | greptime | information_schema | profiling | block_ops_in | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | | greptime | information_schema | profiling | block_ops_out | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | | greptime | information_schema | profiling | context_involuntary | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | From 19f50c27a2f8ce96fe2d1e222ea27de8fa3c2bc4 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Wed, 11 Jun 2025 16:42:38 +0000 Subject: [PATCH 27/31] feat/show-process-list: ### Add Error Handling for Process Listing - **`src/catalog/src/error.rs`**: Introduced a new error variant `ListProcess` to handle failures in listing frontend nodes. - **`src/catalog/src/process_manager.rs`**: Updated `local_processes` and `list_all_processes` methods to return the new error type, adding context for error handling. - **`src/catalog/src/system_schema/information_schema/process_list.rs`**: Modified `make_process_list` to propagate errors using the new error handling mechanism. - **`src/servers/src/grpc/frontend_grpc_handler.rs`**: Enhanced error handling in the `list_process` method to log errors and return appropriate gRPC status codes. Signed-off-by: Lei, HUANG --- src/catalog/src/error.rs | 8 ++++++++ src/catalog/src/process_manager.rs | 16 +++++++++------- .../information_schema/process_list.rs | 2 +- src/servers/src/grpc/frontend_grpc_handler.rs | 12 +++++++++--- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs index c4fdf1cdc1ec..165f99f28f5a 100644 --- a/src/catalog/src/error.rs +++ b/src/catalog/src/error.rs @@ -277,6 +277,13 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to list frontend nodes"))] + ListProcess { + source: common_frontend::error::Error, + #[snafu(implicit)] + location: Location, + }, } impl Error { @@ -345,6 +352,7 @@ impl ErrorExt for Error { Error::GetViewCache { source, .. } | Error::GetTableCache { source, .. } => { source.status_code() } + Error::ListProcess { source, .. } => source.status_code(), } } diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index c548d48e228b..4f7ce1bffb8f 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -22,6 +22,9 @@ use common_frontend::selector::{FrontendSelector, MetaClientSelector}; use common_telemetry::{debug, info}; use common_time::util::current_time_millis; use meta_client::MetaClientRef; +use snafu::ResultExt; + +use crate::error; pub type ProcessManagerRef = Arc; @@ -94,10 +97,7 @@ impl ProcessManager { } /// List local running processes in given catalog. - pub fn local_processes( - &self, - catalog: Option<&str>, - ) -> common_frontend::error::Result> { + pub fn local_processes(&self, catalog: Option<&str>) -> error::Result> { let catalogs = self.catalogs.read().unwrap(); let result = if let Some(catalog) = catalog { if let Some(catalogs) = catalogs.get(catalog) { @@ -117,18 +117,20 @@ impl ProcessManager { pub async fn list_all_processes( &self, catalog: Option<&str>, - ) -> common_frontend::error::Result> { + ) -> error::Result> { let mut processes = vec![]; if let Some(remote_frontend_selector) = self.frontend_selector.as_ref() { let frontends = remote_frontend_selector .select(|node| node.peer.addr != self.server_addr) - .await?; + .await + .context(error::ListProcessSnafu)?; for mut f in frontends { processes.extend( f.list_process(ListProcessRequest { catalog: catalog.unwrap_or_default().to_string(), }) - .await? + .await + .context(error::ListProcessSnafu)? .processes, ); } diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index 050279bb58f5..edfb317c8124 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -122,7 +122,7 @@ async fn make_process_list( let predicates = Predicates::from_scan_request(&Some(request)); let current_time = current_time_millis(); // todo(hl): find a way to extract user catalog to filter queries from other users. - let queries = process_manager.local_processes(None).unwrap(); + let queries = process_manager.local_processes(None)?; let mut id_builder = StringVectorBuilder::with_capacity(queries.len()); let mut catalog_builder = StringVectorBuilder::with_capacity(queries.len()); diff --git a/src/servers/src/grpc/frontend_grpc_handler.rs b/src/servers/src/grpc/frontend_grpc_handler.rs index 83d9ab82a91c..f12ae77a972a 100644 --- a/src/servers/src/grpc/frontend_grpc_handler.rs +++ b/src/servers/src/grpc/frontend_grpc_handler.rs @@ -15,7 +15,8 @@ use api::v1::frontend::frontend_server::Frontend; use api::v1::frontend::{ListProcessRequest, ListProcessResponse}; use catalog::process_manager::ProcessManagerRef; -use tonic::{Request, Response, Status}; +use common_telemetry::error; +use tonic::{Code, Request, Response, Status}; #[derive(Clone)] pub struct FrontendGrpcHandler { @@ -40,7 +41,12 @@ impl Frontend for FrontendGrpcHandler { } else { Some(list_process_request.catalog.as_str()) }; - let processes = self.process_manager.local_processes(catalog).unwrap(); - Ok(Response::new(ListProcessResponse { processes })) + match self.process_manager.local_processes(catalog) { + Ok(processes) => Ok(Response::new(ListProcessResponse { processes })), + Err(e) => { + error!(e; "Failed to handle list process request"); + Err(Status::new(Code::Internal, e.to_string())) + } + } } } From 1e9a9b2cb4b9baea1004bb357c864f7e562f98c0 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Thu, 12 Jun 2025 02:54:15 +0000 Subject: [PATCH 28/31] feat/show-process-list: Update `greptime-proto` Dependency and Remove `frontend_client` Method - **Cargo.lock** and **Cargo.toml**: Updated the `greptime-proto` dependency to a new revision (`5f6119ac7952878d39dcde0343c4bf828d18ffc8`). - **src/client/src/client.rs**: Removed the `frontend_client` method from the `Client` implementation. Signed-off-by: Lei, HUANG --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/client/src/client.rs | 11 ----------- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c63bf765c3a5..464cf5177342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5141,7 +5141,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=58649407b5f502b947705e133b2f66c3a9049caa#58649407b5f502b947705e133b2f66c3a9049caa" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=5f6119ac7952878d39dcde0343c4bf828d18ffc8#5f6119ac7952878d39dcde0343c4bf828d18ffc8" dependencies = [ "prost 0.13.5", "serde", diff --git a/Cargo.toml b/Cargo.toml index a4e8b83a5ecc..aee5d1c892ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,7 +133,7 @@ etcd-client = "0.14" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "58649407b5f502b947705e133b2f66c3a9049caa" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "5f6119ac7952878d39dcde0343c4bf828d18ffc8" } hex = "0.4" http = "1" humantime = "2.1" diff --git a/src/client/src/client.rs b/src/client/src/client.rs index 06047b0f4022..a604bf5124ad 100644 --- a/src/client/src/client.rs +++ b/src/client/src/client.rs @@ -15,7 +15,6 @@ use std::sync::Arc; use api::v1::flow::flow_client::FlowClient as PbFlowClient; -use api::v1::frontend::frontend_client::FrontendClient; use api::v1::health_check_client::HealthCheckClient; use api::v1::prometheus_gateway_client::PrometheusGatewayClient; use api::v1::region::region_client::RegionClient as PbRegionClient; @@ -191,16 +190,6 @@ impl Client { Ok((addr, client)) } - pub fn frontend_client(&self) -> Result> { - let (_, channel) = self.find_channel()?; - let client = FrontendClient::new(channel) - .max_decoding_message_size(self.max_grpc_recv_message_size()) - .max_encoding_message_size(self.max_grpc_send_message_size()) - .accept_compressed(CompressionEncoding::Zstd) - .send_compressed(CompressionEncoding::Zstd); - Ok(client) - } - pub fn make_prometheus_gateway_client(&self) -> Result> { let (_, channel) = self.find_channel()?; let client = PrometheusGatewayClient::new(channel) From e516ae49e4f9b571e8de267050a0243517d58228 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Thu, 12 Jun 2025 02:58:46 +0000 Subject: [PATCH 29/31] feat/show-process-list: ### Add Query Registration with Pre-Generated ID - **`process_manager.rs`**: Introduced `register_query_with_id` method to allow registering queries with a pre-generated ID. This includes creating a `ProcessInfo` instance and inserting it into the catalog. Added `next_id` method to generate the next process ID. Signed-off-by: Lei, HUANG --- src/catalog/src/process_manager.rs | 11 +++++++++-- src/frontend/src/instance.rs | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs index 4f7ce1bffb8f..fa79848dde30 100644 --- a/src/catalog/src/process_manager.rs +++ b/src/catalog/src/process_manager.rs @@ -49,15 +49,16 @@ impl ProcessManager { } impl ProcessManager { - /// Registers a submitted query. + /// Registers a submitted query. Use the provided id if present. pub fn register_query( self: &Arc, catalog: String, schemas: Vec, query: String, client: String, + id: Option, ) -> Ticket { - let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let id = id.unwrap_or_else(|| self.next_id.fetch_add(1, Ordering::Relaxed)); let process = ProcessInfo { id, catalog: catalog.clone(), @@ -80,6 +81,11 @@ impl ProcessManager { } } + /// Generates the next process id. + pub fn next_id(&self) -> u64 { + self.next_id.fetch_add(1, Ordering::Relaxed) + } + /// De-register a query from process list. pub fn deregister_query(&self, catalog: String, id: u64) { if let Entry::Occupied(mut o) = self.catalogs.write().unwrap().entry(catalog) { @@ -167,6 +173,7 @@ mod tests { vec!["test".to_string()], "SELECT * FROM table".to_string(), "".to_string(), + None, ); let running_processes = process_manager.local_processes(None).unwrap(); diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index e4c5a9add189..cc1838aff17f 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -184,6 +184,7 @@ impl Instance { vec![query_ctx.current_schema()], stmt.to_string(), "unknown".to_string(), + None, ); let output = match stmt { From 5e56349217d45f0dd8eefa2b480c157a79080875 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Thu, 12 Jun 2025 06:20:44 +0000 Subject: [PATCH 30/31] feat/show-process-list: ### Update Process List Retrieval Method - **File**: `process_list.rs` - Updated the method for retrieving process lists from `local_processes` to `list_all_processes` to support asynchronous operations. Signed-off-by: Lei, HUANG --- .../src/system_schema/information_schema/process_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog/src/system_schema/information_schema/process_list.rs b/src/catalog/src/system_schema/information_schema/process_list.rs index edfb317c8124..46a94bd458fa 100644 --- a/src/catalog/src/system_schema/information_schema/process_list.rs +++ b/src/catalog/src/system_schema/information_schema/process_list.rs @@ -122,7 +122,7 @@ async fn make_process_list( let predicates = Predicates::from_scan_request(&Some(request)); let current_time = current_time_millis(); // todo(hl): find a way to extract user catalog to filter queries from other users. - let queries = process_manager.local_processes(None)?; + let queries = process_manager.list_all_processes(None).await?; let mut id_builder = StringVectorBuilder::with_capacity(queries.len()); let mut catalog_builder = StringVectorBuilder::with_capacity(queries.len()); From d01e1514012abac0d8d3738915fae3d7a52de5be Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" Date: Thu, 12 Jun 2025 06:22:25 +0000 Subject: [PATCH 31/31] feat/show-process-list: ### Update error handling in `error.rs` - Refined status code handling for `CreateChannel` error by delegating to `source.status_code()`. - Separated `ListProcess` and `CreateChannel` error handling for clarity. Signed-off-by: Lei, HUANG --- src/common/frontend/src/error.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/frontend/src/error.rs b/src/common/frontend/src/error.rs index ec3c1158dd1b..f4046a1fd966 100644 --- a/src/common/frontend/src/error.rs +++ b/src/common/frontend/src/error.rs @@ -67,7 +67,8 @@ impl ErrorExt for Error { External { source, .. } => source.status_code(), Meta { source, .. } => source.status_code(), ParseProcessId { .. } => StatusCode::InvalidArguments, - ListProcess { .. } | CreateChannel { .. } => StatusCode::External, + ListProcess { .. } => StatusCode::External, + CreateChannel { source, .. } => source.status_code(), } }