feat(query): Add columns to display index sizes in system.tables (#18411)

* tmp

fix

feat(query): Add columns to display index sizes in `system.tables`

* fix tests

Author: b41sh

src/meta/app/src/schema/table.rs

@@ -241,9 +241,18 @@ pub struct TableStatistics {
     pub data_bytes: u64,
     /// Size of data compressed in bytes
     pub compressed_data_bytes: u64,
-    /// Size of index data in bytes
+    /// Size of all index data in bytes
     pub index_data_bytes: u64,
-
+    /// Size of bloom index in bytes
+    pub bloom_index_size: Option<u64>,
+    /// Size of ngram index in bytes
+    pub ngram_index_size: Option<u64>,
+    /// Size of inverted index in bytes
+    pub inverted_index_size: Option<u64>,
+    /// Size of vector index in bytes
+    pub vector_index_size: Option<u64>,
+    /// Size of virtual column in bytes
+    pub virtual_column_size: Option<u64>,
     /// number of segments
     pub number_of_segments: Option<u64>,
 

src/meta/proto-conv/src/table_from_to_protobuf_impl.rs

@@ -305,6 +305,11 @@ impl FromToProto for mt::TableStatistics {
             index_data_bytes: p.index_data_bytes,
             number_of_segments: p.number_of_segments,
             number_of_blocks: p.number_of_blocks,
+            bloom_index_size: p.bloom_index_size,
+            ngram_index_size: p.ngram_index_size,
+            inverted_index_size: p.inverted_index_size,
+            vector_index_size: p.vector_index_size,
+            virtual_column_size: p.virtual_column_size,
         };
 
         Ok(v)
@@ -320,6 +325,11 @@ impl FromToProto for mt::TableStatistics {
             index_data_bytes: self.index_data_bytes,
             number_of_segments: self.number_of_segments,
             number_of_blocks: self.number_of_blocks,
+            bloom_index_size: self.bloom_index_size,
+            ngram_index_size: self.ngram_index_size,
+            inverted_index_size: self.inverted_index_size,
+            vector_index_size: self.vector_index_size,
+            virtual_column_size: self.virtual_column_size,
         };
         Ok(p)
     }

src/meta/proto-conv/src/util.rs

@@ -167,6 +167,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (135, "2025-07-16: Add: UDFServer.immutable, UDFScript.immutable"),
     (136, "2025-07-17: Add: Task"),
     (137, "2025-07-22: Add: GrantConnectionObject and UserPrivilegeType AccessConnection, AccessConnection"),
+    (138, "2025-07-23: Add: TableStatistics add index size"),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

@@ -129,3 +129,4 @@ mod v134_add_sequence_meta_storage_version;
 mod v135_udf_immutable;
 mod v136_add_task;
 mod v137_add_grant_object_connection;
+mod v138_table_statistics;

src/meta/proto-conv/tests/it/proto_conv.rs

@@ -258,6 +258,11 @@ fn new_table_statistics() -> databend_common_meta_app::schema::TableStatistics {
         data_bytes: 200,
         compressed_data_bytes: 15,
         index_data_bytes: 20,
+        bloom_index_size: None,
+        ngram_index_size: None,
+        inverted_index_size: None,
+        vector_index_size: None,
+        virtual_column_size: None,
         number_of_segments: Some(1),
         number_of_blocks: Some(2),
     }

src/meta/proto-conv/tests/it/v043_table_statistics.rs

@@ -36,6 +36,11 @@ fn test_decode_v43_table_statistics() -> anyhow::Result<()> {
         data_bytes: 200,
         compressed_data_bytes: 15,
         index_data_bytes: 20,
+        bloom_index_size: None,
+        ngram_index_size: None,
+        inverted_index_size: None,
+        vector_index_size: None,
+        virtual_column_size: None,
         number_of_segments: Some(1),
         number_of_blocks: Some(2),
     };

src/meta/proto-conv/tests/it/v129_vector_datatype.rs

@@ -108,7 +108,6 @@ fn test_decode_v129_schema() -> anyhow::Result<()> {
     ];
     let want = || TableSchema::new(fields.clone());
     common::test_pb_from_to(func_name!(), want())?;
-    // common::test_pb_from_to2(func_name!(), want())?;
     common::test_load_old(func_name!(), table_schema_v129.as_slice(), 129, want())?;
     Ok(())
 }
@@ -264,7 +263,6 @@ fn test_decode_v129_table_meta() -> anyhow::Result<()> {
         }},
     };
     common::test_pb_from_to(func_name!(), want())?;
-    // common::test_pb_from_to2(func_name!(), want())?;
     common::test_load_old(func_name!(), table_meta_v129.as_slice(), 129, want())?;
 
     Ok(())

src/meta/proto-conv/tests/it/v138_table_statistics.rs

@@ -0,0 +1,52 @@
+// Copyright 2023 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use databend_common_meta_app::schema::TableStatistics;
+use fastrace::func_name;
+
+use crate::common;
+
+// These bytes are built when a new version in introduced,
+// and are kept for backward compatibility test.
+//
+// *************************************************************
+// * These messages should never be updated,                   *
+// * only be added when a new version is added,                *
+// * or be removed when an old version is no longer supported. *
+// *************************************************************
+//
+// The message bytes are built from the output of `test_pb_from_to()`
+#[test]
+fn test_decode_v138_table_statistics() -> anyhow::Result<()> {
+    let bytes = vec![
+        8, 100, 16, 128, 8, 24, 128, 6, 32, 128, 8, 40, 1, 48, 2, 56, 128, 2, 64, 128, 1, 72, 128,
+        1, 80, 128, 4, 160, 6, 138, 1, 168, 6, 24,
+    ];
+    let want = || TableStatistics {
+        number_of_rows: 100,
+        data_bytes: 1024,
+        compressed_data_bytes: 768,
+        index_data_bytes: 1024,
+        bloom_index_size: Some(256),
+        ngram_index_size: Some(128),
+        inverted_index_size: Some(128),
+        vector_index_size: Some(512),
+        virtual_column_size: None,
+        number_of_segments: Some(1),
+        number_of_blocks: Some(2),
+    };
+
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), bytes.as_slice(), 138, want())
+}

src/meta/protos/proto/table.proto

@@ -188,6 +188,17 @@ message TableStatistics {
 
   // Number of blocks
   optional uint64 number_of_blocks = 6;
+
+  // Size of bloom index in bytes
+  optional uint64 bloom_index_size = 7;
+  // Size of ngram index in bytes
+  optional uint64 ngram_index_size = 8;
+  // Size of inverted index in bytes
+  optional uint64 inverted_index_size = 9;
+  // Size of vector index in bytes
+  optional uint64 vector_index_size = 10;
+  // Size of virtual column in bytes
+  optional uint64 virtual_column_size = 11;
 }
 
 message DatabaseIdTableName {

src/query/catalog/src/table.rs

@@ -571,6 +571,12 @@ pub struct TableStatistics {
     pub data_size: Option<u64>,
     pub data_size_compressed: Option<u64>,
     pub index_size: Option<u64>,
+    pub bloom_index_size: Option<u64>,
+    pub ngram_index_size: Option<u64>,
+    pub inverted_index_size: Option<u64>,
+    pub vector_index_size: Option<u64>,
+    pub virtual_column_size: Option<u64>,
+
     pub number_of_blocks: Option<u64>,
     pub number_of_segments: Option<u64>,
 }