[Cherry-pick to branch-1.3] [#14066] fix(starrocks): Map index types when loading StarRocks tables (#14067)

api/src/main/java/org/apache/gravitino/rel/indexes/Index.java

@@ -92,7 +92,8 @@ enum IndexType {
     /**
      * Bitmap index is a type of database index that uses bit arrays (bitmaps) to represent the
      * presence or absence of values in a column, enabling efficient querying and data retrieval.
-     * Currently, this type is only applicable to Lance.
+     * Currently, this type is applicable to Lance, is accepted when creating Doris tables, and is
+     * reported when loading StarRocks tables.
      */
     BITMAP,
     /**
@@ -103,12 +104,14 @@ enum IndexType {
     /**
      * INVERTED index is a data structure used to optimize full-text searches by mapping terms to
      * their locations within a dataset, allowing for quick retrieval of documents containing
-     * specific words or phrases. Currently, this type is only applicable to Lance.
+     * specific words or phrases. Currently, this type is applicable to Lance and Doris, and is
+     * reported when loading StarRocks tables.
      */
     INVERTED,
     /**
      * VECTOR index is used to optimize similarity searches in high-dimensional vector spaces.
-     * Currently, this type is only applicable to Lance.
+     * Currently, this type is applicable to Lance and Doris, and is reported when loading StarRocks
+     * tables.
      */
     VECTOR,
     /** IVF_FLAT (Inverted File with Flat quantization) is an indexing method used for efficient */

catalogs/catalog-jdbc-starrocks/build.gradle.kts

@@ -50,6 +50,7 @@ dependencies {
   testImplementation(libs.awaitility)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
+  testImplementation(libs.mockito.core)
   testImplementation(libs.mysql.driver)
   testImplementation(libs.postgresql.driver)
   testImplementation(libs.testcontainers)

catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java

@@ -32,10 +32,12 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -260,12 +262,24 @@ protected List<Index> getIndexes(Connection connection, String databaseName, Str
     try (PreparedStatement preparedStatement = connection.prepareStatement(sql);
         ResultSet resultSet = preparedStatement.executeQuery()) {
 
+      // SHOW INDEX lists only secondary indexes, one row per single-column index; table keys are
+      // not included.
       List<Index> indexes = new ArrayList<>();
       while (resultSet.next()) {
         String indexName = resultSet.getString("Key_name");
         String columnName = resultSet.getString("Column_name");
-        indexes.add(
-            Indexes.of(Index.IndexType.PRIMARY_KEY, indexName, new String[][] {{columnName}}));
+        String starRocksIndexType = resultSet.getString("Index_type");
+        Index.IndexType indexType = mapStarRocksIndexType(starRocksIndexType);
+        if (indexType == null) {
+          LOG.warn(
+              "Skip StarRocks index '{}' on table {}.{} with unsupported type '{}'",
+              indexName,
+              databaseName,
+              tableName,
+              starRocksIndexType);
+          continue;
+        }
+        indexes.add(Indexes.of(indexType, indexName, new String[][] {{columnName}}));
       }
       return indexes;
     } catch (SQLException e) {
@@ -630,4 +644,31 @@ private String updateColumnTypeFieldDefinition(
             .build();
     return appendColumnDefinition(newColumn, sqlBuilder).toString();
   }
+
+  /**
+   * Maps the {@code Index_type} value of StarRocks {@code SHOW INDEX} to a Gravitino index type.
+   * The value may carry index properties, e.g. {@code GIN("parser" = "english")}; only the type
+   * name is used and the properties are not kept, since StarRocks indexes cannot be created or
+   * altered through Gravitino.
+   *
+   * @param indexType the {@code Index_type} value returned by StarRocks
+   * @return the Gravitino index type, or null if the type has no Gravitino equivalent
+   */
+  @Nullable
+  private static Index.IndexType mapStarRocksIndexType(@Nullable String indexType) {
+    if (indexType == null) {
+      return null;
+    }
+    String typeName = StringUtils.substringBefore(indexType, "(").trim();
+    switch (typeName.toUpperCase(Locale.ROOT)) {
+      case "BITMAP":
+        return Index.IndexType.BITMAP;
+      case "GIN":
+        return Index.IndexType.INVERTED;
+      case "VECTOR":
+        return Index.IndexType.VECTOR;
+      default:
+        return null;
+    }
+  }
 }

catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java

@@ -846,4 +846,29 @@ void testStringDefaultValueRoundTrips(String defaultValue) throws SQLException {
       TABLE_OPERATIONS.drop(databaseName, tableName);
     }
   }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"PRIMARY", "UNIQUE", "DUPLICATE"})
+  void testLoadTableReportsOnlySecondaryIndexes(String keyType) throws SQLException {
+    String tableName = GravitinoITUtils.genRandomName("starrocks_index");
+    try (Connection connection = DATA_SOURCE.getConnection();
+        Statement statement = connection.createStatement()) {
+      statement.executeUpdate(
+          String.format(
+              "CREATE TABLE `%s`.`%s` (`id` INT NOT NULL, `region` VARCHAR(32) NOT NULL, "
+                  + "INDEX `idx_region` (`region`) USING BITMAP) "
+                  + "%s KEY(`id`, `region`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 "
+                  + "PROPERTIES (\"replication_num\" = \"1\")",
+              databaseName, tableName, keyType));
+    }
+    try {
+      JdbcTable table = TABLE_OPERATIONS.load(databaseName, tableName);
+      Assertions.assertEquals(1, table.index().length);
+      Assertions.assertEquals(Index.IndexType.BITMAP, table.index()[0].type());
+      Assertions.assertEquals("idx_region", table.index()[0].name());
+      Assertions.assertArrayEquals(new String[][] {{"region"}}, table.index()[0].fieldNames());
+    } finally {
+      TABLE_OPERATIONS.drop(databaseName, tableName);
+    }
+  }
 }

catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java

@@ -19,6 +19,9 @@
 package org.apache.gravitino.catalog.starrocks.operation;
 
 import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -46,6 +49,7 @@
 import org.junit.jupiter.params.provider.EmptySource;
 import org.junit.jupiter.params.provider.EnumSource;
 import org.junit.jupiter.params.provider.NullSource;
+import org.mockito.Mockito;
 
 public class TestStarRocksTableOperationsSqlGeneration {
 
@@ -100,6 +104,11 @@ public String alterTableSql(String tableName, TableChange... changes) {
       return generateAlterTableSql("database", tableName, changes);
     }
 
+    List<Index> indexes(Connection connection, String databaseName, String tableName)
+        throws SQLException {
+      return getIndexes(connection, databaseName, tableName);
+    }
+
     @Override
     protected JdbcTable getOrCreateTable(
         String databaseName, String tableName, JdbcTable lazyLoadCreateTable) {
@@ -411,6 +420,43 @@ public void testAddAutoIncrementColumnIsRejected() {
         "StarRocks does not support adding an auto-increment column: col2", exception.getMessage());
   }
 
+  @Test
+  void testGetIndexesMapsTypesAndSkipsUnsupported() throws SQLException {
+    TestableStarRocksTableOperations ops = new TestableStarRocksTableOperations();
+    Connection connection = Mockito.mock(Connection.class);
+    PreparedStatement statement = Mockito.mock(PreparedStatement.class);
+    ResultSet resultSet = Mockito.mock(ResultSet.class);
+    Mockito.when(connection.prepareStatement("SHOW INDEX FROM `table` FROM `database`"))
+        .thenReturn(statement);
+    Mockito.when(statement.executeQuery()).thenReturn(resultSet);
+    Mockito.when(resultSet.next()).thenReturn(true, true, true, true, true, true, false);
+    Mockito.when(resultSet.getString("Key_name"))
+        .thenReturn("idx_region", "idx_ngram", "idx_text", "idx_vector", "idx_unknown", "idx_null");
+    Mockito.when(resultSet.getString("Column_name"))
+        .thenReturn("region", "name", "text", "embedding", "c5", "c6");
+    Mockito.when(resultSet.getString("Index_type"))
+        .thenReturn(
+            "BITMAP",
+            "NGRAMBF(\"gram_num\" = \"4\")",
+            "GIN(\"parser\" = \"english\")",
+            "VECTOR(\"index_type\" = \"hnsw\", \"dim\" = \"3\")",
+            "UNKNOWN",
+            null);
+
+    List<Index> indexes = ops.indexes(connection, "database", "table");
+
+    Assertions.assertEquals(3, indexes.size());
+    Assertions.assertEquals(Index.IndexType.BITMAP, indexes.get(0).type());
+    Assertions.assertEquals("idx_region", indexes.get(0).name());
+    Assertions.assertArrayEquals(new String[][] {{"region"}}, indexes.get(0).fieldNames());
+    Assertions.assertEquals(Index.IndexType.INVERTED, indexes.get(1).type());
+    Assertions.assertEquals("idx_text", indexes.get(1).name());
+    Assertions.assertArrayEquals(new String[][] {{"text"}}, indexes.get(1).fieldNames());
+    Assertions.assertEquals(Index.IndexType.VECTOR, indexes.get(2).type());
+    Assertions.assertEquals("idx_vector", indexes.get(2).name());
+    Assertions.assertArrayEquals(new String[][] {{"embedding"}}, indexes.get(2).fieldNames());
+  }
+
   @Test
   void testRenameColumnQuotesIdentifiers() {
     TestableStarRocksTableOperations ops = new TestableStarRocksTableOperations("id", "order-id");

clients/client-python/gravitino/api/rel/indexes/index.py

@@ -50,6 +50,15 @@ class IndexType(str, Enum):
         columns that are not designated as the primary key. The uniqueness constraint imposed by
         UNIQUE KEY helps in avoiding redundancy and ensuring data accuracy in the database."""
 
+        BITMAP = "BITMAP"
+        """Bitmap index, which uses bit arrays to represent the presence of values in a column."""
+
+        INVERTED = "INVERTED"
+        """Inverted index, which maps terms to their locations to speed up full-text searches."""
+
+        VECTOR = "VECTOR"
+        """Vector index, which speeds up similarity searches in high-dimensional vector spaces."""
+
     @abstractmethod
     def type(self) -> IndexType:
         """Returns the type of the index. eg: `PRIMARY_KEY` and `UNIQUE_KEY`.

clients/client-python/tests/unittests/dto/rel/test_index_serdes.py

@@ -116,3 +116,21 @@ def test_index_serdes(self):
         json_dict = json.loads(json_string)
         serialized_dict = json.loads(mock_data_class.to_json())
         self.assertDictEqual(json_dict, serialized_dict)
+
+    def test_index_serdes_bitmap_inverted_vector(self):
+        for index_type in (
+            Index.IndexType.BITMAP,
+            Index.IndexType.INVERTED,
+            Index.IndexType.VECTOR,
+        ):
+            index = IndexSerdes.deserialize(
+                {
+                    "indexType": index_type.value,
+                    "name": "idx",
+                    "fieldNames": [["col"]],
+                }
+            )
+            self.assertEqual(index_type, index.type())
+            self.assertEqual(
+                index_type.value, IndexSerdes.serialize(index)["indexType"]
+            )

docs/jdbc-starrocks-catalog.md

@@ -149,7 +149,15 @@ Unsupported for now.
 
 ### Table Indexes
 
-Unsupported. Creating a table with non-empty index definitions is rejected before the table is created.
+Creating a table with non-empty index definitions is rejected before the table is created.
+
+When loading a table, Gravitino reports StarRocks secondary indexes with the following type mapping. Indexes of other types (e.g. `NGRAMBF`) are not reported, and table keys are not reported as indexes.
+
+| StarRocks Index Type | Gravitino Index Type |
+|----------------------|----------------------|
+| `BITMAP`             | `BITMAP`             |
+| `GIN`                | `INVERTED`           |
+| `VECTOR`             | `VECTOR`             |
 
 ### Table Partitioning