Replace segment index file path with index file id within segment index meta (milvus-io#19866)

Signed-off-by: yun.zhang <[email protected]>

Signed-off-by: yun.zhang <[email protected]>

Author: jaime0815

cmd/tools/migration/meta/210_to_220.go

@@ -3,6 +3,7 @@ package meta
 import (
 	"fmt"
 	"sort"
+	"strings"
 
 	"github.com/milvus-io/milvus/internal/util/typeutil"
 
@@ -142,21 +143,32 @@ func combineToSegmentIndexesMeta220(segmentIndexes SegmentIndexesMeta210, indexB
 			if !ok {
 				return nil, fmt.Errorf("index build meta not found, segment id: %d, index id: %d, index build id: %d", segID, indexID, record.GetBuildID())
 			}
+
+			fileKeys := make([]string, len(buildMeta.GetIndexFilePaths()))
+			for i, filePath := range buildMeta.GetIndexFilePaths() {
+				parts := strings.Split(filePath, "/")
+				if len(parts) == 0 {
+					return nil, fmt.Errorf("invaild index file path: %s", filePath)
+				}
+
+				fileKeys[i] = parts[len(parts)-1]
+			}
+
 			segmentIndexModel := &model.SegmentIndex{
-				SegmentID:      segID,
-				CollectionID:   record.GetCollectionID(),
-				PartitionID:    record.GetPartitionID(),
-				NumRows:        buildMeta.GetReq().GetNumRows(),
-				IndexID:        indexID,
-				BuildID:        record.GetBuildID(),
-				NodeID:         buildMeta.GetNodeID(),
-				IndexVersion:   buildMeta.GetIndexVersion(),
-				IndexState:     buildMeta.GetState(),
-				FailReason:     buildMeta.GetFailReason(),
-				IsDeleted:      buildMeta.GetMarkDeleted(),
-				CreateTime:     record.GetCreateTime(),
-				IndexFilePaths: buildMeta.GetIndexFilePaths(),
-				IndexSize:      buildMeta.GetSerializeSize(),
+				SegmentID:     segID,
+				CollectionID:  record.GetCollectionID(),
+				PartitionID:   record.GetPartitionID(),
+				NumRows:       buildMeta.GetReq().GetNumRows(),
+				IndexID:       indexID,
+				BuildID:       record.GetBuildID(),
+				NodeID:        buildMeta.GetNodeID(),
+				IndexVersion:  buildMeta.GetIndexVersion(),
+				IndexState:    buildMeta.GetState(),
+				FailReason:    buildMeta.GetFailReason(),
+				IsDeleted:     buildMeta.GetMarkDeleted(),
+				CreateTime:    record.GetCreateTime(),
+				IndexFileKeys: fileKeys,
+				IndexSize:     buildMeta.GetSerializeSize(),
 			}
 			segmentIndexModels.AddRecord(segID, indexID, segmentIndexModel)
 		}

internal/indexcoord/garbage_collector.go

@@ -22,6 +22,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/milvus-io/milvus/internal/util/metautil"
+
 	"go.uber.org/zap"
 
 	"github.com/milvus-io/milvus-proto/go-api/commonpb"
@@ -241,16 +243,18 @@ func (gc *garbageCollector) recycleUnusedIndexFiles() {
 					continue
 				}
 				log.Ctx(gc.ctx).Info("index meta can be recycled, recycle index files", zap.Int64("buildID", buildID))
-				canRecycle, indexFilePaths := gc.metaTable.GetIndexFilePathByBuildID(buildID)
+				canRecycle, segIdx := gc.metaTable.GetSegmentIndexByBuildID(buildID)
 				if !canRecycle {
 					// Even if the index is marked as deleted, the index file will not be recycled, wait for the next gc,
 					// and delete all index files about the buildID at one time.
 					log.Ctx(gc.ctx).Warn("IndexCoord garbageCollector can not recycle index files", zap.Int64("buildID", buildID))
 					continue
 				}
-				filesMap := make(map[string]bool)
-				for _, file := range indexFilePaths {
-					filesMap[file] = true
+				filesMap := make(map[string]struct{})
+				for _, fileID := range segIdx.IndexFileKeys {
+					filepath := metautil.BuildSegmentIndexFilePath(gc.chunkManager.RootPath(), segIdx.BuildID, segIdx.IndexVersion,
+						segIdx.PartitionID, segIdx.SegmentID, fileID)
+					filesMap[filepath] = struct{}{}
 				}
 				files, _, err := gc.chunkManager.ListWithPrefix(gc.ctx, key, true)
 				if err != nil {