From 6f428ef9eb83cc0e87db0eed42ede0e600606805 Mon Sep 17 00:00:00 2001 From: Tom Nabarro Date: Tue, 4 Aug 2026 22:58:41 +0100 Subject: [PATCH] DAOS-19385 control: remove per-engine sysdb on format replace Features: control Signed-off-by: Tom Nabarro --- src/control/server/ctl_storage_rpc.go | 33 ++++++++++--- .../server/storage/metadata/provider.go | 46 +++++++++++++++---- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/src/control/server/ctl_storage_rpc.go b/src/control/server/ctl_storage_rpc.go index 14eda4d4fd4..edd5608fc92 100644 --- a/src/control/server/ctl_storage_rpc.go +++ b/src/control/server/ctl_storage_rpc.go @@ -802,22 +802,43 @@ func (cs *ControlService) StorageScan(ctx context.Context, req *ctlpb.StorageSca return resp, nil } -func (cs *ControlService) formatMetadata(instances []Engine, reformat bool) (bool, error) { +func (cs *ControlService) formatMetadata(instances []Engine, reformat, replace bool) (bool, error) { // Format control metadata first, if needed if needs, err := cs.storage.ControlMetadataNeedsFormat(); err != nil { return false, errors.Wrap(err, "detecting if metadata format is needed") } else if needs || reformat { + // Full format needed engineIdxs := make([]uint, len(instances)) for i, eng := range instances { engineIdxs[i] = uint(eng.Index()) } - cs.log.Debug("formatting control metadata storage") + cs.log.Debug("formatting control metadata storage (all engines)") if err := cs.storage.FormatControlMetadata(engineIdxs); err != nil { return false, errors.Wrap(err, "formatting control metadata storage") } return true, nil + } else if replace { + // Selective format: only format engines that need SCM format + var needFormatIdxs []uint + for idx, eng := range instances { + needs, err := eng.GetStorage().ScmNeedsFormat() + if err != nil { + return false, errors.Wrapf(err, "detecting if engine %d SCM needs format", idx) + } + if needs { + needFormatIdxs = append(needFormatIdxs, uint(eng.Index())) + } + } + + if len(needFormatIdxs) > 0 { + cs.log.Debugf("formatting control metadata storage for engines %v (--replace)", needFormatIdxs) + if err := cs.storage.FormatControlMetadata(needFormatIdxs); err != nil { + return false, errors.Wrap(err, "formatting control metadata storage") + } + return true, nil + } } cs.log.Debug("no control metadata format needed") @@ -1059,10 +1080,10 @@ func (cs *ControlService) StorageFormat(ctx context.Context, req *ctlpb.StorageF return resp, nil } - // DAOS-15947: control_metadata format is valid in --replace case where multiple engines - // require replacement or format on the same host. No need to handle independently for - // individual engine as if control_metadata is missing then it needs to be created. - mdFormatted, err := cs.formatMetadata(instances, req.Reformat) + // DAOS-15947, DAOS-19385: control_metadata format is required in --replace case + // to ensure old rank metadata is cleared. Only engines requiring SCM format will + // have their control_metadata subdirectories reformatted, preserving healthy engines. + mdFormatted, err := cs.formatMetadata(instances, req.Reformat, req.Replace) if err != nil { return nil, err } diff --git a/src/control/server/storage/metadata/provider.go b/src/control/server/storage/metadata/provider.go index 41e5de98f4f..3c89aa18de0 100644 --- a/src/control/server/storage/metadata/provider.go +++ b/src/control/server/storage/metadata/provider.go @@ -198,18 +198,44 @@ func (p *Provider) isUsableFS(fs *system.FsType, path string) bool { func (p *Provider) setupDataDir(req storage.MetadataFormatRequest) error { perms := os.FileMode(0775) - if err := p.sys.RemoveAll(req.DataPath); err != nil { - return errors.Wrap(err, "removing old control metadata subdirectory") - } - - if err := p.sys.Mkdir(req.DataPath, perms); err != nil { - return errors.Wrap(err, "creating control metadata subdirectory") - } - - if err := p.sys.Chown(req.DataPath, req.OwnerUID, req.OwnerGID); err != nil { - return errors.Wrapf(err, "setting ownership of control metadata subdirectory to %d/%d", req.OwnerUID, req.OwnerGID) + // If specific engine indices are provided and DataPath exists, only delete those engines + if len(req.EngineIdxs) > 0 { + if _, err := p.sys.Stat(req.DataPath); err == nil { + // DataPath exists, selectively remove only specified engine directories + p.log.Debugf("selectively removing control metadata for engines %v", req.EngineIdxs) + for _, idx := range req.EngineIdxs { + engPath := storage.ControlMetadataEngineDir(req.DataPath, idx) + if err := p.sys.RemoveAll(engPath); err != nil { + return errors.Wrapf(err, "removing control metadata for engine %d", idx) + } + } + } else if !os.IsNotExist(err) { + return errors.Wrap(err, "checking control metadata subdirectory") + } else { + // DataPath doesn't exist, create it + p.log.Debugf("creating control metadata subdirectory %q", req.DataPath) + if err := p.sys.Mkdir(req.DataPath, perms); err != nil { + return errors.Wrap(err, "creating control metadata subdirectory") + } + if err := p.sys.Chown(req.DataPath, req.OwnerUID, req.OwnerGID); err != nil { + return errors.Wrapf(err, "setting ownership of control metadata subdirectory to %d/%d", req.OwnerUID, req.OwnerGID) + } + } + } else { + // No specific engines, remove everything (legacy behavior) + p.log.Debugf("removing entire control metadata subdirectory %q", req.DataPath) + if err := p.sys.RemoveAll(req.DataPath); err != nil { + return errors.Wrap(err, "removing old control metadata subdirectory") + } + if err := p.sys.Mkdir(req.DataPath, perms); err != nil { + return errors.Wrap(err, "creating control metadata subdirectory") + } + if err := p.sys.Chown(req.DataPath, req.OwnerUID, req.OwnerGID); err != nil { + return errors.Wrapf(err, "setting ownership of control metadata subdirectory to %d/%d", req.OwnerUID, req.OwnerGID) + } } + // Create engine subdirectories for specified engines for _, idx := range req.EngineIdxs { engPath := storage.ControlMetadataEngineDir(req.DataPath, idx) if err := p.sys.Mkdir(engPath, perms); err != nil {