[Feature] Member Replace Operation

CHANGELOG.md

@@ -16,6 +16,7 @@
 - (Feature) UpgradeByReplace Flow
 - (Feature) (Platform) ArangoRoute Timeout option
 - (Feature) Delay Action
+- (Feature) MigrateMember Action
 
 ## [1.2.44](https://github.com/arangodb/kube-arangodb/tree/1.2.44) (2025-02-03)
 - (Maintenance) Kubernetes 1.31.1 libraries

README.md

@@ -173,6 +173,7 @@ Flags:
       --deployment.feature.local-volume-replacement-check      Replace volume for local-storage if volume is unschedulable (ex. node is gone) - Required ArangoDB >= 3.8.0
       --deployment.feature.random-pod-names                    Enables generating random pod names - Required ArangoDB >= 3.8.0
       --deployment.feature.rebalancer-v2                       Rebalancer V2 feature - Required ArangoDB >= 3.10.0
+      --deployment.feature.replace-migration                   During member replacement shards are migrated directly to the new server - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.restart-policy-always               Allow to restart containers with always restart policy - Required ArangoDB >= 3.8.0
       --deployment.feature.secured-containers                  Create server's containers with non root privileges. It enables 'ephemeral-volumes' feature implicitly - Required ArangoDB >= 3.8.0
       --deployment.feature.sensitive-information-protection    Hide sensitive information from metrics and logs - Required ArangoDB >= 3.8.0

docs/cli/arangodb_operator.md

@@ -57,6 +57,7 @@ Flags:
       --deployment.feature.local-volume-replacement-check      Replace volume for local-storage if volume is unschedulable (ex. node is gone) - Required ArangoDB >= 3.8.0
       --deployment.feature.random-pod-names                    Enables generating random pod names - Required ArangoDB >= 3.8.0
       --deployment.feature.rebalancer-v2                       Rebalancer V2 feature - Required ArangoDB >= 3.10.0
+      --deployment.feature.replace-migration                   During member replacement shards are migrated directly to the new server - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.restart-policy-always               Allow to restart containers with always restart policy - Required ArangoDB >= 3.8.0
       --deployment.feature.secured-containers                  Create server's containers with non root privileges. It enables 'ephemeral-volumes' feature implicitly - Required ArangoDB >= 3.8.0
       --deployment.feature.sensitive-information-protection    Hide sensitive information from metrics and logs - Required ArangoDB >= 3.8.0

docs/generated/actions.md

@@ -51,6 +51,7 @@ nav_order: 11
 | MemberPhaseUpdate | no | 10m0s | no | Community & Enterprise | Change member phase |
 | ~~MemberRIDUpdate~~ | no | 10m0s | no | Community & Enterprise | Update Run ID of member |
 | MemberStatusSync | no | 10m0s | no | Community & Enterprise | Sync ArangoMember Status with ArangoDeployment Status, to keep Member information up to date |
+| MigrateMember | no | 48h0m0s | yes | Community & Enterprise | Run the data movement actions on the member (migration) |
 | PVCResize | no | 30m0s | no | Community & Enterprise | Start the resize procedure. Updates PVC Requests field |
 | PVCResized | no | 15m0s | no | Community & Enterprise | Waits for PVC resize to be completed |
 | PlaceHolder | no | 10m0s | no | Community & Enterprise | Empty placeholder action |
@@ -148,6 +149,7 @@ spec:
       MemberPhaseUpdate: 10m0s
       MemberRIDUpdate: 10m0s
       MemberStatusSync: 10m0s
+      MigrateMember: 48h0m0s
       PVCResize: 30m0s
       PVCResized: 15m0s
       PlaceHolder: 10m0s

internal/actions.yaml

@@ -27,6 +27,10 @@ actions:
   CleanOutMember:
     description: Run the CleanOut job on member
     timeout: 48h
+  MigrateMember:
+    description: Run the data movement actions on the member (migration)
+    timeout: 48h
+    optional: true
   ShutdownMember:
     description: Sends Shutdown requests and waits for container to be stopped
     timeout: 30m

pkg/deployment/agency/definitions.go

@@ -47,6 +47,7 @@ const (
 
 	SupervisionKey            = "Supervision"
 	SupervisionMaintenanceKey = "Maintenance"
+	SupervisionHealthKey      = "Health"
 
 	TargetJobToDoKey     = "ToDo"
 	TargetJobPendingKey  = "Pending"
@@ -74,6 +75,7 @@ func GetAgencyReadRequest(elements ...[]string) ReadRequest {
 func GetAgencyReadRequestFields() ReadRequest {
 	return GetAgencyReadRequest([]string{
 		GetAgencyKey(ArangoKey, SupervisionKey, SupervisionMaintenanceKey),
+		GetAgencyKey(ArangoKey, SupervisionKey, SupervisionHealthKey),
 		GetAgencyKey(ArangoKey, PlanKey, PlanCollectionsKey),
 		GetAgencyKey(ArangoKey, PlanKey, PlanDatabasesKey),
 		GetAgencyKey(ArangoKey, PlanKey, PlanDBServersKey),

pkg/deployment/agency/state/state.go

@@ -61,10 +61,6 @@ type Plan struct {
 	Coordinators ServerMap[string] `json:"Coordinators,omitempty"`
 }
 
-type Supervision struct {
-	Maintenance Timestamp `json:"Maintenance,omitempty"`
-}
-
 type ShardCountDetails struct {
 	Leader, Follower int
 }

pkg/deployment/agency/state/supervision.go

@@ -0,0 +1,48 @@
+//
+// DISCLAIMER
+//
+// Copyright 2025 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package state
+
+type Supervision struct {
+	Maintenance Timestamp `json:"Maintenance,omitempty"`
+
+	Health ServerMap[SupervisionHealthServer] `json:"Health,omitempty"`
+}
+
+type SupervisionHealthServerStatus string
+
+const (
+	SupervisionHealthServerStatusGood SupervisionHealthServerStatus = "GOOD"
+)
+
+type SupervisionHealthServerSyncStatus string
+
+const (
+	SupervisionHealthServerSyncStatusServing SupervisionHealthServerSyncStatus = "SERVING"
+)
+
+type SupervisionHealthServer struct {
+	Status     SupervisionHealthServerStatus     `json:"Status,omitempty"`
+	SyncStatus SupervisionHealthServerSyncStatus `json:"SyncStatus,omitempty"`
+}
+
+func (s SupervisionHealthServer) IsHealthy() bool {
+	return s.Status == SupervisionHealthServerStatusGood && s.SyncStatus == SupervisionHealthServerSyncStatusServing
+}

pkg/deployment/client/rebalance.execute.go

@@ -0,0 +1,83 @@
+//
+// DISCLAIMER
+//
+// Copyright 2025 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package client
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
+)
+
+type RebalanceExecuteRequest struct {
+	Version int `json:"version"`
+
+	Moves RebalanceExecuteRequestMoves `json:"moves,omitempty"`
+}
+
+type RebalanceExecuteRequestMoves []RebalanceExecuteRequestMove
+
+type RebalanceExecuteRequestMove struct {
+	Database   string `json:"database"`
+	Collection string `json:"collection"`
+	Shard      string `json:"shard"`
+
+	From string `json:"from"`
+	To   string `json:"to"`
+
+	IsLeader bool `json:"isLeader"`
+}
+
+func (c *client) RebalanceExecuteMoves(ctx context.Context, moves ...RebalanceExecuteRequestMove) error {
+	return c.RebalanceExecute(ctx, &RebalanceExecuteRequest{
+		Version: 1,
+		Moves:   moves,
+	})
+}
+
+func (c *client) RebalanceExecute(ctx context.Context, request *RebalanceExecuteRequest) error {
+	req, err := c.c.NewRequest(http.MethodPost, "/_admin/cluster/rebalance/execute")
+	if err != nil {
+		return err
+	}
+
+	request = util.InitType(request)
+
+	// Always set to 1
+	request.Version = 1
+
+	if r, err := req.SetBody(request); err != nil {
+		return err
+	} else {
+		req = r
+	}
+
+	resp, err := c.c.Do(ctx, req)
+	if err != nil {
+		return err
+	}
+
+	if err := resp.CheckStatus(http.StatusAccepted); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/deployment/client/rebalance.get.go

@@ -0,0 +1,59 @@
+//
+// DISCLAIMER
+//
+// Copyright 2025 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package client
+
+import (
+	"context"
+	"net/http"
+)
+
+type RebalanceGetResponse struct {
+	Result RebalanceGetResponseResult `json:"result,omitempty"`
+}
+
+type RebalanceGetResponseResult struct {
+	PendingMoveShards int `json:"pendingMoveShards"`
+	TodoMoveShards    int `json:"todoMoveShards"`
+}
+
+func (c *client) RebalanceGet(ctx context.Context) (RebalanceGetResponse, error) {
+	req, err := c.c.NewRequest(http.MethodGet, "/_admin/cluster/rebalance")
+	if err != nil {
+		return RebalanceGetResponse{}, err
+	}
+
+	resp, err := c.c.Do(ctx, req)
+	if err != nil {
+		return RebalanceGetResponse{}, err
+	}
+
+	if err := resp.CheckStatus(http.StatusOK); err != nil {
+		return RebalanceGetResponse{}, err
+	}
+
+	var d RebalanceGetResponse
+
+	if err := resp.ParseBody("", &d); err != nil {
+		return RebalanceGetResponse{}, err
+	}
+
+	return d, nil
+}