diff --git a/engine/storage/raft_storage/raft_state.go b/engine/storage/raft_storage/raft_state.go index 7073d99..97c98f2 100644 --- a/engine/storage/raft_storage/raft_state.go +++ b/engine/storage/raft_storage/raft_state.go @@ -334,7 +334,7 @@ func (e *raftStatePersistence) loadOrBootstrap(initialPeers []uint64) error { } e.hardState = &raftpb.HardState{} - e.confState = &raftpb.ConfState{Nodes: peers} + e.confState = &raftpb.ConfState{Voters: peers} e.snapshot = &raftpb.Snapshot{Metadata: &raftpb.SnapshotMetadata{ConfState: cloneConfState(e.confState)}} return e.db.Update(func(txn *badger.Txn) error { if err := txn.Set(e.key(initializedSuffix), []byte{1}); err != nil { diff --git a/proto/pkg/raftpb/raftpb.pb.go b/proto/pkg/raftpb/raftpb.pb.go index 5bdc1a3..31f19d0 100644 --- a/proto/pkg/raftpb/raftpb.pb.go +++ b/proto/pkg/raftpb/raftpb.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v7.34.1 +// protoc v7.35.0 // source: raftpb.proto package raftpb @@ -26,6 +26,9 @@ type EntryType int32 const ( EntryType_EntryNormal EntryType = 0 EntryType_EntryConfChange EntryType = 1 + // Reserved for the future joint-consensus implementation. Simple + // one-at-a-time changes continue to use EntryConfChange. + EntryType_EntryConfChangeV2 EntryType = 2 ) // Enum value maps for EntryType. @@ -33,10 +36,12 @@ var ( EntryType_name = map[int32]string{ 0: "EntryNormal", 1: "EntryConfChange", + 2: "EntryConfChangeV2", } EntryType_value = map[string]int32{ - "EntryNormal": 0, - "EntryConfChange": 1, + "EntryNormal": 0, + "EntryConfChange": 1, + "EntryConfChangeV2": 2, } ) @@ -146,8 +151,10 @@ func (MessageType) EnumDescriptor() ([]byte, []int) { type ConfChangeType int32 const ( - ConfChangeType_AddNode ConfChangeType = 0 - ConfChangeType_RemoveNode ConfChangeType = 1 + ConfChangeType_AddNode ConfChangeType = 0 + ConfChangeType_RemoveNode ConfChangeType = 1 + ConfChangeType_UpdateNode ConfChangeType = 2 + ConfChangeType_AddLearnerNode ConfChangeType = 3 ) // Enum value maps for ConfChangeType. @@ -155,10 +162,14 @@ var ( ConfChangeType_name = map[int32]string{ 0: "AddNode", 1: "RemoveNode", + 2: "UpdateNode", + 3: "AddLearnerNode", } ConfChangeType_value = map[string]int32{ - "AddNode": 0, - "RemoveNode": 1, + "AddNode": 0, + "RemoveNode": 1, + "UpdateNode": 2, + "AddLearnerNode": 3, } ) @@ -546,10 +557,18 @@ func (x *HardState) GetCommit() uint64 { } type ConfState struct { - state protoimpl.MessageState `protogen:"open.v1"` - Nodes []uint64 `protobuf:"varint,1,rep,packed,name=nodes,proto3" json:"nodes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + // Field 1 used to be named "nodes". Renaming it is wire compatible and + // makes the voting role explicit. + Voters []uint64 `protobuf:"varint,1,rep,packed,name=voters,proto3" json:"voters,omitempty"` + Learners []uint64 `protobuf:"varint,2,rep,packed,name=learners,proto3" json:"learners,omitempty"` + // These fields mirror etcd/raft's ConfState layout so joint consensus can + // be added without another durable-format migration. + VotersOutgoing []uint64 `protobuf:"varint,3,rep,packed,name=voters_outgoing,json=votersOutgoing,proto3" json:"voters_outgoing,omitempty"` + LearnersNext []uint64 `protobuf:"varint,4,rep,packed,name=learners_next,json=learnersNext,proto3" json:"learners_next,omitempty"` + AutoLeave bool `protobuf:"varint,5,opt,name=auto_leave,json=autoLeave,proto3" json:"auto_leave,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ConfState) Reset() { @@ -582,13 +601,41 @@ func (*ConfState) Descriptor() ([]byte, []int) { return file_raftpb_proto_rawDescGZIP(), []int{5} } -func (x *ConfState) GetNodes() []uint64 { +func (x *ConfState) GetVoters() []uint64 { + if x != nil { + return x.Voters + } + return nil +} + +func (x *ConfState) GetLearners() []uint64 { + if x != nil { + return x.Learners + } + return nil +} + +func (x *ConfState) GetVotersOutgoing() []uint64 { if x != nil { - return x.Nodes + return x.VotersOutgoing } return nil } +func (x *ConfState) GetLearnersNext() []uint64 { + if x != nil { + return x.LearnersNext + } + return nil +} + +func (x *ConfState) GetAutoLeave() bool { + if x != nil { + return x.AutoLeave + } + return false +} + type ConfChange struct { state protoimpl.MessageState `protogen:"open.v1"` ChangeType ConfChangeType `protobuf:"varint,1,opt,name=change_type,json=changeType,proto3,enum=raftpb.ConfChangeType" json:"change_type,omitempty"` @@ -683,18 +730,24 @@ const file_raftpb_proto_rawDesc = "" + "\tHardState\x12\x12\n" + "\x04term\x18\x01 \x01(\x04R\x04term\x12\x12\n" + "\x04vote\x18\x02 \x01(\x04R\x04vote\x12\x16\n" + - "\x06commit\x18\x03 \x01(\x04R\x06commit\"!\n" + - "\tConfState\x12\x14\n" + - "\x05nodes\x18\x01 \x03(\x04R\x05nodes\"x\n" + + "\x06commit\x18\x03 \x01(\x04R\x06commit\"\xac\x01\n" + + "\tConfState\x12\x16\n" + + "\x06voters\x18\x01 \x03(\x04R\x06voters\x12\x1a\n" + + "\blearners\x18\x02 \x03(\x04R\blearners\x12'\n" + + "\x0fvoters_outgoing\x18\x03 \x03(\x04R\x0evotersOutgoing\x12#\n" + + "\rlearners_next\x18\x04 \x03(\x04R\flearnersNext\x12\x1d\n" + + "\n" + + "auto_leave\x18\x05 \x01(\bR\tautoLeave\"x\n" + "\n" + "ConfChange\x127\n" + "\vchange_type\x18\x01 \x01(\x0e2\x16.raftpb.ConfChangeTypeR\n" + "changeType\x12\x17\n" + "\anode_id\x18\x02 \x01(\x04R\x06nodeId\x12\x18\n" + - "\acontext\x18\x03 \x01(\fR\acontext*1\n" + + "\acontext\x18\x03 \x01(\fR\acontext*H\n" + "\tEntryType\x12\x0f\n" + "\vEntryNormal\x10\x00\x12\x13\n" + - "\x0fEntryConfChange\x10\x01*\xf3\x01\n" + + "\x0fEntryConfChange\x10\x01\x12\x15\n" + + "\x11EntryConfChangeV2\x10\x02*\xf3\x01\n" + "\vMessageType\x12\n" + "\n" + "\x06MsgHup\x10\x00\x12\v\n" + @@ -709,11 +762,14 @@ const file_raftpb_proto_rawDesc = "" + "\fMsgHeartbeat\x10\b\x12\x18\n" + "\x14MsgHeartbeatResponse\x10\t\x12\x15\n" + "\x11MsgTransferLeader\x10\v\x12\x11\n" + - "\rMsgTimeoutNow\x10\f*-\n" + + "\rMsgTimeoutNow\x10\f*Q\n" + "\x0eConfChangeType\x12\v\n" + "\aAddNode\x10\x00\x12\x0e\n" + "\n" + - "RemoveNode\x10\x01B2Z0github.com/Aetherance/kv/proto/pkg/raftpb;raftpbb\x06proto3" + "RemoveNode\x10\x01\x12\x0e\n" + + "\n" + + "UpdateNode\x10\x02\x12\x12\n" + + "\x0eAddLearnerNode\x10\x03B2Z0github.com/Aetherance/kv/proto/pkg/raftpb;raftpbb\x06proto3" var ( file_raftpb_proto_rawDescOnce sync.Once diff --git a/proto/proto/raftpb.proto b/proto/proto/raftpb.proto index b044909..54f7d28 100644 --- a/proto/proto/raftpb.proto +++ b/proto/proto/raftpb.proto @@ -6,6 +6,9 @@ option go_package = "github.com/Aetherance/kv/proto/pkg/raftpb;raftpb"; enum EntryType { EntryNormal = 0; EntryConfChange = 1; + // Reserved for the future joint-consensus implementation. Simple + // one-at-a-time changes continue to use EntryConfChange. + EntryConfChangeV2 = 2; } message Entry { @@ -61,12 +64,23 @@ message HardState { } message ConfState { - repeated uint64 nodes = 1; + // Field 1 used to be named "nodes". Renaming it is wire compatible and + // makes the voting role explicit. + repeated uint64 voters = 1; + repeated uint64 learners = 2; + + // These fields mirror etcd/raft's ConfState layout so joint consensus can + // be added without another durable-format migration. + repeated uint64 voters_outgoing = 3; + repeated uint64 learners_next = 4; + bool auto_leave = 5; } enum ConfChangeType { - AddNode = 0; - RemoveNode = 1; + AddNode = 0; + RemoveNode = 1; + UpdateNode = 2; + AddLearnerNode = 3; } message ConfChange { diff --git a/raft/raft.go b/raft/raft.go index 177a0d2..6d1f904 100644 --- a/raft/raft.go +++ b/raft/raft.go @@ -47,6 +47,10 @@ func (st StateType) String() string { // so that the proposer can be notified and fail fast. var ErrProposalDropped = errors.New("raft proposal dropped") +// ErrConfChangePending is returned when a second configuration change is +// proposed before the previous one has been applied. +var ErrConfChangePending = errors.New("raft configuration change already pending") + // Config contains the parameters to start a raft. type Config struct { // ID is the identity of the local raft. ID cannot be 0. @@ -101,12 +105,6 @@ func (c *Config) validate() error { return nil } -// Progress represents a follower’s progress in the view of the leader. Leader maintains -// progresses of all followers, and sends entries to the follower based on its progress. -type Progress struct { - Match, Next uint64 -} - type Raft struct { id uint64 @@ -116,7 +114,10 @@ type Raft struct { // the log RaftLog *RaftLog - // log replication progress of each peers + // Tracker owns the voter/learner configuration and replication progress. + Tracker *ProgressTracker + // Prs is kept as a compatibility alias for tests and callers that inspect + // progress. All membership mutation goes through Tracker. Prs map[uint64]*Progress // this peer's role @@ -167,21 +168,15 @@ func newRaft(c *Config) *Raft { panic(err.Error()) } - prs := make(map[uint64]*Progress) - for _, id := range c.peers { - prs[id] = &Progress{} - } - randElectionTimeout := c.ElectionTick + rand.Intn(c.ElectionTick) hs, cs, _ := c.PersistentState.InitialState() - if len(prs) == 0 { - for _, id := range cs.Nodes { - prs[id] = &Progress{} - } - } - raftLog := newLog(c.PersistentState) + tracker := newProgressTracker() + if len(c.peers) > 0 { + cs = &pb.ConfState{Voters: append([]uint64(nil), c.peers...)} + } + tracker.restore(cs, raftLog.LastIndex()) if c.Applied > 0 { if c.Applied < raftLog.entries[0].Index || c.Applied > raftLog.committed { panic("applied index is outside the committed raft log") @@ -192,12 +187,13 @@ func newRaft(c *Config) *Raft { return &Raft{ id: c.ID, RaftLog: raftLog, - Prs: prs, + Tracker: tracker, + Prs: tracker.Progress, State: StateFollower, Term: hs.Term, Vote: hs.Vote, Lead: None, - votes: make(map[uint64]bool), + votes: tracker.Votes, msgs: make([]*pb.Message, 0), heartbeatTimeout: c.HeartbeatTick, electionTimeout: c.ElectionTick, @@ -209,6 +205,9 @@ func newRaft(c *Config) *Raft { // current commit index to the given peer. Returns true if a message was sent. func (r *Raft) sendAppend(to uint64) bool { pr := r.Prs[to] + if pr == nil || pr.PendingSnapshot != 0 { + return false + } prevLogIndex := pr.Next - 1 prevLogTerm, err := r.RaftLog.Term(prevLogIndex) if err != nil { @@ -249,13 +248,14 @@ func (r *Raft) tick() { switch r.State { case StateFollower, StateCandidate: r.electionElapsed++ - if r.electionElapsed >= r.randomElectionTimeout { + if r.Tracker.isVoter(r.id) && r.electionElapsed >= r.randomElectionTimeout { r.electionElapsed = 0 r.campaign() } case StateLeader: r.heartbeatElapsed++ if r.heartbeatElapsed >= r.heartbeatTimeout { + r.heartbeatElapsed = 0 for id := range r.Prs { if id != r.id { r.sendHeartbeat(id) @@ -266,15 +266,18 @@ func (r *Raft) tick() { } func (r *Raft) campaign() { + if !r.Tracker.isVoter(r.id) { + return + } r.becomeCandidate() - if len(r.Prs) == 1 { + if len(r.Tracker.Voters) == 1 { r.becomeLeader() return } logIdx := r.RaftLog.LastIndex() logTerm, _ := r.RaftLog.Term(logIdx) - for prId, _ := range r.Prs { + for prId := range r.Tracker.Voters { if prId != r.id { r.msgs = append(r.msgs, &pb.Message{ From: r.id, @@ -300,22 +303,31 @@ func (r *Raft) becomeFollower(term uint64, lead uint64) { // becomeCandidate transform this peer's state to candidate func (r *Raft) becomeCandidate() { + if !r.Tracker.isVoter(r.id) { + return + } r.State = StateCandidate r.Term++ r.Vote = r.id - r.votes = make(map[uint64]bool) - r.votes[r.id] = true + r.Tracker.resetVotes() + r.Tracker.recordVote(r.id, true) + r.votes = r.Tracker.Votes r.electionElapsed = 0 r.randomElectionTimeout = r.electionTimeout + rand.Intn(r.electionTimeout) } // becomeLeader transform this peer's state to leader func (r *Raft) becomeLeader() { - // Your Code Here (2A). + if !r.Tracker.isVoter(r.id) { + return + } // NOTE: Leader should propose a noop entry on its term r.State = StateLeader r.Lead = r.id r.heartbeatElapsed = 0 + // Conservatively block new configuration proposals until the existing log + // tail is applied. It may contain a change proposed by the previous leader. + r.PendingConfIndex = r.RaftLog.LastIndex() lastIdx := r.RaftLog.LastIndex() for id := range r.Prs { @@ -348,8 +360,10 @@ func (r *Raft) becomeLeader() { // Step the entrance of handle message, see `MessageType` // on `eraftpb.proto` for what msgs should be handled func (r *Raft) Step(m *pb.Message) error { - // Your Code Here (2A). if m.MsgType == pb.MessageType_MsgHup && r.State != StateLeader { + if !r.Tracker.isVoter(r.id) { + return nil + } r.campaign() return nil } @@ -365,9 +379,11 @@ func (r *Raft) Step(m *pb.Message) error { return nil } - if m.MsgType == pb.MessageType_MsgPropose && r.State == StateLeader { - r.handlePropose(m) - return nil + if m.MsgType == pb.MessageType_MsgPropose { + if r.State != StateLeader { + return ErrProposalDropped + } + return r.handlePropose(m) } if m.Term > r.Term { @@ -502,6 +518,13 @@ func (r *Raft) handleHeartbeat(m *pb.Message) { } func (r *Raft) handleRequestVote(m *pb.Message) { + if !r.Tracker.isVoter(r.id) || !r.Tracker.isVoter(m.From) { + r.msgs = append(r.msgs, &pb.Message{ + From: r.id, To: m.From, Term: r.Term, + MsgType: pb.MessageType_MsgRequestVoteResponse, Reject: true, + }) + return + } lastIdx := r.RaftLog.LastIndex() lastTerm, _ := r.RaftLog.Term(lastIdx) @@ -537,22 +560,32 @@ func (r *Raft) handleRequestVote(m *pb.Message) { } func (r *Raft) handleRequestVoteResp(m *pb.Message) { - r.votes[m.From] = !m.Reject - voteCount := 0 - for _, v := range r.votes { - if v { - voteCount++ - } - } - if voteCount > len(r.Prs)/2 { + r.Tracker.recordVote(m.From, !m.Reject) + r.votes = r.Tracker.Votes + granted, rejected := r.Tracker.tallyVotes() + if granted >= r.Tracker.quorum() { r.becomeLeader() - } else if len(r.votes) == len(r.Prs) { + } else if rejected >= r.Tracker.quorum() { r.becomeFollower(r.Term, None) } } -func (r *Raft) handlePropose(m *pb.Message) { +func (r *Raft) handlePropose(m *pb.Message) error { + pendingConfChange := r.PendingConfIndex > r.RaftLog.applied + for _, entry := range m.Entries { + if entry.EntryType != pb.EntryType_EntryConfChange { + continue + } + if pendingConfChange { + return ErrConfChangePending + } + pendingConfChange = true + } + for _, entry := range m.Entries { + if entry.EntryType == pb.EntryType_EntryConfChange { + r.PendingConfIndex = r.RaftLog.LastIndex() + 1 + } entry.Term = r.Term entry.Index = r.RaftLog.LastIndex() + 1 r.RaftLog.entries = append(r.RaftLog.entries, entry) @@ -569,15 +602,26 @@ func (r *Raft) handlePropose(m *pb.Message) { r.sendAppend(id) } } + return nil } func (r *Raft) handleAppendResponse(m *pb.Message) { + pr := r.Prs[m.From] + if pr == nil { + return + } + pr.RecentActive = true success := !m.Reject if success { - r.Prs[m.From].Match = m.Index - r.Prs[m.From].Next = m.Index + 1 + pr.Match = m.Index + pr.Next = m.Index + 1 + if pr.PendingSnapshot != 0 && m.Index >= pr.PendingSnapshot { + pr.PendingSnapshot = 0 + } } else { - r.Prs[m.From].Next-- + if pr.Next > 1 { + pr.Next-- + } r.sendAppend(m.From) } @@ -591,27 +635,25 @@ func (r *Raft) handleAppendResponse(m *pb.Message) { } func (r *Raft) maybeCommit() bool { - for idx := r.RaftLog.committed + 1; idx <= r.RaftLog.LastIndex(); idx++ { - term, _ := r.RaftLog.Term(idx) - if term != r.Term { - continue - } - count := 0 - for _, pr := range r.Prs { - if pr.Match >= idx { - count++ - } - } - if count > len(r.Prs)/2 { - r.RaftLog.committed = idx - return true - } + index := r.Tracker.committed() + if index <= r.RaftLog.committed { + return false + } + term, err := r.RaftLog.Term(index) + if err != nil || term != r.Term { + return false } - return false + r.RaftLog.committed = index + return true } func (r *Raft) handleHeartbeatResponse(m *pb.Message) { - if r.Prs[m.From].Match < r.RaftLog.LastIndex() { + pr := r.Prs[m.From] + if pr == nil { + return + } + pr.RecentActive = true + if pr.Match < r.RaftLog.LastIndex() { r.sendAppend(m.From) } } @@ -620,6 +662,8 @@ func (r *Raft) handleHeartbeatResponse(m *pb.Message) { func (r *Raft) handleSnapshot(m *pb.Message) { snapshot := m.Snapshot if snapshot.Metadata.Index <= r.RaftLog.committed { + r.msgs = append(r.msgs, &pb.Message{From: r.id, To: m.From, Term: r.Term, + MsgType: pb.MessageType_MsgAppendResponse, Index: r.RaftLog.committed}) return } r.RaftLog.entries = []*pb.Entry{{Term: snapshot.Metadata.Term, Index: snapshot.Metadata.Index}} @@ -628,11 +672,11 @@ func (r *Raft) handleSnapshot(m *pb.Message) { r.RaftLog.applied = snapshot.Metadata.Index r.RaftLog.pendingSnapshot = snapshot - r.Prs = make(map[uint64]*Progress) - for _, node := range snapshot.Metadata.ConfState.Nodes { - r.Prs[node] = &Progress{Next: snapshot.Metadata.Index + 1} - } + r.Tracker.restore(snapshot.Metadata.ConfState, snapshot.Metadata.Index) + r.Prs = r.Tracker.Progress r.Lead = m.From + r.msgs = append(r.msgs, &pb.Message{From: r.id, To: m.From, Term: r.Term, + MsgType: pb.MessageType_MsgAppendResponse, Index: snapshot.Metadata.Index}) } func (r *Raft) sendSnapshot(to uint64) { @@ -647,14 +691,43 @@ func (r *Raft) sendSnapshot(to uint64) { &pb.Message{MsgType: pb.MessageType_MsgSnapshot, From: r.id, To: to, Term: r.Term, Snapshot: snapshot}, ) r.Prs[to].Next = snapshot.Metadata.Index + 1 + r.Prs[to].PendingSnapshot = snapshot.Metadata.Index } // addNode add a new node to raft group func (r *Raft) addNode(id uint64) { - // Your Code Here (3A). + r.Tracker.makeVoter(id, r.RaftLog.LastIndex()) + r.Prs = r.Tracker.Progress + if r.State == StateLeader && id == r.id { + if pr := r.Prs[id]; pr != nil { + pr.Match = r.RaftLog.LastIndex() + pr.Next = pr.Match + 1 + } + } +} + +// addLearner adds a non-voting replication target. +func (r *Raft) addLearner(id uint64) { + r.Tracker.addLearner(id, r.RaftLog.LastIndex()) + r.Prs = r.Tracker.Progress } // removeNode remove a node from raft group func (r *Raft) removeNode(id uint64) { - // Your Code Here (3A). + r.Tracker.remove(id) + r.Prs = r.Tracker.Progress + if len(r.Tracker.Voters) == 0 { + return + } + if id == r.id && r.State == StateLeader { + r.becomeFollower(r.Term, None) + return + } + if r.State == StateLeader && r.maybeCommit() { + for peerID := range r.Prs { + if peerID != r.id { + r.sendAppend(peerID) + } + } + } } diff --git a/raft/raft_test.go b/raft/raft_test.go index 93a925f..bf69f2a 100644 --- a/raft/raft_test.go +++ b/raft/raft_test.go @@ -1038,7 +1038,7 @@ func TestRestoreSnapshot2C(t *testing.T) { Metadata: &pb.SnapshotMetadata{ Index: 11, // magic number Term: 11, // magic number - ConfState: &pb.ConfState{Nodes: []uint64{1, 2, 3}}, + ConfState: &pb.ConfState{Voters: []uint64{1, 2, 3}}, }, } @@ -1053,8 +1053,8 @@ func TestRestoreSnapshot2C(t *testing.T) { t.Errorf("log.lastTerm = %d, want %d", mustTerm(sm.RaftLog.Term(s.Metadata.Index)), s.Metadata.Term) } sg := nodes(sm) - if !reflect.DeepEqual(sg, s.Metadata.ConfState.Nodes) { - t.Errorf("sm.Nodes = %+v, want %+v", sg, s.Metadata.ConfState.Nodes) + if !reflect.DeepEqual(sg, s.Metadata.ConfState.Voters) { + t.Errorf("sm.Voters = %+v, want %+v", sg, s.Metadata.ConfState.Voters) } } @@ -1071,7 +1071,7 @@ func TestRestoreIgnoreSnapshot2C(t *testing.T) { Metadata: &pb.SnapshotMetadata{ Index: commit, Term: 1, - ConfState: &pb.ConfState{Nodes: []uint64{1, 2}}, + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, }, } @@ -1088,7 +1088,7 @@ func TestProvideSnap2C(t *testing.T) { Metadata: &pb.SnapshotMetadata{ Index: 11, // magic number Term: 11, // magic number - ConfState: &pb.ConfState{Nodes: []uint64{1, 2}}, + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, }, } storage := NewMemoryState() @@ -1118,7 +1118,7 @@ func TestRestoreFromSnapMsg2C(t *testing.T) { Metadata: &pb.SnapshotMetadata{ Index: 11, // magic number Term: 11, // magic number - ConfState: &pb.ConfState{Nodes: []uint64{1, 2}}, + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, }, } m := &pb.Message{MsgType: pb.MessageType_MsgSnapshot, From: 1, Term: 2, Snapshot: &s} @@ -1136,7 +1136,7 @@ func TestRestoreFromSnapWithOverlapingPeersMsg2C(t *testing.T) { Metadata: &pb.SnapshotMetadata{ Index: 11, // magic number Term: 11, // magic number - ConfState: &pb.ConfState{Nodes: []uint64{2, 3, 4}}, + ConfState: &pb.ConfState{Voters: []uint64{2, 3, 4}}, }, } m := &pb.Message{MsgType: pb.MessageType_MsgSnapshot, From: 1, Term: 2, Snapshot: &s} @@ -1148,7 +1148,7 @@ func TestRestoreFromSnapWithOverlapingPeersMsg2C(t *testing.T) { t.Errorf("sm.Lead = %d, want 1", sm.Lead) } - nodes := s.Metadata.ConfState.Nodes + nodes := s.Metadata.ConfState.Voters if len(nodes) != len(sm.Prs) { t.Errorf("len(sm.Prs) = %d, want %d", len(sm.Prs), len(nodes)) } @@ -1170,7 +1170,7 @@ func TestSlowNodeRestore2C(t *testing.T) { } lead := nt.peers[1].(*Raft) nextEnts(lead, nt.storage[1]) - nt.storage[1].CreateSnapshot(lead.RaftLog.applied, &pb.ConfState{Nodes: nodes(lead)}, nil) + nt.storage[1].CreateSnapshot(lead.RaftLog.applied, &pb.ConfState{Voters: nodes(lead)}, nil) nt.storage[1].Compact(lead.RaftLog.applied) nt.recover() diff --git a/raft/rawnode.go b/raft/rawnode.go index f699a34..a36a34f 100644 --- a/raft/rawnode.go +++ b/raft/rawnode.go @@ -82,6 +82,15 @@ type PrevStates struct { // NewRawNode returns a new RawNode given configuration and a list of raft peers. func NewRawNode(config *Config) (*RawNode, error) { + if config == nil { + return nil, errors.New("raft: nil config") + } + if err := config.validate(); err != nil { + return nil, err + } + if _, _, err := config.PersistentState.InitialState(); err != nil { + return nil, err + } raft := newRaft(config) return &RawNode{ @@ -123,6 +132,7 @@ func (rn *RawNode) ProposeConfChange(cc *pb.ConfChange) error { ent := pb.Entry{EntryType: pb.EntryType_EntryConfChange, Data: data} return rn.Raft.Step(&pb.Message{ MsgType: pb.MessageType_MsgPropose, + From: rn.Raft.id, Entries: []*pb.Entry{&ent}, }) } @@ -130,17 +140,22 @@ func (rn *RawNode) ProposeConfChange(cc *pb.ConfChange) error { // ApplyConfChange applies a config change to the local node. func (rn *RawNode) ApplyConfChange(cc *pb.ConfChange) *pb.ConfState { if cc.NodeId == None { - return &pb.ConfState{Nodes: nodes(rn.Raft)} + return rn.Raft.Tracker.confState() } switch cc.ChangeType { case pb.ConfChangeType_AddNode: rn.Raft.addNode(cc.NodeId) case pb.ConfChangeType_RemoveNode: rn.Raft.removeNode(cc.NodeId) + case pb.ConfChangeType_UpdateNode: + // Address metadata is owned by the application. The Raft voting + // configuration is unchanged. + case pb.ConfChangeType_AddLearnerNode: + rn.Raft.addLearner(cc.NodeId) default: panic("unexpected conf type") } - return &pb.ConfState{Nodes: nodes(rn.Raft)} + return rn.Raft.Tracker.confState() } // Step advances the state machine using the given message. @@ -235,6 +250,29 @@ func (rn *RawNode) GetProgress() map[uint64]Progress { return prs } +// ConfState returns a point-in-time copy of the active voter/learner set. +func (rn *RawNode) ConfState() *pb.ConfState { return rn.Raft.Tracker.confState() } + +func (rn *RawNode) IsVoter(id uint64) bool { return rn.Raft.Tracker.isVoter(id) } + +func (rn *RawNode) IsLearner(id uint64) bool { return rn.Raft.Tracker.isLearner(id) } + +func (rn *RawNode) LeaderID() uint64 { return rn.Raft.Lead } + +func (rn *RawNode) CommitIndex() uint64 { return rn.Raft.RaftLog.committed } + +func (rn *RawNode) AppliedIndex() uint64 { return rn.Raft.RaftLog.applied } + +// ReportSnapshot makes a failed snapshot eligible for retry. A successful send +// remains pending until the follower acknowledges the installed index. +func (rn *RawNode) ReportSnapshot(id uint64, success bool) { + progress := rn.Raft.Prs[id] + if progress == nil || progress.PendingSnapshot == 0 || success { + return + } + progress.PendingSnapshot = 0 +} + // TransferLeader tries to transfer leadership to the given transferee. func (rn *RawNode) TransferLeader(transferee uint64) { _ = rn.Raft.Step(&pb.Message{MsgType: pb.MessageType_MsgTransferLeader, From: transferee}) diff --git a/raft/rawnode_test.go b/raft/rawnode_test.go index c089a5f..886f0ab 100644 --- a/raft/rawnode_test.go +++ b/raft/rawnode_test.go @@ -83,7 +83,7 @@ func TestRawNodeRestart2AC(t *testing.T) { func TestRawNodeRestartFromSnapshot2C(t *testing.T) { snap := pb.Snapshot{ Metadata: &pb.SnapshotMetadata{ - ConfState: &pb.ConfState{Nodes: []uint64{1, 2}}, + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, Index: 2, Term: 1, }, diff --git a/raft/tracker.go b/raft/tracker.go new file mode 100644 index 0000000..25fe2d5 --- /dev/null +++ b/raft/tracker.go @@ -0,0 +1,166 @@ +package raft + +import ( + "sort" + + pb "github.com/Aetherance/kv/proto/pkg/raftpb" +) + +// Progress represents a peer's replication progress in the leader's view. +// Learners are replicated to exactly like voters, but never take part in +// elections or quorum calculations. +type Progress struct { + Match uint64 + Next uint64 + IsLearner bool + RecentActive bool + PendingSnapshot uint64 +} + +// ProgressTracker owns the active Raft configuration and the replication +// progress for every voter and learner. Its shape intentionally follows +// etcd/raft's tracker, while this first implementation supports only simple +// one-at-a-time configuration changes. +type ProgressTracker struct { + Voters map[uint64]struct{} + Learners map[uint64]struct{} + Progress map[uint64]*Progress + Votes map[uint64]bool +} + +func newProgressTracker() *ProgressTracker { + return &ProgressTracker{ + Voters: make(map[uint64]struct{}), + Learners: make(map[uint64]struct{}), + Progress: make(map[uint64]*Progress), + Votes: make(map[uint64]bool), + } +} + +func (t *ProgressTracker) restore(cs *pb.ConfState, lastIndex uint64) { + t.Voters = make(map[uint64]struct{}) + t.Learners = make(map[uint64]struct{}) + t.Progress = make(map[uint64]*Progress) + if cs == nil { + return + } + for _, id := range cs.Voters { + if id == None { + continue + } + t.Voters[id] = struct{}{} + t.Progress[id] = &Progress{Next: lastIndex + 1} + } + for _, id := range cs.Learners { + if id == None { + continue + } + if _, voter := t.Voters[id]; voter { + continue + } + t.Learners[id] = struct{}{} + t.Progress[id] = &Progress{Next: lastIndex + 1, IsLearner: true} + } +} + +func (t *ProgressTracker) confState() *pb.ConfState { + return &pb.ConfState{ + Voters: sortedIDs(t.Voters), + Learners: sortedIDs(t.Learners), + } +} + +func (t *ProgressTracker) voterNodes() []uint64 { return sortedIDs(t.Voters) } + +func (t *ProgressTracker) learnerNodes() []uint64 { return sortedIDs(t.Learners) } + +func (t *ProgressTracker) isVoter(id uint64) bool { + _, ok := t.Voters[id] + return ok +} + +func (t *ProgressTracker) isLearner(id uint64) bool { + _, ok := t.Learners[id] + return ok +} + +func (t *ProgressTracker) quorum() int { return len(t.Voters)/2 + 1 } + +func (t *ProgressTracker) committed() uint64 { + if len(t.Voters) == 0 { + return 0 + } + matches := make([]uint64, 0, len(t.Voters)) + for id := range t.Voters { + if pr := t.Progress[id]; pr != nil { + matches = append(matches, pr.Match) + } else { + matches = append(matches, 0) + } + } + sort.Slice(matches, func(i, j int) bool { return matches[i] < matches[j] }) + return matches[len(matches)-t.quorum()] +} + +func (t *ProgressTracker) resetVotes() { t.Votes = make(map[uint64]bool) } + +func (t *ProgressTracker) recordVote(id uint64, granted bool) { + if !t.isVoter(id) { + return + } + if _, seen := t.Votes[id]; !seen { + t.Votes[id] = granted + } +} + +func (t *ProgressTracker) tallyVotes() (granted, rejected int) { + for id, vote := range t.Votes { + if !t.isVoter(id) { + continue + } + if vote { + granted++ + } else { + rejected++ + } + } + return granted, rejected +} + +func (t *ProgressTracker) addLearner(id, lastIndex uint64) { + if id == None || t.isVoter(id) || t.isLearner(id) { + return + } + t.Learners[id] = struct{}{} + t.Progress[id] = &Progress{Next: lastIndex + 1, IsLearner: true} +} + +func (t *ProgressTracker) makeVoter(id, lastIndex uint64) { + if id == None || t.isVoter(id) { + return + } + if pr := t.Progress[id]; pr != nil { + delete(t.Learners, id) + pr.IsLearner = false + t.Voters[id] = struct{}{} + return + } + t.Voters[id] = struct{}{} + t.Progress[id] = &Progress{Next: lastIndex + 1} +} + +func (t *ProgressTracker) remove(id uint64) { + delete(t.Voters, id) + delete(t.Learners, id) + delete(t.Progress, id) + delete(t.Votes, id) +} + +func sortedIDs(set map[uint64]struct{}) []uint64 { + ids := make([]uint64, 0, len(set)) + for id := range set { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + return ids +} diff --git a/raft/tracker_test.go b/raft/tracker_test.go new file mode 100644 index 0000000..efdb31e --- /dev/null +++ b/raft/tracker_test.go @@ -0,0 +1,91 @@ +package raft + +import ( + "errors" + "testing" + + pb "github.com/Aetherance/kv/proto/pkg/raftpb" +) + +func TestLearnerDoesNotAffectQuorum(t *testing.T) { + r := newTestRaft(1, []uint64{1, 2, 3}, 10, 1, NewMemoryState()) + r.becomeCandidate() + r.becomeLeader() + r.addLearner(4) + + lastIndex := r.RaftLog.LastIndex() + r.Prs[4].Match = lastIndex + r.Prs[4].Next = lastIndex + 1 + if r.maybeCommit() || r.RaftLog.committed != 0 { + t.Fatalf("leader plus learner committed without a voter quorum: %d", r.RaftLog.committed) + } + + r.Prs[2].Match = lastIndex + r.Prs[2].Next = lastIndex + 1 + if !r.maybeCommit() || r.RaftLog.committed != lastIndex { + t.Fatalf("voter quorum did not commit index %d: committed=%d", lastIndex, r.RaftLog.committed) + } +} + +func TestLearnerCannotCampaignOrVote(t *testing.T) { + storage := NewMemoryState() + storage.snapshot.Metadata.ConfState = &pb.ConfState{Voters: []uint64{1}, Learners: []uint64{2}} + r := newRaft(&Config{ID: 2, ElectionTick: 10, HeartbeatTick: 1, PersistentState: storage}) + + if err := r.Step(&pb.Message{MsgType: pb.MessageType_MsgHup}); err != nil { + t.Fatalf("campaign: %v", err) + } + if r.State != StateFollower || r.Term != 0 { + t.Fatalf("learner campaigned: state=%s term=%d", r.State, r.Term) + } + if err := r.Step(&pb.Message{MsgType: pb.MessageType_MsgRequestVote, From: 1, To: 2, Term: 1}); err != nil { + t.Fatalf("vote request: %v", err) + } + if len(r.msgs) != 1 || !r.msgs[0].Reject { + t.Fatalf("learner vote response = %+v, want rejection", r.msgs) + } +} + +func TestPromotingLearnerPreservesProgress(t *testing.T) { + r := newTestRaft(1, []uint64{1}, 10, 1, NewMemoryState()) + r.becomeCandidate() + r.becomeLeader() + r.addLearner(2) + r.Prs[2].Match = 9 + r.Prs[2].Next = 10 + r.Prs[2].RecentActive = true + + r.addNode(2) + if !r.Tracker.isVoter(2) || r.Tracker.isLearner(2) || r.Prs[2].IsLearner { + t.Fatalf("promoted member has wrong tracker state: %+v", r.Prs[2]) + } + if r.Prs[2].Match != 9 || r.Prs[2].Next != 10 || !r.Prs[2].RecentActive { + t.Fatalf("promotion reset replication progress: %+v", r.Prs[2]) + } +} + +func TestOnlyOneUnappliedConfChangeIsAccepted(t *testing.T) { + r := newTestRaft(1, []uint64{1}, 10, 1, NewMemoryState()) + r.becomeCandidate() + r.becomeLeader() + change := &pb.Entry{EntryType: pb.EntryType_EntryConfChange} + proposal := func(entries ...*pb.Entry) error { + return r.Step(&pb.Message{MsgType: pb.MessageType_MsgPropose, From: 1, Entries: entries}) + } + + if err := proposal(change); err != nil { + t.Fatalf("first conf change: %v", err) + } + lastIndex := r.RaftLog.LastIndex() + if err := proposal(&pb.Entry{EntryType: pb.EntryType_EntryConfChange}); !errors.Is(err, ErrConfChangePending) { + t.Fatalf("second conf change error = %v, want ErrConfChangePending", err) + } + if r.RaftLog.LastIndex() != lastIndex { + t.Fatalf("rejected conf change changed last index to %d", r.RaftLog.LastIndex()) + } + + r.RaftLog.applied = r.PendingConfIndex + if err := proposal(&pb.Entry{EntryType: pb.EntryType_EntryConfChange}); err != nil { + t.Fatalf("conf change after apply: %v", err) + } +} diff --git a/raft/util.go b/raft/util.go index 1c9d943..791a374 100644 --- a/raft/util.go +++ b/raft/util.go @@ -20,7 +20,6 @@ import ( "io/ioutil" "os" "os/exec" - "sort" "strings" pb "github.com/Aetherance/kv/proto/pkg/raftpb" @@ -57,12 +56,7 @@ func mustTerm(term uint64, err error) uint64 { } func nodes(r *Raft) []uint64 { - nodes := make([]uint64, 0, len(r.Prs)) - for id := range r.Prs { - nodes = append(nodes, id) - } - sort.Sort(uint64Slice(nodes)) - return nodes + return r.Tracker.voterNodes() } func diffu(a, b string) string {