raft: consolidate all append message sending

This PR consolidates all decision-making about sending append messages
into a single maybeSendAppend method. Previously, the behaviour depended
on the sendIfEmpty flag which was set/unset depending on the context in
which the method is called. This is unnecessary because the Progress
struct contains enough information about the leader->follower flow
state, so maybeSendAppend can be made stand-alone.

In follow-up PRs, the consolidated maybeSendAppend method will be used
to implement a more flexible message flow control.

Epic: CRDB-37515
Release note: none

Author: pav-kv

pkg/raft/doc.go

@@ -314,7 +314,7 @@ stale log entries:
 	rafthttp package.
 
 	'MsgApp' contains log entries to replicate. A leader calls bcastAppend,
-	which calls sendAppend, which sends soon-to-be-replicated logs in 'MsgApp'
+	which calls maybeSendAppend, which sends soon-to-be-replicated logs in 'MsgApp'
 	type. When 'MsgApp' is passed to candidate's Step method, candidate reverts
 	back to follower, because it indicates that there is a valid leader sending
 	'MsgApp' messages. Candidate and follower respond to this message in
@@ -352,8 +352,8 @@ stale log entries:
 
 	'MsgSnap' requests to install a snapshot message. When a node has just
 	become a leader or the leader receives 'MsgProp' message, it calls
-	'bcastAppend' method, which then calls 'sendAppend' method to each
-	follower. In 'sendAppend', if a leader fails to get term or entries,
+	'bcastAppend' method, which then calls 'maybeSendAppend' method to each
+	follower. In 'maybeSendAppend', if a leader fails to get term or entries,
 	the leader requests snapshot by sending 'MsgSnap' type message.
 
 	'MsgSnapStatus' tells the result of snapshot install message. When a
@@ -375,7 +375,7 @@ stale log entries:
 	'MsgHeartbeatResp' is a response to 'MsgHeartbeat'. When 'MsgHeartbeatResp'
 	is passed to leader's Step method, the leader knows which follower
 	responded. And only when the leader's last committed index is greater than
-	follower's Match index, the leader runs 'sendAppend` method.
+	follower's Match index, the leader runs 'maybeSendAppend` method.
 
 	'MsgUnreachable' tells that request(message) wasn't delivered. When
 	'MsgUnreachable' is passed to leader's Step method, the leader discovers

pkg/raft/raft.go

@@ -579,24 +579,24 @@ func (r *raft) send(m pb.Message) {
 	}
 }
 
-// sendAppend sends an append RPC with new entries (if any) and the
-// current commit index to the given peer.
-func (r *raft) sendAppend(to uint64) {
-	r.maybeSendAppend(to, true)
-}
-
-// maybeSendAppend sends an append RPC with new entries to the given peer,
-// if necessary. Returns true if a message was sent. The sendIfEmpty
-// argument controls whether messages with no entries will be sent
-// ("empty" messages are useful to convey updated Commit indexes, but
-// are undesirable when we're sending multiple messages in a batch).
+// maybeSendAppend sends an append RPC with log entries (if any) that are not
+// yet known to be replicated in the given peer's log, as well as the current
+// commit index. Usually it sends a MsgApp message, but in some cases (e.g. the
+// log has been compacted) it can send a MsgSnap.
+//
+// In some cases, the MsgApp message can have zero entries, and yet be sent.
+// When the follower log is not fully up-to-date, we must send a MsgApp
+// periodically so that eventually the flow is either accepted or rejected. Not
+// doing so can result in replication stall, in cases when a MsgApp is dropped.
 //
-// TODO(pav-kv): make invocation of maybeSendAppend stateless. The Progress
-// struct contains all the state necessary for deciding whether to send a
-// message.
-func (r *raft) maybeSendAppend(to uint64, sendIfEmpty bool) bool {
+// Returns true if a message was sent, or false otherwise. A message is not sent
+// if the follower log and commit index are up-to-date, the flow is paused (for
+// reasons like in-flight limits), or the message could not be constructed.
+func (r *raft) maybeSendAppend(to uint64) bool {
 	pr := r.trk.Progress[to]
-	if pr.IsPaused() {
+
+	last, commit := r.raftLog.lastIndex(), r.raftLog.committed
+	if !pr.ShouldSendMsgApp(last, commit) {
 		return false
 	}
 
@@ -608,36 +608,26 @@ func (r *raft) maybeSendAppend(to uint64, sendIfEmpty bool) bool {
 		return r.maybeSendSnapshot(to, pr)
 	}
 
-	var ents []pb.Entry
-	// In a throttled StateReplicate only send empty MsgApp, to ensure progress.
-	// Otherwise, if we had a full Inflights and all inflight messages were in
-	// fact dropped, replication to that follower would stall. Instead, an empty
-	// MsgApp will eventually reach the follower (heartbeats responses prompt the
-	// leader to send an append), allowing it to be acked or rejected, both of
-	// which will clear out Inflights.
-	if pr.State != tracker.StateReplicate || !pr.Inflights.Full() {
-		ents, err = r.raftLog.entries(pr.Next, r.maxMsgSize)
-	}
-	if len(ents) == 0 && !sendIfEmpty {
-		return false
-	}
-	// TODO(pav-kv): move this check up to where err is returned.
-	if err != nil { // send a snapshot if we failed to get the entries
-		return r.maybeSendSnapshot(to, pr)
+	var entries []pb.Entry
+	if pr.CanSendEntries(last) {
+		if entries, err = r.raftLog.entries(pr.Next, r.maxMsgSize); err != nil {
+			// Send a snapshot if we failed to get the entries.
+			return r.maybeSendSnapshot(to, pr)
+		}
 	}
 
-	// Send the actual MsgApp otherwise, and update the progress accordingly.
+	// Send the MsgApp, and update the progress accordingly.
 	r.send(pb.Message{
 		To:      to,
 		Type:    pb.MsgApp,
 		Index:   prevIndex,
 		LogTerm: prevTerm,
-		Entries: ents,
-		Commit:  r.raftLog.committed,
+		Entries: entries,
+		Commit:  commit,
 		Match:   pr.Match,
 	})
-	pr.SentEntries(len(ents), uint64(payloadsSize(ents)))
-	pr.SentCommit(r.raftLog.committed)
+	pr.SentEntries(len(entries), uint64(payloadsSize(entries)))
+	pr.SentCommit(commit)
 	return true
 }
 
@@ -696,7 +686,7 @@ func (r *raft) bcastAppend() {
 		if id == r.id {
 			return
 		}
-		r.sendAppend(id)
+		r.maybeSendAppend(id)
 	})
 }
 
@@ -1450,7 +1440,7 @@ func stepLeader(r *raft, m pb.Message) error {
 				if pr.State == tracker.StateReplicate {
 					pr.BecomeProbe()
 				}
-				r.sendAppend(m.From)
+				r.maybeSendAppend(m.From)
 			}
 		} else {
 			// We want to update our tracking if the response updates our
@@ -1486,21 +1476,13 @@ func stepLeader(r *raft, m pb.Message) error {
 
 				if r.maybeCommit() {
 					r.bcastAppend()
-				} else if r.id != m.From && pr.CanBumpCommit(r.raftLog.committed) {
-					// This node may be missing the latest commit index, so send it.
-					// NB: this is not strictly necessary because the periodic heartbeat
-					// messages deliver commit indices too. However, a message sent now
-					// may arrive earlier than the next heartbeat fires.
-					r.sendAppend(m.From)
 				}
-				// We've updated flow control information above, which may
-				// allow us to send multiple (size-limited) in-flight messages
-				// at once (such as when transitioning from probe to
-				// replicate, or when freeTo() covers multiple messages). If
-				// we have more entries to send, send as many messages as we
-				// can (without sending empty messages for the commit index)
+				// We've updated flow control information above, which may allow us to
+				// send multiple (size-limited) in-flight messages at once (such as when
+				// transitioning from probe to replicate, or when freeTo() covers
+				// multiple messages). Send as many messages as we can.
 				if r.id != m.From {
-					for r.maybeSendAppend(m.From, false /* sendIfEmpty */) {
+					for r.maybeSendAppend(m.From) {
 					}
 				}
 				// Transfer leadership is in progress.
@@ -1512,24 +1494,8 @@ func stepLeader(r *raft, m pb.Message) error {
 		}
 	case pb.MsgHeartbeatResp:
 		pr.RecentActive = true
-		pr.MsgAppFlowPaused = false
-
-		// NB: if the follower is paused (full Inflights), this will still send an
-		// empty append, allowing it to recover from situations in which all the
-		// messages that filled up Inflights in the first place were dropped. Note
-		// also that the outgoing heartbeat already communicated the commit index.
-		//
-		// If the follower is fully caught up but also in StateProbe (as can happen
-		// if ReportUnreachable was called), we also want to send an append (it will
-		// be empty) to allow the follower to transition back to StateReplicate once
-		// it responds.
-		//
-		// Note that StateSnapshot typically satisfies pr.Match < lastIndex, but
-		// `pr.Paused()` is always true for StateSnapshot, so sendAppend is a
-		// no-op.
-		if pr.Match < r.raftLog.lastIndex() || pr.State == tracker.StateProbe {
-			r.sendAppend(m.From)
-		}
+		pr.PauseMsgAppProbes(false)
+		r.maybeSendAppend(m.From)
 
 	case pb.MsgSnapStatus:
 		if pr.State != tracker.StateSnapshot {
@@ -1585,7 +1551,8 @@ func stepLeader(r *raft, m pb.Message) error {
 			r.sendTimeoutNow(leadTransferee)
 			r.logger.Infof("%x sends MsgTimeoutNow to %x immediately as %x already has up-to-date log", r.id, leadTransferee, leadTransferee)
 		} else {
-			r.sendAppend(leadTransferee)
+			pr.PauseMsgAppProbes(false)
+			r.maybeSendAppend(leadTransferee)
 		}
 	}
 	return nil
@@ -1957,21 +1924,14 @@ func (r *raft) switchToConfig(cfg tracker.Config, trk tracker.ProgressMap) pb.Co
 		return cs
 	}
 
-	if r.maybeCommit() {
-		// If the configuration change means that more entries are committed now,
-		// broadcast/append to everyone in the updated config.
-		r.bcastAppend()
-	} else {
-		// Otherwise, still probe the newly added replicas; there's no reason to
-		// let them wait out a heartbeat interval (or the next incoming
-		// proposal).
-		r.trk.Visit(func(id uint64, pr *tracker.Progress) {
-			if id == r.id {
-				return
-			}
-			r.maybeSendAppend(id, false /* sendIfEmpty */)
-		})
-	}
+	r.maybeCommit()
+	// If the configuration change means that more entries are committed now,
+	// broadcast/append to everyone in the updated config.
+	//
+	// Otherwise, still probe the newly added replicas; there's no reason to let
+	// them wait out a heartbeat interval (or the next incoming proposal).
+	r.bcastAppend()
+
 	// If the leadTransferee was removed or demoted, abort the leadership transfer.
 	if _, tOK := r.trk.Config.Voters.IDs()[r.leadTransferee]; !tOK && r.leadTransferee != 0 {
 		r.abortLeaderTransfer()

pkg/raft/raft_test.go

@@ -134,7 +134,7 @@ func TestProgressResumeByHeartbeatResp(t *testing.T) {
 	assert.False(t, r.trk.Progress[2].MsgAppFlowPaused)
 	r.trk.Progress[2].MsgAppFlowPaused = true
 	r.Step(pb.Message{From: 2, To: 1, Type: pb.MsgHeartbeatResp})
-	assert.False(t, r.trk.Progress[2].MsgAppFlowPaused)
+	assert.True(t, r.trk.Progress[2].MsgAppFlowPaused)
 }
 
 func TestProgressPaused(t *testing.T) {
@@ -2070,7 +2070,7 @@ func TestSendAppendForProgressProbe(t *testing.T) {
 			// loop. After that, the follower is paused until a heartbeat response is
 			// received.
 			mustAppendEntry(r, pb.Entry{Data: []byte("somedata")})
-			r.sendAppend(2)
+			r.maybeSendAppend(2)
 			msg := r.readMessages()
 			assert.Len(t, msg, 1)
 			assert.Zero(t, msg[0].Index)
@@ -2079,7 +2079,7 @@ func TestSendAppendForProgressProbe(t *testing.T) {
 		assert.True(t, r.trk.Progress[2].MsgAppFlowPaused)
 		for j := 0; j < 10; j++ {
 			mustAppendEntry(r, pb.Entry{Data: []byte("somedata")})
-			r.sendAppend(2)
+			r.maybeSendAppend(2)
 			assert.Empty(t, r.readMessages())
 		}
 
@@ -2112,7 +2112,7 @@ func TestSendAppendForProgressReplicate(t *testing.T) {
 
 	for i := 0; i < 10; i++ {
 		mustAppendEntry(r, pb.Entry{Data: []byte("somedata")})
-		r.sendAppend(2)
+		r.maybeSendAppend(2)
 		msgs := r.readMessages()
 		assert.Len(t, msgs, 1, "#%d", i)
 	}
@@ -2127,7 +2127,7 @@ func TestSendAppendForProgressSnapshot(t *testing.T) {
 
 	for i := 0; i < 10; i++ {
 		mustAppendEntry(r, pb.Entry{Data: []byte("somedata")})
-		r.sendAppend(2)
+		r.maybeSendAppend(2)
 		msgs := r.readMessages()
 		assert.Empty(t, msgs, "#%d", i)
 	}
@@ -3647,10 +3647,10 @@ func TestLogReplicationWithReorderedMessage(t *testing.T) {
 
 	// r1 sends 2 MsgApp messages to r2.
 	mustAppendEntry(r1, pb.Entry{Data: []byte("somedata")})
-	r1.sendAppend(2)
+	r1.maybeSendAppend(2)
 	req1 := expectOneMessage(t, r1)
 	mustAppendEntry(r1, pb.Entry{Data: []byte("somedata")})
-	r1.sendAppend(2)
+	r1.maybeSendAppend(2)
 	req2 := expectOneMessage(t, r1)
 
 	// r2 receives the second MsgApp first due to reordering.

pkg/raft/testdata/replicate_pause.txt

@@ -76,6 +76,9 @@ deliver-msgs drop=3
 dropped: 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1_12"]
 dropped: 1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_1_13"]
 dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:11 Entries:[1/14 EntryNormal "prop_1_14"]
+dropped: 1->3 MsgApp Term:1 Log:1/14 Commit:12
+dropped: 1->3 MsgApp Term:1 Log:1/14 Commit:13
+dropped: 1->3 MsgApp Term:1 Log:1/14 Commit:14
 
 
 # Repeat committing 3 entries.

pkg/raft/testdata/slow_follower_after_compaction.txt

@@ -88,6 +88,8 @@ deliver-msgs drop=3
 ----
 dropped: 1->3 MsgApp Term:1 Log:1/14 Commit:14 Entries:[1/15 EntryNormal "prop_1_15"]
 dropped: 1->3 MsgApp Term:1 Log:1/15 Commit:14 Entries:[1/16 EntryNormal "prop_1_16"]
+dropped: 1->3 MsgApp Term:1 Log:1/16 Commit:15
+dropped: 1->3 MsgApp Term:1 Log:1/16 Commit:16
 
 # Truncate the leader's log beyond node 3 log size.
 compact 1 17

pkg/raft/tracker/inflights.go

@@ -32,6 +32,8 @@ type Inflights struct {
 	count int    // number of inflight messages in the buffer
 	bytes uint64 // number of inflight bytes
 
+	// TODO(pav-kv): do not store the limits here, pass them to methods. For flow
+	// control, we need to support dynamic limits.
 	size     int    // the max number of inflight messages
 	maxBytes uint64 // the max total byte size of inflight messages
 

pkg/raft/tracker/progress.go

@@ -30,6 +30,9 @@ import (
 // NB(tbg): Progress is basically a state machine whose transitions are mostly
 // strewn around `*raft.raft`. Additionally, some fields are only used when in a
 // certain State. All of this isn't ideal.
+//
+// TODO(pav-kv): consolidate all flow control state changes here. Much of the
+// transitions in raft.go logically belong here.
 type Progress struct {
 	// Match is the index up to which the follower's log is known to match the
 	// leader's.
@@ -122,7 +125,7 @@ type Progress struct {
 // ResetState moves the Progress into the specified State, resetting MsgAppFlowPaused,
 // PendingSnapshot, and Inflights.
 func (pr *Progress) ResetState(state StateType) {
-	pr.MsgAppFlowPaused = false
+	pr.PauseMsgAppProbes(false)
 	pr.PendingSnapshot = 0
 	pr.State = state
 	pr.Inflights.reset()
@@ -166,20 +169,25 @@ func (pr *Progress) BecomeSnapshot(snapshoti uint64) {
 //
 // Must be used with StateProbe or StateReplicate.
 func (pr *Progress) SentEntries(entries int, bytes uint64) {
-	switch pr.State {
-	case StateReplicate:
-		if entries > 0 {
-			pr.Next += uint64(entries)
-			pr.Inflights.Add(pr.Next-1, bytes)
-		}
-		// If this message overflows the in-flights tracker, or it was already full,
-		// consider this message being a probe, so that the flow is paused.
-		pr.MsgAppFlowPaused = pr.Inflights.Full()
-	case StateProbe:
-		pr.MsgAppFlowPaused = true
-	default:
-		panic(fmt.Sprintf("sending append in unhandled state %s", pr.State))
+	if pr.State == StateReplicate && entries > 0 {
+		pr.Next += uint64(entries)
+		pr.Inflights.Add(pr.Next-1, bytes)
 	}
+	pr.PauseMsgAppProbes(true)
+}
+
+// PauseMsgAppProbes pauses or unpauses empty MsgApp messages flow, depending on
+// the passed-in bool.
+func (pr *Progress) PauseMsgAppProbes(pause bool) {
+	pr.MsgAppFlowPaused = pause
+}
+
+// CanSendEntries returns true if the flow control state allows sending at least
+// one log entry to this follower.
+//
+// Must be used with StateProbe or StateReplicate.
+func (pr *Progress) CanSendEntries(lastIndex uint64) bool {
+	return pr.Next <= lastIndex && (pr.State == StateProbe || !pr.Inflights.Full())
 }
 
 // CanBumpCommit returns true if sending the given commit index can potentially
@@ -206,7 +214,6 @@ func (pr *Progress) MaybeUpdate(n uint64) bool {
 	}
 	pr.Match = n
 	pr.Next = max(pr.Next, n+1) // invariant: Match < Next
-	pr.MsgAppFlowPaused = false
 	return true
 }
 
@@ -247,7 +254,7 @@ func (pr *Progress) MaybeDecrTo(rejected, matchHint uint64) bool {
 	pr.Next = max(min(rejected, matchHint+1), pr.Match+1)
 	// Regress the sentCommit since it unlikely has been applied.
 	pr.sentCommit = min(pr.sentCommit, pr.Next-1)
-	pr.MsgAppFlowPaused = false
+	pr.PauseMsgAppProbes(false)
 	return true
 }
 
@@ -262,14 +269,65 @@ func (pr *Progress) IsPaused() bool {
 	case StateProbe:
 		return pr.MsgAppFlowPaused
 	case StateReplicate:
-		return pr.MsgAppFlowPaused
+		return pr.MsgAppFlowPaused && pr.Inflights.Full()
 	case StateSnapshot:
 		return true
 	default:
 		panic("unexpected state")
 	}
 }
 
+// ShouldSendMsgApp returns true if the leader should send a MsgApp to the
+// follower represented by this Progress. The given last and commit index of the
+// leader log help determining if there is outstanding workload, and contribute
+// to this decision-making.
+//
+// In StateProbe, a message is sent periodically. The flow is paused after every
+// message, and un-paused on a heartbeat response. This ensures that probes are
+// not too frequent, and eventually the MsgApp is either accepted or rejected.
+//
+// In StateReplicate, generally a message is sent if there are log entries that
+// are not yet in-flight, and the in-flight limits are not exceeded. Otherwise,
+// we don't send a message, or send a "probe" message in a few situations.
+//
+// A probe message (containing no log entries) is sent if the follower's commit
+// index can be updated, or there hasn't been a probe message recently. We must
+// send a message periodically even if all log entries are in-flight, in order
+// to guarantee that eventually the flow is either accepted or rejected.
+//
+// In StateSnapshot, we do not send append messages.
+func (pr *Progress) ShouldSendMsgApp(last, commit uint64) bool {
+	switch pr.State {
+	case StateProbe:
+		return !pr.MsgAppFlowPaused
+	case StateReplicate:
+		// Send a MsgApp containing the latest commit index if:
+		//	- our commit index exceeds the in-flight commit index, and
+		//	- sending it can commit at least one of the in-flight entries.
+		if pr.CanBumpCommit(commit) {
+			return true
+		}
+		// If the follower's log is up-to-date, don't send a MsgApp.
+		if pr.Match >= last {
+			return false
+		}
+		// Don't send a MsgApp if we are in a throttled replication state, i.e.
+		// pr.Inflights.Full() && pr.MsgAppFlowPaused.
+		if pr.IsPaused() {
+			return false
+		}
+		// We are here if the follower's log is not up-to-date, and the flow is not
+		// paused We can always send a MsgApp, except when everything is already
+		// in-flight, and the last MsgApp was recent.
+		return pr.Next <= last || !pr.MsgAppFlowPaused
+
+	case StateSnapshot:
+		return false
+	default:
+		panic("unexpected state")
+	}
+}
+
 func (pr *Progress) String() string {
 	var buf strings.Builder
 	fmt.Fprintf(&buf, "%s match=%d next=%d", pr.State, pr.Match, pr.Next)

pkg/raft/tracker/progress_test.go

@@ -47,7 +47,7 @@ func TestProgressIsPaused(t *testing.T) {
 		{StateProbe, false, false},
 		{StateProbe, true, true},
 		{StateReplicate, false, false},
-		{StateReplicate, true, true},
+		{StateReplicate, true, false},
 		{StateSnapshot, false, true},
 		{StateSnapshot, true, true},
 	}
@@ -61,8 +61,11 @@ func TestProgressIsPaused(t *testing.T) {
 	}
 }
 
-// TestProgressResume ensures that MaybeUpdate and MaybeDecrTo will reset
-// MsgAppFlowPaused.
+// TestProgressResume ensures that MaybeDecrTo resets MsgAppFlowPaused, and
+// MaybeUpdate does not.
+//
+// TODO(pav-kv): there is little sense in testing these micro-behaviours in the
+// struct. We should test the visible behaviour instead.
 func TestProgressResume(t *testing.T) {
 	p := &Progress{
 		Next:             2,
@@ -72,7 +75,7 @@ func TestProgressResume(t *testing.T) {
 	assert.False(t, p.MsgAppFlowPaused)
 	p.MsgAppFlowPaused = true
 	p.MaybeUpdate(2)
-	assert.False(t, p.MsgAppFlowPaused)
+	assert.True(t, p.MsgAppFlowPaused)
 }
 
 func TestProgressBecomeProbe(t *testing.T) {