From 49dad987575d8a81be1b7d9b44885c6512ddc38b Mon Sep 17 00:00:00 2001 From: Guangcun Li Date: Tue, 17 Nov 2026 13:38:01 +0800 Subject: [PATCH 1/2] RATIS-2814. Fix replyPendingRequest() ignoring the replacement RetryCache entry --- .../ratis/server/impl/RaftServerImpl.java | 10 +- .../server/impl/LeaderElectionTests.java | 106 ++++++++++++++++++ 2 files changed, 111 insertions(+), 5 deletions(-) diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java index 73b6ce7f15..cb9818e45a 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java @@ -1561,9 +1561,9 @@ private CompletableFuture replyPendingRequest( if (getInfo().isLeader() && cacheEntry.isCompletedNormally()) { LOG.warn("{} retry cache entry of leader should be pending: {}", this, cacheEntry); } - if (cacheEntry.isFailed()) { - retryCache.refreshEntry(new CacheEntry(cacheEntry.getKey())); - } + final CacheEntry entryToComplete = cacheEntry.isFailed() + ? retryCache.refreshEntry(new CacheEntry(cacheEntry.getKey())) + : cacheEntry; return stateMachineFuture.whenComplete((reply, exception) -> { getTransactionManager().remove(termIndex); @@ -1581,9 +1581,9 @@ private CompletableFuture replyPendingRequest( // update pending request final LeaderStateImpl leader = role.getLeaderState().orElse(null); if (leader != null) { - leader.replyPendingRequest(termIndex, r, cacheEntry); + leader.replyPendingRequest(termIndex, r, entryToComplete); } else { - cacheEntry.updateResult(r); + entryToComplete.updateResult(r); } }); } diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java index ae055a59ab..d0b04d6790 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java @@ -24,7 +24,10 @@ import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl; import org.apache.ratis.metrics.impl.DefaultTimekeeperImpl; import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.protocol.ClientInvocationId; import org.apache.ratis.protocol.RaftClientReply; +import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.protocol.RaftGroupId; import org.apache.ratis.protocol.RaftGroupMemberId; import org.apache.ratis.protocol.RaftPeer; @@ -57,6 +60,10 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -71,7 +78,10 @@ import org.apache.ratis.statemachine.StateMachine; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; @@ -281,6 +291,102 @@ public void testTransferLeader() throws Exception { } } + @Test + public void testRetryCacheCompletesAfterLeaderTransfer() throws Exception { + try (final MiniRaftCluster cluster = newCluster(3)) { + cluster.start(); + waitForLeader(cluster); + // Give peer connections time to recover from connection failures during startup. + TimeDuration.valueOf(3, TimeUnit.SECONDS).sleep(); + final RaftServerImpl oldLeader = (RaftServerImpl) waitForLeader(cluster); + + try (RaftClient client = cluster.createClient(oldLeader.getId())) { + final RaftServer.Division target = cluster.getFollowers().get(0); + final RaftClientRequest request = cluster.newRaftClientRequest( + ClientId.randomId(), oldLeader.getId(), 100L, new RaftTestUtil.SimpleMessage("W")); + final ClientInvocationId invocationId = ClientInvocationId.valueOf(request); + + final CountDownLatch reachedAppend = new CountDownLatch(1); + final CountDownLatch resumeAppend = new CountDownLatch(1); + final ExecutorService writer = Executors.newSingleThreadExecutor(); + + CodeInjectionForTesting.put(RaftServerImpl.APPEND_TRANSACTION, (localId, remoteId, args) -> { + // The cache entry exists, but appendTransaction has yet to check the role. + reachedAppend.countDown(); + try { + assertTrue(resumeAppend.await(60, TimeUnit.SECONDS), "Timed out waiting to resume the write"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted while pausing the write", e); + } + return true; + }); + + try { + // Submit a single attempt to the old leader without client-side retry. + final Future> submission = + writer.submit(() -> oldLeader.submitClientRequestAsync(request)); + + assertTrue(reachedAppend.await(10, TimeUnit.SECONDS), "Write did not reach appendTransaction"); + + final RetryCacheImpl.CacheEntry originalCacheEntry = oldLeader.getRetryCache().getIfPresent(invocationId); + assertNotNull(originalCacheEntry, "W should already be in the retry cache"); + assertFalse(originalCacheEntry.getReplyFuture().isDone()); + + assertTrue(client.admin().transferLeadership(target.getId(), oldLeader.getId(), 20000).isSuccess()); + assertEquals(target.getId(), waitForLeader(cluster).getId()); + RaftTestUtil.waitFor( + () -> oldLeader.getInfo().isFollower() && !oldLeader.getRole().getLeaderState().isPresent(), + 10, 5000); + + resumeAppend.countDown(); + + final RaftClientReply reply = submission.get(10, TimeUnit.SECONDS).get(10, TimeUnit.SECONDS); + assertNotNull(reply.getNotLeaderException()); + assertSame(originalCacheEntry, oldLeader.getRetryCache().getIfPresent(invocationId)); + assertTrue(originalCacheEntry.isFailed()); + + final RaftServerImpl newLeader = (RaftServerImpl) target; + final RaftClientRequest retry = cluster.newRaftClientRequest( + request.getClientId(), newLeader.getId(), request.getCallId(), request.getMessage()); + final RaftClientReply committed = newLeader.submitClientRequestAsync(retry) + .get(10, TimeUnit.SECONDS); + assertTrue(committed.isSuccess(), committed::toString); + + final RaftClientRequest marker = cluster.newRaftClientRequest( + request.getClientId(), newLeader.getId(), request.getCallId() + 1, + new RaftTestUtil.SimpleMessage("after W")); + final ClientInvocationId markerId = ClientInvocationId.valueOf(marker); + final RaftClientReply markerReply = newLeader.submitClientRequestAsync(marker) + .get(10, TimeUnit.SECONDS); + assertTrue(markerReply.isSuccess(), markerReply::toString); + + // Wait for the marker's reply to be cached on the old leader. + // With BaseStateMachine, the updater runs W's reply callback before it processes the marker. + RaftTestUtil.waitFor(() -> { + final RetryCacheImpl.CacheEntry markerEntry = oldLeader.getRetryCache().getIfPresent(markerId); + return markerEntry != null && markerEntry.isCompletedNormally(); + }, 10, 10000); + + final RetryCacheImpl.CacheEntry refreshedCacheEntry = oldLeader.getRetryCache().getIfPresent(invocationId); + assertNotNull(refreshedCacheEntry); + assertNotSame(originalCacheEntry, refreshedCacheEntry); + + final RaftClientReply cachedReply = refreshedCacheEntry.getReplyFuture().getNow(null); + assertNotNull(cachedReply, "Applying W should complete the replacement retry-cache entry"); + assertTrue(cachedReply.isSuccess(), cachedReply::toString); + } finally { + resumeAppend.countDown(); + CodeInjectionForTesting.remove(RaftServerImpl.APPEND_TRANSACTION); + writer.shutdown(); + if (!writer.awaitTermination(10, TimeUnit.SECONDS)) { + writer.shutdownNow(); + } + } + } + } + } + @Test public void testRiskyLeadershipTransfer() throws Exception { final RaftProperties p = getProperties(); From b491dc8525df5eb8388eda3e9e0b676fc3bd212e Mon Sep 17 00:00:00 2001 From: Guangcun Li Date: Wed, 18 Nov 2026 20:20:43 +0800 Subject: [PATCH 2/2] RATIS-2814. Simplify the retry cache regression test --- .../org/apache/ratis/RetryCacheTests.java | 31 +++++ .../server/impl/LeaderElectionTests.java | 106 ------------------ .../ratis/server/impl/RetryCacheTestUtil.java | 9 ++ 3 files changed, 40 insertions(+), 106 deletions(-) diff --git a/ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java b/ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java index 0674c3ed94..95aff1a83a 100644 --- a/ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java @@ -17,6 +17,7 @@ */ package org.apache.ratis; +import org.apache.ratis.server.RetryCache; import org.apache.ratis.server.impl.MiniRaftCluster; import org.apache.ratis.server.impl.PeerChanges; import org.apache.ratis.RaftTestUtil.SimpleMessage; @@ -165,4 +166,34 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception { Assertions.assertEquals(0, count(cluster.getLeader().getRaftLog(), oldLastApplied + 1)); } } + + @Test + public void testApplyCompletesReplacementOfFailedEntry() throws Exception { + runWithNewCluster(NUM_SERVERS, this::runTestApplyCompletesReplacementOfFailedEntry); + } + + void runTestApplyCompletesReplacementOfFailedEntry(CLUSTER cluster) throws Exception { + final RaftPeerId leaderId = RaftTestUtil.waitForLeader(cluster).getId(); + final RaftServer.Division follower = cluster.getFollowers().get(0); + + try (final RaftClient client = cluster.createClient(leaderId)) { + final long callId = 999; + final RetryCache.Entry failed = RetryCacheTestUtil.createFailedEntry(follower, client.getId(), + callId, new IllegalStateException("Injected failure")); + + final RaftClientRequest r = cluster.newRaftClientRequest(client.getId(), leaderId, + callId, new SimpleMessage("message")); + assertReply(client.getClientRpc().sendRequest(r), client, callId); + + JavaUtils.attempt(() -> { + final RetryCache.Entry entry = RetryCacheTestUtil.get(follower, client.getId(), callId); + Assertions.assertNotNull(entry, "No retry cache entry on the follower"); + Assertions.assertNotSame(failed, entry); + + final RaftClientReply reply = entry.getReplyFuture().getNow(null); + Assertions.assertNotNull(reply, "The replacement entry is not completed"); + assertReply(reply, client, callId); + }, 20, TimeDuration.valueOf(100, TimeUnit.MILLISECONDS), "wait for the follower to apply", LOG); + } + } } diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java index d0b04d6790..ae055a59ab 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java @@ -24,10 +24,7 @@ import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl; import org.apache.ratis.metrics.impl.DefaultTimekeeperImpl; import org.apache.ratis.proto.RaftProtos; -import org.apache.ratis.protocol.ClientId; -import org.apache.ratis.protocol.ClientInvocationId; import org.apache.ratis.protocol.RaftClientReply; -import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.protocol.RaftGroupId; import org.apache.ratis.protocol.RaftGroupMemberId; import org.apache.ratis.protocol.RaftPeer; @@ -60,10 +57,6 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -78,10 +71,7 @@ import org.apache.ratis.statemachine.StateMachine; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNotSame; -import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; @@ -291,102 +281,6 @@ public void testTransferLeader() throws Exception { } } - @Test - public void testRetryCacheCompletesAfterLeaderTransfer() throws Exception { - try (final MiniRaftCluster cluster = newCluster(3)) { - cluster.start(); - waitForLeader(cluster); - // Give peer connections time to recover from connection failures during startup. - TimeDuration.valueOf(3, TimeUnit.SECONDS).sleep(); - final RaftServerImpl oldLeader = (RaftServerImpl) waitForLeader(cluster); - - try (RaftClient client = cluster.createClient(oldLeader.getId())) { - final RaftServer.Division target = cluster.getFollowers().get(0); - final RaftClientRequest request = cluster.newRaftClientRequest( - ClientId.randomId(), oldLeader.getId(), 100L, new RaftTestUtil.SimpleMessage("W")); - final ClientInvocationId invocationId = ClientInvocationId.valueOf(request); - - final CountDownLatch reachedAppend = new CountDownLatch(1); - final CountDownLatch resumeAppend = new CountDownLatch(1); - final ExecutorService writer = Executors.newSingleThreadExecutor(); - - CodeInjectionForTesting.put(RaftServerImpl.APPEND_TRANSACTION, (localId, remoteId, args) -> { - // The cache entry exists, but appendTransaction has yet to check the role. - reachedAppend.countDown(); - try { - assertTrue(resumeAppend.await(60, TimeUnit.SECONDS), "Timed out waiting to resume the write"); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new AssertionError("Interrupted while pausing the write", e); - } - return true; - }); - - try { - // Submit a single attempt to the old leader without client-side retry. - final Future> submission = - writer.submit(() -> oldLeader.submitClientRequestAsync(request)); - - assertTrue(reachedAppend.await(10, TimeUnit.SECONDS), "Write did not reach appendTransaction"); - - final RetryCacheImpl.CacheEntry originalCacheEntry = oldLeader.getRetryCache().getIfPresent(invocationId); - assertNotNull(originalCacheEntry, "W should already be in the retry cache"); - assertFalse(originalCacheEntry.getReplyFuture().isDone()); - - assertTrue(client.admin().transferLeadership(target.getId(), oldLeader.getId(), 20000).isSuccess()); - assertEquals(target.getId(), waitForLeader(cluster).getId()); - RaftTestUtil.waitFor( - () -> oldLeader.getInfo().isFollower() && !oldLeader.getRole().getLeaderState().isPresent(), - 10, 5000); - - resumeAppend.countDown(); - - final RaftClientReply reply = submission.get(10, TimeUnit.SECONDS).get(10, TimeUnit.SECONDS); - assertNotNull(reply.getNotLeaderException()); - assertSame(originalCacheEntry, oldLeader.getRetryCache().getIfPresent(invocationId)); - assertTrue(originalCacheEntry.isFailed()); - - final RaftServerImpl newLeader = (RaftServerImpl) target; - final RaftClientRequest retry = cluster.newRaftClientRequest( - request.getClientId(), newLeader.getId(), request.getCallId(), request.getMessage()); - final RaftClientReply committed = newLeader.submitClientRequestAsync(retry) - .get(10, TimeUnit.SECONDS); - assertTrue(committed.isSuccess(), committed::toString); - - final RaftClientRequest marker = cluster.newRaftClientRequest( - request.getClientId(), newLeader.getId(), request.getCallId() + 1, - new RaftTestUtil.SimpleMessage("after W")); - final ClientInvocationId markerId = ClientInvocationId.valueOf(marker); - final RaftClientReply markerReply = newLeader.submitClientRequestAsync(marker) - .get(10, TimeUnit.SECONDS); - assertTrue(markerReply.isSuccess(), markerReply::toString); - - // Wait for the marker's reply to be cached on the old leader. - // With BaseStateMachine, the updater runs W's reply callback before it processes the marker. - RaftTestUtil.waitFor(() -> { - final RetryCacheImpl.CacheEntry markerEntry = oldLeader.getRetryCache().getIfPresent(markerId); - return markerEntry != null && markerEntry.isCompletedNormally(); - }, 10, 10000); - - final RetryCacheImpl.CacheEntry refreshedCacheEntry = oldLeader.getRetryCache().getIfPresent(invocationId); - assertNotNull(refreshedCacheEntry); - assertNotSame(originalCacheEntry, refreshedCacheEntry); - - final RaftClientReply cachedReply = refreshedCacheEntry.getReplyFuture().getNow(null); - assertNotNull(cachedReply, "Applying W should complete the replacement retry-cache entry"); - assertTrue(cachedReply.isSuccess(), cachedReply::toString); - } finally { - resumeAppend.countDown(); - CodeInjectionForTesting.remove(RaftServerImpl.APPEND_TRANSACTION); - writer.shutdown(); - if (!writer.awaitTermination(10, TimeUnit.SECONDS)) { - writer.shutdownNow(); - } - } - } - } - } - @Test public void testRiskyLeadershipTransfer() throws Exception { final RaftProperties p = getProperties(); diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/RetryCacheTestUtil.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/RetryCacheTestUtil.java index f90f914179..8e3f2fa4da 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/RetryCacheTestUtil.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/RetryCacheTestUtil.java @@ -56,6 +56,15 @@ public static boolean isFailed(RetryCache.Entry entry) { return ((RetryCacheImpl.CacheEntry)entry).isFailed(); } + public static RetryCache.Entry createFailedEntry(RaftServer.Division server, ClientId clientId, + long callId, Throwable t) { + final RetryCacheImpl cache = (RetryCacheImpl) server.getRetryCache(); + final RetryCacheImpl.CacheEntry entry = + cache.getOrCreateEntry(ClientInvocationId.valueOf(clientId, callId)); + entry.failWithException(t); + return entry; + } + public static void assertFailure(RetryCache cache, LogEntryProto logEntry, boolean isFailed) { if(logEntry.hasStateMachineLogEntry()) { final ClientInvocationId invocationId = ClientInvocationId.valueOf(logEntry.getStateMachineLogEntry());