SOLR-17413: ulog replay should copy SolrQueryRequest (#2765)

SolrQueryRequest is a non-threadsafe type, but was being shared across
executor threads during UpdateLog replay.  This introduces a number of
issues, not the least being a ConcurrentModificationException if
multiple threads happen to tweak the request 'context' simultaneously.

This commit fixes this issue by giving each executor thread a unique
LocalSolrQueryRequest instance to use.

Author: gerlowskija

solr/CHANGES.txt

@@ -81,6 +81,8 @@ Bug Fixes
 
 * SOLR-17464: Fixed Http2SolrClient bug in that 'requestAsync' triggered NPE when using a shared Jetty client (Jason Gerlowski, James Dyer)
 
+* SOLR-17413: Fixed UpdateLog replay bug that shared thread-unsafe SolrQueryRequest objects across threads (Jason Gerlowski, David Smiley, Houston Putman)
+
 Dependency Upgrades
 ---------------------
 (No changes)

solr/core/src/java/org/apache/solr/update/UpdateLog.java

@@ -63,6 +63,7 @@
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.MapSolrParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.CollectionUtil;
@@ -1998,6 +1999,14 @@ public String toString() {
   protected RecoveryInfo recoveryInfo;
 
   class LogReplayer implements Runnable {
+    private final SolrParams BASE_REPLAY_PARAMS =
+        new MapSolrParams(
+            Map.of(
+                DISTRIB_UPDATE_PARAM,
+                FROMLEADER.toString(),
+                DistributedUpdateProcessor.LOG_REPLAY,
+                "true"));
+
     private Logger loglog = log; // set to something different?
 
     Deque<TransactionLog> translogs;
@@ -2024,10 +2033,7 @@ public LogReplayer(List<TransactionLog> translogs, boolean activeLog, boolean in
 
     @Override
     public void run() {
-      ModifiableSolrParams params = new ModifiableSolrParams();
-      params.set(DISTRIB_UPDATE_PARAM, FROMLEADER.toString());
-      params.set(DistributedUpdateProcessor.LOG_REPLAY, "true");
-      req = new LocalSolrQueryRequest(uhandler.core, params);
+      req = new LocalSolrQueryRequest(uhandler.core, BASE_REPLAY_PARAMS);
       rsp = new SolrQueryResponse();
       // setting request info will help logging
       SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
@@ -2100,7 +2106,10 @@ public void doReplay(TransactionLog translog) {
         ThreadLocal<UpdateRequestProcessor> procThreadLocal =
             ThreadLocal.withInitial(
                 () -> {
-                  var proc = processorChain.createProcessor(req, rsp);
+                  // SolrQueryRequest is not thread-safe, so use a copy when creating URPs
+                  final var localRequest =
+                      new LocalSolrQueryRequest(uhandler.core, BASE_REPLAY_PARAMS);
+                  var proc = processorChain.createProcessor(localRequest, rsp);
                   procPool.add(proc);
                   return proc;
                 });