[#1805] feat(spark): Support Spark 4 in client-spark/extension (#2749)

Spark 4 switched the UI servlet API from javax.servlet (Jetty 9) to
jakarta.servlet (Jetty 11+). WebUIPage.render must override its parent
signature exactly, so the same .scala file cannot serve both.

Route extension through a ServletCompat type alias backed by two
version-specific source roots (scala-javax / scala-jakarta). The
build-helper-maven-plugin picks one via ${extension.servlet.source.dir},
switched by a spark4 profile in this module's pom that merges with the
root pom's spark4 profile.

Other changes:
- Add client-spark/extension to the root pom's spark4 profile modules.
- Add rss-client-spark-ui dependency to client-spark/spark4/pom.xml,
  mirroring client-spark/spark3/pom.xml.
- Bump build-helper-maven-plugin 1.10 -> 3.6.0.
- Contributor notes document new-variant and new-spark-version steps,
  IDE re-import on profile switch, and ~/.m2 hygiene.

Verified: extension compiles cleanly under spark3 / spark3.0 / spark3.2 /
spark3.2.0 / spark3.3 / spark3.4 / spark3.5 / spark4 profiles.

Author: LuciferYang

client-spark/extension/pom.xml

@@ -33,6 +33,31 @@
     <packaging>jar</packaging>
     <name>Apache Uniffle Client spark ui</name>
 
+    <!--
+        WebUIPage.render's request parameter uses javax.servlet in Spark 3
+        and jakarta.servlet in Spark 4. A single ShufflePage.scala cannot
+        override both signatures, so the module ships two source roots
+        (src/main/scala-javax and src/main/scala-jakarta), each with its
+        own ServletCompat aliasing HttpServletRequest to the matching
+        package. build-helper-maven-plugin picks one via
+        ${extension.servlet.source.dir}; the javax variant is the default
+        and the spark4 profile at the bottom of this pom overrides it.
+
+        To add another servlet flavor, create src/main/scala-<flavor>/
+        with a mirrored ServletCompat (same aliases), then point a new
+        profile at it. Keep aliases in lock-step across every variant.
+
+        rss-client-spark-ui is published under a single Maven coordinate,
+        but different profiles produce incompatible bytecode (servlet +
+        Scala binary). Run `mvn clean` between profiles locally; shipping
+        multiple profile builds to the same remote coordinate is out of
+        scope for this module.
+    -->
+    <properties>
+        <!-- Overridden by the spark4 profile below. -->
+        <extension.servlet.source.dir>src/main/scala-javax</extension.servlet.source.dir>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>org.apache.uniffle</groupId>
@@ -72,6 +97,24 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>add-servlet-compat-source</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${extension.servlet.source.dir}</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
@@ -90,4 +133,14 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <!-- Merged with the root pom's spark4 profile by shared id. -->
+            <id>spark4</id>
+            <properties>
+                <extension.servlet.source.dir>src/main/scala-jakarta</extension.servlet.source.dir>
+            </properties>
+        </profile>
+    </profiles>
 </project>

client-spark/extension/src/main/scala-jakarta/org/apache/spark/ui/ServletCompat.scala

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+// jakarta.servlet variant of ServletCompat, picked up by the spark4
+// profile (see client-spark/extension/pom.xml). Every alias here must
+// also exist in the scala-javax/ variant — only the underlying servlet
+// package may differ.
+private[ui] object ServletCompat {
+  type HttpServletRequest = jakarta.servlet.http.HttpServletRequest
+}

client-spark/extension/src/main/scala-javax/org/apache/spark/ui/ServletCompat.scala

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+// javax.servlet variant of ServletCompat, picked up by the default
+// ${extension.servlet.source.dir} (see client-spark/extension/pom.xml).
+// Every alias here must also exist in the scala-jakarta/ variant —
+// only the underlying servlet package may differ.
+private[ui] object ServletCompat {
+  type HttpServletRequest = javax.servlet.http.HttpServletRequest
+}

client-spark/extension/src/main/scala/org/apache/spark/ui/ShufflePage.scala

@@ -18,11 +18,11 @@
 package org.apache.spark.ui
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.ui.ServletCompat.HttpServletRequest
 import org.apache.spark.util.Utils
 import org.apache.spark.{AggregatedShuffleMetric, AggregatedShuffleReadMetric, AggregatedShuffleWriteMetric, AggregatedTaskInfoUIData, ShuffleType}
 
 import java.util.concurrent.ConcurrentHashMap
-import javax.servlet.http.HttpServletRequest
 import scala.collection.JavaConverters.{collectionAsScalaIterableConverter, mapAsScalaMapConverter}
 import scala.xml.{Node, NodeSeq}
 

client-spark/spark4/pom.xml

@@ -59,6 +59,11 @@
             <artifactId>rss-client-spark-common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.uniffle</groupId>
+            <artifactId>rss-client-spark-ui</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-all</artifactId>

pom.xml

@@ -945,7 +945,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>build-helper-maven-plugin</artifactId>
-          <version>1.10</version>
+          <version>3.6.0</version>
         </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
@@ -2008,6 +2008,7 @@
         <module>client-spark/spark4-shaded</module>
         <module>integration-test/spark-common</module>
         <module>integration-test/spark4</module>
+        <module>client-spark/extension</module>
       </modules>
       <dependencyManagement>
         <dependencies>