[#1805] feat(spark): Support Spark 4 in client-spark/extension

LuciferYang · LuciferYang · commit aedf48a26ee4 · 2026-05-13T07:58:08.000+08:00
Spark 4 switched the UI servlet API from javax.servlet (Jetty 9) to
jakarta.servlet (Jetty 11+). WebUIPage.render must override its parent
signature exactly, so the same .scala file cannot serve both.

Route extension through a ServletCompat type alias backed by two
version-specific source roots (scala-javax / scala-jakarta). The
build-helper-maven-plugin picks one via ${extension.servlet.source.dir},
switched by a spark4 profile in this module's pom that merges with the
root pom's spark4 profile.

Other changes:
- Add client-spark/extension to the root pom's spark4 profile modules.
- Add rss-client-spark-ui dependency to client-spark/spark4/pom.xml,
  mirroring client-spark/spark3/pom.xml.
- Bump build-helper-maven-plugin 1.10 -&gt; 3.6.0.
- Contributor notes document new-variant and new-spark-version steps,
  IDE re-import on profile switch, and ~/.m2 hygiene.

Verified: extension compiles cleanly under spark3 / spark3.0 / spark3.2 /
spark3.2.0 / spark3.3 / spark3.4 / spark3.5 / spark4 profiles.
diff --git a/client-spark/extension/pom.xml b/client-spark/extension/pom.xml
@@ -33,6 +33,56 @@
     <packaging>jar</packaging>
     <name>Apache Uniffle Client spark ui</name>
 
+    <!--
+        Servlet API compatibility across Spark versions.
+
+        Spark 3.x and earlier depend on javax.servlet (Jetty 9 era).
+        Spark 4.x switched to jakarta.servlet (Jetty 11+ era).
+        These APIs cannot coexist in one .scala file because Spark's
+        WebUIPage.render(request) must override the parent signature exactly.
+
+        Strategy: a pair of version-specific source roots, each exposing the
+        same alias `org.apache.spark.ui.ServletCompat.HttpServletRequest`
+        pointing to the appropriate underlying class. The active profile
+        selects one via ${extension.servlet.source.dir}. The javax variant is
+        the module-level default; profiles override it when needed.
+
+        To add support for a new Spark major version:
+          1. If it reuses an existing servlet variant (javax or jakarta),
+             add the new spark profile to the root pom.xml, then add a
+             matching <profile> with the same id here in this module's pom
+             whose <properties> sets ${extension.servlet.source.dir} to the
+             matching scala-* directory. Maven merges same-id profiles, so
+             both activate together under -P<new-id>. The javax variant is
+             already the default, so reusing it needs no profile override
+             (for example, all existing spark3.* profiles in the root pom
+             use the javax default without a matching profile in this pom).
+          2. If it introduces a new servlet variant, create a new source root
+             `src/main/scala-<variant>` with a mirrored ServletCompat (same
+             aliases, different underlying types), then follow step 1.
+          3. Keep every scala-* variant of ServletCompat.scala in sync: every
+             alias must exist in every variant; only the underlying package
+             may differ.
+
+        Notes for contributors:
+          * When an IDE (IntelliJ IDEA, VSCode) is told to activate a
+            different spark profile, re-import the project so the selected
+            scala-* source root is re-resolved.
+          * `rss-client-spark-ui` is published under a single Maven coordinate
+            regardless of profile, but different profiles produce incompatible
+            bytecode (servlet variant and Scala binary version both differ).
+            Do not leave jars from different profiles mixed in your local
+            `~/.m2` cache; run `mvn clean` when switching profiles. Publishing
+            more than one profile's build to the same coordinate in a remote
+            repository is not supported by this module; artifact partitioning
+            (classifier or profile-specific artifactId) is out of scope here.
+    -->
+    <properties>
+        <!-- Default: javax.servlet variant (Spark 2.x / 3.x).
+             Overridden by the local `spark4` profile below. -->
+        <extension.servlet.source.dir>src/main/scala-javax</extension.servlet.source.dir>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>org.apache.uniffle</groupId>
@@ -72,6 +122,24 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>add-servlet-compat-source</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${extension.servlet.source.dir}</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
@@ -90,4 +158,20 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <!--
+                This profile shares its id with the root pom's `spark4` profile.
+                Maven merges same-id profiles across the reactor, so activating
+                `-Pspark4` triggers both. This module only contributes the
+                source-dir override so that Spark 4 builds pick up the
+                jakarta.servlet variant of ServletCompat.
+            -->
+            <id>spark4</id>
+            <properties>
+                <extension.servlet.source.dir>src/main/scala-jakarta</extension.servlet.source.dir>
+            </properties>
+        </profile>
+    </profiles>
 </project>
diff --git a/client-spark/extension/src/main/scala-jakarta/org/apache/spark/ui/ServletCompat.scala b/client-spark/extension/src/main/scala-jakarta/org/apache/spark/ui/ServletCompat.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+// Servlet type aliases for Spark 4.x (jakarta.servlet variant).
+// Selected by build-helper-maven-plugin via ${extension.servlet.source.dir}
+// when the spark4 profile is active; see client-spark/extension/pom.xml.
+//
+// Usage: `import ServletCompat.HttpServletRequest`, then reference the bare
+// `HttpServletRequest` symbol. Callers must reside in package
+// `org.apache.spark.ui` because the object is `private[ui]`.
+//
+// IMPORTANT: this file must stay in sync with every other ServletCompat.scala
+// under src/main/scala-*. Every alias must exist in every variant; only the
+// underlying servlet package may differ. When adding or renaming an alias
+// here, make the same change in every other variant.
+private[ui] object ServletCompat {
+  type HttpServletRequest = jakarta.servlet.http.HttpServletRequest
+}
diff --git a/client-spark/extension/src/main/scala-javax/org/apache/spark/ui/ServletCompat.scala b/client-spark/extension/src/main/scala-javax/org/apache/spark/ui/ServletCompat.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+// Servlet type aliases for Spark 2.x / 3.x (javax.servlet variant).
+// Selected by build-helper-maven-plugin via ${extension.servlet.source.dir}
+// when the spark4 profile is not active; see client-spark/extension/pom.xml.
+//
+// Usage: `import ServletCompat.HttpServletRequest`, then reference the bare
+// `HttpServletRequest` symbol. Callers must reside in package
+// `org.apache.spark.ui` because the object is `private[ui]`.
+//
+// IMPORTANT: this file must stay in sync with every other ServletCompat.scala
+// under src/main/scala-*. Every alias must exist in every variant; only the
+// underlying servlet package may differ. When adding or renaming an alias
+// here, make the same change in every other variant.
+private[ui] object ServletCompat {
+  type HttpServletRequest = javax.servlet.http.HttpServletRequest
+}
diff --git a/client-spark/extension/src/main/scala/org/apache/spark/ui/ShufflePage.scala b/client-spark/extension/src/main/scala/org/apache/spark/ui/ShufflePage.scala
@@ -18,11 +18,11 @@
 package org.apache.spark.ui
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.ui.ServletCompat.HttpServletRequest
 import org.apache.spark.util.Utils
 import org.apache.spark.{AggregatedShuffleMetric, AggregatedShuffleReadMetric, AggregatedShuffleWriteMetric, AggregatedTaskInfoUIData, ShuffleType}
 
 import java.util.concurrent.ConcurrentHashMap
-import javax.servlet.http.HttpServletRequest
 import scala.collection.JavaConverters.{collectionAsScalaIterableConverter, mapAsScalaMapConverter}
 import scala.xml.{Node, NodeSeq}
 
diff --git a/client-spark/spark4/pom.xml b/client-spark/spark4/pom.xml
@@ -59,6 +59,11 @@
             <artifactId>rss-client-spark-common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.uniffle</groupId>
+            <artifactId>rss-client-spark-ui</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-all</artifactId>
diff --git a/pom.xml b/pom.xml
@@ -945,7 +945,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>build-helper-maven-plugin</artifactId>
-          <version>1.10</version>
+          <version>3.6.0</version>
         </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
@@ -2008,6 +2008,7 @@
         <module>client-spark/spark4-shaded</module>
         <module>integration-test/spark-common</module>
         <module>integration-test/spark4</module>
+        <module>client-spark/extension</module>
       </modules>
       <dependencyManagement>
         <dependencies>
