Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

YARN-11786. Upgrade hadoop-yarn-server-timelineservice-hbase-tests to Support Trunk Compilation and Remove compatible hadoop version. #7453

Open
wants to merge 6 commits into
base: trunk
Choose a base branch
from
+991 −296
Open

YARN-11786. Upgrade hadoop-yarn-server-timelineservice-hbase-tests to Support Trunk Compilation and Remove compatible hadoop version. #7453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c264409
REMOVE-COMPATIBLE-HADOOP-VERSION
slfan1989 Mar 2, 2025
54f79ce
fix junit test.
slfan1989 Mar 4, 2025
07e1538
fix junit test.
slfan1989 Mar 4, 2025
6049652
YARN-11786. Improve Some Code.
slfan1989 Mar 5, 2025
75fc637
YARN-11786. Upgrade hadoop-yarn-server-timelineservice-hbase-tests to…
slfan1989 Mar 6, 2025
6c24c0c
Merge branch 'apache:trunk' into remove-compatible-hadoop-version
slfan1989 Mar 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion BUILDING.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ Maven build goals:
YARN Application Timeline Service V2 build options:

YARN Timeline Service v.2 chooses Apache HBase as the primary backing storage. The supported
version of Apache HBase is 2.5.8.
version of Apache HBase is 2.6.1.

Snappy build options:

Expand Down
2 changes: 1 addition & 1 deletion hadoop-project/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>2.0</snakeyaml.version>
<sshd.version>2.11.0</sshd.version>
<hbase.version>2.5.8-hadoop3</hbase.version>
<hbase.version>2.6.1-hadoop3</hbase.version>
<junit.version>4.13.2</junit.version>
<junit.jupiter.version>5.8.2</junit.jupiter.version>
<junit.vintage.version>5.8.2</junit.vintage.version>
Expand Down
4 changes: 3 additions & 1 deletion ...n-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import javax.xml.bind.annotation.XmlElement;

import com.fasterxml.jackson.annotation.JsonInclude;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand All @@ -27,6 +28,7 @@
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
@JsonInclude(JsonInclude.Include.NON_NULL)
public class FlowRunEntity extends HierarchicalTimelineEntity {
public static final String USER_INFO_KEY =
TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "USER";
Expand Down Expand Up @@ -107,7 +109,7 @@ public void setRunId(long runId) {
addInfo(FLOW_RUN_ID_INFO_KEY, runId);
}

public long getStartTime() {
public Long getStartTime() {
return getCreatedTime();
}

Expand Down
4 changes: 3 additions & 1 deletion ...va/org/apache/hadoop/yarn/api/records/timelineservice/writer/TimelineEntitySetWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,13 @@
public class TimelineEntitySetWriter implements MessageBodyWriter<Set<TimelineEntity>> {

private ObjectMapper objectMapper = new ObjectMapper();
private String timelineEntityType =
"java.util.Set<org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity>";

@Override
public boolean isWriteable(Class<?> type, Type genericType,
Annotation[] annotations, MediaType mediaType) {
return true;
return timelineEntityType.equals(genericType.getTypeName());
}

@Override
Expand Down
37 changes: 16 additions & 21 deletions ...ect/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hbase-compatible-hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
Expand All @@ -104,10 +103,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand All @@ -131,13 +126,22 @@
<artifactId>junit-platform-launcher</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-server</artifactId>
<scope>test</scope>
</dependency>

<!-- 'mvn dependency:analyze' fails to detect use of this direct
dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>${hbase-compatible-hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
Expand Down Expand Up @@ -242,13 +246,6 @@
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>1.19.4</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>javax.ws.rs</groupId>
<artifactId>jsr311-api</artifactId>
Expand Down Expand Up @@ -369,18 +366,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hbase-compatible-hadoop.version}</version>
<type>test-jar</type>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
Expand All @@ -393,7 +385,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hbase-compatible-hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
Expand All @@ -412,7 +403,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hbase-compatible-hadoop.version}</version>
<type>test-jar</type>
<scope>test</scope>
<exclusions>
Expand All @@ -430,7 +420,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
<version>${hbase-compatible-hadoop.version}</version>
<scope>test</scope>
</dependency>

Expand Down Expand Up @@ -507,6 +496,12 @@
<artifactId>mockito-inline</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jettison</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
81 changes: 41 additions & 40 deletions ...apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,32 +22,29 @@
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;

import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.util.List;

import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;

import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.reader.TimelineEntityReader;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.HttpUrlConnectorProvider;
import org.junit.Assert;

import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;

/**
* Test Base for TimelineReaderServer HBase tests.
*/
Expand Down Expand Up @@ -109,19 +106,25 @@ protected void addFilters(Configuration conf) {
}

protected Client createClient() {
ClientConfig cfg = new DefaultClientConfig();
cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
return new Client(
new URLConnectionClientHandler(new DummyURLConnectionFactory()), cfg);
final ClientConfig cc = new ClientConfig();
cc.connectorProvider(getHttpURLConnectionFactory());
return ClientBuilder.newClient(cc)
.register(TimelineEntityReader.class)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not familiar with differences in the new Jersey version, but it seems like the prior code was much more driven by introspection and annotations instead of needing to register a lot of custom readers. Is there any way to make it more like the prior code? I'm concerned about risk of backward-incompatible changes in the manual approach and potential error-prone evolution of the code if fields are changed in the future.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a good question. I believe we need to update some parts of the code to align with Jersey 2's standards, as certain parts of the code are not well-supported by the latest standards.

I first became aware of the issue with Jersey during the release of Hadoop 3.4.0, when aajisaka mentioned that, with Hadoop supporting JDK 11, there was an area that needed optimization — Jersey support. For more details, you can refer to HADOOP-15984 (updating Jersey from 1.19 to 2.x).

Jersey is a framework that supports RESTful WebService. When upgrading from version 1.x to 2.x, compatibility issues arise because these two versions are completely incompatible, with even basic methods being different.

To address this, we define custom readers because Jersey 2.x only supports automatic JSON conversion for basic data types and does not support automatic conversion for more complex types. More details can be found in the official documentation.

While writing these readers, I used com.fasterxml.jackson.databind.ObjectMapper to implement automatic conversion between objects and JSON. However, ObjectMapper cannot handle null pointer exceptions, which leads to the first question — the need to assign initial values to certain variables.

After upgrading to JUnit 5, we will configure a new JDK 17 CI pipeline. Following that, I will begin the deeper adaptation and upgrade of Jersey 2.

Copy link
Contributor

@cnauroth cnauroth Mar 4, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@slfan1989 , thank you for the reply. This is helpful information.

It sounds like some of this is just unavoidable with this Jersey upgrade. The biggest thing I'm concerned about is the potential to break YARN's user-facing APIs. Please correct me if I'm wrong, but I think we want to maintain full API compatibility in the 3.5.0 release. If so, how can we be confident about this? Maybe we need to spin up some kind of test of a 3.4 client against a 3.5 server?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cnauroth Thank you for your message. We need to ensure the compatibility of the interfaces. For the YARN REST API, most interfaces have no compatibility issues. Even after upgrading to Jersey 2, the changes to the server's WebService are minimal, usually just adding readers and writers. While there are still a few interfaces that need re-validation, I have a general understanding of the areas that need to be checked. We can work together to complete this task, which must be done before the release of 3.5.0.

.register(TimelineEntitySetReader.class)
.register(TimelineEntityListReader.class)
.register(FlowActivityEntityReader.class)
.register(FlowRunEntityReader.class)
.register(FlowActivityEntitySetReader.class)
.register(FlowActivityEntityListReader.class)
.register(FlowRunEntitySetReader.class);
}

protected ClientResponse getResponse(Client client, URI uri)
protected Response getResponse(Client client, URI uri)
throws Exception {
ClientResponse resp =
client.resource(uri).accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
Response resp =
client.target(uri).request(MediaType.APPLICATION_JSON).get();
if (resp == null || resp.getStatusInfo()
.getStatusCode() != ClientResponse.Status.OK.getStatusCode()) {
.getStatusCode() != HttpURLConnection.HTTP_OK) {
String msg = "";
if (resp != null) {
msg = String.valueOf(resp.getStatusInfo().getStatusCode());
Expand All @@ -132,39 +135,37 @@ protected ClientResponse getResponse(Client client, URI uri)
return resp;
}

protected void verifyHttpResponse(Client client, URI uri, Status status) {
ClientResponse resp =
client.resource(uri).accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
protected void verifyHttpResponse(Client client, URI uri, Response.Status status) {
Response resp = client.target(uri).request(MediaType.APPLICATION_JSON).get();
assertNotNull(resp);
assertTrue("Response from server should have been " + status,
resp.getStatusInfo().getStatusCode() == status.getStatusCode());
System.out.println("Response is: " + resp.getEntity(String.class));
System.out.println("Response is: " + resp.readEntity(String.class));
}

protected List<FlowActivityEntity> verifyFlowEntites(Client client, URI uri,
int noOfEntities) throws Exception {
ClientResponse resp = getResponse(client, uri);
Response resp = getResponse(client, uri);
List<FlowActivityEntity> entities =
resp.getEntity(new GenericType<List<FlowActivityEntity>>() {
resp.readEntity(new GenericType<List<FlowActivityEntity>>() {
});
assertNotNull(entities);
assertEquals(noOfEntities, entities.size());
return entities;
}

protected static class DummyURLConnectionFactory
implements HttpURLConnectionFactory {

@Override
public HttpURLConnection getHttpURLConnection(final URL url)
throws IOException {
try {
return (HttpURLConnection) url.openConnection();
} catch (UndeclaredThrowableException e) {
throw new IOException(e.getCause());
}
}
@VisibleForTesting
protected HttpUrlConnectorProvider getHttpURLConnectionFactory() {
return new HttpUrlConnectorProvider().connectionFactory(
url -> {
HttpURLConnection conn;
try {
conn = (HttpURLConnection) url.openConnection();
} catch (Exception e) {
throw new IOException(e);
}
return conn;
});
}

protected static HBaseTestingUtility getHBaseTestingUtility() {
Expand Down
Loading