Skip to content

Commit

Permalink
YARN-11262. Fix CheckStyle & Junit Test.
Browse files Browse the repository at this point in the history
  • Loading branch information
slfan1989 committed Mar 9, 2025
1 parent 2b3c0d9 commit 5aa469d
Show file tree
Hide file tree
Showing 24 changed files with 1,065 additions and 1,034 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.junit.jupiter.api.BeforeEach;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
Expand All @@ -33,7 +34,6 @@
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.junit.Before;

public abstract class ACLsTestBase {

Expand Down Expand Up @@ -65,7 +65,7 @@ public abstract class ACLsTestBase {
YarnRPC rpc;
InetSocketAddress rmAddress;

@Before
@BeforeEach
public void setup() throws InterruptedException, IOException {
conf = createConfiguration();
rpc = YarnRPC.create(conf);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,18 +56,21 @@
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;

import org.apache.hadoop.yarn.util.resource.Resources;

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.mockito.Mockito;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;

/**
* Test class for {@link ActivitiesManager}.
Expand All @@ -92,24 +95,24 @@ public class TestActivitiesManager {

@BeforeEach
public void setup() {
rmContext = Mockito.mock(RMContext.class);
rmContext = mock(RMContext.class);
Configuration conf = new Configuration();
Mockito.when(rmContext.getYarnConfiguration()).thenReturn(conf);
ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
Mockito.when(scheduler.getMinimumResourceCapability())
when(rmContext.getYarnConfiguration()).thenReturn(conf);
ResourceScheduler scheduler = mock(ResourceScheduler.class);
when(scheduler.getMinimumResourceCapability())
.thenReturn(Resources.none());
Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
LeafQueue mockQueue = Mockito.mock(LeafQueue.class);
when(rmContext.getScheduler()).thenReturn(scheduler);
LeafQueue mockQueue = mock(LeafQueue.class);
Map<ApplicationId, RMApp> rmApps = new ConcurrentHashMap<>();
Mockito.doReturn(rmApps).when(rmContext).getRMApps();
doReturn(rmApps).when(rmContext).getRMApps();
apps = new ArrayList<>();
for (int i = 0; i < NUM_APPS; i++) {
ApplicationAttemptId appAttemptId =
TestUtils.getMockApplicationAttemptId(i, 0);
RMApp mockApp = Mockito.mock(RMApp.class);
Mockito.doReturn(appAttemptId.getApplicationId()).when(mockApp)
RMApp mockApp = mock(RMApp.class);
doReturn(appAttemptId.getApplicationId()).when(mockApp)
.getApplicationId();
Mockito.doReturn(FinalApplicationStatus.UNDEFINED).when(mockApp)
doReturn(FinalApplicationStatus.UNDEFINED).when(mockApp)
.getFinalApplicationStatus();
rmApps.put(appAttemptId.getApplicationId(), mockApp);
FiCaSchedulerApp app =
Expand Down Expand Up @@ -164,12 +167,12 @@ public void testRecordingDifferentNodeActivitiesInMultiThreads()
}
// Check activities for all nodes should be recorded and every node should
// have only one allocation information.
Assertions.assertEquals(NUM_NODES,
assertEquals(NUM_NODES,
activitiesManager.historyNodeAllocations.size());
for (List<List<NodeAllocation>> nodeAllocationsForThisNode :
activitiesManager.historyNodeAllocations.values()) {
Assertions.assertEquals(1, nodeAllocationsForThisNode.size());
Assertions.assertEquals(1, nodeAllocationsForThisNode.get(0).size());
assertEquals(1, nodeAllocationsForThisNode.size());
assertEquals(1, nodeAllocationsForThisNode.get(0).size());
}
}

Expand Down Expand Up @@ -208,7 +211,7 @@ public void testRecordingSchedulerActivitiesForMultiNodesInMultiThreads()
future.get();
}
// Check activities for multi-nodes should be recorded only once
Assertions.assertEquals(1, activitiesManager.historyNodeAllocations.size());
assertEquals(1, activitiesManager.historyNodeAllocations.size());
}


Expand Down Expand Up @@ -255,9 +258,9 @@ public void testRecordingAppActivitiesInMultiThreads()
Queue<AppAllocation> appAllocations =
activitiesManager.completedAppAllocations
.get(randomApp.getApplicationId());
Assertions.assertEquals(nTasks, appAllocations.size());
assertEquals(nTasks, appAllocations.size());
for(AppAllocation aa : appAllocations) {
Assertions.assertEquals(NUM_NODES, aa.getAllocationAttempts().size());
assertEquals(NUM_NODES, aa.getAllocationAttempts().size());
}
}

Expand Down Expand Up @@ -297,15 +300,15 @@ public void testAppActivitiesTTL() throws Exception {
AppActivitiesInfo appActivitiesInfo = newActivitiesManager
.getAppActivitiesInfo(app.getApplicationId(), null, null, null, -1,
false, 3);
Assertions.assertEquals(numActivities,
assertEquals(numActivities,
appActivitiesInfo.getAllocations().size());
// sleep until all app activities expired
Thread.sleep(cleanupIntervalMs + appActivitiesTTL);
// there should be no remaining app activities
appActivitiesInfo = newActivitiesManager
.getAppActivitiesInfo(app.getApplicationId(), null, null, null, -1,
false, 3);
Assertions.assertEquals(0,
assertEquals(0,
appActivitiesInfo.getAllocations().size());
}

Expand Down Expand Up @@ -346,12 +349,12 @@ public void testAppActivitiesPerformance() {
AppActivitiesInfo appActivitiesInfo = activitiesManager
.getAppActivitiesInfo(app.getApplicationId(), null, null, null, -1,
false, 100);
Assertions.assertEquals(numActivities,
assertEquals(numActivities,
appActivitiesInfo.getAllocations().size());
Assertions.assertEquals(1,
assertEquals(1,
appActivitiesInfo.getAllocations().get(0).getChildren()
.size());
Assertions.assertEquals(numNodes,
assertEquals(numNodes,
appActivitiesInfo.getAllocations().get(0).getChildren()
.get(0).getChildren().size());
return null;
Expand All @@ -364,15 +367,15 @@ public void testAppActivitiesPerformance() {
AppActivitiesInfo appActivitiesInfo = activitiesManager
.getAppActivitiesInfo(app.getApplicationId(), null, null,
RMWSConsts.ActivitiesGroupBy.DIAGNOSTIC, -1, false, 100);
Assertions.assertEquals(numActivities,
assertEquals(numActivities,
appActivitiesInfo.getAllocations().size());
Assertions.assertEquals(1,
assertEquals(1,
appActivitiesInfo.getAllocations().get(0).getChildren()
.size());
Assertions.assertEquals(1,
assertEquals(1,
appActivitiesInfo.getAllocations().get(0).getChildren()
.get(0).getChildren().size());
Assertions.assertEquals(numNodes,
assertEquals(numNodes,
appActivitiesInfo.getAllocations().get(0).getChildren()
.get(0).getChildren().get(0).getNodeIds().size());
return null;
Expand All @@ -385,14 +388,14 @@ public void testAppActivitiesPerformance() {
AppActivitiesInfo appActivitiesInfo = activitiesManager
.getAppActivitiesInfo(app.getApplicationId(), null, null,
RMWSConsts.ActivitiesGroupBy.DIAGNOSTIC, -1, true, 100);
Assertions.assertEquals(1, appActivitiesInfo.getAllocations().size());
Assertions.assertEquals(1,
assertEquals(1, appActivitiesInfo.getAllocations().size());
assertEquals(1,
appActivitiesInfo.getAllocations().get(0).getChildren()
.size());
Assertions.assertEquals(1,
assertEquals(1,
appActivitiesInfo.getAllocations().get(0).getChildren()
.get(0).getChildren().size());
Assertions.assertEquals(numNodes,
assertEquals(numNodes,
appActivitiesInfo.getAllocations().get(0).getChildren()
.get(0).getChildren().get(0).getNodeIds().size());
return null;
Expand All @@ -417,46 +420,46 @@ public void testAppActivitiesMaxQueueLengthUpdate()
for (int i = 0; i < numNodes; i++) {
mockNodes.put(NodeId.newInstance("node" + i, 0), mock(RMNode.class));
}
CapacityScheduler cs = Mockito.mock(CapacityScheduler.class);
RMContext mockRMContext = Mockito.mock(RMContext.class);
Mockito.when(mockRMContext.getRMNodes()).thenReturn(mockNodes);
Mockito.when(mockRMContext.getYarnConfiguration()).thenReturn(conf);
Mockito.when(mockRMContext.getScheduler()).thenReturn(cs);
CapacityScheduler cs = mock(CapacityScheduler.class);
RMContext mockRMContext = mock(RMContext.class);
when(mockRMContext.getRMNodes()).thenReturn(mockNodes);
when(mockRMContext.getYarnConfiguration()).thenReturn(conf);
when(mockRMContext.getScheduler()).thenReturn(cs);
/*
* Test for async-scheduling with multi-node placement disabled
*/
Mockito.when(cs.isMultiNodePlacementEnabled()).thenReturn(false);
when(cs.isMultiNodePlacementEnabled()).thenReturn(false);
int numAsyncSchedulerThreads = 3;
Mockito.when(cs.getNumAsyncSchedulerThreads())
when(cs.getNumAsyncSchedulerThreads())
.thenReturn(numAsyncSchedulerThreads);
ActivitiesManager newActivitiesManager =
new ActivitiesManager(mockRMContext);
Assertions.assertEquals(1,
assertEquals(1,
newActivitiesManager.getAppActivitiesMaxQueueLength());
newActivitiesManager.init(conf);
newActivitiesManager.start();
GenericTestUtils.waitFor(
() -> newActivitiesManager.getAppActivitiesMaxQueueLength()
== numNodes * numAsyncSchedulerThreads, 100, 3000);
Assertions.assertEquals(15,
assertEquals(15,
newActivitiesManager.getAppActivitiesMaxQueueLength());
/*
* Test for HB-driven scheduling with multi-node placement disabled
*/
Mockito.when(cs.getNumAsyncSchedulerThreads()).thenReturn(0);
when(cs.getNumAsyncSchedulerThreads()).thenReturn(0);
GenericTestUtils.waitFor(
() -> newActivitiesManager.getAppActivitiesMaxQueueLength()
== numNodes * 1.2, 100, 3000);
Assertions.assertEquals(6,
assertEquals(6,
newActivitiesManager.getAppActivitiesMaxQueueLength());
/*
* Test for scheduling with multi-node placement enabled
*/
Mockito.when(cs.isMultiNodePlacementEnabled()).thenReturn(true);
when(cs.isMultiNodePlacementEnabled()).thenReturn(true);
GenericTestUtils.waitFor(
() -> newActivitiesManager.getAppActivitiesMaxQueueLength()
== configuredAppActivitiesMaxQueueLength, 100, 3000);
Assertions.assertEquals(1,
assertEquals(1,
newActivitiesManager.getAppActivitiesMaxQueueLength());
}

Expand Down Expand Up @@ -484,8 +487,8 @@ public class TestingActivitiesManager extends ActivitiesManager {

public TestingActivitiesManager(RMContext rmContext) {
super(rmContext);
super.completedNodeAllocations = Mockito.spy(new ConcurrentHashMap<>());
Mockito.doAnswer((invocationOnMock) -> {
super.completedNodeAllocations = spy(new ConcurrentHashMap<>());
doAnswer((invocationOnMock) -> {
NodeId nodeId = (NodeId) invocationOnMock.getArguments()[0];
List<NodeAllocation> nodeAllocations =
(List<NodeAllocation>) invocationOnMock.getArguments()[1];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,10 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.util.Clock;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

Expand Down Expand Up @@ -106,7 +107,7 @@ public void waitNumberOfLiveContainersFromApp(FiCaSchedulerApp app,
waitNum++;
}

Assertions.fail();
fail();
}

public void waitNumberOfReservedContainersFromApp(FiCaSchedulerApp app,
Expand All @@ -122,7 +123,7 @@ public void waitNumberOfReservedContainersFromApp(FiCaSchedulerApp app,
waitNum++;
}

Assertions.fail();
fail();
}

public void waitNumberOfLiveContainersOnNodeFromApp(FiCaSchedulerNode node,
Expand All @@ -144,7 +145,7 @@ public void waitNumberOfLiveContainersOnNodeFromApp(FiCaSchedulerNode node,
waitNum++;
}

Assertions.fail(
fail(
"Check #live-container-on-node-from-app, actual=" + total + " expected="
+ expected);
}
Expand All @@ -160,6 +161,6 @@ public void checkNumberOfPreemptionCandidateFromApp(
}
}

Assertions.assertEquals(expected, total);
assertEquals(expected, total);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,8 @@
import java.util.List;
import java.util.Map;

import org.junit.jupiter.api.Assertions;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;

public final class CapacitySchedulerQueueHelpers {

Expand Down Expand Up @@ -386,7 +385,7 @@ public static void checkQueueStructureCapacities(CapacityScheduler cs,
CSQueue rootQueue = cs.getRootQueue();
for (Map.Entry<String, ExpectedCapacities> entry : capacities.entrySet()) {
CSQueue queue = findQueue(rootQueue, entry.getKey());
Assertions.assertNotNull(queue);
assertNotNull(queue);
assertQueueCapacities(queue, entry.getValue());
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,14 @@
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.jupiter.api.Assertions;

import java.io.IOException;
import java.util.Set;

import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfAmbiguousQueue;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

Expand All @@ -84,7 +85,7 @@ public static void checkPendingResource(MockRM rm, String queueName, int memory,
String label) {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
CSQueue queue = cs.getQueue(queueName);
Assertions.assertEquals(
assertEquals(
memory,
queue.getQueueResourceUsage()
.getPending(label == null ? RMNodeLabelsManager.NO_LABEL : label)
Expand All @@ -96,7 +97,7 @@ public static void checkPendingResourceGreaterThanZero(MockRM rm, String queueNa
String label) {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
CSQueue queue = cs.getQueue(queueName);
Assertions.assertTrue(queue.getQueueResourceUsage()
assertTrue(queue.getQueueResourceUsage()
.getPending(label == null ? RMNodeLabelsManager.NO_LABEL : label)
.getMemorySize() > 0);
}
Expand Down Expand Up @@ -230,11 +231,11 @@ public static NodeManager registerNode(ResourceManager rm, String hostName,
}

public static void checkApplicationResourceUsage(int expected, Application application) {
Assertions.assertEquals(expected, application.getUsedResources().getMemorySize());
assertEquals(expected, application.getUsedResources().getMemorySize());
}

public static void checkNodeResourceUsage(int expected, NodeManager node) {
Assertions.assertEquals(expected, node.getUsed().getMemorySize());
assertEquals(expected, node.getUsed().getMemorySize());
node.checkResourceUsage();
}
}
Loading

0 comments on commit 5aa469d

Please sign in to comment.