Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HDDS-11347. Add rocks_tools_native lib check in Ozone CLI checknative subcommand #7101

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

HDDS-11347. Add rocks_tools_native lib check in Ozone CLI checknative subcommand #7101

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6c20423
HDDS-11347. Add rocks_tools_native lib check in Ozone CLI checknative…
smengcl Aug 20, 2024
217a79a
Check new output line in TestCheckNative.
smengcl Aug 20, 2024
bc9c74b
Add test case when rocks-tools is actually loaded.
smengcl Aug 23, 2024
056db62
loadLibrary
smengcl Aug 23, 2024
d26277b
Use `NativeLibraryLoader.getInstance().loadLibrary()`.
smengcl Sep 5, 2024
55a1e8c
Merge remote-tracking branch 'asf/master' into HDDS-11347-checknative
smengcl Sep 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;

import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;

/**
* Class to load Native Libraries.
*/
Expand Down Expand Up @@ -67,6 +69,10 @@ public static NativeLibraryLoader getInstance() {
return instance;
}

public static String getJniLibraryFileName() {
return appendLibOsSuffix("lib" + ROCKS_TOOLS_NATIVE_LIBRARY_NAME);
}

public static String getJniLibraryFileName(String libraryName) {
return appendLibOsSuffix("lib" + libraryName);
}
Expand Down Expand Up @@ -99,9 +105,12 @@ private static String appendLibOsSuffix(String libraryFileName) {
return libraryFileName + getLibOsSuffix();
}

public static boolean isLibraryLoaded() {
return isLibraryLoaded(ROCKS_TOOLS_NATIVE_LIBRARY_NAME);
}

public static boolean isLibraryLoaded(final String libraryName) {
return getInstance().librariesLoaded
.getOrDefault(libraryName, false);
return getInstance().librariesLoaded.getOrDefault(libraryName, false);
}

public synchronized boolean loadLibrary(final String libraryName, final List<String> dependentFiles) {
Expand Down
27 changes: 23 additions & 4 deletions hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@
package org.apache.hadoop.ozone.shell.checknative;

import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.apache.hadoop.util.NativeCodeLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine;

/**
Expand All @@ -30,18 +33,21 @@
description = "Checks if native libraries are loaded")
public class CheckNative extends GenericCli {

private static final Logger LOG =
LoggerFactory.getLogger(CheckNative.class);

public static void main(String[] argv) {
new CheckNative().run(argv);
}

@Override
public Void call() throws Exception {
boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
boolean nativeHadoopLoaded = org.apache.hadoop.util.NativeCodeLoader.isNativeCodeLoaded();
String hadoopLibraryName = "";
String isalDetail = "";
boolean isalLoaded = false;
if (nativeHadoopLoaded) {
hadoopLibraryName = NativeCodeLoader.getLibraryName();
hadoopLibraryName = org.apache.hadoop.util.NativeCodeLoader.getLibraryName();

isalDetail = ErasureCodeNative.getLoadingFailureReason();
if (isalDetail != null) {
Expand All @@ -50,12 +56,25 @@ public Void call() throws Exception {
isalDetail = ErasureCodeNative.getLibraryName();
isalLoaded = true;
}

}
System.out.println("Native library checking:");
System.out.printf("hadoop: %b %s%n", nativeHadoopLoaded,
hadoopLibraryName);
System.out.printf("ISA-L: %b %s%n", isalLoaded, isalDetail);

// Attempt to load the rocks tools lib
try {
ManagedRawSSTFileReader.loadLibrary();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we use NativeLibraryLoader.load(NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME) instead? Since we are checking if the rocks tools library gets loaded.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good idea. Done.

} catch (NativeLibraryNotLoadedException e) {
LOG.debug("Failed to load rocks-tools library", e);
}

boolean nativeRocksToolsLoaded = org.apache.hadoop.hdds.utils.NativeLibraryLoader.isLibraryLoaded();
String rocksToolsDetail = "";
if (nativeRocksToolsLoaded) {
rocksToolsDetail = org.apache.hadoop.hdds.utils.NativeLibraryLoader.getJniLibraryFileName();
}
System.out.printf("rocks-tools: %b %s%n", nativeRocksToolsLoaded, rocksToolsDetail);
return null;
}
}
18 changes: 18 additions & 0 deletions hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.checknative;

import org.apache.hadoop.ozone.shell.checknative.CheckNative;
import org.apache.ozone.test.tag.Native;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.AfterAll;
Expand All @@ -27,6 +28,7 @@
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;

import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
import static org.assertj.core.api.Assertions.assertThat;
import static java.nio.charset.StandardCharsets.UTF_8;

Expand Down Expand Up @@ -59,6 +61,22 @@ public void testCheckNativeNotLoaded() throws UnsupportedEncodingException {
assertThat(stdOut).contains("Native library checking:");
assertThat(stdOut).contains("hadoop: false");
assertThat(stdOut).contains("ISA-L: false");
assertThat(stdOut).contains("rocks-tools: false");
}

@Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
@Test
public void testCheckNativeRocksToolsLoaded() throws UnsupportedEncodingException {
outputStream.reset();
new CheckNative()
.run(new String[] {});
// trims multiple spaces
String stdOut = outputStream.toString(DEFAULT_ENCODING)
.replaceAll(" +", " ");
assertThat(stdOut).contains("Native library checking:");
assertThat(stdOut).contains("hadoop: false");
assertThat(stdOut).contains("ISA-L: false");
assertThat(stdOut).contains("rocks-tools: true");
}

@AfterEach
Expand Down