From 6c20423d80c3139494b700e2300189aa3ff01a78 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Tue, 20 Aug 2024 16:10:40 -0700 Subject: [PATCH 1/5] HDDS-11347. Add rocks_tools_native lib check in Ozone CLI checknative subcommand --- .../hadoop/hdds/utils/NativeLibraryLoader.java | 13 +++++++++++-- .../hadoop/ozone/shell/checknative/CheckNative.java | 13 +++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java index ce424c930e1..d93933dee36 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java @@ -36,6 +36,8 @@ import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; + /** * Class to load Native Libraries. */ @@ -67,6 +69,10 @@ public static NativeLibraryLoader getInstance() { return instance; } + public static String getJniLibraryFileName() { + return appendLibOsSuffix("lib" + ROCKS_TOOLS_NATIVE_LIBRARY_NAME); + } + public static String getJniLibraryFileName(String libraryName) { return appendLibOsSuffix("lib" + libraryName); } @@ -99,9 +105,12 @@ private static String appendLibOsSuffix(String libraryFileName) { return libraryFileName + getLibOsSuffix(); } + public static boolean isLibraryLoaded() { + return isLibraryLoaded(ROCKS_TOOLS_NATIVE_LIBRARY_NAME); + } + public static boolean isLibraryLoaded(final String libraryName) { - return getInstance().librariesLoaded - .getOrDefault(libraryName, false); + return getInstance().librariesLoaded.getOrDefault(libraryName, false); } public synchronized boolean loadLibrary(final String libraryName, final List dependentFiles) { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java index 63b2b425c64..ed474eeb109 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; -import org.apache.hadoop.util.NativeCodeLoader; import picocli.CommandLine; /** @@ -36,12 +35,12 @@ public static void main(String[] argv) { @Override public Void call() throws Exception { - boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded(); + boolean nativeHadoopLoaded = org.apache.hadoop.util.NativeCodeLoader.isNativeCodeLoaded(); String hadoopLibraryName = ""; String isalDetail = ""; boolean isalLoaded = false; if (nativeHadoopLoaded) { - hadoopLibraryName = NativeCodeLoader.getLibraryName(); + hadoopLibraryName = org.apache.hadoop.util.NativeCodeLoader.getLibraryName(); isalDetail = ErasureCodeNative.getLoadingFailureReason(); if (isalDetail != null) { @@ -50,12 +49,18 @@ public Void call() throws Exception { isalDetail = ErasureCodeNative.getLibraryName(); isalLoaded = true; } - } System.out.println("Native library checking:"); System.out.printf("hadoop: %b %s%n", nativeHadoopLoaded, hadoopLibraryName); System.out.printf("ISA-L: %b %s%n", isalLoaded, isalDetail); + + boolean nativeRocksToolsLoaded = org.apache.hadoop.hdds.utils.NativeLibraryLoader.isLibraryLoaded(); + String rocksToolsDetail = ""; + if (nativeRocksToolsLoaded) { + rocksToolsDetail = org.apache.hadoop.hdds.utils.NativeLibraryLoader.getJniLibraryFileName(); + } + System.out.printf("rocks-tools: %b %s%n", nativeRocksToolsLoaded, rocksToolsDetail); return null; } } From 217a79af5bd70d5df2a815add76f49589177c805 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Tue, 20 Aug 2024 16:15:31 -0700 Subject: [PATCH 2/5] Check new output line in TestCheckNative. --- .../org/apache/hadoop/ozone/checknative/TestCheckNative.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java index 8e291056330..47075fb6a23 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java @@ -59,6 +59,7 @@ public void testCheckNativeNotLoaded() throws UnsupportedEncodingException { assertThat(stdOut).contains("Native library checking:"); assertThat(stdOut).contains("hadoop: false"); assertThat(stdOut).contains("ISA-L: false"); + assertThat(stdOut).contains("rocks-tools: false"); } @AfterEach From bc9c74b15b66a7556e584762f1984b7a7ba5c5ec Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Fri, 23 Aug 2024 00:34:10 -0700 Subject: [PATCH 3/5] Add test case when rocks-tools is actually loaded. --- .../ozone/checknative/TestCheckNative.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java index 47075fb6a23..0dc1fde57fa 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/checknative/TestCheckNative.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.checknative; import org.apache.hadoop.ozone.shell.checknative.CheckNative; +import org.apache.ozone.test.tag.Native; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterAll; @@ -27,6 +28,7 @@ import java.io.PrintStream; import java.io.UnsupportedEncodingException; +import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; import static org.assertj.core.api.Assertions.assertThat; import static java.nio.charset.StandardCharsets.UTF_8; @@ -62,6 +64,21 @@ public void testCheckNativeNotLoaded() throws UnsupportedEncodingException { assertThat(stdOut).contains("rocks-tools: false"); } + @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) + @Test + public void testCheckNativeRocksToolsLoaded() throws UnsupportedEncodingException { + outputStream.reset(); + new CheckNative() + .run(new String[] {}); + // trims multiple spaces + String stdOut = outputStream.toString(DEFAULT_ENCODING) + .replaceAll(" +", " "); + assertThat(stdOut).contains("Native library checking:"); + assertThat(stdOut).contains("hadoop: false"); + assertThat(stdOut).contains("ISA-L: false"); + assertThat(stdOut).contains("rocks-tools: true"); + } + @AfterEach public void setUp() { outputStream.reset(); From 056db6220a5f6f630853e93d5c39163316c8b234 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Fri, 23 Aug 2024 05:41:36 -0700 Subject: [PATCH 4/5] loadLibrary --- .../ozone/shell/checknative/CheckNative.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java index ed474eeb109..44707b64962 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java @@ -19,7 +19,11 @@ package org.apache.hadoop.ozone.shell.checknative; import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import picocli.CommandLine; /** @@ -29,6 +33,9 @@ description = "Checks if native libraries are loaded") public class CheckNative extends GenericCli { + private static final Logger LOG = + LoggerFactory.getLogger(CheckNative.class); + public static void main(String[] argv) { new CheckNative().run(argv); } @@ -55,6 +62,13 @@ public Void call() throws Exception { hadoopLibraryName); System.out.printf("ISA-L: %b %s%n", isalLoaded, isalDetail); + // Attempt to load the rocks tools lib + try { + ManagedRawSSTFileReader.loadLibrary(); + } catch (NativeLibraryNotLoadedException e) { + LOG.debug("Failed to load rocks-tools library", e); + } + boolean nativeRocksToolsLoaded = org.apache.hadoop.hdds.utils.NativeLibraryLoader.isLibraryLoaded(); String rocksToolsDetail = ""; if (nativeRocksToolsLoaded) { From d26277b8b1df929d83e2d6860be60c6dba9261e5 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:53:41 -0700 Subject: [PATCH 5/5] Use `NativeLibraryLoader.getInstance().loadLibrary()`. --- .../ozone/shell/checknative/CheckNative.java | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java index 44707b64962..f19548a1fa7 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/checknative/CheckNative.java @@ -19,13 +19,15 @@ package org.apache.hadoop.ozone.shell.checknative; import org.apache.hadoop.hdds.cli.GenericCli; -import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; -import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; +import org.apache.hadoop.hdds.utils.NativeLibraryLoader; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import picocli.CommandLine; +import java.util.Collections; + +import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; + /** * CLI command to check if native libraries are loaded. */ @@ -33,9 +35,6 @@ description = "Checks if native libraries are loaded") public class CheckNative extends GenericCli { - private static final Logger LOG = - LoggerFactory.getLogger(CheckNative.class); - public static void main(String[] argv) { new CheckNative().run(argv); } @@ -62,17 +61,13 @@ public Void call() throws Exception { hadoopLibraryName); System.out.printf("ISA-L: %b %s%n", isalLoaded, isalDetail); - // Attempt to load the rocks tools lib - try { - ManagedRawSSTFileReader.loadLibrary(); - } catch (NativeLibraryNotLoadedException e) { - LOG.debug("Failed to load rocks-tools library", e); - } - - boolean nativeRocksToolsLoaded = org.apache.hadoop.hdds.utils.NativeLibraryLoader.isLibraryLoaded(); + // Attempt to load the rocks-tools lib + boolean nativeRocksToolsLoaded = NativeLibraryLoader.getInstance().loadLibrary( + ROCKS_TOOLS_NATIVE_LIBRARY_NAME, + Collections.singletonList(ManagedRocksObjectUtils.getRocksDBLibFileName())); String rocksToolsDetail = ""; if (nativeRocksToolsLoaded) { - rocksToolsDetail = org.apache.hadoop.hdds.utils.NativeLibraryLoader.getJniLibraryFileName(); + rocksToolsDetail = NativeLibraryLoader.getJniLibraryFileName(); } System.out.printf("rocks-tools: %b %s%n", nativeRocksToolsLoaded, rocksToolsDetail); return null;