diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java index 1ef2119b688fd..6da4935075296 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java @@ -17,14 +17,11 @@ */ package org.apache.hadoop.io; -import java.lang.reflect.Field; +import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.security.AccessController; -import java.security.PrivilegedAction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import sun.misc.Unsafe; import org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes; @@ -57,8 +54,7 @@ private static Comparer lexicographicalComparerJavaImpl() { /** - * Provides a lexicographical comparer implementation; either a Java - * implementation or a faster implementation based on {@link Unsafe}. + * Provides a lexicographical comparer Java implementation. * *

Uses reflection to gracefully fall back to the Java implementation if * {@code Unsafe} isn't available. @@ -131,34 +127,19 @@ public int compareTo(byte[] buffer1, int offset1, int length1, private enum UnsafeComparer implements Comparer { INSTANCE; - static final Unsafe theUnsafe; - /** The offset to the first element in a byte array. */ static final int BYTE_ARRAY_BASE_OFFSET; static { - theUnsafe = (Unsafe) AccessController.doPrivileged( - new PrivilegedAction() { - @Override - public Object run() { - try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - return f.get(null); - } catch (NoSuchFieldException e) { - // It doesn't matter what we throw; - // it's swallowed in getBestComparer(). - throw new Error(); - } catch (IllegalAccessException e) { - throw new Error(); - } - } - }); - - BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class); + /** + * For a byte[] array, the base offset is always 0 because byte is the most fundamental data type in Java, + * and it does not have any additional header information. + * Therefore, it can be safely assumed that its base offset is 0. + */ + BYTE_ARRAY_BASE_OFFSET = 0; // sanity check - this should never fail - if (theUnsafe.arrayIndexScale(byte[].class) != 1) { + if (Byte.BYTES != 1) { throw new AssertionError(); } } @@ -207,8 +188,8 @@ public int compareTo(byte[] buffer1, int offset1, int length1, * On the other hand, it is substantially faster on 64-bit. */ for (i = 0; i < strideLimit; i += stride) { - long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i); - long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i); + long lw = getLongFromByteBuffer(buffer1, offset1Adj + i); + long rw = getLongFromByteBuffer(buffer2, offset2Adj + i); if (lw != rw) { if (!littleEndian) { @@ -241,5 +222,11 @@ public int compareTo(byte[] buffer1, int offset1, int length1, return length1 - length2; } } + + /** Get the long value from the specified offset. */ + private static long getLongFromByteBuffer(byte[] buffer, int offset) { + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer); + return byteBuffer.order(ByteOrder.nativeOrder()).getLong(offset); + } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index 0a469e3024b59..5895a3da5d73f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -17,13 +17,14 @@ */ package org.apache.hadoop.io.nativeio; +import java.io.BufferedReader; import java.io.File; import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.io.RandomAccessFile; -import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; @@ -45,7 +46,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import sun.misc.Unsafe; import org.apache.hadoop.classification.VisibleForTesting; @@ -898,16 +898,35 @@ static long getMemlockLimit() { */ static long getOperatingSystemPageSize() { try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - Unsafe unsafe = (Unsafe)f.get(null); - return unsafe.pageSize(); + if (isAvailable()) { + return getOperatingSystemPageSize0(); + } else { + String os = System.getProperty("os.name").toLowerCase(); + ProcessBuilder processBuilder; + if (os.contains("mac") || os.contains("linux") || os.contains("unix")) { + processBuilder = new ProcessBuilder("getconf", "PAGESIZE"); + } else { + return 4096; + } + Process process = processBuilder.start(); + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream()))) { + String line = reader.readLine(); + if (line != null && !line.isEmpty()) { + return Long.parseLong(line.trim()); + } else { + return 4096; + } + } + } } catch (Throwable e) { LOG.warn("Unable to get operating system page size. Guessing 4096.", e); return 4096; } } + private static native long getOperatingSystemPageSize0(); + private static class CachedUid { final long timestamp; final String username; diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index f720d993770ca..acc6b5389ae19 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -1483,6 +1483,25 @@ JNIEnv *env, jclass clazz) #endif } +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_getOperatingSystemPageSize0( +JNIEnv *env, jclass clazz) +{ +#ifdef UNIX + long pageSize = sysconf(_SC_PAGESIZE); + if (pageSize < 0) { + // Guess 4096 + return 4096; + } + return (jlong) pageSize; +#endif +#ifdef WINDOWS + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + return (jlong) sysInfo.dwPageSize; +#endif +} + JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_copyFileUnbuffered0( JNIEnv *env, jclass clazz, jstring jsrc, jstring jdst) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java index c6f7a50368152..52f66cfa83f8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; @@ -37,8 +38,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import sun.misc.Unsafe; - import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; @@ -54,19 +53,6 @@ public class ShortCircuitShm { protected static final int BYTES_PER_SLOT = 64; - private static final Unsafe unsafe = safetyDance(); - - private static Unsafe safetyDance() { - try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - return (Unsafe)f.get(null); - } catch (Throwable e) { - LOG.error("failed to load misc.Unsafe", e); - } - return null; - } - /** * Calculate the usable size of a shared memory segment. * We round down to a multiple of the slot size and do some validation. @@ -259,9 +245,9 @@ public class Slot { private static final long ANCHORABLE_FLAG = 1L<<62; /** - * The slot address in memory. + * The slot address in memory. Use AtomicLong instead of Unsafe */ - private final long slotAddress; + private final AtomicLong slotAddress; /** * BlockId of the block this slot is used for. @@ -269,7 +255,7 @@ public class Slot { private final ExtendedBlockId blockId; Slot(long slotAddress, ExtendedBlockId blockId) { - this.slotAddress = slotAddress; + this.slotAddress = new AtomicLong(slotAddress); this.blockId = blockId; } @@ -307,41 +293,39 @@ public SlotId getSlotId() { */ public int getSlotIdx() { return Ints.checkedCast( - (slotAddress - baseAddress) / BYTES_PER_SLOT); + (slotAddress.get() - baseAddress) / BYTES_PER_SLOT); } /** * Clear the slot. */ void clear() { - unsafe.putLongVolatile(null, this.slotAddress, 0); + slotAddress.set(0); } private boolean isSet(long flag) { - long prev = unsafe.getLongVolatile(null, this.slotAddress); + long prev = slotAddress.get(); return (prev & flag) != 0; } private void setFlag(long flag) { long prev; do { - prev = unsafe.getLongVolatile(null, this.slotAddress); + prev = slotAddress.get(); if ((prev & flag) != 0) { return; } - } while (!unsafe.compareAndSwapLong(null, this.slotAddress, - prev, prev | flag)); + } while (!slotAddress.compareAndSet(prev, prev | flag)); } private void clearFlag(long flag) { long prev; do { - prev = unsafe.getLongVolatile(null, this.slotAddress); + prev = slotAddress.get(); if ((prev & flag) == 0) { return; } - } while (!unsafe.compareAndSwapLong(null, this.slotAddress, - prev, prev & (~flag))); + } while (!slotAddress.compareAndSet(prev, prev & (~flag))); } public boolean isValid() { @@ -369,7 +353,7 @@ public void makeUnanchorable() { } public boolean isAnchored() { - long prev = unsafe.getLongVolatile(null, this.slotAddress); + long prev = slotAddress.get(); // Slot is no longer valid. return (prev & VALID_FLAG) != 0 && ((prev & 0x7fffffff) != 0); } @@ -385,7 +369,7 @@ public boolean isAnchored() { public boolean addAnchor() { long prev; do { - prev = unsafe.getLongVolatile(null, this.slotAddress); + prev = slotAddress.get(); if ((prev & VALID_FLAG) == 0) { // Slot is no longer valid. return false; @@ -398,8 +382,7 @@ public boolean addAnchor() { // Too many other threads have anchored the slot (2 billion?) return false; } - } while (!unsafe.compareAndSwapLong(null, this.slotAddress, - prev, prev + 1)); + } while (!slotAddress.compareAndSet(prev, prev + 1)); return true; } @@ -409,12 +392,11 @@ public boolean addAnchor() { public void removeAnchor() { long prev; do { - prev = unsafe.getLongVolatile(null, this.slotAddress); + prev = slotAddress.get(); Preconditions.checkState((prev & 0x7fffffff) != 0, "Tried to remove anchor for slot " + slotAddress +", which was " + "not anchored."); - } while (!unsafe.compareAndSwapLong(null, this.slotAddress, - prev, prev - 1)); + } while (!slotAddress.compareAndSet(prev, prev - 1)); } @Override @@ -473,11 +455,6 @@ public ShortCircuitShm(ShmId shmId, FileInputStream stream) throw new UnsupportedOperationException( "DfsClientShm is not yet implemented for Windows."); } - if (unsafe == null) { - throw new UnsupportedOperationException( - "can't use DfsClientShm because we failed to " + - "load misc.Unsafe."); - } this.shmId = shmId; this.mmappedLength = getUsableLength(stream); this.baseAddress = POSIX.mmap(stream.getFD(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java index 1cd97800e33cb..d8dad3b695539 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java @@ -130,32 +130,23 @@ public final boolean cancel(boolean mayInterruptIfRunning) { AtomicHelper helper; try { - helper = new UnsafeAtomicHelper(); - } catch (Throwable unsafeFailure) { - // catch absolutely everything and fall through to our 'SafeAtomicHelper' - // The access control checks that ARFU does means the caller class has - // to be AbstractFuture - // instead of SafeAtomicHelper, so we annoyingly define these here - try { - helper = - new SafeAtomicHelper( - newUpdater(Waiter.class, Thread.class, "thread"), - newUpdater(Waiter.class, Waiter.class, "next"), - newUpdater(AbstractFuture.class, Waiter.class, "waiters"), - newUpdater(AbstractFuture.class, Listener.class, "listeners"), - newUpdater(AbstractFuture.class, Object.class, "value")); - } catch (Throwable atomicReferenceFieldUpdaterFailure) { - // Some Android 5.0.x Samsung devices have bugs in JDK reflection APIs - // that cause getDeclaredField to throw a NoSuchFieldException when - // the field is definitely there. - // For these users fallback to a suboptimal implementation, based on - // synchronized. This will be a definite performance hit to those users. - log.log(Level.SEVERE, "UnsafeAtomicHelper is broken!", unsafeFailure); - log.log( - Level.SEVERE, "SafeAtomicHelper is broken!", - atomicReferenceFieldUpdaterFailure); - helper = new SynchronizedHelper(); - } + helper = + new SafeAtomicHelper( + newUpdater(Waiter.class, Thread.class, "thread"), + newUpdater(Waiter.class, Waiter.class, "next"), + newUpdater(AbstractFuture.class, Waiter.class, "waiters"), + newUpdater(AbstractFuture.class, Listener.class, "listeners"), + newUpdater(AbstractFuture.class, Object.class, "value")); + } catch (Throwable atomicReferenceFieldUpdaterFailure) { + // Some Android 5.0.x Samsung devices have bugs in JDK reflection APIs + // that cause getDeclaredField to throw a NoSuchFieldException when + // the field is definitely there. + // For these users fallback to a suboptimal implementation, based on + // synchronized. This will be a definite performance hit to those users. + log.log( + Level.SEVERE, "SafeAtomicHelper is broken!", + atomicReferenceFieldUpdaterFailure); + helper = new SynchronizedHelper(); } ATOMIC_HELPER = helper; @@ -1035,115 +1026,6 @@ abstract boolean casValue( AbstractFuture future, Object expect, Object update); } - /** - * {@link AtomicHelper} based on {@link sun.misc.Unsafe}. - *

- *

Static initialization of this class will fail if the - * {@link sun.misc.Unsafe} object cannot be accessed. - */ - private static final class UnsafeAtomicHelper extends AtomicHelper { - static final sun.misc.Unsafe UNSAFE; - static final long LISTENERS_OFFSET; - static final long WAITERS_OFFSET; - static final long VALUE_OFFSET; - static final long WAITER_THREAD_OFFSET; - static final long WAITER_NEXT_OFFSET; - - static { - sun.misc.Unsafe unsafe = null; - try { - unsafe = sun.misc.Unsafe.getUnsafe(); - } catch (SecurityException tryReflectionInstead) { - try { - unsafe = - AccessController.doPrivileged( - new PrivilegedExceptionAction() { - @Override - public sun.misc.Unsafe run() throws Exception { - Class k = sun.misc.Unsafe.class; - for (java.lang.reflect.Field f : k.getDeclaredFields()) { - f.setAccessible(true); - Object x = f.get(null); - if (k.isInstance(x)) { - return k.cast(x); - } - } - throw new NoSuchFieldError("the Unsafe"); - } - }); - } catch (PrivilegedActionException e) { - throw new RuntimeException( - "Could not initialize intrinsics", e.getCause()); - } - } - try { - Class abstractFuture = AbstractFuture.class; - WAITERS_OFFSET = unsafe - .objectFieldOffset(abstractFuture.getDeclaredField("waiters")); - LISTENERS_OFFSET = unsafe - .objectFieldOffset(abstractFuture.getDeclaredField("listeners")); - VALUE_OFFSET = unsafe - .objectFieldOffset(abstractFuture.getDeclaredField("value")); - WAITER_THREAD_OFFSET = unsafe - .objectFieldOffset(Waiter.class.getDeclaredField("thread")); - WAITER_NEXT_OFFSET = unsafe - .objectFieldOffset(Waiter.class.getDeclaredField("next")); - UNSAFE = unsafe; - } catch (Exception e) { - throwIfUnchecked(e); - throw new RuntimeException(e); - } - } - - public static void throwIfUnchecked(Throwable throwable) { - Preconditions.checkNotNull(throwable); - if (throwable instanceof RuntimeException) { - throw (RuntimeException) throwable; - } - if (throwable instanceof Error) { - throw (Error) throwable; - } - } - - @Override - void putThread(Waiter waiter, Thread newValue) { - UNSAFE.putObject(waiter, WAITER_THREAD_OFFSET, newValue); - } - - @Override - void putNext(Waiter waiter, Waiter newValue) { - UNSAFE.putObject(waiter, WAITER_NEXT_OFFSET, newValue); - } - - /** - * Performs a CAS operation on the {@link #waiters} field. - */ - @Override - boolean casWaiters(AbstractFuture future, Waiter expect, Waiter - update) { - return UNSAFE - .compareAndSwapObject(future, WAITERS_OFFSET, expect, update); - } - - /** - * Performs a CAS operation on the {@link #listeners} field. - */ - @Override - boolean casListeners( - AbstractFuture future, Listener expect, Listener update) { - return UNSAFE - .compareAndSwapObject(future, LISTENERS_OFFSET, expect, update); - } - - /** - * Performs a CAS operation on the {@link #value} field. - */ - @Override - boolean casValue(AbstractFuture future, Object expect, Object update) { - return UNSAFE.compareAndSwapObject(future, VALUE_OFFSET, expect, update); - } - } - /** * {@link AtomicHelper} based on {@link AtomicReferenceFieldUpdater}. */