diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/ByteArraySupportPartialEvaluationTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/ByteArraySupportPartialEvaluationTest.java index 3873a69bd3f0..2543f87a7d3a 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/ByteArraySupportPartialEvaluationTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/ByteArraySupportPartialEvaluationTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -69,6 +69,21 @@ public int execute(VirtualFrame frame) { } } + static class GetShortUnalignedNonStableNode extends AbstractTestNode { + @CompilationFinal(dimensions = 0) byte[] bytes; + final int offset; + + GetShortUnalignedNonStableNode(String hex, int offset) { + this.bytes = hexToBytes(hex); + this.offset = offset; + } + + @Override + public int execute(VirtualFrame frame) { + return BYTES.getShortUnaligned(bytes, offset); + } + } + static class GetIntNode extends AbstractTestNode { @CompilationFinal(dimensions = 1) byte[] bytes; final int offset; @@ -99,6 +114,21 @@ public int execute(VirtualFrame frame) { } } + static class GetIntUnalignedNonStableNode extends AbstractTestNode { + @CompilationFinal(dimensions = 0) byte[] bytes; + final int offset; + + GetIntUnalignedNonStableNode(String hex, int offset) { + this.bytes = hexToBytes(hex); + this.offset = offset; + } + + @Override + public int execute(VirtualFrame frame) { + return BYTES.getIntUnaligned(bytes, offset); + } + } + static class GetLongNode extends LongNode { @CompilationFinal(dimensions = 1) byte[] bytes; final int offset; @@ -129,6 +159,21 @@ public long execute(VirtualFrame frame) { } } + static class GetLongUnalignedNonStableNode extends LongNode { + @CompilationFinal(dimensions = 0) byte[] bytes; + final int offset; + + GetLongUnalignedNonStableNode(String hex, int offset) { + this.bytes = hexToBytes(hex); + this.offset = offset; + } + + @Override + public long execute(VirtualFrame frame) { + return BYTES.getLongUnaligned(bytes, offset); + } + } + private static byte[] hexToBytes(String s) { int len = s.length(); byte[] data = new byte[len / 2]; @@ -222,4 +267,44 @@ public void testGetLongUnaligned() { assertPartialEvalEquals(constLongRootNode(0x1122334455667788L), new LongRootNode(new GetLongUnalignedNode("00000000000000008877665544332211", 8))); assertPartialEvalEquals(constLongRootNode(0x1122334455667788L), new LongRootNode(new GetLongUnalignedNode("008877665544332211", 1))); } + + @Test + public void testGetUnalignedFromNonStableArray() { + assertPartialEvalEquals(readShortRootNode(), new RootTestNode("getShortUnaligned", new GetShortUnalignedNonStableNode("0089abcdef", 1))); + assertPartialEvalEquals(readIntRootNode(), new RootTestNode("getIntUnaligned", new GetIntUnalignedNonStableNode("000089abcdef", 2))); + assertPartialEvalEquals(readLongRootNode(), new LongRootNode(new GetLongUnalignedNonStableNode("008877665544332211", 1))); + } + + private static RootTestNode readShortRootNode() { + return new RootTestNode("readShort", new AbstractTestNode() { + private final byte[] bytes = new byte[8]; + + @Override + public int execute(VirtualFrame frame) { + return BYTES.getShort(bytes, 0); + } + }); + } + + private static RootTestNode readIntRootNode() { + return new RootTestNode("readInt", new AbstractTestNode() { + private final byte[] bytes = new byte[8]; + + @Override + public int execute(VirtualFrame frame) { + return BYTES.getInt(bytes, 0); + } + }); + } + + private static LongRootNode readLongRootNode() { + return new LongRootNode(new LongNode() { + private final byte[] bytes = new byte[8]; + + @Override + public long execute(VirtualFrame frame) { + return BYTES.getLong(bytes, 0); + } + }); + } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/nodes/ObjectLocationIdentity.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/nodes/ObjectLocationIdentity.java index c1c9c25571d3..b5d81b80d3a1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/nodes/ObjectLocationIdentity.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/nodes/ObjectLocationIdentity.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,6 +35,13 @@ /** * A {@link LocationIdentity} wrapping an object. + * + * Used by Truffle unsafe accesses to associate unique location identities with DynamicObject and + * FrameWithoutBoxing accesses, backed by non-null object references (compared by identity). The + * compiler can assume that accesses with different location identities (except for "any") do not + * interfere with each other (or when they do are constrained by memory barriers), even when they + * may access the same relative memory address (array or field offset) of objects of the same array + * or instance class. */ public final class ObjectLocationIdentity extends LocationIdentity implements JavaConstantFormattable { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/substitutions/TruffleGraphBuilderPlugins.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/substitutions/TruffleGraphBuilderPlugins.java index 8b8771469608..48690aa498ad 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/substitutions/TruffleGraphBuilderPlugins.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/substitutions/TruffleGraphBuilderPlugins.java @@ -28,6 +28,7 @@ import static jdk.graal.compiler.replacements.PEGraphDecoder.Options.MaximumLoopExplosionCount; import java.lang.reflect.Type; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -40,14 +41,17 @@ import com.oracle.truffle.compiler.TruffleCompilationTask; +import jdk.graal.compiler.core.common.NumUtil; import jdk.graal.compiler.core.common.calc.CanonicalCondition; import jdk.graal.compiler.core.common.memory.MemoryOrderMode; +import jdk.graal.compiler.core.common.type.IntegerStamp; import jdk.graal.compiler.core.common.type.ObjectStamp; import jdk.graal.compiler.core.common.type.Stamp; import jdk.graal.compiler.core.common.type.StampFactory; import jdk.graal.compiler.core.common.type.StampPair; import jdk.graal.compiler.core.common.type.TypeReference; import jdk.graal.compiler.debug.DebugContext; +import jdk.graal.compiler.debug.GraalError; import jdk.graal.compiler.graph.Node; import jdk.graal.compiler.lir.gen.ArithmeticLIRGeneratorTool.RoundingMode; import jdk.graal.compiler.nodes.CallTargetNode; @@ -61,6 +65,7 @@ import jdk.graal.compiler.nodes.InvokeNode; import jdk.graal.compiler.nodes.LogicConstantNode; import jdk.graal.compiler.nodes.LogicNode; +import jdk.graal.compiler.nodes.NamedLocationIdentity; import jdk.graal.compiler.nodes.NodeView; import jdk.graal.compiler.nodes.PiArrayNode; import jdk.graal.compiler.nodes.PiNode; @@ -133,6 +138,7 @@ import jdk.vm.ci.meta.DeoptimizationReason; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.MemoryAccessProvider; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -172,6 +178,7 @@ public static void registerInvocationPlugins(InvocationPlugins plugins, KnownTru registerDynamicObjectPlugins(plugins, types, canDelayIntrinsification, providers.getConstantReflection()); registerBufferPlugins(plugins, types, canDelayIntrinsification); registerMemorySegmentPlugins(plugins, types, canDelayIntrinsification); + registerByteArraySupportPlugins(plugins, canDelayIntrinsification); } private static void registerTruffleSafepointPlugins(InvocationPlugins plugins, KnownTruffleTypes types, boolean canDelayIntrinsification) { @@ -1332,18 +1339,17 @@ static class CustomizedUnsafeStorePlugin extends RequiredInvocationPlugin { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode object, ValueNode offset, ValueNode value, ValueNode location) { - ValueNode locationArgument = location; - if (locationArgument.isConstant()) { + if (location.isConstant()) { LocationIdentity locationIdentity; boolean forceLocation; - if (locationArgument.isNullConstant()) { + if (location.isNullConstant()) { locationIdentity = LocationIdentity.any(); forceLocation = false; - } else if (locationArgument.asJavaConstant().equals(anyConstant)) { + } else if (location.asJavaConstant().equals(anyConstant)) { locationIdentity = LocationIdentity.any(); forceLocation = true; } else { - locationIdentity = ObjectLocationIdentity.create(locationArgument.asJavaConstant()); + locationIdentity = ObjectLocationIdentity.create(location.asJavaConstant()); forceLocation = true; } b.add(new RawStoreNode(object, offset, value, kind, locationIdentity, true, null, forceLocation)); @@ -1377,7 +1383,7 @@ static void logPerformanceWarningLocationNotConstant(ValueNode location, Resolve debug.dump(DebugContext.VERBOSE_LEVEL, graph, "perf warn: Location argument is not a partial evaluation constant: %s", location); } } catch (Throwable t) { - debug.handle(t); + throw debug.handle(t); } } } @@ -1411,7 +1417,7 @@ static void logPerformanceWarningUnsafeCastArgNotConst(ResolvedJavaMethod target debug.dump(DebugContext.VERBOSE_LEVEL, graph, "perf warn: unsafeCast arguments could not reduce to a constant: %s, %s, %s", type, nonNull, isExactType); } } catch (Throwable t) { - debug.handle(t); + throw debug.handle(t); } } } @@ -1419,8 +1425,7 @@ static void logPerformanceWarningUnsafeCastArgNotConst(ResolvedJavaMethod target static BailoutException failPEConstant(GraphBuilderContext b, ValueNode value) { StringBuilder sb = new StringBuilder(); sb.append(value); - if (value instanceof ValuePhiNode) { - ValuePhiNode valuePhi = (ValuePhiNode) value; + if (value instanceof ValuePhiNode valuePhi) { sb.append(" ("); for (Node n : valuePhi.inputs()) { sb.append(n); @@ -1443,8 +1448,7 @@ private PEConstantPlugin(boolean canDelayIntrinsification, Type... argumentTypes @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { ValueNode curValue = value; - if (curValue instanceof BoxNode) { - BoxNode boxNode = (BoxNode) curValue; + if (curValue instanceof BoxNode boxNode) { curValue = boxNode.getValue(); } if (curValue.isConstant()) { @@ -1457,4 +1461,121 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec } } + + private static void registerByteArraySupportPlugins(InvocationPlugins plugins, boolean canDelayIntrinsification) { + Registration r = new Registration(plugins, "com.oracle.truffle.api.memory.UnsafeByteArraySupport"); + r.register(new UnsafeGetUnalignedPlugin("unsafeGetShortUnaligned", JavaKind.Short, canDelayIntrinsification)); + r.register(new UnsafeGetUnalignedPlugin("unsafeGetIntUnaligned", JavaKind.Int, canDelayIntrinsification)); + r.register(new UnsafeGetUnalignedPlugin("unsafeGetLongUnaligned", JavaKind.Long, canDelayIntrinsification)); + } + + private static class UnsafeGetUnalignedPlugin extends OptionalInvocationPlugin { + private final boolean canDelayIntrinsification; + private final JavaKind resultKind; + + UnsafeGetUnalignedPlugin(String name, JavaKind resultKind, boolean canDelayIntrinsification) { + super(name, byte[].class, long.class); + this.canDelayIntrinsification = canDelayIntrinsification; + this.resultKind = resultKind; + assert resultKind == JavaKind.Short || resultKind == JavaKind.Int || resultKind == JavaKind.Long : resultKind; + GraalError.guarantee(ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN, "only supported on little-endian architecture"); + } + + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, + ValueNode bufferNode, ValueNode byteOffsetNode) { + if (bufferNode instanceof ConstantNode bufferConstNode && byteOffsetNode.isConstant()) { + if (bufferConstNode.getStableDimension() == 1) { // implies non-null + JavaConstant bufferConst = bufferConstNode.asJavaConstant(); + long byteOffset = byteOffsetNode.asJavaConstant().asLong(); + JavaConstant value = readUnaligned(b, resultKind, bufferConst, byteOffset); + if (value != null && (bufferConstNode.isDefaultStable() || !value.isDefaultForKind())) { + b.addPush(resultKind, ConstantNode.forPrimitive(value, b.getGraph())); + return true; + } + } + } else if (canDelayIntrinsification) { + return false; + } + b.addPush(resultKind, new RawLoadNode(bufferNode, byteOffsetNode, resultKind, NamedLocationIdentity.getArrayLocation(JavaKind.Byte), MemoryOrderMode.PLAIN)); + return true; + } + + /** + * Reads a short, int, or long value from a potentially unaligned offset in a byte[] array. + * Performs a single aligned read if the address is aligned, otherwise combines the results + * of multiple reads of the next narrower naturally aligned width or individual bytes. + * + * @param resultKind value kind, either short, int, or long + * @param base byte[] array constant, with stable dimensions = 1 + * @param byteOffset byte[] index, not including array base offset + * @return result value constant or {@code null} if out of bounds + */ + @SuppressWarnings("fallthrough") + private static JavaConstant readUnaligned(GraphBuilderContext b, JavaKind resultKind, JavaConstant base, long byteOffset) { + ConstantReflectionProvider constantReflection = b.getConstantReflection(); + MemoryAccessProvider memoryAccessProvider = constantReflection.getMemoryAccessProvider(); + long displacement = b.getMetaAccess().getArrayBaseOffset(JavaKind.Byte) + byteOffset; + int resultBytes = resultKind.getByteCount(); + if (displacement % resultBytes == 0) { + // Already aligned, so we can read the value directly. + IntegerStamp accessStamp = StampFactory.forInteger(resultKind.getBitCount()); + return (JavaConstant) accessStamp.readConstant(memoryAccessProvider, base, displacement); + } + + // Figure out if we can read the value in wider-than-byte aligned parts. + JavaKind alignedKind = null; + switch (resultKind) { + case Long: + if (displacement % Integer.BYTES == 0) { + alignedKind = JavaKind.Int; + break; + } + // fallthrough + case Int: + if (displacement % Short.BYTES == 0) { + alignedKind = JavaKind.Short; + break; + } + break; + } + if (alignedKind != null) { + long value = 0; + long mask = NumUtil.getNbitNumberLong(alignedKind.getBitCount()); + IntegerStamp accessStamp = StampFactory.forInteger(alignedKind.getBitCount()); + for (int byteCount = 0; byteCount < resultBytes; byteCount += alignedKind.getByteCount()) { + var part = (JavaConstant) accessStamp.readConstant(memoryAccessProvider, base, displacement + byteCount); + if (part == null) { + /* + * Should not normally happen if base+displacement is aligned and in bounds; + * but in the unexpected case that the read fails, handle it gracefully. + */ + return null; + } + value |= ((part.asLong() & mask) << (byteCount * Byte.SIZE)); + } + return JavaConstant.forPrimitive(resultKind, value); + } + + // Displacement is odd, so we have to read the value byte-by-byte. + assert displacement % 2 != 0 : displacement; + long value = 0; + int byteOffsetAsInt = NumUtil.safeToInt(byteOffset); + for (int byteCount = 0; byteCount < resultBytes; byteCount += 2) { + JavaConstant b0 = constantReflection.readArrayElement(base, byteOffsetAsInt + byteCount); + JavaConstant b1 = constantReflection.readArrayElement(base, byteOffsetAsInt + byteCount + 1); + if (b0 == null || b1 == null) { + /* + * Byte offset is out of bounds. This is not necessarily an error since it + * depends on control flow / bounds checks if this read is actually reachable, + * so we must not fail compilation. We can either deoptimize here or fall back + * to a normal unsafe read. + */ + return null; + } + value |= (b0.asInt() & 0xffL | ((b1.asInt() & 0xffL) << Byte.SIZE)) << (byteCount * Byte.SIZE); + } + return JavaConstant.forPrimitive(resultKind, value); + } + } } diff --git a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/memory/UnsafeByteArraySupport.java b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/memory/UnsafeByteArraySupport.java index 7d19a4898c21..a489741bcab9 100644 --- a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/memory/UnsafeByteArraySupport.java +++ b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/memory/UnsafeByteArraySupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -41,8 +41,6 @@ package com.oracle.truffle.api.memory; -import sun.misc.Unsafe; - import java.lang.reflect.Field; import java.nio.ByteOrder; import java.security.AccessController; @@ -50,18 +48,14 @@ import com.oracle.truffle.api.CompilerDirectives; +import sun.misc.Unsafe; + /** * Implementation of {@link ByteArraySupport} using {@link Unsafe}. *

* Bytes ordering is native endianness ({@link ByteOrder#nativeOrder}). */ final class UnsafeByteArraySupport extends ByteArraySupport { - /** - * Partial evaluation does not constant-fold unaligned accesses, so in compiled code we - * decompose unaligned accesses into multiple aligned accesses that can be constant-folded. This - * optimization is only tested on little-endian platforms. - */ - private static final boolean OPTIMIZED_UNALIGNED_SUPPORTED = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN; @SuppressWarnings("deprecation") private static final Unsafe UNSAFE = AccessController.doPrivileged(new PrivilegedAction() { @Override @@ -204,13 +198,8 @@ public short getShortUnaligned(byte[] buffer, int byteOffset) throws IndexOutOfB @Override public short getShortUnaligned(byte[] buffer, long byteOffset) throws IndexOutOfBoundsException { - if (CompilerDirectives.inCompiledCode() && OPTIMIZED_UNALIGNED_SUPPORTED) { - if (byteOffset % Short.BYTES == 0) { - return getShort(buffer, byteOffset); - } else { - return (short) ((getByte(buffer, byteOffset) & 0xFF) | - ((getByte(buffer, byteOffset + 1) & 0xFF) << Byte.SIZE)); - } + if (CompilerDirectives.inCompiledCode()) { + return unsafeGetShortUnaligned(buffer, byteOffset); } else { return getShort(buffer, byteOffset); } @@ -223,15 +212,8 @@ public int getIntUnaligned(byte[] buffer, int byteOffset) throws IndexOutOfBound @Override public int getIntUnaligned(byte[] buffer, long byteOffset) throws IndexOutOfBoundsException { - if (CompilerDirectives.inCompiledCode() && OPTIMIZED_UNALIGNED_SUPPORTED) { - if (byteOffset % Integer.BYTES == 0) { - return getInt(buffer, byteOffset); - } else { - return (getByte(buffer, byteOffset) & 0xFF) | - ((getByte(buffer, byteOffset + 1) & 0xFF) << Byte.SIZE) | - ((getByte(buffer, byteOffset + 2) & 0xFF) << Byte.SIZE * 2) | - ((getByte(buffer, byteOffset + 3) & 0xFF) << Byte.SIZE * 3); - } + if (CompilerDirectives.inCompiledCode()) { + return unsafeGetIntUnaligned(buffer, byteOffset); } else { return getInt(buffer, byteOffset); } @@ -244,19 +226,8 @@ public long getLongUnaligned(byte[] buffer, int byteOffset) throws IndexOutOfBou @Override public long getLongUnaligned(byte[] buffer, long byteOffset) throws IndexOutOfBoundsException { - if (CompilerDirectives.inCompiledCode() && OPTIMIZED_UNALIGNED_SUPPORTED) { - if (byteOffset % Long.BYTES == 0) { - return getLong(buffer, byteOffset); - } else { - return (getByte(buffer, byteOffset) & 0xFFL) | - ((getByte(buffer, byteOffset + 1) & 0xFFL) << Byte.SIZE) | - ((getByte(buffer, byteOffset + 2) & 0xFFL) << Byte.SIZE * 2) | - ((getByte(buffer, byteOffset + 3) & 0xFFL) << Byte.SIZE * 3) | - ((getByte(buffer, byteOffset + 4) & 0xFFL) << Byte.SIZE * 4) | - ((getByte(buffer, byteOffset + 5) & 0xFFL) << Byte.SIZE * 5) | - ((getByte(buffer, byteOffset + 6) & 0xFFL) << Byte.SIZE * 6) | - ((getByte(buffer, byteOffset + 7) & 0xFFL) << Byte.SIZE * 7); - } + if (CompilerDirectives.inCompiledCode()) { + return unsafeGetLongUnaligned(buffer, byteOffset); } else { return getLong(buffer, byteOffset); } @@ -542,4 +513,148 @@ public long compareAndExchangeLong(byte[] buffer, long byteOffset, long expected } while (!UNSAFE.compareAndSwapLong(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + wordOffset, fullWord, x)); return expected; } + + /** + * Intrinsic candidate. + * + * Partial evaluation does not constant-fold unaligned accesses, so in compiled code we + * decompose unaligned accesses into multiple aligned accesses that can be constant-folded. + */ + private static short unsafeGetShortUnaligned(byte[] buffer, long byteOffset) { + if (byteOffset % Short.BYTES == 0) { + return UNSAFE.getShort(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset); + } else { + return makeShort(UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 1)); + } + } + + /** + * Intrinsic candidate. + * + * Partial evaluation does not constant-fold unaligned accesses, so in compiled code we + * decompose unaligned accesses into multiple aligned accesses that can be constant-folded. + */ + private static int unsafeGetIntUnaligned(byte[] buffer, long byteOffset) { + if (byteOffset % Integer.BYTES == 0) { + return UNSAFE.getInt(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset); + } else if (byteOffset % Short.BYTES == 0) { + return makeInt(UNSAFE.getShort(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset), + UNSAFE.getShort(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + Short.BYTES)); + } else { + return makeInt(UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 1), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 2), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 3)); + } + } + + /** + * Intrinsic candidate. + * + * Partial evaluation does not constant-fold unaligned accesses, so in compiled code we + * decompose unaligned accesses into multiple aligned accesses that can be constant-folded. + */ + private static long unsafeGetLongUnaligned(byte[] buffer, long byteOffset) { + if (byteOffset % Long.BYTES == 0) { + return UNSAFE.getLong(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset); + } else if (byteOffset % Integer.BYTES == 0) { + return makeLong(UNSAFE.getInt(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset), + UNSAFE.getInt(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 4)); + } else if (byteOffset % Short.BYTES == 0) { + return makeLong(UNSAFE.getShort(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset), + UNSAFE.getShort(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + Short.BYTES), + UNSAFE.getShort(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + Short.BYTES * 2), + UNSAFE.getShort(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + Short.BYTES * 3)); + } else { + return makeLong(UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 1), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 2), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 3), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 4), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 5), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 6), + UNSAFE.getByte(buffer, Unsafe.ARRAY_BYTE_BASE_OFFSET + byteOffset + 7)); + } + } + + private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) { + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + return (i0 & 0xffL) | + ((i1 & 0xffL) << Byte.SIZE) | + ((i2 & 0xffL) << Byte.SIZE * 2) | + ((i3 & 0xffL) << Byte.SIZE * 3) | + ((i4 & 0xffL) << Byte.SIZE * 4) | + ((i5 & 0xffL) << Byte.SIZE * 5) | + ((i6 & 0xffL) << Byte.SIZE * 6) | + ((i7 & 0xffL) << Byte.SIZE * 7); + } else { + return ((i0 & 0xffL) << Byte.SIZE * 7) | + ((i1 & 0xffL) << Byte.SIZE * 6) | + ((i2 & 0xffL) << Byte.SIZE * 5) | + ((i3 & 0xffL) << Byte.SIZE * 4) | + ((i4 & 0xffL) << Byte.SIZE * 3) | + ((i5 & 0xffL) << Byte.SIZE * 2) | + ((i6 & 0xffL) << Byte.SIZE) | + (i7 & 0xffL); + } + } + + private static long makeLong(short i0, short i1, short i2, short i3) { + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + return (i0 & 0xffffL) | + ((i1 & 0xffffL) << Short.SIZE) | + ((i2 & 0xffffL) << Short.SIZE * 2) | + ((i3 & 0xffffL) << Short.SIZE * 3); + } else { + return ((i0 & 0xffffL) << Short.SIZE * 3) | + ((i1 & 0xffffL) << Short.SIZE * 2) | + ((i2 & 0xffffL) << Short.SIZE) | + (i3 & 0xffffL); + } + } + + private static long makeLong(int i0, int i1) { + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + return (i0 & 0xffffffffL) | + ((i1 & 0xffffffffL) << Integer.SIZE); + } else { + return ((i0 & 0xffffffffL) << Integer.SIZE) | + (i1 & 0xffffffffL); + } + } + + private static int makeInt(short i0, short i1) { + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + return (i0 & 0xffff) | + ((i1 & 0xffff) << Short.SIZE); + } else { + return ((i0 & 0xffff) << Short.SIZE) | + (i1 & 0xffff); + } + } + + private static int makeInt(byte i0, byte i1, byte i2, byte i3) { + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + return (i0 & 0xff) | + ((i1 & 0xff) << Byte.SIZE) | + ((i2 & 0xff) << Byte.SIZE * 2) | + ((i3 & 0xff) << Byte.SIZE * 3); + } else { + return ((i0 & 0xff) << Byte.SIZE * 3) | + ((i1 & 0xff) << Byte.SIZE * 2) | + ((i2 & 0xff) << Byte.SIZE) | + (i3 & 0xff); + } + } + + private static short makeShort(byte i0, byte i1) { + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + return (short) ((i0 & 0xff) | + (i1 & 0xff) << Byte.SIZE); + } else { + return (short) ((i0 & 0xff) << Byte.SIZE | + (i1 & 0xff)); + } + } }