8303215: Make thread stacks not use huge pages

src/hotspot/os/linux/os_linux.cpp

@@ -981,6 +981,15 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
   }
   assert(is_aligned(stack_size, os::vm_page_size()), "stack_size not aligned");
 
+  // Add an additional page to the stack size to reduce its chances of getting large page aligned
+  // so that the stack does not get backed by a transparent huge page.
+  size_t default_large_page_size = os::large_page_size();
+  if (default_large_page_size != 0 &&
+      stack_size >= default_large_page_size &&
+      is_aligned(stack_size, default_large_page_size)) {
+    stack_size += os::vm_page_size();
+  }
+
   int status = pthread_attr_setstacksize(&attr, stack_size);
   assert_status(status == 0, status, "pthread_attr_setstacksize");
 
@@ -4143,6 +4152,10 @@ bool os::Linux::setup_large_page_type(size_t page_size) {
 }
 
 void os::large_page_init() {
+  // Always initialize the default large page size even if large pages are not being used.
+  size_t large_page_size = Linux::setup_large_page_size();
+
+  // Handle the case where we do not want to use huge pages
   if (!UseLargePages &&
       !UseTransparentHugePages &&
       !UseHugeTLBFS &&
@@ -4160,7 +4173,6 @@ void os::large_page_init() {
     return;
   }
 
-  size_t large_page_size = Linux::setup_large_page_size();
   UseLargePages          = Linux::setup_large_page_type(large_page_size);
 
   set_coredump_filter(LARGEPAGES_BIT);

src/hotspot/os_cpu/linux_aarch64/globals_linux_aarch64.hpp

@@ -30,10 +30,18 @@
 // (see globals.hpp)
 
 define_pd_global(bool, DontYieldALot,            false);
-define_pd_global(intx, ThreadStackSize,          2048); // 0 => use system default
-define_pd_global(intx, VMThreadStackSize,        2048);
 
-define_pd_global(intx, CompilerThreadStackSize,  2048);
+// Set default stack sizes < 2MB so as to prevent stacks from getting
+// large-page aligned and backed by THPs on systems where 2MB is the
+// default huge page size. For non-JavaThreads, glibc may add an additional
+// guard page to the total stack size, so to keep the default sizes same
+// for all the following flags, we set them to 2 pages less than 2MB. On
+// systems where 2MB is the default large page size, 4KB is most commonly
+// the regular page size.
+define_pd_global(intx, ThreadStackSize,          2040); // 0 => use system default
+define_pd_global(intx, VMThreadStackSize,        2040);
+
+define_pd_global(intx, CompilerThreadStackSize,  2040);
 
 define_pd_global(uintx,JVMInvokeMethodSlack,     8192);