0001-ntdll-Use-kernel-soft-dirty-flags-for-write-watches-.patch

From af8ab05b7931136a920a28cb73074c25bca1def9 Mon Sep 17 00:00:00 2001
From: Paul Gofman <pgofman@codeweavers.com>
Date: Fri, 8 May 2020 14:32:09 +0300
Subject: [PATCH 0424/1562] ntdll: Use kernel soft dirty flags for write
 watches support.

Requires kernel patches to have effect.
---
 dlls/kernel32/tests/virtual.c |  54 +++++++++++-
 dlls/ntdll/unix/virtual.c     | 153 +++++++++++++++++++++++++++++++---
 2 files changed, 191 insertions(+), 16 deletions(-)

diff --git a/dlls/kernel32/tests/virtual.c b/dlls/kernel32/tests/virtual.c
index c54fb16ba15..445f519f409 100644
--- a/dlls/kernel32/tests/virtual.c
+++ b/dlls/kernel32/tests/virtual.c
@@ -2127,15 +2127,61 @@ static void test_write_watch(void)
     ok( count == 1, "wrong count %Iu\n", count );
     ok( results[0] == base + 5*pagesize, "wrong result %p\n", results[0] );
 
+    ret = pResetWriteWatch( base, size );
+    ok( !ret, "pResetWriteWatch failed %u\n", GetLastError() );
+
+    ret = VirtualProtect( base, 6*pagesize, PAGE_READWRITE, &old_prot );
+    ok( ret, "VirtualProtect failed error %u\n", GetLastError() );
+    ok( old_prot == PAGE_NOACCESS, "wrong old prot %x\n", old_prot );
+
+    base[3*pagesize + 200] = 3;
+    base[5*pagesize + 200] = 3;
+
     ret = VirtualFree( base, size, MEM_DECOMMIT );
     ok( ret, "VirtualFree failed %lu\n", GetLastError() );
 
     count = 64;
     ret = pGetWriteWatch( 0, base, size, results, &count, &pagesize );
-    ok( !ret, "GetWriteWatch failed %lu\n", GetLastError() );
-    ok( count == 1 || broken(count == 0), /* win98 */
-        "wrong count %Iu\n", count );
-    if (count) ok( results[0] == base + 5*pagesize, "wrong result %p\n", results[0] );
+    ok( !ret, "GetWriteWatch failed %u\n", GetLastError() );
+    ok( !count, "wrong count %lu\n", count );
+
+    base = VirtualAlloc( base, size, MEM_COMMIT, PAGE_READWRITE );
+    ok(!!base, "VirtualAlloc failed.\n");
+
+    count = 64;
+    ret = pGetWriteWatch( 0, base, size, results, &count, &pagesize );
+    ok( !ret, "GetWriteWatch failed %u\n", GetLastError() );
+    ok( !count, "wrong count %lu\n", count );
+
+    base[3*pagesize + 200] = 3;
+    ret = VirtualProtect( base, 6*pagesize, PAGE_READWRITE, &old_prot );
+    ok( ret, "VirtualProtect failed error %u\n", GetLastError() );
+    ok( old_prot == PAGE_READWRITE, "wrong old prot %x\n", old_prot );
+
+    base[5*pagesize + 200] = 3;
+    count = 64;
+    ret = pGetWriteWatch( 0, base, size, results, &count, &pagesize );
+    ok( !ret, "GetWriteWatch failed %u\n", GetLastError() );
+    ok( count == 2, "wrong count %lu\n", count );
+    ok( results[0] == base + 3*pagesize && results[1] == base + 5*pagesize, "wrong result %p\n", results[0] );
+
+    ret = VirtualFree( base, size, MEM_DECOMMIT );
+    ok( ret, "VirtualFree failed %u\n", GetLastError() );
+
+    count = 64;
+    ret = pGetWriteWatch( 0, base, size, results, &count, &pagesize );
+    ok( !ret, "GetWriteWatch failed %u\n", GetLastError() );
+    todo_wine ok( count == 1, "wrong count %lu\n", count );
+    ok( results[0] == base + 3*pagesize, "wrong result %p\n", results[0] );
+
+    base = VirtualAlloc( base, size, MEM_COMMIT, PAGE_READWRITE );
+    ok(!!base, "VirtualAlloc failed.\n");
+
+    count = 64;
+    ret = pGetWriteWatch( 0, base, size, results, &count, &pagesize );
+    ok( !ret, "GetWriteWatch failed %u\n", GetLastError() );
+    todo_wine ok( count == 1, "wrong count %lu\n", count );
+    ok( results[0] == base + 3*pagesize, "wrong result %p\n", results[0] );
 
     VirtualFree( base, 0, MEM_RELEASE );
 }
diff --git a/dlls/ntdll/unix/virtual.c b/dlls/ntdll/unix/virtual.c
index 636a699303a..aab67d9b08f 100644
--- a/dlls/ntdll/unix/virtual.c
+++ b/dlls/ntdll/unix/virtual.c
@@ -214,6 +214,13 @@ static BYTE **pages_vprot;
 static BYTE *pages_vprot;
 #endif
 
+static BOOL use_kernel_writewatch;
+static int pagemap_fd, pagemap_reset_fd, clear_refs_fd;
+#define PAGE_FLAGS_BUFFER_LENGTH 1024
+#define PM_SOFT_DIRTY_PAGE (1ull << 57)
+
+static void reset_write_watches( void *base, SIZE_T size );
+
 static struct file_view *view_block_start, *view_block_end, *next_free_view;
 #ifdef _WIN64
 static const size_t view_block_size = 0x200000;
@@ -1163,7 +1170,7 @@ static int get_unix_prot( BYTE vprot )
         if (vprot & VPROT_WRITE) prot |= PROT_WRITE | PROT_READ;
         if (vprot & VPROT_WRITECOPY) prot |= PROT_WRITE | PROT_READ;
         if (vprot & VPROT_EXEC) prot |= PROT_EXEC | PROT_READ;
-        if (vprot & VPROT_WRITEWATCH) prot &= ~PROT_WRITE;
+        if (vprot & VPROT_WRITEWATCH && !use_kernel_writewatch) prot &= ~PROT_WRITE;
     }
     if (!prot) prot = PROT_NONE;
     return prot;
@@ -1539,6 +1546,13 @@ static NTSTATUS create_view( struct file_view **view_ret, void *base, size_t siz
         TRACE( "forcing exec permission on %p-%p\n", base, (char *)base + size - 1 );
         mprotect( base, size, unix_prot | PROT_EXEC );
     }
+
+    if (vprot & VPROT_WRITEWATCH && use_kernel_writewatch)
+    {
+        madvise( view->base, view->size, MADV_NOHUGEPAGE );
+        reset_write_watches( view->base, view->size );
+    }
+
     return STATUS_SUCCESS;
 }
 
@@ -1656,7 +1670,7 @@ static void mprotect_range( void *base, size_t size, BYTE set, BYTE clear )
  */
 static BOOL set_vprot( struct file_view *view, void *base, size_t size, BYTE vprot )
 {
-    if (view->protect & VPROT_WRITEWATCH)
+    if (!use_kernel_writewatch && view->protect & VPROT_WRITEWATCH)
     {
         /* each page may need different protections depending on write watch flag */
         set_page_vprot_bits( base, size, vprot & ~VPROT_WRITEWATCH, ~vprot & ~VPROT_WRITEWATCH );
@@ -1734,8 +1748,24 @@ static void update_write_watches( void *base, size_t size, size_t accessed_size
  */
 static void reset_write_watches( void *base, SIZE_T size )
 {
-    set_page_vprot_bits( base, size, VPROT_WRITEWATCH, 0 );
-    mprotect_range( base, size, 0, 0 );
+    if (use_kernel_writewatch)
+    {
+        char buffer[17];
+        ssize_t ret;
+
+        memset(buffer, 0, sizeof(buffer));
+        buffer[0] = '6';
+        *(void **)&buffer[1] = base;
+        *(void **)&buffer[1 + 8] = (char *)base + size;
+
+        if ((ret = write(clear_refs_fd, buffer, sizeof(buffer))) != sizeof(buffer))
+            ERR("Could not clear soft dirty bits, ret %zd, error %s.\n", ret, strerror(errno));
+    }
+    else
+    {
+        set_page_vprot_bits( base, size, VPROT_WRITEWATCH, 0 );
+        mprotect_range( base, size, 0, 0 );
+    }
 }
 
 
@@ -2068,7 +2098,11 @@ static NTSTATUS map_view( struct file_view **view_ret, void *base, size_t size,
 
         view->protect = vprot | VPROT_PLACEHOLDER;
         set_vprot( view, base, size, vprot );
-        if (vprot & VPROT_WRITEWATCH) reset_write_watches( base, size );
+        if (vprot & VPROT_WRITEWATCH)
+        {
+            madvise( base, size, MADV_NOHUGEPAGE );
+            reset_write_watches( base, size );
+        }
         *view_ret = view;
         return STATUS_SUCCESS;
     }
@@ -2247,6 +2281,9 @@ static NTSTATUS decommit_pages( struct file_view *view, size_t start, size_t siz
     if (!size) size = view->size;
     if (anon_mmap_fixed( (char *)view->base + start, size, PROT_NONE, 0 ) != MAP_FAILED)
     {
+        if (use_kernel_writewatch && view->protect & VPROT_WRITEWATCH)
+            madvise( view->base, view->size, MADV_NOHUGEPAGE );
+
         set_page_vprot_bits( (char *)view->base + start, size, 0, VPROT_COMMITTED );
         return STATUS_SUCCESS;
     }
@@ -3353,6 +3390,7 @@ void virtual_init(void)
     size_t size;
     int i;
     pthread_mutexattr_t attr;
+    const char *env_var;
 
     pthread_mutexattr_init( &attr );
     pthread_mutexattr_settype( &attr, PTHREAD_MUTEX_RECURSIVE );
@@ -3366,6 +3404,24 @@ void virtual_init(void)
     host_addr_space_limit = address_space_limit;
 #endif
 
+    if (!((env_var = getenv("WINE_DISABLE_KERNEL_WRITEWATCH")) && atoi(env_var))
+            && (pagemap_reset_fd = open("/proc/self/pagemap_reset", O_RDONLY | O_CLOEXEC)) != -1)
+    {
+        use_kernel_writewatch = TRUE;
+        if ((pagemap_fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC)) == -1)
+        {
+            ERR("Could not open pagemap file, error %s.\n", strerror(errno));
+            exit(-1);
+        }
+        if ((clear_refs_fd = open("/proc/self/clear_refs", O_WRONLY | O_CLOEXEC)) == -1)
+        {
+            ERR("Could not open clear_refs file, error %s.\n", strerror(errno));
+            exit(-1);
+        }
+        if (ERR_ON(virtual))
+            MESSAGE("wine: using kernel write watches (experimental).\n");
+    }
+
     if (preload_info && *preload_info)
         for (i = 0; (*preload_info)[i].size; i++)
             mmap_add_reserved_area( (*preload_info)[i].addr, (*preload_info)[i].size );
@@ -4126,7 +4182,7 @@ NTSTATUS virtual_handle_fault( EXCEPTION_RECORD *rec, void *stack )
         }
         else ret = grow_thread_stack( page, &stack_info );
     }
-    else if (err & EXCEPTION_WRITE_FAULT)
+    else if (!use_kernel_writewatch && err & EXCEPTION_WRITE_FAULT)
     {
         if (vprot & VPROT_WRITEWATCH)
         {
@@ -4220,11 +4276,11 @@ static NTSTATUS check_write_access( void *base, size_t size, BOOL *has_write_wat
     for (i = 0; i < size; i += page_size)
     {
         BYTE vprot = get_page_vprot( addr + i );
-        if (vprot & VPROT_WRITEWATCH) *has_write_watch = TRUE;
+        if (!use_kernel_writewatch && vprot & VPROT_WRITEWATCH) *has_write_watch = TRUE;
         if (!(get_unix_prot( vprot & ~VPROT_WRITEWATCH ) & PROT_WRITE))
             return STATUS_INVALID_USER_BUFFER;
     }
-    if (*has_write_watch)
+    if (!use_kernel_writewatch && *has_write_watch)
         mprotect_range( addr, size, 0, VPROT_WRITEWATCH );  /* temporarily enable write access */
     return STATUS_SUCCESS;
 }
@@ -6134,17 +6190,90 @@ NTSTATUS WINAPI NtGetWriteWatch( HANDLE process, ULONG flags, PVOID base, SIZE_T
         char *addr = base;
         char *end = addr + size;
 
-        while (pos < *count && addr < end)
+        if (use_kernel_writewatch)
         {
-            if (!(get_page_vprot( addr ) & VPROT_WRITEWATCH)) addresses[pos++] = addr;
-            addr += page_size;
+            static UINT64 buffer[PAGE_FLAGS_BUFFER_LENGTH];
+            unsigned int i, length;
+            ssize_t read_length;
+
+            if (flags & WRITE_WATCH_FLAG_RESET)
+            {
+                if (is_win64)
+                {
+                    addresses[0] = end;
+                    if ((read_length = pread(pagemap_reset_fd, addresses, *count * sizeof(*addresses),
+                            ((ULONG_PTR)addr >> page_shift) * sizeof(*addresses))) == -1)
+                    {
+                        ERR("Error reading page flags, read_length %zd, error %s.\n", read_length, strerror(errno));
+                        status = STATUS_INVALID_ADDRESS;
+                        goto done;
+                    }
+                    *count = read_length / sizeof(*addresses);
+                    *granularity = page_size;
+                    goto done;
+                }
+
+                while (pos < *count && addr < end)
+                {
+                    length = min(PAGE_FLAGS_BUFFER_LENGTH, *count - pos);
+
+                    buffer[0] = (ULONG_PTR)end;
+                    if ((read_length = pread(pagemap_reset_fd, buffer, length * sizeof(*buffer),
+                            ((ULONG_PTR)addr >> page_shift) * sizeof(*buffer))) == -1)
+                    {
+                        ERR("Error reading page flags, read_length %zd, error %s.\n", read_length, strerror(errno));
+                        status = STATUS_INVALID_ADDRESS;
+                        goto done;
+                    }
+                    read_length /= sizeof(*buffer);
+                    for (i = 0; i < read_length; ++i)
+                    {
+                        assert(pos < *count);
+                        addresses[pos++] = (void *)(ULONG_PTR)buffer[i];
+                    }
+                    if (read_length < length)
+                        break;
+                    addr = (char *)(ULONG_PTR)buffer[read_length - 1] + page_size;
+                }
+            }
+            else
+            {
+                while (pos < *count && addr < end)
+                {
+                    length = min(PAGE_FLAGS_BUFFER_LENGTH, (end - addr) >> page_shift);
+
+                    if ((read_length = pread(pagemap_fd, buffer, length * sizeof(*buffer),
+                            ((ULONG_PTR)addr >> page_shift) * sizeof(*buffer))) != length * sizeof(*buffer))
+                    {
+                        ERR("Error reading page flags, read_length %zd, error %s.\n", read_length, strerror(errno));
+                        status = STATUS_INVALID_ADDRESS;
+                        goto done;
+                    }
+                    for (i = 0; i < length && pos < *count; ++i)
+                    {
+                        if (buffer[i] & PM_SOFT_DIRTY_PAGE)
+                            addresses[pos++] = addr;
+
+                        addr += page_size;
+                    }
+                }
+            }
+        }
+        else
+        {
+            while (pos < *count && addr < end)
+            {
+                if (!(get_page_vprot( addr ) & VPROT_WRITEWATCH)) addresses[pos++] = addr;
+                addr += page_size;
+            }
+            if (flags & WRITE_WATCH_FLAG_RESET) reset_write_watches( base, addr - (char *)base );
         }
-        if (flags & WRITE_WATCH_FLAG_RESET) reset_write_watches( base, addr - (char *)base );
         *count = pos;
         *granularity = page_size;
     }
     else status = STATUS_INVALID_PARAMETER;
 
+done:
     server_leave_uninterrupted_section( &virtual_mutex, &sigset );
     return status;
 }