Fix mark the staging file sparse on Windows

crates/hydra-cli/src/download.rs

@@ -439,29 +439,13 @@ fn targets_for(
 /// Proxy setting for a URL, honouring `--no-proxy` semantics via the environment.
 /// How many bytes of `path` are really there.
 ///
-/// A resume record is authoritative when present. Otherwise fall back to allocated
-/// blocks, which for a sparse file is far smaller than its apparent length. On a
-/// filesystem that does not report blocks, treat a file whose apparent length equals
-/// its allocation as fully present.
+/// A resume record is authoritative when present. Otherwise fall back to the
+/// allocation, which for a sparse file is far smaller than its apparent length.
 fn bytes_present(path: &Path, sidecar: Option<&Sidecar>) -> u64 {
     if let Some(sc) = sidecar {
         return sc.bytes_done();
     }
-    let Ok(md) = std::fs::metadata(path) else {
-        return 0;
-    };
-    #[cfg(unix)]
-    {
-        use std::os::unix::fs::MetadataExt;
-        let allocated = md.blocks() * 512;
-        // Allocation is rounded up to a block, so it can exceed the apparent length by
-        // less than one block; clamp rather than report more bytes than the file has.
-        allocated.min(md.len())
-    }
-    #[cfg(not(unix))]
-    {
-        md.len()
-    }
+    hya_net::allocated_bytes(path).unwrap_or(0)
 }
 
 /// An Outcome with every field at a neutral value, for tests that care about two of them.

crates/hydra-gui/src/app.rs

@@ -9562,8 +9562,8 @@ pub fn sanitize_spans(spans: &[(u64, u64)], size: Option<u64>) -> Vec<(u64, u64)
 ///
 /// Length must match the recorded size exactly — `SparseSink::create` calls
 /// `set_len(size)` up front, so any other length means another object or a
-/// truncation. On unix the allocated blocks are checked against the bytes
-/// the spans claim: holes are not allocated, so a `.part` that received one
+/// truncation. The allocated bytes are checked against the bytes the spans
+/// claim: holes are not allocated, so a `.part` that received one
 /// byte cannot pass for one that received everything (file length says
 /// nothing about completeness — the lesson the CLI already recorded).
 pub fn part_matches(part: &std::path::Path, size: Option<u64>, held: &[(u64, u64)]) -> bool {
@@ -9578,40 +9578,13 @@ pub fn part_matches(part: &std::path::Path, size: Option<u64>, held: &[(u64, u64
             return false;
         }
     }
-    #[cfg(unix)]
-    {
-        use std::os::unix::fs::MetadataExt;
-        let allocated = meta.blocks().saturating_mul(512);
-        let claimed = sum_spans(held);
-        // Filesystems account allocation in blocks; allow one block of
-        // rounding at each end of every span.
-        let slop = (held.len() as u64 + 1).saturating_mul(2 * 4096);
-        if allocated.saturating_add(slop) < claimed {
-            return false;
-        }
-    }
-    #[cfg(target_os = "windows")]
-    {
-        use std::io::Read;
-        // On Windows, detect sparse files by checking if we can read actual
-        // data from the claimed parts. A file created with set_len but no
-        // actual data written will fail this check.
-        if !held.is_empty() {
-            if let Ok(mut file) = std::fs::File::open(part) {
-                let mut buf = vec![0u8; 4096.min(meta.len() as usize)];
-                // Try to read from the first claimed part
-                if let Ok(n) = file.read(&mut buf) {
-                    if n == 0 {
-                        // File is sparse: no data could be read
-                        return false;
-                    }
-                }
-            }
-        }
-    }
-    #[cfg(not(any(unix, target_os = "windows")))]
-    let _ = held;
-    true
+    let Ok(allocated) = hya_net::allocated_bytes(part) else {
+        return false;
+    };
+    // Filesystems account allocation in blocks; allow one block of rounding
+    // at each end of every span.
+    let slop = (held.len() as u64 + 1).saturating_mul(2 * 4096);
+    allocated.saturating_add(slop) >= sum_spans(held)
 }
 
 /// What Escape means in window `kind`: its Cancel button, or nothing for
@@ -12346,14 +12319,20 @@ mod tests {
         // Missing file never resumes.
         assert!(!part_matches(&path, Some(1000), &[(0, 1000)]));
 
-        // Sparse file: set_len only, no byte written. Length matches, but
-        // nothing is allocated — claiming the whole object must fail.
-        let f = std::fs::File::create(&path).unwrap();
-        f.set_len(1_000_000).unwrap();
-        drop(f);
-        assert!(!part_matches(&path, Some(1_000_000), &[(0, 1_000_000)]));
+        // A staging file as the engine makes it, no byte written. Length
+        // matches, but nothing is allocated — claiming the whole object must fail.
+        const SIZE: u64 = 64 << 20;
+        const HEAD: u64 = 1 << 20;
+        let sink = hya_net::SparseSink::create(&path.to_string_lossy(), SIZE).unwrap();
+        assert!(!part_matches(&path, Some(SIZE), &[(0, SIZE)]));
         // ...while claiming nothing (fresh start) is fine.
-        assert!(part_matches(&path, Some(1_000_000), &[]));
+        assert!(part_matches(&path, Some(SIZE), &[]));
+
+        // Only the head arrived: its claim holds, the whole object's does not.
+        sink.write_at(0, &vec![0xCD; HEAD as usize]).unwrap();
+        drop(sink);
+        assert!(part_matches(&path, Some(SIZE), &[(0, HEAD)]));
+        assert!(!part_matches(&path, Some(SIZE), &[(0, SIZE)]));
 
         // Fully written file passes.
         std::fs::write(&path, vec![0xAB; 1_000_000]).unwrap();

crates/hydra-net/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hya-net"
-version = "0.6.3"
+version = "0.6.4"
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
@@ -36,7 +36,15 @@ serde_json = { workspace = true }
 # Chromium on Windows wraps its master key with DPAPI, whose only interface is
 # `CryptUnprotectData`. One function, declared by the same crate the rest of the
 # workspace already builds against rather than by a hand-written extern block.
-windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security_Cryptography"] }
+# The staging file needs FSCTL_SET_SPARSE and GetCompressedFileSizeW too: NTFS
+# zero-fills a non-sparse file up to every far-offset write.
+windows-sys = { version = "0.61", features = [
+    "Win32_Foundation",
+    "Win32_Security_Cryptography",
+    "Win32_Storage_FileSystem",
+    "Win32_System_IO",
+    "Win32_System_Ioctl",
+] }
 
 [dev-dependencies]
 tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "io-util", "time", "fs", "sync"] }

crates/hydra-net/src/lib.rs

@@ -566,7 +566,7 @@ pub use http::{
     probe_size_via_range, probe_via_get, Probe, Redirect,
 };
 pub use redirect::{html_redirect, html_redirect_target};
-pub use sink::SparseSink;
+pub use sink::{allocated_bytes, SparseSink};
 pub use transfer::{
     run_transfer, run_transfer_cancellable, run_transfer_into, run_transfer_observed,
     run_transfer_paced, run_transfer_tick, run_transfer_with_reserves, Bench, OnSubstitute,

crates/hydra-net/src/sink.rs

@@ -42,6 +42,7 @@ impl SparseSink {
             .write(true)
             .read(true)
             .open(path)?;
+        mark_sparse(&file);
         file.set_len(size)?;
         Ok(SparseSink {
             file: Some(file),
@@ -187,6 +188,78 @@ impl SparseSink {
     }
 }
 
+/// Ask NTFS for holes. Without the sparse flag, `set_len` only moves the end of
+/// file: the first write far into it makes the kernel zero-fill every byte below,
+/// synchronously inside that `WriteFile` — gigabytes of disk writes that block
+/// the engine and cannot be cancelled. Best effort: FAT32 and exFAT have no
+/// sparse files, and there is nothing cheaper to fall back to.
+#[cfg(windows)]
+fn mark_sparse(file: &std::fs::File) {
+    use std::os::windows::io::AsRawHandle;
+    use windows_sys::Win32::System::{Ioctl::FSCTL_SET_SPARSE, IO::DeviceIoControl};
+    let mut returned = 0u32;
+    // SAFETY: the handle is open for writing and outlives the call; a null input
+    // buffer means "set sparse", and FSCTL_SET_SPARSE has no output buffer.
+    unsafe {
+        DeviceIoControl(
+            file.as_raw_handle(),
+            FSCTL_SET_SPARSE,
+            std::ptr::null(),
+            0,
+            std::ptr::null_mut(),
+            0,
+            &mut returned,
+            std::ptr::null_mut(),
+        );
+    }
+}
+
+#[cfg(not(windows))]
+fn mark_sparse(_file: &std::fs::File) {}
+
+/// Bytes the filesystem has actually allocated for `path`, capped at its length.
+///
+/// The staging file is full-length from its first byte, so its length says nothing
+/// about what arrived; for a sparse file the allocation does. On a filesystem without
+/// holes this is the full length.
+pub fn allocated_bytes(path: &std::path::Path) -> io::Result<u64> {
+    let meta = std::fs::metadata(path)?;
+    Ok(raw_allocation(path, &meta)?.min(meta.len()))
+}
+
+#[cfg(unix)]
+fn raw_allocation(_path: &std::path::Path, meta: &std::fs::Metadata) -> io::Result<u64> {
+    use std::os::unix::fs::MetadataExt;
+    Ok(meta.blocks().saturating_mul(512))
+}
+
+#[cfg(windows)]
+fn raw_allocation(path: &std::path::Path, _meta: &std::fs::Metadata) -> io::Result<u64> {
+    use std::os::windows::ffi::OsStrExt;
+    use windows_sys::Win32::Foundation::{GetLastError, SetLastError, NO_ERROR};
+    use windows_sys::Win32::Storage::FileSystem::{GetCompressedFileSizeW, INVALID_FILE_SIZE};
+    let wide: Vec<u16> = path.as_os_str().encode_wide().chain([0]).collect();
+    let mut high = 0u32;
+    // INVALID_FILE_SIZE is also a legitimate low word; only the last error tells the
+    // two apart, so clear it first.
+    // SAFETY: `wide` is NUL-terminated and outlives the call; `high` is a valid
+    // out-pointer; the error calls have no preconditions.
+    let (low, err) = unsafe {
+        SetLastError(NO_ERROR);
+        let low = GetCompressedFileSizeW(wide.as_ptr(), &mut high);
+        (low, GetLastError())
+    };
+    if low == INVALID_FILE_SIZE && err != NO_ERROR {
+        return Err(io::Error::from_raw_os_error(err as i32));
+    }
+    Ok((u64::from(high) << 32) | u64::from(low))
+}
+
+#[cfg(not(any(unix, windows)))]
+fn raw_allocation(_path: &std::path::Path, meta: &std::fs::Metadata) -> io::Result<u64> {
+    Ok(meta.len())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -257,4 +330,54 @@ mod tests {
         );
         let _ = std::fs::remove_file(&p);
     }
+
+    fn scratch(tag: &str) -> std::path::PathBuf {
+        std::env::temp_dir().join(format!("hydra_sink_{tag}_{}", std::process::id()))
+    }
+
+    #[test]
+    fn a_far_write_allocates_only_the_bytes_written() {
+        // Issue #414: on NTFS without the sparse flag, this one 64 KiB write near the
+        // end made the kernel write 256 MiB of zeros below it before returning.
+        let path = scratch("far_write");
+        let _ = std::fs::remove_file(&path);
+        const SIZE: u64 = 256 << 20;
+        let sink = SparseSink::create(&path.to_string_lossy(), SIZE).unwrap();
+        sink.write_at(SIZE - (64 << 10), &[0x5A; 64 << 10]).unwrap();
+        drop(sink);
+
+        let allocated = allocated_bytes(&path).unwrap();
+        assert_eq!(std::fs::metadata(&path).unwrap().len(), SIZE);
+        assert!(
+            allocated < 16 << 20,
+            "a staging file must stay sparse below its first write, got {allocated} bytes allocated"
+        );
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn a_fully_written_file_reports_its_whole_length_allocated() {
+        let path = scratch("full");
+        std::fs::write(&path, vec![0x11; 3 * 4096 + 5]).unwrap();
+        assert_eq!(allocated_bytes(&path).unwrap(), 3 * 4096 + 5);
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn allocation_of_a_missing_file_is_an_error() {
+        assert!(allocated_bytes(&scratch("missing")).is_err());
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn the_staging_file_carries_the_sparse_attribute() {
+        use std::os::windows::fs::MetadataExt;
+        use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_SPARSE_FILE;
+        let path = scratch("attr");
+        let _ = std::fs::remove_file(&path);
+        drop(SparseSink::create(&path.to_string_lossy(), 1 << 20).unwrap());
+        let attrs = std::fs::metadata(&path).unwrap().file_attributes();
+        assert_ne!(attrs & FILE_ATTRIBUTE_SPARSE_FILE, 0);
+        let _ = std::fs::remove_file(&path);
+    }
 }