diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 5f08aa3a..aadc3b23 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -13,6 +13,8 @@ ignore = [ "RUSTSEC-2024-0436", "RUSTSEC-2026-0206", "RUSTSEC-2026-0192", + # pin-utils via wasmtimer under iced_futures 0.14. + "RUSTSEC-2026-0441", # Unmaintained, no known vulnerability: zbus_names, zvariant and # zvariant_derive via zbus 5.19 (latest) under ksni, dark-light, mundy # and system_shutdown. diff --git a/Cargo.lock b/Cargo.lock index fe3deaeb..3d461697 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,6 +288,45 @@ dependencies = [ "libloading", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -314,9 +353,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +checksum = "f426a31126c735126297bc5ea7a521263c9faff4f63ca415267ee625651c4f37" dependencies = [ "async-task", "concurrent-queue", @@ -339,7 +378,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 1.1.5", + "rustix 1.1.6", "slab", "windows-sys 0.61.2", ] @@ -370,7 +409,7 @@ dependencies = [ "cfg-if", "event-listener", "futures-lite", - "rustix 1.1.5", + "rustix 1.1.6", ] [[package]] @@ -396,7 +435,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 1.1.5", + "rustix 1.1.6", "signal-hook-registry", "slab", "windows-sys 0.61.2", @@ -460,7 +499,7 @@ dependencies = [ "anyhow", "arrayvec", "log", - "nom", + "nom 8.0.0", "num-rational", "v_frame", ] @@ -480,13 +519,19 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bit-set" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -495,6 +540,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bit_field" version = "0.10.3" @@ -671,7 +725,7 @@ checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" dependencies = [ "bitflags 2.13.2", "polling", - "rustix 1.1.5", + "rustix 1.1.6", "slab", "tracing", ] @@ -695,7 +749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" dependencies = [ "calloop 0.14.4", - "rustix 1.1.5", + "rustix 1.1.6", "wayland-backend", "wayland-client", ] @@ -1217,7 +1271,7 @@ dependencies = [ "document-features", "mio", "parking_lot", - "rustix 1.1.5", + "rustix 1.1.6", "signal-hook", "signal-hook-mio", "winapi", @@ -1332,12 +1386,38 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "data-url" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + [[package]] name = "derive_more" version = "2.1.1" @@ -1401,7 +1481,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1584,7 +1664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1946,7 +2026,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix 1.1.5", + "rustix 1.1.6", "windows-link", ] @@ -2334,6 +2414,7 @@ dependencies = [ "hya-core", "memchr", "pbkdf2 0.13.0", + "rcgen", "reed-solomon-simd", "rustls", "rustls-pki-types", @@ -2387,7 +2468,7 @@ name = "hya-plugin-cli" version = "1.2.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "clap", "embed-resource", "hya-plugin", @@ -2434,6 +2515,7 @@ dependencies = [ "sha2 0.11.0", "tar", "tokio", + "winreg 0.56.0", "zip", ] @@ -3265,6 +3347,12 @@ dependencies = [ "paste", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "minisign" version = "0.9.1" @@ -3464,6 +3552,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -3499,6 +3597,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-derive" version = "0.4.2" @@ -4021,6 +4125,15 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -4170,6 +4283,16 @@ dependencies = [ "hmac 0.13.0", ] +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -4273,7 +4396,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix 1.1.5", + "rustix 1.1.6", "windows-sys 0.61.2", ] @@ -4318,6 +4441,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -4377,7 +4506,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", - "bit-vec", + "bit-vec 0.8.0", "bitflags 2.13.2", "num-traits", "rand", @@ -4604,6 +4733,20 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "read-fonts" version = "0.35.0" @@ -4859,6 +5002,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "rustix" version = "0.38.44" @@ -4874,15 +5026,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5289,7 +5441,7 @@ dependencies = [ "libc", "log", "memmap2", - "rustix 1.1.5", + "rustix 1.1.6", "thiserror 2.0.21", "wayland-backend", "wayland-client", @@ -5352,7 +5504,7 @@ dependencies = [ "objc2-quartz-core 0.3.2", "raw-window-handle", "redox_syscall 0.5.18", - "rustix 1.1.5", + "rustix 1.1.6", "tiny-xlib", "tracing", "wasm-bindgen", @@ -5566,6 +5718,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "synstructure" version = "0.14.0" @@ -5625,8 +5788,8 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix 1.1.5", - "windows-sys 0.61.2", + "rustix 1.1.6", + "windows-sys 0.52.0", ] [[package]] @@ -5692,6 +5855,36 @@ dependencies = [ "zune-jpeg 0.5.15", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-skia" version = "0.11.4" @@ -6069,7 +6262,7 @@ version = "0.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80be9b06fbae3b8b303400ab20778c80bbaf338f563afe567cf3c9eea17b47ef" dependencies = [ - "base64", + "base64 0.22.1", "data-url", "flate2", "fontdb", @@ -6356,7 +6549,7 @@ checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078" dependencies = [ "cc", "downcast-rs", - "rustix 1.1.5", + "rustix 1.1.6", "scoped-tls", "smallvec", "wayland-sys", @@ -6369,7 +6562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" dependencies = [ "bitflags 2.13.2", - "rustix 1.1.5", + "rustix 1.1.6", "wayland-backend", "wayland-scanner", ] @@ -6391,7 +6584,7 @@ version = "0.31.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a52d18780be9b1314328a3de5f930b73d2200112e3849ca6cb11822793fb34d" dependencies = [ - "rustix 1.1.5", + "rustix 1.1.6", "wayland-client", "xcursor", ] @@ -6555,7 +6748,7 @@ checksum = "27a75de515543b1897b26119f93731b385a19aea165a1ec5f0e3acecc229cae7" dependencies = [ "arrayvec", "bit-set", - "bit-vec", + "bit-vec 0.8.0", "bitflags 2.13.2", "bytemuck", "cfg_aliases", @@ -6697,7 +6890,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7138,7 +7331,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d6f32a0ff4a9f6f01231eb2059cc85479330739333e0e58cadf03b6af2cca10" dependencies = [ "cfg-if", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7175,7 +7368,7 @@ dependencies = [ "libc", "libloading", "once_cell", - "rustix 1.1.5", + "rustix 1.1.6", "x11rb-protocol 0.13.2", ] @@ -7186,7 +7379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a8885a854a8bfdf87a301e53e41b17c5f8f33639903131338b997b1eb614f44" dependencies = [ "gethostname", - "rustix 1.1.5", + "rustix 1.1.6", "x11rb-protocol 0.14.0", ] @@ -7202,6 +7395,24 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acf4d1bc32aa46eec18caa634ec3cf4c05bfa151f12b93b510b15190f69a1ca8" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + [[package]] name = "xattr" version = "1.6.1" @@ -7209,7 +7420,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix 1.1.5", + "rustix 1.1.6", ] [[package]] @@ -7266,6 +7477,16 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yazi" version = "0.2.1" @@ -7292,7 +7513,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "synstructure", + "synstructure 0.14.0", ] [[package]] @@ -7317,7 +7538,7 @@ dependencies = [ "hex", "libc", "ordered-stream", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_repr", "tracing", @@ -7409,7 +7630,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "synstructure", + "synstructure 0.14.0", ] [[package]] diff --git a/crates/hydra-cli/src/queue.rs b/crates/hydra-cli/src/queue.rs index d40c2fd8..7f53c2fd 100644 --- a/crates/hydra-cli/src/queue.rs +++ b/crates/hydra-cli/src/queue.rs @@ -236,6 +236,21 @@ impl Queue { } } + /// Put a running item back in line because its session is ending, not + /// because anyone asked it to stop: the next session starts it first, + /// ahead of the items that were still waiting behind it. The interrupted + /// run is not a failed attempt. + pub fn interrupt(&mut self, id: u64) { + if let Some(i) = self.get_mut(id) { + if i.state == State::Running { + i.state = State::Queued; + i.rate = 0.0; + i.owner_pid = None; + i.attempts = i.attempts.saturating_sub(1); + } + } + } + /// Resume a paused or failed item, clearing its attempt count so an /// explicit user action is not throttled by earlier automatic retries. pub fn resume(&mut self, id: u64) { @@ -692,6 +707,47 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// #396: quitting mid-transfer put the item on hold, so the next session + /// skipped it and started the one queued behind it. + #[test] + fn a_quit_mid_transfer_resumes_that_item_first_in_the_next_session() { + let dir = std::env::temp_dir().join(format!("hydra_q_quit_{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("queue.json"); + let mut q = Queue::new(1); + for n in 1..=5 { + q.add(vec![format!("http://a/{n}")], format!("{n}.bin").into()); + } + for id in [1, 2] { + q.mark_running(id); + q.finish(id, None, None); + } + assert_eq!(q.to_start(), vec![3]); + q.mark_running(3); + q.progress(3, 4096, Some(8192), 1.0e6); + + q.interrupt(3); + q.save(&path).unwrap(); + let back = Queue::load(&path).unwrap().unwrap(); + + let item = back.get(3).unwrap(); + assert_eq!(item.state, State::Queued); + assert_eq!(item.done_bytes, 4096, "the bytes already held are kept"); + assert_eq!(item.attempts, 0, "an interrupted run is not a failure"); + assert_eq!(item.owner_pid, None); + assert_eq!(back.to_start(), vec![3], "#3 resumes; #4 keeps waiting"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn interrupt_leaves_a_user_pause_alone() { + let mut q = q3(); + q.mark_running(1); + q.pause(1); + q.interrupt(1); + assert_eq!(q.get(1).unwrap().state, State::Paused); + } + #[test] fn a_reloaded_queue_demotes_running_items_only_when_the_owner_is_dead() { // Two cases that must NOT be conflated. A crash leaves Running items whose owner diff --git a/crates/hydra-cli/src/tui.rs b/crates/hydra-cli/src/tui.rs index 52e9142e..e1f930f5 100644 --- a/crates/hydra-cli/src/tui.rs +++ b/crates/hydra-cli/src/tui.rs @@ -1421,14 +1421,15 @@ async fn run_interactive( return Ok(0); } - // Anything still running is recorded as paused, not lost: its bytes and - // sidecar are on disk and `-c` or a later session picks them up. + // Anything still running goes back in line, not on hold: its bytes and + // sidecar are on disk and the next session resumes it before anything + // that was waiting behind it. Pausing here read as a user's pause. for stop in stops.values() { stop.store(true, Ordering::Relaxed); } for (id, h) in running.drain() { let _ = h.await; - q.pause(id); + q.interrupt(id); } let _ = q.save(&queue_path); let (_, _, _, failed) = q.counts(); diff --git a/crates/hydra-gui/src/app.rs b/crates/hydra-gui/src/app.rs index 2773f325..24db5d92 100644 --- a/crates/hydra-gui/src/app.rs +++ b/crates/hydra-gui/src/app.rs @@ -3639,6 +3639,7 @@ impl App { if let Some(d) = self.item(id) { if d.state != DlState::Complete { let _ = std::fs::remove_file(d.part_file()); + let _ = std::fs::remove_dir_all(d.track_dir()); } else if remove_file { let _ = std::fs::remove_file(d.full_path()); } @@ -9378,6 +9379,7 @@ fn reroute(d: &DownloadItem, fi: &FileInfoState) -> (ProxyChoice, bool) { /// owns its `.part`. fn reset_for_redownload(d: &mut DownloadItem) { let _ = std::fs::remove_file(d.part_file()); + let _ = std::fs::remove_dir_all(d.track_dir()); d.held.clear(); d.downloaded = 0; d.disp_progress = 0.0; diff --git a/crates/hydra-gui/src/engine.rs b/crates/hydra-gui/src/engine.rs index 8bfcfbe8..c3f218c2 100644 --- a/crates/hydra-gui/src/engine.rs +++ b/crates/hydra-gui/src/engine.rs @@ -694,14 +694,19 @@ async fn resolve_link( // to them. let requested = url.clone(); let first = target_via(route.http(), &parse_url(&url).ok()?, Vec::new(), user_agent); + let mut headers = headers.to_vec(); for _ in 0..10 { let u = parse_url(&url).ok()?; let t = target_via(route.http(), &u, Vec::new(), user_agent).with_headers_from( &first, - headers.to_vec(), + headers.clone(), Some(user_agent.to_string()), ); let p = probe_resilient(connector, &t).await.ok()?; + if let Some(referer) = referer_fallback(&p, &u, &headers) { + headers.push(format!("Referer: {referer}")); + continue; + } if p.is_redirect() { url = join_url(&u, p.location.as_deref().unwrap_or(""))?; continue; @@ -1958,6 +1963,31 @@ struct Primary { delta: f64, } +/// The `Referer` to ask again with when `p` is a `403` to a request that +/// carried none: the object's own origin, as a link on its own site would +/// send. Hotlink-protected hosts (erome's CDN) refuse a pasted link outright +/// and serve it to any same-site referer. +fn referer_fallback(p: &Probe, u: &ParsedUrl, headers: &[String]) -> Option { + let has_referer = headers + .iter() + .any(|h| h.to_ascii_lowercase().starts_with("referer:")); + if p.status != 403 || has_referer || u.ftp { + return None; + } + let scheme = if u.tls { "https" } else { "http" }; + let default_port = if u.tls { 443 } else { 80 }; + let host = if u.host.contains(':') && !u.host.starts_with('[') { + format!("[{}]", u.host) + } else { + u.host.clone() + }; + Some(if u.port == default_port { + format!("{scheme}://{host}/") + } else { + format!("{scheme}://{host}:{}/", u.port) + }) +} + /// Probe `spec.url`, following header and HTML redirects and falling forward /// through the mirror list when a lead mirror cannot be reached at all. A /// dead mirror is removed from `spec.mirrors` so nothing probes it again. @@ -2053,6 +2083,17 @@ async fn resolve_primary( return None; } }; + let fallback_referer = match &answer { + Ok(p) if spec.referer.is_none() => referer_fallback(p, &u, &spec.plugin_headers), + _ => None, + }; + if let Some(referer) = fallback_referer { + crate::log::debug(&format!( + "#{id} 403 with no referer; asking again as linked from {referer}" + )); + spec.referer = Some(referer); + continue; + } match answer { Ok(p) if p.is_redirect() => { let loc = p.location.clone().unwrap_or_default(); @@ -8835,6 +8876,7 @@ mod intercepted_page_e2e_tests { let seen = requests.clone(); let page_requested = Arc::new(tokio::sync::Notify::new()); let requested = page_requested.clone(); + let own_referer = format!("referer: {url}/"); let task = tokio::spawn(async move { loop { let (socket, _) = listener.accept().await.unwrap(); @@ -8848,6 +8890,14 @@ mod intercepted_page_e2e_tests { request.push_str(&line); } seen.lock().unwrap().push(request.clone()); + let hotlinked = request.split_whitespace().nth(1) == Some("/hotlink.bin") + && !request + .lines() + .any(|l| l.eq_ignore_ascii_case(own_referer.trim_end())); + if hotlinked { + let _ = socket.get_mut().write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await; + continue; + } if request.split_whitespace().nth(1) == Some("/redirect.zip") { let _ = socket.get_mut().write_all(b"HTTP/1.1 302 Found\r\nLocation: /login\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await; continue; @@ -8959,6 +9009,81 @@ mod intercepted_page_e2e_tests { events } + /// #398: a CDN that refuses any request without a same-site `Referer` + /// answered a pasted link with `403`. + #[tokio::test] + async fn gui_e2e_a_hotlink_403_is_asked_again_as_linked_from_its_own_site() { + let dir = tempfile::tempdir().unwrap(); + let body = b"protected bytes".to_vec(); + let origin = origin(body.clone(), Some("video/mp4"), None, true, false).await; + let spec = spec(&origin, dir.path(), "hotlink.bin"); + + let events = download(spec.clone()).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Finished { .. })), + "{events:?}" + ); + assert_eq!(tokio::fs::read(&spec.final_path).await.unwrap(), body); + let requests = origin.requests.lock().unwrap(); + let referer = format!("Referer: {}/", origin.url); + let first_with = requests.iter().position(|r| r.contains(&referer)); + let first_with = first_with.expect("the second probe carries it"); + assert!( + first_with > 0, + "the first ask is the bare one: {requests:?}" + ); + assert!( + requests[first_with..].iter().all(|r| r.contains(&referer)), + "every request after the refusal carries it: {requests:?}" + ); + } + + #[tokio::test] + async fn gui_e2e_a_referer_the_user_gave_is_never_replaced() { + let dir = tempfile::tempdir().unwrap(); + let origin = origin(b"x".to_vec(), None, None, true, false).await; + let mut spec = spec(&origin, dir.path(), "hotlink.bin"); + spec.referer = Some("https://elsewhere.example/page".into()); + + let events = download(spec).await; + + assert!( + events + .iter() + .any(|e| matches!(e, Event::Failed { error, .. } if error.contains("403"))), + "{events:?}" + ); + let requests = origin.requests.lock().unwrap(); + assert!(requests + .iter() + .all(|r| r.contains("Referer: https://elsewhere.example/page"))); + } + + #[test] + fn referer_fallback_names_the_origin_and_only_for_a_bare_403() { + let probe = |status| Probe { + status, + ..Probe::default() + }; + let u = parse_url("https://v46.erome.com/7232/a_720p.mp4").unwrap(); + assert_eq!( + referer_fallback(&probe(403), &u, &[]).as_deref(), + Some("https://v46.erome.com/") + ); + let odd = parse_url("http://cdn.example:8080/f").unwrap(); + assert_eq!( + referer_fallback(&probe(403), &odd, &[]).as_deref(), + Some("http://cdn.example:8080/") + ); + assert_eq!(referer_fallback(&probe(404), &u, &[]), None); + assert_eq!(referer_fallback(&probe(200), &u, &[]), None); + let given = ["referer: https://www.erome.com/".to_string()]; + assert_eq!(referer_fallback(&probe(403), &u, &given), None); + let ftp = parse_url("ftp://ftp.example/f").unwrap(); + assert_eq!(referer_fallback(&probe(403), &ftp, &[]), None); + } + #[tokio::test] async fn gui_e2e_html_block_pages_never_replace_the_requested_binary() { for (index, (kind, disposition, ranges, early, large)) in [ diff --git a/crates/hydra-gui/src/engine/plugin.rs b/crates/hydra-gui/src/engine/plugin.rs index a4fb20b0..24a9aafe 100644 --- a/crates/hydra-gui/src/engine/plugin.rs +++ b/crates/hydra-gui/src/engine/plugin.rs @@ -285,9 +285,28 @@ async fn run_with_root( )); return true; } - let size = selected.tracks.iter().try_fold(0u64, |n, &i| { - plan.tracks[i].size.and_then(|s| n.checked_add(s)) - }); + let scratch = TrackDir::at(std::path::Path::new(&spec.temp_path)); + let mut saved = scratch.load(); + saved.resize_with(selected.tracks.len(), SavedTrack::default); + for (entry, &i) in saved.iter_mut().zip(&selected.tracks) { + if entry.id != plan.tracks[i].id { + *entry = SavedTrack { + id: plan.tracks[i].id.clone(), + ..SavedTrack::default() + }; + } + } + let mut sizes: Vec> = selected + .tracks + .iter() + .zip(&saved) + .map(|(&i, entry)| entry.size.or(plan.tracks[i].size)) + .collect(); + let mut size = total(&sizes); + let ranges = selected + .tracks + .iter() + .all(|&i| plan.tracks[i].ranges != hya_plugin_api::Ranges::Deny); let title = plan.title.as_deref().unwrap_or(&plan.id); let name = hya_net::filename::portable(title).unwrap_or_else(|| "download".into()); let ext = if extract { @@ -335,27 +354,32 @@ async fn run_with_root( let _ = tx.send(Event::Probed { id: spec.id, size, - ranges: false, + ranges, file_name, }); let start = std::time::Instant::now(); let mut completed = 0; - let scratch_parent = std::path::Path::new(&spec.temp_path) - .parent() - .unwrap_or_else(|| std::path::Path::new(".")); - if let Err(e) = std::fs::create_dir_all(scratch_parent) { + if let Err(e) = std::fs::create_dir_all(scratch.path()) { fail(format!("plugin {plugin}: {e}")); return true; } - let scratch = match tempfile::tempdir_in(scratch_parent) { - Ok(dir) => dir, - Err(e) => { - fail(format!("plugin {plugin}: {e}")); - return true; - } - }; let mut outputs = Vec::new(); for (ordinal, &i) in selected.tracks.iter().enumerate() { + let part = scratch.path().join(format!("track-{ordinal}.part")); + let output = scratch.path().join(format!("track-{ordinal}")); + let entry = &mut saved[ordinal]; + if let Some(done) = entry + .size + .filter(|&n| entry.done && file_len(&output) == Some(n)) + { + completed += done; + outputs.push((i, output.to_string_lossy().into_owned())); + continue; + } + if entry.done || !part.exists() { + entry.held.clear(); + } + entry.done = false; let mut attempt = 0; loop { let track = plan.tracks[i].clone(); @@ -399,8 +423,8 @@ async fn run_with_root( max_connections: source.max_connections.map(|n| n as usize), }) .collect(); - child.held.clear(); - child.expected_size = None; + child.held = saved[ordinal].held.clone(); + child.expected_size = saved[ordinal].size; child.attested_size = track.size; child.attested_digest = track.digest.clone(); child.pieces = None; @@ -414,16 +438,8 @@ async fn run_with_root( child.conns = 1; child.force_stream = true; } - child.temp_path = scratch - .path() - .join(format!("track-{ordinal}.part")) - .to_string_lossy() - .into_owned(); - child.final_path = scratch - .path() - .join(format!("track-{ordinal}")) - .to_string_lossy() - .into_owned(); + child.temp_path = part.to_string_lossy().into_owned(); + child.final_path = output.to_string_lossy().into_owned(); let destination = child.final_path.clone(); let (events, mut rx) = tokio::sync::mpsc::unbounded_channel(); let task = super::run_file_download( @@ -436,17 +452,77 @@ async fn run_with_root( tokio::pin!(task); let mut finished = None; let mut failure = None; + let mut task_done = false; loop { - tokio::select! { - () = &mut task => { while let Ok(event) = rx.try_recv() { if let Event::Finished {size,..} = event { finished = Some(size); } else if let Event::Failed {error,..} = event {failure = Some(error);} } break; } - event = rx.recv() => match event { - Some(Event::Finished {size,..}) => finished = Some(size), - Some(Event::Progress {done,rate,eta,conns,..}) => { let _ = tx.send(Event::Progress { id:spec.id,done:completed+done,rate,eta,conns,held:vec![],recorded:None }); } - Some(Event::Failed {error,..}) => {failure = Some(error);break;} - Some(Event::Stopped {done,..}) => { let _ = tx.send(Event::Stopped { id:spec.id,done:completed+done,held:vec![] }); return true; } - Some(Event::Status {line,..}) => { let _ = tx.send(Event::Status {id:spec.id,line:format!("{plugin}: {line}")}); } - _ => {} + // The task's last events may still be queued when it returns, + // a Stop among them; polling it again after that would panic. + let event = if task_done { + match rx.try_recv() { + Ok(event) => event, + Err(_) => break, + } + } else { + tokio::select! { + () = &mut task => { task_done = true; continue; } + event = rx.recv() => match event { + Some(event) => event, + None => { task_done = true; continue; } + }, + } + }; + match event { + Event::Finished { size, .. } => finished = Some(size), + Event::Probed { size: Some(n), .. } => { + saved[ordinal].size = Some(n); + sizes[ordinal] = Some(n); + if total(&sizes) != size { + size = total(&sizes); + let _ = tx.send(Event::Probed { + id: spec.id, + size, + ranges, + file_name: None, + }); + } + } + Event::Progress { + done, + rate, + eta, + conns, + .. + } => { + let _ = tx.send(Event::Progress { + id: spec.id, + done: completed + done, + rate, + eta, + conns, + held: vec![], + recorded: None, + }); } + Event::Failed { error, .. } => { + failure = Some(error); + break; + } + Event::Stopped { done, held, .. } => { + saved[ordinal].held = held; + scratch.keep(&saved); + let _ = tx.send(Event::Stopped { + id: spec.id, + done: completed + done, + held: vec![], + }); + return true; + } + Event::Status { line, .. } => { + let _ = tx.send(Event::Status { + id: spec.id, + line: format!("{plugin}: {line}"), + }); + } + _ => {} } } if let Some(error) = failure { @@ -467,8 +543,9 @@ async fn run_with_root( fail(format!("plugin {plugin}, track {}: {error}", track.id)); return true; } - let Some(size) = finished else { + let Some(track_size) = finished else { if cancel.load(Ordering::Relaxed) { + scratch.keep(&saved); let _ = tx.send(Event::Stopped { id: spec.id, done: completed, @@ -479,7 +556,13 @@ async fn run_with_root( } return true; }; - completed += size; + saved[ordinal] = SavedTrack { + id: track.id.clone(), + size: Some(track_size), + held: vec![], + done: true, + }; + completed += track_size; outputs.push((i, destination)); break; } @@ -601,6 +684,74 @@ async fn run_with_root( true } +/// One selected track's progress, kept across a Stop so the next start +/// resumes it instead of fetching it again from byte zero. +#[derive(Clone, Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)] +struct SavedTrack { + id: String, + size: Option, + held: Vec<(u64, u64)>, + done: bool, +} + +/// Where a plugin download stages its tracks: [`crate::model::track_dir`] +/// beside the item's `.part`, so a restarted transfer finds them again. +/// Removed when it is dropped, unless a Stop asked to [`keep`](Self::keep) +/// it. +struct TrackDir { + path: std::path::PathBuf, + keep: std::cell::Cell, +} + +impl TrackDir { + const STATE: &'static str = "resume.json"; + + fn at(part: &std::path::Path) -> Self { + TrackDir { + path: crate::model::track_dir(part), + keep: std::cell::Cell::new(false), + } + } + + fn path(&self) -> &std::path::Path { + &self.path + } + + fn load(&self) -> Vec { + std::fs::read(self.path.join(Self::STATE)) + .ok() + .and_then(|raw| serde_json::from_slice(&raw).ok()) + .unwrap_or_default() + } + + fn keep(&self, tracks: &[SavedTrack]) { + let saved = serde_json::to_vec(tracks) + .map_err(std::io::Error::other) + .and_then(|json| std::fs::write(self.path.join(Self::STATE), json)); + // Without the record the held spans cannot be trusted, and keeping + // the files would only leave them behind. + self.keep.set(saved.is_ok()); + } +} + +impl Drop for TrackDir { + fn drop(&mut self) { + if !self.keep.get() { + let _ = std::fs::remove_dir_all(&self.path); + } + } +} + +fn total(sizes: &[Option]) -> Option { + sizes + .iter() + .try_fold(0u64, |n, size| size.and_then(|s| n.checked_add(s))) +} + +fn file_len(path: &std::path::Path) -> Option { + std::fs::metadata(path).ok().map(|m| m.len()) +} + async fn run_transfer( spec: &StartSpec, plugin: &str, @@ -1020,6 +1171,196 @@ mod tests { assert!(write_sidecar(&source, &destination).is_err()); assert_eq!(std::fs::read(&destination).unwrap(), b"WEBVTT"); } + /// Install a resolver that answers every claim with `plan`. + fn install_fixture( + root: &std::path::Path, + package: &std::path::Path, + plan: &hya_plugin_api::Plan, + ) { + let reply = serde_json::to_vec(&hya_plugin_api::Resolve::Plan(plan.clone())).unwrap(); + let encoded = reply + .iter() + .map(|b| format!("\\{b:02x}")) + .collect::(); + let module=wat::parse_str(format!(r#"(module + (memory (export "memory") 4) (global $heap (mut i32) (i32.const 10000)) + (data (i32.const 128) "{encoded}") + (func (export "hydra_api") (result i32) i32.const 1) + (func (export "hydra_alloc") (param $n i32) (result i32) (local $p i32) global.get $heap local.set $p global.get $heap local.get $n i32.add global.set $heap local.get $p) + (func (export "hydra_call") (param i32 i32 i32 i32) (result i64) i64.const {}))"#,hya_plugin_api::abi::pack(128,reply.len() as u32))).unwrap(); + std::fs::write(package.join("plugin.wasm"), module).unwrap(); + std::fs::write(package.join("hydra-plugin.toml"),"id='test.gui'\nname='GUI fixture'\nversion='1.0.0'\napi=1\nmodule='plugin.wasm'\nhooks=['resolve']\nclaims=['https://example.com/*']\n[permissions]\nsources=['127.0.0.1']\n").unwrap(); + let mut manager = hya_plugin::manager::Manager::open(root.into()).unwrap(); + let manifest = hya_plugin::manager::Manager::inspect(package) + .unwrap() + .manifest; + manager.install(package, manifest.permissions).unwrap(); + drop(manager); + } + + /// #328: a Stop part-way through a plugin download threw away every + /// track already fetched, and Resume started again from byte zero. + #[tokio::test] + async fn a_stopped_plugin_download_resumes_its_finished_tracks() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let root = tempfile::tempdir().unwrap(); + let package = tempfile::tempdir().unwrap(); + let output = tempfile::tempdir().unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let hold_sub = Arc::new(AtomicBool::new(true)); + let sub_asked = Arc::new(tokio::sync::Notify::new()); + let file_gets = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let (hold, asked, gets) = (hold_sub.clone(), sub_asked.clone(), file_gets.clone()); + let server = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let (hold, asked, gets) = (hold.clone(), asked.clone(), gets.clone()); + tokio::spawn(async move { + let mut request = Vec::new(); + let mut byte = [0]; + while !request.ends_with(b"\r\n\r\n") { + if socket.read(&mut byte).await.unwrap_or(0) == 0 { + return; + } + request.push(byte[0]); + } + let request = String::from_utf8_lossy(&request); + let sub = request.contains("/sub "); + if sub && hold.load(Ordering::Relaxed) { + asked.notify_one(); + std::future::pending::<()>().await; + } + if !sub && request.starts_with("GET ") { + gets.fetch_add(1, Ordering::Relaxed); + } + let body = if sub { + b"WEBVTT\n".as_slice() + } else { + b"first track".as_slice() + }; + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + if !request.starts_with("HEAD ") { + let _ = socket.write_all(body).await; + } + }); + } + }); + let plan: hya_plugin_api::Plan = serde_json::from_value(serde_json::json!({"id":"gui","title":"GUI file","tracks":[ + {"id":"file","kind":"audio","container":"bin","ranges":"deny","sources":[{"url":format!("http://{address}/file")}]}, + {"id":"sub","kind":"subtitle","language":"en","container":"vtt","ranges":"deny","sources":[{"url":format!("http://{address}/sub")}]} + ]})).unwrap(); + install_fixture(root.path(), package.path(), &plan); + let mut spec = StartSpec::plain(); + spec.id = 77; + spec.url = "https://example.com/file".into(); + spec.proxy = crate::model::ProxyChoice::Direct; + spec.temp_path = output + .path() + .join("result.bin.part") + .to_string_lossy() + .into_owned(); + spec.final_path = output + .path() + .join("result.bin") + .to_string_lossy() + .into_owned(); + spec.plugin_plan = Some(crate::plugins::PlanInfo { + plugin: "test.gui".into(), + plan, + preferences: Preferences { + include_files: true, + subtitle_languages: vec!["en".into()], + ..Default::default() + }, + }); + let final_path = Arc::new(Mutex::new(spec.final_path.clone())); + let pace = Pace::pair( + Arc::new(super::super::RateLimiter::unlimited()), + Arc::new(super::super::RateLimiter::unlimited()), + ); + let cancel = Arc::new(AtomicBool::new(false)); + let (tx, mut events) = tokio::sync::mpsc::unbounded_channel(); + let stopper = { + let cancel = cancel.clone(); + tokio::spawn(async move { + sub_asked.notified().await; + cancel.store(true, Ordering::Relaxed); + }) + }; + let root_path: std::path::PathBuf = root.path().into(); + assert!(run_with_root(&spec, &cancel, &pace, &final_path, &tx, root_path.clone()).await); + stopper.await.unwrap(); + let stopped: Vec<_> = std::iter::from_fn(|| events.try_recv().ok()).collect(); + assert!( + stopped + .iter() + .any(|e| matches!(e, Event::Stopped { done: 11, .. })), + "{stopped:?}" + ); + assert!( + stopped + .iter() + .any(|e| matches!(e, Event::Probed { ranges: false, .. })), + "tracks that refuse ranges cannot resume mid-file" + ); + let tracks = crate::model::track_dir(std::path::Path::new(&spec.temp_path)); + assert!( + tracks.join("track-0").exists(), + "the finished track is kept" + ); + + hold_sub.store(false, Ordering::Relaxed); + cancel.store(false, Ordering::Relaxed); + assert!(run_with_root(&spec, &cancel, &pace, &final_path, &tx, root_path).await); + let resumed: Vec<_> = std::iter::from_fn(|| events.try_recv().ok()).collect(); + assert!( + resumed.iter().any(|e| matches!(e, Event::Finished { .. })), + "{resumed:?}" + ); + assert_eq!(std::fs::read(&spec.final_path).unwrap(), b"first track"); + assert_eq!( + file_gets.load(Ordering::Relaxed), + 1, + "the finished track is not fetched again" + ); + assert!( + !tracks.exists(), + "a finished download leaves no staging behind" + ); + server.abort(); + } + + #[test] + fn track_state_survives_only_a_kept_dir() { + let dir = tempfile::tempdir().unwrap(); + let part = dir.path().join("x.mkv.part"); + let saved = vec![SavedTrack { + id: "137".into(), + size: Some(9), + held: vec![(0, 4)], + done: false, + }]; + { + let tracks = TrackDir::at(&part); + std::fs::create_dir_all(tracks.path()).unwrap(); + tracks.keep(&saved); + } + let tracks = TrackDir::at(&part); + assert_eq!(tracks.load(), saved); + drop(tracks); + assert!( + !crate::model::track_dir(&part).exists(), + "not kept this time" + ); + assert_eq!(total(&[Some(2), Some(3)]), Some(5)); + assert_eq!(total(&[Some(2), None]), None); + } + #[tokio::test] async fn gui_engine_executes_installed_plan_saves_sidecar_and_cleans_failed_tracks() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -1070,27 +1411,7 @@ mod tests { {"id":"file","kind":"audio","container":"bin","ranges":"deny","sources":[{"url":format!("http://{address}/file")}]}, {"id":"sub","kind":"subtitle","language":"en","container":"vtt","ranges":"deny","sources":[{"url":format!("http://{address}/sub")}]} ]})).unwrap(); - let reply = serde_json::to_vec(&hya_plugin_api::Resolve::Plan(plan.clone())).unwrap(); - let encoded = reply - .iter() - .map(|b| format!("\\{b:02x}")) - .collect::(); - let module=wat::parse_str(format!(r#"(module - (memory (export "memory") 4) (global $heap (mut i32) (i32.const 10000)) - (data (i32.const 128) "{encoded}") - (func (export "hydra_api") (result i32) i32.const 1) - (func (export "hydra_alloc") (param $n i32) (result i32) (local $p i32) global.get $heap local.set $p global.get $heap local.get $n i32.add global.set $heap local.get $p) - (func (export "hydra_call") (param i32 i32 i32 i32) (result i64) i64.const {}))"#,hya_plugin_api::abi::pack(128,reply.len() as u32))).unwrap(); - std::fs::write(package.path().join("plugin.wasm"), module).unwrap(); - std::fs::write(package.path().join("hydra-plugin.toml"),"id='test.gui'\nname='GUI fixture'\nversion='1.0.0'\napi=1\nmodule='plugin.wasm'\nhooks=['resolve']\nclaims=['https://example.com/*']\n[permissions]\nsources=['127.0.0.1']\n").unwrap(); - let mut manager = hya_plugin::manager::Manager::open(root.path().into()).unwrap(); - let manifest = hya_plugin::manager::Manager::inspect(package.path()) - .unwrap() - .manifest; - manager - .install(package.path(), manifest.permissions) - .unwrap(); - drop(manager); + install_fixture(root.path(), package.path(), &plan); let mut spec = StartSpec::plain(); spec.id = 99; spec.url = "https://example.com/file".into(); diff --git a/crates/hydra-gui/src/model.rs b/crates/hydra-gui/src/model.rs index 0c581ca1..c43f2a1e 100644 --- a/crates/hydra-gui/src/model.rs +++ b/crates/hydra-gui/src/model.rs @@ -326,6 +326,11 @@ impl DownloadItem { } } + /// Where a plugin download stages its tracks between runs. + pub fn track_dir(&self) -> PathBuf { + track_dir(&self.part_file()) + } + pub fn progress(&self) -> f32 { // A live recording has no size, so bytes can say nothing about how // far along it is — but a recording with a time limit does have a @@ -1404,6 +1409,14 @@ pub struct Settings { pub window_pos: Option<(f32, f32)>, } +/// The folder beside a `.part` that a plugin download keeps its separate +/// tracks in until they are combined into the one file. +pub fn track_dir(part: &std::path::Path) -> PathBuf { + let mut dir = part.as_os_str().to_owned(); + dir.push(".tracks"); + PathBuf::from(dir) +} + /// The User-Agent a download added by hand sends unless Options picks another. pub const DEFAULT_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko"; diff --git a/crates/hydra-net/Cargo.toml b/crates/hydra-net/Cargo.toml index 9127fe20..6118098d 100644 --- a/crates/hydra-net/Cargo.toml +++ b/crates/hydra-net/Cargo.toml @@ -44,6 +44,8 @@ criterion = { workspace = true } # Writes the fixture archives the `zipdir` tests read back: a real archiver's # output, ZIP64 included, rather than bytes laid out by the code under test. zip = { version = "9", default-features = false, features = ["deflate"] } +# Mints the root, intermediate and leaf the incomplete-chain TLS tests serve. +rcgen = { version = "0.14", default-features = false, features = ["ring", "pem"] } [[bench]] name = "hot_paths" diff --git a/crates/hydra-net/src/tls.rs b/crates/hydra-net/src/tls.rs index 05702890..715e86a7 100644 --- a/crates/hydra-net/src/tls.rs +++ b/crates/hydra-net/src/tls.rs @@ -193,6 +193,160 @@ impl rustls::client::danger::ServerCertVerifier for AcceptAnyCert { } } +/// Intermediates fetched from certificates' Authority Information Access +/// URLs, and the URL each failed handshake asked for. +/// +/// A server that sends its leaf without the intermediate that signed it is +/// common, and browsers paper over it by fetching the intermediate from the +/// URL the leaf names. What is fetched only ever completes a chain: it must +/// still verify up to a bundled root, so it cannot make an untrusted +/// certificate trusted. +#[derive(Debug, Default)] +struct Aia { + fetched: std::sync::Mutex)>>, + wanted: std::sync::Mutex>, +} + +/// Bounds the cache: an origin can make us fetch, not make us hoard. +const AIA_CACHE_MAX: usize = 32; +/// Issuers fetched per handshake: real chains are two or three deep. +const AIA_MAX_DEPTH: usize = 3; +/// An intermediate is a couple of kilobytes; this is generous. +const AIA_FETCH_CAP: usize = 64 * 1024; + +impl Aia { + fn take_wanted(&self, server: &str) -> Option { + self.wanted + .lock() + .unwrap_or_else(|e| e.into_inner()) + .remove(server) + } + + fn add(&self, url: &str, cert: rustls_pki_types::CertificateDer<'static>) { + let mut fetched = self.fetched.lock().unwrap_or_else(|e| e.into_inner()); + fetched.retain(|(u, _)| u != url); + if fetched.len() >= AIA_CACHE_MAX { + fetched.remove(0); + } + fetched.push((url.to_string(), cert)); + } + + /// The next issuer to fetch for a chain that did not verify: the first + /// CA Issuers URL along it that has not been fetched already, so each + /// pass climbs one level. + fn next_issuer(&self, chain: &[&[u8]]) -> Option { + let fetched = self.fetched.lock().unwrap_or_else(|e| e.into_inner()); + chain + .iter() + .filter_map(|cert| ca_issuers_url(cert)) + .find(|url| !fetched.iter().any(|(u, _)| u == url)) + } +} + +/// The bundled-roots verifier, with the [`Aia`] intermediates offered +/// alongside whatever the server sent. +#[derive(Debug)] +struct AiaVerifier { + inner: Arc, + aia: Arc, +} + +impl rustls::client::danger::ServerCertVerifier for AiaVerifier { + fn verify_server_cert( + &self, + end_entity: &rustls_pki_types::CertificateDer<'_>, + intermediates: &[rustls_pki_types::CertificateDer<'_>], + server_name: &ServerName<'_>, + ocsp: &[u8], + now: rustls_pki_types::UnixTime, + ) -> Result { + let mut chain = intermediates.to_vec(); + chain.extend( + self.aia + .fetched + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .map(|(_, cert)| cert.clone()), + ); + let verdict = self + .inner + .verify_server_cert(end_entity, &chain, server_name, ocsp, now); + if let Err(rustls::Error::InvalidCertificate(rustls::CertificateError::UnknownIssuer)) = + &verdict + { + let path: Vec<&[u8]> = std::iter::once(end_entity.as_ref()) + .chain(chain.iter().map(|c| c.as_ref())) + .collect(); + if let Some(url) = self.aia.next_issuer(&path) { + self.aia + .wanted + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(server_name.to_str().into_owned(), url); + } + } + verdict + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls_pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + self.inner.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls_pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + self.inner.verify_tls13_signature(message, cert, dss) + } + + fn supported_verify_schemes(&self) -> Vec { + self.inner.supported_verify_schemes() + } +} + +/// The `http://` CA Issuers URL in a certificate's Authority Information +/// Access extension. +/// +/// Read straight off the DER: the accessMethod OID for caIssuers is followed +/// by the location as a `[6]` uniformResourceIdentifier. Only `http` is +/// taken; RFC 5280 says that is what the extension carries, and following an +/// `https` one would need the very chain being repaired. +fn ca_issuers_url(cert: &[u8]) -> Option { + const CA_ISSUERS: [u8; 10] = [0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x02]; + let at = cert + .windows(CA_ISSUERS.len()) + .position(|w| w == CA_ISSUERS)? + + CA_ISSUERS.len(); + let rest = cert.get(at..)?; + if *rest.first()? != 0x86 { + return None; + } + let (len, start) = match *rest.get(1)? { + n if n < 0x80 => (n as usize, 2), + 0x81 => (*rest.get(2)? as usize, 3), + _ => return None, + }; + let url = std::str::from_utf8(rest.get(start..start + len)?).ok()?; + url.starts_with("http://").then(|| url.to_string()) +} + +/// A fetched issuer certificate: DER as published, or PEM. +fn issuer_cert(body: Vec) -> Option> { + use rustls_pki_types::pem::PemObject; + if body.first() == Some(&0x30) { + return Some(rustls_pki_types::CertificateDer::from(body)); + } + rustls_pki_types::CertificateDer::from_pem_slice(&body).ok() +} + /// Which IP version to connect over: `-4`, `-6`, or whatever resolves. /// /// A property of the connection rather than of the request, so it lives on the @@ -367,6 +521,8 @@ pub struct TlsCapableConnector { /// redials the origin it was just speaking to — measured at 1.6-2.0 s of setup /// on a live TLS path. pool: crate::pool::SharedPool, + /// Shared with the verifier; `None` when nothing is verified. + aia: Option>, } impl TlsCapableConnector { @@ -379,10 +535,25 @@ impl TlsCapableConnector { pub fn new() -> io::Result { let mut roots = rustls::RootCertStore::empty(); roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + Self::verifying(roots) + } + + fn verifying(roots: rustls::RootCertStore) -> io::Result { + let inner = rustls::client::WebPkiServerVerifier::builder(Arc::new(roots)) + .build() + .map_err(|e| io::Error::other(format!("tls config: {e}")))?; + let aia = Arc::new(Aia::default()); let config = ClientConfig::builder() - .with_root_certificates(roots) + .dangerous() + .with_custom_certificate_verifier(Arc::new(AiaVerifier { + inner, + aia: aia.clone(), + })) .with_no_client_auth(); - Ok(Self::finish(config)) + Ok(Self { + aia: Some(aia), + ..Self::finish(config) + }) } /// The tail every constructor shares: session resumption, because a @@ -398,6 +569,7 @@ impl TlsCapableConnector { proxy: crate::socks::Proxy::none(), family: IpFamily::Any, pool: std::sync::Arc::new(crate::pool::ConnPool::new()), + aia: None, } } @@ -460,41 +632,11 @@ impl Connector for TlsCapableConnector { &'a self, t: &'a Target, ) -> Pin> + Send + 'a>> { - let config = self.config.clone(); - let socks = self.proxy.clone(); - let family = self.family; Box::pin(async move { - // With a SOCKS proxy the TCP connection goes to the PROXY, and the - // handshake asks it to reach the origin. The origin name is sent as a - // name, not an address, so the proxy resolves it — which is the whole - // point when local DNS cannot. - let mut tcp = if socks.kind.is_socks() { - // The family restriction applies to the hop this process actually - // opens, which is the one to the PROXY. Beyond it the proxy - // resolves the origin itself and this client has no say — saying - // otherwise would be a promise we cannot keep. - let mut sock = connect_family(&socks.host, socks.port, family).await?; - let _ = sock.set_nodelay(true); - let (dst_host, dst_port) = t.origin_endpoint(); - crate::socks::handshake(&mut sock, &socks, &dst_host, dst_port).await?; - sock - } else { - let sock = connect_family(&t.host, t.port, family).await?; - let _ = sock.set_nodelay(true); - sock - }; + let tcp = self.dial(t).await?; if !t.tls { return Ok(MaybeTls::Plain(tcp)); } - // Through a proxy, TLS needs a CONNECT tunnel first: absolute-form - // requests only work in cleartext, because the proxy has to read the - // request line, and it cannot read an encrypted one. - // A CONNECT tunnel is only for an HTTP proxy. Through SOCKS the stream is - // already end-to-end with the origin, so a CONNECT would be sent TO the - // origin as a bogus request. - if t.origin.is_some() && !socks.kind.is_socks() { - connect_tunnel(&mut tcp, t.proxy_authority(), t.proxy_authorization()).await?; - } // SNI must be the ORIGIN name, not the socket peer: when routing // through a proxy the socket connects to the proxy but the certificate // being validated belongs to the origin. @@ -505,14 +647,108 @@ impl Connector for TlsCapableConnector { format!("bad TLS name: {sni_host}"), ) })?; - let stream = RustlsConnector::from(config) - .connect(server_name, tcp) - .await?; - Ok(MaybeTls::Tls(Box::new(stream))) + let mut handshake = RustlsConnector::from(self.config.clone()) + .connect(server_name.clone(), tcp) + .await; + // A chain that stops short of its root is completed from the URL + // its top certificate names, one level per pass, then asked again. + for _ in 0..AIA_MAX_DEPTH { + if handshake.is_ok() { + break; + } + let Some(url) = self.aia.as_ref().and_then(|a| a.take_wanted(sni_host)) else { + break; + }; + if !self.fetch_issuer(t, &url).await { + break; + } + let tcp = self.dial(t).await?; + handshake = RustlsConnector::from(self.config.clone()) + .connect(server_name.clone(), tcp) + .await; + } + Ok(MaybeTls::Tls(Box::new(handshake?))) }) } } +impl TlsCapableConnector { + /// The TCP leg to `t`: direct, through SOCKS, or through an HTTP proxy's + /// CONNECT tunnel when `t` is TLS. + async fn dial(&self, t: &Target) -> io::Result { + let socks = &self.proxy; + // With a SOCKS proxy the TCP connection goes to the PROXY, and the + // handshake asks it to reach the origin. The origin name is sent as a + // name, not an address, so the proxy resolves it — which is the whole + // point when local DNS cannot. + let mut tcp = if socks.kind.is_socks() { + // The family restriction applies to the hop this process actually + // opens, which is the one to the PROXY. Beyond it the proxy + // resolves the origin itself and this client has no say — saying + // otherwise would be a promise we cannot keep. + let mut sock = connect_family(&socks.host, socks.port, self.family).await?; + let _ = sock.set_nodelay(true); + let (dst_host, dst_port) = t.origin_endpoint(); + crate::socks::handshake(&mut sock, socks, &dst_host, dst_port).await?; + sock + } else { + let sock = connect_family(&t.host, t.port, self.family).await?; + let _ = sock.set_nodelay(true); + sock + }; + // Through a proxy, TLS needs a CONNECT tunnel first: absolute-form + // requests only work in cleartext, because the proxy has to read the + // request line, and it cannot read an encrypted one. + // A CONNECT tunnel is only for an HTTP proxy. Through SOCKS the stream is + // already end-to-end with the origin, so a CONNECT would be sent TO the + // origin as a bogus request. + if t.tls && t.origin.is_some() && !socks.kind.is_socks() { + connect_tunnel(&mut tcp, t.proxy_authority(), t.proxy_authorization()).await?; + } + Ok(tcp) + } + + /// Fetch the issuer certificate at `url` the way `t` itself is routed, + /// and cache it for the verifier. Whether one was added. + async fn fetch_issuer(&self, t: &Target, url: &str) -> bool { + let Some(rest) = url.strip_prefix("http://") else { + return false; + }; + let (authority, path) = match rest.find('/') { + Some(i) => (&rest[..i], &rest[i..]), + None => (rest, "/"), + }; + let (host, port) = match authority.rsplit_once(':') { + Some((h, p)) => match p.parse() { + Ok(p) => (h, p), + Err(_) => return false, + }, + None => (authority, 80), + }; + let mut aia = if t.origin.is_some() && !self.proxy.kind.is_socks() { + Target::via_proxy(&t.host, t.port, authority, path) + } else { + Target::direct(host, port, path) + }; + aia.headers = t + .headers + .iter() + .filter(|h| crate::is_field(h, "proxy-authorization")) + .cloned() + .collect(); + let Ok(body) = crate::http::fetch_small(self, &aia, AIA_FETCH_CAP).await else { + return false; + }; + match (issuer_cert(body), &self.aia) { + (Some(cert), Some(cache)) => { + cache.add(url, cert); + true + } + _ => false, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -529,6 +765,273 @@ mod tests { drop(c); } + fn der(bytes: &[u8]) -> Vec { + let mut out = Vec::new(); + if bytes.len() < 0x80 { + out.push(bytes.len() as u8); + } else { + out.extend([0x81, bytes.len() as u8]); + } + out.extend(bytes); + out + } + + /// An Authority Information Access extension's value naming `url` as + /// the CA Issuers location. + fn aia_value(url: &str) -> Vec { + let mut access = vec![ + 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x02, 0x86, + ]; + access.extend(der(url.as_bytes())); + let mut description = vec![0x30]; + description.extend(der(&access)); + let mut outer = vec![0x30]; + outer.extend(der(&description)); + outer + } + + #[test] + fn ca_issuers_url_reads_the_http_location_only() { + assert_eq!( + ca_issuers_url(&aia_value("http://yr1.i.lencr.org/")).as_deref(), + Some("http://yr1.i.lencr.org/") + ); + let long = format!("http://ca.example/{}", "a".repeat(150)); + assert_eq!(ca_issuers_url(&aia_value(&long)), Some(long)); + assert_eq!(ca_issuers_url(&aia_value("https://ca.example/i.der")), None); + assert_eq!(ca_issuers_url(&aia_value("ldap://ca.example/cn")), None); + assert_eq!(ca_issuers_url(b"\x30\x03\x02\x01\x00"), None); + } + + #[test] + fn each_pass_asks_for_the_next_issuer_up_the_chain() { + let aia = Aia::default(); + let leaf = aia_value("http://ca.example/intermediate"); + let intermediate = aia_value("http://ca.example/cross"); + assert_eq!( + aia.next_issuer(&[&leaf]).as_deref(), + Some("http://ca.example/intermediate") + ); + aia.add( + "http://ca.example/intermediate", + rustls_pki_types::CertificateDer::from(intermediate.clone()), + ); + assert_eq!( + aia.next_issuer(&[&leaf, &intermediate]).as_deref(), + Some("http://ca.example/cross") + ); + aia.add( + "http://ca.example/cross", + rustls_pki_types::CertificateDer::from(vec![0x30, 0x00]), + ); + assert_eq!(aia.next_issuer(&[&leaf, &intermediate]), None); + } + + #[test] + fn the_issuer_cache_is_bounded() { + let aia = Aia::default(); + for n in 0..AIA_CACHE_MAX + 5 { + aia.add( + &format!("http://ca.example/{n}"), + rustls_pki_types::CertificateDer::from(vec![0x30, n as u8]), + ); + } + let fetched = aia.fetched.lock().unwrap(); + assert_eq!(fetched.len(), AIA_CACHE_MAX); + assert_eq!(fetched[0].0, "http://ca.example/5"); + } + + #[test] + fn a_fetched_issuer_may_be_der_or_pem() { + let der_body = vec![0x30, 0x03, 0x02, 0x01, 0x00]; + assert_eq!( + issuer_cert(der_body.clone()).map(|c| c.as_ref().to_vec()), + Some(der_body.clone()) + ); + let pem = rcgen::CertificateParams::new(vec!["ca.example".into()]) + .unwrap() + .self_signed(&rcgen::KeyPair::generate().unwrap()) + .unwrap(); + assert_eq!( + issuer_cert(pem.pem().into_bytes()).as_ref(), + Some(pem.der()) + ); + assert!(issuer_cert(b"Expired".to_vec()).is_none()); + } + + struct Chain { + root: rustls_pki_types::CertificateDer<'static>, + intermediate: Vec, + leaf: rustls_pki_types::CertificateDer<'static>, + leaf_key: rustls_pki_types::PrivateKeyDer<'static>, + } + + /// root -> intermediate -> leaf for `localhost`, the leaf naming + /// `issuer_url` as where its issuer is published. + fn chain(issuer_url: &str) -> Chain { + use rcgen::{BasicConstraints, CertificateParams, DnType, IsCa, Issuer, KeyPair}; + let ca = |name: &str| { + let mut p = CertificateParams::new(Vec::::new()).unwrap(); + p.distinguished_name.push(DnType::CommonName, name); + p.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + p + }; + let root_key = KeyPair::generate().unwrap(); + let root_params = ca("Hydra Test Root"); + let root = root_params.self_signed(&root_key).unwrap(); + let int_key = KeyPair::generate().unwrap(); + let int_params = ca("Hydra Test Intermediate"); + let intermediate = int_params + .signed_by(&int_key, &Issuer::from_params(&root_params, &root_key)) + .unwrap(); + let leaf_key = KeyPair::generate().unwrap(); + let mut leaf_params = CertificateParams::new(vec!["localhost".into()]).unwrap(); + leaf_params + .distinguished_name + .push(DnType::CommonName, "localhost"); + leaf_params + .custom_extensions + .push(rcgen::CustomExtension::from_oid_content( + &[1, 3, 6, 1, 5, 5, 7, 1, 1], + aia_value(issuer_url), + )); + let leaf = leaf_params + .signed_by(&leaf_key, &Issuer::from_params(&int_params, &int_key)) + .unwrap(); + Chain { + root: root.der().clone(), + intermediate: intermediate.der().to_vec(), + leaf: leaf.der().clone(), + leaf_key: rustls_pki_types::PrivateKeyDer::Pkcs8(leaf_key.serialize_der().into()), + } + } + + /// Serves whatever `body` holds to every plain HTTP request, counting + /// them. The body is filled in after the chain naming this server's port + /// has been minted. + async fn issuer_server() -> ( + u16, + Arc>>, + Arc, + ) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let body = Arc::new(std::sync::Mutex::new(Vec::new())); + let hits = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let (served, counted) = (body.clone(), hits.clone()); + tokio::spawn(async move { + loop { + let (mut s, _) = listener.accept().await.unwrap(); + counted.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let mut buf = [0u8; 2048]; + let _ = s.read(&mut buf).await; + let body = served.lock().unwrap().clone(); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/pkix-cert\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = s.write_all(head.as_bytes()).await; + let _ = s.write_all(&body).await; + } + }); + (port, body, hits) + } + + /// A TLS origin that sends its leaf and nothing else: the misconfiguration + /// `fs20.uploadrar.com` has, with the intermediate left to the client. + async fn leaf_only_origin(chain: &Chain) -> u16 { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let config = rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![chain.leaf.clone()], chain.leaf_key.clone_key()) + .unwrap(); + let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + loop { + let (s, _) = listener.accept().await.unwrap(); + let acceptor = acceptor.clone(); + tokio::spawn(async move { + if let Ok(mut tls) = acceptor.accept(s).await { + let mut buf = [0u8; 1024]; + let _ = tls.read(&mut buf).await; + let _ = tls + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await; + } + }); + } + }); + port + } + + fn trusting(root: &rustls_pki_types::CertificateDer<'static>) -> TlsCapableConnector { + let mut roots = rustls::RootCertStore::empty(); + roots.add(root.clone()).unwrap(); + TlsCapableConnector::verifying(roots).unwrap() + } + + #[tokio::test] + async fn a_chain_missing_its_intermediate_is_completed_from_the_leafs_aia() { + let (issuer_port, body, hits) = issuer_server().await; + let chain = chain(&format!("http://127.0.0.1:{issuer_port}/int.der")); + *body.lock().unwrap() = chain.intermediate.clone(); + let origin = leaf_only_origin(&chain).await; + let connector = trusting(&chain.root); + let t = Target::direct_tls("localhost", origin, "/"); + + let body = crate::http::fetch_small(&connector, &t, 64).await.unwrap(); + assert_eq!(body, b"ok"); + let again = crate::http::fetch_small(&connector, &t, 64).await.unwrap(); + assert_eq!(again, b"ok"); + assert_eq!( + hits.load(std::sync::atomic::Ordering::Relaxed), + 1, + "the intermediate is fetched once and then reused" + ); + } + + #[tokio::test] + async fn an_issuer_that_does_not_chain_to_a_trusted_root_still_fails() { + let (issuer_port, body, _) = issuer_server().await; + let chain = chain(&format!("http://127.0.0.1:{issuer_port}/int.der")); + let impostor = self::chain("http://unused.example/"); + *body.lock().unwrap() = impostor.intermediate.clone(); + let origin = leaf_only_origin(&chain).await; + let connector = trusting(&chain.root); + let t = Target::direct_tls("localhost", origin, "/"); + + let err = crate::http::fetch_small(&connector, &t, 64) + .await + .unwrap_err(); + assert!( + err.to_string().contains("invalid peer certificate"), + "{err}" + ); + } + + #[tokio::test] + async fn an_unreachable_issuer_reports_the_original_failure() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let dead = listener.local_addr().unwrap().port(); + drop(listener); + let chain = chain(&format!("http://127.0.0.1:{dead}/int.der")); + let origin = leaf_only_origin(&chain).await; + let connector = trusting(&chain.root); + let t = Target::direct_tls("localhost", origin, "/"); + + let err = crate::http::fetch_small(&connector, &t, 64) + .await + .unwrap_err(); + assert!(err.to_string().contains("UnknownIssuer"), "{err}"); + } + #[test] fn an_insecure_client_builds() { assert!(TlsCapableConnector::insecure().is_ok()); diff --git a/crates/hydra-updater/Cargo.toml b/crates/hydra-updater/Cargo.toml index 32faec39..256f7f69 100644 --- a/crates/hydra-updater/Cargo.toml +++ b/crates/hydra-updater/Cargo.toml @@ -34,6 +34,8 @@ zip = { version = "9", default-features = false, features = ["deflate"] } # is not clang-parsable, so blake3's C NEON path cannot build when # cross-compiling; the pure-Rust implementation is forced in for Windows. blake3 = { version = "1", features = ["pure"] } +# The setup installer's Add/Remove Programs entry carries the version. +winreg = "0.56" [lints] workspace = true diff --git a/crates/hydra-updater/src/lib.rs b/crates/hydra-updater/src/lib.rs index 6f0dd32f..059c2cc8 100644 --- a/crates/hydra-updater/src/lib.rs +++ b/crates/hydra-updater/src/lib.rs @@ -1178,9 +1178,57 @@ pub fn apply_with( if let Some(bundle) = &bundle { finish_bundle(bundle, opts.version.as_deref(), &mut report); } + if let Some(version) = opts.version.as_deref() { + match set_uninstall_version(install_dir, version) { + Ok(true) => report.notes.push(format!("DisplayVersion -> {version}")), + Ok(false) => {} + Err(e) => report + .notes + .push(format!("DisplayVersion update failed: {e}")), + } + } Ok(report) } +/// Restamp `DisplayVersion` in the setup installer's uninstall entry, which +/// Settings > Apps and every inventory tool read; a swap in place would +/// otherwise leave it naming the version that was first installed. +/// `Ok(false)` off Windows, or when the entry belongs to another install. +#[cfg(windows)] +fn set_uninstall_version(install_dir: &Path, version: &str) -> io::Result { + use winreg::enums::{HKEY_CURRENT_USER, KEY_READ, KEY_SET_VALUE}; + // The Add/Remove Programs entry `hydra-installer.nsi` writes. + const UNINSTALL_KEY: &str = r"Software\Microsoft\Windows\CurrentVersion\Uninstall\Hydra"; + let key = match winreg::RegKey::predef(HKEY_CURRENT_USER) + .open_subkey_with_flags(UNINSTALL_KEY, KEY_READ | KEY_SET_VALUE) + { + Ok(key) => key, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(e) => return Err(e), + }; + let location: String = key.get_value("InstallLocation").unwrap_or_default(); + if !same_install_dir(&location, install_dir) { + return Ok(false); + } + key.set_value("DisplayVersion", &version)?; + Ok(true) +} + +#[cfg(not(windows))] +fn set_uninstall_version(_install_dir: &Path, _version: &str) -> io::Result { + Ok(false) +} + +/// Whether the installer's recorded `InstallLocation` names `install_dir`. +/// Windows paths compare without case, and NSIS may or may not leave a +/// trailing separator on `$INSTDIR`. +#[cfg_attr(not(windows), allow(dead_code))] +fn same_install_dir(location: &str, install_dir: &Path) -> bool { + let trim = |s: &str| s.trim_end_matches(['\\', '/']).replace('/', "\\"); + let location = trim(location); + !location.is_empty() && location.eq_ignore_ascii_case(&trim(&install_dir.to_string_lossy())) +} + /// The short second name every Hydra installer puts next to the CLI, because /// `hydra` is also THC-Hydra and three letters types better. pub const CLI_ALIAS: &str = if cfg!(windows) { "hya.exe" } else { "hya" }; @@ -1552,6 +1600,33 @@ fn user_id() -> String { mod tests { use super::*; + #[test] + fn uninstall_entry_matches_its_own_install_dir_only() { + let dir = Path::new(r"C:\Users\me\AppData\Local\Programs\Hydra"); + assert!(same_install_dir( + r"C:\Users\me\AppData\Local\Programs\Hydra", + dir + )); + assert!(same_install_dir( + r"c:\users\ME\appdata\local\programs\hydra\", + dir + )); + assert!(same_install_dir( + "C:/Users/me/AppData/Local/Programs/Hydra", + dir + )); + assert!(!same_install_dir(r"D:\Portable\Hydra", dir)); + assert!(!same_install_dir("", dir)); + } + + #[test] + fn uninstall_version_is_left_alone_off_windows() { + if cfg!(windows) { + return; + } + assert!(!set_uninstall_version(Path::new("/opt/hydra"), "1.2.0").unwrap()); + } + #[test] fn appimage_asset_is_matched_by_shape() { // Named from the workspace version, which a pre-release tag need not diff --git a/extensions/chrome/README.md b/extensions/chrome/README.md index 21a32515..9b49c942 100644 --- a/extensions/chrome/README.md +++ b/extensions/chrome/README.md @@ -195,11 +195,17 @@ id?, error?}`. overwritten today. The entry for the URL's scheme is the one that is sent, the browser's bypass list (`bypassList` / `passthrough`, including ``) is honoured, and no credentials are involved: neither browser's - API exposes them. Two settings are deliberately *not* passed on, because - neither browser will say what they resolve to: following the **machine's** - proxy (`system` / `autoDetect`) and a **PAC script**, which is a program - neither the extension nor Hydra runs. Both leave the download on whatever - Hydra's Options say. The read is capped at 1.5 s, because a capture awaits + API exposes them. Following the **machine's** proxy (`system` / + `autoDetect`) is deliberately *not* passed on, because the browser will not + say what it resolves to; it leaves the download on whatever Hydra's + Options say. A proxy switcher (ZeroOmega, SwitchyOmega) is followed: in + Chromium its **PAC script** (inline or by URL) is run for the download's + URL in a sandbox page hosted by an offscreen document — MV3 service + workers cannot compile code — with the PAC helpers but no DNS, capped at + 3 s, and its first `PROXY`/`HTTPS`/`SOCKS*` entry is sent (`DIRECT` sends + none). In Firefox the switcher routes requests from `proxy.onRequest`, + which the settings never show, so the `proxyInfo` of the last response + seen from the download's origin is sent instead. The read is capped at 1.5 s, because a capture awaits it and a browser that answers neither way would leave the download parked on a question about a proxy. `proxy` is an **optional permission** and *is* the switch — the API does not exist until the checkbox grants it, so there is diff --git a/extensions/chrome/core.js b/extensions/chrome/core.js index c4ebcb82..fd245ad0 100644 --- a/extensions/chrome/core.js +++ b/extensions/chrome/core.js @@ -460,11 +460,13 @@ async function altBypassed() { // and a second stored flag beside the permission could only ever disagree // with it. // -// Two things are deliberately NOT captured. A browser following the machine's -// proxy ("system" / "autoDetect") will not say what that is, and a PAC script -// is a program neither this extension nor Hydra runs — see the same note in -// hydra-gui's proxy.rs. Both resolve to "no proxy known", which leaves the +// A browser following the machine's proxy ("system" / "autoDetect") will not +// say what that is, and resolves to "no proxy known", which leaves the // download on whatever Options says, exactly as if this were switched off. +// A proxy switcher (ZeroOmega, SwitchyOmega) is the other case people have: +// in Chromium it installs a PAC script, which is run here for the one URL; +// in Firefox it routes each request from a listener the settings never show, +// so the route a response actually took is remembered instead. // A local settings read answers in microseconds. The ceiling is not about // slowness: a capture AWAITS this, and a browser whose namespace neither @@ -618,12 +620,147 @@ function geckoProxy(cfg, url) { ); } +// What a PAC entry's keyword means to Chromium, the one running the script. +// A bare SOCKS there is version 4, unlike the bare word in a settings field. +const PAC_KINDS = { + PROXY: "http", + HTTPS: "https", + SOCKS: "socks4", + SOCKS4: "socks4", + SOCKS5: "socks5", +}; + +/// The first usable entry of a `FindProxyForURL` answer such as +/// `PROXY 127.0.0.1:8080; DIRECT`. DIRECT ahead of any proxy means the +/// browser goes direct, and so does the download. +function pacAnswerProxy(answer) { + for (const entry of String(answer ?? "").split(";")) { + const [kind, where] = entry.trim().split(/\s+/); + if (!kind) continue; + const upper = kind.toUpperCase(); + if (upper === "DIRECT") return null; + const spec = upper in PAC_KINDS ? proxySpec(PAC_KINDS[upper], where) : null; + if (spec) return spec; + } + return null; +} + +// A rule list can run to thousands of entries and the offscreen document may +// have to be created first; still, a capture awaits this, so a script that +// loops or a PAC server that never answers costs a bounded wait and then +// "no proxy known". +const PAC_TIMEOUT_MS = 3000; +const PAC_DOCUMENT = "offscreen.html"; +let pacDocumentCreating = null; + +async function ensurePacDocument() { + const open = await chrome.runtime.getContexts({ contextTypes: ["OFFSCREEN_DOCUMENT"] }); + if (open.length) return; + pacDocumentCreating ??= chrome.offscreen + .createDocument({ + url: PAC_DOCUMENT, + reasons: ["IFRAME_SCRIPTING"], + justification: "Evaluate the browser's proxy auto-config script for a captured download.", + }) + // Losing a race with a service worker instance that created it first + // still leaves a document to talk to. + .catch(() => {}) + .finally(() => { + pacDocumentCreating = null; + }); + await pacDocumentCreating; +} + +async function pacScriptText(pac) { + if (typeof pac?.data === "string" && pac.data.trim()) return pac.data; + if (!pac?.url) return null; + const res = await fetch(pac.url, { cache: "no-store" }); + return res.ok ? res.text() : null; +} + +/// Run Chromium's PAC script for this URL in the sandbox, as the browser +/// would: an https URL is reduced to its origin before the script sees it. +async function evaluatePac(pac, url) { + const script = await pacScriptText(pac); + if (!script) return null; + await ensurePacDocument(); + const u = new URL(url); + const reply = await chrome.runtime.sendMessage({ + target: "hydra-pac", + script, + url: u.protocol === "https:" ? `${u.origin}/` : url, + host: u.hostname.replace(/^\[|\]$/g, ""), + }); + return reply && !reply.error ? pacAnswerProxy(reply.result) : null; +} + +async function pacProxy(pac, url) { + if (!chrome.offscreen?.createDocument || !chrome.runtime.getContexts) return null; + const expired = Symbol("expired"); + let timer; + const deadline = new Promise((resolve) => { + timer = setTimeout(() => resolve(expired), PAC_TIMEOUT_MS); + }); + const answer = evaluatePac(pac, url).catch(() => null); + const got = await Promise.race([answer, deadline]); + clearTimeout(timer); + if (got !== expired) return got; + // A script stuck in a loop holds the sandbox for good; closing the document + // discards it, and the next capture starts a fresh one. + chrome.offscreen.closeDocument().catch(() => {}); + return null; +} + +// Firefox: the proxy each recent response really came through, by origin. +// Bounded, oldest first out; an origin seen again moves to the back. +const OBSERVED_PROXY_ORIGINS = 256; +const observedProxies = new Map(); + +function proxyInfoSpec(info) { + if (!info?.host) return null; + const host = info.host.includes(":") && !info.host.startsWith("[") ? `[${info.host}]` : info.host; + const kinds = { http: "http", https: "https", socks: "socks5" }; + const kind = info.type === "socks4" ? (info.proxyDNS ? "socks4a" : "socks4") : kinds[info.type]; + return kind ? proxySpec(kind, `${host}:${info.port}`) : null; +} + +/// Note the route a Firefox webRequest event reports. Credentials are left +/// out, as in `proxySpec`. +function noteProxyInfo(details) { + let origin; + try { + origin = new URL(details.url).origin; + } catch { + return; + } + observedProxies.delete(origin); + observedProxies.set(origin, proxyInfoSpec(details.proxyInfo)); + if (observedProxies.size > OBSERVED_PROXY_ORIGINS) { + observedProxies.delete(observedProxies.keys().next().value); + } +} + +function observedProxy(url) { + try { + return observedProxies.get(new URL(url).origin) ?? null; + } catch { + return null; + } +} + +if (BROWSER === "Mozilla Firefox") { + chrome.webRequest?.onHeadersReceived.addListener(noteProxyInfo, { urls: [""] }); +} + /// The proxy to hand over with this URL, or null for "Hydra decides". async function captureProxy(url) { const cfg = await proxyConfig(); if (!cfg) return null; // The two browsers answer with different objects; each names itself. - if (typeof cfg.proxyType === "string") return geckoProxy(cfg, url); + if (typeof cfg.proxyType === "string") { + return cfg.proxyType === "manual" ? geckoProxy(cfg, url) : observedProxy(url); + } + if (cfg.mode === "pac_script") return pacProxy(cfg.pacScript, url); if (typeof cfg.mode === "string") return chromiumProxy(cfg, url); return null; } @@ -967,6 +1104,36 @@ async function retryDeferredCaptures() { // therefore whether there is anything to hand back. Resuming one that was // never parked is not harmless: on Gecko it throws, and the throw used to be // swallowed next to a download that had already been cancelled. +// ------------------------------------------------------------ form posts + +// A download that answered a form POST — SharePoint and OneDrive zip a folder +// by posting its file list and tokens to `*.svc.ms/transform/zip` — has its +// request in the body. Hydra is handed a URL and asks with a bodiless GET, +// which the service refuses, so such a download stays with the browser. +const POSTED_TTL_MS = 120_000; +const POSTED_MAX = 64; +const postedUrls = new Map(); + +function notePosted(details) { + if (details.method !== "POST") return; + const url = details.url.split("#")[0]; + postedUrls.delete(url); + postedUrls.set(url, Date.now()); + if (postedUrls.size > POSTED_MAX) postedUrls.delete(postedUrls.keys().next().value); +} + +function wasPosted(...urls) { + return urls.some((url) => { + const at = url && postedUrls.get(url.split("#")[0]); + return !!at && Date.now() - at < POSTED_TTL_MS; + }); +} + +chrome.webRequest?.onBeforeRequest?.addListener(notePosted, { + urls: ["http://*/*", "https://*/*"], + types: ["main_frame", "sub_frame"], +}); + async function decideCapture(item, parked) { const url = captureUrl(item); @@ -984,6 +1151,7 @@ async function decideCapture(item, parked) { }; if (item.byExtensionId === chrome.runtime.id) return giveBack("started by this extension"); + if (wasPosted(url, item.url, item.finalUrl)) return giveBack("answered a form POST"); const why = await offerToHydra({ url, diff --git a/extensions/chrome/manifest.json b/extensions/chrome/manifest.json index fdfe2324..1699faa0 100644 --- a/extensions/chrome/manifest.json +++ b/extensions/chrome/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Hydra Download Manager Integration", - "version": "0.4.1", + "version": "0.4.2", "description": "A browser integration for Hydra: automatic download capture, right-click downloads, and media/HLS/DASH stream sniffing.", "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt7NVgqBOTaCTKcon/gTK0M4VxY4ZKcyh0pWTVQoWfjJrJlTvRHaWFxE4ZbIj+88zNKJBxVa6uMdQKtK2V+NaPYR1NTYyTDh75UErRI+mx5RXa5c2kiwudl2jwHPsYcUU9NPMI4FwqVFaxEetvIS9J91/3ze5l+FbsZDF0Yvc9uR1dYNnnc//nGg1R+uL0/wus4INeccd9COj10+Aka2SeBkvrVlLs63AMFMkPpN9iIBwyhgs01bulVRFbvHK9m/TizLfthMHAyMCkUdCkZr6gEZbYbwNwGwKCUIswomoiVRgZx/FBFYrh6cRlfoLQZUbS7kFTShpfjm+zEGBkWQ3yQIDAQAB", "minimum_chrome_version": "116", @@ -12,10 +12,12 @@ "storage", "tabs", "nativeMessaging", - "webRequest" + "webRequest", + "offscreen" ], "optional_permissions": ["proxy"], "host_permissions": [""], + "sandbox": { "pages": ["pac-sandbox.html"] }, "background": { "service_worker": "background.js" }, "content_scripts": [ { diff --git a/extensions/chrome/offscreen.html b/extensions/chrome/offscreen.html new file mode 100644 index 00000000..5b663bab --- /dev/null +++ b/extensions/chrome/offscreen.html @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/extensions/chrome/offscreen.js b/extensions/chrome/offscreen.js new file mode 100644 index 00000000..b28ed94b --- /dev/null +++ b/extensions/chrome/offscreen.js @@ -0,0 +1,34 @@ +// Copyright (C) 2026 Javad Rajabzadeh +// SPDX-License-Identifier: GPL-3.0-or-later +// +// The offscreen document that holds the PAC sandbox (pac-sandbox.html). The +// service worker has no DOM to frame it in, and the sandbox has no extension +// APIs to be messaged through, so this page only relays: a runtime message in, +// a postMessage to the frame, the frame's answer back. + +const frame = document.getElementById("pac"); +// The document's own load waits for the frame's, and unlike the frame's it +// cannot have fired before this script was parsed. +const loaded = + document.readyState === "complete" + ? Promise.resolve() + : new Promise((resolve) => window.addEventListener("load", resolve, { once: true })); +const pending = new Map(); +let nextId = 0; + +window.addEventListener("message", (event) => { + if (event.source !== frame.contentWindow) return; + const answer = pending.get(event.data?.id); + pending.delete(event.data?.id); + answer?.(event.data); +}); + +chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => { + if (msg?.target !== "hydra-pac") return false; + loaded.then(() => { + const id = nextId++; + pending.set(id, sendResponse); + frame.contentWindow.postMessage({ id, script: msg.script, url: msg.url, host: msg.host }, "*"); + }); + return true; +}); diff --git a/extensions/chrome/pac-sandbox.html b/extensions/chrome/pac-sandbox.html new file mode 100644 index 00000000..4d33357f --- /dev/null +++ b/extensions/chrome/pac-sandbox.html @@ -0,0 +1,7 @@ + + + + + + + diff --git a/extensions/chrome/pac.js b/extensions/chrome/pac.js new file mode 100644 index 00000000..80dfffb9 --- /dev/null +++ b/extensions/chrome/pac.js @@ -0,0 +1,167 @@ +// Copyright (C) 2026 Javad Rajabzadeh +// SPDX-License-Identifier: GPL-3.0-or-later +// +// A PAC script evaluator, run inside the manifest's sandbox page: the one +// place an MV3 extension may compile a string into code. It answers the +// question Chromium asks a proxy switcher's script (ZeroOmega, SwitchyOmega, +// a corporate wpad.dat) for one URL, and nothing else. +// +// There is no DNS here. A PAC helper that would resolve a name answers as if +// the name did not resolve, which is also what the browser does offline; an +// IP literal still resolves to itself, so `isInNet` keeps working for those. + +const PAC_DAYS = ["SUN", "MON", "TUE", "WED", "THU", "FRI", "SAT"]; +const PAC_MONTHS = [ + "JAN", + "FEB", + "MAR", + "APR", + "MAY", + "JUN", + "JUL", + "AUG", + "SEP", + "OCT", + "NOV", + "DEC", +]; +const IPV4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/; + +function convert_addr(ip) { + const m = IPV4.exec(String(ip)); + return m ? ((m[1] << 24) | (m[2] << 16) | (m[3] << 8) | m[4]) >>> 0 : 0; +} + +function dnsResolve(host) { + const h = String(host); + return IPV4.test(h) || h.includes(":") ? h : null; +} + +function isInNet(host, pattern, mask) { + const ip = dnsResolve(host); + if (!ip || !IPV4.test(ip)) return false; + const m = convert_addr(mask); + return (convert_addr(ip) & m) >>> 0 === (convert_addr(pattern) & m) >>> 0; +} + +function shExpMatch(str, pattern) { + const rx = String(pattern) + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, ".*") + .replace(/\?/g, "."); + return new RegExp(`^${rx}$`).test(String(str)); +} + +/// The clock a time helper reads, and its arguments without the trailing +/// "GMT" that switches it to UTC. +function pacClock(args) { + const gmt = args.length > 0 && String(args.at(-1)).toUpperCase() === "GMT"; + const now = new Date(); + const get = (local, utc) => (gmt ? now[utc]() : now[local]()); + return { + args: gmt ? args.slice(0, -1) : args, + day: get("getDate", "getUTCDate"), + month: get("getMonth", "getUTCMonth"), + year: get("getFullYear", "getUTCFullYear"), + weekday: get("getDay", "getUTCDay"), + seconds: + get("getHours", "getUTCHours") * 3600 + + get("getMinutes", "getUTCMinutes") * 60 + + get("getSeconds", "getUTCSeconds"), + }; +} + +/// `lo <= v <= hi`, where a range whose end comes before its start wraps +/// around (FRI..MON, NOV..FEB, 22h..6h). +const inWrappedRange = (v, lo, hi) => (lo <= hi ? lo <= v && v <= hi : v >= lo || v <= hi); + +function weekdayRange(...raw) { + const { args, weekday } = pacClock(raw); + const lo = PAC_DAYS.indexOf(String(args[0]).toUpperCase()); + const hi = args.length > 1 ? PAC_DAYS.indexOf(String(args[1]).toUpperCase()) : lo; + return lo >= 0 && hi >= 0 && inWrappedRange(weekday, lo, hi); +} + +// Each argument is a day (1-31), a month name or a four-digit year; the +// bounds compare as year-month-day numbers built from the fields given. +function dateRange(...raw) { + const clock = pacClock(raw); + const kinds = clock.args.map((a) => { + const month = PAC_MONTHS.indexOf(String(a).toUpperCase()); + if (month >= 0) return ["month", month]; + const n = Number(a); + return n > 31 ? ["year", n] : ["day", n]; + }); + if (kinds.length !== 1 && kinds.length % 2) return false; + const half = kinds.length === 1 ? kinds : kinds.slice(0, kinds.length / 2); + const end = kinds.length === 1 ? kinds : kinds.slice(kinds.length / 2); + const weight = { year: 10000, month: 100, day: 1 }; + const value = (fields) => fields.reduce((sum, [k, v]) => sum + v * weight[k], 0); + const now = value(half.map(([k]) => [k, clock[k]])); + if (half.some(([k]) => k === "year")) return value(half) <= now && now <= value(end); + return inWrappedRange(now, value(half), value(end)); +} + +// timeRange(h), timeRange(h1, h2), (h1, m1, h2, m2) or (h1, m1, s1, h2, m2, s2). +function timeRange(...raw) { + const { args, seconds } = pacClock(raw); + const n = args.map(Number); + if (n.length === 1) return Math.floor(seconds / 3600) === n[0]; + const at = (h, m = 0, s = 0) => h * 3600 + m * 60 + s; + if (n.length === 2) return inWrappedRange(seconds, at(n[0]), at(n[1]) - 1); + if (n.length === 4) return inWrappedRange(seconds, at(n[0], n[1]), at(n[2], n[3])); + if (n.length === 6) return inWrappedRange(seconds, at(n[0], n[1], n[2]), at(n[3], n[4], n[5])); + return false; +} + +const PAC_HELPERS = { + isPlainHostName: (host) => !String(host).includes("."), + dnsDomainIs: (host, domain) => String(host).toLowerCase().endsWith(String(domain).toLowerCase()), + localHostOrDomainIs: (host, hostdom) => { + const h = String(host).toLowerCase(); + const full = String(hostdom).toLowerCase(); + return h === full || (!h.includes(".") && full.startsWith(`${h}.`)); + }, + isResolvable: (host) => dnsResolve(host) !== null, + isInNet, + dnsResolve, + myIpAddress: () => "127.0.0.1", + dnsDomainLevels: (host) => String(host).split(".").length - 1, + shExpMatch, + weekdayRange, + dateRange, + timeRange, + convert_addr, + alert: () => {}, +}; + +// A capture asks once per download and the popup once per open, nearly always +// with the same script: compiling a rule list of thousands of entries again +// each time is the whole cost of answering. +let compiled = { script: null, find: null }; + +/// What the script's `FindProxyForURL` returns for this URL, as a string. +function pacFindProxy(script, url, host) { + if (compiled.script !== script) { + const names = Object.keys(PAC_HELPERS); + const find = new Function( + ...names, + `${script}\n;return typeof FindProxyForURL === "function" ? FindProxyForURL : null;`, + )(...Object.values(PAC_HELPERS)); + if (typeof find !== "function") throw new Error("the script defines no FindProxyForURL"); + compiled = { script, find }; + } + return String(compiled.find(url, host) ?? ""); +} + +globalThis.addEventListener?.("message", (event) => { + const { id, script, url, host } = event.data || {}; + if (typeof id !== "number" || typeof script !== "string") return; + let reply; + try { + reply = { id, result: pacFindProxy(script, url, host) }; + } catch (e) { + reply = { id, error: String(e?.message || e) }; + } + event.source?.postMessage(reply, "*"); +}); diff --git a/extensions/chrome/popup.js b/extensions/chrome/popup.js index e1cb4854..fe8ee637 100644 --- a/extensions/chrome/popup.js +++ b/extensions/chrome/popup.js @@ -204,9 +204,9 @@ const PROXY_OPTIONAL = (chrome.runtime.getManifest().optional_permissions || []) "proxy" ); -// Set, but to something the browser will not spell out for us. +// Direct for this page, or set to something the browser will not spell out. const PROXY_OPAQUE = - "This browser is set to follow the machine's proxy, or a configuration script. Neither says what it resolves to, so downloads take whatever Hydra's Options say."; + "This browser goes direct for this page, or follows the machine's proxy, which it does not say. Downloads take whatever Hydra's Options say."; async function refreshProxy() { if (!PROXY_OPTIONAL) return; diff --git a/extensions/firefox/background.js b/extensions/firefox/background.js index 186d9bf1..0e8d65e9 100644 --- a/extensions/firefox/background.js +++ b/extensions/firefox/background.js @@ -102,11 +102,19 @@ async function closeBlankTab(tabId) { /// Blocking a response to decide costs the round-trip to the app, paid only /// on transfers that were going to be downloads anyway. function interceptResponse(details) { + // core.js observes every response too, but nothing promises its listener + // runs before this one, and the capture below asks for this very route. + noteProxyInfo(details); const headers = Object.fromEntries( (details.responseHeaders || []).map((h) => [h.name.toLowerCase(), h.value]), ); if (!willDownload(headers)) return; if (browserOwned.get(details.url) !== undefined) return; + if (details.method === "POST") { + // Its request is in a body Hydra is never handed; see `notePosted`. + headerVerdicts.set(details.url); + return; + } // The same choice `decideCapture` makes, given the same two URLs: a resolved // URL that is a short-lived signature is dead by Hydra's first retry, and diff --git a/extensions/firefox/manifest.json b/extensions/firefox/manifest.json index dfd0a827..9dea01f7 100644 --- a/extensions/firefox/manifest.json +++ b/extensions/firefox/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Hydra Download Manager Integration", - "version": "0.4.1", + "version": "0.4.2", "description": "A browser integration for Hydra: automatic download capture, right-click downloads, and media/HLS/DASH stream sniffing.", "browser_specific_settings": { "gecko": { diff --git a/extensions/safari/Resources/background.js b/extensions/safari/Resources/background.js index c4ebcb82..fd245ad0 100644 --- a/extensions/safari/Resources/background.js +++ b/extensions/safari/Resources/background.js @@ -460,11 +460,13 @@ async function altBypassed() { // and a second stored flag beside the permission could only ever disagree // with it. // -// Two things are deliberately NOT captured. A browser following the machine's -// proxy ("system" / "autoDetect") will not say what that is, and a PAC script -// is a program neither this extension nor Hydra runs — see the same note in -// hydra-gui's proxy.rs. Both resolve to "no proxy known", which leaves the +// A browser following the machine's proxy ("system" / "autoDetect") will not +// say what that is, and resolves to "no proxy known", which leaves the // download on whatever Options says, exactly as if this were switched off. +// A proxy switcher (ZeroOmega, SwitchyOmega) is the other case people have: +// in Chromium it installs a PAC script, which is run here for the one URL; +// in Firefox it routes each request from a listener the settings never show, +// so the route a response actually took is remembered instead. // A local settings read answers in microseconds. The ceiling is not about // slowness: a capture AWAITS this, and a browser whose namespace neither @@ -618,12 +620,147 @@ function geckoProxy(cfg, url) { ); } +// What a PAC entry's keyword means to Chromium, the one running the script. +// A bare SOCKS there is version 4, unlike the bare word in a settings field. +const PAC_KINDS = { + PROXY: "http", + HTTPS: "https", + SOCKS: "socks4", + SOCKS4: "socks4", + SOCKS5: "socks5", +}; + +/// The first usable entry of a `FindProxyForURL` answer such as +/// `PROXY 127.0.0.1:8080; DIRECT`. DIRECT ahead of any proxy means the +/// browser goes direct, and so does the download. +function pacAnswerProxy(answer) { + for (const entry of String(answer ?? "").split(";")) { + const [kind, where] = entry.trim().split(/\s+/); + if (!kind) continue; + const upper = kind.toUpperCase(); + if (upper === "DIRECT") return null; + const spec = upper in PAC_KINDS ? proxySpec(PAC_KINDS[upper], where) : null; + if (spec) return spec; + } + return null; +} + +// A rule list can run to thousands of entries and the offscreen document may +// have to be created first; still, a capture awaits this, so a script that +// loops or a PAC server that never answers costs a bounded wait and then +// "no proxy known". +const PAC_TIMEOUT_MS = 3000; +const PAC_DOCUMENT = "offscreen.html"; +let pacDocumentCreating = null; + +async function ensurePacDocument() { + const open = await chrome.runtime.getContexts({ contextTypes: ["OFFSCREEN_DOCUMENT"] }); + if (open.length) return; + pacDocumentCreating ??= chrome.offscreen + .createDocument({ + url: PAC_DOCUMENT, + reasons: ["IFRAME_SCRIPTING"], + justification: "Evaluate the browser's proxy auto-config script for a captured download.", + }) + // Losing a race with a service worker instance that created it first + // still leaves a document to talk to. + .catch(() => {}) + .finally(() => { + pacDocumentCreating = null; + }); + await pacDocumentCreating; +} + +async function pacScriptText(pac) { + if (typeof pac?.data === "string" && pac.data.trim()) return pac.data; + if (!pac?.url) return null; + const res = await fetch(pac.url, { cache: "no-store" }); + return res.ok ? res.text() : null; +} + +/// Run Chromium's PAC script for this URL in the sandbox, as the browser +/// would: an https URL is reduced to its origin before the script sees it. +async function evaluatePac(pac, url) { + const script = await pacScriptText(pac); + if (!script) return null; + await ensurePacDocument(); + const u = new URL(url); + const reply = await chrome.runtime.sendMessage({ + target: "hydra-pac", + script, + url: u.protocol === "https:" ? `${u.origin}/` : url, + host: u.hostname.replace(/^\[|\]$/g, ""), + }); + return reply && !reply.error ? pacAnswerProxy(reply.result) : null; +} + +async function pacProxy(pac, url) { + if (!chrome.offscreen?.createDocument || !chrome.runtime.getContexts) return null; + const expired = Symbol("expired"); + let timer; + const deadline = new Promise((resolve) => { + timer = setTimeout(() => resolve(expired), PAC_TIMEOUT_MS); + }); + const answer = evaluatePac(pac, url).catch(() => null); + const got = await Promise.race([answer, deadline]); + clearTimeout(timer); + if (got !== expired) return got; + // A script stuck in a loop holds the sandbox for good; closing the document + // discards it, and the next capture starts a fresh one. + chrome.offscreen.closeDocument().catch(() => {}); + return null; +} + +// Firefox: the proxy each recent response really came through, by origin. +// Bounded, oldest first out; an origin seen again moves to the back. +const OBSERVED_PROXY_ORIGINS = 256; +const observedProxies = new Map(); + +function proxyInfoSpec(info) { + if (!info?.host) return null; + const host = info.host.includes(":") && !info.host.startsWith("[") ? `[${info.host}]` : info.host; + const kinds = { http: "http", https: "https", socks: "socks5" }; + const kind = info.type === "socks4" ? (info.proxyDNS ? "socks4a" : "socks4") : kinds[info.type]; + return kind ? proxySpec(kind, `${host}:${info.port}`) : null; +} + +/// Note the route a Firefox webRequest event reports. Credentials are left +/// out, as in `proxySpec`. +function noteProxyInfo(details) { + let origin; + try { + origin = new URL(details.url).origin; + } catch { + return; + } + observedProxies.delete(origin); + observedProxies.set(origin, proxyInfoSpec(details.proxyInfo)); + if (observedProxies.size > OBSERVED_PROXY_ORIGINS) { + observedProxies.delete(observedProxies.keys().next().value); + } +} + +function observedProxy(url) { + try { + return observedProxies.get(new URL(url).origin) ?? null; + } catch { + return null; + } +} + +if (BROWSER === "Mozilla Firefox") { + chrome.webRequest?.onHeadersReceived.addListener(noteProxyInfo, { urls: [""] }); +} + /// The proxy to hand over with this URL, or null for "Hydra decides". async function captureProxy(url) { const cfg = await proxyConfig(); if (!cfg) return null; // The two browsers answer with different objects; each names itself. - if (typeof cfg.proxyType === "string") return geckoProxy(cfg, url); + if (typeof cfg.proxyType === "string") { + return cfg.proxyType === "manual" ? geckoProxy(cfg, url) : observedProxy(url); + } + if (cfg.mode === "pac_script") return pacProxy(cfg.pacScript, url); if (typeof cfg.mode === "string") return chromiumProxy(cfg, url); return null; } @@ -967,6 +1104,36 @@ async function retryDeferredCaptures() { // therefore whether there is anything to hand back. Resuming one that was // never parked is not harmless: on Gecko it throws, and the throw used to be // swallowed next to a download that had already been cancelled. +// ------------------------------------------------------------ form posts + +// A download that answered a form POST — SharePoint and OneDrive zip a folder +// by posting its file list and tokens to `*.svc.ms/transform/zip` — has its +// request in the body. Hydra is handed a URL and asks with a bodiless GET, +// which the service refuses, so such a download stays with the browser. +const POSTED_TTL_MS = 120_000; +const POSTED_MAX = 64; +const postedUrls = new Map(); + +function notePosted(details) { + if (details.method !== "POST") return; + const url = details.url.split("#")[0]; + postedUrls.delete(url); + postedUrls.set(url, Date.now()); + if (postedUrls.size > POSTED_MAX) postedUrls.delete(postedUrls.keys().next().value); +} + +function wasPosted(...urls) { + return urls.some((url) => { + const at = url && postedUrls.get(url.split("#")[0]); + return !!at && Date.now() - at < POSTED_TTL_MS; + }); +} + +chrome.webRequest?.onBeforeRequest?.addListener(notePosted, { + urls: ["http://*/*", "https://*/*"], + types: ["main_frame", "sub_frame"], +}); + async function decideCapture(item, parked) { const url = captureUrl(item); @@ -984,6 +1151,7 @@ async function decideCapture(item, parked) { }; if (item.byExtensionId === chrome.runtime.id) return giveBack("started by this extension"); + if (wasPosted(url, item.url, item.finalUrl)) return giveBack("answered a form POST"); const why = await offerToHydra({ url, diff --git a/extensions/safari/Resources/manifest.json b/extensions/safari/Resources/manifest.json index 2bbaf604..c9b30765 100644 --- a/extensions/safari/Resources/manifest.json +++ b/extensions/safari/Resources/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Hydra Download Manager Integration", - "version": "0.4.1", + "version": "0.4.2", "description": "A browser integration for Hydra: right-click downloads, link selection, and media/HLS/DASH stream sniffing.", "browser_specific_settings": { "safari": { diff --git a/extensions/safari/Resources/popup.js b/extensions/safari/Resources/popup.js index e1cb4854..fe8ee637 100644 --- a/extensions/safari/Resources/popup.js +++ b/extensions/safari/Resources/popup.js @@ -204,9 +204,9 @@ const PROXY_OPTIONAL = (chrome.runtime.getManifest().optional_permissions || []) "proxy" ); -// Set, but to something the browser will not spell out for us. +// Direct for this page, or set to something the browser will not spell out. const PROXY_OPAQUE = - "This browser is set to follow the machine's proxy, or a configuration script. Neither says what it resolves to, so downloads take whatever Hydra's Options say."; + "This browser goes direct for this page, or follows the machine's proxy, which it does not say. Downloads take whatever Hydra's Options say."; async function refreshProxy() { if (!PROXY_OPTIONAL) return; diff --git a/extensions/safari/manifest.json b/extensions/safari/manifest.json index 2bbaf604..c9b30765 100644 --- a/extensions/safari/manifest.json +++ b/extensions/safari/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Hydra Download Manager Integration", - "version": "0.4.1", + "version": "0.4.2", "description": "A browser integration for Hydra: right-click downloads, link selection, and media/HLS/DASH stream sniffing.", "browser_specific_settings": { "safari": { diff --git a/extensions/tests/capture.test.mjs b/extensions/tests/capture.test.mjs index 14732494..4a0ff826 100644 --- a/extensions/tests/capture.test.mjs +++ b/extensions/tests/capture.test.mjs @@ -18,11 +18,20 @@ const check = (label, cond, extra = "") => { }; const tick = (n = 3) => new Promise((r) => { let i = 0; const f = () => (++i >= n ? r() : setImmediate(f)); setImmediate(f); }); -function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = null, proxy, proxyStyle = "callback", cookieJar = null, frames = null, fetchImpl = null } = {}) { +// The PAC evaluator the Chromium sandbox page runs, loaded the same way: a +// classic script whose declarations land on its global. +function pacSandbox() { + const ctx = vm.createContext({}); + vm.runInContext(readFileSync("extensions/chrome/pac.js", "utf8"), ctx, { filename: "pac.js" }); + return ctx; +} + +function build({ hydraReply = { ok: true }, store = {}, gecko = false, ua = UA, tab = null, proxy, proxyStyle = "callback", cookieJar = null, frames = null, fetchImpl = null, offscreen = null } = {}) { const sent = []; // messages that reached "Hydra" const calls = []; // downloads API calls const cookieQueries = []; // every cookies.getAll details object - const listeners = {}; + const listeners = { headerObservers: [] }; + const pacDoc = { open: false, created: [], closed: 0, asked: [], sandbox: null }; const ev = () => { const l = []; return { addListener: (f) => l.push(f), fire: (...a) => l.map((f) => f(...a)), l }; }; const storage = () => ({ @@ -45,7 +54,39 @@ function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = nul onMessage: (listeners.msg = ev()), getURL: (p) => `chrome-extension://x/${p}`, sendNativeMessage: (_h, _m, cb) => cb(undefined), + // The offscreen document, as the service worker reaches it: "pac" runs + // the real sandbox evaluator, "hang" is a script that never returns. + ...(offscreen + ? { + getContexts: async () => (pacDoc.open ? [{ contextType: "OFFSCREEN_DOCUMENT" }] : []), + sendMessage: async (msg) => { + pacDoc.asked.push(msg); + if (!pacDoc.open || msg.target !== "hydra-pac") throw new Error("Receiving end does not exist."); + if (offscreen === "hang") return new Promise(() => {}); + try { + return { result: (pacDoc.sandbox ??= pacSandbox()).pacFindProxy(msg.script, msg.url, msg.host) }; + } catch (e) { + return { error: String(e) }; + } + }, + } + : {}), }, + ...(offscreen + ? { + offscreen: { + createDocument: async (opts) => { + if (pacDoc.open) throw new Error("Only a single offscreen document may be created."); + pacDoc.created.push(opts); + pacDoc.open = true; + }, + closeDocument: async () => { + pacDoc.open = false; + pacDoc.closed++; + }, + }, + } + : {}), storage: { local: storage(), session: storage() }, action: { setBadgeBackgroundColor: async () => {}, setBadgeText: async () => {}, setTitle: async () => {}, @@ -112,6 +153,7 @@ function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = nul }, onHeadersReceived: { addListener: (fn, filter, spec) => { + if (!spec?.includes("blocking")) return listeners.headerObservers.push(fn); listeners.headers = fn; listeners.headersFilter = filter; listeners.headersSpec = spec; @@ -120,6 +162,9 @@ function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = nul onBeforeRedirect: { addListener: (fn) => { listeners.redirect = fn; }, }, + onBeforeRequest: { + addListener: (fn, filter) => { listeners.request = fn; listeners.requestFilter = filter; }, + }, }, }; @@ -138,7 +183,7 @@ function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = nul } const ctx = vm.createContext({ - chrome, navigator: { userAgent: UA }, WebSocket: FakeWS, console, + chrome, navigator: { userAgent: ua }, WebSocket: FakeWS, console, setTimeout, clearTimeout, setInterval, clearInterval, setImmediate, fetch: fetchImpl ?? (async () => { throw new Error("no net"); }), // A fresh vm context has the ECMAScript built-ins only; URL and friends // are web globals the service worker relies on, so hand them in. @@ -168,19 +213,26 @@ function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = nul // registered `types` filter the way the browser gates it: a test that called // the listener directly would pass against a filter the response never // reaches. Returns whatever the listener returns — the blocking verdict. - const headResponse = ({ url, headers = {}, type = "main_frame", tabId = 7, requestId = "r1" }) => + const headResponse = ({ url, headers = {}, type = "main_frame", tabId = 7, requestId = "r1", proxyInfo, method = "GET" }) => (listeners.headersFilter?.types || []).includes(type) ? listeners.headers?.({ - url, type, tabId, requestId, + url, type, tabId, requestId, proxyInfo, method, responseHeaders: Object.entries(headers).map(([name, value]) => ({ name, value })), }) : undefined; + // Any response at all, as Firefox shows it to a non-blocking observer. + const observed = ({ url, proxyInfo = null, type = "xmlhttprequest" }) => + listeners.headerObservers.forEach((fn) => fn({ url, type, tabId: 7, requestId: "o1", proxyInfo })); + // A request as the browser announces it before sending, gated like the rest. + const requested = ({ url, method = "GET", type = "sub_frame" }) => + (listeners.requestFilter?.types || []).includes(type) && + listeners.request?.({ url, method, type, tabId: 7, requestId: "q1" }); const redirected = ({ url, redirectUrl, requestId = "r1" }) => listeners.redirect?.({ url, redirectUrl, requestId, type: "main_frame", tabId: 7 }); return { sent, calls, onCreated, onDetermining, onMenu, send, store, respond, - headResponse, redirected, cookieQueries, + headResponse, redirected, requested, cookieQueries, observed, pacDoc, sniffFilter: () => listeners.sniffFilter, headersSpec: () => listeners.headersSpec, }; @@ -217,6 +269,49 @@ function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = nul check("capture: it is NOT resumed once Hydra took it", !h.calls.some(([c]) => c === "resume")); } +// ------------------------------------------- 1b. a download a form POSTed +// +// SharePoint zips a shared folder by posting the file list and its tokens to +// the zip service from a hidden frame. Hydra would replay the URL as a GET +// with no body, and the service answers that with an error. +{ + const h = build(); + await tick(4); + const zip = "https://northeurope1-mediap.svc.ms/transform/zip?cs=abc"; + h.requested({ url: zip, method: "POST" }); + const item = { id: 11, url: zip, filename: "share.zip", mime: "application/zip", totalBytes: 0 }; + h.onCreated.fire(item); + await tick(3); + h.onDetermining.fire(item, () => h.calls.push(["suggest"])); + await tick(6); + check("form post: the zip is not sent to Hydra", !h.sent.some((m) => m.type === "download"), JSON.stringify(h.sent)); + check("form post: the browser keeps it", h.calls.some(([c]) => c === "resume")); + + const plain = "https://cdn.example/after.zip"; + h.requested({ url: plain, method: "GET" }); + const next = { id: 12, url: plain, filename: "after.zip", mime: "application/zip", totalBytes: 5e6 }; + h.onCreated.fire(next); + await tick(3); + h.onDetermining.fire(next, () => h.calls.push(["suggest"])); + await tick(6); + check("form post: a GET download is still captured", h.sent.some((m) => m.type === "download" && m.url === plain)); +} +{ + const h = build({ gecko: true }); + await tick(4); + const zip = "https://northeurope1-mediap.svc.ms/transform/zip?cs=abc"; + const verdict = await h.headResponse({ + url: zip, + type: "sub_frame", + method: "POST", + headers: { "Content-Type": "application/zip", "Content-Disposition": "attachment; filename=share.zip" }, + }); + check("form post (gecko): the response is let through", verdict === undefined, JSON.stringify(verdict)); + h.onCreated.fire({ id: 13, url: zip, filename: "share.zip", mime: "application/zip" }); + await tick(6); + check("form post (gecko): nothing reaches Hydra", !h.sent.some((m) => m.type === "download"), JSON.stringify(h.sent)); +} + // ------------------------------------- 2. a type outside the capture list { const h = build(); @@ -836,9 +931,9 @@ function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = nul } { // "system" is the browser saying it follows the machine and will not say - // what that is; a PAC script is a program neither side runs. Both leave - // the download on whatever Hydra's Options say. - for (const mode of ["system", "direct", "auto_detect", "pac_script"]) { + // what that is. That, and going direct, leave the download on whatever + // Hydra's Options say. + for (const mode of ["system", "direct", "auto_detect"]) { const h = build({ proxy: { mode, rules: { singleProxy: { scheme: "socks5", host: "h", port: 1 } } } }); await tick(4); await h.send({ type: "download-url", url: "https://cdn.example/pack.zip" }); @@ -846,6 +941,162 @@ function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = nul h.sent.at(-1)?.proxy === null, JSON.stringify(h.sent.at(-1)?.proxy)); } } +// ZeroOmega's Auto Switch profile, as Chromium holds it: a generated PAC +// script that picks a profile per host and ends in a FindProxyForURL answer. +const SWITCHER_PAC = ` +var FindProxyForURL = function(init, profiles) { + return function(url, host) { + "use strict"; + var result = init, scheme = url.substr(0, url.indexOf(":")); + do { + if (!profiles[result]) return result; + result = profiles[result]; + if (typeof result === "function") result = result(url, host, scheme); + } while (typeof result !== "string" || result.charCodeAt(0) === 43); + return result; + }; +}("+auto switch", { + "+auto switch": function(url, host, scheme) { + "use strict"; + if (/(?:^|\\.)blocked\\.example$/.test(host)) return "+proxy"; + if (shExpMatch(host, "*.tunnel.example")) return "+socks"; + return "DIRECT"; + }, + "+proxy": function(url, host, scheme) { + "use strict"; + return "PROXY 127.0.0.1:8080; DIRECT"; + }, + "+socks": function(url, host, scheme) { + "use strict"; + return "SOCKS5 127.0.0.1:1080; SOCKS 127.0.0.1:1080"; + } +});`; +{ + // ZeroOmega's Auto Switch in Chromium: the browser's setting is a PAC + // script, and what it resolves to depends on the URL. + const h = build({ offscreen: "pac", proxy: { mode: "pac_script", pacScript: { data: SWITCHER_PAC, mandatory: true } } }); + await tick(4); + await h.send({ type: "download-url", url: "https://cdn.blocked.example/pack.zip" }); + check("pac: a host the switcher routes through its proxy takes that proxy", + h.sent.at(-1)?.proxy === "http://127.0.0.1:8080", JSON.stringify(h.sent.at(-1)?.proxy)); + await h.send({ type: "download-url", url: "https://files.tunnel.example/pack.zip" }); + check("pac: a SOCKS5 profile is handed over as socks5", + h.sent.at(-1)?.proxy === "socks5://127.0.0.1:1080", JSON.stringify(h.sent.at(-1)?.proxy)); + await h.send({ type: "download-url", url: "https://open.example/pack.zip" }); + check("pac: a host the script sends DIRECT gets no proxy", + h.sent.at(-1)?.proxy === null, JSON.stringify(h.sent.at(-1)?.proxy)); + check("pac: one offscreen document serves every capture", + h.pacDoc.created.length === 1 && h.pacDoc.created[0].reasons.includes("IFRAME_SCRIPTING"), + JSON.stringify(h.pacDoc.created)); + // Chromium hands the script only the origin of an https URL; a rule keyed + // on a path must not see one here that the browser would not show it. + check("pac: an https URL reaches the script as its origin only", + h.pacDoc.asked[0]?.url === "https://cdn.blocked.example/" && h.pacDoc.asked[0]?.host === "cdn.blocked.example", + JSON.stringify(h.pacDoc.asked[0])); + + check("pac: the popup is told the route for the page in front", + (await h.send({ type: "proxy-status" }))?.spec === null); +} +{ + // ZeroOmega's PAC profile points at a URL rather than inlining the script. + const fetchImpl = async (url) => ({ ok: url === "http://pac.example/proxy.pac", text: async () => SWITCHER_PAC }); + const h = build({ offscreen: "pac", fetchImpl, proxy: { mode: "pac_script", pacScript: { url: "http://pac.example/proxy.pac" } } }); + await tick(4); + await h.send({ type: "download-url", url: "http://www.blocked.example/pack.zip" }); + check("pac: a script fetched by URL is run the same way", + h.sent.at(-1)?.proxy === "http://127.0.0.1:8080", JSON.stringify(h.sent.at(-1)?.proxy)); +} +{ + // A script that throws, or one with no FindProxyForURL, is not a proxy. + for (const data of ["throw new Error('boom')", "var nothing = 1;"]) { + const h = build({ offscreen: "pac", proxy: { mode: "pac_script", pacScript: { data } } }); + await tick(4); + await h.send({ type: "download-url", url: "https://cdn.example/pack.zip" }); + check(`pac: a broken script leaves Hydra to decide (${data})`, + h.sent.at(-1)?.proxy === null, JSON.stringify(h.sent.at(-1)?.proxy)); + } +} +{ + // A script that never returns: the capture still goes out, without a + // proxy, and the stuck document is discarded for the next one. + const h = build({ offscreen: "hang", proxy: { mode: "pac_script", pacScript: { data: SWITCHER_PAC } } }); + await tick(4); + const sent = h.send({ type: "download-url", url: "https://cdn.blocked.example/pack.zip" }); + const raced = await Promise.race([sent.then(() => "answered"), new Promise((r) => setTimeout(() => r("hung"), 6000))]); + check("pac: a script that never answers does not park the download", + raced === "answered" && h.sent.at(-1)?.proxy === null, JSON.stringify([raced, h.sent.at(-1)?.proxy])); + check("pac: and the stuck document is closed", h.pacDoc.closed === 1, String(h.pacDoc.closed)); +} +{ + // A browser with no offscreen API (Firefox reports PAC as autoConfig and + // never reaches this) has nowhere to run the script: no proxy known. + const h = build({ proxy: { mode: "pac_script", pacScript: { data: SWITCHER_PAC } } }); + await tick(4); + await h.send({ type: "download-url", url: "https://cdn.blocked.example/pack.zip" }); + check("pac: without an offscreen document the script is not run", + h.sent.at(-1)?.proxy === null, JSON.stringify(h.sent.at(-1)?.proxy)); +} + +const FIREFOX_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0"; +{ + // ZeroOmega in Firefox routes each request from `proxy.onRequest`; the + // settings still say "system", and only the response says where it went. + const h = build({ gecko: true, ua: FIREFOX_UA, proxy: { proxyType: "system" } }); + await tick(4); + h.observed({ url: "https://cdn.example/page.js", proxyInfo: { type: "socks", host: "127.0.0.1", port: 1080, proxyDNS: true, username: "u" } }); + h.observed({ url: "https://open.example/page.js", proxyInfo: { type: "direct", host: "", port: -1 } }); + await h.send({ type: "download-url", url: "https://cdn.example/pack.zip" }); + check("firefox route: the proxy a response from that origin took is sent", + h.sent.at(-1)?.proxy === "socks5://127.0.0.1:1080", JSON.stringify(h.sent.at(-1)?.proxy)); + await h.send({ type: "download-url", url: "https://open.example/pack.zip" }); + check("firefox route: an origin reached directly gets no proxy", + h.sent.at(-1)?.proxy === null, JSON.stringify(h.sent.at(-1)?.proxy)); + await h.send({ type: "download-url", url: "https://never-seen.example/pack.zip" }); + check("firefox route: an origin never seen is left to Hydra", + h.sent.at(-1)?.proxy === null, JSON.stringify(h.sent.at(-1)?.proxy)); + await h.send({ type: "download-url", url: "http://cdn.example/pack.zip" }); + check("firefox route: another scheme is another origin", + h.sent.at(-1)?.proxy === null, JSON.stringify(h.sent.at(-1)?.proxy)); + + // The route changes when the switcher's rules do; the latest one wins. + h.observed({ url: "https://cdn.example/other.js", proxyInfo: { type: "http", host: "::1", port: 3128 } }); + await h.send({ type: "download-url", url: "https://cdn.example/pack.zip" }); + check("firefox route: the newest observation wins, IPv6 bracketed", + h.sent.at(-1)?.proxy === "http://[::1]:3128", JSON.stringify(h.sent.at(-1)?.proxy)); +} +{ + // The download's own response, at the blocking header stage, is the first + // the extension sees from that origin — the capture it triggers must use it. + const h = build({ gecko: true, ua: FIREFOX_UA, proxy: { proxyType: "none" } }); + await tick(4); + const verdict = h.headResponse({ + url: "https://dl.example/pack.zip", + headers: { "content-type": "application/zip", "content-length": "5000000" }, + proxyInfo: { type: "socks4", host: "10.0.0.1", port: 1081, proxyDNS: false }, + }); + await verdict; + const dl = h.sent.find((m) => m.type === "download"); + check("firefox route: the download's own response names its proxy", + dl?.proxy === "socks4://10.0.0.1:1081", JSON.stringify(dl?.proxy)); +} +{ + // Off means off: without the permission the observation is never used. + const h = build({ gecko: true, ua: FIREFOX_UA }); + await tick(4); + h.observed({ url: "https://cdn.example/page.js", proxyInfo: { type: "http", host: "proxy.example", port: 8080 } }); + await h.send({ type: "download-url", url: "https://cdn.example/pack.zip" }); + check("firefox route: without the permission nothing is passed on", + h.sent.at(-1)?.proxy === null, JSON.stringify(h.sent.at(-1)?.proxy)); +} +{ + // Manual settings still decide on their own, bypass list included. + const h = build({ gecko: true, ua: FIREFOX_UA, proxy: { proxyType: "manual", http: "proxy.example:3128", passthrough: ".corp.example" } }); + await tick(4); + h.observed({ url: "https://files.corp.example/x", proxyInfo: { type: "http", host: "other.example", port: 1 } }); + await h.send({ type: "download-url", url: "https://files.corp.example/pack.zip" }); + check("firefox route: manual settings with a bypass are not second-guessed", + h.sent.at(-1)?.proxy === null, JSON.stringify(h.sent.at(-1)?.proxy)); +} { // Firefox's own shape, and the SOCKS fallback for a scheme its per-protocol // fields do not cover. @@ -923,6 +1174,12 @@ function build({ hydraReply = { ok: true }, store = {}, gecko = false, tab = nul const safari = JSON.parse(readFileSync("extensions/safari/manifest.json", "utf8")); check("proxy: Safari has no proxy API, so it does not ask for one", !(safari.optional_permissions || []).includes("proxy")); + check("pac: the sandbox page is declared, so it may compile the script", + (manifest.sandbox?.pages || []).includes("pac-sandbox.html") && manifest.permissions.includes("offscreen"), + JSON.stringify([manifest.sandbox, manifest.permissions])); + const firefox = JSON.parse(readFileSync("extensions/firefox/manifest.json", "utf8")); + check("pac: Firefox, with no offscreen documents, is not asked for one", + !firefox.sandbox && !firefox.permissions.includes("offscreen")); } // ------------------------------------------- 7. what the Firefox manifest pins diff --git a/extensions/tests/pac.test.mjs b/extensions/tests/pac.test.mjs new file mode 100644 index 00000000..84038216 --- /dev/null +++ b/extensions/tests/pac.test.mjs @@ -0,0 +1,107 @@ +// Copyright (C) 2026 Javad Rajabzadeh +// SPDX-License-Identifier: GPL-3.0-or-later +// +// The PAC evaluator the Chromium sandbox page runs, and the helpers a proxy +// switcher's generated script calls. +// Run from the repository root: node extensions/tests/pac.test.mjs +import { readFileSync } from "node:fs"; +import vm from "node:vm"; + +let fails = 0; +const check = (label, cond, extra = "") => { + if (cond) console.log(`ok ${label}`); + else { fails++; console.log(`FAIL ${label} ${extra}`); } +}; + +const posted = []; +const listeners = []; +const ctx = vm.createContext({ addEventListener: (type, fn) => type === "message" && listeners.push(fn) }); +vm.runInContext(readFileSync("extensions/chrome/pac.js", "utf8"), ctx, { filename: "pac.js" }); +const find = (script, url) => ctx.pacFindProxy(script, url, new URL(url).hostname); + +// SwitchyOmega / ZeroOmega's generated shape: a rule list that resolves +// profile names ("+name") until it reaches a PAC answer. +const SWITCHY = ` +var FindProxyForURL = function(init, profiles) { + return function(url, host) { + "use strict"; + var result = init, scheme = url.substr(0, url.indexOf(":")); + do { + if (!profiles[result]) return result; + result = profiles[result]; + if (typeof result === "function") result = result(url, host, scheme); + } while (typeof result !== "string" || result.charCodeAt(0) === 43); + return result; + }; +}("+auto switch", { + "+auto switch": function(url, host, scheme) { + "use strict"; + if (isPlainHostName(host) || shExpMatch(host, "*.local")) return "DIRECT"; + if (isInNet(host, "10.0.0.0", "255.0.0.0")) return "DIRECT"; + if (/(?:^|\\.)google\\.com$/.test(host)) return "+proxy"; + if (shExpMatch(url, "http://*.example.org/files/*")) return "+proxy"; + if (dnsDomainIs(host, ".socks.example")) return "+socks"; + return "DIRECT"; + }, + "+proxy": function() { return "PROXY 127.0.0.1:8080; DIRECT"; }, + "+socks": function() { return "SOCKS5 127.0.0.1:1080; SOCKS 127.0.0.1:1080"; } +});`; + +check("switchy: a rule's host goes to its profile", find(SWITCHY, "https://www.google.com/") === "PROXY 127.0.0.1:8080; DIRECT"); +check("switchy: a URL wildcard rule matches on the full URL", find(SWITCHY, "http://cdn.example.org/files/a.zip") === "PROXY 127.0.0.1:8080; DIRECT"); +check("switchy: and not outside its path", find(SWITCHY, "http://cdn.example.org/other/a.zip") === "DIRECT"); +check("switchy: a domain rule picks the SOCKS profile", find(SWITCHY, "https://a.socks.example/") === "SOCKS5 127.0.0.1:1080; SOCKS 127.0.0.1:1080"); +check("switchy: plain host names stay direct", find(SWITCHY, "http://nas/") === "DIRECT"); +check("switchy: an IP literal in a private net stays direct", find(SWITCHY, "http://10.1.2.3/") === "DIRECT"); +check("switchy: anything else falls through to DIRECT", find(SWITCHY, "https://example.net/") === "DIRECT"); + +// The classic PAC file shape: a top-level function declaration. +const CLASSIC = `function FindProxyForURL(url, host) { + if (localHostOrDomainIs(host, "www.corp.example")) return "DIRECT"; + if (dnsDomainLevels(host) > 2) return "HTTPS secure.example:443"; + return "PROXY proxy.example:3128"; +}`; +check("classic: localHostOrDomainIs takes the bare name", find(CLASSIC, "http://www/") === "DIRECT"); +check("classic: dnsDomainLevels counts the dots", find(CLASSIC, "https://a.b.c.example/") === "HTTPS secure.example:443"); +check("classic: the fallthrough", find(CLASSIC, "https://b.example/") === "PROXY proxy.example:3128"); + +// No DNS: a name never resolves, so a rule keyed on its address does not fire. +check("dns: a name does not resolve", ctx.dnsResolve("example.com") === null && !ctx.isInNet("example.com", "0.0.0.0", "0.0.0.0")); +check("dns: an IP literal resolves to itself", ctx.dnsResolve("192.168.1.9") === "192.168.1.9"); +check("dns: isInNet compares under the mask", ctx.isInNet("192.168.1.9", "192.168.0.0", "255.255.0.0") && !ctx.isInNet("192.169.1.9", "192.168.0.0", "255.255.0.0")); +check("dns: convert_addr is the unsigned address", ctx.convert_addr("255.255.255.255") === 4294967295); +check("shExpMatch: ? is one character", ctx.shExpMatch("a1c", "a?c") && !ctx.shExpMatch("a12c", "a?c")); +check("shExpMatch: dots are literal", !ctx.shExpMatch("axcom", "a.com")); + +// Time helpers, against today's clock. +const now = new Date(); +const DAYS = ["SUN", "MON", "TUE", "WED", "THU", "FRI", "SAT"]; +const MONTHS = ["JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"]; +const today = DAYS[now.getDay()], tomorrow = DAYS[(now.getDay() + 1) % 7], yesterday = DAYS[(now.getDay() + 6) % 7]; +check("weekdayRange: today", ctx.weekdayRange(today)); +check("weekdayRange: a range around today", ctx.weekdayRange(yesterday, tomorrow)); +check("weekdayRange: a range that excludes today", !ctx.weekdayRange(tomorrow, tomorrow)); +check("weekdayRange: a wrapping range from tomorrow to yesterday excludes today", !ctx.weekdayRange(tomorrow, yesterday)); +check("dateRange: this month", ctx.dateRange(MONTHS[now.getMonth()])); +check("dateRange: this year as a range", ctx.dateRange(now.getFullYear() - 1, now.getFullYear() + 1)); +check("dateRange: day and month bounds", ctx.dateRange(1, "JAN", 31, "DEC")); +check("dateRange: next year is not now", !ctx.dateRange(now.getFullYear() + 1)); +check("dateRange: an odd argument count is not a range", !ctx.dateRange(1, "JAN", 31)); +check("timeRange: this hour", ctx.timeRange(now.getHours())); +check("timeRange: the whole day", ctx.timeRange(0, 0, 0, 23, 59, 59)); +check("timeRange: an hour range that ends before this hour", now.getHours() === 0 || !ctx.timeRange(0, now.getHours())); +check("timeRange: GMT reads UTC", ctx.timeRange(now.getUTCHours(), "GMT")); + +// The sandbox's message protocol: an answer or an error, never a throw. +const ask = (data) => listeners.forEach((fn) => fn({ data, source: { postMessage: (m) => posted.push(m) } })); +ask({ id: 1, script: CLASSIC, url: "https://b.example/", host: "b.example" }); +ask({ id: 2, script: "throw new Error('boom')", url: "https://b.example/", host: "b.example" }); +ask({ id: 3, script: "var x = 1;", url: "https://b.example/", host: "b.example" }); +ask({ hello: "not a PAC request" }); +check("sandbox: an answer comes back under its id", posted[0]?.id === 1 && posted[0]?.result === "PROXY proxy.example:3128", JSON.stringify(posted[0])); +check("sandbox: a throwing script is an error, not a crash", posted[1]?.id === 2 && /boom/.test(posted[1]?.error), JSON.stringify(posted[1])); +check("sandbox: a script without FindProxyForURL is an error", posted[2]?.id === 3 && !!posted[2]?.error, JSON.stringify(posted[2])); +check("sandbox: anything else on the channel is ignored", posted.length === 3); + +console.log(fails ? `\n${fails} FAILED` : "\nall passed"); +process.exit(fails ? 1 : 0);