Consolidate and simplify the measured Vulkan inference improvements

bench/gguf_latency.rs

@@ -1,5 +1,5 @@
 //! Matched GGUF diagnostic; see gguf_latency.cpp for the llama.cpp counterpart.
-//! Usage: gguf_latency model.gguf output-prefix [tune-seconds]
+//! Usage: gguf_latency model.gguf output-prefix [tune-seconds] [all|dense|attention]
 //! GPU selection and precision use the usual SessionConfig environment options.
 
 use meganeura::{Graph, Session, SessionConfig, load::gguf};
@@ -19,7 +19,7 @@ fn run(
     config: &gguf::arch::ModelConfig,
     position: usize,
     count: usize,
-) -> (Vec<f32>, [f64; 3]) {
+) -> (Vec<f32>, [f64; 2]) {
     let tokens: Vec<u32> = (position..position + count)
         .map(|i| 42 + (i % 31) as u32)
         .collect();
@@ -41,18 +41,15 @@ fn run(
     let start = Instant::now();
     session.step();
     let submitted = Instant::now();
-    session.wait();
-    let finished = Instant::now();
     let mut logits = vec![0.0; config.vocab_size];
-    session.read_output_by_index(0, &mut logits);
+    session.wait_read_output(0, &mut logits);
     assert!(logits.iter().all(|x| x.is_finite()));
     let read = Instant::now();
     (
         logits,
         [
             submitted.duration_since(start).as_secs_f64() * 1000.0,
-            finished.duration_since(submitted).as_secs_f64() * 1000.0,
-            read.duration_since(finished).as_secs_f64() * 1000.0,
+            read.duration_since(submitted).as_secs_f64() * 1000.0,
         ],
     )
 }
@@ -61,10 +58,16 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
     let args: Vec<_> = std::env::args().collect();
     assert!(
-        (3..=4).contains(&args.len()),
-        "gguf_latency model.gguf output-prefix [tune-seconds]"
+        (3..=5).contains(&args.len()),
+        "gguf_latency model.gguf output-prefix [tune-seconds] [all|dense|attention]"
     );
     let tune_seconds: u64 = args.get(3).map_or(Ok(0), |s| s.parse())?;
+    let scope = match args.get(4).map(String::as_str).unwrap_or("all") {
+        "all" => meganeura::tune::TuneScope::All,
+        "dense" => meganeura::tune::TuneScope::Dense,
+        "attention" => meganeura::tune::TuneScope::Attention,
+        _ => return Err("expected all, dense or attention tuning scope".into()),
+    };
     let started = Instant::now();
     let model = gguf::load_gguf(Path::new(&args[1]))?;
     let config = gguf::arch::ModelConfig::from_gguf(&model)?;
@@ -87,6 +90,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         session.set_submission_chunks(1);
         if tune_seconds != 0 {
             tuning.push(session.tune_with(meganeura::tune::TuneOptions {
+                scope,
                 max_time: Duration::from_secs(tune_seconds),
                 max_classes: 64,
                 max_scratch_bytes: 256 * 1024 * 1024,
@@ -105,6 +109,22 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         }
         sessions.push(session);
     }
+    let mut scheduling = Vec::new();
+    if tune_seconds != 0 && matches!(scope, meganeura::tune::TuneScope::All) {
+        run(&mut sessions[1], &model, &config, 0, PROMPT);
+        for pos in PROMPT..PROMPT + DECODE {
+            run(&mut sessions[0], &model, &config, pos, 1);
+        }
+        for session in &mut sessions {
+            scheduling.push(
+                session.tune_submissions(meganeura::tune::TuneSubmissionOptions {
+                    max_scratch_bytes: 256 * 1024 * 1024,
+                    min_improvement: 0.01,
+                    ..Default::default()
+                })?,
+            );
+        }
+    }
     let prepare_ms = started.elapsed().as_secs_f64() * 1000.0;
     let mut prefill_ms = Vec::new();
     let mut decode_ms = Vec::new();
@@ -143,9 +163,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         "vocab": config.vocab_size, "prepare_ms": prepare_ms,
         "activations": if f32_activations { "f32" } else { "q8_1" },
         "prefill_ms": prefill_ms, "decode_ms": decode_ms,
-        "decode_record_wait_read_ms": decode_parts_ms,
+        "decode_record_finish_ms": decode_parts_ms,
         "dispatches": [sessions[1].plan().dispatches.len(), sessions[0].plan().dispatches.len()],
         "tuning": tuning,
+        "submission_tuning": scheduling,
     });
     std::fs::write(
         format!("{}.json", args[2]),

examples/diagnose_fusion.rs

@@ -34,7 +34,7 @@ fn shader_name(s: &ShaderEntry) -> String {
 /// epilogue absorption on the scalar plan inspected here.
 fn is_scalar_matmul(d: &Dispatch) -> bool {
     use ShaderEntry::*;
-    !d.use_coop
+    !d.use_coop()
         && matches!(
             d.shader,
             MatMul | MatMulAT | MatMulBT | FusedMatMulAdd | FusedMatMulATAdd | FusedMatMulBTAdd
@@ -71,9 +71,6 @@ fn count_consumers(plan: &ExecutionPlan) -> HashMap<BufferRef, Vec<usize>> {
         for b in &d.input_buffers {
             m.entry(*b).or_default().push(i);
         }
-        for b in &d.epilogue_buffers {
-            m.entry(*b).or_default().push(i);
-        }
     }
     m
 }
@@ -142,7 +139,7 @@ fn diagnose(plan: &ExecutionPlan) -> Vec<Finding> {
         if matches!(
             d.shader,
             ShaderEntry::MatMul | ShaderEntry::MatMulBT | ShaderEntry::MatMulAT
-        ) && !d.use_coop
+        ) && !d.use_coop()
             && d.input_buffers.len() >= 2
         {
             for (slot_idx, in_buf) in d.input_buffers[..2].iter().enumerate() {
@@ -164,7 +161,7 @@ fn diagnose(plan: &ExecutionPlan) -> Vec<Finding> {
         // Pattern 3: MatMul → (single consumer) Add/BiasAdd with a matmul-fused variant
         // already existing (FusedMatMulAdd). Count cases where this is being done in
         // two dispatches.
-        if matches!(d.shader, ShaderEntry::Add | ShaderEntry::BiasAdd) && d.pointwise.is_none() {
+        if matches!(d.shader, ShaderEntry::Add | ShaderEntry::BiasAdd) && d.pointwise().is_none() {
             for (slot_idx, in_buf) in d.input_buffers.iter().enumerate() {
                 if !external.contains(in_buf)
                     && let Some(&prod_i) = producer.get(in_buf)

examples/gpu_compare.rs

@@ -26,9 +26,9 @@ fn bench_matmul(n: usize, warmup: usize, iters: usize) -> (f64, &'static str) {
         .iter()
         .find(|d| matches!(d.shader, meganeura::compile::ShaderEntry::MatMul))
         .map(|d| {
-            if d.use_coop {
+            if d.use_coop() {
                 "coop"
-            } else if d.use_small_tiles {
+            } else if d.use_small_tiles() {
                 "small"
             } else {
                 "tile"

examples/matmul_throughput.rs

@@ -98,13 +98,13 @@ fn bench_shape(
         .map(|d| {
             let kernel = match d.shader {
                 meganeura::compile::ShaderEntry::MatMulGemv => "gemv",
-                _ if d.use_coop => "coop",
-                _ if d.use_small_tiles => "small",
+                _ if d.use_coop() => "coop",
+                _ if d.use_small_tiles() => "small",
                 _ => "tile",
             };
             (
-                if d.use_coop { 1 } else { 0 },
-                if d.use_small_tiles { 1 } else { 0 },
+                if d.use_coop() { 1 } else { 0 },
+                if d.use_small_tiles() { 1 } else { 0 },
                 d.workgroups[0] * d.workgroups[1] * d.workgroups[2],
                 kernel,
             )

src/cache.rs

@@ -9,9 +9,8 @@ use std::{io, path::Path};
 
 /// Increment whenever the serialized execution plan or build pipeline changes
 /// in a way that can make an older plan unsafe to reuse.
-// Version 7 invalidates plans that may have replaced a generated pointwise DAG
-// with its legacy shader sentinel while fusing a matmul epilogue.
-const CACHE_FORMAT_VERSION: u32 = 7;
+// Version 9 represents mutually exclusive dispatch implementations as an enum.
+const CACHE_FORMAT_VERSION: u32 = 9;
 
 /// Cached execution plan with a graph fingerprint for invalidation.
 #[derive(Serialize, Deserialize)]