icy_term: keep up with streamed sixel video (#191)

* Leave the scrollback out of the render snapshot

TextScreen::render_snapshot() cloned the whole screen, scrollback
included, and each scrollback line keeps a rendered pixel strip
(640x16 RGBA, about 40 KB). With the default 2000 lines full, every UI
frame copied about 80 MB while holding the screen lock: 39 ms per
frame. The terminal thread waited for the lock long enough to drop
from about 28 to about 15 sixel frames a second, so a 20 fps sixel
stream fell further and further behind.

The renderer never reads the scrollback (viewing it goes through
snapshot_scrollback()), so the snapshot now copies everything else and
starts with an empty scrollback. Time under the lock per frame drops
to 0.16 ms.

* Skip the sixel display pause when a newer sixel is queued

The terminal thread sleeps 20 ms after each sixel so the UI can show
every frame of an animation. On Windows that sleep takes about 31 ms,
while decoding a 460x360 frame takes under 1 ms, so the pause caps a
sixel stream at about 28 frames a second however far behind the
terminal is. Over SSH nothing slows the host down, so the lag then
grows without limit.

On a live connection the pause is now skipped when the host has
already sent a newer sixel. Every frame is still decoded and placed;
the UI shows the newest one, the way SyncTERM and xterm degrade. When
the terminal keeps up, and for local playback, every frame still gets
its pause, so sixel animations are unchanged.

* Don't pause after sixels on a live connection

The 20 ms pause after each sixel lets the UI show every frame of an
animation played back locally. On a connection the host paces its own
frames, and the pause only costs time: NESDOOR streams NES video at
up to 60 frames a second, and the terminal thread spent about 80% of
each second asleep in these pauses. It ran at the very edge of the
stream, fell behind on every burst, and keystrokes waited behind the
pauses too.

The previous change skipped the pause only when a newer sixel was
already queued, which does not help when frames arrive one per read.
The pause now applies only without a connection, as in SyncTERM and
xterm, which draw sixels as fast as they arrive.

* Process received data in the tick that reads it

Each tick processed what was already pending and only then read from
the connection, so new data waited a full tick (about 13 ms) before
it reached the screen. Process it right after the read instead.

With baud emulation the second pass releases nothing until enough time
has passed, so emulated speeds are unchanged.

Author: NuSkooler

crates/icy_engine/src/text_buffer/text_screen.rs

@@ -100,7 +100,21 @@ impl TextPane for TextScreen {
 
 impl Screen for TextScreen {
     fn render_snapshot(&self) -> Option<Box<dyn Screen>> {
-        Some(Box::new(self.clone()))
+        // Rendering never reads the scrollback. It keeps a rendered pixel strip per line, so a full
+        // scrollback is tens of megabytes, and copying it every frame would hold the screen lock
+        // long enough to stall the terminal thread.
+        Some(Box::new(Self {
+            caret: self.caret.clone(),
+            buffer: self.buffer.clone(),
+            current_layer: self.current_layer,
+            selection_opt: self.selection_opt,
+            mouse_fields: self.mouse_fields.clone(),
+            saved_caret_pos: self.saved_caret_pos,
+            saved_caret_state: self.saved_caret_state.clone(),
+            scan_lines: self.scan_lines,
+            scrollback_buffer: ScrollbackBuffer::new(),
+            unicode: self.unicode.clone(),
+        }))
     }
 
     fn buffer_type(&self) -> crate::BufferType {

crates/icy_engine/tests/text_screen.rs

@@ -1529,3 +1529,21 @@ fn test_kitty_keyboard_resets_with_the_terminal() {
     screen.reset_terminal();
     assert!(!screen.terminal_state().kitty_keyboard.is_active());
 }
+
+#[test]
+fn test_render_snapshot_leaves_out_scrollback() {
+    let mut screen = TextScreen::new(Size::new(80, 25));
+    screen.terminal_state_mut().is_terminal_buffer = true;
+    let mut parser = AnsiParser::new();
+    for line in 0..40 {
+        parser.parse(format!("\x1b[3{}mline {line}\r\n", line % 8).as_bytes(), &mut ScreenSink::new(&mut screen));
+    }
+    assert!(!screen.scrollback_buffer.chunks.is_empty());
+
+    let mut snapshot = screen.render_snapshot().unwrap();
+    let region = Rectangle::from(0, 0, screen.resolution().width, screen.resolution().height);
+    let options = RenderOptions::default();
+    assert_eq!(snapshot.render_region_to_rgba(region, &options), screen.render_region_to_rgba(region, &options));
+    let snapshot = snapshot.as_any_mut().downcast_mut::<TextScreen>().unwrap();
+    assert!(snapshot.scrollback_buffer.chunks.is_empty());
+}

crates/icy_term/src/terminal/terminal_thread.rs

@@ -697,24 +697,7 @@ impl TerminalThread {
                         pending_data.append(&mut self.injected_data);
                     }
 
-                    // Process pending data with baud emulation
-                    if pending_offset < pending_data.len() {
-                        let remaining = pending_data.len() - pending_offset;
-                        let bytes_to_send = self.baud_emulator.calculate_bytes_to_send(remaining);
-                        if bytes_to_send > 0 {
-                            let end = pending_offset + bytes_to_send;
-                            let chunk = &pending_data[pending_offset..end];
-                            self.write_to_capture(chunk).await;
-                            self.process_data(chunk).await;
-                            pending_offset = end;
-
-                            // Clear buffer when fully processed
-                            if pending_offset >= pending_data.len() {
-                                pending_data.clear();
-                                pending_offset = 0;
-                            }
-                        }
-                    }
+                    self.process_pending_data(&mut pending_data, &mut pending_offset).await;
 
                     // Check for pending auto-transfers
                     if let Some((protocol_id, is_download, filename)) = self.auto_transfer.take() {
@@ -772,6 +755,8 @@ impl TerminalThread {
                         if let Some(new_data) = self.read_connection_raw(&mut read_buffer).await {
                             if !new_data.is_empty() {
                                 pending_data.extend_from_slice(&new_data);
+                                // Show it now rather than a tick later
+                                self.process_pending_data(&mut pending_data, &mut pending_offset).await;
                             }
                         }
                     }
@@ -780,6 +765,28 @@ impl TerminalThread {
         }
     }
 
+    /// Processes as much of `pending_data` as the baud emulation allows.
+    async fn process_pending_data(&mut self, pending_data: &mut Vec<u8>, pending_offset: &mut usize) {
+        if *pending_offset >= pending_data.len() {
+            return;
+        }
+        let remaining = pending_data.len() - *pending_offset;
+        let bytes_to_send = self.baud_emulator.calculate_bytes_to_send(remaining);
+        if bytes_to_send > 0 {
+            let end = *pending_offset + bytes_to_send;
+            let chunk = &pending_data[*pending_offset..end];
+            self.write_to_capture(chunk).await;
+            self.process_data(chunk).await;
+            *pending_offset = end;
+
+            // Clear buffer when fully processed
+            if *pending_offset >= pending_data.len() {
+                pending_data.clear();
+                *pending_offset = 0;
+            }
+        }
+    }
+
     fn perform_resize(&mut self, width: u16, height: u16) {
         let mut state = self.edit_screen.lock();
         if let Some(editable) = state.as_editable() {
@@ -1768,7 +1775,12 @@ impl TerminalThread {
                                 }
                             }
                         }
-                        tokio::time::sleep(Duration::from_millis(20)).await;
+                        // Pause so the UI can show each frame of a sixel animation played back
+                        // locally. On a live connection the host paces its frames; pausing there
+                        // only puts the terminal behind the stream (and holds up typed keys).
+                        if self.connection.is_none() {
+                            tokio::time::sleep(Duration::from_millis(20)).await;
+                        }
                     }
                     Err(err) => {
                         if shared {
@@ -3161,6 +3173,48 @@ mod tests {
         }
     }
 
+    /// Full-screen style animation: every frame replaces the previous one, and only the last frame is red.
+    fn sixel_frames(count: usize) -> Vec<u8> {
+        let mut data = Vec::new();
+        for frame in 0..count {
+            let color = if frame + 1 == count { "100;0;0" } else { "0;0;100" };
+            data.extend_from_slice(format!("\x1b[1;1H\x1bPq#1;2;{color}#1!16~-!16~\x1b\\").as_bytes());
+        }
+        data
+    }
+
+    #[tokio::test]
+    async fn sixel_frames_from_a_connection_skip_the_display_pause() {
+        let (mut terminal, _, screen) = test_terminal();
+        let frames = sixel_frames(20);
+        let start = std::time::Instant::now();
+
+        // Both all at once and one frame per read, the way a host pacing its frames delivers them.
+        terminal.process_data(&frames).await;
+        for frame in frames.split_inclusive(|byte| *byte == b'\\') {
+            terminal.process_data(frame).await;
+        }
+
+        // Pausing after each of the 40 frames would take at least 800 ms.
+        assert!(start.elapsed() < std::time::Duration::from_millis(200), "{:?}", start.elapsed());
+        let mut screen = screen.lock();
+        let text = screen.as_any_mut().downcast_mut::<TextScreen>().unwrap();
+        let sixels = &text.buffer.layers[0].sixels;
+        assert_eq!(sixels.len(), 1);
+        assert_eq!(&sixels[0].picture_data[..4], &[255, 0, 0, 255]);
+    }
+
+    #[tokio::test]
+    async fn sixel_frames_played_locally_keep_the_display_pause() {
+        let (mut terminal, _, _) = test_terminal();
+        terminal.connection = None;
+        let start = std::time::Instant::now();
+
+        terminal.process_data(&sixel_frames(3)).await;
+
+        assert!(start.elapsed() >= std::time::Duration::from_millis(60), "{:?}", start.elapsed());
+    }
+
     #[test]
     fn auto_login_enter_uses_active_terminal_mapping() {
         assert_eq!(TerminalThread::auto_login_control_code(TerminalEmulation::ATAscii, b'\r'), vec![0x9B]);