Feature: watcher clients (#4487)

* initial implementation

* make Screen aware of watchers - slightly buggy

* fix: only render focused tab to watcher

* track watcher sizes on Screen

* pad watcher output as needed

* truncate watcher sizes as needed

* render on watcher resize

* fix watcher first resize

* allow watchers to quit with esc, ctrl-c or ctrl-q

* fix: send watcher size on initial connection message

* refactor: move  padding logic to output

* fix watcher rendering after all clients disconnected

* fix watcher rendering caching issues

* crop watcher cursor when out of bounds

* remove watchers cleanly on session close

* standardize watcher messages ignoring

* add e2e test for watcher functionality

* update snapshots

* add output tests

* rustfmt

* cleanups

* docs(changelog): add PR

4 files changed, 316 insertions, 2 deletions

diff --git a/src/tests/e2e/cases.rs b/src/tests/e2e/cases.rs
index bf5b6564..d2096cea 100644
--- a/src/tests/e2e/cases.rs
+++ b/src/tests/e2e/cases.rs
@@ -1,7 +1,10 @@
 #![allow(unused)]
 
 use ::insta::assert_snapshot;
-use zellij_utils::{pane_size::Size, position::Position};
+use zellij_utils::{
+    pane_size::Size,
+    position::{Column, Line, Position},
+};
 
 use rand::Rng;
 use regex::Regex;
@@ -2545,3 +2548,206 @@ pub fn pin_floating_panes() {
     let last_snapshot = account_for_races_in_snapshot(last_snapshot);
     assert_snapshot!(last_snapshot);
 }
+#[test]
+#[ignore]
+pub fn watcher_client_functionality() {
+    let fake_win_size = Size {
+        cols: 120,
+        rows: 24,
+    };
+    let mut test_attempts = 10;
+    let session_name = "watcher_client_functionality";
+    let (main_client_snapshot, watcher_snapshot) = loop {
+        RemoteRunner::kill_running_sessions(fake_win_size);
+
+        // Step 1: Create main client and wait for it to load
+        let mut main_client =
+            RemoteRunner::new_with_session_name(fake_win_size, session_name, false)
+                .dont_panic()
+                .add_step(Step {
+                    name: "Wait for app to load",
+                    instruction: |remote_terminal: RemoteTerminal| -> bool {
+                        remote_terminal.status_bar_appears()
+                            && remote_terminal.cursor_position_is(3, 2)
+                    },
+                });
+        main_client.run_all_steps();
+
+        // Step 2: Start a background process that produces periodic output
+        main_client = main_client.add_step(Step {
+            name: "Start background output loop",
+            instruction: |mut remote_terminal: RemoteTerminal| -> bool {
+                remote_terminal.send_key(b"{ for i in 1 2 3; do sleep 1; echo \"WATCHER_OUTPUT_$i\"; done & } 2>/dev/null");
+                std::thread::sleep(std::time::Duration::from_millis(100));
+                remote_terminal.send_key(&ENTER);
+                true
+            },
+        });
+        main_client.run_all_steps();
+
+        // Step 3: Wait for first output line to appear
+        main_client = main_client.add_step(Step {
+            name: "Wait for first background output",
+            instruction: |remote_terminal: RemoteTerminal| -> bool {
+                let found = remote_terminal.snapshot_contains("WATCHER_OUTPUT_1");
+                found
+            },
+        });
+        main_client.run_all_steps();
+
+        // Step 4: Attach watcher client
+        let mut watcher = RemoteRunner::new_watcher_session(fake_win_size, session_name)
+            .dont_panic()
+            .add_step(Step {
+                name: "Wait for watcher to connect",
+                instruction: |remote_terminal: RemoteTerminal| -> bool {
+                    remote_terminal.status_bar_appears()
+                    // && remote_terminal.cursor_position_is(3, 2)
+                },
+            });
+        watcher.run_all_steps();
+
+        // Step 5: Main client splits pane right
+        main_client = main_client.add_step(Step {
+            name: "Split pane to the right",
+            instruction: |mut remote_terminal: RemoteTerminal| -> bool {
+                if remote_terminal.status_bar_appears() && remote_terminal.cursor_position_is(1, 7)
+                {
+                    remote_terminal.send_key(&PANE_MODE);
+                    std::thread::sleep(std::time::Duration::from_millis(100));
+                    remote_terminal.send_key(&SPLIT_RIGHT_IN_PANE_MODE);
+                    std::thread::sleep(std::time::Duration::from_millis(100));
+                    true
+                } else {
+                    false
+                }
+            },
+        });
+        main_client.run_all_steps();
+
+        // Step 6: Verify watcher sees the split pane
+        watcher = watcher.add_step(Step {
+            name: "Watcher sees split pane",
+            instruction: |remote_terminal: RemoteTerminal| -> bool {
+                remote_terminal.cursor_position_is(63, 2) && remote_terminal.snapshot_contains("┐┌")
+                // vertical split indicator
+            },
+        });
+        watcher.run_all_steps();
+
+        // Step 7: Watcher tries to open a new tab (should be ignored)
+        watcher = watcher.add_step(Step {
+            name: "Watcher attempts to open new tab",
+            instruction: |mut remote_terminal: RemoteTerminal| -> bool {
+                remote_terminal.send_key(&TAB_MODE);
+                std::thread::sleep(std::time::Duration::from_millis(100));
+                remote_terminal.send_key(&NEW_TAB_IN_TAB_MODE);
+                std::thread::sleep(std::time::Duration::from_millis(500));
+                remote_terminal.send_key(b"some text that should not appear...");
+                std::thread::sleep(std::time::Duration::from_millis(500));
+                true
+            },
+        });
+        watcher.run_all_steps();
+
+        // Step 8: Verify main client didn't receive the tab command
+        main_client = main_client.add_step(Step {
+            name: "Verify no new tab was created",
+            instruction: |remote_terminal: RemoteTerminal| -> bool {
+                // Should still be in Tab #1, not Tab #2
+                remote_terminal.snapshot_contains("Tab #1")
+                    && !remote_terminal.snapshot_contains("Tab #2")
+            },
+        });
+        main_client.run_all_steps();
+
+        // Step 9: Watcher sends mouse click (should be ignored)
+        watcher = watcher.add_step(Step {
+            name: "Watcher attempts mouse click",
+            instruction: |mut remote_terminal: RemoteTerminal| -> bool {
+                let click_position = Position {
+                    line: Line(10),
+                    column: Column(10),
+                };
+                remote_terminal.send_key(&sgr_mouse_report(click_position, 0)); // left click
+                std::thread::sleep(std::time::Duration::from_millis(500));
+                true
+            },
+        });
+        watcher.run_all_steps();
+
+        // Step 10: Main client detaches
+        main_client = main_client.add_step(Step {
+            name: "Main client detaches",
+            instruction: |mut remote_terminal: RemoteTerminal| -> bool {
+                remote_terminal.send_key(&SESSION_MODE);
+                std::thread::sleep(std::time::Duration::from_millis(100));
+                remote_terminal.send_key(&DETACH_IN_SESSION_MODE);
+                true
+            },
+        });
+        main_client.run_all_steps();
+
+        // Step 11: Verify watcher receives background output even with no main client
+        watcher = watcher.add_step(Step {
+            name: "Watcher sees WATCHER_OUTPUT_2",
+            instruction: |remote_terminal: RemoteTerminal| -> bool {
+                remote_terminal.snapshot_contains("WATCHER_OUTPUT_2")
+            },
+        });
+        watcher.run_all_steps();
+
+        watcher = watcher.add_step(Step {
+            name: "Watcher sees WATCHER_OUTPUT_3",
+            instruction: |remote_terminal: RemoteTerminal| -> bool {
+                remote_terminal.snapshot_contains("WATCHER_OUTPUT_3")
+            },
+        });
+        watcher.run_all_steps();
+
+        // Step 12: Main client re-attaches
+        let mut main_client_reattached =
+            RemoteRunner::new_existing_session(fake_win_size, session_name)
+                .dont_panic()
+                .add_step(Step {
+                    name: "Main client re-attaches",
+                    instruction: |remote_terminal: RemoteTerminal| -> bool {
+                        std::thread::sleep(std::time::Duration::from_millis(500)); // wait for watcher
+                                                                                   // to fail running
+                                                                                   // commands
+                        remote_terminal.status_bar_appears()
+                            && remote_terminal.snapshot_contains("┐┌")
+                    },
+                });
+        main_client_reattached.run_all_steps();
+
+        watcher.run_all_steps();
+
+        // Step 15: Take final snapshots
+        let main_client_snapshot = main_client_reattached.take_snapshot_after(Step {
+            name: "Take main client final snapshot",
+            instruction: |remote_terminal: RemoteTerminal| -> bool {
+                remote_terminal.status_bar_appears()
+            },
+        });
+
+        let watcher_snapshot = watcher.take_snapshot_after(Step {
+            name: "Take watcher final snapshot",
+            instruction: |remote_terminal: RemoteTerminal| -> bool {
+                remote_terminal.status_bar_appears()
+            },
+        });
+
+        if (main_client_reattached.test_timed_out || watcher.test_timed_out) && test_attempts > 0 {
+            test_attempts -= 1;
+            continue;
+        } else {
+            break (main_client_snapshot, watcher_snapshot);
+        }
+    };
+
+    let main_client_snapshot = account_for_races_in_snapshot(main_client_snapshot);
+    let watcher_snapshot = account_for_races_in_snapshot(watcher_snapshot);
+    assert_snapshot!(main_client_snapshot);
+    assert_snapshot!(watcher_snapshot);
+}
diff --git a/src/tests/e2e/remote_runner.rs b/src/tests/e2e/remote_runner.rs
index 54b6c468..a9b345b7 100644
--- a/src/tests/e2e/remote_runner.rs
+++ b/src/tests/e2e/remote_runner.rs
@@ -180,6 +180,20 @@ fn attach_to_existing_session(channel: &mut ssh2::Channel, session_name: &str) {
     std::thread::sleep(std::time::Duration::from_secs(3)); // wait until Zellij stops parsing startup ANSI codes from the terminal STDIN
 }
 
+fn watch_existing_session(channel: &mut ssh2::Channel, session_name: &str) {
+    channel
+        .write_all(
+            format!(
+                "{} {} watch {}\n",
+                SET_ENV_VARIABLES, ZELLIJ_EXECUTABLE_LOCATION, session_name
+            )
+            .as_bytes(),
+        )
+        .unwrap();
+    channel.flush().unwrap();
+    std::thread::sleep(std::time::Duration::from_secs(3)); // wait until Zellij stops parsing startup ANSI codes from the terminal STDIN
+}
+
 fn start_zellij_without_frames(channel: &mut ssh2::Channel) {
     stop_zellij(channel);
     channel
@@ -688,6 +702,44 @@ impl RemoteRunner {
             reader_thread,
         }
     }
+    pub fn new_watcher_session(win_size: Size, session_name: &str) -> Self {
+        let sess = ssh_connect_without_timeout();
+        let mut channel = sess.channel_session().unwrap();
+        let mut rows = Dimension::fixed(win_size.rows);
+        let mut cols = Dimension::fixed(win_size.cols);
+        rows.set_inner(win_size.rows);
+        cols.set_inner(win_size.cols);
+        let pane_geom = PaneGeom {
+            x: 0,
+            y: 0,
+            rows,
+            cols,
+            stacked: None,
+            is_pinned: false,
+            logical_position: None,
+        };
+        setup_remote_environment(&mut channel, win_size);
+        watch_existing_session(&mut channel, session_name);
+        let channel = Arc::new(Mutex::new(channel));
+        let last_snapshot = Arc::new(Mutex::new(String::new()));
+        let cursor_coordinates = Arc::new(Mutex::new((0, 0)));
+        sess.set_blocking(false);
+        let reader_thread =
+            read_from_channel(&channel, &last_snapshot, &cursor_coordinates, &pane_geom);
+        RemoteRunner {
+            steps: vec![],
+            channel,
+            currently_running_step: None,
+            current_step_index: 0,
+            retries_left: RETRIES,
+            retry_pause_ms: 100,
+            test_timed_out: false,
+            panic_on_no_retries_left: true,
+            last_snapshot,
+            cursor_coordinates,
+            reader_thread,
+        }
+    }
     pub fn new_without_frames(win_size: Size) -> Self {
         let sess = ssh_connect();
         let mut channel = sess.channel_session().unwrap();
@@ -797,7 +849,7 @@ impl RemoteRunner {
                 self.retries_left = RETRIES;
                 self.current_step_index += 1;
             } else {
-                self.retries_left -= 1;
+                self.retries_left = self.retries_left.saturating_sub(1);
                 std::thread::sleep(std::time::Duration::from_millis(self.retry_pause_ms as u64));
             }
         }
diff --git a/src/tests/e2e/snapshots/zellij__tests__e2e__cases__watcher_client_functionality-2.snap b/src/tests/e2e/snapshots/zellij__tests__e2e__cases__watcher_client_functionality-2.snap
new file mode 100644
index 00000000..cc4bec39
--- /dev/null
+++ b/src/tests/e2e/snapshots/zellij__tests__e2e__cases__watcher_client_functionality-2.snap
@@ -0,0 +1,28 @@
+---
+source: src/tests/e2e/cases.rs
+expression: watcher_snapshot
+---
+ Zellij (watcher_client_functionality)  Tab #1 
+┌ Pane #1 ─────────────────────────────────────────────────┐┌ Pane #2 ─────────────────────────────────────────────────┐
+│$ { for i in 1 2 3; do sleep 1; echo "WATCHER_OUTPUT_$i"; ││$ █                                                       │
+│done & } 2>/dev/null                                      ││                                                          │
+│$                                                         ││                                                          │
+│$ WATCHER_OUTPUT_1                                        ││                                                          │
+│WATCHER_OUTPUT_2                                          ││                                                          │
+│WATCHER_OUTPUT_3                                          ││                                                          │
+│$                                                         ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+└──────────────────────────────────────────────────────────┘└──────────────────────────────────────────────────────────┘
+ Ctrl + <g> LOCK  <p> PANE  <t> TAB  <n> RESIZE  <h> MOVE  <s> SEARCH  <o> SESSION  <q> QUIT        ...
diff --git a/src/tests/e2e/snapshots/zellij__tests__e2e__cases__watcher_client_functionality.snap b/src/tests/e2e/snapshots/zellij__tests__e2e__cases__watcher_client_functionality.snap
new file mode 100644
index 00000000..cc6509e0
--- /dev/null
+++ b/src/tests/e2e/snapshots/zellij__tests__e2e__cases__watcher_client_functionality.snap
@@ -0,0 +1,28 @@
+---
+source: src/tests/e2e/cases.rs
+expression: main_client_snapshot
+---
+ Zellij (watcher_client_functionality)  Tab #1 
+┌ Pane #1 ─────────────────────────────────────────────────┐┌ Pane #2 ─────────────────────────────────────────────────┐
+│$ { for i in 1 2 3; do sleep 1; echo "WATCHER_OUTPUT_$i"; ││$ █                                                       │
+│done & } 2>/dev/null                                      ││                                                          │
+│$                                                         ││                                                          │
+│$ WATCHER_OUTPUT_1                                        ││                                                          │
+│WATCHER_OUTPUT_2                                          ││                                                          │
+│WATCHER_OUTPUT_3                                          ││                                                          │
+│$                                                         ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+│                                                          ││                                                          │
+└──────────────────────────────────────────────────────────┘└──────────────────────────────────────────────────────────┘
+ Ctrl + <g> LOCK  <p> PANE  <t> TAB  <n> RESIZE  <h> MOVE  <s> SEARCH  <o> SESSION  <q> QUIT        ...