diff options
| author | Aram Drevekenin <aram@poor.dev> | 2026-07-31 19:19:49 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-07-31 19:19:49 +0200 |
| commit | 0e6e4404027a187f1399a43bf91bdbd13d9636e1 (patch) | |
| tree | b334f14704939620e846caa30ab925fefca34d70 /zellij-server/src/output | |
| parent | fix(build): remove manpage, honor CARGO_TARGET_DIR, add cargo argument passth... (diff) | |
| download | sl0ppij-0e6e4404027a187f1399a43bf91bdbd13d9636e1.tar.gz sl0ppij-0e6e4404027a187f1399a43bf91bdbd13d9636e1.tar.xz sl0ppij-0e6e4404027a187f1399a43bf91bdbd13d9636e1.zip | |
Support the Kitty Image Protocol (#5428)
* initial implementation - kinda working
* tests and fixes
* fix bounds
* tests and compatibility fixes
* fix: cursor scroll, clears, deletes, key tolerance, and host id range
* fix: keep images alive across resize, relayout and history scroll
* anchor to lines, not pixels
* fix image/text alignments
* config to allow disabling
* fix pinned pane issue
* never let host graphics replies reach the keyboard parser
* fix nested session
* properly handle fullscreening a pane
* rustfmt
* cleanup of ignored test
* add pr
Diffstat (limited to 'zellij-server/src/output')
| -rw-r--r-- | zellij-server/src/output/mod.rs | 580 | ||||
| -rw-r--r-- | zellij-server/src/output/unit/output_tests.rs | 477 |
2 files changed, 1053 insertions, 4 deletions
diff --git a/zellij-server/src/output/mod.rs b/zellij-server/src/output/mod.rs index e85bae6b..6254bf00 100644 --- a/zellij-server/src/output/mod.rs +++ b/zellij-server/src/output/mod.rs @@ -4,6 +4,7 @@ use crate::panes::Row; use crate::panes::Selection; use crate::{ + panes::kitty_graphics::store::{InternalImageId, KittyImageStore}, panes::sixel::SixelImageStore, panes::terminal_character::{AnsiCode, CharacterStyles}, panes::{LinkHandler, PaneId, TerminalCharacter, DEFAULT_STYLES, EMPTY_TERMINAL_CHARACTER}, @@ -13,7 +14,7 @@ use std::cell::RefCell; use std::fmt::Write; use std::rc::Rc; use std::{ - collections::{HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, str, }; use zellij_utils::data::{HighlightLayer, PaneContents, PaneRenderReport}; @@ -202,6 +203,7 @@ fn serialize_chunks( styled_underlines: bool, osc8_hyperlinks: bool, max_size: Option<Size>, + kitty_input: Option<KittyFrameInput>, ) -> Result<String> { let err_context = || "failed to serialize input chunks".to_string(); @@ -296,6 +298,14 @@ fn serialize_chunks( vte_output.push_str(sixel_vte); vte_output.push_str(restore_cursor_position); } + if let Some(kitty_input) = kitty_input { + let kitty_vte = serialize_kitty_frame(kitty_input).with_context(err_context)?; + if !kitty_vte.is_empty() { + vte_output.push_str("\u{1b}[s"); + vte_output.push_str(&kitty_vte); + vte_output.push_str("\u{1b}[u"); + } + } Ok(vte_output) } @@ -350,14 +360,287 @@ fn adjust_middle_segment_for_wide_chars( )) } +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct KittyChunkKey { + pub pane_id: PaneId, + pub placement_uid: u64, + pub sub_index: u32, +} + +#[derive(Debug, Clone)] +pub struct HostPlacementRecord { + pub host_image_id: u32, + pub host_placement_id: u32, + pub image_key: (InternalImageId, Option<(u16, u16)>), + pub geometry: KittyImageChunk, +} + +#[derive(Debug, Clone)] +pub struct HostKittyState { + pub transmitted: HashMap<(InternalImageId, Option<(u16, u16)>), u32>, + pub live_placements: HashMap<KittyChunkKey, HostPlacementRecord>, + next_host_image_id: u32, + next_host_placement_id: u32, +} + +impl Default for HostKittyState { + fn default() -> Self { + HostKittyState { + transmitted: HashMap::new(), + live_placements: HashMap::new(), + next_host_image_id: 2_000_000_000, + next_host_placement_id: 1, + } + } +} + +pub struct KittyFrameInput<'a> { + pub chunks_by_pane: HashMap<PaneId, Vec<KittyImageChunk>>, + pub rendered_panes: HashSet<PaneId>, + pub visible_panes: Option<&'a HashSet<PaneId>>, + pub kitty_image_store: &'a mut KittyImageStore, + pub host_state: &'a mut HostKittyState, + pub host_display_cleared: bool, +} + +fn pane_sort_key(pane_id: PaneId) -> (u8, u32) { + match pane_id { + PaneId::Terminal(id) => (0, id), + PaneId::Plugin(id) => (1, id), + } +} + +fn emit_kitty_transmit( + out: &mut String, + host_image_id: u32, + width: usize, + height: usize, + b64: &str, +) -> Result<()> { + let err_context = "failed to serialize kitty transmit"; + let mut parts: Vec<&str> = vec![]; + let mut index = 0; + while index < b64.len() { + let end = std::cmp::min(index + 4096, b64.len()); + parts.push(&b64[index..end]); + index = end; + } + if parts.is_empty() { + parts.push(""); + } + let last = parts.len() - 1; + for (part_index, part) in parts.iter().enumerate() { + if part_index == 0 { + write!( + out, + "\u{1b}_Ga=t,q=2,f=32,t=d,i={},s={},v={},m={};{}\u{1b}\\", + host_image_id, + width, + height, + if last == 0 { 0 } else { 1 }, + part + ) + .context(err_context)?; + } else { + write!( + out, + "\u{1b}_Gq=2,m={};{}\u{1b}\\", + if part_index == last { 0 } else { 1 }, + part + ) + .context(err_context)?; + } + } + Ok(()) +} + +fn serialize_kitty_frame(kitty_input: KittyFrameInput) -> Result<String> { + let err_context = "failed to serialize kitty frame"; + let KittyFrameInput { + mut chunks_by_pane, + rendered_panes, + visible_panes, + kitty_image_store, + host_state, + host_display_cleared, + } = kitty_input; + let mut out = String::new(); + if host_display_cleared { + host_state.live_placements.clear(); + host_state.transmitted.clear(); + } + let mut freed: Vec<((InternalImageId, Option<(u16, u16)>), u32)> = host_state + .transmitted + .iter() + .filter(|(image_key, _)| kitty_image_store.get(image_key.0).is_none()) + .map(|(image_key, host_id)| (*image_key, *host_id)) + .collect(); + freed.sort_by_key(|(_, host_id)| *host_id); + for (image_key, host_id) in freed { + write!(out, "\u{1b}_Ga=d,q=2,d=I,i={}\u{1b}\\", host_id).context(err_context)?; + host_state.transmitted.remove(&image_key); + host_state + .live_placements + .retain(|_, record| record.image_key != image_key); + } + if let Some(visible_panes) = visible_panes { + let mut to_delete: Vec<(KittyChunkKey, u32, u32)> = host_state + .live_placements + .iter() + .filter(|(key, _)| !visible_panes.contains(&key.pane_id)) + .map(|(key, record)| (key.clone(), record.host_image_id, record.host_placement_id)) + .collect(); + to_delete.sort_by_key(|(key, _, _)| { + (pane_sort_key(key.pane_id), key.placement_uid, key.sub_index) + }); + for (key, host_image_id, host_placement_id) in to_delete { + write!( + out, + "\u{1b}_Ga=d,q=2,d=i,i={},p={}\u{1b}\\", + host_image_id, host_placement_id + ) + .context(err_context)?; + host_state.live_placements.remove(&key); + } + } + let mut rendered: Vec<PaneId> = rendered_panes.into_iter().collect(); + rendered.sort_by_key(|pane_id| pane_sort_key(*pane_id)); + for pane_id in rendered { + let chunks = chunks_by_pane.remove(&pane_id).unwrap_or_default(); + let mut by_uid: BTreeMap<u64, Vec<KittyImageChunk>> = BTreeMap::new(); + for chunk in chunks { + by_uid.entry(chunk.placement_uid).or_default().push(chunk); + } + let mut current: BTreeMap<KittyChunkKey, KittyImageChunk> = BTreeMap::new(); + for (placement_uid, mut group) in by_uid { + group.sort_by_key(|chunk| (chunk.cell_y, chunk.cell_x)); + for (sub_index, chunk) in group.into_iter().enumerate() { + current.insert( + KittyChunkKey { + pane_id, + placement_uid, + sub_index: sub_index as u32, + }, + chunk, + ); + } + } + let mut stale: Vec<(KittyChunkKey, u32, u32)> = host_state + .live_placements + .iter() + .filter(|(key, _)| key.pane_id == pane_id && !current.contains_key(key)) + .map(|(key, record)| (key.clone(), record.host_image_id, record.host_placement_id)) + .collect(); + stale.sort_by_key(|(key, _, _)| (key.placement_uid, key.sub_index)); + for (key, host_image_id, host_placement_id) in stale { + write!( + out, + "\u{1b}_Ga=d,q=2,d=i,i={},p={}\u{1b}\\", + host_image_id, host_placement_id + ) + .context(err_context)?; + host_state.live_placements.remove(&key); + } + for (key, chunk) in current { + let variant = if chunk.scaled_px.is_some() { + Some(chunk.dest_cells) + } else { + None + }; + let image_key = (chunk.internal_image_id, variant); + let host_image_id = match host_state.transmitted.get(&image_key) { + Some(host_image_id) => *host_image_id, + None => { + let raster_dims = match variant { + Some(cells) => { + match kitty_image_store.scaled_variant(chunk.internal_image_id, cells) { + Some(_) => chunk.scaled_px, + None => None, + } + }, + None => kitty_image_store + .get(chunk.internal_image_id) + .map(|image| (image.width as usize, image.height as usize)), + }; + let (width, height) = match raster_dims { + Some(dims) => dims, + None => continue, + }; + let b64 = match kitty_image_store.base64_for(chunk.internal_image_id, variant) { + Some(b64) => b64, + None => continue, + }; + let host_image_id = host_state.next_host_image_id; + host_state.next_host_image_id += 1; + emit_kitty_transmit(&mut out, host_image_id, width, height, &b64)?; + host_state.transmitted.insert(image_key, host_image_id); + host_image_id + }, + }; + let existing = host_state.live_placements.get(&key).map(|record| { + ( + record.host_placement_id, + record.geometry, + record.host_image_id, + ) + }); + let host_placement_id = match existing { + Some((_, geometry, existing_host_image_id)) + if geometry == chunk && existing_host_image_id == host_image_id => + { + continue; + }, + Some((host_placement_id, _, _)) => host_placement_id, + None => { + let host_placement_id = host_state.next_host_placement_id; + host_state.next_host_placement_id += 1; + host_placement_id + }, + }; + vte_goto_instruction(chunk.cell_x, chunk.cell_y, &mut out).context(err_context)?; + write!( + out, + "\u{1b}_Ga=p,q=2,i={},p={},x={},y={},w={},h={},X={},Y={},z={},C=1\u{1b}\\", + host_image_id, + host_placement_id, + chunk.source_px_x, + chunk.source_px_y, + chunk.source_px_width, + chunk.source_px_height, + chunk.cell_offset_x, + chunk.cell_offset_y, + chunk.z_index + ) + .context(err_context)?; + host_state.live_placements.insert( + key, + HostPlacementRecord { + host_image_id, + host_placement_id, + image_key, + geometry: chunk, + }, + ); + } + } + Ok(out) +} + #[derive(Clone, Debug, Default)] pub struct Output { pre_vte_instructions: HashMap<ClientId, Vec<String>>, post_vte_instructions: HashMap<ClientId, Vec<String>>, client_character_chunks: HashMap<ClientId, Vec<CharacterChunk>>, sixel_chunks: HashMap<ClientId, Vec<SixelImageChunk>>, + client_kitty_chunks: HashMap<ClientId, HashMap<PaneId, Vec<KittyImageChunk>>>, + client_rendered_kitty_panes: HashMap<ClientId, HashSet<PaneId>>, + client_kitty_visible_panes: HashMap<ClientId, HashSet<PaneId>>, + clients_with_cleared_host_display: HashSet<ClientId>, link_handler: Option<Rc<RefCell<LinkHandler>>>, sixel_image_store: Rc<RefCell<SixelImageStore>>, + kitty_image_store: Rc<RefCell<KittyImageStore>>, + kitty_host_capabilities: Rc<RefCell<HashMap<ClientId, bool>>>, + kitty_host_state: Rc<RefCell<HashMap<ClientId, HostKittyState>>>, character_cell_size: Rc<RefCell<Option<SizeInPixels>>>, floating_panes_stack: Option<FloatingPanesStack>, styled_underlines: bool, @@ -373,12 +656,18 @@ impl Output { character_cell_size: Rc<RefCell<Option<SizeInPixels>>>, styled_underlines: bool, osc8_hyperlinks: bool, + kitty_image_store: Rc<RefCell<KittyImageStore>>, + kitty_host_capabilities: Rc<RefCell<HashMap<ClientId, bool>>>, + kitty_host_state: Rc<RefCell<HashMap<ClientId, HostKittyState>>>, ) -> Self { Output { sixel_image_store, character_cell_size, styled_underlines, osc8_hyperlinks, + kitty_image_store, + kitty_host_capabilities, + kitty_host_state, ..Default::default() } } @@ -453,6 +742,17 @@ impl Output { entry.push(String::from(vte_instruction)); } } + pub fn mark_host_display_cleared_for_clients( + &mut self, + client_ids: impl Iterator<Item = ClientId>, + ) { + for client_id in client_ids { + self.clients_with_cleared_host_display.insert(client_id); + } + } + pub fn mark_host_display_cleared_for_client(&mut self, client_id: ClientId) { + self.clients_with_cleared_host_display.insert(client_id); + } pub fn add_post_vte_instruction_to_client( &mut self, client_id: ClientId, @@ -517,6 +817,63 @@ impl Output { } } } + pub fn add_kitty_image_chunks_to_client( + &mut self, + client_id: ClientId, + pane_id: PaneId, + kitty_image_chunks: Vec<KittyImageChunk>, + z_index: Option<usize>, + ) { + self.client_rendered_kitty_panes + .entry(client_id) + .or_insert_with(HashSet::new) + .insert(pane_id); + let mut kitty_chunks = match ( + *self.character_cell_size.borrow(), + &self.floating_panes_stack, + ) { + (Some(character_cell_size), Some(floating_panes_stack)) => floating_panes_stack + .visible_kitty_image_chunks(kitty_image_chunks, z_index, &character_cell_size), + _ => kitty_image_chunks, + }; + self.client_kitty_chunks + .entry(client_id) + .or_insert_with(HashMap::new) + .entry(pane_id) + .or_insert_with(Vec::new) + .append(&mut kitty_chunks); + } + pub fn add_kitty_image_chunks_to_multiple_clients( + &mut self, + pane_id: PaneId, + kitty_image_chunks: Vec<KittyImageChunk>, + client_ids: impl Iterator<Item = ClientId>, + z_index: Option<usize>, + ) { + let kitty_chunks = match ( + *self.character_cell_size.borrow(), + &self.floating_panes_stack, + ) { + (Some(character_cell_size), Some(floating_panes_stack)) => floating_panes_stack + .visible_kitty_image_chunks(kitty_image_chunks, z_index, &character_cell_size), + _ => kitty_image_chunks, + }; + for client_id in client_ids { + self.client_rendered_kitty_panes + .entry(client_id) + .or_insert_with(HashSet::new) + .insert(pane_id); + self.client_kitty_chunks + .entry(client_id) + .or_insert_with(HashMap::new) + .entry(pane_id) + .or_insert_with(Vec::new) + .append(&mut kitty_chunks.clone()); + } + } + pub fn set_kitty_visible_panes(&mut self, client_id: ClientId, pane_ids: HashSet<PaneId>) { + self.client_kitty_visible_panes.insert(client_id, pane_ids); + } pub fn serialize(&mut self) -> Result<HashMap<ClientId, String>> { let err_context = || "failed to serialize output to clients".to_string(); @@ -526,6 +883,7 @@ impl Output { let mut client_serialized_render_instructions = String::new(); // append pre-vte instructions for this client + let host_display_cleared = self.clients_with_cleared_host_display.remove(&client_id); if let Some(pre_vte_instructions_for_client) = self.pre_vte_instructions.remove(&client_id) { @@ -534,6 +892,32 @@ impl Output { } } + let kitty_chunks_by_pane = self.client_kitty_chunks.remove(&client_id); + let kitty_rendered_panes = self.client_rendered_kitty_panes.remove(&client_id); + let kitty_visible_panes = self.client_kitty_visible_panes.remove(&client_id); + let client_host_is_kitty_capable = self + .kitty_host_capabilities + .borrow() + .get(&client_id) + .copied() + .unwrap_or(false); + let mut kitty_image_store; + let mut kitty_host_state_map; + let kitty_input = if client_host_is_kitty_capable { + kitty_image_store = self.kitty_image_store.borrow_mut(); + kitty_host_state_map = self.kitty_host_state.borrow_mut(); + Some(KittyFrameInput { + chunks_by_pane: kitty_chunks_by_pane.unwrap_or_default(), + rendered_panes: kitty_rendered_panes.unwrap_or_default(), + visible_panes: kitty_visible_panes.as_ref(), + kitty_image_store: &mut *kitty_image_store, + host_state: kitty_host_state_map.entry(client_id).or_default(), + host_display_cleared, + }) + } else { + None + }; + // append the actual vte client_serialized_render_instructions.push_str( &serialize_chunks( @@ -544,6 +928,7 @@ impl Output { self.styled_underlines, self.osc8_hyperlinks, None, // No size constraints for regular rendering + kitty_input, ) .with_context(err_context)?, ); // TODO: less allocations? @@ -612,6 +997,7 @@ impl Output { self.styled_underlines, self.osc8_hyperlinks, max_size, + None, ) .with_context(err_context)?, ); @@ -645,11 +1031,29 @@ impl Output { || !self.post_vte_instructions.is_empty() || self.client_character_chunks.values().any(|c| !c.is_empty()) || self.sixel_chunks.values().any(|c| !c.is_empty()) + || self + .client_kitty_chunks + .values() + .any(|chunks_by_pane| chunks_by_pane.values().any(|c| !c.is_empty())) + || self.has_pending_kitty_host_deletes() + } + pub fn has_pending_kitty_host_deletes(&self) -> bool { + let kitty_image_store = self.kitty_image_store.borrow(); + self.kitty_host_state.borrow().values().any(|host_state| { + host_state + .transmitted + .keys() + .any(|(internal_id, _)| kitty_image_store.get(*internal_id).is_none()) + }) } pub fn has_rendered_assets(&self) -> bool { // pre_vte and post_vte are not considered rendered assets as they should not be visible self.client_character_chunks.values().any(|c| !c.is_empty()) || self.sixel_chunks.values().any(|c| !c.is_empty()) + || self + .client_kitty_chunks + .values() + .any(|chunks_by_pane| chunks_by_pane.values().any(|c| !c.is_empty())) } pub fn cursor_is_visible( &mut self, @@ -762,6 +1166,178 @@ impl FloatingPanesStack { } chunks_to_check } + pub fn visible_kitty_image_chunks( + &self, + mut kitty_image_chunks: Vec<KittyImageChunk>, + z_index: Option<usize>, + character_cell_size: &SizeInPixels, + ) -> Vec<KittyImageChunk> { + let z_index = z_index.unwrap_or(0); + let mut chunks_to_check: Vec<KittyImageChunk> = kitty_image_chunks.drain(..).collect(); + let panes_to_check = self.layers.iter().skip(z_index); + for pane_geom in panes_to_check { + let chunks_to_check_against_this_pane: Vec<KittyImageChunk> = + chunks_to_check.drain(..).collect(); + for k_chunk in chunks_to_check_against_this_pane { + let mut uncovered_chunks = + self.remove_covered_kitty_parts(pane_geom, &k_chunk, character_cell_size); + chunks_to_check.append(&mut uncovered_chunks); + } + } + chunks_to_check + } + fn remove_covered_kitty_parts( + &self, + pane_geom: &PaneGeom, + k_chunk: &KittyImageChunk, + character_cell_size: &SizeInPixels, + ) -> Vec<KittyImageChunk> { + let rounded_kitty_image_pixel_height = + if k_chunk.source_px_height % character_cell_size.height > 0 { + let modulus = k_chunk.source_px_height % character_cell_size.height; + k_chunk.source_px_height + (character_cell_size.height - modulus) + } else { + k_chunk.source_px_height + }; + let rounded_kitty_image_pixel_width = + if k_chunk.source_px_width % character_cell_size.width > 0 { + let modulus = k_chunk.source_px_width % character_cell_size.width; + k_chunk.source_px_width + (character_cell_size.width - modulus) + } else { + k_chunk.source_px_width + }; + + let pane_top_edge = pane_geom.y * character_cell_size.height; + let pane_left_edge = pane_geom.x * character_cell_size.width; + let pane_bottom_edge = (pane_geom.y + pane_geom.rows.as_usize().saturating_sub(1)) + * character_cell_size.height; + let pane_right_edge = + (pane_geom.x + pane_geom.cols.as_usize().saturating_sub(1)) * character_cell_size.width; + let k_chunk_top_edge = k_chunk.cell_y * character_cell_size.height; + let k_chunk_bottom_edge = k_chunk_top_edge + rounded_kitty_image_pixel_height; + let k_chunk_left_edge = k_chunk.cell_x * character_cell_size.width; + let k_chunk_right_edge = k_chunk_left_edge + rounded_kitty_image_pixel_width; + + let mut uncovered_chunks = vec![]; + let pane_covers_chunk_completely = pane_top_edge <= k_chunk_top_edge + && pane_bottom_edge >= k_chunk_bottom_edge + && pane_left_edge <= k_chunk_left_edge + && pane_right_edge >= k_chunk_right_edge; + let pane_intersects_with_chunk_vertically = (pane_left_edge >= k_chunk_left_edge + && pane_left_edge <= k_chunk_right_edge) + || (pane_right_edge >= k_chunk_left_edge && pane_right_edge <= k_chunk_right_edge) + || (pane_left_edge <= k_chunk_left_edge && pane_right_edge >= k_chunk_right_edge); + let pane_intersects_with_chunk_horizontally = (pane_top_edge >= k_chunk_top_edge + && pane_top_edge <= k_chunk_bottom_edge) + || (pane_bottom_edge >= k_chunk_top_edge && pane_bottom_edge <= k_chunk_bottom_edge) + || (pane_top_edge <= k_chunk_top_edge && pane_bottom_edge >= k_chunk_bottom_edge); + if pane_covers_chunk_completely { + return uncovered_chunks; + } + if pane_top_edge >= k_chunk_top_edge + && pane_top_edge <= k_chunk_bottom_edge + && pane_intersects_with_chunk_vertically + { + let top_image_chunk = KittyImageChunk { + cell_x: k_chunk.cell_x, + cell_y: k_chunk.cell_y, + source_px_x: k_chunk.source_px_x, + source_px_y: k_chunk.source_px_y, + source_px_width: rounded_kitty_image_pixel_width, + source_px_height: pane_top_edge - k_chunk_top_edge, + ..*k_chunk + }; + uncovered_chunks.push(top_image_chunk); + } + if pane_bottom_edge <= k_chunk_bottom_edge + && pane_bottom_edge >= k_chunk_top_edge + && pane_intersects_with_chunk_vertically + { + let bottom_image_chunk = KittyImageChunk { + cell_x: k_chunk.cell_x, + cell_y: (pane_bottom_edge / character_cell_size.height) + 1, + source_px_x: k_chunk.source_px_x, + source_px_y: k_chunk.source_px_y + + (pane_bottom_edge - k_chunk_top_edge) + + character_cell_size.height, + source_px_width: rounded_kitty_image_pixel_width, + source_px_height: (rounded_kitty_image_pixel_height + - (pane_bottom_edge - k_chunk_top_edge)) + .saturating_sub(character_cell_size.height), + ..*k_chunk + }; + uncovered_chunks.push(bottom_image_chunk); + } + if pane_left_edge >= k_chunk_left_edge + && pane_left_edge <= k_chunk_right_edge + && pane_intersects_with_chunk_horizontally + { + let source_px_y = if k_chunk_top_edge < pane_top_edge { + k_chunk.source_px_y + (pane_top_edge - k_chunk_top_edge) + } else { + k_chunk.source_px_y + }; + let max_image_height = if k_chunk_top_edge < pane_top_edge { + rounded_kitty_image_pixel_height.saturating_sub(pane_top_edge - k_chunk_top_edge) + } else { + rounded_kitty_image_pixel_height + }; + let left_image_chunk = KittyImageChunk { + cell_x: k_chunk.cell_x, + cell_y: std::cmp::max(k_chunk.cell_y, pane_top_edge / character_cell_size.height), + source_px_x: k_chunk.source_px_x, + source_px_y, + source_px_width: rounded_kitty_image_pixel_width + .saturating_sub(k_chunk_right_edge.saturating_sub(pane_left_edge)), + source_px_height: std::cmp::min( + pane_bottom_edge - pane_top_edge + character_cell_size.height, + max_image_height, + ), + ..*k_chunk + }; + uncovered_chunks.push(left_image_chunk); + } + if pane_right_edge <= k_chunk_right_edge + && pane_right_edge >= k_chunk_left_edge + && pane_intersects_with_chunk_horizontally + { + let source_px_y = if k_chunk_top_edge < pane_top_edge { + k_chunk.source_px_y + (pane_top_edge - k_chunk_top_edge) + } else { + k_chunk.source_px_y + }; + let max_image_height = if k_chunk_top_edge < pane_top_edge { + rounded_kitty_image_pixel_height.saturating_sub(pane_top_edge - k_chunk_top_edge) + } else { + rounded_kitty_image_pixel_height + }; + let source_px_x = k_chunk.source_px_x + + (pane_right_edge - k_chunk_left_edge) + + character_cell_size.width; + let right_image_chunk = KittyImageChunk { + cell_x: (pane_right_edge / character_cell_size.width) + 1, + cell_y: std::cmp::max(k_chunk.cell_y, pane_top_edge / character_cell_size.height), + source_px_x, + source_px_y, + source_px_width: (rounded_kitty_image_pixel_width + .saturating_sub(pane_right_edge - k_chunk_left_edge)) + .saturating_sub(character_cell_size.width), + source_px_height: std::cmp::min( + pane_bottom_edge - pane_top_edge + character_cell_size.height, + max_image_height, + ), + ..*k_chunk + }; + uncovered_chunks.push(right_image_chunk); + } + if uncovered_chunks.is_empty() { + uncovered_chunks.push(*k_chunk); + } + uncovered_chunks + .into_iter() + .filter(|chunk| chunk.source_px_width > 0 && chunk.source_px_height > 0) + .collect() + } fn remove_covered_parts( &self, pane_geom: &PaneGeom, @@ -1022,6 +1598,8 @@ pub struct CharacterChunk { selection_and_colors: Vec<HighlightSelection>, } +pub use crate::panes::kitty_graphics::grid_state::KittyImageChunk; + #[derive(Debug, Clone, Copy, Default)] pub struct SixelImageChunk { pub cell_x: usize, diff --git a/zellij-server/src/output/unit/output_tests.rs b/zellij-server/src/output/unit/output_tests.rs index 2328376c..e3a2322c 100644 --- a/zellij-server/src/output/unit/output_tests.rs +++ b/zellij-server/src/output/unit/output_tests.rs @@ -1,10 +1,17 @@ -use super::super::{CharacterChunk, FloatingPanesStack, Output, OutputBuffer, SixelImageChunk}; +use super::super::{ + CharacterChunk, FloatingPanesStack, HostKittyState, KittyImageChunk, Output, OutputBuffer, + SixelImageChunk, +}; +use crate::panes::kitty_graphics::parser::{ + DecodedImage, KittyAction, KittyCommandParser, KittyFormat, +}; +use crate::panes::kitty_graphics::store::{InternalImageId, KittyImageStore}; use crate::panes::sixel::SixelImageStore; use crate::panes::terminal_character::AnsiCode; -use crate::panes::{LinkHandler, Row, TerminalCharacter}; +use crate::panes::{LinkHandler, PaneId, Row, TerminalCharacter}; use crate::ClientId; use std::cell::RefCell; -use std::collections::{HashSet, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::rc::Rc; use zellij_utils::pane_size::{Dimension, PaneGeom, Size, SizeInPixels}; @@ -22,6 +29,9 @@ fn create_test_output() -> Output { character_cell_size, styled_underlines, osc8_hyperlinks, + Rc::new(RefCell::new(KittyImageStore::default())), + Rc::new(RefCell::new(HashMap::new())), + Rc::new(RefCell::new(HashMap::new())), ) } @@ -1111,3 +1121,464 @@ fn test_pane_defaults_preserved_when_right_covered() { "Remaining chunk should preserve pane_default_bg" ); } + +type KittyTestParts = ( + Rc<RefCell<KittyImageStore>>, + Rc<RefCell<HashMap<ClientId, bool>>>, + Rc<RefCell<HashMap<ClientId, HostKittyState>>>, +); + +fn create_test_kitty_parts() -> KittyTestParts { + let kitty_image_store = Rc::new(RefCell::new(KittyImageStore::default())); + let capabilities = Rc::new(RefCell::new(HashMap::new())); + capabilities.borrow_mut().insert(1, true); + let host_state = Rc::new(RefCell::new(HashMap::new())); + (kitty_image_store, capabilities, host_state) +} + +fn create_test_kitty_output(parts: &KittyTestParts) -> Output { + let sixel_image_store = Rc::new(RefCell::new(SixelImageStore::default())); + let character_cell_size = Rc::new(RefCell::new(Some(SizeInPixels { + height: 20, + width: 10, + }))); + Output::new( + sixel_image_store, + character_cell_size, + true, + true, + parts.0.clone(), + parts.1.clone(), + parts.2.clone(), + ) +} + +fn store_test_kitty_image( + kitty_image_store: &Rc<RefCell<KittyImageStore>>, + width: u32, + height: u32, +) -> InternalImageId { + kitty_image_store + .borrow_mut() + .store_image(DecodedImage { + bytes: vec![255u8; (width * height * 4) as usize], + width, + height, + format: KittyFormat::Rgba32, + }) + .unwrap() +} + +fn kitty_chunk( + internal_image_id: InternalImageId, + placement_uid: u64, + cell_x: usize, + cell_y: usize, +) -> KittyImageChunk { + KittyImageChunk { + cell_x, + cell_y, + internal_image_id, + source_px_x: 0, + source_px_y: 0, + source_px_width: 30, + source_px_height: 40, + cell_offset_x: 0, + cell_offset_y: 0, + z_index: 0, + dest_cells: (3, 2), + scaled_px: None, + placement_uid, + } +} + +fn run_kitty_frame( + parts: &KittyTestParts, + chunks: Vec<KittyImageChunk>, + floating_panes_stack: Option<FloatingPanesStack>, +) -> String { + let mut output = create_test_kitty_output(parts); + let client_ids: HashSet<ClientId> = create_test_clients(1); + let link_handler = Rc::new(RefCell::new(LinkHandler::new())); + output.add_clients(&client_ids, link_handler, floating_panes_stack); + output.add_kitty_image_chunks_to_client(1, PaneId::Terminal(1), chunks, None); + output.serialize().unwrap().remove(&1).unwrap_or_default() +} + +fn run_kitty_frame_with_host_clear(parts: &KittyTestParts, chunks: Vec<KittyImageChunk>) -> String { + let mut output = create_test_kitty_output(parts); + let client_ids: HashSet<ClientId> = create_test_clients(1); + let link_handler = Rc::new(RefCell::new(LinkHandler::new())); + output.add_clients(&client_ids, link_handler, None); + output.add_pre_vte_instruction_to_client(1, "\u{1b}[m\u{1b}[2J"); + output.mark_host_display_cleared_for_clients(std::iter::once(1)); + output.add_kitty_image_chunks_to_client(1, PaneId::Terminal(1), chunks, None); + output.serialize().unwrap().remove(&1).unwrap_or_default() +} + +fn run_kitty_frame_with_unmarked_clear_string( + parts: &KittyTestParts, + chunks: Vec<KittyImageChunk>, +) -> String { + let mut output = create_test_kitty_output(parts); + let client_ids: HashSet<ClientId> = create_test_clients(1); + let link_handler = Rc::new(RefCell::new(LinkHandler::new())); + output.add_clients(&client_ids, link_handler, None); + output.add_pre_vte_instruction_to_client(1, "\u{1b}[m\u{1b}[2J"); + output.add_kitty_image_chunks_to_client(1, PaneId::Terminal(1), chunks, None); + output.serialize().unwrap().remove(&1).unwrap_or_default() +} + +fn parse_kitty_placement_crops(output: &str) -> Vec<(usize, usize, usize, usize, usize, usize)> { + let marker = "\u{1b}_Ga=p,q=2,"; + let mut crops = vec![]; + let mut search_start = 0; + while let Some(position) = output[search_start..].find(marker) { + let absolute_position = search_start + position; + let after = &output[absolute_position + marker.len()..]; + let end = after.find("\u{1b}\\").unwrap(); + let mut params: HashMap<&str, &str> = HashMap::new(); + for key_value in after[..end].split(',') { + let mut parts = key_value.splitn(2, '='); + let key = parts.next().unwrap(); + let value = parts.next().unwrap_or(""); + params.insert(key, value); + } + let before = output[..absolute_position] + .strip_suffix("\u{1b}[m") + .unwrap(); + let goto_start = before.rfind("\u{1b}[").unwrap(); + let coordinates = &before[goto_start + 2..before.len() - 1]; + let mut coordinate_parts = coordinates.split(';'); + let row: usize = coordinate_parts.next().unwrap().parse().unwrap(); + let column: usize = coordinate_parts.next().unwrap().parse().unwrap(); + crops.push(( + column - 1, + row - 1, + params["x"].parse().unwrap(), + params["y"].parse().unwrap(), + params["w"].parse().unwrap(), + params["h"].parse().unwrap(), + )); + search_start = absolute_position + marker.len(); + } + crops +} + +#[test] +fn kitty_transmit_only_once_across_frames() { + let parts = create_test_kitty_parts(); + let internal = store_test_kitty_image(&parts.0, 30, 40); + let frame_a = run_kitty_frame(&parts, vec![kitty_chunk(internal, 1, 0, 0)], None); + let frame_b = run_kitty_frame(&parts, vec![kitty_chunk(internal, 1, 0, 0)], None); + let combined = format!("{}{}", frame_a, frame_b); + assert_eq!(combined.matches("\u{1b}_Ga=t").count(), 1); + assert!(!frame_b.contains("\u{1b}_G")); +} + +#[test] +fn kitty_placement_bytes_with_negative_z() { + let parts = create_test_kitty_parts(); + let internal = store_test_kitty_image(&parts.0, 30, 40); + let mut chunk = kitty_chunk(internal, 1, 5, 3); + chunk.z_index = -1; + let output = run_kitty_frame(&parts, vec![chunk], None); + assert!(output.contains("\u{1b}_Ga=t,q=2,f=32,t=d,i=2000000000,s=30,v=40,m=1;")); + assert!(output.contains("\u{1b}_Gq=2,m=0;")); + let placement = "\u{1b}[4;6H\u{1b}[m\u{1b}_Ga=p,q=2,i=2000000000,p=1,x=0,y=0,w=30,h=40,X=0,Y=0,z=-1,C=1\u{1b}\\"; + assert!(output.contains(placement)); + let save_position = output.find("\u{1b}[s").unwrap(); + let transmit_position = output.find("\u{1b}_Ga=t").unwrap(); + let restore_position = output.rfind("\u{1b}[u").unwrap(); + assert!(save_position < transmit_position); + assert!(transmit_position < restore_position); +} + +#[test] +fn kitty_host_display_clear_forces_replacement_next_frame() { + let parts = create_test_kitty_parts(); + let internal = store_test_kitty_image(&parts.0, 30, 40); + let frame_1 = run_kitty_frame(&parts, vec![kitty_chunk(internal, 1, 0, 0)], None); + assert_eq!(frame_1.matches("\u{1b}_Ga=p,q=2,").count(), 1); + + let frame_2 = run_kitty_frame(&parts, vec![kitty_chunk(internal, 1, 0, 0)], None); + assert!(!frame_2.contains("\u{1b}_Ga=p,q=2,")); + + let frame_3 = run_kitty_frame_with_host_clear(&parts, vec![kitty_chunk(internal, 1, 0, 0)]); + assert!( + frame_3.contains("\u{1b}[2J"), + "frame must carry the host clear" + ); + assert_eq!( + frame_3.matches("\u{1b}_Ga=t").count(), + 1, + "a host display clear must force the image to be re-transmitted (host may drop image data on 2J)" + ); + assert_eq!( + frame_3.matches("\u{1b}_Ga=p,q=2,").count(), + 1, + "a host display clear must force the placement to be re-emitted" + ); + let clear_pos = frame_3.find("\u{1b}[2J").unwrap(); + let transmit_pos = frame_3.find("\u{1b}_Ga=t").unwrap(); + let place_pos = frame_3.find("\u{1b}_Ga=p,q=2,").unwrap(); + assert!( + clear_pos < transmit_pos && transmit_pos < place_pos, + "after a host clear the order must be 2J, then re-transmit, then re-place" + ); +} + +#[test] +fn kitty_clear_string_without_explicit_mark_does_not_invalidate_host_state() { + let parts = create_test_kitty_parts(); + let internal = store_test_kitty_image(&parts.0, 30, 40); + let frame_1 = run_kitty_frame(&parts, vec![kitty_chunk(internal, 1, 0, 0)], None); + assert_eq!(frame_1.matches("\u{1b}_Ga=t").count(), 1); + + let frame_2 = + run_kitty_frame_with_unmarked_clear_string(&parts, vec![kitty_chunk(internal, 1, 0, 0)]); + assert!( + frame_2.contains("\u{1b}[2J"), + "frame must carry the clear string" + ); + assert!( + !frame_2.contains("\u{1b}_G"), + "host state invalidation must depend on the explicit flag, not on byte sniffing" + ); +} + +#[test] +fn kitty_diff_move_remove_free_retransmit() { + let parts = create_test_kitty_parts(); + let internal = store_test_kitty_image(&parts.0, 30, 40); + let frame_1 = run_kitty_frame(&parts, vec![kitty_chunk(internal, 1, 2, 2)], None); + assert_eq!(frame_1.matches("\u{1b}_Ga=t").count(), 1); + assert!(frame_1.contains("\u{1b}_Ga=p,q=2,i=2000000000,p=1,")); + let frame_2 = run_kitty_frame(&parts, vec![kitty_chunk(internal, 1, 2, 5)], None); + assert!(frame_2.contains("\u{1b}_Ga=p,q=2,i=2000000000,p=1,")); + assert!(!frame_2.contains("a=d")); + assert!(!frame_2.contains("\u{1b}_Ga=t")); + let frame_3 = run_kitty_frame(&parts, vec![], None); + assert!(frame_3.contains("\u{1b}_Ga=d,q=2,d=i,i=2000000000,p=1\u{1b}\\")); + assert!(!frame_3.contains("d=I")); + parts.0.borrow_mut().free(internal); + let frame_4 = run_kitty_frame(&parts, vec![], None); + assert!(frame_4.contains("\u{1b}_Ga=d,q=2,d=I,i=2000000000\u{1b}\\")); + let new_internal = store_test_kitty_image(&parts.0, 30, 40); + let frame_5 = run_kitty_frame(&parts, vec![kitty_chunk(new_internal, 2, 0, 0)], None); + assert!(frame_5.contains("\u{1b}_Ga=t,q=2,f=32,t=d,i=2000000001,")); +} + +#[test] +fn kitty_occlusion_crops_exclude_covered_quarter() { + let parts = create_test_kitty_parts(); + let internal = store_test_kitty_image(&parts.0, 40, 80); + let pane_geom = create_pane_geom(2, 0, 2, 2); + let floating_panes_stack = FloatingPanesStack { + layers: vec![pane_geom], + }; + let mut chunk = kitty_chunk(internal, 1, 0, 0); + chunk.source_px_width = 40; + chunk.source_px_height = 80; + chunk.dest_cells = (4, 4); + let output = run_kitty_frame(&parts, vec![chunk], Some(floating_panes_stack)); + let crops = parse_kitty_placement_crops(&output); + let crop_set: HashSet<(usize, usize, usize, usize, usize, usize)> = + crops.iter().copied().collect(); + let expected_crop_set: HashSet<(usize, usize, usize, usize, usize, usize)> = + [(0, 0, 0, 0, 20, 40), (0, 2, 0, 40, 40, 40)] + .into_iter() + .collect(); + assert_eq!(crop_set, expected_crop_set); + let image_area = 40 * 80; + let covered_area = (4 - 2) * 10 * ((2 - 0) * 20); + let union_area: usize = crops.iter().map(|(_, _, _, _, w, h)| w * h).sum(); + assert_eq!(union_area, image_area - covered_area); + let covered_x_range = 20..40; + let covered_y_range = 0..40; + for (cell_x, cell_y, _, _, w, h) in &crops { + let absolute_x = cell_x * 10; + let absolute_y = cell_y * 20; + let intersects_horizontally = + absolute_x < covered_x_range.end && absolute_x + w > covered_x_range.start; + let intersects_vertically = + absolute_y < covered_y_range.end && absolute_y + h > covered_y_range.start; + assert!( + !(intersects_horizontally && intersects_vertically), + "crop at ({}, {}) size {}x{} intersects the covered pane rect", + cell_x, + cell_y, + w, + h + ); + } +} + +#[test] +fn kitty_capability_gating_suppresses_all_apc() { + let parts_with_false = create_test_kitty_parts(); + parts_with_false.1.borrow_mut().insert(1, false); + let internal = store_test_kitty_image(&parts_with_false.0, 30, 40); + let output = run_kitty_frame( + &parts_with_false, + vec![kitty_chunk(internal, 1, 0, 0)], + None, + ); + assert!(!output.contains("\u{1b}_G")); + assert!(!parts_with_false.2.borrow().contains_key(&1)); + + let parts_with_absent = create_test_kitty_parts(); + parts_with_absent.1.borrow_mut().clear(); + let internal = store_test_kitty_image(&parts_with_absent.0, 30, 40); + let output = run_kitty_frame( + &parts_with_absent, + vec![kitty_chunk(internal, 1, 0, 0)], + None, + ); + assert!(!output.contains("\u{1b}_G")); + assert!(!parts_with_absent.2.borrow().contains_key(&1)); +} + +#[test] +fn is_dirty_with_kitty_chunks_and_pending_deletes() { + let parts = create_test_kitty_parts(); + let internal = store_test_kitty_image(&parts.0, 30, 40); + let mut output = create_test_kitty_output(&parts); + let client_ids = create_test_clients(1); + let link_handler = Rc::new(RefCell::new(LinkHandler::new())); + output.add_clients(&client_ids, link_handler, None); + output.add_kitty_image_chunks_to_client( + 1, + PaneId::Terminal(1), + vec![kitty_chunk(internal, 1, 0, 0)], + None, + ); + assert!(output.is_dirty()); + assert!(output.has_rendered_assets()); + + let pending_parts = create_test_kitty_parts(); + let mut transmitted = HashMap::new(); + transmitted.insert((1 as InternalImageId, None), 2_000_000_000u32); + pending_parts.2.borrow_mut().insert( + 1, + HostKittyState { + transmitted, + ..Default::default() + }, + ); + let pending_output = create_test_kitty_output(&pending_parts); + assert!(pending_output.is_dirty()); +} + +fn extract_kitty_commands(output: &str) -> Vec<Vec<u8>> { + let bytes = output.as_bytes(); + let mut commands = Vec::new(); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == 0x1b + && index + 2 < bytes.len() + && bytes[index + 1] == b'_' + && bytes[index + 2] == b'G' + { + let body_start = index + 3; + let mut end = body_start; + while end + 1 < bytes.len() && !(bytes[end] == 0x1b && bytes[end + 1] == b'\\') { + end += 1; + } + commands.push(bytes[body_start..end].to_vec()); + index = end + 2; + } else { + index += 1; + } + } + commands +} + +#[test] +fn kitty_emitted_bytes_roundtrip_through_our_parser() { + let parts = create_test_kitty_parts(); + let internal = store_test_kitty_image(&parts.0, 40, 80); + let mut chunk = kitty_chunk(internal, 1, 2, 3); + chunk.source_px_x = 5; + chunk.source_px_y = 7; + chunk.source_px_width = 20; + chunk.source_px_height = 40; + chunk.cell_offset_x = 3; + chunk.cell_offset_y = 4; + chunk.z_index = -1; + chunk.dest_cells = (2, 2); + let output = run_kitty_frame(&parts, vec![chunk.clone()], None); + let commands = extract_kitty_commands(&output); + assert!( + !commands.is_empty(), + "expected at least one emitted kitty command" + ); + + let mut parser = KittyCommandParser::new(); + let mut decoded_image: Option<(u32, u32, KittyFormat, usize)> = None; + let mut placement: Option<crate::panes::kitty_graphics::parser::KittyCommand> = None; + for raw in &commands { + match parser.parse(raw) { + Some(Ok(command)) => match command.action { + KittyAction::Transmit => { + let image = command + .image + .as_ref() + .expect("transmit command must carry decoded image data"); + decoded_image = + Some((image.width, image.height, image.format, image.bytes.len())); + }, + KittyAction::Display => placement = Some(command), + _ => {}, + }, + Some(Err(err)) => panic!("emitted command failed to parse: {:?}", err.code), + None => {}, + } + } + + let (width, height, format, byte_len) = + decoded_image.expect("transmit round-trip produced no decoded image"); + assert_eq!((width, height), (40, 80)); + assert_eq!(format, KittyFormat::Rgba32); + assert_eq!(byte_len, (40 * 80 * 4) as usize); + + let placement = placement.expect("no placement command round-tripped"); + assert_eq!(placement.source_x, chunk.source_px_x as u32); + assert_eq!(placement.source_y, chunk.source_px_y as u32); + assert_eq!(placement.source_w, chunk.source_px_width as u32); + assert_eq!(placement.source_h, chunk.source_px_height as u32); + assert_eq!(placement.cell_offset_x, chunk.cell_offset_x); + assert_eq!(placement.cell_offset_y, chunk.cell_offset_y); + assert_eq!(placement.z_index, chunk.z_index); + assert!(placement.suppress_cursor_movement); +} + +#[test] +fn kitty_host_ids_stay_within_signed_32_bit_range() { + let parts = create_test_kitty_parts(); + for id in 0..8u64 { + let internal = store_test_kitty_image(&parts.0, 4, 4); + let output = run_kitty_frame(&parts, vec![kitty_chunk(internal, id, 0, 0)], None); + for command in extract_kitty_commands(&output) { + let text = String::from_utf8(command).unwrap(); + for segment in text.split([',', ';']) { + if let Some(value) = segment.strip_prefix("i=") { + let numeric: i64 = value.parse().unwrap(); + assert!( + numeric <= i32::MAX as i64, + "host image id {} exceeds signed 32-bit range (Konsole parses with toInt)", + numeric + ); + } + if let Some(value) = segment.strip_prefix("p=") { + let numeric: i64 = value.parse().unwrap(); + assert!( + numeric <= i32::MAX as i64, + "host placement id {} exceeds signed 32-bit range", + numeric + ); + } + } + } + } +} |
