image rendering and visual selection

This commit is contained in:
2026-07-07 14:50:49 +02:00
parent 04b6776ebf
commit 4727e51d1b
20 changed files with 4476 additions and 2427 deletions
+279 -107
View File
@@ -11,7 +11,7 @@ use std::time::Instant;
use base64::Engine;
use flate2::read::ZlibDecoder;
use image::{codecs::gif::GifDecoder, AnimationDecoder, ImageFormat};
use image::{AnimationDecoder, ImageFormat, codecs::gif::GifDecoder};
/// Action to perform with the graphics command.
#[derive(Clone, Copy, Debug, PartialEq, Default)]
@@ -221,8 +221,10 @@ impl GraphicsCommand {
}
}
let is_animation =
matches!(cmd.action, Action::AnimationFrame | Action::AnimationControl);
let is_animation = matches!(
cmd.action,
Action::AnimationFrame | Action::AnimationControl
);
// Second pass: parse all keys with correct interpretation
for (key, value) in pairs {
@@ -355,7 +357,11 @@ impl GraphicsCommand {
}
// Decode base64 payload
log::debug!("Parsing payload: len={}, content={:?}", payload_part.len(), std::str::from_utf8(payload_part).ok());
log::debug!(
"Parsing payload: len={}, content={:?}",
payload_part.len(),
std::str::from_utf8(payload_part).ok()
);
if !payload_part.is_empty() {
if let Ok(payload_str) = std::str::from_utf8(payload_part) {
if let Ok(decoded) = base64_decode(payload_str) {
@@ -455,8 +461,13 @@ pub fn decode_gif(
return Err(GraphicsError::GifDecodeFailed);
}
log::debug!("Decoded GIF: {}x{}, {} frames, {}ms total duration",
width, height, frames.len(), total_duration_ms);
log::debug!(
"Decoded GIF: {}x{}, {} frames, {}ms total duration",
width,
height,
frames.len(),
total_duration_ms
);
let first_frame = frames[0].data.clone();
@@ -484,7 +495,7 @@ pub fn decode_gif(
pub fn decode_webm(
path: &str,
) -> Result<(u32, u32, Vec<u8>, Option<AnimationData>), GraphicsError> {
use ffmpeg::format::{input, Pixel};
use ffmpeg::format::{Pixel, input};
use ffmpeg::media::Type;
use ffmpeg::software::scaling::{
context::Context as ScalingContext, flag::Flags,
@@ -823,7 +834,9 @@ pub struct ImageStorage {
current_chunked_id: Option<u32>,
/// Next auto-generated image ID.
next_id: u32,
/// Flag indicating images have changed and need re-upload to GPU.
/// Images that have been updated and need re-upload to GPU.
pub dirty_images: std::collections::HashSet<u32>,
/// Flag indicating placements have changed and need re-render.
pub dirty: bool,
}
@@ -843,6 +856,7 @@ impl ImageStorage {
chunk_buffer: HashMap::new(),
current_chunked_id: None,
next_id: 1,
dirty_images: std::collections::HashSet::new(),
dirty: false,
}
}
@@ -861,12 +875,12 @@ impl ImageStorage {
if cmd.more_chunks {
// Use explicit image_id if provided, otherwise use the current chunked transfer ID
let id = cmd.image_id.or(self.current_chunked_id).unwrap_or(0);
// If this chunk has an explicit ID, it starts a new chunked transfer
if cmd.image_id.is_some() {
self.current_chunked_id = cmd.image_id;
}
let buffer = self.chunk_buffer.entry(id).or_default();
buffer.data.extend_from_slice(&cmd.payload);
if buffer.command.is_none() {
@@ -878,10 +892,10 @@ impl ImageStorage {
// Check if this completes a chunked transfer
// Use explicit image_id if provided, otherwise use the current chunked transfer ID
let id = cmd.image_id.or(self.current_chunked_id).unwrap_or(0);
// Clear the current chunked transfer ID since we're completing it
self.current_chunked_id = None;
if let Some(mut buffer) = self.chunk_buffer.remove(&id) {
buffer.data.extend_from_slice(&cmd.payload);
if let Some(mut buffered_cmd) = buffer.command {
@@ -966,8 +980,15 @@ impl ImageStorage {
cell_width,
cell_height,
);
log::debug!("Placed image id={} at col={} row={}, cols={} rows={}, placements={}",
id, cursor_col, cursor_row, cols, rows, self.placements.len());
log::debug!(
"Placed image id={} at col={} row={}, cols={} rows={}, placements={}",
id,
cursor_col,
cursor_row,
cols,
rows,
self.placements.len()
);
Some(PlacementResult {
cols,
rows,
@@ -998,6 +1019,7 @@ impl ImageStorage {
let virtual_placement = cmd.unicode_placeholder;
if self.images.contains_key(&id) {
log::debug!("Put image {}: found in storage", id);
let (cols, rows) = self.place_image(
cmd,
cursor_col,
@@ -1013,6 +1035,11 @@ impl ImageStorage {
};
(self.format_response(cmd, Ok(id)), Some(placement_result))
} else {
log::warn!(
"Put image {}: NOT found in storage! (storage size: {})",
id,
self.images.len()
);
(
self.format_response(cmd, Err(GraphicsError::ImageNotFound)),
None,
@@ -1022,37 +1049,47 @@ impl ImageStorage {
/// Handle a delete command.
fn handle_delete(&mut self, cmd: &GraphicsCommand) {
log::debug!(
"Delete command: target={:?}, id={:?}",
cmd.delete_target,
cmd.image_id
);
match &cmd.delete_target {
DeleteTarget::All => {
log::debug!("Deleting all images and placements");
self.images.clear();
self.placements.clear();
self.dirty = true;
}
DeleteTarget::ById(id) => {
let id = cmd.image_id.unwrap_or(*id);
self.images.remove(&id);
log::debug!("Removing all placements of image by id={}", id);
self.placements.retain(|p| p.image_id != id);
self.dirty = true;
}
DeleteTarget::AtCursor => {
// Would need cursor position - simplified for now
log::debug!("Deleting placements at cursor");
self.placements.clear();
self.dirty = true;
}
_ => {
// Other delete modes not yet implemented
log::debug!("Unhandled delete target: {:?}", cmd.delete_target);
}
}
}
/// Handle an animation frame command (a=f).
/// This adds a frame to an existing image's animation.
fn handle_animation_frame(&mut self, mut cmd: GraphicsCommand) -> Option<String> {
fn handle_animation_frame(
&mut self,
mut cmd: GraphicsCommand,
) -> Option<String> {
let id = match cmd.image_id {
Some(id) => id,
None => {
log::warn!("AnimationFrame without image_id");
return self.format_response(&cmd, Err(GraphicsError::MissingId));
return self
.format_response(&cmd, Err(GraphicsError::MissingId));
}
};
@@ -1075,15 +1112,25 @@ impl ImageStorage {
Ok(p) => p.trim().to_string(),
Err(_) => {
log::warn!("Invalid file path in animation frame");
return self.format_response(&cmd, Err(GraphicsError::FileReadFailed));
return self.format_response(
&cmd,
Err(GraphicsError::FileReadFailed),
);
}
};
log::debug!("Reading animation frame from file: {}", path);
match std::fs::read(&path) {
Ok(data) => cmd.payload = data,
Err(e) => {
log::warn!("Failed to read animation frame file {}: {}", path, e);
return self.format_response(&cmd, Err(GraphicsError::FileReadFailed));
log::warn!(
"Failed to read animation frame file {}: {}",
path,
e
);
return self.format_response(
&cmd,
Err(GraphicsError::FileReadFailed),
);
}
}
// Delete temp file after reading
@@ -1095,20 +1142,38 @@ impl ImageStorage {
let shm_name = match std::str::from_utf8(&cmd.payload) {
Ok(p) => p.trim().to_string(),
Err(_) => {
log::warn!("Invalid shared memory name in animation frame");
return self.format_response(&cmd, Err(GraphicsError::FileReadFailed));
log::warn!(
"Invalid shared memory name in animation frame"
);
return self.format_response(
&cmd,
Err(GraphicsError::FileReadFailed),
);
}
};
let shm_path = format!("/dev/shm/{}", shm_name);
log::debug!("Reading animation frame from shared memory: {}", shm_path);
log::debug!(
"Reading animation frame from shared memory: {}",
shm_path
);
match std::fs::read(&shm_path) {
Ok(data) => {
log::debug!("Read {} bytes from shared memory", data.len());
log::debug!(
"Read {} bytes from shared memory",
data.len()
);
cmd.payload = data;
}
Err(e) => {
log::warn!("Failed to read animation frame shm {}: {}", shm_path, e);
return self.format_response(&cmd, Err(GraphicsError::FileReadFailed));
log::warn!(
"Failed to read animation frame shm {}: {}",
shm_path,
e
);
return self.format_response(
&cmd,
Err(GraphicsError::FileReadFailed),
);
}
}
// Remove shared memory object after reading
@@ -1144,7 +1209,10 @@ impl ImageStorage {
Format::Gif => {
// Unlikely, but handle it
log::warn!("GIF format in animation frame - not supported");
return self.format_response(&cmd, Err(GraphicsError::UnsupportedFormat));
return self.format_response(
&cmd,
Err(GraphicsError::UnsupportedFormat),
);
}
};
@@ -1153,18 +1221,23 @@ impl ImageStorage {
Some(img) => img,
None => {
log::warn!("AnimationFrame for non-existent image {}", id);
return self.format_response(&cmd, Err(GraphicsError::ImageNotFound));
return self
.format_response(&cmd, Err(GraphicsError::ImageNotFound));
}
};
// Expected size for a full frame
let expected_size = (image.width * image.height * 4) as usize;
// Initialize animation if this image doesn't have one yet
// This MUST happen before compositing so that frame 0 exists for c=1
if image.animation.is_none() {
// Debug: check base image alpha values
let transparent_count = image.data.chunks(4).filter(|p| p.len() == 4 && p[3] < 255).count();
let transparent_count = image
.data
.chunks(4)
.filter(|p| p.len() == 4 && p[3] < 255)
.count();
let total_pixels = image.data.len() / 4;
log::debug!(
"Creating animation base frame: {}/{} pixels have alpha < 255, data len = {}",
@@ -1172,7 +1245,7 @@ impl ImageStorage {
total_pixels,
image.data.len()
);
let base_frame = AnimationFrame {
data: image.data.clone(),
duration_ms: 100, // Default for base frame
@@ -1187,7 +1260,7 @@ impl ImageStorage {
loops_remaining: DEFAULT_ANIMATION_LOOPS,
});
}
// Composite the frame onto the base frame if needed
// GIF animations typically use delta frames where transparent pixels
// should show through to the previous frame
@@ -1200,18 +1273,20 @@ impl ImageStorage {
} else {
(base_frame_num as usize).saturating_sub(1)
};
if base_idx < anim.frames.len() {
let base_data = &anim.frames[base_idx].data;
if frame_data.len() == expected_size && base_data.len() == expected_size {
if frame_data.len() == expected_size
&& base_data.len() == expected_size
{
// Both frames are full size - composite them
// composition_mode: 0 = alpha blend, 1 = overwrite
let mut composited = base_data.clone();
for i in (0..expected_size).step_by(4) {
let src_a = frame_data[i + 3];
if src_a == 255 {
// Fully opaque source - just copy
composited[i] = frame_data[i];
@@ -1231,25 +1306,36 @@ impl ImageStorage {
let src_g = frame_data[i + 1] as u32;
let src_b = frame_data[i + 2] as u32;
let src_a32 = src_a as u32;
let dst_r = composited[i] as u32;
let dst_g = composited[i + 1] as u32;
let dst_b = composited[i + 2] as u32;
let dst_a = composited[i + 3] as u32;
// Standard alpha compositing: out = src + dst * (1 - src_a)
let inv_a = 255 - src_a32;
composited[i] = ((src_r * src_a32 + dst_r * inv_a) / 255) as u8;
composited[i + 1] = ((src_g * src_a32 + dst_g * inv_a) / 255) as u8;
composited[i + 2] = ((src_b * src_a32 + dst_b * inv_a) / 255) as u8;
composited[i + 3] = (src_a32 + dst_a * inv_a / 255).min(255) as u8;
composited[i] =
((src_r * src_a32 + dst_r * inv_a) / 255)
as u8;
composited[i + 1] =
((src_g * src_a32 + dst_g * inv_a) / 255)
as u8;
composited[i + 2] =
((src_b * src_a32 + dst_b * inv_a) / 255)
as u8;
composited[i + 3] =
(src_a32 + dst_a * inv_a / 255).min(255)
as u8;
}
}
// else: src_a == 0, keep base pixel (already in composited)
}
// Debug: check alpha values
let transparent_count = composited.chunks(4).filter(|p| p.len() == 4 && p[3] < 255).count();
let transparent_count = composited
.chunks(4)
.filter(|p| p.len() == 4 && p[3] < 255)
.count();
let total_pixels = composited.len() / 4;
if transparent_count > 0 {
log::debug!(
@@ -1258,9 +1344,11 @@ impl ImageStorage {
total_pixels
);
}
composited
} else if frame_data.len() < expected_size && base_data.len() == expected_size {
} else if frame_data.len() < expected_size
&& base_data.len() == expected_size
{
// Partial frame data - just use base for now
log::debug!(
"Frame data size {} < expected {}, using base frame {}",
@@ -1277,7 +1365,10 @@ impl ImageStorage {
}
} else {
// Base frame doesn't exist yet (shouldn't happen now), pad the data
log::warn!("Base frame {} doesn't exist, padding data", base_frame_num);
log::warn!(
"Base frame {} doesn't exist, padding data",
base_frame_num
);
let mut data = frame_data;
data.resize(expected_size, 0);
data
@@ -1310,7 +1401,7 @@ impl ImageStorage {
// Add the new frame (animation is guaranteed to exist now)
if let Some(ref mut anim) = image.animation {
let frame_num = cmd.edit_frame.unwrap_or(0);
if frame_num > 0 && (frame_num as usize) <= anim.frames.len() {
// Replace existing frame (1-indexed)
anim.frames[frame_num as usize - 1] = frame;
@@ -1319,7 +1410,7 @@ impl ImageStorage {
anim.total_duration_ms += duration_ms as u64;
anim.frames.push(frame);
}
log::debug!(
"Added animation frame to image {}: now {} frames, {}ms total",
id,
@@ -1329,7 +1420,7 @@ impl ImageStorage {
}
self.dirty = true;
// Return OK response (quiet mode respected)
if cmd.quiet >= 1 {
None
@@ -1340,12 +1431,16 @@ impl ImageStorage {
/// Handle an animation control command (a=a).
/// This controls playback of an animated image.
fn handle_animation_control(&mut self, cmd: &GraphicsCommand) -> Option<String> {
fn handle_animation_control(
&mut self,
cmd: &GraphicsCommand,
) -> Option<String> {
let id = match cmd.image_id {
Some(id) => id,
None => {
log::warn!("AnimationControl without image_id");
return self.format_response(cmd, Err(GraphicsError::MissingId));
return self
.format_response(cmd, Err(GraphicsError::MissingId));
}
};
@@ -1361,7 +1456,8 @@ impl ImageStorage {
Some(img) => img,
None => {
log::warn!("AnimationControl for non-existent image {}", id);
return self.format_response(cmd, Err(GraphicsError::ImageNotFound));
return self
.format_response(cmd, Err(GraphicsError::ImageNotFound));
}
};
@@ -1378,7 +1474,11 @@ impl ImageStorage {
AnimationState::Loading
}
3 => {
log::debug!("Animation {} running ({} frames)", id, anim.frames.len());
log::debug!(
"Animation {} running ({} frames)",
id,
anim.frames.len()
);
// Reset frame start when starting animation
anim.frame_start = None;
anim.looping = true;
@@ -1394,7 +1494,11 @@ impl ImageStorage {
anim.current_frame = frame_num as usize - 1; // 1-indexed to 0-indexed
// No need to clone - renderer uses current_frame_data()
anim.frame_start = None; // Reset timing
log::debug!("Animation {} jumped to frame {}", id, frame_num);
log::debug!(
"Animation {} jumped to frame {}",
id,
frame_num
);
}
}
@@ -1407,7 +1511,11 @@ impl ImageStorage {
anim.looping = true;
anim.loops_remaining = Some(loop_count);
}
log::debug!("Animation {} loop count set to {:?}", id, anim.loops_remaining);
log::debug!(
"Animation {} loop count set to {:?}",
id,
anim.loops_remaining
);
}
self.dirty = true;
@@ -1470,7 +1578,9 @@ impl ImageStorage {
}
// Delete temp file after reading
if cmd.transmission == Transmission::TempFile && file_path.is_none() {
if cmd.transmission == Transmission::TempFile
&& file_path.is_none()
{
let _ = std::fs::remove_file(&path);
}
}
@@ -1504,7 +1614,9 @@ impl ImageStorage {
// Payload is already the data
// Try to detect format from magic bytes if format is default
if cmd.format == Format::Rgba && cmd.payload.len() >= 6 {
if &cmd.payload[0..6] == b"GIF89a" || &cmd.payload[0..6] == b"GIF87a" {
if &cmd.payload[0..6] == b"GIF89a"
|| &cmd.payload[0..6] == b"GIF87a"
{
cmd.format = Format::Gif;
}
}
@@ -1535,26 +1647,36 @@ impl ImageStorage {
(w, h, d, None)
}
Format::Rgba => {
let w = cmd.width.ok_or(GraphicsError::MissingDimensions)?;
let h = cmd.height.ok_or(GraphicsError::MissingDimensions)?;
let w =
cmd.width.ok_or(GraphicsError::MissingDimensions)?;
let h =
cmd.height.ok_or(GraphicsError::MissingDimensions)?;
let expected_size = (w * h * 4) as usize;
if cmd.payload.len() != expected_size {
log::warn!(
"RGBA image size mismatch: declared {}x{} ({} bytes expected), got {} bytes",
w, h, expected_size, cmd.payload.len()
w,
h,
expected_size,
cmd.payload.len()
);
return Err(GraphicsError::InvalidData);
}
(w, h, cmd.payload.clone(), None)
}
Format::Rgb => {
let w = cmd.width.ok_or(GraphicsError::MissingDimensions)?;
let h = cmd.height.ok_or(GraphicsError::MissingDimensions)?;
let w =
cmd.width.ok_or(GraphicsError::MissingDimensions)?;
let h =
cmd.height.ok_or(GraphicsError::MissingDimensions)?;
let expected_size = (w * h * 3) as usize;
if cmd.payload.len() != expected_size {
log::warn!(
"RGB image size mismatch: declared {}x{} ({} bytes expected), got {} bytes",
w, h, expected_size, cmd.payload.len()
w,
h,
expected_size,
cmd.payload.len()
);
return Err(GraphicsError::InvalidData);
}
@@ -1581,6 +1703,7 @@ impl ImageStorage {
animation,
},
);
self.dirty_images.insert(id);
self.dirty = true;
Ok(id)
@@ -1631,8 +1754,13 @@ impl ImageStorage {
};
// Handle relative positioning
let (final_col, final_row) = if let (Some(p_id), Some(q_id)) = (cmd.parent_image_id, cmd.parent_placement_id) {
if let Some(parent) = self.placements.iter().find(|p| p.image_id == p_id && p.placement_id == q_id) {
let (final_col, final_row) = if let Some(p_id) = cmd.parent_image_id {
let q_id = cmd.parent_placement_id.unwrap_or(0);
if let Some(parent) = self
.placements
.iter()
.find(|p| p.image_id == p_id && p.placement_id == q_id)
{
let col = parent.col as i32 + cmd.h_offset;
let row = parent.row as i32 + cmd.v_offset;
(col.max(0) as usize, row.max(0) as usize)
@@ -1671,12 +1799,9 @@ impl ImageStorage {
y_offset: cmd.y_offset,
};
// Remove existing placement with same ID if present
if cmd.placement_id.is_some() {
self.placements.retain(|p| {
p.image_id != id || p.placement_id != placement.placement_id
});
}
let pid = cmd.placement_id.unwrap_or(0);
self.placements
.retain(|p| p.image_id != id || p.placement_id != pid);
self.placements.push(placement);
self.dirty = true;
@@ -1748,6 +1873,24 @@ impl ImageStorage {
&self.placements
}
/// Shift all image placements by a delta (e.g., when scrollback buffer wraps).
pub fn shift_placements(&mut self, delta: isize) {
if delta == 0 {
return;
}
self.placements.retain_mut(|p| {
let new_row = p.row as isize + delta;
if new_row < 0 {
false
} else {
p.row = new_row as usize;
true
}
});
self.dirty = true;
}
/// Get an image by ID.
pub fn get_image(&self, id: u32) -> Option<&ImageData> {
self.images.get(&id)
@@ -1756,6 +1899,7 @@ impl ImageStorage {
/// Clear the dirty flag.
pub fn clear_dirty(&mut self) {
self.dirty = false;
self.dirty_images.clear();
}
/// Update animations and return list of image IDs that changed frames.
@@ -1774,13 +1918,19 @@ impl ImageStorage {
// Initialize frame start time if not set
if anim.frame_start.is_none() {
anim.frame_start = Some(now);
log::debug!("Animation {} started, {} frames, first frame {}ms",
id, anim.frames.len(), anim.frames[0].duration_ms);
log::debug!(
"Animation {} started, {} frames, first frame {}ms",
id,
anim.frames.len(),
anim.frames[0].duration_ms
);
}
let frame_start = anim.frame_start.unwrap();
let elapsed = now.duration_since(frame_start).as_millis() as u32;
let current_frame_duration = anim.frames[anim.current_frame].duration_ms;
let elapsed =
now.duration_since(frame_start).as_millis() as u32;
let current_frame_duration =
anim.frames[anim.current_frame].duration_ms;
if elapsed >= current_frame_duration {
// Advance to next frame
@@ -1790,36 +1940,56 @@ impl ImageStorage {
if anim.looping {
// Check loop count
if let Some(ref mut loops) = anim.loops_remaining {
if *loops > 0 {
log::debug!("Animation {} looping, {} loops remaining", id, *loops - 1);
*loops -= 1;
anim.current_frame = 0;
} else {
log::debug!("Animation {} stopped: no more loops", id);
// No more loops, stop
anim.state = AnimationState::Stopped;
continue;
}
} else {
log::debug!("Animation {} looping indefinitely", id);
// Infinite looping
anim.current_frame = 0;
}
}
log::debug!("Animation {} reached end, looping={}", id, anim.looping);
if !anim.looping {
log::debug!("Animation {} stopping (looping=false)", id);
}
// else: stay on last frame
} else {
if *loops > 0 {
log::debug!(
"Animation {} looping, {} loops remaining",
id,
*loops - 1
);
*loops -= 1;
anim.current_frame = 0;
} else {
log::debug!(
"Animation {} stopped: no more loops",
id
);
// No more loops, stop
anim.state = AnimationState::Stopped;
continue;
}
} else {
log::debug!(
"Animation {} looping indefinitely",
id
);
// Infinite looping
anim.current_frame = 0;
}
}
log::debug!(
"Animation {} reached end, looping={}",
id,
anim.looping
);
if !anim.looping {
log::debug!(
"Animation {} stopping (looping=false)",
id
);
}
// else: stay on last frame
} else {
anim.current_frame = next_frame;
}
log::debug!("Animation {} frame {} -> {} (elapsed {}ms >= {}ms)",
id, old_frame, anim.current_frame, elapsed, current_frame_duration);
log::debug!(
"Animation {} frame {} -> {} (elapsed {}ms >= {}ms)",
id,
old_frame,
anim.current_frame,
elapsed,
current_frame_duration
);
// Just update frame index - no data clone needed!
// The renderer will use current_frame_data() to get the right frame.
@@ -1841,7 +2011,9 @@ impl ImageStorage {
self.images.values().any(|img| {
img.animation
.as_ref()
.map(|a| a.state == AnimationState::Running && a.frames.len() > 1)
.map(|a| {
a.state == AnimationState::Running && a.frames.len() > 1
})
.unwrap_or(false)
})
}