zach/zterm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

image rendering and visual selection

Browse Source
This commit is contained in:
zach
2026-07-07 14:50:49 +02:00
parent 04b6776ebf
commit 4727e51d1b
20 changed files with 4476 additions and 2427 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/image_renderer.rs
+176 -89
View File
@@ -3,9 +3,9 @@
//! This module handles GPU-accelerated rendering of images in the terminal,
//! supporting the Kitty Graphics Protocol for inline image display.
use std::collections::HashMap;
use crate::gpu_types::{ImageUniforms, PaneId};
use crate::graphics::{ImageData, ImagePlacement, ImageStorage};
use std::collections::HashMap;
// ═══════════════════════════════════════════════════════════════════════════════
// GPU IMAGE
@@ -43,7 +43,6 @@ pub struct ImageRenderer {
pub alignment: u64,
}
impl ImageRenderer {
/// Create a new ImageRenderer with the necessary GPU resources.
pub fn new(device: &wgpu::Device) -> Self {
@@ -60,42 +59,49 @@ impl ImageRenderer {
});
// Create bind group layout for uniforms (binding 0)
let uniform_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("Image Uniform Layout"),
entries: &[wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: true,
min_binding_size: None,
},
count: None,
}],
});
// Create bind group layout for textures (binding 1, 2)
let texture_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("Image Texture Layout"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: true },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
let uniform_layout =
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("Image Uniform Layout"),
entries: &[wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::VERTEX
| wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: true,
min_binding_size: None,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 2,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
],
});
}],
});
// Create bind group layout for textures (binding 1, 2)
let texture_layout =
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("Image Texture Layout"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float {
filterable: true,
},
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 2,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(
wgpu::SamplerBindingType::Filtering,
),
count: None,
},
],
});
// Create a large uniform buffer for all image renders in a frame
// Max 256 images per frame (65536 / 256)
@@ -108,20 +114,26 @@ impl ImageRenderer {
});
// Create the uniform bind group
let uniform_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("Image Uniform Bind Group"),
layout: &uniform_layout,
entries: &[wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding {
buffer: &uniform_buffer,
offset: 0,
size: std::num::NonZeroU64::new(std::mem::size_of::<ImageUniforms>() as u64),
}),
}],
});
let uniform_bind_group =
device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("Image Uniform Bind Group"),
layout: &uniform_layout,
entries: &[wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::Buffer(
wgpu::BufferBinding {
buffer: &uniform_buffer,
offset: 0,
size: std::num::NonZeroU64::new(
std::mem::size_of::<ImageUniforms>() as u64,
),
},
),
}],
});
let alignment = device.limits().min_uniform_buffer_offset_alignment as u64;
let alignment =
device.limits().min_uniform_buffer_offset_alignment as u64;
Self {
uniform_layout,
@@ -155,14 +167,28 @@ impl ImageRenderer {
}
/// Upload an image to the GPU, creating or updating its texture.
pub fn upload_image(&mut self, device: &wgpu::Device, queue: &wgpu::Queue, pane_id: PaneId, image: &ImageData) {
log::debug!("upload_image: pane_id={:?}, id={}, width={}, height={}, data_len={}", pane_id, image.id, image.width, image.height, image.data.len());
pub fn upload_image(
&mut self,
device: &wgpu::Device,
queue: &wgpu::Queue,
pane_id: PaneId,
image: &ImageData,
) {
log::debug!(
"upload_image: pane_id={:?}, id={}, width={}, height={}, data_len={}",
pane_id,
image.id,
image.width,
image.height,
image.data.len()
);
// Get current frame data (handles animation frames automatically)
let data = image.current_frame_data();
// Check if we already have this image
if let Some(existing) = self.textures.get(&(pane_id, image.id)) {
if existing.width == image.width && existing.height == image.height {
if existing.width == image.width && existing.height == image.height
{
// Same dimensions, just update the data
queue.write_texture(
wgpu::TexelCopyTextureInfo {
@@ -200,7 +226,8 @@ impl ImageRenderer {
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8UnormSrgb,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
usage: wgpu::TextureUsages::TEXTURE_BINDING
| wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
@@ -228,7 +255,10 @@ impl ImageRenderer {
let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some(&format!("Image {} (pane {:?}) Bind Group", image.id, pane_id)),
label: Some(&format!(
"Image {} (pane {:?}) Bind Group",
image.id, pane_id
)),
layout: &self.texture_layout,
entries: &[
wgpu::BindGroupEntry {
@@ -241,15 +271,17 @@ impl ImageRenderer {
},
],
});
self.textures.insert((pane_id, image.id), GpuImage {
texture,
view,
bind_group,
width: image.width,
height: image.height,
});
self.textures.insert(
(pane_id, image.id),
GpuImage {
texture,
view,
bind_group,
width: image.width,
height: image.height,
},
);
log::debug!(
"Uploaded image {} ({}x{}) to GPU",
@@ -262,45 +294,73 @@ impl ImageRenderer {
/// Remove an image from the GPU.
pub fn remove_image(&mut self, pane_id: PaneId, image_id: u32) {
if self.textures.remove(&(pane_id, image_id)).is_some() {
log::debug!("Removed image {} (pane {:?}) from GPU", image_id, pane_id);
log::debug!(
"Removed image {} (pane {:?}) from GPU",
image_id,
pane_id
);
}
}
/// Sync images from terminal's image storage to GPU.
/// Uploads new/changed images and removes deleted ones.
/// Also updates animation frames.
pub fn sync_images(&mut self, device: &wgpu::Device, queue: &wgpu::Queue, pane_id: PaneId, storage: &mut ImageStorage) {
pub fn sync_images(
&mut self,
device: &wgpu::Device,
queue: &wgpu::Queue,
pane_id: PaneId,
storage: &mut ImageStorage,
) {
// Update animations and get list of changed image IDs
let changed_ids = storage.update_animations();
log::debug!("Sync images: pane_id={:?}, changed_ids={:?}, dirty={}", pane_id, changed_ids, storage.dirty);
log::debug!(
"Sync images: pane_id={:?}, changed_ids={:?}, dirty={}",
pane_id,
changed_ids,
storage.dirty
);
// Re-upload frames that changed due to animation
for id in &changed_ids {
if let Some(image) = storage.get_image(*id) {
self.upload_image(device, queue, pane_id, image);
}
}
if !storage.dirty && changed_ids.is_empty() {
log::debug!(
"Sync images: skipping upload (not dirty, no animations)"
);
return;
}
// Upload all images (upload_image handles deduplication)
for image in storage.images().values() {
self.upload_image(device, queue, pane_id, image);
// Upload images that were marked as dirty (newly transmitted or modified)
for id in &storage.dirty_images {
log::debug!("Sync images: uploading dirty image id={:?}", id);
if let Some(image) = storage.get_image(*id) {
self.upload_image(device, queue, pane_id, image);
}
}
storage.clear_dirty();
}
/// Remove images from the GPU that are not present in the provided set of active images.
/// active_images is a set of (pane_id, image_id) tuples.
pub fn gc_images(&mut self, active_images: &std::collections::HashSet<(PaneId, u32)>) {
let gpu_ids: Vec<(PaneId, u32)> = self.textures.keys().copied().collect();
pub fn gc_images(
&mut self,
active_images: &std::collections::HashSet<(PaneId, u32)>,
) {
let gpu_ids: Vec<(PaneId, u32)> =
self.textures.keys().copied().collect();
let mut removed_count = 0;
for id in gpu_ids {
if !active_images.contains(&id) {
log::debug!(
"GC: removing image {:?} as it is no longer active",
id
);
self.remove_image(id.0, id.1);
removed_count += 1;
}
@@ -327,40 +387,67 @@ impl ImageRenderer {
visible_rows: usize,
dim_factor: f32,
) -> Vec<(u32, ImageUniforms)> {
log::debug!("prepare_image_renders: pane={:?}, placements={}, scrollback={}, offset={}, rows={}", pane_id, placements.len(), scrollback_len, scroll_offset, visible_rows);
log::debug!(
"prepare_image_renders: pane={:?}, placements={}, scrollback={}, offset={}, rows={}",
pane_id,
placements.len(),
scrollback_len,
scroll_offset,
visible_rows
);
let mut renders = Vec::new();
for placement in placements {
// Check if we have the GPU texture for this image
let gpu_image = match self.textures.get(&(pane_id, placement.image_id)) {
Some(img) => img,
None => {
log::debug!("Image {} not found in GPU cache for pane {:?}", placement.image_id, pane_id);
continue;
},
};
let gpu_image =
match self.textures.get(&(pane_id, placement.image_id)) {
Some(img) => img,
None => {
log::debug!(
"Image {} not found in GPU cache for pane {:?}",
placement.image_id,
pane_id
);
continue;
}
};
// Convert absolute row to visible screen row
// placement.row is absolute (scrollback_len_at_placement + cursor_row)
// visible_row = absolute_row - scrollback_len + scroll_offset
let absolute_row = placement.row as isize;
let visible_row = absolute_row - scrollback_len as isize + scroll_offset as isize;
let visible_row =
absolute_row - scrollback_len as isize + scroll_offset as isize;
// Check if image is visible on screen
// Image spans from visible_row to visible_row + placement.rows
let image_bottom = visible_row + placement.rows as isize;
if image_bottom < 0 || visible_row >= visible_rows as isize {
log::debug!("Image {} culled: visible_row={}, image_bottom={}, visible_rows={}", placement.image_id, visible_row, image_bottom, visible_rows);
log::debug!(
"Image {} culled: visible_row={}, image_bottom={}, visible_rows={}",
placement.image_id,
visible_row,
image_bottom,
visible_rows
);
continue; // Image is completely off-screen
}
// Calculate display position in pixels
let pos_x = pane_x + (placement.col as f32 * cell_width) + placement.x_offset as f32;
let pos_y = pane_y + (visible_row as f32 * cell_height) + placement.y_offset as f32;
let pos_x = pane_x
+ (placement.col as f32 * cell_width)
+ placement.x_offset as f32;
let pos_y = pane_y
+ (visible_row as f32 * cell_height)
+ placement.y_offset as f32;
log::debug!(
"Image render: pane_x={} col={} cell_width={} x_offset={} => pos_x={}",
pane_x, placement.col, cell_width, placement.x_offset, pos_x
pane_x,
placement.col,
cell_width,
placement.x_offset,
pos_x
);
// Calculate display size in pixels