diff --git a/apps/desktop/src-tauri/src/editor_window.rs b/apps/desktop/src-tauri/src/editor_window.rs index e85b900776..12415f6199 100644 --- a/apps/desktop/src-tauri/src/editor_window.rs +++ b/apps/desktop/src-tauri/src/editor_window.rs @@ -37,7 +37,7 @@ async fn do_prewarm(app: AppHandle, path: PathBuf) -> PendingResult { GpuOutputFormat::Rgba => WSFrameFormat::Rgba, }; WSFrame { - data: frame.data, + data: Arc::new(frame.data.into_vec()), width: frame.width, height: frame.height, stride: frame.y_stride, @@ -254,7 +254,7 @@ impl EditorInstances { GpuOutputFormat::Rgba => WSFrameFormat::Rgba, }; WSFrame { - data: frame.data, + data: Arc::new(frame.data.into_vec()), width: frame.width, height: frame.height, stride: frame.y_stride, diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index 29efdb5890..1fde2c3485 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -776,6 +776,18 @@ fn get_codec_and_options( _ => "veryfast", }, ); + if !matches!(preset, H264Preset::Slow | H264Preset::Medium) { + let thread_count = thread::available_parallelism() + .map(|v| v.get()) + .unwrap_or(4); + options.set("threads", &thread_count.to_string()); + options.set("rc-lookahead", "10"); + options.set("b-adapt", "1"); + options.set("aq-mode", "1"); + options.set("ref", "2"); + options.set("subme", "2"); + options.set("trellis", "0"); + } } else { options.set( "preset", diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 7ec1efc98f..76f0ea1c55 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -3,7 +3,9 @@ use cap_editor::{AudioRenderer, get_audio_segments}; use cap_enc_ffmpeg::{AudioEncoder, aac::AACEncoder, h264::H264Encoder, mp4::*}; use cap_media_info::{RawVideoFormat, VideoInfo}; use cap_project::XY; -use cap_rendering::{Nv12RenderedFrame, ProjectUniforms, RenderSegment}; +use cap_rendering::{ + GpuOutputFormat, Nv12RenderedFrame, ProjectUniforms, RenderSegment, SharedNv12Buffer, +}; use futures::FutureExt; use image::ImageBuffer; use serde::Deserialize; @@ -152,16 +154,14 @@ impl Mp4ExportSettings { base: ExporterBase, output_size: (u32, u32), fps: u32, - mut on_progress: impl FnMut(u32) -> bool + Send + 'static, + on_progress: impl FnMut(u32) -> bool + Send + 'static, mode: ExportNv12Mode, ) -> Result { let pipeline_start = std::time::Instant::now(); let output_path = base.output_path.clone(); let meta = &base.studio_meta; - let (tx_image_data, mut video_rx) = - tokio::sync::mpsc::channel::<(Nv12RenderedFrame, u32)>(32); - let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::(32); + let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::(4); let mut video_info = VideoInfo::from_raw(RawVideoFormat::Nv12, output_size.0, output_size.1, fps); @@ -169,16 +169,15 @@ impl Mp4ExportSettings { let audio_segments = get_audio_segments(&base.segments); - let mut audio_renderer = audio_segments + let has_audio = audio_segments .first() .filter(|_| !base.project_config.audio.mute) - .map(|_| AudioRenderer::new(audio_segments.clone())); - let has_audio = audio_renderer.is_some(); + .is_some(); - let stop_after_frames_sent = mode.stop_after_frames_sent; let record_first_queued_ms = mode.record_first_queued_ms_since_pipeline; let nv12_render_startup_breakdown_ms = mode.nv12_render_startup_breakdown_ms; + let project_for_audio = base.project_config.clone(); let pipeline_start_for_encoder = pipeline_start; let encoder_thread = tokio::task::spawn_blocking(move || { trace!("Creating MP4File encoder (NV12 path)"); @@ -206,6 +205,12 @@ impl Mp4ExportSettings { info!("Created MP4File encoder (NV12, external conversion, export settings)"); + let mut audio_renderer = if has_audio { + Some(AudioRenderer::new(audio_segments)) + } else { + None + }; + let mut reusable_frame = ffmpeg::frame::Video::new( ffmpeg::format::Pixel::NV12, output_size.0, @@ -214,9 +219,42 @@ impl Mp4ExportSettings { let mut converted_frame: Option = None; let mut encoded_frames = 0u32; let encode_start = std::time::Instant::now(); + let sample_rate = u64::from(AudioRenderer::SAMPLE_RATE); + let fps_u64 = u64::from(fps); + let mut audio_sample_cursor = 0u64; while let Ok(input) = frame_rx.recv() { - fill_nv12_frame(&mut reusable_frame, &input); + if encoded_frames == 0 + && let Some(audio) = &mut audio_renderer + { + audio.set_playhead(0.0, &project_for_audio); + } + + let audio_frame = audio_renderer.as_mut().and_then(|audio| { + let n = u64::from(input.frame_number); + let end = ((n + 1) * sample_rate) / fps_u64; + if end <= audio_sample_cursor { + return None; + } + let pts = audio_sample_cursor as i64; + let samples = (end - audio_sample_cursor) as usize; + audio_sample_cursor = end; + audio + .render_frame(samples, &project_for_audio) + .map(|mut frame| { + frame.set_pts(Some(pts)); + frame + }) + }); + + fill_nv12_frame_direct( + &mut reusable_frame, + &input.nv12_data, + input.width, + input.height, + input.y_stride, + input.frame_number as i64, + ); encoder .queue_video_frame_reusable( &mut reusable_frame, @@ -224,7 +262,7 @@ impl Mp4ExportSettings { Duration::MAX, ) .map_err(|err| err.to_string())?; - if let Some(audio) = input.audio { + if let Some(audio) = audio_frame { encoder.queue_audio_frame(audio); } encoded_frames += 1; @@ -263,149 +301,11 @@ impl Mp4ExportSettings { }) .then(|r| async { r.map_err(|e| e.to_string()).and_then(|v| v) }); - let render_task = tokio::spawn({ - let project = base.project_config.clone(); - let project_path = base.project_path.clone(); - async move { - let mut frame_count = 0; - let mut first_frame_data: Option = None; - let sample_rate = u64::from(AudioRenderer::SAMPLE_RATE); - let fps_u64 = u64::from(fps); - let mut audio_sample_cursor = 0u64; - let mut consecutive_timeouts = 0u32; - const MAX_CONSECUTIVE_TIMEOUTS: u32 = 3; - - loop { - let timeout_secs = if frame_count == 0 { 120 } else { 90 }; - let (frame, frame_number) = match tokio::time::timeout( - Duration::from_secs(timeout_secs), - video_rx.recv(), - ) - .await - { - Err(_) => { - consecutive_timeouts += 1; - - if consecutive_timeouts >= MAX_CONSECUTIVE_TIMEOUTS { - tracing::error!( - frame_count = frame_count, - timeout_secs = timeout_secs, - consecutive_timeouts = consecutive_timeouts, - "Export render_task timed out {} consecutive times - aborting", - MAX_CONSECUTIVE_TIMEOUTS - ); - return Err(format!( - "Export timed out {MAX_CONSECUTIVE_TIMEOUTS} times consecutively after {timeout_secs}s each waiting for frame {frame_count} - GPU/decoder may be unresponsive" - )); - } - - tracing::warn!( - frame_count = frame_count, - timeout_secs = timeout_secs, - consecutive_timeouts = consecutive_timeouts, - "Frame receive timed out, waiting for next frame..." - ); - continue; - } - Ok(Some(v)) => { - consecutive_timeouts = 0; - v - } - Ok(None) => { - tracing::debug!( - frame_count = frame_count, - "Render channel closed - rendering complete" - ); - break; - } - }; - - if !(on_progress)(frame_count) { - return Err("Export cancelled".to_string()); - } - - let frame_width = frame.width; - let frame_height = frame.height; - let nv12_data = ensure_nv12_data(frame); - - if frame_count == 0 { - first_frame_data = Some(FirstFrameNv12 { - data: nv12_data.clone(), - width: frame_width, - height: frame_height, - y_stride: frame_width, - }); - if let Some(audio) = &mut audio_renderer { - audio.set_playhead(0.0, &project); - } - } - - let audio_frame = audio_renderer.as_mut().and_then(|audio| { - let n = u64::from(frame_number); - let end = ((n + 1) * sample_rate) / fps_u64; - if end <= audio_sample_cursor { - return None; - } - let pts = audio_sample_cursor as i64; - let samples = (end - audio_sample_cursor) as usize; - audio_sample_cursor = end; - audio.render_frame(samples, &project).map(|mut frame| { - frame.set_pts(Some(pts)); - frame - }) - }); - - if frame_tx - .send(Nv12ExportFrame { - audio: audio_frame, - nv12_data, - width: frame_width, - height: frame_height, - y_stride: frame_width, - pts: frame_number as i64, - }) - .is_err() - { - warn!("Renderer task sender dropped. Exiting"); - return Ok(()); - } - - frame_count += 1; - } - - drop(frame_tx); - - if let Some(first) = first_frame_data { - let project_path = project_path.clone(); - let screenshot_task = tokio::task::spawn_blocking(move || { - save_screenshot_from_nv12( - &first.data, - first.width, - first.height, - first.y_stride, - &project_path, - ); - }); - - if let Err(e) = screenshot_task.await { - warn!("Screenshot task failed: {e}"); - } - } else { - warn!("No frames were processed, cannot save screenshot or thumbnail"); - } - - Ok::<_, String>(()) - } - }) - .then(|r| async { - r.map_err(|e| e.to_string()) - .and_then(|v| v.map_err(|e| e.to_string())) - }); - - let render_video_task = cap_rendering::render_video_to_channel_nv12( + let stop_after_frames_sent = mode.stop_after_frames_sent; + let render_video_task = export_render_to_channel( &base.render_constants, &base.project_config, - tx_image_data, + frame_tx, &base.recording_meta, meta, base.segments @@ -420,36 +320,41 @@ impl Mp4ExportSettings { &base.recordings, stop_after_frames_sent, nv12_render_startup_breakdown_ms, + on_progress, + base.project_path.clone(), ) .then(|v| async { v.map_err(|e| e.to_string()) }); - tokio::try_join!(encoder_thread, render_video_task, render_task)?; + tokio::try_join!(encoder_thread, render_video_task)?; Ok(output_path) } } -struct FirstFrameNv12 { - data: Arc>, +struct ExportFrame { + nv12_data: SharedNv12Buffer, width: u32, height: u32, y_stride: u32, + frame_number: u32, } -struct Nv12ExportFrame { - nv12_data: Arc>, +struct FirstFrameNv12 { + data: SharedNv12Buffer, width: u32, height: u32, y_stride: u32, - pts: i64, - audio: Option, } -fn ensure_nv12_data(frame: Nv12RenderedFrame) -> Arc> { - use cap_rendering::GpuOutputFormat; - +fn nv12_from_rendered_frame(frame: Nv12RenderedFrame) -> ExportFrame { if frame.format != GpuOutputFormat::Rgba { - return frame.data; + return ExportFrame { + width: frame.width, + height: frame.height, + y_stride: frame.y_stride, + frame_number: frame.frame_number, + nv12_data: frame.data, + }; } tracing::warn!( @@ -510,7 +415,13 @@ fn ensure_nv12_data(frame: Nv12RenderedFrame) -> Arc> { } } - return Arc::new(result); + return ExportFrame { + nv12_data: SharedNv12Buffer::from_vec(result), + width, + height, + y_stride: width, + frame_number: frame.frame_number, + }; } } @@ -518,20 +429,33 @@ fn ensure_nv12_data(frame: Nv12RenderedFrame) -> Arc> { frame_number = frame.frame_number, "swscale RGBA to NV12 conversion failed, using zeroed NV12" ); - Arc::new(vec![0u8; width as usize * height as usize * 3 / 2]) + ExportFrame { + nv12_data: SharedNv12Buffer::from_vec(vec![0u8; width as usize * height as usize * 3 / 2]), + width, + height, + y_stride: width, + frame_number: frame.frame_number, + } } -fn fill_nv12_frame(frame: &mut ffmpeg::frame::Video, input: &Nv12ExportFrame) { - frame.set_pts(Some(input.pts)); +fn fill_nv12_frame_direct( + frame: &mut ffmpeg::frame::Video, + nv12_data: &[u8], + width: u32, + height: u32, + y_stride: u32, + pts: i64, +) { + frame.set_pts(Some(pts)); - let width = input.width as usize; - let height = input.height as usize; - let y_stride = input.y_stride as usize; + let width = width as usize; + let height = height as usize; + let y_stride = y_stride as usize; let y_plane_size = y_stride * height; - let y_src = &input.nv12_data[..y_plane_size.min(input.nv12_data.len())]; - let uv_src = if y_plane_size < input.nv12_data.len() { - &input.nv12_data[y_plane_size..] + let y_src = &nv12_data[..y_plane_size.min(nv12_data.len())]; + let uv_src = if y_plane_size < nv12_data.len() { + &nv12_data[y_plane_size..] } else { &[] }; @@ -574,6 +498,28 @@ fn fill_nv12_frame(frame: &mut ffmpeg::frame::Video, input: &Nv12ExportFrame) { } } +#[cfg(test)] +struct Nv12ExportFrame { + nv12_data: SharedNv12Buffer, + width: u32, + height: u32, + y_stride: u32, + pts: i64, + audio: Option, +} + +#[cfg(test)] +fn fill_nv12_frame(frame: &mut ffmpeg::frame::Video, input: &Nv12ExportFrame) { + fill_nv12_frame_direct( + frame, + &input.nv12_data, + input.width, + input.height, + input.y_stride, + input.pts, + ); +} + fn save_screenshot_from_nv12( nv12_data: &[u8], width: u32, @@ -615,6 +561,142 @@ fn save_screenshot_from_nv12( let _ = rgb_img.save(&screenshot_path); } +use cap_project::{ProjectConfiguration, RecordingMeta, StudioRecordingMeta}; +use cap_rendering::{ProjectRecordingsMeta, RenderVideoConstants}; + +const FRAME_RECEIVE_INITIAL_TIMEOUT_SECS: u64 = 120; +const FRAME_RECEIVE_STEADY_TIMEOUT_SECS: u64 = 90; +const MAX_CONSECUTIVE_FRAME_TIMEOUTS: u32 = 3; + +#[allow(clippy::too_many_arguments)] +async fn export_render_to_channel( + constants: &RenderVideoConstants, + project: &ProjectConfiguration, + sender: std::sync::mpsc::SyncSender, + recording_meta: &RecordingMeta, + meta: &StudioRecordingMeta, + render_segments: Vec, + fps: u32, + resolution_base: XY, + recordings: &ProjectRecordingsMeta, + stop_after_frames_sent: Option, + startup_breakdown_ms: Option>>>, + mut on_progress: impl FnMut(u32) -> bool + Send + 'static, + project_path: PathBuf, +) -> Result<(), cap_rendering::RenderingError> { + let (tx_image_data, mut video_rx) = tokio::sync::mpsc::channel::<(Nv12RenderedFrame, u32)>(2); + + let screenshot_project_path = project_path; + + let render_result = { + let render_future = cap_rendering::render_video_to_channel_nv12( + constants, + project, + tx_image_data, + recording_meta, + meta, + render_segments, + fps, + resolution_base, + recordings, + stop_after_frames_sent, + startup_breakdown_ms, + ); + + let forward_future = async { + let mut first_frame_data: Option = None; + let mut frame_count = 0u32; + let mut consecutive_timeouts = 0u32; + + loop { + let timeout_secs = if frame_count == 0 { + FRAME_RECEIVE_INITIAL_TIMEOUT_SECS + } else { + FRAME_RECEIVE_STEADY_TIMEOUT_SECS + }; + + let Some((frame, _frame_number)) = (match tokio::time::timeout( + Duration::from_secs(timeout_secs), + video_rx.recv(), + ) + .await + { + Ok(frame) => { + consecutive_timeouts = 0; + frame + } + Err(_) => { + consecutive_timeouts += 1; + + if consecutive_timeouts >= MAX_CONSECUTIVE_FRAME_TIMEOUTS { + return Err(cap_rendering::RenderingError::ImageLoadError(format!( + "Export timed out {MAX_CONSECUTIVE_FRAME_TIMEOUTS} times consecutively after {timeout_secs}s each waiting for frame {frame_count}" + ))); + } + + warn!( + frame_count = frame_count, + timeout_secs = timeout_secs, + consecutive_timeouts = consecutive_timeouts, + "Timed out waiting for rendered frame" + ); + continue; + } + }) else { + break; + }; + + if !(on_progress)(frame_count) { + return Err(cap_rendering::RenderingError::ImageLoadError( + "Export cancelled".to_string(), + )); + } + + let export_frame = nv12_from_rendered_frame(frame); + + if first_frame_data.is_none() { + first_frame_data = Some(FirstFrameNv12 { + data: export_frame.nv12_data.clone(), + width: export_frame.width, + height: export_frame.height, + y_stride: export_frame.y_stride, + }); + } + + if sender.send(export_frame).is_err() { + warn!("Encoder dropped, stopping render forwarding"); + break; + } + + frame_count += 1; + } + + drop(sender); + + if let Some(first) = first_frame_data { + let pp = screenshot_project_path; + let _screenshot_task = tokio::task::spawn_blocking(move || { + save_screenshot_from_nv12( + first.data.as_ref(), + first.width, + first.height, + first.y_stride, + &pp, + ); + }); + } + + Ok::<_, cap_rendering::RenderingError>(()) + }; + + tokio::try_join!(render_future, forward_future) + }; + + render_result?; + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -658,7 +740,7 @@ mod tests { } let input = Nv12ExportFrame { - nv12_data: Arc::new(nv12_data.clone()), + nv12_data: SharedNv12Buffer::from_vec(nv12_data.clone()), width, height, y_stride: width, @@ -689,12 +771,12 @@ mod tests { } #[test] - fn ensure_nv12_data_passthrough_for_nv12_format() { + fn nv12_from_rendered_frame_passthrough_for_nv12_format() { use cap_rendering::{GpuOutputFormat, Nv12RenderedFrame}; let data = vec![1u8, 2, 3, 4, 5, 6]; let frame = Nv12RenderedFrame { - data: std::sync::Arc::new(data.clone()), + data: SharedNv12Buffer::from_vec(data.clone()), width: 4, height: 2, y_stride: 4, @@ -703,8 +785,8 @@ mod tests { format: GpuOutputFormat::Nv12, }; - let result = ensure_nv12_data(frame); - assert_eq!(*result, data); + let result = nv12_from_rendered_frame(frame); + assert_eq!(*result.nv12_data, data); } #[test] diff --git a/crates/rendering/src/cursor_interpolation.rs b/crates/rendering/src/cursor_interpolation.rs index ad0dec8d85..3611945f46 100644 --- a/crates/rendering/src/cursor_interpolation.rs +++ b/crates/rendering/src/cursor_interpolation.rs @@ -236,47 +236,105 @@ pub fn interpolate_cursor_with_click_spring( ); interpolate_timeline(&timeline, time_ms) } else { - if cursor.moves[0].time_ms > time_ms { - let event = &cursor.moves[0]; - return Some(InterpolatedCursorPosition { - position: Coord::new(XY { - x: event.x, - y: event.y, - }), - velocity: XY::new(0.0, 0.0), - cursor_id: event.cursor_id.clone(), - }); + interpolate_raw_cursor(cursor, time_ms) + } +} + +fn interpolate_raw_cursor( + cursor: &CursorEvents, + time_ms: f64, +) -> Option { + if cursor.moves.is_empty() { + return None; + } + + if cursor.moves[0].time_ms > time_ms { + let event = &cursor.moves[0]; + return Some(InterpolatedCursorPosition { + position: Coord::new(XY { + x: event.x, + y: event.y, + }), + velocity: XY::new(0.0, 0.0), + cursor_id: event.cursor_id.clone(), + }); + } + + if let Some(event) = cursor.moves.last() + && event.time_ms <= time_ms + { + return Some(InterpolatedCursorPosition { + position: Coord::new(XY { + x: event.x, + y: event.y, + }), + velocity: XY::new(0.0, 0.0), + cursor_id: event.cursor_id.clone(), + }); + } + + cursor.moves.windows(2).find_map(|chunk| { + if time_ms >= chunk[0].time_ms && time_ms < chunk[1].time_ms { + let c = &chunk[0]; + let next = &chunk[1]; + let delta_ms = (next.time_ms - c.time_ms) as f32; + let dt = (delta_ms / 1000.0).max(0.000_1); + let velocity = XY::new(((next.x - c.x) as f32) / dt, ((next.y - c.y) as f32) / dt); + Some(InterpolatedCursorPosition { + position: Coord::new(XY { x: c.x, y: c.y }), + velocity, + cursor_id: c.cursor_id.clone(), + }) + } else { + None } + }) +} - if let Some(event) = cursor.moves.last() - && event.time_ms <= time_ms - { - return Some(InterpolatedCursorPosition { - position: Coord::new(XY { - x: event.x, - y: event.y, - }), - velocity: XY::new(0.0, 0.0), - cursor_id: event.cursor_id.clone(), - }); +pub struct PrecomputedCursorTimeline { + timeline: Vec, + raw_cursor: CursorEvents, + has_smoothing: bool, +} + +impl PrecomputedCursorTimeline { + pub fn new( + cursor: &CursorEvents, + smoothing: Option, + click_spring: Option, + ) -> Self { + if cursor.moves.is_empty() || smoothing.is_none() { + return Self { + timeline: vec![], + raw_cursor: cursor.clone(), + has_smoothing: false, + }; } - cursor.moves.windows(2).find_map(|chunk| { - if time_ms >= chunk[0].time_ms && time_ms < chunk[1].time_ms { - let c = &chunk[0]; - let next = &chunk[1]; - let delta_ms = (next.time_ms - c.time_ms) as f32; - let dt = (delta_ms / 1000.0).max(0.000_1); - let velocity = XY::new(((next.x - c.x) as f32) / dt, ((next.y - c.y) as f32) / dt); - Some(InterpolatedCursorPosition { - position: Coord::new(XY { x: c.x, y: c.y }), - velocity, - cursor_id: c.cursor_id.clone(), - }) - } else { - None - } - }) + let smoothing_config = smoothing.unwrap(); + let filtered_moves = filter_cursor_shake(&cursor.moves); + let prepared_moves = decimate_cursor_moves(filtered_moves.as_ref()); + let timeline = build_smoothed_timeline( + cursor, + prepared_moves.as_ref(), + smoothing_config, + click_spring, + ); + + Self { + timeline, + raw_cursor: cursor.clone(), + has_smoothing: true, + } + } + + pub fn interpolate(&self, time_secs: f32) -> Option { + let time_ms = (time_secs * 1000.0) as f64; + if self.has_smoothing { + interpolate_timeline(&self.timeline, time_ms) + } else { + interpolate_raw_cursor(&self.raw_cursor, time_ms) + } } } @@ -567,9 +625,8 @@ mod tests { .map(|i| cursor_move(i as f64 * 2.0, 0.5 + i as f64 * 1e-6, 0.5)) .collect(); let decimated = decimate_cursor_moves(&moves); - match decimated { - Cow::Owned(v) => assert!(v.len() < moves.len()), - Cow::Borrowed(_) => {} + if let Cow::Owned(v) = decimated { + assert!(v.len() < moves.len()); } } diff --git a/crates/rendering/src/frame_pipeline.rs b/crates/rendering/src/frame_pipeline.rs index a66fdb52e7..5872fb426d 100644 --- a/crates/rendering/src/frame_pipeline.rs +++ b/crates/rendering/src/frame_pipeline.rs @@ -1,4 +1,5 @@ -use std::sync::Arc; +use std::ops::Deref; +use std::sync::{Arc, Mutex}; use std::time::Instant; use tokio::sync::oneshot; use wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; @@ -7,6 +8,95 @@ use crate::{ProjectUniforms, RenderingError}; const GPU_BUFFER_WAIT_TIMEOUT_SECS: u64 = 10; +pub struct NV12BufferPool { + buffers: Arc>>>, +} + +struct PooledNv12Buffer { + data: Vec, + pool: Option>>>>, +} + +impl Drop for PooledNv12Buffer { + fn drop(&mut self) { + let Some(pool) = self.pool.take() else { + return; + }; + + let mut data = std::mem::take(&mut self.data); + data.clear(); + + if let Ok(mut buffers) = pool.lock() { + buffers.push(data); + } + } +} + +#[derive(Clone)] +pub struct SharedNv12Buffer(Arc); + +impl SharedNv12Buffer { + fn new(data: Vec, pool: Option>>>>) -> Self { + Self(Arc::new(PooledNv12Buffer { data, pool })) + } + + pub fn from_vec(data: Vec) -> Self { + Self::new(data, None) + } + + pub fn from_arc_vec(data: Arc>) -> Self { + Self::from_vec(Arc::unwrap_or_clone(data)) + } + + pub fn into_vec(self) -> Vec { + match Arc::try_unwrap(self.0) { + Ok(mut inner) => { + inner.pool = None; + std::mem::take(&mut inner.data) + } + Err(shared) => shared.data.clone(), + } + } +} + +impl AsRef<[u8]> for SharedNv12Buffer { + fn as_ref(&self) -> &[u8] { + &self.0.data + } +} + +impl Deref for SharedNv12Buffer { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.0.data + } +} + +impl NV12BufferPool { + pub fn new(pre_alloc: usize) -> Self { + Self { + buffers: Arc::new(Mutex::new(Vec::with_capacity(pre_alloc))), + } + } + + pub fn acquire(&self, size: usize) -> Vec { + if let Ok(mut buffers) = self.buffers.lock() + && let Some(pos) = buffers.iter().position(|b| b.capacity() >= size) + { + let mut buf = buffers.swap_remove(pos); + buf.clear(); + return buf; + } + + Vec::with_capacity(size) + } + + pub fn wrap(&self, data: Vec) -> SharedNv12Buffer { + SharedNv12Buffer::new(data, Some(Arc::clone(&self.buffers))) + } +} + pub struct RgbaToNv12Converter { pipeline: wgpu::ComputePipeline, bind_group_layout: wgpu::BindGroupLayout, @@ -304,9 +394,10 @@ impl PendingNv12Readback { RenderingError::BufferMapWaitingFailed } - pub async fn wait( + pub async fn wait_with_pool( mut self, device: &wgpu::Device, + buffer_pool: Option<&mut NV12BufferPool>, ) -> Result { let Some(mut rx) = self.rx.take() else { return Err(self.cancel()); @@ -348,7 +439,15 @@ impl PendingNv12Readback { let buffer_slice = self.buffer.slice(..); let data = buffer_slice.get_mapped_range(); - let nv12_data = Arc::new(data.to_vec()); + let data_len = data.len(); + + let nv12_data = if let Some(pool) = buffer_pool { + let mut buf = pool.acquire(data_len); + buf.extend_from_slice(&data); + pool.wrap(buf) + } else { + SharedNv12Buffer::from_vec(data.to_vec()) + }; drop(data); self.buffer.unmap(); @@ -375,7 +474,7 @@ pub enum GpuOutputFormat { } pub struct Nv12RenderedFrame { - pub data: Arc>, + pub data: SharedNv12Buffer, pub width: u32, pub height: u32, pub y_stride: u32, @@ -398,7 +497,7 @@ impl Nv12RenderedFrame { } pub fn into_data(self) -> Vec { - Arc::unwrap_or_clone(self.data) + self.data.into_vec() } pub fn y_plane(&self) -> &[u8] { @@ -870,19 +969,20 @@ pub async fn finish_encoder( Ok(previous_frame) } -pub async fn finish_encoder_nv12( +pub async fn finish_encoder_nv12_pooled( session: &mut RenderSession, nv12_converter: &mut RgbaToNv12Converter, device: &wgpu::Device, queue: &wgpu::Queue, uniforms: &ProjectUniforms, mut encoder: wgpu::CommandEncoder, + buffer_pool: Option<&mut NV12BufferPool>, ) -> Result, RenderingError> { let width = uniforms.output_size.0; let height = uniforms.output_size.1; let previous_frame = if let Some(prev) = nv12_converter.take_pending() { - Some(prev.wait(device).await?) + Some(prev.wait_with_pool(device, buffer_pool).await?) } else { None }; @@ -915,7 +1015,7 @@ pub async fn finish_encoder_nv12( } else { let rgba_frame = finish_encoder(session, device, queue, uniforms, encoder).await?; Ok(rgba_frame.map(|f| Nv12RenderedFrame { - data: f.data, + data: SharedNv12Buffer::from_arc_vec(f.data), width: f.width, height: f.height, y_stride: f.padded_bytes_per_row, diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 2f5d4616b5..b47423f352 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -6,10 +6,14 @@ use cap_project::{ use composite_frame::CompositeVideoFrameUniforms; use core::f64; use cursor_interpolation::{ - InterpolatedCursorPosition, interpolate_cursor, interpolate_cursor_with_click_spring, + InterpolatedCursorPosition, PrecomputedCursorTimeline, interpolate_cursor, + interpolate_cursor_with_click_spring, }; use decoder::{AsyncVideoDecoderHandle, spawn_decoder}; -use frame_pipeline::{RenderSession, finish_encoder, finish_encoder_nv12, flush_pending_readback}; +use frame_pipeline::{ + NV12BufferPool, RenderSession, finish_encoder, finish_encoder_nv12_pooled, + flush_pending_readback, +}; use futures::future::OptionFuture; use layers::{ Background, BackgroundLayer, BlurLayer, CameraLayer, CaptionsLayer, CursorLayer, DisplayLayer, @@ -46,7 +50,7 @@ pub mod zoom_focus_interpolation; pub use coord::*; pub use decoder::{DecodedFrame, DecoderStatus, DecoderType, PixelFormat}; -pub use frame_pipeline::{GpuOutputFormat, Nv12RenderedFrame, RenderedFrame}; +pub use frame_pipeline::{GpuOutputFormat, Nv12RenderedFrame, RenderedFrame, SharedNv12Buffer}; pub use project_recordings::{ProjectRecordingsMeta, SegmentRecordings, Video}; use mask::interpolate_masks; @@ -457,10 +461,22 @@ pub async fn render_video_to_channel( let click_spring = project.cursor.click_spring_config(); - let mut zoom_focus_interpolators: Vec = render_segments + let precomputed_cursor_timelines: Vec> = render_segments .iter() .map(|segment| { - ZoomFocusInterpolator::new( + Arc::new(PrecomputedCursorTimeline::new( + &segment.cursor, + cursor_smoothing, + Some(click_spring), + )) + }) + .collect(); + + let mut zoom_focus_interpolators: Vec = render_segments + .iter() + .zip(precomputed_cursor_timelines.iter()) + .map(|(segment, precomputed_cursor)| { + ZoomFocusInterpolator::new_with_precomputed_cursor( &segment.cursor, cursor_smoothing, click_spring, @@ -471,6 +487,7 @@ pub async fn render_video_to_channel( .as_ref() .map(|t| t.zoom_segments.as_slice()) .unwrap_or(&[]), + Some(precomputed_cursor.clone()), ) }) .collect(); @@ -563,8 +580,9 @@ pub async fn render_video_to_channel( consecutive_failures = 0; let zoom_focus_interp = &zoom_focus_interpolators[segment_clip_index]; + let precomputed_cursor = &precomputed_cursor_timelines[segment_clip_index]; - let uniforms = ProjectUniforms::new( + let uniforms = ProjectUniforms::new_with_precomputed_cursor( constants, project, current_frame_number, @@ -574,6 +592,7 @@ pub async fn render_video_to_channel( &segment_frames, duration, zoom_focus_interp, + precomputed_cursor, ); let next_frame_number = frame_number; @@ -690,7 +709,11 @@ pub async fn render_video_to_channel( frame_number = current_frame_number, segment_time = segment_time, consecutive_failures = consecutive_failures, - max_retries = DECODE_MAX_RETRIES, + max_retries = if is_initial_frame { + DECODE_MAX_RETRIES_INITIAL + } else { + DECODE_MAX_RETRIES_STEADY + }, "Frame decode failed after retries - using previous frame" ); let mut fallback = last_frame.clone(); @@ -702,7 +725,11 @@ pub async fn render_video_to_channel( tracing::error!( frame_number = current_frame_number, segment_time = segment_time, - max_retries = DECODE_MAX_RETRIES, + max_retries = if is_initial_frame { + DECODE_MAX_RETRIES_INITIAL + } else { + DECODE_MAX_RETRIES_STEADY + }, "First frame decode failed after retries - cannot continue" ); continue; @@ -762,11 +789,23 @@ pub async fn render_video_to_channel_nv12( let click_spring = project.cursor.click_spring_config(); + let precomputed_cursor_timelines: Vec> = render_segments + .iter() + .map(|segment| { + Arc::new(PrecomputedCursorTimeline::new( + &segment.cursor, + cursor_smoothing, + Some(click_spring), + )) + }) + .collect(); + let zoom_build_start = Instant::now(); let mut zoom_focus_interpolators: Vec = render_segments .iter() - .map(|segment| { - ZoomFocusInterpolator::new( + .zip(precomputed_cursor_timelines.iter()) + .map(|(segment, precomputed_cursor)| { + ZoomFocusInterpolator::new_with_precomputed_cursor( &segment.cursor, cursor_smoothing, click_spring, @@ -777,9 +816,13 @@ pub async fn render_video_to_channel_nv12( .as_ref() .map(|t| t.zoom_segments.as_slice()) .unwrap_or(&[]), + Some(precomputed_cursor.clone()), ) }) .collect(); + for interp in &mut zoom_focus_interpolators { + interp.ensure_precomputed_until(duration as f32 + 1.0); + } let zoom_focus_interpolators_construct_ms = zoom_build_start.elapsed().as_millis() as u64; let mut frame_number = 0; @@ -882,8 +925,9 @@ pub async fn render_video_to_channel_nv12( consecutive_failures = 0; let zoom_focus_interp = &zoom_focus_interpolators[segment_clip_index]; + let precomputed_cursor = &precomputed_cursor_timelines[segment_clip_index]; - let uniforms = ProjectUniforms::new( + let uniforms = ProjectUniforms::new_with_precomputed_cursor( constants, project, current_frame_number, @@ -893,6 +937,7 @@ pub async fn render_video_to_channel_nv12( &segment_frames, duration, zoom_focus_interp, + precomputed_cursor, ); let next_frame_number = frame_number; @@ -1112,7 +1157,11 @@ pub async fn render_video_to_channel_nv12( frame_number = current_frame_number, segment_time = segment_time, consecutive_failures = consecutive_failures, - max_retries = DECODE_MAX_RETRIES, + max_retries = if is_initial_frame { + DECODE_MAX_RETRIES_INITIAL + } else { + DECODE_MAX_RETRIES_STEADY + }, "Frame decode failed after retries - using previous NV12 frame" ); let mut fallback = last_frame.clone_metadata_with_data(); @@ -1129,7 +1178,11 @@ pub async fn render_video_to_channel_nv12( tracing::error!( frame_number = current_frame_number, segment_time = segment_time, - max_retries = DECODE_MAX_RETRIES, + max_retries = if is_initial_frame { + DECODE_MAX_RETRIES_INITIAL + } else { + DECODE_MAX_RETRIES_STEADY + }, "First frame decode failed after retries - cannot continue" ); continue; @@ -1157,7 +1210,8 @@ pub async fn render_video_to_channel_nv12( Ok(()) } -const DECODE_MAX_RETRIES: u32 = 5; +const DECODE_MAX_RETRIES_INITIAL: u32 = 5; +const DECODE_MAX_RETRIES_STEADY: u32 = 2; async fn decode_segment_frames_with_retry( decoders: &RecordingSegmentDecoders, @@ -1169,13 +1223,18 @@ async fn decode_segment_frames_with_retry( ) -> Option { let mut result = None; let mut retry_count = 0u32; + let max_retries = if is_initial_frame { + DECODE_MAX_RETRIES_INITIAL + } else { + DECODE_MAX_RETRIES_STEADY + }; - while result.is_none() && retry_count < DECODE_MAX_RETRIES { + while result.is_none() && retry_count < max_retries { if retry_count > 0 { let delay = if is_initial_frame { 500 * (retry_count as u64 + 1) } else { - 50 * retry_count as u64 + 10 }; tokio::time::sleep(std::time::Duration::from_millis(delay)).await; } @@ -1192,7 +1251,7 @@ async fn decode_segment_frames_with_retry( if result.is_none() { retry_count += 1; - if retry_count < DECODE_MAX_RETRIES { + if retry_count < max_retries { tracing::warn!( frame_number = current_frame_number, segment_time = segment_time, @@ -2062,6 +2121,83 @@ impl ProjectUniforms { segment_frames: &DecodedSegmentFrames, total_duration: f64, zoom_focus_interpolator: &ZoomFocusInterpolator, + ) -> Self { + let cursor_smoothing = (!project.cursor.raw).then_some(SpringMassDamperSimulationConfig { + tension: project.cursor.tension, + mass: project.cursor.mass, + friction: project.cursor.friction, + }); + let click_spring_cfg = project.cursor.click_spring_config(); + + let cursor_interp_fn = |time: f32| -> Option { + match cursor_smoothing { + Some(cfg) => interpolate_cursor_with_click_spring( + cursor_events, + time, + Some(cfg), + Some(click_spring_cfg), + ), + None => interpolate_cursor(cursor_events, time, None), + } + }; + + Self::new_inner( + constants, + project, + frame_number, + fps, + resolution_base, + cursor_events, + segment_frames, + total_duration, + zoom_focus_interpolator, + &cursor_interp_fn, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn new_with_precomputed_cursor( + constants: &RenderVideoConstants, + project: &ProjectConfiguration, + frame_number: u32, + fps: u32, + resolution_base: XY, + cursor_events: &CursorEvents, + segment_frames: &DecodedSegmentFrames, + total_duration: f64, + zoom_focus_interpolator: &ZoomFocusInterpolator, + precomputed_cursor: &PrecomputedCursorTimeline, + ) -> Self { + let cursor_interp_fn = |time: f32| -> Option { + precomputed_cursor.interpolate(time) + }; + + Self::new_inner( + constants, + project, + frame_number, + fps, + resolution_base, + cursor_events, + segment_frames, + total_duration, + zoom_focus_interpolator, + &cursor_interp_fn, + ) + } + + #[allow(clippy::too_many_arguments)] + fn new_inner( + constants: &RenderVideoConstants, + project: &ProjectConfiguration, + frame_number: u32, + fps: u32, + resolution_base: XY, + _cursor_events: &CursorEvents, + segment_frames: &DecodedSegmentFrames, + total_duration: f64, + zoom_focus_interpolator: &ZoomFocusInterpolator, + cursor_interp_fn: &dyn Fn(f32) -> Option, ) -> Self { let options = &constants.options; let output_size = Self::get_output_size(options, project, resolution_base); @@ -2099,44 +2235,10 @@ impl ProjectUniforms { let crop = Self::get_crop(options, project); - let cursor_smoothing = (!project.cursor.raw).then_some(SpringMassDamperSimulationConfig { - tension: project.cursor.tension, - mass: project.cursor.mass, - friction: project.cursor.friction, - }); - - let click_spring_cfg = project.cursor.click_spring_config(); - - let interpolated_cursor = match cursor_smoothing { - Some(cfg) => interpolate_cursor_with_click_spring( - cursor_events, - cursor_time_for_interp, - Some(cfg), - Some(click_spring_cfg), - ), - None => interpolate_cursor(cursor_events, cursor_time_for_interp, None), - }; - - let prev_interpolated_cursor = match cursor_smoothing { - Some(cfg) => interpolate_cursor_with_click_spring( - cursor_events, - prev_cursor_time_for_interp, - Some(cfg), - Some(click_spring_cfg), - ), - None => interpolate_cursor(cursor_events, prev_cursor_time_for_interp, None), - }; - + let interpolated_cursor = cursor_interp_fn(cursor_time_for_interp); + let prev_interpolated_cursor = cursor_interp_fn(prev_cursor_time_for_interp); let lookback_t = (cursor_time_for_interp - 0.4).max(0.0); - let past_cursor_for_tilt = match cursor_smoothing { - Some(cfg) => interpolate_cursor_with_click_spring( - cursor_events, - lookback_t, - Some(cfg), - Some(click_spring_cfg), - ), - None => interpolate_cursor(cursor_events, lookback_t, None), - }; + let past_cursor_for_tilt = cursor_interp_fn(lookback_t); let cursor_x_axis_tilt_radians = if let (Some(cur), Some(past)) = (&interpolated_cursor, past_cursor_for_tilt) { @@ -2202,15 +2304,7 @@ impl ProjectUniforms { let boundary_recording_time = (current_recording_time as f64 - (frame_time as f64 - prev.end)) .clamp(0.0, prev.end) as f32; - match cursor_smoothing { - Some(cfg) => interpolate_cursor_with_click_spring( - cursor_events, - boundary_recording_time, - Some(cfg), - Some(click_spring_cfg), - ), - None => interpolate_cursor(cursor_events, boundary_recording_time, None), - } + cursor_interp_fn(boundary_recording_time) }) .map(|c| Coord::::new(c.position.coord)); @@ -2238,15 +2332,7 @@ impl ProjectUniforms { let boundary_recording_time = (prev_recording_time as f64 - (prev_frame_time as f64 - prev.end)) .clamp(0.0, prev.end) as f32; - match cursor_smoothing { - Some(cfg) => interpolate_cursor_with_click_spring( - cursor_events, - boundary_recording_time, - Some(cfg), - Some(click_spring_cfg), - ), - None => interpolate_cursor(cursor_events, boundary_recording_time, None), - } + cursor_interp_fn(boundary_recording_time) }) .map(|c| Coord::::new(c.position.coord)); @@ -2667,6 +2753,7 @@ pub struct FrameRenderer<'a> { constants: &'a RenderVideoConstants, session: Option, nv12_converter: Option, + nv12_buffer_pool: NV12BufferPool, } impl<'a> FrameRenderer<'a> { @@ -2677,6 +2764,7 @@ impl<'a> FrameRenderer<'a> { constants, session: None, nv12_converter: None, + nv12_buffer_pool: NV12BufferPool::new(6), } } @@ -2803,7 +2891,11 @@ impl<'a> FrameRenderer<'a> { ) -> Option> { let nv12_converter = self.nv12_converter.as_mut()?; let pending = nv12_converter.take_pending()?; - Some(pending.wait(&self.constants.device).await) + Some( + pending + .wait_with_pool(&self.constants.device, Some(&mut self.nv12_buffer_pool)) + .await, + ) } pub async fn render_nv12( @@ -2812,6 +2904,112 @@ impl<'a> FrameRenderer<'a> { uniforms: ProjectUniforms, cursor: &CursorEvents, layers: &mut RendererLayers, + ) -> Result, RenderingError> { + if self.constants.is_software_adapter { + return self + .render_nv12_software_path(segment_frames, uniforms, cursor, layers) + .await; + } + + self.render_nv12_gpu_path(segment_frames, uniforms, cursor, layers) + .await + } + + async fn render_nv12_software_path( + &mut self, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: &CursorEvents, + layers: &mut RendererLayers, + ) -> Result, RenderingError> { + let rgba_frame = self + .render(segment_frames, uniforms.clone(), cursor, layers) + .await?; + + let Some(rgba_frame) = rgba_frame else { + return Ok(None); + }; + + let width = rgba_frame.width; + let height = rgba_frame.height; + let padded_bytes_per_row = rgba_frame.padded_bytes_per_row; + let frame_number = rgba_frame.frame_number; + let target_time_ns = rgba_frame.target_time_ns; + + let nv12_size = (width as usize) * (height as usize) * 3 / 2; + let mut nv12_buf = self.nv12_buffer_pool.acquire(nv12_size); + + let y_stride = width as usize; + let uv_stride = width as usize; + let y_plane_size = y_stride * height as usize; + let uv_plane_size = uv_stride * (height as usize / 2); + nv12_buf.resize(y_plane_size + uv_plane_size, 0); + + let src_data = &rgba_frame.data; + let src_stride = padded_bytes_per_row as usize; + + for row in 0..height as usize { + let src_row = &src_data[row * src_stride..row * src_stride + width as usize * 4]; + let y_row = &mut nv12_buf[row * y_stride..(row + 1) * y_stride]; + for col in 0..width as usize { + let r = src_row[col * 4] as i32; + let g = src_row[col * 4 + 1] as i32; + let b = src_row[col * 4 + 2] as i32; + y_row[col] = ((16 + ((65 * r + 129 * g + 25 * b + 128) >> 8)) as u8).clamp(16, 235); + } + } + + let uv_offset = y_plane_size; + for row in 0..(height as usize / 2) { + let src_row0 = + &src_data[row * 2 * src_stride..row * 2 * src_stride + width as usize * 4]; + let src_row1 = &src_data + [(row * 2 + 1) * src_stride..(row * 2 + 1) * src_stride + width as usize * 4]; + let uv_row = + &mut nv12_buf[uv_offset + row * uv_stride..uv_offset + (row + 1) * uv_stride]; + for col in 0..(width as usize / 2) { + let r = (src_row0[col * 8] as i32 + + src_row0[col * 8 + 4] as i32 + + src_row1[col * 8] as i32 + + src_row1[col * 8 + 4] as i32 + + 2) + / 4; + let g = (src_row0[col * 8 + 1] as i32 + + src_row0[col * 8 + 5] as i32 + + src_row1[col * 8 + 1] as i32 + + src_row1[col * 8 + 5] as i32 + + 2) + / 4; + let b = (src_row0[col * 8 + 2] as i32 + + src_row0[col * 8 + 6] as i32 + + src_row1[col * 8 + 2] as i32 + + src_row1[col * 8 + 6] as i32 + + 2) + / 4; + uv_row[col * 2] = + ((128 + ((-38 * r - 74 * g + 112 * b + 128) >> 8)) as u8).clamp(16, 240); + uv_row[col * 2 + 1] = + ((128 + ((112 * r - 94 * g - 18 * b + 128) >> 8)) as u8).clamp(16, 240); + } + } + + Ok(Some(frame_pipeline::Nv12RenderedFrame { + data: self.nv12_buffer_pool.wrap(nv12_buf), + width, + height, + y_stride: width, + frame_number, + target_time_ns, + format: frame_pipeline::GpuOutputFormat::Nv12, + })) + } + + async fn render_nv12_gpu_path( + &mut self, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: &CursorEvents, + layers: &mut RendererLayers, ) -> Result, RenderingError> { let mut last_error = None; @@ -2874,13 +3072,14 @@ impl<'a> FrameRenderer<'a> { &uniforms, ); - match finish_encoder_nv12( + match finish_encoder_nv12_pooled( session, nv12_converter, &self.constants.device, &self.constants.queue, &uniforms, encoder, + Some(&mut self.nv12_buffer_pool), ) .await { diff --git a/crates/rendering/src/zoom_focus_interpolation.rs b/crates/rendering/src/zoom_focus_interpolation.rs index e354d4d127..5162bfbfb2 100644 --- a/crates/rendering/src/zoom_focus_interpolation.rs +++ b/crates/rendering/src/zoom_focus_interpolation.rs @@ -3,7 +3,8 @@ use cap_project::{ClickSpringConfig, CursorEvents, ScreenMovementSpring, XY, Zoo use crate::{ Coord, RawDisplayUVSpace, cursor_interpolation::{ - InterpolatedCursorPosition, interpolate_cursor, interpolate_cursor_with_click_spring, + InterpolatedCursorPosition, PrecomputedCursorTimeline, interpolate_cursor, + interpolate_cursor_with_click_spring, }, spring_mass_damper::{SpringMassDamperSimulation, SpringMassDamperSimulationConfig}, }; @@ -130,6 +131,7 @@ pub struct ZoomFocusInterpolator { events: Option>, precompute_sim: Option, cursor_events: std::sync::Arc, + precomputed_cursor: Option>, cursor_smoothing: Option, click_spring: ClickSpringConfig, screen_spring: ScreenMovementSpring, @@ -145,12 +147,33 @@ impl ZoomFocusInterpolator { screen_spring: ScreenMovementSpring, duration_secs: f64, zoom_segments: &[ZoomSegment], + ) -> Self { + Self::new_with_precomputed_cursor( + cursor_events, + cursor_smoothing, + click_spring, + screen_spring, + duration_secs, + zoom_segments, + None, + ) + } + + pub fn new_with_precomputed_cursor( + cursor_events: &CursorEvents, + cursor_smoothing: Option, + click_spring: ClickSpringConfig, + screen_spring: ScreenMovementSpring, + duration_secs: f64, + zoom_segments: &[ZoomSegment], + precomputed_cursor: Option>, ) -> Self { let segment_clusters = Self::build_segment_clusters(cursor_events, zoom_segments); Self { events: None, precompute_sim: None, cursor_events: std::sync::Arc::new(cursor_events.clone()), + precomputed_cursor, cursor_smoothing, click_spring, screen_spring, @@ -166,12 +189,33 @@ impl ZoomFocusInterpolator { screen_spring: ScreenMovementSpring, duration_secs: f64, zoom_segments: &[ZoomSegment], + ) -> Self { + Self::new_arc_with_precomputed_cursor( + cursor_events, + cursor_smoothing, + click_spring, + screen_spring, + duration_secs, + zoom_segments, + None, + ) + } + + pub fn new_arc_with_precomputed_cursor( + cursor_events: std::sync::Arc, + cursor_smoothing: Option, + click_spring: ClickSpringConfig, + screen_spring: ScreenMovementSpring, + duration_secs: f64, + zoom_segments: &[ZoomSegment], + precomputed_cursor: Option>, ) -> Self { let segment_clusters = Self::build_segment_clusters(cursor_events.as_ref(), zoom_segments); Self { events: None, precompute_sim: None, cursor_events, + precomputed_cursor, cursor_smoothing, click_spring, screen_spring, @@ -204,6 +248,10 @@ impl ZoomFocusInterpolator { } fn interpolate_cursor_at(&self, time_secs: f32) -> Option { + if let Some(precomputed_cursor) = self.precomputed_cursor.as_ref() { + return precomputed_cursor.interpolate(time_secs); + } + match self.cursor_smoothing { Some(cfg) => interpolate_cursor_with_click_spring( self.cursor_events.as_ref(),