diff --git a/.gitignore b/.gitignore index 64320734e4..c84f53be62 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ dist-ssr .env.development .env.test target +target-agent .cursorrules .github/hooks @@ -68,4 +69,5 @@ scripts/releases-backfill-data.txt .opencode/ analysis/ analysis/plans/ -.ralphy \ No newline at end of file +.ralphy +tmp/ \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index c73653976b..960e308337 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ All Rust code must respect these workspace-level lints defined in `Cargo.toml`: **Clippy lints (all denied):** - `dbg_macro` — Never use `dbg!()` in code; use proper logging instead. - `let_underscore_future` — Never write `let _ = async_fn()` which silently drops futures; await or explicitly handle them. -- `unchecked_duration_subtraction` — Use `saturating_sub` instead of `-` for `Duration` to avoid panics. +- `unchecked_time_subtraction` — Use `saturating_sub` instead of `-` for `Duration` to avoid panics. - `collapsible_if` — Merge nested `if` statements: use `if a && b { }` instead of `if a { if b { } }`. - `clone_on_copy` — Don't call `.clone()` on `Copy` types; just copy them directly. - `redundant_closure` — Use function references directly: `iter.map(foo)` instead of `iter.map(|x| foo(x))`. diff --git a/CLAUDE.md b/CLAUDE.md index c6279ee04b..de7026dd37 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -380,7 +380,7 @@ All Rust code must respect these workspace-level lints defined in `Cargo.toml`. **Clippy lints (all denied — code MUST NOT contain these patterns):** - `dbg_macro` — Never use `dbg!()` in code; use proper logging (`tracing::debug!`, etc.) instead. - `let_underscore_future` — Never write `let _ = async_fn()` which silently drops futures; await or explicitly handle them. -- `unchecked_duration_subtraction` — Use `duration.saturating_sub(other)` instead of `duration - other` to avoid panics on underflow. +- `unchecked_time_subtraction` — Use `duration.saturating_sub(other)` instead of `duration - other` to avoid panics on underflow. - `collapsible_if` — Merge nested `if` statements: write `if a && b { }` instead of `if a { if b { } }`. - `clone_on_copy` — Don't call `.clone()` on `Copy` types (integers, bools, etc.); just copy them directly. - `redundant_closure` — Use function references directly: `iter.map(foo)` instead of `iter.map(|x| foo(x))`. diff --git a/Cargo.toml b/Cargo.toml index f39d35f4c6..44097ca757 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,7 @@ unused_must_use = "deny" [workspace.lints.clippy] dbg_macro = "deny" let_underscore_future = "deny" -unchecked_duration_subtraction = "deny" +unchecked_time_subtraction = "deny" collapsible_if = "deny" clone_on_copy = "deny" redundant_closure = "deny" diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index 0eba1e546a..40993c1db3 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -336,8 +336,14 @@ pub async fn generate_export_preview( let zoom_focus_interpolator = ZoomFocusInterpolator::new( &render_segment.cursor, cursor_smoothing, + project_config.cursor.click_spring_config(), project_config.screen_movement_spring, total_duration, + project_config + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), ); let uniforms = ProjectUniforms::new( @@ -482,8 +488,14 @@ pub async fn generate_export_preview_fast( let zoom_focus_interpolator = ZoomFocusInterpolator::new( &segment_media.cursor, cursor_smoothing, + project_config.cursor.click_spring_config(), project_config.screen_movement_spring, total_duration, + project_config + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), ); let uniforms = ProjectUniforms::new( diff --git a/apps/desktop/src-tauri/src/posthog.rs b/apps/desktop/src-tauri/src/posthog.rs index 068b925054..d1ea697695 100644 --- a/apps/desktop/src-tauri/src/posthog.rs +++ b/apps/desktop/src-tauri/src/posthog.rs @@ -2,8 +2,11 @@ use std::{ sync::{OnceLock, PoisonError, RwLock}, time::Duration, }; +use tauri::AppHandle; use tracing::error; +use crate::auth::AuthStore; + #[derive(Debug)] pub enum PostHogEvent { MultipartUploadComplete { @@ -22,36 +25,42 @@ pub enum PostHogEvent { }, } -impl From for posthog_rs::Event { - fn from(event: PostHogEvent) -> Self { - match event { - PostHogEvent::MultipartUploadComplete { - duration, - length, - size, - } => { - let mut e = posthog_rs::Event::new_anon("multipart_upload_complete"); - e.insert_prop("duration", duration.as_secs()) - .map_err(|err| error!("Error adding PostHog property: {err:?}")) - .ok(); - e.insert_prop("length", length.as_secs()) - .map_err(|err| error!("Error adding PostHog property: {err:?}")) - .ok(); - e.insert_prop("size", size) - .map_err(|err| error!("Error adding PostHog property: {err:?}")) - .ok(); - e - } - PostHogEvent::MultipartUploadFailed { duration, error } => { - let mut e = posthog_rs::Event::new_anon("multipart_upload_failed"); - e.insert_prop("duration", duration.as_secs()) - .map_err(|err| error!("Error adding PostHog property: {err:?}")) - .ok(); - e.insert_prop("error", error) - .map_err(|err| error!("Error adding PostHog property: {err:?}")) - .ok(); - e - } +fn posthog_event(event: PostHogEvent, distinct_id: Option<&str>) -> posthog_rs::Event { + match event { + PostHogEvent::MultipartUploadComplete { + duration, + length, + size, + } => { + let mut e = match distinct_id { + Some(distinct_id) => { + posthog_rs::Event::new("multipart_upload_complete", distinct_id) + } + None => posthog_rs::Event::new_anon("multipart_upload_complete"), + }; + e.insert_prop("duration", duration.as_secs()) + .map_err(|err| error!("Error adding PostHog property: {err:?}")) + .ok(); + e.insert_prop("length", length.as_secs()) + .map_err(|err| error!("Error adding PostHog property: {err:?}")) + .ok(); + e.insert_prop("size", size) + .map_err(|err| error!("Error adding PostHog property: {err:?}")) + .ok(); + e + } + PostHogEvent::MultipartUploadFailed { duration, error } => { + let mut e = match distinct_id { + Some(distinct_id) => posthog_rs::Event::new("multipart_upload_failed", distinct_id), + None => posthog_rs::Event::new_anon("multipart_upload_failed"), + }; + e.insert_prop("duration", duration.as_secs()) + .map_err(|err| error!("Error adding PostHog property: {err:?}")) + .ok(); + e.insert_prop("error", error) + .map_err(|err| error!("Error adding PostHog property: {err:?}")) + .ok(); + e } } } @@ -76,10 +85,14 @@ pub fn set_server_url(url: &str) { static API_SERVER_IS_CAP_CLOUD: OnceLock>> = OnceLock::new(); -pub fn async_capture_event(event: PostHogEvent) { +pub fn async_capture_event(app: &AppHandle, event: PostHogEvent) { if option_env!("VITE_POSTHOG_KEY").is_some() { + let distinct_id = AuthStore::get(app) + .ok() + .flatten() + .and_then(|auth| auth.user_id); tokio::spawn(async move { - let mut e: posthog_rs::Event = event.into(); + let mut e = posthog_event(event, distinct_id.as_deref()); e.insert_prop("cap_version", env!("CARGO_PKG_VERSION")) .map_err(|err| error!("Error adding PostHog property: {err:?}")) diff --git a/apps/desktop/src-tauri/src/screenshot_editor.rs b/apps/desktop/src-tauri/src/screenshot_editor.rs index 6dec290f6f..472be597e4 100644 --- a/apps/desktop/src-tauri/src/screenshot_editor.rs +++ b/apps/desktop/src-tauri/src/screenshot_editor.rs @@ -356,8 +356,14 @@ impl ScreenshotEditorInstances { let zoom_focus_interpolator = ZoomFocusInterpolator::new( &cursor_events, None, + current_config.cursor.click_spring_config(), current_config.screen_movement_spring, 0.0, + current_config + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), ); let uniforms = ProjectUniforms::new( diff --git a/apps/desktop/src-tauri/src/upload.rs b/apps/desktop/src-tauri/src/upload.rs index bbe106bd2c..9aafd41b70 100644 --- a/apps/desktop/src-tauri/src/upload.rs +++ b/apps/desktop/src-tauri/src/upload.rs @@ -146,22 +146,25 @@ pub async fn upload_video( emit_upload_complete(app, &video_id); - async_capture_event(match &video_result { - Ok(meta) => PostHogEvent::MultipartUploadComplete { - duration: start.elapsed(), - length: meta - .as_ref() - .map(|v| Duration::from_secs(v.duration_in_secs as u64)) - .unwrap_or_default(), - size: std::fs::metadata(file_path) - .map(|m| ((m.len() as f64) / 1_000_000.0) as u64) - .unwrap_or_default(), - }, - Err(err) => PostHogEvent::MultipartUploadFailed { - duration: start.elapsed(), - error: err.to_string(), + async_capture_event( + app, + match &video_result { + Ok(meta) => PostHogEvent::MultipartUploadComplete { + duration: start.elapsed(), + length: meta + .as_ref() + .map(|v| Duration::from_secs(v.duration_in_secs as u64)) + .unwrap_or_default(), + size: std::fs::metadata(file_path) + .map(|m| ((m.len() as f64) / 1_000_000.0) as u64) + .unwrap_or_default(), + }, + Err(err) => PostHogEvent::MultipartUploadFailed { + duration: start.elapsed(), + error: err.to_string(), + }, }, - }); + ); let _ = (video_result?, thumbnail_result?); @@ -399,29 +402,32 @@ impl InstantMultipartUpload { handle: spawn_actor(async move { let start = Instant::now(); let result = Self::run( - app, + app.clone(), file_path.clone(), pre_created_video, recording_dir, realtime_upload_done, ) .await; - async_capture_event(match &result { - Ok(meta) => PostHogEvent::MultipartUploadComplete { - duration: start.elapsed(), - length: meta - .as_ref() - .map(|v| Duration::from_secs(v.duration_in_secs as u64)) - .unwrap_or_default(), - size: std::fs::metadata(file_path) - .map(|m| ((m.len() as f64) / 1_000_000.0) as u64) - .unwrap_or_default(), - }, - Err(err) => PostHogEvent::MultipartUploadFailed { - duration: start.elapsed(), - error: err.to_string(), + async_capture_event( + &app, + match &result { + Ok(meta) => PostHogEvent::MultipartUploadComplete { + duration: start.elapsed(), + length: meta + .as_ref() + .map(|v| Duration::from_secs(v.duration_in_secs as u64)) + .unwrap_or_default(), + size: std::fs::metadata(file_path) + .map(|m| ((m.len() as f64) / 1_000_000.0) as u64) + .unwrap_or_default(), + }, + Err(err) => PostHogEvent::MultipartUploadFailed { + duration: start.elapsed(), + error: err.to_string(), + }, }, - }); + ); result.map(|_| ()) }), diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index aee1def1de..91117b6786 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -224,7 +224,7 @@ type CursorPresetValues = { friction: number; }; -const DEFAULT_CURSOR_MOTION_BLUR = 0.5; +const DEFAULT_CURSOR_MOTION_BLUR = 1.0; const CURSOR_TYPE_OPTIONS = [ { @@ -619,6 +619,16 @@ export function ConfigSidebar() { step={1} /> + }> + setProject("cursor", "rotationAmount", v[0])} + minValue={0} + maxValue={1} + step={0.01} + formatTooltip={(value) => `${Math.round(value * 100)}%`} + /> + } diff --git a/apps/desktop/src/utils/frame-worker.ts b/apps/desktop/src/utils/frame-worker.ts index 8720428559..07786d9ff9 100644 --- a/apps/desktop/src/utils/frame-worker.ts +++ b/apps/desktop/src/utils/frame-worker.ts @@ -417,7 +417,6 @@ function drainAndRenderLatestSharedWebGPU(maxDrain: number): boolean { if (renderMode !== "webgpu" || !webgpuRenderer) return false; let latest: { bytes: Uint8Array; release: () => void } | null = null; - let drained = 0; for (let i = 0; i < maxDrain; i += 1) { const borrowed = consumer.borrow(0); @@ -427,7 +426,6 @@ function drainAndRenderLatestSharedWebGPU(maxDrain: number): boolean { latest.release(); } latest = { bytes: borrowed.data, release: borrowed.release }; - drained += 1; } if (!latest) return false; diff --git a/apps/desktop/src/utils/socket.ts b/apps/desktop/src/utils/socket.ts index 7d4985ca39..4d173f7eb6 100644 --- a/apps/desktop/src/utils/socket.ts +++ b/apps/desktop/src/utils/socket.ts @@ -624,7 +624,6 @@ export function createImageDataWS( let renderFrameCount = 0; let minFrameTime = Number.MAX_VALUE; let maxFrameTime = 0; - const getLocalFpsStats = (): FpsStats => ({ fps: frameCount > 0 && frameTimeSum > 0 @@ -699,7 +698,6 @@ export function createImageDataWS( const yStride = meta.getUint32(0, true); const height = meta.getUint32(4, true); const width = meta.getUint32(8, true); - const frameNumber = meta.getUint32(12, true); if (width > 0 && height > 0) { const ySize = yStride * height; @@ -765,7 +763,6 @@ export function createImageDataWS( const yStride = meta.getUint32(0, true); const height = meta.getUint32(4, true); const width = meta.getUint32(8, true); - const frameNumber = meta.getUint32(12, true); if (width > 0 && height > 0) { const ySize = yStride * height; diff --git a/apps/desktop/src/utils/webgpu-renderer.ts b/apps/desktop/src/utils/webgpu-renderer.ts index 08ef45ad23..560bb759ac 100644 --- a/apps/desktop/src/utils/webgpu-renderer.ts +++ b/apps/desktop/src/utils/webgpu-renderer.ts @@ -299,8 +299,7 @@ export function renderFrameWebGPU( colorAttachments: [ { view: currentTexture.createView(), - clearValue: { r: 0, g: 0, b: 0, a: 1 }, - loadOp: "clear", + loadOp: "load", storeOp: "store", }, ], @@ -413,8 +412,7 @@ export function renderNv12FrameWebGPU( colorAttachments: [ { view: context.getCurrentTexture().createView(), - clearValue: { r: 0, g: 0, b: 0, a: 1 }, - loadOp: "clear", + loadOp: "load", storeOp: "store", }, ], diff --git a/apps/web/actions/analytics/track-user-signed-up.ts b/apps/web/actions/analytics/track-user-signed-up.ts index 652a55e4a1..2ffb33c6f0 100644 --- a/apps/web/actions/analytics/track-user-signed-up.ts +++ b/apps/web/actions/analytics/track-user-signed-up.ts @@ -5,6 +5,8 @@ import { getCurrentUser } from "@cap/database/auth/session"; import { users } from "@cap/database/schema"; import { sql } from "drizzle-orm"; +const SIGNUP_TRACKING_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; + type UserPreferences = { notifications?: { pauseComments: boolean; @@ -17,6 +19,29 @@ type UserPreferences = { }; } | null; +const getAffectedRows = (result: unknown) => { + if (Array.isArray(result)) { + return ( + (result[0] as { affectedRows?: number } | undefined)?.affectedRows ?? 0 + ); + } + + return (result as { affectedRows?: number } | undefined)?.affectedRows ?? 0; +}; + +const getCreatedAtTime = (value: unknown) => { + if (value instanceof Date) { + return value.getTime(); + } + + if (typeof value === "string" || typeof value === "number") { + const parsed = new Date(value).getTime(); + return Number.isNaN(parsed) ? null : parsed; + } + + return null; +}; + export async function checkAndMarkUserSignedUpTracked(): Promise<{ shouldTrack: boolean; }> { @@ -33,16 +58,25 @@ export async function checkAndMarkUserSignedUpTracked(): Promise<{ return { shouldTrack: false }; } - const [result] = await db() + const createdAtTime = getCreatedAtTime(currentUser.created_at); + + if ( + createdAtTime === null || + Date.now() - createdAtTime > SIGNUP_TRACKING_WINDOW_MS + ) { + return { shouldTrack: false }; + } + + const result = await db() .update(users) .set({ preferences: sql`JSON_SET(COALESCE(${users.preferences}, JSON_OBJECT()), '$.trackedEvents.user_signed_up', true)`, }) .where( - sql`(${users.id} = ${currentUser.id}) AND (${users.created_at} >= CURRENT_DATE()) AND JSON_CONTAINS(COALESCE(${users.preferences}, JSON_OBJECT()), CAST(true AS JSON), '$.trackedEvents.user_signed_up') = 0`, + sql`(${users.id} = ${currentUser.id}) AND JSON_CONTAINS(COALESCE(${users.preferences}, JSON_OBJECT()), CAST(true AS JSON), '$.trackedEvents.user_signed_up') = 0`, ); - return { shouldTrack: result.affectedRows > 0 }; + return { shouldTrack: getAffectedRows(result) > 0 }; } catch { return { shouldTrack: false }; } diff --git a/apps/web/app/(org)/login/form.tsx b/apps/web/app/(org)/login/form.tsx index 3d631c0f02..50ffff2733 100644 --- a/apps/web/app/(org)/login/form.tsx +++ b/apps/web/app/(org)/login/form.tsx @@ -105,7 +105,11 @@ export function LoginForm() { }, [emailSent]); const handleGoogleSignIn = () => { - trackEvent("auth_started", { method: "google", is_signup: true }); + trackEvent("auth_started", { + method: "google", + is_signup: false, + auth_surface: "login", + }); signIn("google", { ...(next && next.length > 0 ? { callbackUrl: next } : {}), }); @@ -267,7 +271,8 @@ export function LoginForm() { setLoading(true); trackEvent("auth_started", { method: "email", - is_signup: !oauthError, + is_signup: false, + auth_surface: "login", }); const normalizedEmail = email.trim().toLowerCase(); signIn("email", { @@ -284,6 +289,9 @@ export function LoginForm() { setEmailSent(true); setLastEmailSentTime(Date.now()); trackEvent("auth_email_sent", { + method: "email", + is_signup: false, + auth_surface: "login", email_domain: normalizedEmail.split("@")[1], }); const params = new URLSearchParams({ diff --git a/apps/web/app/(org)/signup/form.tsx b/apps/web/app/(org)/signup/form.tsx index 7ec96287e2..c35635d417 100644 --- a/apps/web/app/(org)/signup/form.tsx +++ b/apps/web/app/(org)/signup/form.tsx @@ -105,7 +105,11 @@ export function SignupForm() { }, [emailSent]); const handleGoogleSignIn = () => { - trackEvent("auth_started", { method: "google", is_signup: true }); + trackEvent("auth_started", { + method: "google", + is_signup: true, + auth_surface: "signup", + }); signIn("google", { ...(next && next.length > 0 ? { callbackUrl: next } : {}), }); @@ -268,6 +272,7 @@ export function SignupForm() { trackEvent("auth_started", { method: "email", is_signup: true, + auth_surface: "signup", }); const normalizedEmail = email.trim().toLowerCase(); signIn("email", { @@ -284,6 +289,9 @@ export function SignupForm() { setEmailSent(true); setLastEmailSentTime(Date.now()); trackEvent("auth_email_sent", { + method: "email", + is_signup: true, + auth_surface: "signup", email_domain: normalizedEmail.split("@")[1], }); const params = new URLSearchParams({ diff --git a/apps/web/app/api/webhooks/stripe/route.ts b/apps/web/app/api/webhooks/stripe/route.ts index adb2a8cc21..f5d84e3dd6 100644 --- a/apps/web/app/api/webhooks/stripe/route.ts +++ b/apps/web/app/api/webhooks/stripe/route.ts @@ -334,7 +334,12 @@ export const POST = async (req: Request) => { price_id: subscription.items.data[0]?.price.id, quantity: inviteQuota, is_onboarding: session.metadata?.isOnBoarding === "true", - platform: session.metadata?.platform === "web", + platform: + session.metadata?.platform === "desktop" + ? "desktop" + : session.metadata?.platform === "web" + ? "web" + : "unknown", is_first_purchase: isFirstPurchase, is_guest_checkout: isGuestCheckout, }, diff --git a/apps/web/app/s/[videoId]/_components/AuthOverlay.tsx b/apps/web/app/s/[videoId]/_components/AuthOverlay.tsx index eb7a580895..75a3c83408 100644 --- a/apps/web/app/s/[videoId]/_components/AuthOverlay.tsx +++ b/apps/web/app/s/[videoId]/_components/AuthOverlay.tsx @@ -135,9 +135,15 @@ const StepOne = ({ setLastResendTime: (time: number | null) => void; emailId: string; }) => { - const videoId = useParams().videoId; + const rawVideoId = useParams().videoId; + const videoId = Array.isArray(rawVideoId) ? rawVideoId[0] : rawVideoId; const handleGoogleSignIn = () => { - trackEvent("auth_started", { method: "google", is_signup: true }); + trackEvent("auth_started", { + method: "google", + is_signup: false, + auth_surface: "share_overlay", + video_id: videoId, + }); setLoading(true); signIn("google", { redirect: false, @@ -153,8 +159,15 @@ const StepOne = ({ if (!email) return; setLoading(true); + const normalizedEmail = email.trim().toLowerCase(); + trackEvent("auth_started", { + method: "email", + is_signup: false, + auth_surface: "share_overlay", + video_id: videoId, + }); signIn("email", { - email: email.trim().toLowerCase(), + email: normalizedEmail, redirect: false, }) .then((res) => { @@ -163,6 +176,13 @@ const StepOne = ({ setEmailSent(true); setStep(2); setLastResendTime(Date.now()); + trackEvent("auth_email_sent", { + method: "email", + is_signup: false, + auth_surface: "share_overlay", + email_domain: normalizedEmail.split("@").at(1), + video_id: videoId, + }); toast.success("Email sent - check your inbox!"); } else { toast.error("Error sending email - try again?"); diff --git a/crates/audio/src/latency.rs b/crates/audio/src/latency.rs index ea0874f22a..364c5d3faf 100644 --- a/crates/audio/src/latency.rs +++ b/crates/audio/src/latency.rs @@ -945,7 +945,7 @@ mod windows { } #[cfg(test)] -#[allow(clippy::unchecked_duration_subtraction)] +#[allow(clippy::unchecked_time_subtraction)] mod tests { use super::*; use std::time::Instant; diff --git a/crates/cap-test/src/suites/performance.rs b/crates/cap-test/src/suites/performance.rs index c109b920fe..b8ea76e94f 100644 --- a/crates/cap-test/src/suites/performance.rs +++ b/crates/cap-test/src/suites/performance.rs @@ -482,8 +482,15 @@ async fn benchmark_playback( let zoom_focus_interpolator = ZoomFocusInterpolator::new( &segment_media.cursor, cursor_smoothing, + context.project.cursor.click_spring_config(), context.project.screen_movement_spring, duration, + context + .project + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), ); let uniforms = ProjectUniforms::new( @@ -671,8 +678,15 @@ async fn render_single_frame(context: &FixtureContext) -> Result<()> { let zoom_focus_interpolator = ZoomFocusInterpolator::new( &segment_media.cursor, cursor_smoothing, + context.project.cursor.click_spring_config(), context.project.screen_movement_spring, duration, + context + .project + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), ); let uniforms = ProjectUniforms::new( &context.render_constants, diff --git a/crates/editor/examples/playback-pipeline-benchmark.rs b/crates/editor/examples/playback-pipeline-benchmark.rs index e59172ee9f..7acedd2b1a 100644 --- a/crates/editor/examples/playback-pipeline-benchmark.rs +++ b/crates/editor/examples/playback-pipeline-benchmark.rs @@ -356,8 +356,14 @@ async fn run_full_pipeline_benchmark( let zoom_focus_interpolator = ZoomFocusInterpolator::new( &segment_media.cursor, cursor_smoothing, + project.cursor.click_spring_config(), project.screen_movement_spring, duration, + project + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), ); let uniforms = ProjectUniforms::new( @@ -518,8 +524,14 @@ async fn run_scrubbing_benchmark( let zoom_focus_interpolator = ZoomFocusInterpolator::new( &segment_media.cursor, cursor_smoothing, + project.cursor.click_spring_config(), project.screen_movement_spring, duration, + project + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), ); let uniforms = ProjectUniforms::new( diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 3ceca87143..0796e5d9ca 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -83,7 +83,7 @@ impl Renderer { let total_frames = (30_f64 * max_duration).ceil() as u32; - let (tx, rx) = mpsc::channel(8); + let (tx, rx) = mpsc::channel(64); let this = Self { rx, @@ -150,6 +150,7 @@ impl Renderer { continue; }; + let mut drained_count = 0u32; let queue_drain_start = Instant::now(); while let Ok(msg) = self.rx.try_recv() { match msg { @@ -166,6 +167,7 @@ impl Renderer { finished, cursor, }; + drained_count += 1; } RendererMessage::Stop { finished } => { let _ = current.finished.send(()); @@ -178,6 +180,10 @@ impl Renderer { } } + if drained_count > 0 { + let _ = frame_renderer.flush_pipeline_nv12().await; + } + match frame_renderer .render_immediate_nv12( current.segment_frames, @@ -208,7 +214,6 @@ impl RendererHandle { cursor: Arc, ) { let (finished_tx, _finished_rx) = oneshot::channel(); - let _ = self.tx.try_send(RendererMessage::RenderFrame { segment_frames, uniforms, @@ -217,6 +222,22 @@ impl RendererHandle { }); } + pub fn render_frame_blocking( + &self, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: Arc, + ) { + let (finished_tx, _finished_rx) = oneshot::channel(); + let msg = RendererMessage::RenderFrame { + segment_frames, + uniforms, + finished: finished_tx, + cursor, + }; + let _ = self.tx.blocking_send(msg); + } + pub async fn stop(&self) { let (tx, rx) = oneshot::channel(); if self diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index bf42055aa6..cc1f823dbe 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -530,8 +530,10 @@ impl EditorInstance { let zoom_focus_interpolator = ZoomFocusInterpolator::new_arc( segment_medias.cursor.clone(), cursor_smoothing, + project.cursor.click_spring_config(), project.screen_movement_spring, total_duration, + project.timeline.as_ref().map(|t| t.zoom_segments.as_slice()).unwrap_or(&[]), ); let uniforms = ProjectUniforms::new( diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index f246a2e740..7e9e09f668 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -288,6 +288,7 @@ impl Playback { let prefetch_time = frame_num as f64 / fps_f64; if prefetch_time >= prefetch_duration { + next_prefetch_frame = next_prefetch_frame.saturating_add(1); break; } @@ -465,8 +466,13 @@ impl Playback { let mut last_stats_time = Instant::now(); let stats_interval = Duration::from_secs(2); - let warmup_target_frames = 10usize; - let warmup_after_first_timeout = Duration::from_millis(500); + let is_mid_start = self.start_frame_number > 0; + let warmup_target_frames = if is_mid_start { 30 } else { 10 }; + let warmup_after_first_timeout = if is_mid_start { + Duration::from_millis(800) + } else { + Duration::from_millis(500) + }; let warmup_no_frames_timeout = Duration::from_secs(5); let warmup_start = Instant::now(); let mut first_frame_time: Option = None; @@ -492,7 +498,7 @@ impl Playback { return; } - match prefetch_rx.recv_timeout(Duration::from_millis(100)) { + match prefetch_rx.recv_timeout(Duration::from_millis(50)) { Ok(prefetched) => { if prefetched.frame_number >= frame_number { prefetch_buffer.push_back(prefetched); @@ -500,6 +506,16 @@ impl Playback { first_frame_time = Some(Instant::now()); } } + while prefetch_buffer.len() < warmup_target_frames { + match prefetch_rx.try_recv() { + Ok(p) => { + if p.frame_number >= frame_number { + prefetch_buffer.push_back(p); + } + } + Err(_) => break, + } + } } Err(std_mpsc::RecvTimeoutError::Timeout) => {} Err(std_mpsc::RecvTimeoutError::Disconnected) => break, @@ -555,15 +571,7 @@ impl Playback { continue; } - while prefetch_buffer - .front() - .is_some_and(|f| f.frame_number < frame_number) - { - prefetch_buffer.pop_front(); - } - if prefetch_buffer.len() >= PREFETCH_BUFFER_SIZE { - prefetch_buffer.retain(|p| p.frame_number >= frame_number); - } + prefetch_buffer.retain(|p| p.frame_number >= frame_number); let drain_budget = 16usize; let mut drained = 0usize; while prefetch_buffer.len() < PREFETCH_BUFFER_SIZE && drained < drain_budget { @@ -615,8 +623,19 @@ impl Playback { let _ = frame_request_tx.send(frame_number); let wait_ms = if total_frames_rendered < 15 { 100 } else { 50 }; - match prefetch_rx.recv_timeout(Duration::from_millis(wait_ms)) { - Ok(prefetched) => { + let prefetched_opt = + match prefetch_rx.recv_timeout(Duration::from_millis(wait_ms)) { + Ok(p) => Some(p), + Err(std_mpsc::RecvTimeoutError::Timeout) => { + prefetch_rx.recv_timeout(Duration::from_millis(400)).ok() + } + Err(std_mpsc::RecvTimeoutError::Disconnected) => { + break 'playback; + } + }; + + match prefetched_opt { + Some(prefetched) => { if prefetched.frame_number == frame_number { Some(( Arc::new(prefetched.segment_frames), @@ -636,16 +655,51 @@ impl Playback { continue; } } - Err(_) => { + None => { frame_number = frame_number.saturating_add(1); total_frames_skipped += 1; + let _ = frame_request_tx.send(frame_number); + let _ = playback_position_tx.send(frame_number); + if has_audio + && audio_playhead_tx + .send(frame_number as f64 / fps_f64) + .is_err() + { + break 'playback; + } continue; } } } else { - let _ = frame_request_tx.send(frame_number); + let min_buffered = prefetch_buffer.iter().map(|p| p.frame_number).min(); + if let Some(next_available_frame) = min_buffered + && next_available_frame > frame_number + { + let jumped = next_available_frame - frame_number; + frame_number = next_available_frame; + total_frames_skipped += jumped as u64; + let _ = frame_request_tx.send(frame_number); + let _ = playback_position_tx.send(frame_number); + if has_audio + && audio_playhead_tx + .send(frame_number as f64 / fps_f64) + .is_err() + { + break 'playback; + } + continue; + } frame_number = frame_number.saturating_add(1); total_frames_skipped += 1; + let _ = frame_request_tx.send(frame_number); + let _ = playback_position_tx.send(frame_number); + if has_audio + && audio_playhead_tx + .send(frame_number as f64 / fps_f64) + .is_err() + { + break 'playback; + } continue; } }; @@ -675,8 +729,14 @@ impl Playback { let zoom_focus_interpolator = ZoomFocusInterpolator::new_arc( segment_media.cursor.clone(), cursor_smoothing, + cached_project.cursor.click_spring_config(), cached_project.screen_movement_spring, duration, + cached_project + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), ); let uniforms = ProjectUniforms::new( @@ -691,7 +751,7 @@ impl Playback { &zoom_focus_interpolator, ); - self.renderer.render_frame( + self.renderer.render_frame_blocking( Arc::unwrap_or_clone(segment_frames), uniforms, segment_media.cursor.clone(), diff --git a/crates/enc-avfoundation/src/mp4.rs b/crates/enc-avfoundation/src/mp4.rs index 7a5dbbf516..a2d747dfd0 100644 --- a/crates/enc-avfoundation/src/mp4.rs +++ b/crates/enc-avfoundation/src/mp4.rs @@ -698,12 +698,16 @@ impl MP4Encoder { dts: cm::Time::invalid(), }; - let timed_frame = match pending.raw_frame.copy_with_new_timing(&[timing]) { + let timed_frame = match pending + .raw_frame + .copy_with_new_timing(&[timing]) + .or_else(|_| rebuild_video_sample_buf(&pending.raw_frame, timing)) + { Ok(f) => f, Err(e) => { warn!( ?e, - "Failed to copy pending sample buffer with new timing, skipping frame" + "Failed to rebuild pending sample buffer with new timing, skipping frame" ); return Ok(()); } @@ -1139,7 +1143,11 @@ mod tests { path } - fn create_pixel_buffer_pool(width: usize, height: usize) -> arc::R { + fn create_pixel_buffer_pool_with_format( + width: usize, + height: usize, + pixel_format: cidre::cv::PixelFormat, + ) -> arc::R { use cidre::{cf, cv}; let min_count_num = cf::Number::from_usize(8); @@ -1158,7 +1166,7 @@ mod tests { cv::pixel_buffer::keys::height().as_ref(), ]; let pixel_buf_attr_values: [&cf::Type; 3] = [ - cv::PixelFormat::_420V.to_cf_number().as_ref(), + pixel_format.to_cf_number().as_ref(), width_num.as_ref(), height_num.as_ref(), ]; @@ -1168,6 +1176,10 @@ mod tests { cv::PixelBufPool::new(Some(pool_attrs.as_ref()), Some(pixel_buf_attrs.as_ref())).unwrap() } + fn create_pixel_buffer_pool(width: usize, height: usize) -> arc::R { + create_pixel_buffer_pool_with_format(width, height, cidre::cv::PixelFormat::_420V) + } + fn create_test_video_frame( pool: &cidre::cv::PixelBufPool, pts_us: i64, @@ -1202,6 +1214,62 @@ mod tests { frame } + fn run_camera_encoder_scenario( + name: &str, + video_info: VideoInfo, + pool_format: cidre::cv::PixelFormat, + frame_timings: &[(i64, i64)], + ) -> Result { + let output = test_output_path(name); + let mut encoder = + MP4Encoder::init(output.clone(), video_info, None, None).map_err(|e| e.to_string())?; + let pool = create_pixel_buffer_pool_with_format( + video_info.width as usize, + video_info.height as usize, + pool_format, + ); + + let mut appended = 0usize; + + for &(pts_us, duration_us) in frame_timings { + let frame = create_test_video_frame(&pool, pts_us, duration_us); + let timestamp = Duration::from_micros(pts_us.max(0) as u64); + + match encoder.queue_video_frame(frame, timestamp) { + Ok(()) => appended += 1, + Err(QueueFrameError::NotReadyForMore) => {} + Err(QueueFrameError::WriterFailed(err)) => { + let _ = std::fs::remove_file(&output); + return Err(format!( + "WriterFailed after {appended} frames at pts={pts_us}us: {err}" + )); + } + Err(QueueFrameError::Failed) => { + let _ = std::fs::remove_file(&output); + return Err(format!("Failed after {appended} frames at pts={pts_us}us")); + } + Err(err) => { + let _ = std::fs::remove_file(&output); + return Err(format!( + "Error after {appended} frames at pts={pts_us}us: {err}" + )); + } + } + } + + let end_pts_us = frame_timings + .last() + .map(|(pts, dur)| pts + dur) + .unwrap_or(1_000_000); + encoder + .finish(Some(Duration::from_micros(end_pts_us.max(0) as u64))) + .map_err(|e| e.to_string())?; + + let _ = std::fs::remove_file(&output); + + Ok(appended) + } + fn wireless_audio_config() -> cap_media_info::AudioInfo { cap_media_info::AudioInfo { sample_rate: 48000, @@ -2317,6 +2385,130 @@ mod tests { ); } + #[test] + fn raw_uyvy_camera_sample_bufs_do_not_fail_writer() { + let output = test_output_path("uyvy_camera_ok"); + + let (mut asset_writer, mut video_input) = + setup_raw_writer(&output, 1920, 1080, 60.0, true).unwrap(); + let pool = create_pixel_buffer_pool_with_format(1920, 1080, cidre::cv::PixelFormat::_2VUY); + + let timings: Vec<(i64, i64)> = (0..120i64).map(|i| (i * 16_666, 16_666)).collect(); + + let result = feed_frames_to_writer(&mut video_input, &asset_writer, &pool, &timings, 0); + + let finish = finish_raw_writer(&mut video_input, &mut asset_writer, 120 * 16_666); + + let _ = std::fs::remove_file(&output); + + assert!( + result.is_ok() && finish.is_ok(), + "Raw UYVY camera sample buffers should append successfully. result={result:?} finish={finish:?}" + ); + } + + #[test] + fn camera_writer_scenario_matrix_1080p_uyvy() { + let video_60 = VideoInfo::from_raw_ffmpeg(ffmpeg::format::Pixel::UYVY422, 1920, 1080, 60); + let frame_60 = 16_666i64; + let frame_30 = 33_333i64; + + let clean_60: Vec<(i64, i64)> = (0..240i64).map(|i| (i * frame_60, frame_60)).collect(); + + let two_backward_anomalies_60: Vec<(i64, i64)> = (0..240i64) + .scan(0i64, |pts, i| { + let current = *pts; + let step = match i { + 4 => frame_60 - 49_000, + 5 => frame_60 - 14_000, + _ => frame_60, + }; + *pts = (*pts + step).max(0); + Some((current, frame_60)) + }) + .collect(); + + let bursty_60: Vec<(i64, i64)> = (0..240i64) + .scan(0i64, |pts, i| { + let current = *pts; + let step = match i % 12 { + 0 | 1 | 2 => 8_333, + 3 => 33_333, + 4 | 5 | 6 => 12_000, + _ => frame_60, + }; + *pts += step; + Some((current, frame_60)) + }) + .collect(); + + let duplicates_60: Vec<(i64, i64)> = (0..240i64) + .map(|i| { + let pts = if i == 60 || i == 61 { + 60 * frame_60 + } else { + i * frame_60 + }; + (pts, frame_60) + }) + .collect(); + + let video_30 = VideoInfo::from_raw_ffmpeg(ffmpeg::format::Pixel::UYVY422, 1920, 1080, 30); + let camera_overlay_like_30: Vec<(i64, i64)> = (0..180i64) + .scan(0i64, |pts, i| { + let current = *pts; + let step = match i % 10 { + 0 => 16_666, + 1 => 50_000, + 5 => 20_000, + _ => frame_30, + }; + *pts += step; + Some((current, frame_30)) + }) + .collect(); + + let scenarios = vec![ + ("cam_clean_1080p60_uyvy", video_60, clean_60, true), + ( + "cam_two_backward_1080p60_uyvy", + video_60, + two_backward_anomalies_60, + true, + ), + ("cam_bursty_1080p60_uyvy", video_60, bursty_60, true), + ( + "cam_duplicate_pts_1080p60_uyvy", + video_60, + duplicates_60, + true, + ), + ( + "cam_overlay_like_1080p30_uyvy", + video_30, + camera_overlay_like_30, + true, + ), + ]; + + for (name, video_info, timings, should_succeed) in scenarios { + let result = run_camera_encoder_scenario( + name, + video_info, + cidre::cv::PixelFormat::_2VUY, + &timings, + ); + if should_succeed { + assert!(result.is_ok(), "{name} should succeed, got {result:?}"); + } else { + assert!( + result.is_err(), + "{name} should fail to reproduce writer rejection" + ); + } + } + } + fn generate_drift_timings( frame_count: usize, base_interval_us: i64, @@ -3302,6 +3494,25 @@ impl SampleBufExt for cm::SampleBuf { } } +fn rebuild_video_sample_buf( + frame: &cm::SampleBuf, + timing: SampleTimingInfo, +) -> os::Result> { + if let Some(image_buf) = frame.image_buf() { + let format_desc = cm::VideoFormatDesc::with_image_buf(image_buf)?; + cm::SampleBuf::with_image_buf( + image_buf, + true, + None, + std::ptr::null(), + &format_desc, + &timing, + ) + } else { + frame.copy_with_new_timing(&[timing]) + } +} + fn writer_status_name(writer: &av::AssetWriter) -> &'static str { use av::asset::writer::Status; match writer.status() { diff --git a/crates/export/examples/export_startup_time.rs b/crates/export/examples/export_startup_time.rs new file mode 100644 index 0000000000..a7c6b044ac --- /dev/null +++ b/crates/export/examples/export_startup_time.rs @@ -0,0 +1,58 @@ +use cap_export::{ + ExporterBase, + mp4::{ExportCompression, Mp4ExportSettings}, +}; +use cap_project::XY; +use std::{env, path::PathBuf, time::Instant}; + +#[tokio::main] +async fn main() -> Result<(), String> { + let project_path = env::args() + .nth(1) + .map(PathBuf::from) + .ok_or_else(|| "usage: export_startup_time ".to_string())?; + + let settings = Mp4ExportSettings { + fps: 60, + resolution_base: XY { x: 3840, y: 2160 }, + compression: ExportCompression::Maximum, + custom_bpp: None, + force_ffmpeg_decoder: false, + }; + + let temp_out = tempfile::Builder::new() + .suffix(".mp4") + .tempfile() + .map_err(|e| e.to_string())?; + let temp_path = temp_out.path().to_path_buf(); + + let build_start = Instant::now(); + let base = ExporterBase::builder(project_path.clone()) + .with_output_path(temp_path) + .build() + .await + .map_err(|e| e.to_string())?; + let build_ms = build_start.elapsed().as_millis() as u64; + + let total_frames = base.total_frames(settings.fps); + + let pipeline_start = Instant::now(); + let bench = settings + .benchmark_first_frame_with_breakdown(base) + .await + .map_err(|e| e.to_string())?; + let benchmark_total_ms = pipeline_start.elapsed().as_millis() as u64; + + let line = serde_json::json!({ + "project": project_path.to_string_lossy(), + "build_ms": build_ms, + "ms_to_first_frame_queued_since_export_pipeline_start": bench.ms_to_first_frame_queued_since_export_pipeline_start, + "nv12_render_startup_breakdown_ms": bench.nv12_render_startup_breakdown_ms, + "benchmark_wall_ms_including_encoder_finish": benchmark_total_ms, + "total_frames_at_60fps": total_frames, + "note": "ms_to_first_frame_queued is from NV12 pipeline start until first queue_video_frame_reusable succeeds (editor export path)", + }); + println!("{line}"); + + Ok(()) +} diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 4cd014069e..7ec1efc98f 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -8,7 +8,14 @@ use futures::FutureExt; use image::ImageBuffer; use serde::Deserialize; use specta::Type; -use std::{path::PathBuf, sync::Arc, time::Duration}; +use std::{ + path::PathBuf, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; use tracing::{info, trace, warn}; #[derive(Deserialize, Type, Clone, Copy, Debug)] @@ -30,6 +37,20 @@ impl ExportCompression { } } +#[derive(Clone, Default)] +struct ExportNv12Mode { + stop_after_frames_sent: Option, + record_first_queued_ms_since_pipeline: Option>, + nv12_render_startup_breakdown_ms: + Option>>>, +} + +#[derive(Debug, serde::Serialize)] +pub struct FirstFrameQueuedBenchmark { + pub ms_to_first_frame_queued_since_export_pipeline_start: u64, + pub nv12_render_startup_breakdown_ms: Option, +} + #[derive(Deserialize, Type, Clone, Copy, Debug)] pub struct Mp4ExportSettings { pub fps: u32, @@ -69,7 +90,61 @@ impl Mp4ExportSettings { height = output_size.1, "Exporting with NV12 pipeline (GPU when possible, CPU fallback otherwise)" ); - self.export_nv12(base, output_size, fps, on_progress).await + self.export_nv12( + base, + output_size, + fps, + on_progress, + ExportNv12Mode::default(), + ) + .await + } + + pub async fn benchmark_first_frame_with_breakdown( + self, + base: ExporterBase, + ) -> Result { + let fps = self.fps; + let output_size = ProjectUniforms::get_output_size( + &base.render_constants.options, + &base.project_config, + self.resolution_base, + ); + let first_ms = Arc::new(AtomicU64::new(u64::MAX)); + let first_ms_enc = Arc::clone(&first_ms); + let breakdown = Arc::new(Mutex::new(None)); + let breakdown_enc = Arc::clone(&breakdown); + self.export_nv12( + base, + output_size, + fps, + |_| true, + ExportNv12Mode { + stop_after_frames_sent: Some(1), + record_first_queued_ms_since_pipeline: Some(first_ms_enc), + nv12_render_startup_breakdown_ms: Some(breakdown_enc), + }, + ) + .await?; + let v = first_ms.load(Ordering::Relaxed); + if v == u64::MAX { + return Err("first frame was not queued to the encoder".to_string()); + } + let nv12_render_startup_breakdown_ms = breakdown.lock().ok().and_then(|mut g| g.take()); + Ok(FirstFrameQueuedBenchmark { + ms_to_first_frame_queued_since_export_pipeline_start: v, + nv12_render_startup_breakdown_ms, + }) + } + + pub async fn benchmark_ms_to_first_frame_queued( + self, + base: ExporterBase, + ) -> Result { + Ok(self + .benchmark_first_frame_with_breakdown(base) + .await? + .ms_to_first_frame_queued_since_export_pipeline_start) } async fn export_nv12( @@ -78,7 +153,9 @@ impl Mp4ExportSettings { output_size: (u32, u32), fps: u32, mut on_progress: impl FnMut(u32) -> bool + Send + 'static, + mode: ExportNv12Mode, ) -> Result { + let pipeline_start = std::time::Instant::now(); let output_path = base.output_path.clone(); let meta = &base.studio_meta; @@ -98,6 +175,11 @@ impl Mp4ExportSettings { .map(|_| AudioRenderer::new(audio_segments.clone())); let has_audio = audio_renderer.is_some(); + let stop_after_frames_sent = mode.stop_after_frames_sent; + let record_first_queued_ms = mode.record_first_queued_ms_since_pipeline; + let nv12_render_startup_breakdown_ms = mode.nv12_render_startup_breakdown_ms; + + let pipeline_start_for_encoder = pipeline_start; let encoder_thread = tokio::task::spawn_blocking(move || { trace!("Creating MP4File encoder (NV12 path)"); @@ -146,6 +228,13 @@ impl Mp4ExportSettings { encoder.queue_audio_frame(audio); } encoded_frames += 1; + if encoded_frames == 1 + && let Some(atom) = record_first_queued_ms.as_ref() + { + let ms = pipeline_start_for_encoder.elapsed().as_millis() as u64; + let _ = + atom.compare_exchange(u64::MAX, ms, Ordering::Relaxed, Ordering::Relaxed); + } } let encode_elapsed = encode_start.elapsed(); @@ -329,6 +418,8 @@ impl Mp4ExportSettings { fps, self.resolution_base, &base.recordings, + stop_after_frames_sent, + nv12_render_startup_breakdown_ms, ) .then(|v| async { v.map_err(|e| e.to_string()) }); diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 612bb691fa..02f14c823a 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -438,9 +438,9 @@ pub struct ClickSpringConfig { impl Default for ClickSpringConfig { fn default() -> Self { Self { - tension: 700.0, + tension: 530.0, mass: 1.0, - friction: 30.0, + friction: 40.0, } } } @@ -520,7 +520,7 @@ impl Default for CursorConfiguration { mass: 3.0, friction: 70.0, raw: false, - motion_blur: 0.5, + motion_blur: 1.0, use_svg: true, rotation_amount: Self::default_rotation_amount(), base_rotation: 0.0, @@ -543,7 +543,7 @@ impl CursorConfiguration { } fn default_rotation_amount() -> f32 { - 0.5 + 0.15 } pub fn cursor_type(&self) -> &CursorType { @@ -1118,7 +1118,7 @@ fn camera_config_needs_migration(value: &Value) -> bool { impl ProjectConfiguration { fn default_screen_motion_blur() -> f32 { - 0.5 + 1.0 } pub fn validate(&self) -> Result<(), AnnotationValidationError> { diff --git a/crates/recording/examples/camera-writer-repro.rs b/crates/recording/examples/camera-writer-repro.rs new file mode 100644 index 0000000000..20c96497d0 --- /dev/null +++ b/crates/recording/examples/camera-writer-repro.rs @@ -0,0 +1,390 @@ +#[cfg(not(target_os = "macos"))] +fn main() { + eprintln!("camera-writer-repro is only available on macOS"); +} + +#[cfg(target_os = "macos")] +fn main() -> anyhow::Result<()> { + use cap_camera::{CameraInfo, CapturedFrame, Format}; + use cap_camera_ffmpeg::CapturedFrameExt; + use cap_enc_avfoundation::{MP4Encoder, QueueFrameError}; + use cap_media_info::VideoInfo; + use cidre::{arc, cm}; + use std::{ + cmp::Ordering, + env, + path::PathBuf, + sync::mpsc::sync_channel, + time::{Duration, Instant}, + }; + + #[derive(Clone)] + struct ObservedFrame { + sample_buf: arc::R, + timestamp: Duration, + subtype: String, + width: u32, + height: u32, + ffmpeg_video_info: Option, + ffmpeg_error: Option, + } + + #[derive(Clone)] + struct ProbeTarget { + camera: CameraInfo, + format: Format, + } + + #[derive(Clone)] + struct ProbeSummary { + camera_name: String, + width: u32, + height: u32, + fps: f32, + received: usize, + queue_failures: Vec, + ffmpeg_failures: Vec, + elapsed_ms: u128, + } + + unsafe impl Send for ObservedFrame {} + unsafe impl Sync for ObservedFrame {} + + fn bool_flag(args: &[String], flag: &str) -> bool { + args.iter().any(|arg| arg == flag) + } + + fn value_flag(args: &[String], flag: &str) -> Option { + args.windows(2) + .find(|window| window[0] == flag) + .map(|window| window[1].clone()) + } + + fn select_camera( + cameras: &[CameraInfo], + preferred: Option<&str>, + ) -> anyhow::Result { + if let Some(camera) = preferred.and_then(|preferred_name| { + cameras + .iter() + .find(|camera| camera.display_name().contains(preferred_name)) + }) { + return Ok(camera.clone()); + } + + if let Some(camera) = cameras + .iter() + .find(|camera| camera.display_name() == "MacBook Pro Camera") + { + return Ok(camera.clone()); + } + + if let Some(camera) = cameras + .iter() + .find(|camera| !camera.display_name().contains("Desk View")) + { + return Ok(camera.clone()); + } + + cameras + .first() + .cloned() + .ok_or_else(|| anyhow::anyhow!("No cameras available")) + } + + fn sorted_formats(camera: &CameraInfo) -> anyhow::Result> { + let mut formats = camera + .formats() + .ok_or_else(|| anyhow::anyhow!("No formats reported for {}", camera.display_name()))?; + + formats.sort_by(|a, b| { + let target_aspect_ratio = 16.0 / 9.0; + let aspect_ratio_a = a.width() as f32 / a.height() as f32; + let aspect_ratio_b = b.width() as f32 / b.height() as f32; + let aspect_cmp_a = (aspect_ratio_a - target_aspect_ratio).abs(); + let aspect_cmp_b = (aspect_ratio_b - target_aspect_ratio).abs(); + let aspect_cmp = aspect_cmp_a.partial_cmp(&aspect_cmp_b); + let resolution_cmp = (a.width() * a.height()).cmp(&(b.width() * b.height())); + let fr_cmp = a.frame_rate().partial_cmp(&b.frame_rate()); + + aspect_cmp + .unwrap_or(Ordering::Equal) + .then(resolution_cmp.reverse()) + .then(fr_cmp.unwrap_or(Ordering::Equal).reverse()) + }); + + Ok(formats) + } + + fn choose_default_format(camera: &CameraInfo) -> anyhow::Result { + let formats = sorted_formats(camera)?; + if let Some(format) = formats.iter().find(|format| { + format.frame_rate() >= 30.0 && format.width() < 2000 && format.height() < 2000 + }) { + return Ok(format.clone()); + } + + formats + .first() + .cloned() + .ok_or_else(|| anyhow::anyhow!("No usable formats for {}", camera.display_name())) + } + + fn collect_probe_targets( + all_cameras: bool, + format_limit: usize, + preferred_camera: Option<&str>, + ) -> anyhow::Result> { + let cameras = cap_camera::list_cameras().collect::>(); + let selected_cameras = if all_cameras { + cameras + } else { + vec![select_camera(&cameras, preferred_camera)?] + }; + + let mut targets = Vec::new(); + + for camera in selected_cameras { + let formats = sorted_formats(&camera)?; + + if format_limit <= 1 { + targets.push(ProbeTarget { + camera: camera.clone(), + format: choose_default_format(&camera)?, + }); + continue; + } + + for format in formats.into_iter().take(format_limit) { + targets.push(ProbeTarget { + camera: camera.clone(), + format, + }); + } + } + + Ok(targets) + } + + fn observe_frame(frame: &CapturedFrame, fps: u32) -> ObservedFrame { + let sample_buf = frame.native().sample_buf().clone(); + let (subtype, width, height) = match sample_buf.image_buf() { + Some(image_buf) => { + let width = image_buf.width() as u32; + let height = image_buf.height() as u32; + let subtype = sample_buf + .format_desc() + .map(|desc| { + let mut bytes = desc.media_sub_type().to_be_bytes(); + cidre::four_cc_to_str(&mut bytes).to_string() + }) + .unwrap_or_else(|| "unknown".to_string()); + (subtype, width, height) + } + None => ("no-image-buf".to_string(), 0, 0), + }; + + let (ffmpeg_video_info, ffmpeg_error) = match frame.as_ffmpeg() { + Ok(ff_frame) => ( + Some(VideoInfo::from_raw_ffmpeg( + ff_frame.format(), + ff_frame.width(), + ff_frame.height(), + fps, + )), + None, + ), + Err(error) => (None, Some(error.to_string())), + }; + + ObservedFrame { + sample_buf, + timestamp: frame.timestamp, + subtype, + width, + height, + ffmpeg_video_info, + ffmpeg_error, + } + } + + fn run_probe( + target: ProbeTarget, + frame_limit: usize, + timeout_secs: u64, + ) -> anyhow::Result { + let fps = target.format.frame_rate().round().max(1.0) as u32; + let output_path = PathBuf::from(format!( + "/tmp/cap-camera-writer-repro-{}-{}x{}-{}.mp4", + target.camera.display_name().replace(' ', "-"), + target.format.width(), + target.format.height(), + fps + )); + let _ = std::fs::remove_file(&output_path); + + println!( + "Probe camera='{}' format={}x{} @ {:.2}fps output={}", + target.camera.display_name(), + target.format.width(), + target.format.height(), + target.format.frame_rate(), + output_path.display() + ); + + let (tx, rx) = sync_channel::(frame_limit.max(1) * 2); + let started = Instant::now(); + let handle = target + .camera + .start_capturing(target.format.clone(), move |frame| { + let observed = observe_frame(&frame, fps); + let _ = tx.try_send(observed); + })?; + + let mut encoder: Option = None; + let mut received = 0usize; + let mut first_timestamp = None; + let mut queue_failures = Vec::new(); + let mut ffmpeg_failures = Vec::new(); + let deadline = Instant::now() + Duration::from_secs(timeout_secs); + + while received < frame_limit && Instant::now() < deadline { + let remaining = deadline.saturating_duration_since(Instant::now()); + let Ok(frame) = rx.recv_timeout(remaining.min(Duration::from_millis(500))) else { + continue; + }; + + let first = *first_timestamp.get_or_insert(frame.timestamp); + let rel_ms = frame.timestamp.saturating_sub(first).as_millis(); + let timing = frame.sample_buf.timing_info(0).ok(); + let pts_us = timing + .as_ref() + .map(|timing| timing.pts.value * 1_000_000 / timing.pts.scale.max(1) as i64); + let dur_us = timing.as_ref().map(|timing| { + timing.duration.value * 1_000_000 / timing.duration.scale.max(1) as i64 + }); + + println!( + "frame={received} rel_ms={rel_ms} subtype={} size={}x{} pts_us={pts_us:?} dur_us={dur_us:?}", + frame.subtype, frame.width, frame.height + ); + + if let Some(error) = &frame.ffmpeg_error { + ffmpeg_failures.push(error.clone()); + } + + if encoder.is_none() { + if let Some(video_info) = frame.ffmpeg_video_info { + encoder = Some( + MP4Encoder::init(output_path.clone(), video_info, None, None) + .map_err(|error| anyhow::anyhow!(error.to_string()))?, + ); + } else { + break; + } + } + + let result = encoder + .as_mut() + .expect("encoder initialized") + .queue_video_frame(frame.sample_buf.clone(), frame.timestamp); + + println!("queue_result={result:?}"); + + match result { + Ok(()) | Err(QueueFrameError::NotReadyForMore) => {} + Err(QueueFrameError::WriterFailed(err)) => { + queue_failures.push(format!("WriterFailed/{err}")); + break; + } + Err(QueueFrameError::Failed) => { + queue_failures.push("Failed".to_string()); + break; + } + Err(err) => { + queue_failures.push(err.to_string()); + break; + } + } + + received += 1; + } + + drop(handle); + + if let Some(mut encoder) = encoder { + let finish_ts = first_timestamp + .map(|first| first + Duration::from_secs(2)) + .unwrap_or(Duration::from_secs(1)); + let finish_result = encoder.finish(Some(finish_ts)); + println!("finish_result={finish_result:?}"); + } + + Ok(ProbeSummary { + camera_name: target.camera.display_name().to_string(), + width: target.format.width(), + height: target.format.height(), + fps: target.format.frame_rate(), + received, + queue_failures, + ffmpeg_failures, + elapsed_ms: started.elapsed().as_millis(), + }) + } + + let args = env::args().collect::>(); + let preferred_camera = + value_flag(&args, "--camera").or_else(|| env::var("CAP_CAMERA_NAME").ok()); + let frame_limit = value_flag(&args, "--frames") + .and_then(|value| value.parse::().ok()) + .unwrap_or(12); + let timeout_secs = value_flag(&args, "--timeout") + .and_then(|value| value.parse::().ok()) + .unwrap_or(8); + let format_limit = value_flag(&args, "--formats") + .and_then(|value| value.parse::().ok()) + .unwrap_or(1); + let all_cameras = bool_flag(&args, "--all-cameras"); + let list_only = bool_flag(&args, "--list"); + + let targets = collect_probe_targets(all_cameras, format_limit, preferred_camera.as_deref())?; + + if targets.is_empty() { + return Err(anyhow::anyhow!("No probe targets")); + } + + if list_only { + for target in &targets { + println!( + "camera='{}' format={}x{} @ {:.2}fps", + target.camera.display_name(), + target.format.width(), + target.format.height(), + target.format.frame_rate() + ); + } + return Ok(()); + } + + let mut summaries = Vec::new(); + + for target in targets { + let summary = run_probe(target, frame_limit, timeout_secs)?; + println!( + "summary camera='{}' format={}x{} @ {:.2}fps received={} queue_failures={:?} ffmpeg_failures={:?} elapsed_ms={}", + summary.camera_name, + summary.width, + summary.height, + summary.fps, + summary.received, + summary.queue_failures, + summary.ffmpeg_failures, + summary.elapsed_ms + ); + summaries.push(summary); + } + + println!("final_summaries={}", summaries.len()); + + Ok(()) +} diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs index 0fa5464a26..b79b3685f5 100644 --- a/crates/recording/src/output_pipeline/core.rs +++ b/crates/recording/src/output_pipeline/core.rs @@ -31,6 +31,44 @@ const LARGE_FORWARD_JUMP_SECS: f64 = 2.0; const HEALTH_CHANNEL_CAPACITY: usize = 32; +pub(crate) enum BlockingThreadFinish { + Clean, + Failed(anyhow::Error), + TimedOut(anyhow::Error), +} + +fn join_blocking_thread( + handle: std::thread::JoinHandle>, + label: &str, +) -> anyhow::Result<()> { + match handle.join() { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(anyhow!("{label} returned error: {error:#}")), + Err(panic_payload) => Err(anyhow!("{label} panicked during finish: {panic_payload:?}")), + } +} + +pub(crate) fn spawn_blocking_thread_timeout_cleanup( + handle: std::thread::JoinHandle>, + label: &str, +) -> std::sync::mpsc::Receiver> { + let (tx, rx) = std::sync::mpsc::channel(); + let label = label.to_string(); + std::thread::spawn(move || { + let result = join_blocking_thread(handle, &label); + match &result { + Ok(()) => warn!(worker = %label, "Timed-out blocking worker later exited cleanly"), + Err(error) => error!( + worker = %label, + error = %error, + "Timed-out blocking worker later exited with failure" + ), + } + let _ = tx.send(result); + }); + rx +} + #[derive(Debug, Clone)] pub enum PipelineHealthEvent { FrameDropRateHigh { rate_pct: f64 }, @@ -50,6 +88,57 @@ pub fn emit_health(tx: &HealthSender, event: PipelineHealthEvent) { let _ = tx.try_send(event); } +pub(crate) fn wait_for_blocking_thread_finish( + handle: std::thread::JoinHandle>, + timeout: Duration, + label: &str, +) -> BlockingThreadFinish { + let start = Instant::now(); + + loop { + if handle.is_finished() { + return match join_blocking_thread(handle, label) { + Ok(()) => BlockingThreadFinish::Clean, + Err(error) => BlockingThreadFinish::Failed(error), + }; + } + + if start.elapsed() > timeout { + drop(spawn_blocking_thread_timeout_cleanup(handle, label)); + return BlockingThreadFinish::TimedOut(anyhow!( + "{label} did not finish within {:?}", + timeout + )); + } + + std::thread::sleep(Duration::from_millis(50)); + } +} + +pub(crate) fn combine_finish_errors( + primary: anyhow::Error, + secondary: anyhow::Error, +) -> anyhow::Error { + anyhow!("{primary:#}; {secondary:#}") +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MuxStreamKind { + Video, + Audio, +} + +fn mux_send_error(kind: MuxStreamKind, frame_count: u64, error: anyhow::Error) -> anyhow::Error { + match kind { + MuxStreamKind::Video => { + anyhow!("Video muxer stopped accepting frames at frame {frame_count}: {error}") + } + MuxStreamKind::Audio => { + anyhow!("Audio muxer stopped accepting frames at frame {frame_count}: {error}") + } + } +} + struct AudioTimestampGenerator { sample_rate: u32, total_samples: u64, @@ -1039,16 +1128,7 @@ async fn finish_build( .then(async move |res| { let muxer_res = muxer.lock().await.finish(timestamps.instant().elapsed()); - let _ = done_tx.send(match (res, muxer_res) { - (Err(e), _) | (_, Err(e)) => Err(e), - (_, Ok(muxer_streams_res)) => { - if let Err(e) = muxer_streams_res { - warn!("Muxer streams had failure: {e:#}"); - } - - Ok(()) - } - }); + let _ = done_tx.send(resolve_pipeline_completion(res, muxer_res)); }), ); @@ -1057,6 +1137,17 @@ async fn finish_build( Ok(()) } +fn resolve_pipeline_completion( + task_result: anyhow::Result<()>, + muxer_result: anyhow::Result>, +) -> anyhow::Result<()> { + match (task_result, muxer_result) { + (Err(error), _) | (_, Err(error)) => Err(error), + (_, Ok(Ok(()))) => Ok(()), + (_, Ok(Err(error))) => Err(anyhow!("Muxer finish failed: {error:#}")), + } +} + async fn setup_video_source( video_config: TVideo::Config, setup_ctx: &mut SetupCtx, @@ -1186,8 +1277,7 @@ fn spawn_video_encoder, TVideo: V } if let Err(e) = muxer.lock().await.send_video_frame(frame, duration) { - warn!("Video encoder stopped accepting frames: {e}"); - break; + return Err(mux_send_error(MuxStreamKind::Video, frame_count, e)); } } @@ -1380,10 +1470,11 @@ impl PreparedAudioSources { .await .send_audio_frame(silence_frame, sample_based_before) { - warn!( - "Audio encoder stopped accepting frames (silence): {e}" - ); - break; + return Err(mux_send_error( + MuxStreamKind::Audio, + frame_count, + e, + )); } } } @@ -1410,8 +1501,7 @@ impl PreparedAudioSources { } if let Err(e) = muxer.lock().await.send_audio_frame(frame, timestamp) { - warn!("Audio encoder stopped accepting frames: {e}"); - break; + return Err(mux_send_error(MuxStreamKind::Audio, frame_count, e)); } } Ok::<(), anyhow::Error>(()) @@ -2296,4 +2386,387 @@ mod tests { assert!(tracker.total_forward_skew_secs > 2.0); } } + + mod finish_build { + use super::*; + + #[test] + fn treats_inner_muxer_finish_error_as_failure() { + let result = resolve_pipeline_completion( + Ok(()), + Ok(Err(anyhow!("fragmented audio trailer write failed"))), + ); + + let error = result.expect_err("inner muxer failure should fail the pipeline"); + assert!( + error + .to_string() + .contains("fragmented audio trailer write failed"), + "error should include the muxer failure reason" + ); + } + + #[test] + fn preserves_task_failure_over_muxer_finish_success() { + let result = + resolve_pipeline_completion(Err(anyhow!("capture-video failed")), Ok(Ok(()))); + + let error = result.expect_err("task failure should fail the pipeline"); + assert!( + error.to_string().contains("capture-video failed"), + "error should include the task failure reason" + ); + } + + #[test] + fn succeeds_only_when_tasks_and_muxer_finish_succeed() { + resolve_pipeline_completion(Ok(()), Ok(Ok(()))) + .expect("pipeline should succeed when all work succeeds"); + } + } + + mod pipeline_mux_send_failures { + use super::*; + + #[derive(Clone, Copy)] + struct TestVideoFrame { + timestamp: Timestamp, + } + + impl VideoFrame for TestVideoFrame { + fn timestamp(&self) -> Timestamp { + self.timestamp + } + } + + #[derive(Clone, Copy)] + struct FailingVideoMuxerConfig { + fail_after_frame: u64, + } + + struct FailingVideoMuxer { + fail_after_frame: u64, + sent_frames: u64, + } + + impl Muxer for FailingVideoMuxer { + type Config = FailingVideoMuxerConfig; + + fn setup( + config: Self::Config, + _output_path: PathBuf, + _video_config: Option, + _audio_config: Option, + _pause_flag: Arc, + _tasks: &mut TaskPool, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + Ok(Self { + fail_after_frame: config.fail_after_frame, + sent_frames: 0, + }) + } + } + + fn finish(&mut self, _timestamp: Duration) -> anyhow::Result> { + Ok(Ok(())) + } + } + + impl AudioMuxer for FailingVideoMuxer { + fn send_audio_frame( + &mut self, + _frame: AudioFrame, + _timestamp: Duration, + ) -> anyhow::Result<()> { + Ok(()) + } + } + + impl VideoMuxer for FailingVideoMuxer { + type VideoFrame = TestVideoFrame; + + fn send_video_frame( + &mut self, + _frame: Self::VideoFrame, + _timestamp: Duration, + ) -> anyhow::Result<()> { + self.sent_frames += 1; + if self.sent_frames >= self.fail_after_frame { + return Err(anyhow!("video mux send failed")); + } + Ok(()) + } + } + + #[derive(Clone, Copy)] + struct FailingAudioMuxerConfig { + fail_after_frame: u64, + } + + struct FailingAudioMuxer { + fail_after_frame: u64, + sent_frames: u64, + } + + impl Muxer for FailingAudioMuxer { + type Config = FailingAudioMuxerConfig; + + fn setup( + config: Self::Config, + _output_path: PathBuf, + _video_config: Option, + _audio_config: Option, + _pause_flag: Arc, + _tasks: &mut TaskPool, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + Ok(Self { + fail_after_frame: config.fail_after_frame, + sent_frames: 0, + }) + } + } + + fn finish(&mut self, _timestamp: Duration) -> anyhow::Result> { + Ok(Ok(())) + } + } + + impl AudioMuxer for FailingAudioMuxer { + fn send_audio_frame( + &mut self, + _frame: AudioFrame, + _timestamp: Duration, + ) -> anyhow::Result<()> { + self.sent_frames += 1; + if self.sent_frames >= self.fail_after_frame { + return Err(anyhow!("audio mux send failed")); + } + Ok(()) + } + } + + fn test_video_info() -> VideoInfo { + VideoInfo::from_raw(cap_media_info::RawVideoFormat::Bgra, 16, 16, 30) + } + + fn test_audio_info() -> AudioInfo { + AudioInfo::new_raw( + cap_media_info::Sample::F32(cap_media_info::Type::Packed), + 48_000, + 2, + ) + } + + #[tokio::test] + async fn pipeline_done_future_surfaces_video_mux_send_failure() { + let temp_dir = tempfile::tempdir().expect("temp dir should be created"); + let timestamps = Timestamps::now(); + let (video_tx, video_rx) = flume::bounded(4); + let pipeline = OutputPipeline::builder(temp_dir.path().join("video.mp4")) + .with_video::>(ChannelVideoSourceConfig::new( + test_video_info(), + video_rx, + )) + .with_timestamps(timestamps) + .build::(FailingVideoMuxerConfig { + fail_after_frame: 1, + }) + .await + .expect("pipeline should build"); + let done_fut = pipeline.done_fut(); + + video_tx + .send_async(TestVideoFrame { + timestamp: Timestamp::Instant(timestamps.instant() + Duration::from_millis(33)), + }) + .await + .expect("video frame should send"); + drop(video_tx); + + let done_error = done_fut + .await + .expect_err("done future should fail when mux-video rejects a frame"); + assert!( + done_error.to_string().contains("Task mux-video failed"), + "done future should surface the mux-video task failure" + ); + assert!( + done_error + .to_string() + .contains("Video muxer stopped accepting frames at frame 1"), + "done future should retain the send-failure context" + ); + + let stop_error = match pipeline.stop().await { + Ok(_) => panic!("stop should fail when mux-video rejects a frame"), + Err(error) => error, + }; + assert!( + stop_error + .to_string() + .contains("Video muxer stopped accepting frames at frame 1"), + "stop should propagate the mux-video send failure" + ); + } + + #[tokio::test] + async fn pipeline_done_future_surfaces_audio_mux_send_failure() { + let temp_dir = tempfile::tempdir().expect("temp dir should be created"); + let timestamps = Timestamps::now(); + let (mut audio_tx, audio_rx) = mpsc::channel(4); + let pipeline = OutputPipeline::builder(temp_dir.path().join("audio.ogg")) + .with_audio_source::(ChannelAudioSourceConfig::new( + test_audio_info(), + audio_rx, + )) + .with_timestamps(timestamps) + .build::(FailingAudioMuxerConfig { + fail_after_frame: 1, + }) + .await + .expect("pipeline should build"); + let done_fut = pipeline.done_fut(); + + audio_tx + .try_send(AudioFrame::new( + test_audio_info().empty_frame(960), + Timestamp::Instant(timestamps.instant() + Duration::from_millis(20)), + )) + .expect("audio frame should send"); + drop(audio_tx); + + let done_error = done_fut + .await + .expect_err("done future should fail when mux-audio rejects a frame"); + assert!( + done_error.to_string().contains("Task mux-audio failed"), + "done future should surface the mux-audio task failure" + ); + assert!( + done_error + .to_string() + .contains("Audio muxer stopped accepting frames at frame 1"), + "done future should retain the send-failure context" + ); + + let stop_error = match pipeline.stop().await { + Ok(_) => panic!("stop should fail when mux-audio rejects a frame"), + Err(error) => error, + }; + assert!( + stop_error + .to_string() + .contains("Audio muxer stopped accepting frames at frame 1"), + "stop should propagate the mux-audio send failure" + ); + } + } + + mod blocking_thread_finish { + use super::*; + + #[test] + fn returns_clean_when_thread_exits_successfully() { + let handle = std::thread::spawn(|| Ok(())); + + match wait_for_blocking_thread_finish(handle, Duration::from_millis(100), "test-worker") + { + BlockingThreadFinish::Clean => {} + BlockingThreadFinish::Failed(error) => { + panic!("expected clean shutdown, got failure: {error:#}"); + } + BlockingThreadFinish::TimedOut(error) => { + panic!("expected clean shutdown, got timeout: {error:#}"); + } + } + } + + #[test] + fn returns_failure_when_thread_returns_error() { + let handle = std::thread::spawn(|| Err(anyhow!("encoder worker failed"))); + + match wait_for_blocking_thread_finish(handle, Duration::from_millis(100), "test-worker") + { + BlockingThreadFinish::Failed(error) => { + assert!( + error.to_string().contains("encoder worker failed"), + "error should include the worker failure reason" + ); + } + BlockingThreadFinish::Clean => { + panic!("expected failure when worker returns an error"); + } + BlockingThreadFinish::TimedOut(error) => { + panic!("expected failure, got timeout: {error:#}"); + } + } + } + + #[test] + fn returns_timeout_when_thread_does_not_exit_in_time() { + let handle = std::thread::spawn(|| { + std::thread::sleep(Duration::from_millis(100)); + Ok(()) + }); + + match wait_for_blocking_thread_finish(handle, Duration::from_millis(5), "test-worker") { + BlockingThreadFinish::TimedOut(error) => { + assert!( + error + .to_string() + .contains("test-worker did not finish within"), + "error should include the timeout reason" + ); + } + BlockingThreadFinish::Clean => { + panic!("expected timeout when worker exceeds deadline"); + } + BlockingThreadFinish::Failed(error) => { + panic!("expected timeout, got failure: {error:#}"); + } + } + } + + #[test] + fn timeout_cleanup_reports_late_success() { + let handle = std::thread::spawn(|| { + std::thread::sleep(Duration::from_millis(25)); + Ok(()) + }); + + let cleanup_rx = spawn_blocking_thread_timeout_cleanup(handle, "test-worker"); + let result = cleanup_rx + .recv_timeout(Duration::from_millis(250)) + .expect("cleanup worker should report eventual completion"); + + result.expect("cleanup worker should observe a clean exit"); + } + + #[test] + fn timeout_cleanup_reports_late_failure() { + let handle = std::thread::spawn(|| { + std::thread::sleep(Duration::from_millis(25)); + Err(anyhow!("late worker failure")) + }); + + let cleanup_rx = spawn_blocking_thread_timeout_cleanup(handle, "test-worker"); + let error = cleanup_rx + .recv_timeout(Duration::from_millis(250)) + .expect("cleanup worker should report eventual completion") + .expect_err("cleanup worker should surface a late failure"); + + assert!( + error.to_string().contains("late worker failure"), + "error should include the late worker failure" + ); + } + } } diff --git a/crates/recording/src/output_pipeline/macos.rs b/crates/recording/src/output_pipeline/macos.rs index ed82a1a103..a020cd8c5e 100644 --- a/crates/recording/src/output_pipeline/macos.rs +++ b/crates/recording/src/output_pipeline/macos.rs @@ -1,5 +1,8 @@ use crate::{ - output_pipeline::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoFrame, VideoMuxer}, + output_pipeline::{ + AudioFrame, AudioMuxer, BlockingThreadFinish, Muxer, TaskPool, VideoFrame, VideoMuxer, + wait_for_blocking_thread_finish, + }, sources::screen_capture, }; use anyhow::anyhow; @@ -73,20 +76,9 @@ fn wait_for_worker( timeout: Duration, worker_name: &str, ) -> anyhow::Result<()> { - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - return match handle.join() { - Ok(res) => res, - Err(panic_payload) => Err(anyhow!("{worker_name} panicked: {panic_payload:?}")), - }; - } - - if start.elapsed() > timeout { - return Err(anyhow!("{worker_name} did not finish within {:?}", timeout)); - } - - std::thread::sleep(Duration::from_millis(50)); + match wait_for_blocking_thread_finish(handle, timeout, worker_name) { + BlockingThreadFinish::Clean => Ok(()), + BlockingThreadFinish::Failed(error) | BlockingThreadFinish::TimedOut(error) => Err(error), } } diff --git a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs index 1077fdfc0d..5540e694bb 100644 --- a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs @@ -1,3 +1,4 @@ +use super::core::{BlockingThreadFinish, combine_finish_errors, wait_for_blocking_thread_finish}; use crate::{ AudioFrame, AudioMuxer, Muxer, SharedPauseState, TaskPool, VideoMuxer, output_pipeline::NativeCameraFrame, screen_capture, @@ -88,6 +89,52 @@ impl FrameDropTracker { } } +fn finish_encoder_thread( + handle: JoinHandle>, + label: &str, +) -> BlockingThreadFinish { + wait_for_blocking_thread_finish(handle, Duration::from_secs(5), label) +} + +fn finish_segmented_encoder( + mut state: EncoderState, + timestamp: Duration, + thread_label: &str, + finish_label: &str, +) -> anyhow::Result<()> { + if let Err(error) = state.video_tx.send(None) { + trace!("{thread_label} channel already closed during finish: {error}"); + } + + let thread_result = state + .encoder_handle + .take() + .map(|handle| finish_encoder_thread(handle, thread_label)) + .unwrap_or(BlockingThreadFinish::Clean); + + let thread_error = match thread_result { + BlockingThreadFinish::Clean => None, + BlockingThreadFinish::Failed(error) => Some(error), + BlockingThreadFinish::TimedOut(error) => return Err(error), + }; + + let finalize_error = match state.encoder.lock() { + Ok(mut encoder) => encoder + .finish_with_timestamp(timestamp) + .map_err(|error| anyhow!("{finish_label}: {error:#}")) + .err(), + Err(_) => Some(anyhow!( + "{finish_label}: encoder mutex poisoned - recording may be corrupt or incomplete" + )), + }; + + match (thread_error, finalize_error) { + (None, None) => Ok(()), + (Some(error), None) | (None, Some(error)) => Err(error), + (Some(primary), Some(secondary)) => Err(combine_finish_errors(primary, secondary)), + } +} + struct EncoderState { video_tx: SyncSender, Duration)>>, encoder: Arc>, @@ -174,54 +221,15 @@ impl Muxer for MacOSFragmentedM4SMuxer { } fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - if let Some(mut state) = self.state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("M4S encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - match handle.join() { - Err(panic_payload) => { - warn!( - "M4S encoder thread panicked during finish: {:?}", - panic_payload - ); - } - Ok(Err(e)) => { - warn!("M4S encoder thread returned error: {e}"); - } - Ok(Ok(())) => {} - } - break; - } - if start.elapsed() > timeout { - warn!( - "M4S encoder thread did not finish within {:?}, abandoning", - timeout - ); - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - - match state.encoder.lock() { - Ok(mut encoder) => { - if let Err(e) = encoder.finish_with_timestamp(timestamp) { - warn!("Failed to finish segmented encoder: {e}"); - } - } - Err(_) => { - error!("Encoder mutex poisoned during finish - encoder thread likely panicked"); - return Ok(Err(anyhow!( - "Encoder mutex poisoned - recording may be corrupt or incomplete" - ))); - } - } + if let Some(state) = self.state.take() + && let Err(error) = finish_segmented_encoder( + state, + timestamp, + "M4S encoder", + "Failed to finish segmented encoder", + ) + { + return Ok(Err(error)); } Ok(Ok(())) @@ -405,7 +413,7 @@ impl VideoMuxer for MacOSFragmentedM4SMuxer { self.frame_drops.record_drop(); } std::sync::mpsc::TrySendError::Disconnected(_) => { - trace!("M4S encoder channel disconnected"); + return Err(anyhow!("M4S encoder channel disconnected")); } }, } @@ -668,56 +676,15 @@ impl Muxer for MacOSFragmentedM4SCameraMuxer { } fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - if let Some(mut state) = self.state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("M4S camera encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - match handle.join() { - Err(panic_payload) => { - warn!( - "M4S camera encoder thread panicked during finish: {:?}", - panic_payload - ); - } - Ok(Err(e)) => { - warn!("M4S camera encoder thread returned error: {e}"); - } - Ok(Ok(())) => {} - } - break; - } - if start.elapsed() > timeout { - warn!( - "M4S camera encoder thread did not finish within {:?}, abandoning", - timeout - ); - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - - match state.encoder.lock() { - Ok(mut encoder) => { - if let Err(e) = encoder.finish_with_timestamp(timestamp) { - warn!("Failed to finish camera segmented encoder: {e}"); - } - } - Err(_) => { - error!( - "Camera encoder mutex poisoned during finish - encoder thread likely panicked" - ); - return Ok(Err(anyhow!( - "Camera encoder mutex poisoned - recording may be corrupt or incomplete" - ))); - } - } + if let Some(state) = self.state.take() + && let Err(error) = finish_segmented_encoder( + state, + timestamp, + "M4S camera encoder", + "Failed to finish camera segmented encoder", + ) + { + return Ok(Err(error)); } Ok(Ok(())) @@ -903,7 +870,7 @@ impl VideoMuxer for MacOSFragmentedM4SCameraMuxer { self.frame_drops.record_drop(); } std::sync::mpsc::TrySendError::Disconnected(_) => { - trace!("M4S camera encoder channel disconnected"); + return Err(anyhow!("M4S camera encoder channel disconnected")); } }, } diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index 78cb1d0037..8e378668d6 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -1,3 +1,4 @@ +use super::core::{BlockingThreadFinish, combine_finish_errors, wait_for_blocking_thread_finish}; use crate::{ AudioFrame, AudioMuxer, Muxer, SharedPauseState, TaskPool, VideoMuxer, output_pipeline::{NativeCameraFrame, camera_frame_to_ffmpeg}, @@ -92,6 +93,93 @@ impl FrameDropTracker { } } +fn finish_encoder_thread( + handle: JoinHandle>, + label: &str, +) -> BlockingThreadFinish { + wait_for_blocking_thread_finish(handle, Duration::from_secs(5), label) +} + +trait FinishableEncoderState { + fn close_channel(&self, thread_label: &str); + fn take_handle(&mut self) -> Option>>; + fn finish_encoder(&self, timestamp: Duration, finish_label: &str) -> anyhow::Result<()>; +} + +impl FinishableEncoderState for EncoderState { + fn close_channel(&self, thread_label: &str) { + if let Err(error) = self.video_tx.send(None) { + trace!("{thread_label} channel already closed during finish: {error}"); + } + } + + fn take_handle(&mut self) -> Option>> { + self.encoder_handle.take() + } + + fn finish_encoder(&self, timestamp: Duration, finish_label: &str) -> anyhow::Result<()> { + match self.encoder.lock() { + Ok(mut encoder) => encoder + .finish_with_timestamp(timestamp) + .map_err(|error| anyhow!("{finish_label}: {error:#}")), + Err(_) => Err(anyhow!( + "{finish_label}: encoder mutex poisoned - recording may be corrupt or incomplete" + )), + } + } +} + +impl FinishableEncoderState for CameraEncoderState { + fn close_channel(&self, thread_label: &str) { + if let Err(error) = self.video_tx.send(None) { + trace!("{thread_label} channel already closed during finish: {error}"); + } + } + + fn take_handle(&mut self) -> Option>> { + self.encoder_handle.take() + } + + fn finish_encoder(&self, timestamp: Duration, finish_label: &str) -> anyhow::Result<()> { + match self.encoder.lock() { + Ok(mut encoder) => encoder + .finish_with_timestamp(timestamp) + .map_err(|error| anyhow!("{finish_label}: {error:#}")), + Err(_) => Err(anyhow!( + "{finish_label}: encoder mutex poisoned - recording may be corrupt or incomplete" + )), + } + } +} + +fn finish_segmented_encoder_impl( + state: &mut impl FinishableEncoderState, + timestamp: Duration, + thread_label: &str, + finish_label: &str, +) -> anyhow::Result<()> { + state.close_channel(thread_label); + + let thread_result = state + .take_handle() + .map(|handle| finish_encoder_thread(handle, thread_label)) + .unwrap_or(BlockingThreadFinish::Clean); + + let thread_error = match thread_result { + BlockingThreadFinish::Clean => None, + BlockingThreadFinish::Failed(error) => Some(error), + BlockingThreadFinish::TimedOut(error) => return Err(error), + }; + + let finalize_error = state.finish_encoder(timestamp, finish_label).err(); + + match (thread_error, finalize_error) { + (None, None) => Ok(()), + (Some(error), None) | (None, Some(error)) => Err(error), + (Some(primary), Some(secondary)) => Err(combine_finish_errors(primary, secondary)), + } +} + struct EncoderState { video_tx: SyncSender>, encoder: Arc>, @@ -182,54 +270,15 @@ impl Muxer for WindowsFragmentedM4SMuxer { } fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - if let Some(mut state) = self.state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Windows M4S encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - match handle.join() { - Err(panic_payload) => { - warn!( - "Windows M4S encoder thread panicked during finish: {:?}", - panic_payload - ); - } - Ok(Err(e)) => { - warn!("Windows M4S encoder thread returned error: {e}"); - } - Ok(Ok(())) => {} - } - break; - } - if start.elapsed() > timeout { - warn!( - "Windows M4S encoder thread did not finish within {:?}, abandoning", - timeout - ); - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - - match state.encoder.lock() { - Ok(mut encoder) => { - if let Err(e) = encoder.finish_with_timestamp(timestamp) { - warn!("Failed to finish segmented encoder: {e}"); - } - } - Err(_) => { - error!("Encoder mutex poisoned during finish - encoder thread likely panicked"); - return Ok(Err(anyhow!( - "Encoder mutex poisoned - recording may be corrupt or incomplete" - ))); - } - } + if let Some(mut state) = self.state.take() + && let Err(error) = finish_segmented_encoder_impl( + &mut state, + timestamp, + "Windows M4S encoder", + "Failed to finish segmented encoder", + ) + { + return Ok(Err(error)); } Ok(Ok(())) @@ -508,7 +557,7 @@ impl VideoMuxer for WindowsFragmentedM4SMuxer { self.frame_drops.record_frame(); } Err(_) => { - trace!("Windows M4S encoder channel disconnected"); + return Err(anyhow!("Windows M4S encoder channel disconnected")); } } } @@ -614,56 +663,15 @@ impl Muxer for WindowsFragmentedM4SCameraMuxer { } fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - if let Some(mut state) = self.state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Windows M4S camera encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - match handle.join() { - Err(panic_payload) => { - warn!( - "Windows M4S camera encoder thread panicked during finish: {:?}", - panic_payload - ); - } - Ok(Err(e)) => { - warn!("Windows M4S camera encoder thread returned error: {e}"); - } - Ok(Ok(())) => {} - } - break; - } - if start.elapsed() > timeout { - warn!( - "Windows M4S camera encoder thread did not finish within {:?}, abandoning", - timeout - ); - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - - match state.encoder.lock() { - Ok(mut encoder) => { - if let Err(e) = encoder.finish_with_timestamp(timestamp) { - warn!("Failed to finish camera segmented encoder: {e}"); - } - } - Err(_) => { - error!( - "Camera encoder mutex poisoned during finish - encoder thread likely panicked" - ); - return Ok(Err(anyhow!( - "Camera encoder mutex poisoned - recording may be corrupt or incomplete" - ))); - } - } + if let Some(mut state) = self.state.take() + && let Err(error) = finish_segmented_encoder_impl( + &mut state, + timestamp, + "Windows M4S camera encoder", + "Failed to finish camera segmented encoder", + ) + { + return Ok(Err(error)); } Ok(Ok(())) @@ -986,7 +994,7 @@ impl VideoMuxer for WindowsFragmentedM4SCameraMuxer { self.frame_drops.record_frame(); } Err(_) => { - trace!("Windows M4S camera encoder channel disconnected"); + return Err(anyhow!("Windows M4S camera encoder channel disconnected")); } } } diff --git a/crates/recording/src/output_pipeline/win_segmented.rs b/crates/recording/src/output_pipeline/win_segmented.rs index ae256a2b59..1edaf97daf 100644 --- a/crates/recording/src/output_pipeline/win_segmented.rs +++ b/crates/recording/src/output_pipeline/win_segmented.rs @@ -1,3 +1,4 @@ +use super::core::{BlockingThreadFinish, combine_finish_errors, wait_for_blocking_thread_finish}; use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, fragmentation, screen_capture}; use anyhow::{Context, anyhow}; use cap_media_info::{AudioInfo, VideoInfo}; @@ -233,6 +234,8 @@ impl Muxer for WindowsSegmentedMuxer { fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { let segment_path = self.current_segment_path(); let segment_start = self.segment_start_time; + let mut finish_error = None; + let mut final_segment = None; if let Some(mut state) = self.current_state.take() { if let Err(e) = state.video_tx.send(None) { @@ -240,26 +243,19 @@ impl Muxer for WindowsSegmentedMuxer { } if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "Screen encoder thread panicked during finish: {:?}", - panic_payload - ); - } - break; + match wait_for_blocking_thread_finish( + handle, + Duration::from_secs(5), + "Screen encoder thread", + ) { + BlockingThreadFinish::Clean => {} + BlockingThreadFinish::Failed(error) => { + finish_error = Some(error); } - if start.elapsed() > timeout { - warn!( - "Screen encoder thread did not finish within {:?}, abandoning", - timeout - ); - break; + BlockingThreadFinish::TimedOut(error) => { + self.write_in_progress_manifest(); + return Ok(Err(error)); } - std::thread::sleep(Duration::from_millis(50)); } } @@ -267,7 +263,13 @@ impl Muxer for WindowsSegmentedMuxer { .output .lock() .map_err(|_| anyhow!("Failed to lock output"))?; - output.write_trailer()?; + if let Err(error) = output.write_trailer() { + let error = anyhow!("Failed to write screen trailer: {error:#}"); + finish_error = Some(match finish_error.take() { + Some(existing) => combine_finish_errors(existing, error), + None => error, + }); + } fragmentation::sync_file(&segment_path); @@ -275,7 +277,7 @@ impl Muxer for WindowsSegmentedMuxer { let final_duration = timestamp.saturating_sub(start); let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - self.completed_segments.push(SegmentInfo { + final_segment = Some(SegmentInfo { path: segment_path, index: self.current_index, duration: final_duration, @@ -284,7 +286,19 @@ impl Muxer for WindowsSegmentedMuxer { } } - self.finalize_manifest(); + if let Some(error) = finish_error { + self.write_in_progress_manifest(); + return Ok(Err(error)); + } + + if let Some(segment) = final_segment { + self.completed_segments.push(segment); + } + + if let Err(error) = self.finalize_manifest() { + self.write_in_progress_manifest(); + return Ok(Err(error)); + } Ok(Ok(())) } @@ -328,7 +342,7 @@ impl WindowsSegmentedMuxer { } } - fn finalize_manifest(&self) { + fn finalize_manifest(&self) -> anyhow::Result<()> { let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); let manifest = Manifest { @@ -354,12 +368,12 @@ impl WindowsSegmentedMuxer { }; let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write final manifest to {}: {e}", + fragmentation::atomic_write_json(&manifest_path, &manifest).map_err(|error| { + anyhow!( + "Failed to write final manifest to {}: {error}", manifest_path.display() - ); - } + ) + }) } fn create_segment(&mut self) -> anyhow::Result<()> { @@ -615,6 +629,8 @@ impl WindowsSegmentedMuxer { let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); let segment_duration = timestamp.saturating_sub(segment_start); let completed_segment_path = self.current_segment_path(); + let mut rotation_error = None; + let mut rotated_segment = None; if let Some(mut state) = self.current_state.take() { if let Err(e) = state.video_tx.send(None) { @@ -622,26 +638,19 @@ impl WindowsSegmentedMuxer { } if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "Screen encoder thread panicked during rotation: {:?}", - panic_payload - ); - } - break; + match wait_for_blocking_thread_finish( + handle, + Duration::from_secs(5), + "Screen encoder thread during rotation", + ) { + BlockingThreadFinish::Clean => {} + BlockingThreadFinish::Failed(error) => { + rotation_error = Some(error); } - if start.elapsed() > timeout { - warn!( - "Screen encoder thread did not finish within {:?} during rotation, abandoning", - timeout - ); - break; + BlockingThreadFinish::TimedOut(error) => { + self.write_in_progress_manifest(); + return Err(error); } - std::thread::sleep(Duration::from_millis(50)); } } @@ -649,7 +658,13 @@ impl WindowsSegmentedMuxer { .output .lock() .map_err(|_| anyhow!("Failed to lock output"))?; - output.write_trailer()?; + if let Err(error) = output.write_trailer() { + let error = anyhow!("Failed to write rotated screen trailer: {error:#}"); + rotation_error = Some(match rotation_error.take() { + Some(existing) => combine_finish_errors(existing, error), + None => error, + }); + } fragmentation::sync_file(&completed_segment_path); @@ -657,16 +672,24 @@ impl WindowsSegmentedMuxer { .ok() .map(|m| m.len()); - self.completed_segments.push(SegmentInfo { + rotated_segment = Some(SegmentInfo { path: completed_segment_path, index: self.current_index, duration: segment_duration, file_size, }); + } + + if let Some(error) = rotation_error { + self.write_in_progress_manifest(); + return Err(error); + } - self.write_manifest(); + if let Some(segment) = rotated_segment { + self.completed_segments.push(segment); } + self.write_manifest(); self.frame_drops.reset(); self.current_index += 1; self.segment_start_time = Some(timestamp); @@ -769,7 +792,7 @@ impl VideoMuxer for WindowsSegmentedMuxer { self.frame_drops.record_drop(); } std::sync::mpsc::TrySendError::Disconnected(_) => { - trace!("Screen encoder channel disconnected"); + return Err(anyhow!("Screen encoder channel disconnected")); } } } diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs index b9f2e52bb3..c99bf96059 100644 --- a/crates/recording/src/output_pipeline/win_segmented_camera.rs +++ b/crates/recording/src/output_pipeline/win_segmented_camera.rs @@ -1,3 +1,4 @@ +use super::core::{BlockingThreadFinish, combine_finish_errors, wait_for_blocking_thread_finish}; use crate::output_pipeline::win::{CameraBuffers, NativeCameraFrame, upload_mf_buffer_to_texture}; use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, fragmentation}; use anyhow::{Context, anyhow}; @@ -231,6 +232,8 @@ impl Muxer for WindowsSegmentedCameraMuxer { fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { let segment_path = self.current_segment_path(); let segment_start = self.segment_start_time; + let mut finish_error = None; + let mut final_segment = None; if let Some(mut state) = self.current_state.take() { if let Err(e) = state.video_tx.send(None) { @@ -238,26 +241,19 @@ impl Muxer for WindowsSegmentedCameraMuxer { } if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "Camera encoder thread panicked during finish: {:?}", - panic_payload - ); - } - break; + match wait_for_blocking_thread_finish( + handle, + Duration::from_secs(5), + "Camera encoder thread", + ) { + BlockingThreadFinish::Clean => {} + BlockingThreadFinish::Failed(error) => { + finish_error = Some(error); } - if start.elapsed() > timeout { - warn!( - "Camera encoder thread did not finish within {:?}, abandoning", - timeout - ); - break; + BlockingThreadFinish::TimedOut(error) => { + self.write_in_progress_manifest(); + return Ok(Err(error)); } - std::thread::sleep(Duration::from_millis(50)); } } @@ -265,7 +261,13 @@ impl Muxer for WindowsSegmentedCameraMuxer { .output .lock() .map_err(|_| anyhow!("Failed to lock output"))?; - output.write_trailer()?; + if let Err(error) = output.write_trailer() { + let error = anyhow!("Failed to write camera trailer: {error:#}"); + finish_error = Some(match finish_error.take() { + Some(existing) => combine_finish_errors(existing, error), + None => error, + }); + } fragmentation::sync_file(&segment_path); @@ -273,7 +275,7 @@ impl Muxer for WindowsSegmentedCameraMuxer { let final_duration = timestamp.saturating_sub(start); let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - self.completed_segments.push(SegmentInfo { + final_segment = Some(SegmentInfo { path: segment_path, index: self.current_index, duration: final_duration, @@ -282,7 +284,19 @@ impl Muxer for WindowsSegmentedCameraMuxer { } } - self.finalize_manifest(); + if let Some(error) = finish_error { + self.write_in_progress_manifest(); + return Ok(Err(error)); + } + + if let Some(segment) = final_segment { + self.completed_segments.push(segment); + } + + if let Err(error) = self.finalize_manifest() { + self.write_in_progress_manifest(); + return Ok(Err(error)); + } Ok(Ok(())) } @@ -326,7 +340,7 @@ impl WindowsSegmentedCameraMuxer { } } - fn finalize_manifest(&self) { + fn finalize_manifest(&self) -> anyhow::Result<()> { let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); let manifest = Manifest { @@ -352,12 +366,12 @@ impl WindowsSegmentedCameraMuxer { }; let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write final manifest to {}: {e}", + fragmentation::atomic_write_json(&manifest_path, &manifest).map_err(|error| { + anyhow!( + "Failed to write final manifest to {}: {error}", manifest_path.display() - ); - } + ) + }) } fn create_segment(&mut self, first_frame: &NativeCameraFrame) -> anyhow::Result<()> { @@ -643,6 +657,8 @@ impl WindowsSegmentedCameraMuxer { let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); let segment_duration = timestamp.saturating_sub(segment_start); let completed_segment_path = self.current_segment_path(); + let mut rotation_error = None; + let mut rotated_segment = None; if let Some(mut state) = self.current_state.take() { if let Err(e) = state.video_tx.send(None) { @@ -650,26 +666,19 @@ impl WindowsSegmentedCameraMuxer { } if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "Camera encoder thread panicked during rotation: {:?}", - panic_payload - ); - } - break; + match wait_for_blocking_thread_finish( + handle, + Duration::from_secs(5), + "Camera encoder thread during rotation", + ) { + BlockingThreadFinish::Clean => {} + BlockingThreadFinish::Failed(error) => { + rotation_error = Some(error); } - if start.elapsed() > timeout { - warn!( - "Camera encoder thread did not finish within {:?} during rotation, abandoning", - timeout - ); - break; + BlockingThreadFinish::TimedOut(error) => { + self.write_in_progress_manifest(); + return Err(error); } - std::thread::sleep(Duration::from_millis(50)); } } @@ -677,7 +686,13 @@ impl WindowsSegmentedCameraMuxer { .output .lock() .map_err(|_| anyhow!("Failed to lock output"))?; - output.write_trailer()?; + if let Err(error) = output.write_trailer() { + let error = anyhow!("Failed to write rotated camera trailer: {error:#}"); + rotation_error = Some(match rotation_error.take() { + Some(existing) => combine_finish_errors(existing, error), + None => error, + }); + } fragmentation::sync_file(&completed_segment_path); @@ -685,16 +700,24 @@ impl WindowsSegmentedCameraMuxer { .ok() .map(|m| m.len()); - self.completed_segments.push(SegmentInfo { + rotated_segment = Some(SegmentInfo { path: completed_segment_path, index: self.current_index, duration: segment_duration, file_size, }); + } + + if let Some(error) = rotation_error { + self.write_in_progress_manifest(); + return Err(error); + } - self.write_manifest(); + if let Some(segment) = rotated_segment { + self.completed_segments.push(segment); } + self.write_manifest(); self.frame_drops.reset(); self.current_index += 1; self.segment_start_time = Some(timestamp); @@ -795,7 +818,7 @@ impl VideoMuxer for WindowsSegmentedCameraMuxer { self.frame_drops.record_drop(); } std::sync::mpsc::TrySendError::Disconnected(_) => { - trace!("Camera encoder channel disconnected"); + return Err(anyhow!("Camera encoder channel disconnected")); } } } diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 4d575f6782..6256af1906 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -27,19 +27,21 @@ use crate::output_pipeline::{ use anyhow::{Context as _, anyhow, bail}; use cap_media_info::VideoInfo; use cap_project::{ - CursorEvents, MultipleSegments, Platform, RecordingMeta, RecordingMetaInner, + CursorEvents, MultipleSegment, MultipleSegments, Platform, RecordingMeta, RecordingMetaInner, StudioRecordingMeta, StudioRecordingStatus, }; use cap_timestamp::{Timestamp, Timestamps}; use futures::{FutureExt, StreamExt, future::OptionFuture, stream::FuturesUnordered}; use kameo::{Actor as _, prelude::*}; use relative_path::RelativePathBuf; +use serde::Serialize; use std::{ path::{Path, PathBuf}, + pin::Pin, sync::Arc, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use tokio::sync::watch; +use tokio::{sync::watch, task::JoinHandle}; use tracing::{Instrument, debug, error_span, info, trace, warn}; #[allow(clippy::large_enum_variant)] @@ -324,6 +326,8 @@ struct Pipeline { pub camera: Option, pub system_audio: Option, pub cursor: Option, + pub track_failures: SharedTrackFailures, + pub watcher_task: Option>, } struct FinishedPipeline { @@ -334,6 +338,136 @@ struct FinishedPipeline { pub camera: Option, pub system_audio: Option, pub cursor: Option, + pub track_failures: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +enum RecordingTrackKind { + Display, + Microphone, + Camera, + SystemAudio, +} + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +enum TrackFailureStage { + Runtime, + Stop, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct TrackFailureRecord { + track: RecordingTrackKind, + stage: TrackFailureStage, + error: String, +} + +type SharedTrackFailures = Arc>>; + +#[derive(Debug, Clone, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +struct RecordingFailureDiagnostics { + version: u32, + segments: Vec, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +struct SegmentFailureDiagnostics { + segment_index: u32, + start: f64, + end: f64, + track_failures: Vec, +} + +struct SegmentOutput { + meta: MultipleSegment, + diagnostics: Option, +} + +fn record_track_failure( + failures: &SharedTrackFailures, + track: RecordingTrackKind, + stage: TrackFailureStage, + error: impl Into, +) { + let error = error.into(); + match failures.lock() { + Ok(mut failures) => failures.push(TrackFailureRecord { + track, + stage, + error, + }), + Err(poisoned) => poisoned.into_inner().push(TrackFailureRecord { + track, + stage, + error, + }), + } +} + +fn take_track_failures(failures: &SharedTrackFailures) -> Vec { + match failures.lock() { + Ok(mut failures) => std::mem::take(&mut *failures), + Err(poisoned) => { + let mut failures = poisoned.into_inner(); + std::mem::take(&mut *failures) + } + } +} + +fn has_track_failure(failures: &SharedTrackFailures, track: RecordingTrackKind) -> bool { + match failures.lock() { + Ok(failures) => failures.iter().any(|failure| failure.track == track), + Err(poisoned) => poisoned + .into_inner() + .iter() + .any(|failure| failure.track == track), + } +} + +fn finalize_optional_track( + track: RecordingTrackKind, + result: Result, anyhow::Error>, + failures: &SharedTrackFailures, +) -> Option { + match result { + Ok(value) => value, + Err(error) => { + warn!(?track, error = %error, "Optional recording track failed during stop"); + if !has_track_failure(failures, track) { + record_track_failure(failures, track, TrackFailureStage::Stop, error.to_string()); + } + None + } + } +} + +fn build_recording_failure_diagnostics( + segments: &[SegmentFailureDiagnostics], +) -> Option { + if segments.is_empty() { + None + } else { + Some(RecordingFailureDiagnostics { + version: 1, + segments: segments.to_vec(), + }) + } +} + +fn write_recording_failure_diagnostics( + recording_dir: &Path, + diagnostics: &RecordingFailureDiagnostics, +) -> Result<(), RecordingError> { + std::fs::write( + recording_dir.join("recording-diagnostics.json"), + serde_json::to_string_pretty(diagnostics)?, + )?; + Ok(()) } impl Pipeline { @@ -349,38 +483,72 @@ impl Pipeline { cursor.actor.stop(); } - let system_audio = match system_audio.transpose() { - Ok(value) => value, - Err(err) => { - warn!("system audio pipeline failed during stop: {err:#}"); - None - } - }; + if let Some(watcher_task) = self.watcher_task.take() + && let Err(error) = watcher_task.await + { + warn!(error = %error, "Studio recording watcher task ended unexpectedly"); + } Ok(FinishedPipeline { start_time: self.start_time, - screen: screen.context("screen")?, - microphone: microphone.transpose().context("microphone")?, - camera: camera.transpose().context("camera")?, - system_audio, + screen: screen.context("display")?, + microphone: finalize_optional_track( + RecordingTrackKind::Microphone, + microphone.transpose(), + &self.track_failures, + ), + camera: finalize_optional_track( + RecordingTrackKind::Camera, + camera.transpose(), + &self.track_failures, + ), + system_audio: finalize_optional_track( + RecordingTrackKind::SystemAudio, + system_audio.transpose(), + &self.track_failures, + ), cursor: self.cursor, + track_failures: take_track_failures(&self.track_failures), }) } - fn spawn_watcher(&self, completion_tx: watch::Sender>>) { - let mut futures = FuturesUnordered::new(); - futures.push(self.screen.done_fut()); + fn spawn_watcher( + &mut self, + completion_tx: watch::Sender>>, + ) { + let mut futures = FuturesUnordered::< + Pin< + Box< + dyn futures::Future< + Output = (RecordingTrackKind, bool, Result<(), PipelineDoneError>), + > + Send, + >, + >, + >::new(); + futures.push(Box::pin({ + let done_fut = self.screen.done_fut(); + async move { (RecordingTrackKind::Display, true, done_fut.await) } + })); if let Some(ref microphone) = self.microphone { - futures.push(microphone.done_fut()); + futures.push(Box::pin({ + let done_fut = microphone.done_fut(); + async move { (RecordingTrackKind::Microphone, false, done_fut.await) } + })); } if let Some(ref camera) = self.camera { - futures.push(camera.done_fut()); + futures.push(Box::pin({ + let done_fut = camera.done_fut(); + async move { (RecordingTrackKind::Camera, false, done_fut.await) } + })); } if let Some(ref system_audio) = self.system_audio { - futures.push(system_audio.done_fut()); + futures.push(Box::pin({ + let done_fut = system_audio.done_fut(); + async move { (RecordingTrackKind::SystemAudio, false, done_fut.await) } + })); } // Ensure non-video pipelines stop promptly when the video pipeline completes @@ -405,15 +573,26 @@ impl Pipeline { }); } - tokio::spawn(async move { - while let Some(res) = futures.next().await { - if let Err(err) = res - && completion_tx.borrow().is_none() - { - let _ = completion_tx.send(Some(Err(err))); + let track_failures = self.track_failures.clone(); + self.watcher_task = Some(tokio::spawn(async move { + while let Some((track, required, res)) = futures.next().await { + if let Err(err) = res { + if required { + if completion_tx.borrow().is_none() { + let _ = completion_tx.send(Some(Err(err))); + } + } else { + warn!(?track, error = %err, "Optional recording track failed during runtime"); + record_track_failure( + &track_failures, + track, + TrackFailureStage::Runtime, + err.to_string(), + ); + } } } - }); + })); } } @@ -683,8 +862,10 @@ async fn stop_recording( } }; - let segment_metas: Vec<_> = futures::stream::iter(segments) - .then(async |s| { + let segment_outputs: Vec<_> = segments + .into_iter() + .enumerate() + .map(|(segment_index, s)| { let to_start_time = |timestamp: Timestamp| timestamp.signed_duration_since_secs(s.pipeline.start_time); @@ -727,73 +908,91 @@ async fn stop_recording( raw_display_start }; - MultipleSegment { - display: VideoMeta { - path: make_relative(&s.pipeline.screen.path), - fps: s - .pipeline - .screen - .video_info - .map(|v| v.fps()) - .unwrap_or_else(|| { + let diagnostics = + (!s.pipeline.track_failures.is_empty()).then(|| SegmentFailureDiagnostics { + segment_index: segment_index as u32, + start: s.start, + end: s.end, + track_failures: s.pipeline.track_failures.clone(), + }); + + SegmentOutput { + meta: MultipleSegment { + display: VideoMeta { + path: make_relative(&s.pipeline.screen.path), + fps: s + .pipeline + .screen + .video_info + .map(|v| v.fps()) + .unwrap_or_else(|| { + tracing::warn!( + "Screen video_info missing, using default fps: {}", + DEFAULT_FPS + ); + DEFAULT_FPS + }), + start_time: Some(display_start_time), + device_id: None, + }, + camera: s.pipeline.camera.map(|camera| VideoMeta { + path: make_relative(&camera.path), + fps: camera.video_info.map(|v| v.fps()).unwrap_or_else(|| { tracing::warn!( - "Screen video_info missing, using default fps: {}", + "Camera video_info missing, using default fps: {}", DEFAULT_FPS ); DEFAULT_FPS }), - start_time: Some(display_start_time), - device_id: None, - }, - camera: s.pipeline.camera.map(|camera| VideoMeta { - path: make_relative(&camera.path), - fps: camera.video_info.map(|v| v.fps()).unwrap_or_else(|| { - tracing::warn!( - "Camera video_info missing, using default fps: {}", - DEFAULT_FPS - ); - DEFAULT_FPS + start_time: camera_start_time, + device_id: s.camera_device_id.clone(), }), - start_time: camera_start_time, - device_id: s.camera_device_id.clone(), - }), - mic: s.pipeline.microphone.map(|mic| AudioMeta { - path: make_relative(&mic.path), - start_time: mic_start_time, - device_id: s.mic_device_id.clone(), - }), - system_audio: s.pipeline.system_audio.map(|audio| { - let raw_sys_start = to_start_time(audio.first_timestamp); - let sys_start_time = if let Some(mic_start) = mic_start_time { - let sync_offset = raw_sys_start - mic_start; - if sync_offset.abs() > 0.030 { - mic_start - } else { - raw_sys_start - } - } else { - let sync_offset = raw_sys_start - display_start_time; - if sync_offset.abs() > 0.030 { - display_start_time + mic: s.pipeline.microphone.map(|mic| AudioMeta { + path: make_relative(&mic.path), + start_time: mic_start_time, + device_id: s.mic_device_id.clone(), + }), + system_audio: s.pipeline.system_audio.map(|audio| { + let raw_sys_start = to_start_time(audio.first_timestamp); + let sys_start_time = if let Some(mic_start) = mic_start_time { + let sync_offset = raw_sys_start - mic_start; + if sync_offset.abs() > 0.030 { + mic_start + } else { + raw_sys_start + } } else { - raw_sys_start + let sync_offset = raw_sys_start - display_start_time; + if sync_offset.abs() > 0.030 { + display_start_time + } else { + raw_sys_start + } + }; + AudioMeta { + path: make_relative(&audio.path), + start_time: Some(sys_start_time), + device_id: None, } - }; - AudioMeta { - path: make_relative(&audio.path), - start_time: Some(sys_start_time), - device_id: None, - } - }), - cursor: s - .pipeline - .cursor - .as_ref() - .map(|cursor| make_relative(&cursor.output_path)), + }), + cursor: s + .pipeline + .cursor + .as_ref() + .map(|cursor| make_relative(&cursor.output_path)), + }, + diagnostics, } }) - .collect::>() - .await; + .collect(); + let segment_failure_diagnostics: Vec<_> = segment_outputs + .iter() + .filter_map(|segment| segment.diagnostics.clone()) + .collect(); + let segment_metas: Vec<_> = segment_outputs + .into_iter() + .map(|segment| segment.meta) + .collect(); let needs_remux = if fragmented { segment_metas.iter().any(|seg| { @@ -838,6 +1037,16 @@ async fn stop_recording( .write(&recording_dir) .map_err(RecordingError::from)?; + if let Some(diagnostics) = build_recording_failure_diagnostics(&segment_failure_diagnostics) + && let Err(error) = write_recording_failure_diagnostics(&recording_dir, &diagnostics) + { + warn!( + error = %error, + path = %recording_dir.join("recording-diagnostics.json").display(), + "Failed to persist recording diagnostics sidecar" + ); + } + Ok(CompletedRecording { project_path: recording_dir, meta, @@ -891,7 +1100,7 @@ impl SegmentPipelineFactory { next_cursors_id: u32, ) -> anyhow::Result { let segment_start_time = Timestamps::now(); - let pipeline = create_segment_pipeline( + let mut pipeline = create_segment_pipeline( &self.segments_dir, &self.cursors_dir, self.index, @@ -1261,6 +1470,8 @@ async fn create_segment_pipeline( camera, cursor, system_audio, + track_failures: Arc::new(std::sync::Mutex::new(Vec::new())), + watcher_task: None, }) } @@ -1299,3 +1510,430 @@ fn write_in_progress_meta(recording_dir: &Path) -> anyhow::Result<()> { meta.save_for_project() .map_err(|e| anyhow!("Failed to save in-progress meta: {:?}", e)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::output_pipeline::{ + AudioMuxer, ChannelAudioSource, ChannelAudioSourceConfig, ChannelVideoSource, + ChannelVideoSourceConfig, Muxer, TaskPool, VideoFrame, VideoMuxer, + }; + + fn test_finished_output_pipeline() -> FinishedOutputPipeline { + let timestamps = Timestamps::now(); + test_finished_output_pipeline_at( + PathBuf::from("track.mp4"), + Timestamp::Instant(timestamps.instant()), + None, + 1, + ) + } + + fn test_finished_output_pipeline_at( + path: PathBuf, + first_timestamp: Timestamp, + video_info: Option, + video_frame_count: u64, + ) -> FinishedOutputPipeline { + FinishedOutputPipeline { + path, + first_timestamp, + video_info, + video_frame_count, + } + } + + #[derive(Clone, Copy)] + struct TestVideoFrame { + timestamp: Timestamp, + } + + impl VideoFrame for TestVideoFrame { + fn timestamp(&self) -> Timestamp { + self.timestamp + } + } + + struct SuccessfulVideoMuxer; + + impl Muxer for SuccessfulVideoMuxer { + type Config = (); + + fn setup( + _config: Self::Config, + _output_path: PathBuf, + _video_config: Option, + _audio_config: Option, + _pause_flag: Arc, + _tasks: &mut TaskPool, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { Ok(Self) } + } + + fn finish(&mut self, _timestamp: Duration) -> anyhow::Result> { + Ok(Ok(())) + } + } + + impl AudioMuxer for SuccessfulVideoMuxer { + fn send_audio_frame( + &mut self, + _frame: crate::output_pipeline::AudioFrame, + _timestamp: Duration, + ) -> anyhow::Result<()> { + Ok(()) + } + } + + impl VideoMuxer for SuccessfulVideoMuxer { + type VideoFrame = TestVideoFrame; + + fn send_video_frame( + &mut self, + _frame: Self::VideoFrame, + _timestamp: Duration, + ) -> anyhow::Result<()> { + Ok(()) + } + } + + #[derive(Clone, Copy)] + struct FailingAudioMuxerConfig { + fail_after_frame: u64, + } + + struct FailingAudioMuxer { + fail_after_frame: u64, + sent_frames: u64, + } + + impl Muxer for FailingAudioMuxer { + type Config = FailingAudioMuxerConfig; + + fn setup( + config: Self::Config, + _output_path: PathBuf, + _video_config: Option, + _audio_config: Option, + _pause_flag: Arc, + _tasks: &mut TaskPool, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + Ok(Self { + fail_after_frame: config.fail_after_frame, + sent_frames: 0, + }) + } + } + + fn finish(&mut self, _timestamp: Duration) -> anyhow::Result> { + Ok(Ok(())) + } + } + + impl AudioMuxer for FailingAudioMuxer { + fn send_audio_frame( + &mut self, + _frame: crate::output_pipeline::AudioFrame, + _timestamp: Duration, + ) -> anyhow::Result<()> { + self.sent_frames += 1; + if self.sent_frames >= self.fail_after_frame { + return Err(anyhow!("optional audio mux send failed")); + } + Ok(()) + } + } + + fn test_video_info() -> VideoInfo { + VideoInfo::from_raw(cap_media_info::RawVideoFormat::Bgra, 16, 16, 30) + } + + fn test_audio_info() -> cap_media_info::AudioInfo { + cap_media_info::AudioInfo::new_raw( + cap_media_info::Sample::F32(cap_media_info::Type::Packed), + 48_000, + 2, + ) + } + + #[test] + fn finalize_optional_track_records_stop_failure() { + let failures = Arc::new(std::sync::Mutex::new(Vec::new())); + let output = finalize_optional_track( + RecordingTrackKind::Camera, + Err(anyhow!("camera stop failed")), + &failures, + ); + + assert!(output.is_none()); + + let recorded = take_track_failures(&failures); + assert_eq!( + recorded, + vec![TrackFailureRecord { + track: RecordingTrackKind::Camera, + stage: TrackFailureStage::Stop, + error: "camera stop failed".to_string(), + }] + ); + } + + #[test] + fn finalize_optional_track_preserves_successful_track() { + let failures = Arc::new(std::sync::Mutex::new(Vec::new())); + let output = finalize_optional_track( + RecordingTrackKind::Microphone, + Ok(Some(test_finished_output_pipeline())), + &failures, + ); + + assert!(output.is_some()); + assert!(take_track_failures(&failures).is_empty()); + } + + #[test] + fn finalize_optional_track_does_not_duplicate_runtime_failure() { + let failures = Arc::new(std::sync::Mutex::new(Vec::new())); + record_track_failure( + &failures, + RecordingTrackKind::SystemAudio, + TrackFailureStage::Runtime, + "system audio writer failed", + ); + + let output = finalize_optional_track( + RecordingTrackKind::SystemAudio, + Err(anyhow!("system audio writer failed")), + &failures, + ); + + assert!(output.is_none()); + assert_eq!( + take_track_failures(&failures), + vec![TrackFailureRecord { + track: RecordingTrackKind::SystemAudio, + stage: TrackFailureStage::Runtime, + error: "system audio writer failed".to_string(), + }] + ); + } + + #[test] + fn build_recording_failure_diagnostics_skips_clean_recordings() { + assert!(build_recording_failure_diagnostics(&[]).is_none()); + } + + #[test] + fn build_recording_failure_diagnostics_keeps_segment_failures() { + let diagnostics = build_recording_failure_diagnostics(&[SegmentFailureDiagnostics { + segment_index: 2, + start: 10.0, + end: 20.0, + track_failures: vec![ + TrackFailureRecord { + track: RecordingTrackKind::Microphone, + stage: TrackFailureStage::Runtime, + error: "microphone writer failed".to_string(), + }, + TrackFailureRecord { + track: RecordingTrackKind::SystemAudio, + stage: TrackFailureStage::Stop, + error: "system audio finalize failed".to_string(), + }, + ], + }]); + + assert_eq!( + diagnostics, + Some(RecordingFailureDiagnostics { + version: 1, + segments: vec![SegmentFailureDiagnostics { + segment_index: 2, + start: 10.0, + end: 20.0, + track_failures: vec![ + TrackFailureRecord { + track: RecordingTrackKind::Microphone, + stage: TrackFailureStage::Runtime, + error: "microphone writer failed".to_string(), + }, + TrackFailureRecord { + track: RecordingTrackKind::SystemAudio, + stage: TrackFailureStage::Stop, + error: "system audio finalize failed".to_string(), + }, + ], + }], + }) + ); + } + + #[tokio::test] + async fn stop_recording_keeps_success_when_diagnostics_sidecar_write_fails() { + let temp_dir = tempfile::tempdir().expect("temp dir should be created"); + let recording_dir = temp_dir.path().join("recording"); + let start_time = Timestamps::now(); + std::fs::create_dir_all(recording_dir.join("content")) + .expect("recording content dir should be created"); + std::fs::create_dir_all(recording_dir.join("recording-diagnostics.json")) + .expect("diagnostics path should be pre-created as a directory"); + + let segment = RecordingSegment { + start: 0.0, + end: 1.0, + pipeline: FinishedPipeline { + start_time, + screen: test_finished_output_pipeline_at( + recording_dir.join("content/display.mp4"), + Timestamp::Instant(start_time.instant() + Duration::from_millis(33)), + Some(test_video_info()), + 1, + ), + microphone: None, + camera: None, + system_audio: None, + cursor: None, + track_failures: vec![TrackFailureRecord { + track: RecordingTrackKind::Microphone, + stage: TrackFailureStage::Runtime, + error: "microphone runtime failure".to_string(), + }], + }, + camera_device_id: None, + mic_device_id: None, + }; + + let completed = stop_recording( + recording_dir.clone(), + vec![segment], + Default::default(), + false, + ) + .await + .expect("diagnostics sidecar failure should not abort stop_recording"); + + assert_eq!(completed.project_path, recording_dir); + assert!( + completed.project_path.join("project-config.json").is_file(), + "project config should still be written" + ); + assert!( + completed + .project_path + .join("recording-diagnostics.json") + .is_dir(), + "the pre-existing diagnostics directory should remain, proving the sidecar write failed" + ); + } + + #[tokio::test] + async fn stop_preserves_display_when_optional_track_fails_during_runtime() { + let temp_dir = tempfile::tempdir().expect("temp dir should be created"); + let timestamps = Timestamps::now(); + let (screen_tx, screen_rx) = flume::bounded(4); + let (completion_tx, completion_rx) = watch::channel(None); + let (mut microphone_tx, microphone_rx) = futures::channel::mpsc::channel(4); + + let screen = OutputPipeline::builder(temp_dir.path().join("display.mp4")) + .with_video::>(ChannelVideoSourceConfig::new( + test_video_info(), + screen_rx, + )) + .with_timestamps(timestamps) + .build::(()) + .await + .expect("display pipeline should build"); + + let microphone = OutputPipeline::builder(temp_dir.path().join("audio-input.ogg")) + .with_audio_source::(ChannelAudioSourceConfig::new( + test_audio_info(), + microphone_rx, + )) + .with_timestamps(timestamps) + .build::(FailingAudioMuxerConfig { + fail_after_frame: 1, + }) + .await + .expect("microphone pipeline should build"); + let microphone_done = microphone.done_fut(); + + let mut pipeline = Pipeline { + start_time: timestamps, + screen, + microphone: Some(microphone), + camera: None, + system_audio: None, + cursor: None, + track_failures: Arc::new(std::sync::Mutex::new(Vec::new())), + watcher_task: None, + }; + pipeline.spawn_watcher(completion_tx); + + screen_tx + .send_async(TestVideoFrame { + timestamp: Timestamp::Instant(timestamps.instant() + Duration::from_millis(33)), + }) + .await + .expect("display frame should send"); + drop(screen_tx); + + microphone_tx + .try_send(crate::output_pipeline::AudioFrame::new( + test_audio_info().empty_frame(960), + Timestamp::Instant(timestamps.instant() + Duration::from_millis(20)), + )) + .expect("microphone frame should send"); + drop(microphone_tx); + + let microphone_error = microphone_done + .await + .expect_err("optional microphone pipeline should fail at runtime"); + assert!( + microphone_error + .to_string() + .contains("Audio muxer stopped accepting frames at frame 1"), + "runtime error should retain the mux send-failure context" + ); + + let finished = pipeline + .stop() + .await + .expect("display success should still allow the recording to stop cleanly"); + + assert_eq!( + finished.screen.video_frame_count, 1, + "display output should be preserved" + ); + assert!( + finished.microphone.is_none(), + "optional microphone output should be dropped after runtime failure" + ); + assert_eq!( + finished.track_failures.len(), + 1, + "runtime failure should be recorded exactly once" + ); + assert!( + completion_rx.borrow().is_none(), + "optional runtime failure should not publish a required-track completion error" + ); + assert_eq!( + finished.track_failures[0].track, + RecordingTrackKind::Microphone + ); + assert_eq!(finished.track_failures[0].stage, TrackFailureStage::Runtime); + assert!( + finished.track_failures[0] + .error + .contains("Audio muxer stopped accepting frames at frame 1"), + "recorded runtime failure should preserve the mux send-failure context" + ); + } +} diff --git a/crates/rendering/src/cursor_interpolation.rs b/crates/rendering/src/cursor_interpolation.rs index 74bd7646cc..ad0dec8d85 100644 --- a/crates/rendering/src/cursor_interpolation.rs +++ b/crates/rendering/src/cursor_interpolation.rs @@ -7,10 +7,26 @@ use crate::{ spring_mass_damper::{SpringMassDamperSimulation, SpringMassDamperSimulationConfig}, }; -const CLICK_REACTION_WINDOW_MS: f64 = 160.0; -const MIN_MASS: f32 = 0.1; +const CLICK_LOOKAHEAD_TARGET_MS: f64 = 500.0; +const CLICK_SPRING_WINDOW_MS: f64 = 175.0; const SHAKE_THRESHOLD_UV: f64 = 0.015; const SHAKE_DETECTION_WINDOW_MS: f64 = 100.0; +const DECIMATE_FPS: f64 = 60.0; +const DECIMATE_MIN_DIST_UV: f64 = 1.0 / 1920.0; +const SIMULATION_STEP_MS: f64 = 1000.0 / 60.0; +const SPRING_SETTLE_EXTRA_MS: f64 = 300.0; + +const DEFAULT_CLICK_SPRING: SpringMassDamperSimulationConfig = SpringMassDamperSimulationConfig { + tension: 530.0, + mass: 1.0, + friction: 40.0, +}; + +const DRAG_SPRING: SpringMassDamperSimulationConfig = SpringMassDamperSimulationConfig { + tension: 1000.0, + mass: 1.0, + friction: 40.0, +}; #[derive(Clone, Copy, PartialEq, Eq, Debug)] enum SpringProfile { @@ -36,16 +52,12 @@ impl CursorSpringPresets { mass: c.mass, friction: c.friction, }) - .unwrap_or(SpringMassDamperSimulationConfig { - tension: 700.0, - mass: 1.0, - friction: 30.0, - }); + .unwrap_or(DEFAULT_CLICK_SPRING); Self { default: base, snappy, - drag: scale_config(base, 0.8, 1.2, 1.3), + drag: DRAG_SPRING, } } @@ -58,23 +70,9 @@ impl CursorSpringPresets { } } -fn scale_config( - base: SpringMassDamperSimulationConfig, - tension_scale: f32, - mass_scale: f32, - friction_scale: f32, -) -> SpringMassDamperSimulationConfig { - SpringMassDamperSimulationConfig { - tension: base.tension * tension_scale, - mass: (base.mass * mass_scale).max(MIN_MASS), - friction: base.friction * friction_scale, - } -} - struct CursorSpringContext<'a> { clicks: &'a [CursorClickEvent], next_click_index: usize, - last_click_time: Option, primary_button_down: bool, } @@ -83,7 +81,6 @@ impl<'a> CursorSpringContext<'a> { Self { clicks, next_click_index: 0, - last_click_time: None, primary_button_down: false, } } @@ -92,7 +89,6 @@ impl<'a> CursorSpringContext<'a> { while let Some(click) = self.clicks.get(self.next_click_index) && click.time_ms <= time_ms { - self.last_click_time = Some(click.time_ms); if click.cursor_num == 0 { self.primary_button_down = click.down; } @@ -101,7 +97,7 @@ impl<'a> CursorSpringContext<'a> { } fn profile(&self, time_ms: f64) -> SpringProfile { - if self.was_recent_click(time_ms) { + if self.has_imminent_click(time_ms) { SpringProfile::Snappy } else if self.primary_button_down { SpringProfile::Drag @@ -110,11 +106,95 @@ impl<'a> CursorSpringContext<'a> { } } - fn was_recent_click(&self, time_ms: f64) -> bool { - self.last_click_time - .map(|t| (time_ms - t).abs() <= CLICK_REACTION_WINDOW_MS) - .unwrap_or(false) + fn has_imminent_click(&self, time_ms: f64) -> bool { + let idx = self.clicks[self.next_click_index..].partition_point(|c| c.time_ms <= time_ms); + self.clicks + .get(self.next_click_index + idx) + .is_some_and(|c| c.time_ms - time_ms <= CLICK_SPRING_WINDOW_MS) + } +} + +fn next_click_within( + clicks: &[CursorClickEvent], + time_ms: f64, + window_ms: f64, +) -> Option<&CursorClickEvent> { + let idx = clicks.partition_point(|c| c.time_ms <= time_ms); + clicks.get(idx).filter(|c| c.time_ms - time_ms <= window_ms) +} + +fn position_at_time(moves: &[CursorMoveEvent], time_ms: f64) -> (f64, f64) { + if moves.is_empty() { + return (0.0, 0.0); + } + if time_ms <= moves[0].time_ms { + return (moves[0].x, moves[0].y); + } + if let Some(last) = moves.last() + && time_ms >= last.time_ms + { + return (last.x, last.y); + } + moves + .windows(2) + .find_map(|w| { + if time_ms >= w[0].time_ms && time_ms < w[1].time_ms { + let dt = w[1].time_ms - w[0].time_ms; + if dt > IDLE_GAP_THRESHOLD_MS { + return Some((w[0].x, w[0].y)); + } + let u = if dt.abs() < 1e-9 { + 0.0 + } else { + (time_ms - w[0].time_ms) / dt + }; + Some(( + w[0].x + (w[1].x - w[0].x) * u, + w[0].y + (w[1].y - w[0].y) * u, + )) + } else { + None + } + }) + .unwrap_or_else(|| { + let l = moves.last().unwrap(); + (l.x, l.y) + }) +} + +const IDLE_GAP_THRESHOLD_MS: f64 = SIMULATION_STEP_MS * 4.0; + +fn position_at_time_hinted( + moves: &[CursorMoveEvent], + time_ms: f64, + hint: &mut usize, +) -> (f64, f64) { + while *hint + 1 < moves.len() && moves[*hint + 1].time_ms <= time_ms { + *hint += 1; + } + + let m = &moves[*hint]; + if *hint + 1 < moves.len() { + let next = &moves[*hint + 1]; + if time_ms >= m.time_ms && time_ms < next.time_ms { + let dt = next.time_ms - m.time_ms; + if dt > IDLE_GAP_THRESHOLD_MS { + return (m.x, m.y); + } + if dt > 1e-9 { + let u = (time_ms - m.time_ms) / dt; + return (m.x + (next.x - m.x) * u, m.y + (next.y - m.y) * u); + } + } + } + (m.x, m.y) +} + +fn cursor_id_at_time(moves: &[CursorMoveEvent], _time_ms: f64, hint: usize) -> &str { + if hint < moves.len() { + return &moves[hint].cursor_id; } + &moves.last().unwrap().cursor_id } #[derive(Debug, Clone)] @@ -139,194 +219,219 @@ pub fn interpolate_cursor_with_click_spring( smoothing: Option, click_spring: Option, ) -> Option { - let time_ms = (time_secs * 1000.0) as f64; - if cursor.moves.is_empty() { return None; } - if cursor.moves[0].time_ms > time_ms { - let event = &cursor.moves[0]; - - return Some(InterpolatedCursorPosition { - position: Coord::new(XY { - x: event.x, - y: event.y, - }), - velocity: XY::new(0.0, 0.0), - cursor_id: event.cursor_id.clone(), - }); - } - - if let Some(event) = cursor.moves.last() - && event.time_ms <= time_ms - { - return Some(InterpolatedCursorPosition { - position: Coord::new(XY { - x: event.x, - y: event.y, - }), - velocity: XY::new(0.0, 0.0), - cursor_id: event.cursor_id.clone(), - }); - } + let time_ms = (time_secs * 1000.0) as f64; if let Some(smoothing_config) = smoothing { let filtered_moves = filter_cursor_shake(&cursor.moves); - let prepared_moves = densify_cursor_moves(filtered_moves.as_ref()); - let events = get_smoothed_cursor_events_with_click_spring( + let prepared_moves = decimate_cursor_moves(filtered_moves.as_ref()); + let timeline = build_smoothed_timeline( cursor, prepared_moves.as_ref(), smoothing_config, click_spring, ); - interpolate_smoothed_position(&events, time_secs as f64, smoothing_config) + interpolate_timeline(&timeline, time_ms) } else { - let (pos, cursor_id, velocity) = cursor.moves.windows(2).find_map(|chunk| { + if cursor.moves[0].time_ms > time_ms { + let event = &cursor.moves[0]; + return Some(InterpolatedCursorPosition { + position: Coord::new(XY { + x: event.x, + y: event.y, + }), + velocity: XY::new(0.0, 0.0), + cursor_id: event.cursor_id.clone(), + }); + } + + if let Some(event) = cursor.moves.last() + && event.time_ms <= time_ms + { + return Some(InterpolatedCursorPosition { + position: Coord::new(XY { + x: event.x, + y: event.y, + }), + velocity: XY::new(0.0, 0.0), + cursor_id: event.cursor_id.clone(), + }); + } + + cursor.moves.windows(2).find_map(|chunk| { if time_ms >= chunk[0].time_ms && time_ms < chunk[1].time_ms { let c = &chunk[0]; let next = &chunk[1]; let delta_ms = (next.time_ms - c.time_ms) as f32; let dt = (delta_ms / 1000.0).max(0.000_1); let velocity = XY::new(((next.x - c.x) as f32) / dt, ((next.y - c.y) as f32) / dt); - Some(( - XY::new(c.x as f32, c.y as f32), - c.cursor_id.clone(), + Some(InterpolatedCursorPosition { + position: Coord::new(XY { x: c.x, y: c.y }), velocity, - )) + cursor_id: c.cursor_id.clone(), + }) } else { None } - })?; - - Some(InterpolatedCursorPosition { - position: Coord::new(XY { - x: pos.x as f64, - y: pos.y as f64, - }), - velocity, - cursor_id, }) } } -#[allow(dead_code)] -fn get_smoothed_cursor_events( - cursor: &CursorEvents, - moves: &[CursorMoveEvent], - smoothing_config: SpringMassDamperSimulationConfig, -) -> Vec { - get_smoothed_cursor_events_with_click_spring(cursor, moves, smoothing_config, None) -} - -fn get_smoothed_cursor_events_with_click_spring( +fn build_smoothed_timeline( cursor: &CursorEvents, moves: &[CursorMoveEvent], smoothing_config: SpringMassDamperSimulationConfig, click_spring: Option, ) -> Vec { - let mut last_time = 0.0; - - let mut events = vec![]; + if moves.is_empty() { + return vec![]; + } - let mut sim = SpringMassDamperSimulation::new(smoothing_config); let presets = CursorSpringPresets::new(smoothing_config, click_spring); let mut context = CursorSpringContext::new(&cursor.clicks); + let mut sim = SpringMassDamperSimulation::new(smoothing_config); - sim.set_position(XY::new(moves[0].x, moves[0].y).map(|v| v as f32)); + let start_pos = XY::new(moves[0].x as f32, moves[0].y as f32); + sim.set_position(start_pos); sim.set_velocity(XY::new(0.0, 0.0)); - sim.set_target_position(sim.position); + sim.set_target_position(start_pos); - if moves[0].time_ms > 0.0 { - events.push(SmoothedCursorEvent { - time: 0.0, - target_position: sim.target_position, - position: sim.position, - velocity: sim.velocity, - cursor_id: moves[0].cursor_id.clone(), - }) - } + let end_time_ms = moves.last().unwrap().time_ms; + let settle_end = end_time_ms + SPRING_SETTLE_EXTRA_MS; - for m in moves.iter() { - let target_position = XY::new(m.x, m.y).map(|v| v as f32); - sim.set_target_position(target_position); + let capacity = ((settle_end / SIMULATION_STEP_MS).ceil() as usize) + 2; + let mut events = Vec::with_capacity(capacity); + let mut move_hint: usize = 0; - context.advance_to(m.time_ms); - let profile = context.profile(m.time_ms); - sim.set_config(presets.config(profile)); + events.push(SmoothedCursorEvent { + time: 0.0, + position: start_pos, + velocity: XY::new(0.0, 0.0), + cursor_id: moves[0].cursor_id.clone(), + }); - sim.run(m.time_ms as f32 - last_time); + let mut t_ms = SIMULATION_STEP_MS; - last_time = m.time_ms as f32; + while t_ms <= settle_end { + let clamped_t = t_ms.min(end_time_ms); - let clamped_position = XY::new( - sim.position.x.clamp(0.0, 1.0), - sim.position.y.clamp(0.0, 1.0), - ); + let (cx, cy) = position_at_time_hinted(moves, clamped_t, &mut move_hint); + let cid = cursor_id_at_time(moves, clamped_t, move_hint).to_string(); + + let target = if let Some(click) = + next_click_within(&cursor.clicks, t_ms, CLICK_LOOKAHEAD_TARGET_MS) + { + let (tx, ty) = position_at_time(moves, click.time_ms.min(end_time_ms)); + XY::new(tx as f32, ty as f32) + } else { + XY::new(cx as f32, cy as f32) + }; + + sim.set_target_position(target); + + context.advance_to(t_ms); + sim.set_config(presets.config(context.profile(t_ms))); + + sim.run(SIMULATION_STEP_MS as f32); events.push(SmoothedCursorEvent { - time: m.time_ms as f32, - target_position, - position: clamped_position, + time: t_ms as f32, + position: XY::new( + sim.position.x.clamp(0.0, 1.0), + sim.position.y.clamp(0.0, 1.0), + ), velocity: sim.velocity, - cursor_id: m.cursor_id.clone(), + cursor_id: cid, }); + + t_ms += SIMULATION_STEP_MS; } events } -fn interpolate_smoothed_position( - smoothed_events: &[SmoothedCursorEvent], - query_time: f64, - smoothing_config: SpringMassDamperSimulationConfig, +fn interpolate_timeline( + events: &[SmoothedCursorEvent], + query_ms: f64, ) -> Option { - if smoothed_events.is_empty() { + if events.is_empty() { return None; } - let mut sim = SpringMassDamperSimulation::new(smoothing_config); + let query = query_ms as f32; - let query_time_ms = (query_time * 1000.0) as f32; + if query <= events[0].time { + let e = &events[0]; + return Some(InterpolatedCursorPosition { + position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), + velocity: e.velocity, + cursor_id: e.cursor_id.clone(), + }); + } - let cursor_id = match smoothed_events - .windows(2) - .find(|chunk| chunk[0].time <= query_time_ms && query_time_ms < chunk[1].time) - { - Some(c) => { - sim.set_position(c[0].position); - sim.set_velocity(c[0].velocity); - sim.set_target_position(c[0].target_position); - sim.run(query_time_ms - c[0].time); - c[0].cursor_id.clone() - } - None => { - let e = smoothed_events.last().unwrap(); - sim.set_position(e.position); - sim.set_velocity(e.velocity); - sim.set_target_position(e.target_position); - sim.run(query_time_ms - e.time); - e.cursor_id.clone() - } + if query >= events.last().unwrap().time { + let e = events.last().unwrap(); + return Some(InterpolatedCursorPosition { + position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), + velocity: e.velocity, + cursor_id: e.cursor_id.clone(), + }); + } + + let first_time = events[0].time; + let step = if events.len() > 1 { + events[1].time - events[0].time + } else { + SIMULATION_STEP_MS as f32 }; - let clamped_position = XY::new( - sim.position.x.clamp(0.0, 1.0) as f64, - sim.position.y.clamp(0.0, 1.0) as f64, - ); + let raw_idx = ((query - first_time) / step) as usize; + let idx = raw_idx.min(events.len().saturating_sub(2)); + + let (a, b) = + if events[idx].time <= query && idx + 1 < events.len() && query < events[idx + 1].time { + (&events[idx], &events[idx + 1]) + } else { + match events + .windows(2) + .find(|w| w[0].time <= query && query < w[1].time) + { + Some(w) => (&w[0], &w[1]), + None => { + let e = events.last().unwrap(); + return Some(InterpolatedCursorPosition { + position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), + velocity: e.velocity, + cursor_id: e.cursor_id.clone(), + }); + } + } + }; + + let dt = b.time - a.time; + let t = if dt.abs() < 1e-6 { + 0.0 + } else { + ((query - a.time) / dt).clamp(0.0, 1.0) + }; + let inv = 1.0 - t; Some(InterpolatedCursorPosition { - position: Coord::new(clamped_position), - velocity: sim.velocity, - cursor_id, + position: Coord::new(XY::new( + (a.position.x * inv + b.position.x * t) as f64, + (a.position.y * inv + b.position.y * t) as f64, + )), + velocity: XY::new( + a.velocity.x * inv + b.velocity.x * t, + a.velocity.y * inv + b.velocity.y * t, + ), + cursor_id: a.cursor_id.clone(), }) } -const CURSOR_FRAME_DURATION_MS: f64 = 1000.0 / 60.0; -const GAP_INTERPOLATION_THRESHOLD_MS: f64 = CURSOR_FRAME_DURATION_MS * 4.0; -const MIN_CURSOR_TRAVEL_FOR_INTERPOLATION: f64 = 0.02; -const MAX_INTERPOLATED_STEPS: usize = 120; - fn filter_cursor_shake<'a>(moves: &'a [CursorMoveEvent]) -> Cow<'a, [CursorMoveEvent]> { if moves.len() < 3 { return Cow::Borrowed(moves); @@ -385,86 +490,48 @@ fn filter_cursor_shake<'a>(moves: &'a [CursorMoveEvent]) -> Cow<'a, [CursorMoveE Cow::Owned(filtered) } -fn densify_cursor_moves<'a>(moves: &'a [CursorMoveEvent]) -> Cow<'a, [CursorMoveEvent]> { +fn decimate_cursor_moves<'a>(moves: &'a [CursorMoveEvent]) -> Cow<'a, [CursorMoveEvent]> { if moves.len() < 2 { return Cow::Borrowed(moves); } - let requires_interpolation = moves.windows(2).any(|window| { - let current = &window[0]; - let next = &window[1]; - should_fill_gap(current, next) - }); - - if !requires_interpolation { - return Cow::Borrowed(moves); - } + let frame_ms = (1000.0 / DECIMATE_FPS).floor(); - let mut dense_moves = Vec::with_capacity(moves.len()); - dense_moves.push(moves[0].clone()); + let mut out = Vec::with_capacity(moves.len()); + out.push(moves[0].clone()); - for i in 0..moves.len() - 1 { - let current = &moves[i]; + for i in 1..moves.len() { + let curr = &moves[i]; + let last_kept = out.last().unwrap(); + if curr.cursor_id != last_kept.cursor_id { + out.push(curr.clone()); + continue; + } + if i + 1 >= moves.len() { + out.push(curr.clone()); + break; + } let next = &moves[i + 1]; - if should_fill_gap(current, next) { - push_interpolated_samples(current, next, &mut dense_moves); - } else { - dense_moves.push(next.clone()); + let quick_succ = next.time_ms - last_kept.time_ms < frame_ms; + let dx = curr.x - last_kept.x; + let dy = curr.y - last_kept.y; + let small = (dx * dx + dy * dy).sqrt() < DECIMATE_MIN_DIST_UV; + if quick_succ || small { + continue; } + out.push(curr.clone()); } - Cow::Owned(dense_moves) -} - -fn should_fill_gap(from: &CursorMoveEvent, to: &CursorMoveEvent) -> bool { - if from.cursor_id != to.cursor_id { - return false; - } - - let dt_ms = (to.time_ms - from.time_ms).max(0.0); - if dt_ms < GAP_INTERPOLATION_THRESHOLD_MS { - return false; - } - - let dx = to.x - from.x; - let dy = to.y - from.y; - let distance = (dx * dx + dy * dy).sqrt(); - - distance >= MIN_CURSOR_TRAVEL_FOR_INTERPOLATION -} - -fn push_interpolated_samples( - from: &CursorMoveEvent, - to: &CursorMoveEvent, - output: &mut Vec, -) { - let dt_ms = (to.time_ms - from.time_ms).max(0.0); - if dt_ms <= 0.0 { - output.push(to.clone()); - return; - } - - let segments = - ((dt_ms / CURSOR_FRAME_DURATION_MS).ceil() as usize).clamp(2, MAX_INTERPOLATED_STEPS); - - for step in 1..segments { - let t = step as f64 / segments as f64; - output.push(CursorMoveEvent { - active_modifiers: to.active_modifiers.clone(), - cursor_id: to.cursor_id.clone(), - time_ms: from.time_ms + dt_ms * t, - x: from.x + (to.x - from.x) * t, - y: from.y + (to.y - from.y) * t, - }); + if out.len() == moves.len() { + Cow::Borrowed(moves) + } else { + Cow::Owned(out) } - - output.push(to.clone()); } #[derive(Debug)] struct SmoothedCursorEvent { time: f32, - target_position: XY, position: XY, velocity: XY, cursor_id: String, @@ -495,56 +562,81 @@ mod tests { } #[test] - fn densify_inserts_samples_for_large_gaps() { - let moves = vec![cursor_move(0.0, 0.1, 0.1), cursor_move(140.0, 0.9, 0.9)]; - - match densify_cursor_moves(&moves) { - Cow::Owned(dense) => { - assert!(dense.len() > moves.len(), "expected interpolated samples"); - assert_eq!( - dense.first().unwrap().time_ms, - moves.first().unwrap().time_ms - ); - assert_eq!(dense.last().unwrap().time_ms, moves.last().unwrap().time_ms); - } - Cow::Borrowed(_) => panic!("expected densified output"), + fn decimate_thins_burst_moves() { + let moves: Vec<_> = (0..20) + .map(|i| cursor_move(i as f64 * 2.0, 0.5 + i as f64 * 1e-6, 0.5)) + .collect(); + let decimated = decimate_cursor_moves(&moves); + match decimated { + Cow::Owned(v) => assert!(v.len() < moves.len()), + Cow::Borrowed(_) => {} } } - #[test] - fn densify_skips_small_gaps_or_cursor_switches() { - let small_gap = vec![cursor_move(0.0, 0.1, 0.1), cursor_move(30.0, 0.2, 0.2)]; - assert!(matches!(densify_cursor_moves(&small_gap), Cow::Borrowed(_))); - - let mut cursor_switch = vec![cursor_move(0.0, 0.1, 0.1), cursor_move(100.0, 0.8, 0.8)]; - cursor_switch[1].cursor_id = "text".into(); - assert!(matches!( - densify_cursor_moves(&cursor_switch), - Cow::Borrowed(_) - )); - } - #[test] fn spring_context_detects_dragging_between_clicks() { - let clicks = vec![click_event(100.0, true), click_event(360.0, false)]; + let clicks = vec![click_event(100.0, true), click_event(500.0, false)]; let mut context = CursorSpringContext::new(&clicks); context.advance_to(280.0); assert_eq!(context.profile(280.0), SpringProfile::Drag); + context.advance_to(450.0); + assert_eq!(context.profile(450.0), SpringProfile::Snappy); + context.advance_to(620.0); assert_eq!(context.profile(620.0), SpringProfile::Default); } #[test] - fn spring_context_switches_to_snappy_near_click_events() { - let clicks = vec![click_event(80.0, true), click_event(140.0, false)]; + fn spring_context_snappy_before_imminent_click() { + let clicks = vec![click_event(200.0, true)]; let mut context = CursorSpringContext::new(&clicks); - context.advance_to(80.0); - assert_eq!(context.profile(80.0), SpringProfile::Snappy); + context.advance_to(50.0); + assert_eq!(context.profile(50.0), SpringProfile::Snappy); + } - context.advance_to(340.0); - assert_eq!(context.profile(340.0), SpringProfile::Default); + #[test] + fn spring_context_default_when_click_far() { + let clicks = vec![click_event(2000.0, true)]; + let mut context = CursorSpringContext::new(&clicks); + + context.advance_to(100.0); + assert_eq!(context.profile(100.0), SpringProfile::Default); + } + + #[test] + fn smoothed_timeline_has_no_jumps() { + let moves = vec![ + cursor_move(0.0, 0.1, 0.1), + cursor_move(100.0, 0.2, 0.2), + cursor_move(200.0, 0.3, 0.3), + cursor_move(450.0, 0.5, 0.5), + cursor_move(600.0, 0.8, 0.3), + ]; + let clicks = vec![click_event(500.0, true)]; + let cursor = CursorEvents { moves, clicks }; + + let smoothing = SpringMassDamperSimulationConfig { + tension: 470.0, + mass: 3.0, + friction: 70.0, + }; + + let mut prev: Option = None; + for t_ms in (0..700).step_by(1) { + let t_secs = t_ms as f32 / 1000.0; + let pos = interpolate_cursor_with_click_spring(&cursor, t_secs, Some(smoothing), None); + if let (Some(p), Some(cur)) = (&prev, &pos) { + let dx = (cur.position.coord.x - p.position.coord.x).abs(); + let dy = (cur.position.coord.y - p.position.coord.y).abs(); + assert!( + dx < 0.02 && dy < 0.02, + "jump at t={t_ms}ms: dx={dx:.6}, dy={dy:.6}" + ); + } + prev = pos; + } } } diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 73f9c62a4a..f72392c984 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -497,15 +497,19 @@ impl AVAssetReaderDecoder { } fn select_best_decoder(&mut self, requested_time: f32) -> (usize, bool) { - let (best_id, _distance, needs_reset) = - self.pool_manager.find_best_decoder_for_time(requested_time); + let decoder_count = self.decoders.len(); + let (best_id, _distance, needs_reset) = self + .pool_manager + .find_best_decoder_for_time(requested_time, decoder_count); - let decoder_idx = best_id.min(self.decoders.len().saturating_sub(1)); + let decoder_idx = best_id.min(decoder_count.saturating_sub(1)); if needs_reset && decoder_idx < self.decoders.len() { self.decoders[decoder_idx].reset(requested_time); - self.pool_manager - .update_decoder_position(best_id, self.decoders[decoder_idx].current_position()); + self.pool_manager.update_decoder_position( + decoder_idx, + self.decoders[decoder_idx].current_position(), + ); } self.active_decoder_idx = decoder_idx; @@ -667,7 +671,6 @@ impl AVAssetReaderDecoder { pts_to_frame(frame.pts().value, Rational::new(1, frame.pts().scale), fps); let position_secs = current_frame as f32 / fps as f32; - last_decoded_position = Some(position_secs); let Some(frame) = frame.image_buf() else { tracing::debug!( @@ -677,6 +680,8 @@ impl AVAssetReaderDecoder { continue; }; + last_decoded_position = Some(position_secs); + let cache_frame = CachedFrame::new(&processor, frame.retained(), current_frame); if first_ever_frame.borrow().is_none() { @@ -721,15 +726,30 @@ impl AVAssetReaderDecoder { let _ = req.sender.send(data.to_decoded_frame()); } else { const MAX_FALLBACK_DISTANCE: u32 = 90; - - let nearest = cache - .range(..=req.frame) - .next_back() - .or_else(|| cache.range(req.frame..).next()); + const MAX_FORWARD_FALLBACK_DISTANCE: u32 = 1; + + let nearest = if is_scrubbing { + cache + .range(..=req.frame) + .next_back() + .or_else(|| cache.range(req.frame..).next()) + } else { + cache + .range(req.frame..) + .next() + .or_else(|| cache.range(..=req.frame).next_back()) + }; if let Some((&frame_num, cached)) = nearest { + let is_forward_or_equal = frame_num >= req.frame; let distance = req.frame.abs_diff(frame_num); - if distance <= MAX_FALLBACK_DISTANCE { + let is_allowed = if is_scrubbing { + distance <= MAX_FALLBACK_DISTANCE + } else { + is_forward_or_equal + && distance <= MAX_FORWARD_FALLBACK_DISTANCE + }; + if is_allowed { let _ = req.sender.send(cached.data().to_decoded_frame()); } @@ -819,8 +839,11 @@ impl AVAssetReaderDecoder { } } - if let Some(pos) = last_decoded_position { - this.pool_manager.update_decoder_position(decoder_idx, pos); + if let Some(last_pos) = last_decoded_position { + let max_req_time = max_requested_frame as f32 / fps as f32; + let capped = last_pos.min(max_req_time); + this.pool_manager + .update_decoder_position(decoder_idx, capped); } let mut unfulfilled_count = 0u32; @@ -837,34 +860,63 @@ impl AVAssetReaderDecoder { const MAX_FALLBACK_DISTANCE: u32 = 90; const MAX_FALLBACK_DISTANCE_EOF: u32 = 300; const MAX_FALLBACK_DISTANCE_NEAR_END: u32 = 180; + const MAX_FORWARD_FALLBACK_DISTANCE: u32 = 1; + + let allow_relaxed_fallback = is_scrubbing + || near_video_end + || decoder_at_eof + || decoder_returned_no_frames; let fallback_distance = if decoder_at_eof || decoder_returned_no_frames { MAX_FALLBACK_DISTANCE_EOF } else if near_video_end { MAX_FALLBACK_DISTANCE_NEAR_END + } else if !allow_relaxed_fallback { + MAX_FORWARD_FALLBACK_DISTANCE } else { MAX_FALLBACK_DISTANCE }; - let nearest = cache - .range(..=req.frame) - .next_back() - .or_else(|| cache.range(req.frame..).next()); + let nearest = if allow_relaxed_fallback { + cache + .range(..=req.frame) + .next_back() + .or_else(|| cache.range(req.frame..).next()) + } else { + cache + .range(req.frame..) + .next() + .or_else(|| cache.range(..=req.frame).next_back()) + }; if let Some((&frame_num, cached)) = nearest { + let is_forward_or_equal = frame_num >= req.frame; let distance = req.frame.abs_diff(frame_num); - if distance <= fallback_distance { + let is_allowed = if allow_relaxed_fallback { + distance <= fallback_distance + } else { + is_forward_or_equal && distance <= fallback_distance + }; + if is_allowed { let _ = req.sender.send(cached.data().to_decoded_frame()); - } else if let Some(ref last) = *last_sent_frame.borrow() { + } else if allow_relaxed_fallback + && let Some(ref last) = *last_sent_frame.borrow() + { let _ = req.sender.send(last.to_decoded_frame()); - } else if let Some(ref first) = *first_ever_frame.borrow() { + } else if allow_relaxed_fallback + && let Some(ref first) = *first_ever_frame.borrow() + { let _ = req.sender.send(first.to_decoded_frame()); } else { unfulfilled_count += 1; } - } else if let Some(ref last) = *last_sent_frame.borrow() { + } else if allow_relaxed_fallback + && let Some(ref last) = *last_sent_frame.borrow() + { let _ = req.sender.send(last.to_decoded_frame()); - } else if let Some(ref first) = *first_ever_frame.borrow() { + } else if allow_relaxed_fallback + && let Some(ref first) = *first_ever_frame.borrow() + { let _ = req.sender.send(first.to_decoded_frame()); } else { unfulfilled_count += 1; diff --git a/crates/rendering/src/decoder/multi_position.rs b/crates/rendering/src/decoder/multi_position.rs index f9aac22c8a..3da95fb32a 100644 --- a/crates/rendering/src/decoder/multi_position.rs +++ b/crates/rendering/src/decoder/multi_position.rs @@ -129,7 +129,11 @@ impl DecoderPoolManager { self.reposition_threshold } - pub fn find_best_decoder_for_time(&mut self, requested_time: f32) -> (usize, f32, bool) { + pub fn find_best_decoder_for_time( + &mut self, + requested_time: f32, + decoder_count: usize, + ) -> (usize, f32, bool) { self.total_accesses += 1; let frame = (requested_time * self.config.fps as f32).floor() as u32; @@ -139,7 +143,11 @@ impl DecoderPoolManager { let mut best_distance = f32::MAX; let mut needs_reset = true; - for position in &self.positions { + if decoder_count == 0 { + return (0, f32::MAX, true); + } + + for position in self.positions.iter().filter(|p| p.id < decoder_count) { let distance = (position.position_secs - requested_time).abs(); let is_usable = position.position_secs <= requested_time && (requested_time - position.position_secs) < self.reposition_threshold; @@ -152,7 +160,7 @@ impl DecoderPoolManager { } if needs_reset { - for position in &self.positions { + for position in self.positions.iter().filter(|p| p.id < decoder_count) { let distance = (position.position_secs - requested_time).abs(); if distance < best_distance { best_distance = distance; diff --git a/crates/rendering/src/layers/cursor.rs b/crates/rendering/src/layers/cursor.rs index 4ed98b0dd5..bfe7ad9601 100644 --- a/crates/rendering/src/layers/cursor.rs +++ b/crates/rendering/src/layers/cursor.rs @@ -11,17 +11,18 @@ use crate::{ STANDARD_CURSOR_HEIGHT, zoom::InterpolatedZoom, }; -const CURSOR_CLICK_DURATION: f64 = 0.25; +const CURSOR_CLICK_DURATION: f64 = 0.13; const CURSOR_CLICK_DURATION_MS: f64 = CURSOR_CLICK_DURATION * 1000.0; -const CLICK_SHRINK_SIZE: f32 = 0.7; +const CLICK_SHRINK_SIZE: f32 = 0.8; const CURSOR_IDLE_MIN_DELAY_MS: f64 = 500.0; const CURSOR_IDLE_FADE_OUT_MS: f64 = 400.0; +const CURSOR_IDLE_RESUME_LOOKAHEAD_MS: f64 = 250.0; const CURSOR_VECTOR_CAP: f32 = 320.0; const CURSOR_MIN_MOTION_NORMALIZED: f32 = 0.01; const CURSOR_MIN_MOTION_PX: f32 = 1.0; const CURSOR_BASELINE_FPS: f32 = 60.0; -const CURSOR_MULTIPLIER: f32 = 3.0; -const CURSOR_MAX_STRENGTH: f32 = 5.0; +const CURSOR_MULTIPLIER: f32 = 1.0; +const CURSOR_MAX_STRENGTH: f32 = 2.0; const VELOCITY_BLEND_RATIO: f32 = 0.7; /// The size to render the svg to. @@ -494,9 +495,9 @@ impl CursorLayer { cursor_opacity, ], rotation_params: [ - uniforms.project.cursor.rotation_amount, - uniforms.project.cursor.base_rotation, 0.0, + uniforms.project.cursor.base_rotation, + uniforms.cursor_x_axis_tilt_radians, 0.0, ], }; @@ -572,6 +573,18 @@ fn compute_cursor_idle_opacity( return 1.0; } + let idx = cursor + .moves + .partition_point(|e| e.time_ms <= current_time_ms); + let has_upcoming_move = cursor + .moves + .get(idx) + .is_some_and(|e| e.time_ms - current_time_ms <= CURSOR_IDLE_RESUME_LOOKAHEAD_MS); + + if has_upcoming_move { + return 1.0; + } + let Some(last_index) = cursor .moves .iter() @@ -634,46 +647,36 @@ fn get_click_t(clicks: &[CursorClickEvent], time_ms: f64) -> f32 { t * t * (3.0 - 2.0 * t) } - let mut prev_i = None; + let next_click = clicks.iter().find(|c| c.time_ms > time_ms); - for (i, clicks) in clicks.windows(2).enumerate() { - let left = &clicks[0]; - let right = &clicks[1]; + if let Some(next) = next_click { + if next.down && next.time_ms - time_ms <= CURSOR_CLICK_DURATION_MS { + return smoothstep( + 0.0, + CURSOR_CLICK_DURATION_MS as f32, + (next.time_ms - time_ms) as f32, + ); + } - if left.time_ms <= time_ms && right.time_ms > time_ms { - prev_i = Some(i); - break; + if !next.down { + return 0.0; } } - let Some(prev_i) = prev_i else { - return 1.0; - }; - - let prev = &clicks[prev_i]; - - if prev.down { - return 0.0; - } + let prev_click = clicks.iter().rev().find(|c| c.time_ms <= time_ms); - if !prev.down && time_ms - prev.time_ms <= CURSOR_CLICK_DURATION_MS { - return smoothstep( - 0.0, - CURSOR_CLICK_DURATION_MS as f32, - (time_ms - prev.time_ms) as f32, - ); - } + if let Some(prev) = prev_click { + if prev.down { + return 0.0; + } - if let Some(next) = clicks.get(prev_i + 1) - && !prev.down - && next.down - && next.time_ms - time_ms <= CURSOR_CLICK_DURATION_MS - { - return smoothstep( - 0.0, - CURSOR_CLICK_DURATION_MS as f32, - (time_ms - next.time_ms).abs() as f32, - ); + if !prev.down && time_ms - prev.time_ms <= CURSOR_CLICK_DURATION_MS { + return smoothstep( + 0.0, + CURSOR_CLICK_DURATION_MS as f32, + (time_ms - prev.time_ms) as f32, + ); + } } 1.0 diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 813f400147..2f5d4616b5 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -5,7 +5,9 @@ use cap_project::{ }; use composite_frame::CompositeVideoFrameUniforms; use core::f64; -use cursor_interpolation::{InterpolatedCursorPosition, interpolate_cursor}; +use cursor_interpolation::{ + InterpolatedCursorPosition, interpolate_cursor, interpolate_cursor_with_click_spring, +}; use decoder::{AsyncVideoDecoderHandle, spawn_decoder}; use frame_pipeline::{RenderSession, finish_encoder, finish_encoder_nv12, flush_pending_readback}; use futures::future::OptionFuture; @@ -15,7 +17,10 @@ use layers::{ }; use specta::Type; use spring_mass_damper::SpringMassDamperSimulationConfig; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; use std::{path::PathBuf, time::Instant}; use tokio::sync::mpsc; @@ -50,6 +55,47 @@ use text::{PreparedText, prepare_texts}; use zoom::*; pub use zoom_focus_interpolation::ZoomFocusInterpolator; +#[derive(Debug, Clone, serde::Serialize)] +pub struct Nv12RenderStartupBreakdownMs { + pub ffmpeg_init_ms: u64, + pub zoom_focus_interpolators_construct_ms: u64, + pub frame_renderer_and_layers_setup_ms: u64, + pub frame_index_zero_zoom_precompute_ms: Option, + pub frame_index_zero_decode_ms: Option, + pub frame_index_zero_render_nv12_ms: Option, + pub frame_index_zero_prefetch_decode_parallel_ms: Option, + pub frame_index_zero_join_wall_ms: Option, + pub first_queued_zoom_precompute_ms: Option, + pub first_queued_decode_ms: Option, + pub first_queued_render_nv12_ms: Option, + pub first_queued_prefetch_decode_parallel_ms: Option, + pub first_queued_join_wall_ms: Option, +} + +impl Nv12RenderStartupBreakdownMs { + fn new_header( + ffmpeg_init_ms: u64, + zoom_focus_interpolators_construct_ms: u64, + frame_renderer_and_layers_setup_ms: u64, + ) -> Self { + Self { + ffmpeg_init_ms, + zoom_focus_interpolators_construct_ms, + frame_renderer_and_layers_setup_ms, + frame_index_zero_zoom_precompute_ms: None, + frame_index_zero_decode_ms: None, + frame_index_zero_render_nv12_ms: None, + frame_index_zero_prefetch_decode_parallel_ms: None, + frame_index_zero_join_wall_ms: None, + first_queued_zoom_precompute_ms: None, + first_queued_decode_ms: None, + first_queued_render_nv12_ms: None, + first_queued_prefetch_decode_parallel_ms: None, + first_queued_join_wall_ms: None, + } + } +} + pub fn is_software_wgpu_adapter(info: &wgpu::AdapterInfo) -> bool { matches!(info.device_type, wgpu::DeviceType::Cpu) || info @@ -409,17 +455,23 @@ pub async fn render_video_to_channel( friction: project.cursor.friction, }); - let zoom_focus_interpolators: Vec = render_segments + let click_spring = project.cursor.click_spring_config(); + + let mut zoom_focus_interpolators: Vec = render_segments .iter() .map(|segment| { - let mut interp = ZoomFocusInterpolator::new( + ZoomFocusInterpolator::new( &segment.cursor, cursor_smoothing, + click_spring, project.screen_movement_spring, duration, - ); - interp.precompute(); - interp + project + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), + ) }) .collect(); @@ -477,6 +529,9 @@ pub async fn render_video_to_channel( let is_initial_frame = current_frame_number == 0 || last_successful_frame.is_none(); let segment_clip_index = segment.recording_clip as usize; + let zoom_until = (current_frame_number as f32 + 1.0) / fps as f32; + zoom_focus_interpolators[segment_clip_index].ensure_precomputed_until(zoom_until); + let segment_frames = if let Some((pf_num, _pf_time, pf_clip, pf_result)) = prefetched_decode.take() { if pf_num == current_frame_number && pf_clip == segment_clip_index { @@ -685,8 +740,12 @@ pub async fn render_video_to_channel_nv12( fps: u32, resolution_base: XY, recordings: &ProjectRecordingsMeta, + stop_after_frames_sent: Option, + startup_breakdown_ms: Option>>>, ) -> Result<(), RenderingError> { + let ffmpeg_init_start = Instant::now(); ffmpeg::init().unwrap(); + let ffmpeg_init_ms = ffmpeg_init_start.elapsed().as_millis() as u64; let start_time = Instant::now(); @@ -701,22 +760,31 @@ pub async fn render_video_to_channel_nv12( friction: project.cursor.friction, }); - let zoom_focus_interpolators: Vec = render_segments + let click_spring = project.cursor.click_spring_config(); + + let zoom_build_start = Instant::now(); + let mut zoom_focus_interpolators: Vec = render_segments .iter() .map(|segment| { - let mut interp = ZoomFocusInterpolator::new( + ZoomFocusInterpolator::new( &segment.cursor, cursor_smoothing, + click_spring, project.screen_movement_spring, duration, - ); - interp.precompute(); - interp + project + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), + ) }) .collect(); + let zoom_focus_interpolators_construct_ms = zoom_build_start.elapsed().as_millis() as u64; let mut frame_number = 0; + let renderer_setup_start = Instant::now(); let mut frame_renderer = FrameRenderer::new(constants); let mut layers = RendererLayers::new_with_options( @@ -736,6 +804,7 @@ pub async fn render_video_to_channel_nv12( camera_dims.map(|(_, h)| h), ); } + let frame_renderer_and_layers_setup_ms = renderer_setup_start.elapsed().as_millis() as u64; let needs_camera = !project.camera.hide; @@ -745,6 +814,11 @@ pub async fn render_video_to_channel_nv12( let mut prefetched_decode: Option<(u32, f64, usize, Option)> = None; + let mut channel_frames_sent = 0u32; + let mut stopped_after_frame_limit = false; + + let mut record_first_frame_nv12_phases = startup_breakdown_ms.is_some(); + loop { if frame_number >= total_frames { break; @@ -770,6 +844,12 @@ pub async fn render_video_to_channel_nv12( let is_initial_frame = current_frame_number == 0 || last_successful_frame.is_none(); let segment_clip_index = segment.recording_clip as usize; + let zoom_pre_start = Instant::now(); + let zoom_until = (current_frame_number as f32 + 1.0) / fps as f32; + zoom_focus_interpolators[segment_clip_index].ensure_precomputed_until(zoom_until); + let this_zoom_pre_ms = zoom_pre_start.elapsed().as_millis() as u64; + + let decode_wall_start = Instant::now(); let segment_frames = if let Some((pf_num, _pf_time, pf_clip, pf_result)) = prefetched_decode.take() { if pf_num == current_frame_number && pf_clip == segment_clip_index { @@ -796,6 +876,7 @@ pub async fn render_video_to_channel_nv12( ) .await }; + let this_decode_ms = decode_wall_start.elapsed().as_millis() as u64; if let Some(segment_frames) = segment_frames { consecutive_failures = 0; @@ -844,38 +925,131 @@ pub async fn render_video_to_channel_nv12( None }; - let render_result = if let Some(prefetch) = prefetch_future { - let (render, decoded) = tokio::join!( - frame_renderer.render_nv12( + let ( + render_result, + first_phase_render_ms, + first_phase_prefetch_ms, + first_phase_join_wall_ms, + ) = if let Some(prefetch) = prefetch_future { + if record_first_frame_nv12_phases { + let join_wall_start = Instant::now(); + let render_fut = async { + let t0 = Instant::now(); + let r = frame_renderer + .render_nv12( + segment_frames, + uniforms, + &render_segment.cursor, + &mut layers, + ) + .await; + (t0.elapsed(), r) + }; + let prefetch_fut = async { + let t0 = Instant::now(); + let d = prefetch.await; + (t0.elapsed(), d) + }; + let ((render_elapsed, render), (prefetch_elapsed, decoded)) = + tokio::join!(render_fut, prefetch_fut); + if let Some((next_seg_time, next_clip_index)) = next_prefetch_meta { + prefetched_decode = + Some((next_frame_number, next_seg_time, next_clip_index, decoded)); + } + ( + render, + Some(render_elapsed.as_millis() as u64), + Some(prefetch_elapsed.as_millis() as u64), + Some(join_wall_start.elapsed().as_millis() as u64), + ) + } else { + let (render, decoded) = tokio::join!( + frame_renderer.render_nv12( + segment_frames, + uniforms, + &render_segment.cursor, + &mut layers, + ), + prefetch + ); + + if let Some((next_seg_time, next_clip_index)) = next_prefetch_meta { + prefetched_decode = + Some((next_frame_number, next_seg_time, next_clip_index, decoded)); + } + + (render, None, None, None) + } + } else if record_first_frame_nv12_phases { + let render_start = Instant::now(); + let render = frame_renderer + .render_nv12( segment_frames, uniforms, &render_segment.cursor, &mut layers, - ), - prefetch - ); - - if let Some((next_seg_time, next_clip_index)) = next_prefetch_meta { - prefetched_decode = - Some((next_frame_number, next_seg_time, next_clip_index, decoded)); - } - - render + ) + .await; + ( + render, + Some(render_start.elapsed().as_millis() as u64), + None, + None, + ) } else { - frame_renderer + let render = frame_renderer .render_nv12( segment_frames, uniforms, &render_segment.cursor, &mut layers, ) - .await + .await; + (render, None, None, None) }; + if current_frame_number == 0 + && let Some(ref slot) = startup_breakdown_ms + && let Ok(mut guard) = slot.lock() + { + let b = guard.get_or_insert(Nv12RenderStartupBreakdownMs::new_header( + ffmpeg_init_ms, + zoom_focus_interpolators_construct_ms, + frame_renderer_and_layers_setup_ms, + )); + b.frame_index_zero_zoom_precompute_ms = Some(this_zoom_pre_ms); + b.frame_index_zero_decode_ms = Some(this_decode_ms); + b.frame_index_zero_render_nv12_ms = first_phase_render_ms; + b.frame_index_zero_prefetch_decode_parallel_ms = first_phase_prefetch_ms; + b.frame_index_zero_join_wall_ms = first_phase_join_wall_ms; + } + match render_result { Ok(Some(frame)) if frame.width > 0 && frame.height > 0 => { + if record_first_frame_nv12_phases { + if let Some(ref slot) = startup_breakdown_ms + && let Ok(mut guard) = slot.lock() + { + let b = guard.get_or_insert(Nv12RenderStartupBreakdownMs::new_header( + ffmpeg_init_ms, + zoom_focus_interpolators_construct_ms, + frame_renderer_and_layers_setup_ms, + )); + b.first_queued_zoom_precompute_ms = Some(this_zoom_pre_ms); + b.first_queued_decode_ms = Some(this_decode_ms); + b.first_queued_render_nv12_ms = first_phase_render_ms; + b.first_queued_prefetch_decode_parallel_ms = first_phase_prefetch_ms; + b.first_queued_join_wall_ms = first_phase_join_wall_ms; + } + record_first_frame_nv12_phases = false; + } last_successful_frame = Some(frame.clone_metadata_with_data()); sender.send((frame, current_frame_number)).await?; + channel_frames_sent += 1; + if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) { + stopped_after_frame_limit = true; + break; + } } Ok(Some(_)) => { tracing::warn!( @@ -888,6 +1062,11 @@ pub async fn render_video_to_channel_nv12( fallback.target_time_ns = (current_frame_number as u64 * 1_000_000_000) / fps as u64; sender.send((fallback, current_frame_number)).await?; + channel_frames_sent += 1; + if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) { + stopped_after_frame_limit = true; + break; + } } } Ok(None) => {} @@ -903,6 +1082,11 @@ pub async fn render_video_to_channel_nv12( fallback.target_time_ns = (current_frame_number as u64 * 1_000_000_000) / fps as u64; sender.send((fallback, current_frame_number)).await?; + channel_frames_sent += 1; + if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) { + stopped_after_frame_limit = true; + break; + } } else { return Err(e); } @@ -936,6 +1120,11 @@ pub async fn render_video_to_channel_nv12( fallback.target_time_ns = (current_frame_number as u64 * 1_000_000_000) / fps as u64; sender.send((fallback, current_frame_number)).await?; + channel_frames_sent += 1; + if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) { + stopped_after_frame_limit = true; + break; + } } else { tracing::error!( frame_number = current_frame_number, @@ -948,7 +1137,8 @@ pub async fn render_video_to_channel_nv12( } } - if let Some(Ok(final_frame)) = frame_renderer.flush_pipeline_nv12().await + if !stopped_after_frame_limit + && let Some(Ok(final_frame)) = frame_renderer.flush_pipeline_nv12().await && final_frame.width > 0 && final_frame.height > 0 { @@ -1225,6 +1415,7 @@ impl RenderVideoConstants { pub struct ProjectUniforms { pub output_size: (u32, u32), pub cursor_size: f32, + pub cursor_x_axis_tilt_radians: f32, pub frame_rate: u32, pub frame_number: u32, display: CompositeVideoFrameUniforms, @@ -1426,8 +1617,8 @@ fn resolve_motion_descriptor( let zoom_metric = analysis.zoom_magnitude; let move_metric = analysis.movement_magnitude; - let zoom_strength = (base_amount * zoom_multiplier).min(2.0); - let move_strength = (base_amount * move_multiplier).min(2.0); + let zoom_strength = base_amount * zoom_multiplier; + let move_strength = base_amount * move_multiplier; if zoom_metric > move_metric && zoom_metric > MOTION_MIN_THRESHOLD && zoom_strength > 0.0 { let zoom_amount = (zoom_metric * zoom_strength).min(MAX_ZOOM_AMOUNT); @@ -1457,10 +1648,10 @@ const SCREEN_MAX_PADDING: f64 = 0.4; const MOTION_BLUR_BASELINE_FPS: f32 = 60.0; const MOTION_MIN_THRESHOLD: f32 = 0.003; -const MOTION_VECTOR_CAP: f32 = 0.85; -const MAX_ZOOM_AMOUNT: f32 = 0.9; -const DISPLAY_MOVE_MULTIPLIER: f32 = 0.6; -const DISPLAY_ZOOM_MULTIPLIER: f32 = 0.45; +const MOTION_VECTOR_CAP: f32 = 2.0; +const MAX_ZOOM_AMOUNT: f32 = 2.0; +const DISPLAY_MOVE_MULTIPLIER: f32 = 1.0; +const DISPLAY_ZOOM_MULTIPLIER: f32 = 1.0; const CAMERA_MULTIPLIER: f32 = 1.0; const CAMERA_ONLY_MULTIPLIER: f32 = 0.45; @@ -1914,11 +2105,49 @@ impl ProjectUniforms { friction: project.cursor.friction, }); - let interpolated_cursor = - interpolate_cursor(cursor_events, cursor_time_for_interp, cursor_smoothing); + let click_spring_cfg = project.cursor.click_spring_config(); - let prev_interpolated_cursor = - interpolate_cursor(cursor_events, prev_cursor_time_for_interp, cursor_smoothing); + let interpolated_cursor = match cursor_smoothing { + Some(cfg) => interpolate_cursor_with_click_spring( + cursor_events, + cursor_time_for_interp, + Some(cfg), + Some(click_spring_cfg), + ), + None => interpolate_cursor(cursor_events, cursor_time_for_interp, None), + }; + + let prev_interpolated_cursor = match cursor_smoothing { + Some(cfg) => interpolate_cursor_with_click_spring( + cursor_events, + prev_cursor_time_for_interp, + Some(cfg), + Some(click_spring_cfg), + ), + None => interpolate_cursor(cursor_events, prev_cursor_time_for_interp, None), + }; + + let lookback_t = (cursor_time_for_interp - 0.4).max(0.0); + let past_cursor_for_tilt = match cursor_smoothing { + Some(cfg) => interpolate_cursor_with_click_spring( + cursor_events, + lookback_t, + Some(cfg), + Some(click_spring_cfg), + ), + None => interpolate_cursor(cursor_events, lookback_t, None), + }; + + let cursor_x_axis_tilt_radians = + if let (Some(cur), Some(past)) = (&interpolated_cursor, past_cursor_for_tilt) { + let delta_x_norm = cur.position.coord.x - past.position.coord.x; + let delta_x_px = delta_x_norm * resolution_base.x as f64; + let deg = + (delta_x_px * 0.03 * project.cursor.rotation_amount as f64).clamp(-20.0, 20.0); + deg.to_radians() as f32 + } else { + 0.0 + }; let zoom_segments = project .timeline @@ -1932,9 +2161,22 @@ impl ProjectUniforms { .map(|t| t.scene_segments.as_slice()) .unwrap_or(&[]); - let zoom_focus = zoom_focus_interpolator.interpolate(current_recording_time); - - let prev_zoom_focus = zoom_focus_interpolator.interpolate(prev_recording_time); + let segments_cursor = SegmentsCursor::new(frame_time as f64, zoom_segments); + let prev_segments_cursor = SegmentsCursor::new(prev_frame_time as f64, zoom_segments); + let recording_time_for_zoom_focus_interpolate = segments_cursor + .segment + .filter(|s| matches!(s.mode, cap_project::ZoomMode::Auto)) + .map(|s| current_recording_time.min(s.end as f32)) + .unwrap_or(current_recording_time); + let prev_recording_time_for_zoom_focus_interpolate = prev_segments_cursor + .segment + .filter(|s| matches!(s.mode, cap_project::ZoomMode::Auto)) + .map(|s| prev_recording_time.min(s.end as f32)) + .unwrap_or(prev_recording_time); + let zoom_focus = + zoom_focus_interpolator.interpolate(recording_time_for_zoom_focus_interpolate); + let prev_zoom_focus = + zoom_focus_interpolator.interpolate(prev_recording_time_for_zoom_focus_interpolate); let actual_cursor_coord = interpolated_cursor .as_ref() @@ -1944,16 +2186,76 @@ impl ProjectUniforms { .as_ref() .map(|c| Coord::::new(c.position.coord)); - let zoom = InterpolatedZoom::new_with_cursor( - SegmentsCursor::new(frame_time as f64, zoom_segments), + let segment_end_focus = segments_cursor + .prev_segment + .filter(|_| segments_cursor.segment.is_none()) + .map(|prev| { + let boundary_recording_time = (current_recording_time as f64 + - (frame_time as f64 - prev.end)) + .clamp(0.0, prev.end) as f32; + zoom_focus_interpolator.interpolate(boundary_recording_time) + }); + let segment_end_cursor = segments_cursor + .prev_segment + .filter(|_| segments_cursor.segment.is_none()) + .and_then(|prev| { + let boundary_recording_time = (current_recording_time as f64 + - (frame_time as f64 - prev.end)) + .clamp(0.0, prev.end) as f32; + match cursor_smoothing { + Some(cfg) => interpolate_cursor_with_click_spring( + cursor_events, + boundary_recording_time, + Some(cfg), + Some(click_spring_cfg), + ), + None => interpolate_cursor(cursor_events, boundary_recording_time, None), + } + }) + .map(|c| Coord::::new(c.position.coord)); + + let zoom = InterpolatedZoom::new_with_cursor_and_end_focus( + segments_cursor, zoom_focus, actual_cursor_coord, + segment_end_focus, + segment_end_cursor, ); - let prev_zoom = InterpolatedZoom::new_with_cursor( - SegmentsCursor::new(prev_frame_time as f64, zoom_segments), + let prev_segment_end_focus = prev_segments_cursor + .prev_segment + .filter(|_| prev_segments_cursor.segment.is_none()) + .map(|prev| { + let boundary_recording_time = (prev_recording_time as f64 + - (prev_frame_time as f64 - prev.end)) + .clamp(0.0, prev.end) as f32; + zoom_focus_interpolator.interpolate(boundary_recording_time) + }); + let prev_segment_end_cursor = prev_segments_cursor + .prev_segment + .filter(|_| prev_segments_cursor.segment.is_none()) + .and_then(|prev| { + let boundary_recording_time = (prev_recording_time as f64 + - (prev_frame_time as f64 - prev.end)) + .clamp(0.0, prev.end) as f32; + match cursor_smoothing { + Some(cfg) => interpolate_cursor_with_click_spring( + cursor_events, + boundary_recording_time, + Some(cfg), + Some(click_spring_cfg), + ), + None => interpolate_cursor(cursor_events, boundary_recording_time, None), + } + }) + .map(|c| Coord::::new(c.position.coord)); + + let prev_zoom = InterpolatedZoom::new_with_cursor_and_end_focus( + prev_segments_cursor, prev_zoom_focus, prev_actual_cursor_coord, + prev_segment_end_focus, + prev_segment_end_cursor, ); let scene = @@ -2333,6 +2635,7 @@ impl ProjectUniforms { Self { output_size, cursor_size: project.cursor.size as f32, + cursor_x_axis_tilt_radians, resolution_base, display, camera, diff --git a/crates/rendering/src/shaders/composite-video-frame.wgsl b/crates/rendering/src/shaders/composite-video-frame.wgsl index f72302e61b..398a558626 100644 --- a/crates/rendering/src/shaders/composite-video-frame.wgsl +++ b/crates/rendering/src/shaders/composite-video-frame.wgsl @@ -158,23 +158,18 @@ fn fs_main(@builtin(position) frag_coord: vec4) -> @location(0) vec4 { return mix(shadow_color, base_color, base_color.a); } - let direction = motion_vec / motion_len; - let stroke = min(motion_len, 0.35); - let num_samples = i32(clamp(6.0 + 24.0 * blur_strength, 6.0, 36.0)); - - for (var i = 1; i < num_samples; i = i + 1) { - let t = f32(i) / f32(num_samples); - let eased = smoothstep(0.0, 1.0, t); - let offset = direction * stroke * eased; - let jitter_seed = target_uv + vec2(t, f32(i) * 0.37); - let jitter = (rand(jitter_seed) - 0.5) * stroke * 0.15; - let sample_uv = target_uv - offset + direction * jitter; + let velocity_uv = motion_vec; + let offset_base = -0.5; + let k = 20; + + for (var i = 0; i < 20; i = i + 1) { + let bias = velocity_uv * (f32(i) / f32(k) + offset_base); + let sample_uv = target_uv + bias; if sample_uv.x >= 0.0 && sample_uv.x <= 1.0 && sample_uv.y >= 0.0 && sample_uv.y <= 1.0 { var sample_color = sample_texture(sample_uv, crop_bounds_uv); sample_color = apply_rounded_corners(sample_color, sample_uv); - let weight = 1.0 - t * 0.8; - let sample_weight = weight * sample_color.a; + let sample_weight = sample_color.a; if sample_weight > 1e-6 { accum += sample_color * sample_weight; weight_sum += sample_weight; @@ -183,27 +178,25 @@ fn fs_main(@builtin(position) frag_coord: vec4) -> @location(0) vec4 { } } else { let center = uniforms.motion_blur_zoom_center; - let to_center = target_uv - center; - let dist = length(to_center); + let dir = center - target_uv; + let dist = length(dir); if dist < 1e-4 || zoom_amount < 1e-4 { return mix(shadow_color, base_color, base_color.a); } - let radial_dir = to_center / dist; - let sample_span = zoom_amount * 1.2; - let num_samples = i32(clamp(8.0 + 26.0 * blur_strength, 8.0, 40.0)); + let scaled_dir = dir * blur_strength; + let max_kernel = 13.0; + + let offset_rand = rand(vec2(target_uv.x * 7.37, target_uv.y * 11.23)); - for (var i = 1; i < num_samples; i = i + 1) { - let t = f32(i) / f32(num_samples); - let offset = radial_dir * sample_span * t; - let jitter_seed = vec2(t, target_uv.x + target_uv.y); - let jitter = (rand(jitter_seed) - 0.5) * zoom_amount * 0.1; - let sample_uv = target_uv - offset + radial_dir * jitter; + for (var i = 0; i < 13; i = i + 1) { + let percent = (f32(i) + offset_rand) / max_kernel; + let weight = 4.0 * (percent - percent * percent); + let sample_uv = target_uv + scaled_dir * percent; if sample_uv.x >= 0.0 && sample_uv.x <= 1.0 && sample_uv.y >= 0.0 && sample_uv.y <= 1.0 { var sample_color = sample_texture(sample_uv, crop_bounds_uv); sample_color = apply_rounded_corners(sample_color, sample_uv); - let weight = 1.0 - t * 0.9; let sample_weight = weight * sample_color.a; if sample_weight > 1e-6 { accum += sample_color * sample_weight; diff --git a/crates/rendering/src/shaders/cursor.wgsl b/crates/rendering/src/shaders/cursor.wgsl index 1e880b27b9..34e5103e67 100644 --- a/crates/rendering/src/shaders/cursor.wgsl +++ b/crates/rendering/src/shaders/cursor.wgsl @@ -20,8 +20,7 @@ var t_cursor: texture_2d; @group(0) @binding(2) var s_cursor: sampler; -const MAX_ROTATION_RADIANS: f32 = 0.25; -const ROTATION_VELOCITY_SCALE: f32 = 0.003; +const MAX_ROTATION_RADIANS: f32 = 0.34906584; fn rotate_point(p: vec2, center: vec2, angle: f32) -> vec2 { let cos_a = cos(angle); @@ -53,13 +52,11 @@ fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { let screen_pos = uniforms.position_size.xy; let cursor_size = uniforms.position_size.zw; - let rotation_amount = uniforms.rotation_params.x; let base_rotation = uniforms.rotation_params.y; + let x_movement_tilt = uniforms.rotation_params.z; - let motion_x = uniforms.motion_vector_strength.x; - let normalized_velocity = clamp(motion_x * ROTATION_VELOCITY_SCALE, -1.0, 1.0); - let velocity_rotation = normalized_velocity * MAX_ROTATION_RADIANS * rotation_amount; - let rotation_angle = velocity_rotation + base_rotation; + let clamped_tilt = clamp(x_movement_tilt, -MAX_ROTATION_RADIANS, MAX_ROTATION_RADIANS); + let rotation_angle = clamped_tilt + base_rotation; let pivot = vec2(0.0, 0.0); let rotated_pos = rotate_point(pos, pivot, rotation_angle); @@ -87,50 +84,26 @@ fn fs_main(input: VertexOutput) -> @location(0) vec4 { } let cursor_size = uniforms.position_size.zw; - let blur_offset_uv = motion_vec * 0.45 / cursor_size; - let blur_len = length(blur_offset_uv); + let velocity_uv = motion_vec / cursor_size; + let vel_len = length(velocity_uv); - if (blur_len < 0.005) { + if (vel_len < 0.005) { return textureSample(t_cursor, s_cursor, input.uv) * opacity; } - let num_samples = 24; - var color_sum = vec4(0.0); - var alpha_sum = 0.0; - var weight_sum = 0.0; + let kernel_size = 21; + let k = kernel_size - 1; + let offset_base = -vel_len / 2.0 / vel_len - 0.5; - let blur_center = 0.3; - let blur_spread = 2.5; + var color = textureSample(t_cursor, s_cursor, input.uv); - for (var i = 0; i < num_samples; i++) { - let t = f32(i) / f32(num_samples - 1); - let centered_t = t - blur_center; - let sample_offset = blur_offset_uv * centered_t; - let sample_uv = input.uv + sample_offset; - - let gauss_t = centered_t * blur_spread; - var weight = exp(-gauss_t * gauss_t); - - if (centered_t > 0.0) { - weight *= 1.0 + centered_t * 0.3; - } - - let sample_color = textureSample(t_cursor, s_cursor, sample_uv); - let premul_rgb = sample_color.rgb * sample_color.a; - color_sum += vec4(premul_rgb * weight, 0.0); - alpha_sum += sample_color.a * weight; - weight_sum += weight; - } - - let avg_alpha = alpha_sum / max(weight_sum, 0.001); - var final_color: vec4; - if (avg_alpha > 0.001) { - let final_rgb = color_sum.rgb / max(alpha_sum, 0.001); - final_color = vec4(final_rgb, avg_alpha); - } else { - final_color = vec4(0.0); + for (var i = 0; i < 20; i++) { + let bias = velocity_uv * (f32(i) / f32(k) + offset_base); + let sample_uv = input.uv + bias; + color += textureSample(t_cursor, s_cursor, sample_uv); } - final_color *= opacity; - return final_color; + color = color / f32(kernel_size); + color *= opacity; + return color; } diff --git a/crates/rendering/src/zoom.rs b/crates/rendering/src/zoom.rs index 73582bdcde..575c0d8eaf 100644 --- a/crates/rendering/src/zoom.rs +++ b/crates/rendering/src/zoom.rs @@ -11,8 +11,8 @@ const SCREEN_SPRING_MASS: f64 = 2.25; #[derive(Debug, Clone, Copy)] pub struct SegmentsCursor<'a> { time: f64, - segment: Option<&'a ZoomSegment>, - prev_segment: Option<&'a ZoomSegment>, + pub segment: Option<&'a ZoomSegment>, + pub prev_segment: Option<&'a ZoomSegment>, segments: &'a [ZoomSegment], } @@ -106,19 +106,35 @@ impl SegmentBounds { ) } + fn snap_to_edges(scalar: f64, snap_ratio: f64) -> f64 { + if snap_ratio <= 0.0 { + return scalar; + } + let lo = snap_ratio; + let hi = 1.0 - snap_ratio + 0.0001; + if hi <= lo { + return 0.5; + } + ((scalar - lo) / (hi - lo)).clamp(0.0, 1.0) + } + fn calculate_follow_center( cursor_pos: (f64, f64), zoom_amount: f64, - _edge_snap_ratio: f64, + edge_snap_ratio: f64, ) -> (f64, f64) { - let viewport_half = 0.5 / zoom_amount; + let snapped = ( + Self::snap_to_edges(cursor_pos.0, edge_snap_ratio), + Self::snap_to_edges(cursor_pos.1, edge_snap_ratio), + ); + let viewport_half = 0.5 / zoom_amount; let min_center = viewport_half; let max_center = 1.0 - viewport_half; ( - cursor_pos.0.clamp(min_center, max_center), - cursor_pos.1.clamp(min_center, max_center), + (min_center + snapped.0 * (max_center - min_center)).clamp(min_center, max_center), + (min_center + snapped.1 * (max_center - min_center)).clamp(min_center, max_center), ) } @@ -126,15 +142,22 @@ impl SegmentBounds { focus_pos: (f64, f64), cursor_pos: (f64, f64), target_zoom: f64, - _edge_snap_ratio: f64, + edge_snap_ratio: f64, ) -> (f64, (f64, f64)) { + let snapped_focus = ( + Self::snap_to_edges(focus_pos.0, edge_snap_ratio), + Self::snap_to_edges(focus_pos.1, edge_snap_ratio), + ); + let viewport_half = 0.5 / target_zoom; let min_center = viewport_half; let max_center = 1.0 - viewport_half; let mut center = ( - focus_pos.0.clamp(min_center, max_center), - focus_pos.1.clamp(min_center, max_center), + (min_center + snapped_focus.0 * (max_center - min_center)) + .clamp(min_center, max_center), + (min_center + snapped_focus.1 * (max_center - min_center)) + .clamp(min_center, max_center), ); let viewport_left = center.0 - viewport_half; @@ -278,6 +301,16 @@ impl InterpolatedZoom { cursor: SegmentsCursor, zoom_focus: Coord, actual_cursor: Option>, + ) -> Self { + Self::new_with_cursor_and_end_focus(cursor, zoom_focus, actual_cursor, None, None) + } + + pub fn new_with_cursor_and_end_focus( + cursor: SegmentsCursor, + zoom_focus: Coord, + actual_cursor: Option>, + segment_end_focus: Option>, + segment_end_cursor: Option>, ) -> Self { let use_instant = cursor.segment.map(|s| s.instant_animation).unwrap_or(false); if use_instant { @@ -285,6 +318,8 @@ impl InterpolatedZoom { cursor, zoom_focus, actual_cursor, + segment_end_focus, + segment_end_cursor, instant_ease, instant_ease, ) @@ -293,6 +328,8 @@ impl InterpolatedZoom { cursor, zoom_focus, actual_cursor, + segment_end_focus, + segment_end_cursor, spring_ease, spring_ease_out, ) @@ -307,6 +344,8 @@ impl InterpolatedZoom { cursor: SegmentsCursor, zoom_focus: Coord, actual_cursor: Option>, + segment_end_focus: Option>, + segment_end_cursor: Option>, ease_in: impl Fn(f32) -> f32 + Copy, ease_out: impl Fn(f32) -> f32 + Copy, ) -> InterpolatedZoom { @@ -319,27 +358,54 @@ impl InterpolatedZoom { let result = match (cursor.prev_segment, cursor.segment) { (Some(prev_segment), None) => { - let zoom_t = - ease_out(t_clamp((cursor.time - prev_segment.end) / ZOOM_DURATION) as f32) - as f64; - - Self { - t: 1.0 - zoom_t, - bounds: { - let prev_segment_bounds = - SegmentBounds::from_segment_with_cursor_constraint( - prev_segment, - zoom_focus, - actual_cursor, - ); + let raw_t = t_clamp((cursor.time - prev_segment.end) / ZOOM_DURATION); + let ease_val = ease_out(raw_t as f32) as f64; + + let ramp = (raw_t * 5.0).min(1.0); + let ramp_smooth = ramp * ramp * (3.0 - 2.0 * ramp); + let zoom_t = ease_val * ramp_smooth; + + let eased_amount = prev_segment.amount * (1.0 - zoom_t) + 1.0 * zoom_t; + + if eased_amount > 1.001 { + let eased_segment = ZoomSegment { + amount: eased_amount, + ..prev_segment.clone() + }; + let is_auto_zoom_out = matches!(prev_segment.mode, cap_project::ZoomMode::Auto); + let focus_for_bounds = if is_auto_zoom_out { + segment_end_focus.unwrap_or(zoom_focus) + } else { + zoom_focus + }; + let cursor_for_bounds = if is_auto_zoom_out { + segment_end_cursor.or(actual_cursor) + } else { + actual_cursor + }; + + let bounds = SegmentBounds::from_segment_with_cursor_constraint( + &eased_segment, + focus_for_bounds, + cursor_for_bounds, + ); + let bounds = if let Some(cursor_coord) = cursor_for_bounds { + Self { t: 1.0, bounds } + .ensure_cursor_visible((cursor_coord.x, cursor_coord.y)) + .bounds + } else { + bounds + }; - SegmentBounds::new( - prev_segment_bounds.top_left * (1.0 - zoom_t) - + default.top_left * zoom_t, - prev_segment_bounds.bottom_right * (1.0 - zoom_t) - + default.bottom_right * zoom_t, - ) - }, + Self { + t: 1.0 - zoom_t, + bounds, + } + } else { + Self { + t: 0.0, + bounds: default, + } } } (None, Some(segment)) => { @@ -392,6 +458,8 @@ impl InterpolatedZoom { SegmentsCursor::new(segment.start, cursor.segments), zoom_focus, actual_cursor, + segment_end_focus, + segment_end_cursor, ease_in, ease_out, ); @@ -425,7 +493,11 @@ impl InterpolatedZoom { }, }; - if is_auto_mode && let Some(cursor_coord) = actual_cursor { + let is_zooming_out = cursor.prev_segment.is_some() && cursor.segment.is_none(); + if is_auto_mode + && !is_zooming_out + && let Some(cursor_coord) = actual_cursor + { return result.ensure_cursor_visible((cursor_coord.x, cursor_coord.y)); } @@ -589,6 +661,8 @@ mod test { c(time, segments), Default::default(), None, + None, + None, |t| t, |t| t, ); @@ -920,4 +994,79 @@ mod test { "Cursor should be visible in interpolated zoom state" ); } + + #[test] + fn zoom_out_boundary_is_continuous() { + let segments = vec![test_segment(2.0, 4.0, 2.0, 0.5, 0.5)]; + + let at_boundary = InterpolatedZoom::new_with_easing_and_cursor( + c(4.0, &segments), + Default::default(), + None, + None, + None, + |t| t, + |t| t, + ); + + let epsilon = 1e-9; + let just_after = InterpolatedZoom::new_with_easing_and_cursor( + c(4.0 + epsilon, &segments), + Default::default(), + None, + None, + None, + |t| t, + |t| t, + ); + + let dx_tl = (just_after.bounds.top_left.x - at_boundary.bounds.top_left.x).abs(); + let dy_tl = (just_after.bounds.top_left.y - at_boundary.bounds.top_left.y).abs(); + let dx_br = (just_after.bounds.bottom_right.x - at_boundary.bounds.bottom_right.x).abs(); + let dy_br = (just_after.bounds.bottom_right.y - at_boundary.bounds.bottom_right.y).abs(); + + let max_jump = dx_tl.max(dy_tl).max(dx_br).max(dy_br); + assert!( + max_jump < 1e-4, + "Bounds jumped {max_jump} at segment boundary; expected near-zero" + ); + } + + #[test] + fn zoom_out_early_frames_are_smooth() { + let segments = vec![test_segment(2.0, 4.0, 2.0, 0.5, 0.5)]; + + let dt = ZOOM_DURATION * 0.01; + let mut prev = InterpolatedZoom::new_with_easing_and_cursor( + c(4.0, &segments), + Default::default(), + None, + None, + None, + |t| t, + |t| t, + ); + + for i in 1..=20 { + let time = 4.0 + dt * i as f64; + let current = InterpolatedZoom::new_with_easing_and_cursor( + c(time, &segments), + Default::default(), + None, + None, + None, + |t| t, + |t| t, + ); + + let dx = (current.bounds.top_left.x - prev.bounds.top_left.x).abs(); + let dy = (current.bounds.top_left.y - prev.bounds.top_left.y).abs(); + assert!( + dx < 0.02 && dy < 0.02, + "Frame jump at step {i} (t={time:.4}): dx={dx:.6} dy={dy:.6}" + ); + + prev = current; + } + } } diff --git a/crates/rendering/src/zoom_focus_interpolation.rs b/crates/rendering/src/zoom_focus_interpolation.rs index c698026254..e354d4d127 100644 --- a/crates/rendering/src/zoom_focus_interpolation.rs +++ b/crates/rendering/src/zoom_focus_interpolation.rs @@ -1,12 +1,21 @@ -use cap_project::{CursorEvents, ScreenMovementSpring, XY, ZoomSegment}; +use cap_project::{ClickSpringConfig, CursorEvents, ScreenMovementSpring, XY, ZoomSegment}; use crate::{ Coord, RawDisplayUVSpace, - cursor_interpolation::interpolate_cursor, + cursor_interpolation::{ + InterpolatedCursorPosition, interpolate_cursor, interpolate_cursor_with_click_spring, + }, spring_mass_damper::{SpringMassDamperSimulation, SpringMassDamperSimulationConfig}, }; +struct ZoomFocusPrecomputeSim { + sim: SpringMassDamperSimulation, + last_integrated_ms: f64, +} + const SAMPLE_INTERVAL_MS: f64 = 8.0; +const CLUSTER_WIDTH_RATIO: f64 = 0.5; +const CLUSTER_HEIGHT_RATIO: f64 = 0.7; #[derive(Clone)] struct SmoothedFocusEvent { @@ -14,106 +23,296 @@ struct SmoothedFocusEvent { position: XY, } +struct ClickCluster { + min_x: f64, + max_x: f64, + min_y: f64, + max_y: f64, + start_time_ms: f64, +} + +impl ClickCluster { + fn new(x: f64, y: f64, time_ms: f64) -> Self { + Self { + min_x: x, + max_x: x, + min_y: y, + max_y: y, + start_time_ms: time_ms, + } + } + + fn can_add(&self, x: f64, y: f64, max_w: f64, max_h: f64) -> bool { + let new_w = self.max_x.max(x) - self.min_x.min(x); + let new_h = self.max_y.max(y) - self.min_y.min(y); + new_w <= max_w && new_h <= max_h + } + + fn add(&mut self, x: f64, y: f64) { + self.min_x = self.min_x.min(x); + self.max_x = self.max_x.max(x); + self.min_y = self.min_y.min(y); + self.max_y = self.max_y.max(y); + } + + fn center(&self) -> (f64, f64) { + ( + (self.min_x + self.max_x) / 2.0, + (self.min_y + self.max_y) / 2.0, + ) + } +} + +fn build_clusters( + cursor_events: &CursorEvents, + segment_start_secs: f64, + segment_end_secs: f64, + zoom_amount: f64, +) -> Vec { + let start_ms = segment_start_secs * 1000.0; + let end_ms = segment_end_secs * 1000.0; + let cluster_w = CLUSTER_WIDTH_RATIO / zoom_amount; + let cluster_h = CLUSTER_HEIGHT_RATIO / zoom_amount; + + let events_in_range: Vec<&cap_project::CursorMoveEvent> = cursor_events + .moves + .iter() + .filter(|m| m.time_ms >= start_ms && m.time_ms <= end_ms) + .collect(); + + if events_in_range.is_empty() { + let fallback = cursor_events + .moves + .iter() + .rev() + .find(|m| m.time_ms <= start_ms) + .or_else(|| cursor_events.moves.iter().find(|m| m.time_ms >= start_ms)); + + if let Some(evt) = fallback { + return vec![ClickCluster::new(evt.x, evt.y, evt.time_ms)]; + } + return vec![]; + } + + let mut clusters = Vec::new(); + let first = events_in_range[0]; + let mut current = ClickCluster::new(first.x, first.y, first.time_ms); + + for evt in &events_in_range[1..] { + if current.can_add(evt.x, evt.y, cluster_w, cluster_h) { + current.add(evt.x, evt.y); + } else { + clusters.push(current); + current = ClickCluster::new(evt.x, evt.y, evt.time_ms); + } + } + clusters.push(current); + + clusters +} + +fn cluster_center_at_time(clusters: &[ClickCluster], time_ms: f64) -> Option<(f64, f64)> { + clusters + .iter() + .rev() + .find(|c| c.start_time_ms <= time_ms) + .or_else(|| clusters.first()) + .map(|c| c.center()) +} + +struct SegmentClusters { + start_secs: f64, + end_secs: f64, + clusters: Vec, +} + pub struct ZoomFocusInterpolator { events: Option>, + precompute_sim: Option, cursor_events: std::sync::Arc, cursor_smoothing: Option, + click_spring: ClickSpringConfig, screen_spring: ScreenMovementSpring, duration_secs: f64, + segment_clusters: Vec, } impl ZoomFocusInterpolator { pub fn new( cursor_events: &CursorEvents, cursor_smoothing: Option, + click_spring: ClickSpringConfig, screen_spring: ScreenMovementSpring, duration_secs: f64, + zoom_segments: &[ZoomSegment], ) -> Self { + let segment_clusters = Self::build_segment_clusters(cursor_events, zoom_segments); Self { events: None, + precompute_sim: None, cursor_events: std::sync::Arc::new(cursor_events.clone()), cursor_smoothing, + click_spring, screen_spring, duration_secs, + segment_clusters, } } pub fn new_arc( cursor_events: std::sync::Arc, cursor_smoothing: Option, + click_spring: ClickSpringConfig, screen_spring: ScreenMovementSpring, duration_secs: f64, + zoom_segments: &[ZoomSegment], ) -> Self { + let segment_clusters = Self::build_segment_clusters(cursor_events.as_ref(), zoom_segments); Self { events: None, + precompute_sim: None, cursor_events, cursor_smoothing, + click_spring, screen_spring, duration_secs, + segment_clusters, } } - pub fn precompute(&mut self) { - if self.events.is_some() { - return; - } - - if self.cursor_events.moves.is_empty() { - self.events = Some(vec![]); - return; - } - - let spring_config = SpringMassDamperSimulationConfig { - tension: self.screen_spring.stiffness, - mass: self.screen_spring.mass, - friction: self.screen_spring.damping, - }; + fn build_segment_clusters( + cursor_events: &CursorEvents, + zoom_segments: &[ZoomSegment], + ) -> Vec { + zoom_segments + .iter() + .filter(|s| matches!(s.mode, cap_project::ZoomMode::Auto)) + .map(|s| SegmentClusters { + start_secs: s.start, + end_secs: s.end, + clusters: build_clusters(cursor_events, s.start, s.end, s.amount), + }) + .collect() + } - let mut sim = SpringMassDamperSimulation::new(spring_config); + fn cluster_focus_at(&self, time_secs: f64) -> Option<(f64, f64)> { + let time_ms = time_secs * 1000.0; + self.segment_clusters + .iter() + .find(|sc| time_secs >= sc.start_secs && time_secs <= sc.end_secs) + .and_then(|sc| cluster_center_at_time(&sc.clusters, time_ms)) + } - let first_cursor = interpolate_cursor(&self.cursor_events, 0.0, self.cursor_smoothing); - let initial_pos = first_cursor - .map(|c| XY::new(c.position.coord.x as f32, c.position.coord.y as f32)) - .unwrap_or(XY::new(0.5, 0.5)); + fn interpolate_cursor_at(&self, time_secs: f32) -> Option { + match self.cursor_smoothing { + Some(cfg) => interpolate_cursor_with_click_spring( + self.cursor_events.as_ref(), + time_secs, + Some(cfg), + Some(self.click_spring), + ), + None => interpolate_cursor(self.cursor_events.as_ref(), time_secs, None), + } + } - sim.set_position(initial_pos); - sim.set_velocity(XY::new(0.0, 0.0)); - sim.set_target_position(initial_pos); + fn focus_target_at(&self, time_secs: f32) -> XY { + if let Some((cx, cy)) = self.cluster_focus_at(time_secs as f64) { + return XY::new(cx as f32, cy as f32); + } - let mut events = vec![SmoothedFocusEvent { - time: 0.0, - position: initial_pos, - }]; + if let Some(cursor) = self.interpolate_cursor_at(time_secs) { + XY::new( + cursor.position.coord.x as f32, + cursor.position.coord.y as f32, + ) + } else { + XY::new(0.5, 0.5) + } + } + pub fn ensure_precomputed_until(&mut self, time_secs: f32) { let duration_ms = self.duration_secs * 1000.0; - let mut current_time_ms = 0.0; - - while current_time_ms < duration_ms { - current_time_ms += SAMPLE_INTERVAL_MS; - let time_secs = (current_time_ms / 1000.0) as f32; - - if let Some(cursor) = - interpolate_cursor(&self.cursor_events, time_secs, self.cursor_smoothing) - { - let target = XY::new( - cursor.position.coord.x as f32, - cursor.position.coord.y as f32, - ); - sim.set_target_position(target); + let need_ms = (f64::from(time_secs) * 1000.0).clamp(0.0, duration_ms); + + if self.cursor_events.moves.is_empty() { + if self.events.is_none() { + self.events = Some(vec![]); } + return; + } - sim.run(SAMPLE_INTERVAL_MS as f32); + if let Some(ref events) = self.events + && let Some(last) = events.last() + && last.time + f64::EPSILON >= need_ms + { + return; + } + if self.events.is_none() { + let spring_config = SpringMassDamperSimulationConfig { + tension: self.screen_spring.stiffness, + mass: self.screen_spring.mass, + friction: self.screen_spring.damping, + }; + let mut sim = SpringMassDamperSimulation::new(spring_config); + let initial_pos = self.focus_target_at(0.0); + sim.set_position(initial_pos); + sim.set_velocity(XY::new(0.0, 0.0)); + sim.set_target_position(initial_pos); + self.events = Some(vec![SmoothedFocusEvent { + time: 0.0, + position: initial_pos, + }]); + self.precompute_sim = Some(ZoomFocusPrecomputeSim { + sim, + last_integrated_ms: 0.0, + }); + } + + loop { + let (next_ms, step_ms) = { + let Some(ps) = self.precompute_sim.as_ref() else { + break; + }; + if ps.last_integrated_ms + f64::EPSILON >= need_ms { + break; + } + let next_ms = (ps.last_integrated_ms + SAMPLE_INTERVAL_MS).min(duration_ms); + if next_ms <= ps.last_integrated_ms + f64::EPSILON { + break; + } + let step_ms = next_ms - ps.last_integrated_ms; + (next_ms, step_ms) + }; + let time_secs = (next_ms / 1000.0) as f32; + let target = self.focus_target_at(time_secs); + let Some(ps) = self.precompute_sim.as_mut() else { + break; + }; + let Some(events) = self.events.as_mut() else { + break; + }; + ps.sim.set_target_position(target); + ps.sim.run(step_ms as f32); + ps.last_integrated_ms = next_ms; events.push(SmoothedFocusEvent { - time: current_time_ms, + time: next_ms, position: XY::new( - sim.position.x.clamp(0.0, 1.0), - sim.position.y.clamp(0.0, 1.0), + ps.sim.position.x.clamp(0.0, 1.0), + ps.sim.position.y.clamp(0.0, 1.0), ), }); } - self.events = Some(events); + if let Some(ps) = self.precompute_sim.as_ref() + && ps.last_integrated_ms + f64::EPSILON >= duration_ms + { + self.precompute_sim = None; + } + } + + pub fn precompute(&mut self) { + self.ensure_precomputed_until(self.duration_secs as f32); } pub fn interpolate(&self, time_secs: f32) -> Coord { @@ -131,16 +330,8 @@ impl ZoomFocusInterpolator { } fn interpolate_direct(&self, time_secs: f32) -> Coord { - if let Some(cursor) = - interpolate_cursor(&self.cursor_events, time_secs, self.cursor_smoothing) - { - Coord::new(XY::new( - cursor.position.coord.x.clamp(0.0, 1.0), - cursor.position.coord.y.clamp(0.0, 1.0), - )) - } else { - Coord::new(XY::new(0.5, 0.5)) - } + let target = self.focus_target_at(time_secs); + Coord::new(XY::new(target.x as f64, target.y as f64)) } fn interpolate_from_events( @@ -188,28 +379,3 @@ impl ZoomFocusInterpolator { Coord::new(XY::new(lerped.x as f64, lerped.y as f64)) } } - -#[allow(dead_code)] -pub fn apply_edge_snap_to_focus( - focus: Coord, - segment: &ZoomSegment, -) -> Coord { - let position = (focus.x, focus.y); - let zoom_amount = segment.amount; - let edge_snap_ratio = segment.edge_snap_ratio; - - let viewport_half = 0.5 / zoom_amount; - let snap_threshold = edge_snap_ratio / zoom_amount; - - let snap_axis = |pos: f64| -> f64 { - if pos < snap_threshold { - viewport_half - } else if pos > 1.0 - snap_threshold { - 1.0 - viewport_half - } else { - pos.clamp(viewport_half, 1.0 - viewport_half) - } - }; - - Coord::new(XY::new(snap_axis(position.0), snap_axis(position.1))) -} diff --git a/package.json b/package.json index 00e3a15d00..fd94015071 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "cap-setup": "dotenv -e .env -- node scripts/setup.js", "analytics:migrate-dub": "dotenv -e .env -- node scripts/analytics/migrate-dub-to-tinybird.js --dry-run", "analytics:migrate-dub:apply": "dotenv -e .env -- node scripts/analytics/migrate-dub-to-tinybird.js --apply", + "analytics:posthog-growth": "dotenv -e .env -- node scripts/analytics/posthog-growth-audit.mjs", "analytics:setup": "dotenv -e .env -- node scripts/analytics/setup-analytics.js", "analytics:check": "dotenv -e .env -- node scripts/analytics/check-analytics.js", "analytics:delete-all": "dotenv -e .env -- node scripts/analytics/delete-all-data.js", diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index a652e80ecf..a47a159ba0 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -93,6 +93,7 @@ declare global { const IconLucideRatio: typeof import('~icons/lucide/ratio.jsx')['default'] const IconLucideRectangleHorizontal: typeof import('~icons/lucide/rectangle-horizontal.jsx')['default'] const IconLucideRefreshCw: typeof import('~icons/lucide/refresh-cw.jsx')['default'] + const IconLucideRotate3d: typeof import('~icons/lucide/rotate3d.jsx')['default'] const IconLucideRotateCcw: typeof import('~icons/lucide/rotate-ccw.jsx')['default'] const IconLucideRotateCw: typeof import('~icons/lucide/rotate-cw.jsx')['default'] const IconLucideSave: typeof import('~icons/lucide/save.jsx')['default']