fix(core): prevent double-offset on pip video visibility and media sync

packages/core/src/runtime/init.test.ts

@@ -477,6 +477,125 @@ describe("initSandboxRuntimeModular", () => {
     expect(hookHost.style.visibility).toBe("visible");
   });
 
+  it("shows pip video at global start time even when host composition starts late", () => {
+    // Regression: resolveStartForElement used to add the host composition's start on top of
+    // the video's own data-start, causing double-offset. A pip video with data-start="45.40"
+    // inside a host at data-start="45.40" would resolve to 90.80 and stay permanently hidden.
+    const root = document.createElement("div");
+    root.setAttribute("data-composition-id", "main");
+    root.setAttribute("data-root", "true");
+    root.setAttribute("data-start", "0");
+    root.setAttribute("data-width", "1920");
+    root.setAttribute("data-height", "1080");
+    document.body.appendChild(root);
+
+    const host = document.createElement("div");
+    host.setAttribute("data-composition-id", "scene-pip");
+    host.setAttribute("data-start", "45.40");
+    host.setAttribute("data-duration", "7.06");
+    root.appendChild(host);
+
+    const innerRoot = document.createElement("div");
+    innerRoot.setAttribute("data-composition-id", "scene-pip");
+    host.appendChild(innerRoot);
+
+    // pip-wired video: data-start is authored in global time (same value as host)
+    const pipVideo = document.createElement("video");
+    pipVideo.setAttribute("data-start", "45.40");
+    pipVideo.setAttribute("data-duration", "7.06");
+    Object.defineProperty(pipVideo, "paused", { value: true, configurable: true });
+    Object.defineProperty(pipVideo, "readyState", { value: 0, configurable: true });
+    Object.defineProperty(pipVideo, "currentTime", {
+      value: 0,
+      writable: true,
+      configurable: true,
+    });
+    pipVideo.load = () => {};
+    innerRoot.appendChild(pipVideo);
+
+    (window as Window & { __timelines?: Record<string, RuntimeTimelineLike> }).__timelines = {
+      main: createMockTimeline(60),
+      "scene-pip": createMockTimeline(7.06),
+    };
+
+    initSandboxRuntimeModular();
+
+    const player = (
+      window as Window & {
+        __player?: { seek: (timeSeconds: number) => void };
+      }
+    ).__player;
+    expect(player).toBeDefined();
+
+    // Before the fix: resolveStartForElement(pipVideo) = 45.40 + 45.40 = 90.80, so the
+    // video would be hidden at t=46 (90.80 > 46). After the fix: start = 45.40, visible.
+    player?.seek(46);
+    expect(pipVideo.style.visibility).toBe("visible");
+
+    player?.seek(53);
+    expect(pipVideo.style.visibility).toBe("hidden");
+
+    player?.seek(44);
+    expect(pipVideo.style.visibility).toBe("hidden");
+  });
+
+  it("shows auto-injected video at host time, not at t=0", () => {
+    const root = document.createElement("div");
+    root.setAttribute("data-composition-id", "main");
+    root.setAttribute("data-root", "true");
+    root.setAttribute("data-start", "0");
+    root.setAttribute("data-width", "1920");
+    root.setAttribute("data-height", "1080");
+    document.body.appendChild(root);
+
+    const host = document.createElement("div");
+    host.setAttribute("data-composition-id", "intro");
+    host.setAttribute("data-start", "10");
+    host.setAttribute("data-duration", "5");
+    root.appendChild(host);
+
+    const innerRoot = document.createElement("div");
+    innerRoot.setAttribute("data-composition-id", "intro");
+    host.appendChild(innerRoot);
+
+    const video = document.createElement("video");
+    video.setAttribute("data-start", "0");
+    video.setAttribute("data-hf-auto-start", "");
+    video.setAttribute("data-duration", "5");
+    Object.defineProperty(video, "paused", { value: true, configurable: true });
+    Object.defineProperty(video, "readyState", { value: 0, configurable: true });
+    Object.defineProperty(video, "currentTime", {
+      value: 0,
+      writable: true,
+      configurable: true,
+    });
+    video.load = () => {};
+    innerRoot.appendChild(video);
+
+    (window as Window & { __timelines?: Record<string, RuntimeTimelineLike> }).__timelines = {
+      main: createMockTimeline(30),
+      intro: createMockTimeline(5),
+    };
+
+    initSandboxRuntimeModular();
+
+    const player = (
+      window as Window & {
+        __player?: { seek: (timeSeconds: number) => void };
+      }
+    ).__player;
+    expect(player).toBeDefined();
+
+    player?.seek(12);
+    expect(video.style.visibility).toBe("visible");
+
+    player?.seek(5);
+    expect(video.style.visibility).toBe("hidden");
+
+    player?.seek(16);
+    expect(video.style.visibility).toBe("hidden");
+  });
+
   it("plays scheduled child timelines without a captured root timeline when audio has failed", () => {
     const raf = createManualRaf();
     vi.spyOn(performance, "now").mockImplementation(() => raf.now());

packages/core/src/runtime/init.ts

@@ -387,6 +387,14 @@ export function initSandboxRuntimeModular(): void {
     });
     return resolver.resolveDurationForElement(element);
   };
+
+  const resolveMediaStartSeconds = (element: Element, fallback = 0): number => {
+    if (!element.hasAttribute("data-hf-auto-start") && element.hasAttribute("data-start")) {
+      return Math.max(0, Number(element.getAttribute("data-start") ?? 0) || 0);
+    }
+    return resolveStartForElement(element, fallback);
+  };
+
   const hasExternalCompositions = !!document.querySelector("[data-composition-src]");
   let hasInlineTemplateCompositions = false;
   {
@@ -456,7 +464,7 @@ export function initSandboxRuntimeModular(): void {
     if (mediaNodes.length === 0) return null;
     let maxWindowEndSeconds = 0;
     for (const node of mediaNodes) {
-      const start = resolveStartForElement(node, 0);
+      const start = resolveMediaStartSeconds(node, 0);
       if (!Number.isFinite(start)) continue;
       const duration = resolveMediaElementDurationSeconds(node);
       if (duration == null || duration <= MIN_VALID_TIMELINE_DURATION_SECONDS) continue;
@@ -1281,11 +1289,11 @@ export function initSandboxRuntimeModular(): void {
         const context = resolveMediaCompositionContext(
           element as HTMLVideoElement | HTMLAudioElement,
         );
-        return resolveStartForElement(element, context.inheritedStart ?? 0);
+        return resolveMediaStartSeconds(element, context.inheritedStart ?? 0);
       },
       resolveDurationSeconds: (element) => {
         const context = resolveMediaCompositionContext(element);
-        const start = resolveStartForElement(element, context.inheritedStart ?? 0);
+        const start = resolveMediaStartSeconds(element, context.inheritedStart ?? 0);
         const mediaStart =
           Number.parseFloat(element.dataset.playbackStart ?? element.dataset.mediaStart ?? "0") ||
           0;
@@ -1329,7 +1337,10 @@ export function initSandboxRuntimeModular(): void {
       const tag = rawNode.tagName.toLowerCase();
       if (tag === "script" || tag === "style" || tag === "link" || tag === "meta") continue;
 
-      const start = resolveStartForElement(rawNode, 0);
+      const start =
+        tag === "video" || tag === "audio"
+          ? resolveMediaStartSeconds(rawNode, 0)
+          : resolveStartForElement(rawNode, 0);
       let duration = resolveDurationForElement(rawNode);
       const compId = rawNode.getAttribute("data-composition-id");
       if (compId) {

packages/core/src/runtime/timeline.ts

@@ -220,7 +220,9 @@ export function collectRuntimeTimelinePayload(params: {
     if (mediaNodes.length === 0) return null;
     let maxWindowEndSeconds = 0;
     for (const mediaNode of mediaNodes) {
-      const start = startResolver.resolveStartForElement(mediaNode, 0);
+      const start = !mediaNode.hasAttribute("data-hf-auto-start")
+        ? Math.max(0, Number(mediaNode.getAttribute("data-start") ?? 0) || 0)
+        : startResolver.resolveStartForElement(mediaNode, 0);
       if (!Number.isFinite(start)) continue;
       const duration = resolveMediaElementDurationSeconds(mediaNode);
       if (duration == null || duration <= 0) continue;

packages/producer/tests/pip-video-late-host/meta.json

@@ -0,0 +1,12 @@
+{
+  "name": "Pip video inside late-starting sub-composition host",
+  "description": "Regression for: resolveStartForElement double-counts the host offset for media elements with explicitly authored data-start. A pip video at data-start='3' inside a host at data-start='3' resolved to 6.0 instead of 3.0, keeping the video permanently hidden during the host's window.",
+  "tags": ["video", "sub-composition", "pip", "regression"],
+  "minPsnr": 25,
+  "maxFrameFailures": 5,
+  "minAudioCorrelation": 0.0,
+  "maxAudioLagWindows": 120,
+  "renderConfig": {
+    "fps": 24
+  }
+}

packages/producer/tests/pip-video-late-host/src/compositions/pip.html

@@ -0,0 +1,63 @@
+<script src="https://cdn.jsdelivr.net/npm/gsap@3.14.2/dist/gsap.min.js"></script>
+<template id="pip-template">
+  <div data-composition-id="pip-scene" data-width="1920" data-height="1080">
+    <!-- Pip video: data-start="3" is global time matching the host start.
+         Before the fix, resolveStartForElement returned 3+3=6, hiding the
+         video for the entire host window (3–6s). -->
+    <video
+      id="pip-video"
+      data-start="3"
+      data-duration="3"
+      data-track-index="0"
+      src="https://gen-os-static.s3.us-east-2.amazonaws.com/astral_assets/uploaded_assets/fb7e48ac_f6bf2ab079394d7ebd0491e7008a9242.mp4"
+      muted
+      playsinline
+      crossorigin="anonymous"
+      style="position: absolute; width: 100%; height: 100%; object-fit: cover;"
+    ></video>
+
+    <!-- Overlay text confirming pip scene is active -->
+    <div
+      id="pip-label"
+      data-start="3"
+      data-duration="3"
+      style="
+        position: absolute;
+        bottom: 80px;
+        left: 0;
+        width: 100%;
+        text-align: center;
+        font-family: system-ui, sans-serif;
+        font-size: 48px;
+        font-weight: 700;
+        color: #fff;
+        text-shadow: 0 2px 8px rgba(0,0,0,0.7);
+        z-index: 10;
+      "
+    >PIP VIDEO (3–6s)</div>
+
+    <style>
+      [data-composition-id="pip-scene"] {
+        position: relative;
+        width: 1920px;
+        height: 1080px;
+        background: #0a0a23;
+      }
+    </style>
+
+    <script>
+      window.__timelines = window.__timelines || {};
+      var tl = gsap.timeline({ paused: true });
+
+      tl.fromTo("#pip-video", { scale: 1.05 }, { scale: 1, duration: 3, ease: "none" }, 0);
+      tl.fromTo(
+        "#pip-label",
+        { y: 20, opacity: 0 },
+        { y: 0, opacity: 1, duration: 0.4, ease: "power2.out" },
+        0.2,
+      );
+
+      window.__timelines["pip-scene"] = tl;
+    </script>
+  </div>
+</template>

packages/producer/tests/pip-video-late-host/src/index.html

@@ -0,0 +1,72 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=1920, height=1080" />
+    <script src="https://cdn.jsdelivr.net/npm/gsap@3.14.2/dist/gsap.min.js"></script>
+    <style>
+      * { margin: 0; padding: 0; box-sizing: border-box; }
+      html, body { width: 1920px; height: 1080px; overflow: hidden; background: #1a1a2e; }
+      #root { position: relative; width: 1920px; height: 1080px; overflow: hidden; }
+      .label {
+        position: absolute;
+        font-family: system-ui, sans-serif;
+        font-size: 64px;
+        font-weight: 700;
+        color: #fff;
+        text-align: center;
+        width: 100%;
+        top: 50%;
+        transform: translateY(-50%);
+      }
+    </style>
+  </head>
+  <body>
+    <div
+      id="root"
+      data-composition-id="main"
+      data-start="0"
+      data-duration="6"
+      data-width="1920"
+      data-height="1080"
+    >
+      <!-- Scene 1: visible 0-3s (solid color background) -->
+      <div
+        id="scene-intro"
+        data-start="0"
+        data-duration="3"
+        data-track-index="0"
+        style="position: absolute; inset: 0; background: #16213e;"
+      >
+        <div class="label">INTRO (0–3s)</div>
+      </div>
+
+      <!-- Scene 2: late-starting host with pip video, visible 3-6s -->
+      <div
+        id="scene-pip-host"
+        data-composition-id="pip-scene"
+        data-composition-src="compositions/pip.html"
+        data-start="3"
+        data-duration="3"
+        data-track-index="1"
+        style="position: absolute; inset: 0;"
+      ></div>
+    </div>
+
+    <script>
+      window.__timelines = window.__timelines || {};
+      var tl = gsap.timeline({ paused: true });
+
+      tl.fromTo("#scene-intro", { opacity: 1 }, { opacity: 1, duration: 3 }, 0);
+      tl.to("#scene-intro", { opacity: 0, duration: 0.3 }, 2.7);
+      tl.fromTo(
+        "#scene-pip-host",
+        { opacity: 0 },
+        { opacity: 1, duration: 0.3, ease: "power2.out" },
+        3,
+      );
+
+      window.__timelines["main"] = tl;
+    </script>
+  </body>
+</html>