Add transparent and matte background modes

README.md

@@ -81,8 +81,22 @@ new EffectsPipeline(stream, { background: { video: 'https://example.com/bg.mp4'
 // solid color — hex string or [r, g, b] floats in [0, 1]
 new EffectsPipeline(stream, { background: { color: '#00b050' } })       // greenscreen
 new EffectsPipeline(stream, { background: { color: [0, 0.7, 0.3] } })
+
+// isolate the subject on a TRANSPARENT background — the matte becomes the
+// canvas alpha channel, so the subject can be composited over anything (e.g.
+// a WebGL scene) with no keying. Premultiplied output.
+new EffectsPipeline(stream, { background: 'transparent' })
+
+// render the raw alpha matte as a white silhouette — a debug view and a
+// reusable mask you can composite against your own full-resolution source.
+new EffectsPipeline(stream, { background: 'matte' })
 ```
 
+> `transparent` / `matte` only carry alpha on a canvas surface (the output
+> `MediaStream`'s video track is always opaque). Route them to a page-owned
+> canvas via `attachPreview(canvas)` + `setPreview({ background: 'transparent' })`,
+> whose context is configured with premultiplied alpha.
+
 Swap at runtime:
 
 ```ts

src/model/backend.ts

@@ -222,6 +222,13 @@ export interface Backend {
     // by RenderOp when the renderer is disabled (true GPU-level passthrough
     // — input frame in, same frame on the canvas).
     CompositePassthrough:    (image: Tensor, target?: RenderTarget) => Presenter;
+    // Transparent: composites image over nothing, using alpha as the canvas
+    // alpha channel — the subject is isolated so whatever sits behind the
+    // canvas shows through. Premultiplied output.
+    CompositeTransparent:    (image: Tensor, alpha: Tensor, target?: RenderTarget) => Presenter;
+    // Matte: renders the raw 1-channel alpha as a premultiplied white
+    // silhouette (rgb = a, alpha = a). Debug view + reusable mask.
+    CompositeMatte:          (alpha: Tensor, target?: RenderTarget) => Presenter;
   };
 
   // Register an additional output canvas under `name` so presenters can target

src/model/backends/webgl/index.ts

@@ -31,6 +31,8 @@ import { CompositeSolidWebGL } from '~/model/backends/webgl/ops/composite_solid.
 import { CompositeImageWebGL } from '~/model/backends/webgl/ops/composite_image.ts'
 import { CompositeImageBilinearWebGL } from '~/model/backends/webgl/ops/composite_image_bilinear.ts'
 import { CompositePassthroughWebGL } from '~/model/backends/webgl/ops/composite_passthrough.ts'
+import { CompositeTransparentWebGL } from '~/model/backends/webgl/ops/composite_transparent.ts'
+import { CompositeMatteWebGL } from '~/model/backends/webgl/ops/composite_matte.ts'
 import { InputWebGL } from '~/model/backends/webgl/ops/input.ts'
 
 export interface WebGLBackendOptions {
@@ -111,6 +113,10 @@ export class WebGLBackend implements Backend {
         new CompositeImageBilinearWebGL(this, image, alpha, bg),
       CompositePassthrough: (image) =>
         new CompositePassthroughWebGL(this, image),
+      CompositeTransparent: (image, alpha) =>
+        new CompositeTransparentWebGL(this, image, alpha),
+      CompositeMatte: (alpha) =>
+        new CompositeMatteWebGL(this, alpha),
     }
   }
 

src/model/backends/webgl/ops/composite_matte.ts

@@ -0,0 +1,62 @@
+import type { Tensor } from '~/model/backend.ts'
+import type { WebGLBackend } from '~/model/backends/webgl/index.ts'
+import type { WebGLTensor } from '~/model/backends/webgl/base_webgl_op.ts'
+import compositeMatteSrc from '~/model/backends/webgl/shaders/composite_matte.glsl'
+
+const QUAD_VERT = `#version 300 es
+const vec2 VERTS[6] = vec2[6](
+    vec2(-1.0,-1.0), vec2(1.0,-1.0), vec2(-1.0,1.0),
+    vec2(-1.0,1.0),  vec2(1.0,-1.0), vec2(1.0,1.0)
+);
+void main() { gl_Position = vec4(VERTS[gl_VertexID], 0.0, 1.0); }`
+
+// Renders the raw 1-channel alpha matte as a premultiplied white silhouette to
+// the canvas (default framebuffer). Alpha only — no image, no background. Not a
+// WebGLOp — produces no Tensor output, lives at the boundary between the model
+// graph and the display surface.
+//
+// Caller invariants:
+//   - canvas.width === alpha.w, canvas.height === alpha.h (no resampling here)
+export class CompositeMatteWebGL {
+  private readonly program:  WebGLProgram
+  private readonly alphaTex: WebGLTexture
+
+  constructor(
+    private readonly backend: WebGLBackend,
+    alpha: Tensor,
+  ) {
+    this.alphaTex = (alpha as WebGLTensor).texture
+
+    const gl = backend.gl
+
+    const vert = gl.createShader(gl.VERTEX_SHADER)!
+    gl.shaderSource(vert, QUAD_VERT)
+    gl.compileShader(vert)
+
+    const frag = gl.createShader(gl.FRAGMENT_SHADER)!
+    gl.shaderSource(frag, compositeMatteSrc)
+    gl.compileShader(frag)
+    if (!gl.getShaderParameter(frag, gl.COMPILE_STATUS))
+      throw new Error(`composite_matte GLSL compile error: ${gl.getShaderInfoLog(frag)}`)
+
+    this.program = gl.createProgram()!
+    gl.attachShader(this.program, vert)
+    gl.attachShader(this.program, frag)
+    gl.linkProgram(this.program)
+    if (!gl.getProgramParameter(this.program, gl.LINK_STATUS))
+      throw new Error(`composite_matte GLSL link error: ${gl.getProgramInfoLog(this.program)}`)
+  }
+
+  run(): void {
+    const gl = this.backend.gl
+    gl.useProgram(this.program)
+
+    gl.activeTexture(gl.TEXTURE0)
+    gl.bindTexture(gl.TEXTURE_2D, this.alphaTex)
+    gl.uniform1i(gl.getUniformLocation(this.program, 'u_alpha'), 0)
+
+    this.backend.bindDisplayFramebuffer()
+    gl.bindVertexArray(null)
+    gl.drawArrays(gl.TRIANGLES, 0, 6)
+  }
+}

src/model/backends/webgl/ops/composite_transparent.ts

@@ -0,0 +1,80 @@
+import type { Tensor } from '~/model/backend.ts'
+import type { WebGLBackend } from '~/model/backends/webgl/index.ts'
+import type { WebGLTensor } from '~/model/backends/webgl/base_webgl_op.ts'
+import compositeTransparentSrc from '~/model/backends/webgl/shaders/composite_transparent.glsl'
+
+const QUAD_VERT = `#version 300 es
+const vec2 VERTS[6] = vec2[6](
+    vec2(-1.0,-1.0), vec2(1.0,-1.0), vec2(-1.0,1.0),
+    vec2(-1.0,1.0),  vec2(1.0,-1.0), vec2(1.0,1.0)
+);
+void main() { gl_Position = vec4(VERTS[gl_VertexID], 0.0, 1.0); }`
+
+// Composites image + alpha over TRANSPARENCY (the matte becomes the canvas
+// alpha channel), writes to the canvas (default framebuffer). Like
+// CompositeSolidWebGL but with no background color — the subject is isolated so
+// whatever sits behind the canvas shows through. Not a WebGLOp — produces no
+// Tensor output, lives at the boundary between the model graph and the display
+// surface.
+//
+// Caller invariants:
+//   - image and alpha share h × w
+//   - canvas.width === image.w, canvas.height === image.h
+//     (the upscaler matches alpha to image res; this compositor does not
+//     resample)
+export class CompositeTransparentWebGL {
+  private readonly program:  WebGLProgram
+  private readonly imageTex: WebGLTexture
+  private readonly alphaTex: WebGLTexture
+
+  constructor(
+    private readonly backend: WebGLBackend,
+    image: Tensor,
+    alpha: Tensor,
+  ) {
+    if (image.h !== alpha.h || image.w !== alpha.w)
+      throw new Error(
+        `CompositeTransparent: image (${image.h}×${image.w}) and alpha ` +
+        `(${alpha.h}×${alpha.w}) must match. Run the upscaler first.`,
+      )
+
+    this.imageTex = (image as WebGLTensor).texture
+    this.alphaTex = (alpha as WebGLTensor).texture
+
+    const gl = backend.gl
+
+    const vert = gl.createShader(gl.VERTEX_SHADER)!
+    gl.shaderSource(vert, QUAD_VERT)
+    gl.compileShader(vert)
+
+    const frag = gl.createShader(gl.FRAGMENT_SHADER)!
+    gl.shaderSource(frag, compositeTransparentSrc)
+    gl.compileShader(frag)
+    if (!gl.getShaderParameter(frag, gl.COMPILE_STATUS))
+      throw new Error(`composite_transparent GLSL compile error: ${gl.getShaderInfoLog(frag)}`)
+
+    this.program = gl.createProgram()!
+    gl.attachShader(this.program, vert)
+    gl.attachShader(this.program, frag)
+    gl.linkProgram(this.program)
+    if (!gl.getProgramParameter(this.program, gl.LINK_STATUS))
+      throw new Error(`composite_transparent GLSL link error: ${gl.getProgramInfoLog(this.program)}`)
+  }
+
+  run(): void {
+    const gl = this.backend.gl
+    gl.useProgram(this.program)
+
+    gl.activeTexture(gl.TEXTURE0)
+    gl.bindTexture(gl.TEXTURE_2D, this.imageTex)
+    gl.uniform1i(gl.getUniformLocation(this.program, 'u_image'), 0)
+
+    gl.activeTexture(gl.TEXTURE1)
+    gl.bindTexture(gl.TEXTURE_2D, this.alphaTex)
+    gl.uniform1i(gl.getUniformLocation(this.program, 'u_alpha'), 1)
+
+    this.backend.bindDisplayFramebuffer()
+    gl.bindVertexArray(null)
+    gl.drawArrays(gl.TRIANGLES, 0, 6)
+  }
+}

src/model/backends/webgl/shaders/composite_matte.glsl

@@ -0,0 +1,23 @@
+#version 300 es
+// Render the raw 1-channel alpha matte as a premultiplied white silhouette
+// (rgb = a, alpha = a). Doubles as a debug view AND a reusable mask: a consumer
+// can composite it against their own full-resolution source (e.g. drawImage
+// with globalCompositeOperation 'destination-in'). No image input — matte only.
+//
+// Assumes the canvas (viewport) matches the alpha texture h×w — no resampling.
+
+precision highp float;
+
+uniform sampler2D u_alpha;   // alpha as NHWC vec4 (value in .r)
+
+out vec4 fragColor;
+
+void main() {
+    // WebGL gl_FragCoord origin is bottom-left; tensor textures are stored
+    // top-down. Flip y so the displayed mask is upright.
+    int H = textureSize(u_alpha, 0).y;
+    ivec2 px = ivec2(int(gl_FragCoord.x), H - 1 - int(gl_FragCoord.y));
+    float a = texelFetch(u_alpha, px, 0).r;
+    // Premultiplied white × matte: (a, a, a, a).
+    fragColor = vec4(vec3(a), a);
+}

src/model/backends/webgl/shaders/composite_transparent.glsl

@@ -0,0 +1,31 @@
+#version 300 es
+// Composite an RGBA image over TRANSPARENCY, gated by a 1-channel alpha mask.
+// Output: premultiplied RGBA with the matte as the alpha channel, so the
+// subject is isolated on a transparent background — whatever sits behind the
+// canvas shows through wherever the matte is 0. Mirrors composite_solid.glsl
+// but drops the background color (the "background" is nothing).
+//
+// Assumes image and alpha textures are the same h×w and that the canvas
+// (viewport) matches that resolution — no resampling here.
+
+precision highp float;
+
+uniform sampler2D u_image;   // image as NHWC vec4 (RGBA in vec4)
+uniform sampler2D u_alpha;   // alpha as NHWC vec4 (value in .r)
+
+out vec4 fragColor;
+
+void main() {
+    // WebGL gl_FragCoord origin is bottom-left, but tensor textures are stored
+    // top-down. Flip y when sampling so the displayed image is upright (matches
+    // composite_solid.glsl).
+    int H = textureSize(u_image, 0).y;
+    ivec2 px = ivec2(int(gl_FragCoord.x), H - 1 - int(gl_FragCoord.y));
+    vec3  fg = texelFetch(u_image, px, 0).rgb;
+    float a  = texelFetch(u_alpha, px, 0).r;
+
+    // Premultiplied output: rgb·a with the matte as the alpha channel. On the
+    // canvas's premultiplied surface this is a correct straight-alpha subject
+    // over transparency.
+    fragColor = vec4(fg * a, a);
+}

src/model/backends/webgpu/index.ts

@@ -31,6 +31,8 @@ import { CompositeSolidWebGPU } from "~/model/backends/webgpu/ops/composite_soli
 import { CompositeImageWebGPU } from "~/model/backends/webgpu/ops/composite_image.ts";
 import { CompositeImageBilinearWebGPU } from "~/model/backends/webgpu/ops/composite_image_bilinear.ts";
 import { CompositePassthroughWebGPU } from "~/model/backends/webgpu/ops/composite_passthrough.ts";
+import { CompositeTransparentWebGPU } from "~/model/backends/webgpu/ops/composite_transparent.ts";
+import { CompositeMatteWebGPU } from "~/model/backends/webgpu/ops/composite_matte.ts";
 import { InputWebGPU } from "~/model/backends/webgpu/ops/input.ts";
 
 const STORAGE = navigator.gpu ? GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST : 0;
@@ -140,6 +142,24 @@ export class WebGPUBackend implements Backend {
           },
         };
       },
+      CompositeTransparent: (image, alpha, target = "main") => {
+        const op = new CompositeTransparentWebGPU(this, image, alpha);
+        return {
+          run: () => {
+            op.setOutput(this.getCurrentDisplayTexture(target));
+            op.run();
+          },
+        };
+      },
+      CompositeMatte: (alpha, target = "main") => {
+        const op = new CompositeMatteWebGPU(this, alpha);
+        return {
+          run: () => {
+            op.setOutput(this.getCurrentDisplayTexture(target));
+            op.run();
+          },
+        };
+      },
     };
   }
 

src/model/backends/webgpu/ops/composite_matte.ts

@@ -0,0 +1,86 @@
+import type { Tensor } from '~/model/backend.ts'
+import type { WebGPUBackend } from '~/model/backends/webgpu/index.ts'
+import type { WebGPUTensor } from '~/model/backends/webgpu/base_webgpu_op.ts'
+import compositeMatteF32Src from '~/model/backends/webgpu/shaders/composite_matte.wgsl'
+import compositeMatteF16Src from '~/model/backends/webgpu/shaders/composite_matte_f16.wgsl'
+
+// Renders the raw 1-channel alpha matte as a premultiplied white silhouette to
+// a canvas swapchain texture. Alpha only — no image, no background. Standalone —
+// not a WebGPUOp.
+//
+// Caller invariants:
+//   - alpha.c === 4
+//   - canvas.width === alpha.w, canvas.height === alpha.h
+//
+// Per-frame contract (handled by Backend.presenters.CompositeMatte wrapper):
+//   compositor.setOutput(backend.getCurrentDisplayTexture(target))
+//   compositor.run()
+export class CompositeMatteWebGPU {
+  private readonly pipeline:      GPURenderPipeline
+  private readonly bindGroup:     GPUBindGroup
+  private readonly uniformBuffer: GPUBuffer
+  private          outputView:    GPUTextureView | null = null
+
+  constructor(
+    private readonly backend: WebGPUBackend,
+    alpha: Tensor,
+  ) {
+    const device = backend.device
+
+    // Params: just the alpha width (u32 = 4 bytes; round up to 16 for uniform
+    // buffer alignment).
+    this.uniformBuffer = device.createBuffer({
+      size: 16,
+      usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
+    })
+    const ab = new ArrayBuffer(16)
+    new Uint32Array(ab, 0, 1)[0] = alpha.w
+    device.queue.writeBuffer(this.uniformBuffer, 0, ab)
+
+    const src = backend.dtype === 'f16' ? compositeMatteF16Src : compositeMatteF32Src
+    const module = device.createShaderModule({ code: src })
+
+    this.pipeline = device.createRenderPipeline({
+      layout: 'auto',
+      vertex:   { module, entryPoint: 'vs' },
+      fragment: { module, entryPoint: 'fs', targets: [{ format: backend.canvasFormat }] },
+      primitive: { topology: 'triangle-list' },
+    })
+
+    this.bindGroup = device.createBindGroup({
+      layout: this.pipeline.getBindGroupLayout(0),
+      entries: [
+        { binding: 0, resource: { buffer: (alpha as WebGPUTensor).buffer } },
+        { binding: 1, resource: { buffer: this.uniformBuffer } },
+      ],
+    })
+  }
+
+  setOutput(texture: GPUTexture): void {
+    this.outputView = texture.createView()
+  }
+
+  run(): void {
+    if (!this.outputView)
+      throw new Error('CompositeMatteWebGPU.run() called before setOutput()')
+
+    const enc = this.backend.device.createCommandEncoder()
+    const pass = enc.beginRenderPass({
+      colorAttachments: [{
+        view:       this.outputView,
+        clearValue: [0, 0, 0, 0],
+        loadOp:     'clear',
+        storeOp:    'store',
+      }],
+    })
+    pass.setPipeline(this.pipeline)
+    pass.setBindGroup(0, this.bindGroup)
+    pass.draw(6)
+    pass.end()
+    this.backend.device.queue.submit([enc.finish()])
+
+    // GPUTexture is invalidated after the next browser paint — force the
+    // caller to set it again before the next frame.
+    this.outputView = null
+  }
+}