From 0665886bb80e23af825d3caa3e96913cf9b61081 Mon Sep 17 00:00:00 2001 From: Jeff Date: Wed, 1 Jul 2026 22:40:38 -0400 Subject: [PATCH] Add `transparent` and `matte` background modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two output modes that expose the matte as real transparency, so a consumer can composite the isolated subject over their own surface (e.g. a WebGL scene) instead of one of the built-in backgrounds: - `transparent` — RGBA = (subject.rgb · a, a): the subject on a transparent background (premultiplied), matte as the canvas alpha channel. - `matte` — RGBA = (a, a, a, a): the raw alpha as a white silhouette; a debug view and a reusable mask. Both plug into the existing pipeline with no context changes — the WebGL context is already alpha:true/premultiplied and the WebGPU canvas is configured alphaMode:'premultiplied'; the stock compositors just hardcode alpha 1.0. They surface through the same `background` normalization and the `attachPreview` / `setPreview` machinery, so `main:'none'` + a `preview:'transparent'` renders the isolated subject straight to a page-owned canvas (the output MediaStream can't carry alpha). Implementation mirrors composite_solid across both backends: - new GLSL / WGSL shaders (+ f16 WGSL variants) and per-backend ops - CompositorTransparent / CompositorMatte effect wrappers - CompositeSpec `transparent` / `matte` modes wired through buildCompositor, sameSpec, and renderer.translateBackgroundFor - 'transparent' / 'matte' keywords in BackgroundInput / Background + normalize - backend.presenters interface + WebGL/WebGPU registration - tests: WebGL pixel-exact premultiplied output + WebGPU compile/submit smoke - README backgrounds section Co-Authored-By: Claude Opus 4.8 --- README.md | 14 +++ src/model/backend.ts | 7 ++ src/model/backends/webgl/index.ts | 6 ++ .../backends/webgl/ops/composite_matte.ts | 62 ++++++++++++ .../webgl/ops/composite_transparent.ts | 80 +++++++++++++++ .../webgl/shaders/composite_matte.glsl | 23 +++++ .../webgl/shaders/composite_transparent.glsl | 31 ++++++ src/model/backends/webgpu/index.ts | 20 ++++ .../backends/webgpu/ops/composite_matte.ts | 86 ++++++++++++++++ .../webgpu/ops/composite_transparent.ts | 97 +++++++++++++++++++ .../webgpu/shaders/composite_matte.wgsl | 39 ++++++++ .../webgpu/shaders/composite_matte_f16.wgsl | 40 ++++++++ .../webgpu/shaders/composite_transparent.wgsl | 44 +++++++++ .../shaders/composite_transparent_f16.wgsl | 44 +++++++++ src/model/effects/compositor_matte.ts | 24 +++++ src/model/effects/compositor_transparent.ts | 27 ++++++ src/model/render_op.ts | 17 +++- src/pipeline/background.ts | 6 ++ src/pipeline/worker/renderer.ts | 4 + tests/effects/composite_alpha.test.ts | 79 +++++++++++++++ 20 files changed, 747 insertions(+), 3 deletions(-) create mode 100644 src/model/backends/webgl/ops/composite_matte.ts create mode 100644 src/model/backends/webgl/ops/composite_transparent.ts create mode 100644 src/model/backends/webgl/shaders/composite_matte.glsl create mode 100644 src/model/backends/webgl/shaders/composite_transparent.glsl create mode 100644 src/model/backends/webgpu/ops/composite_matte.ts create mode 100644 src/model/backends/webgpu/ops/composite_transparent.ts create mode 100644 src/model/backends/webgpu/shaders/composite_matte.wgsl create mode 100644 src/model/backends/webgpu/shaders/composite_matte_f16.wgsl create mode 100644 src/model/backends/webgpu/shaders/composite_transparent.wgsl create mode 100644 src/model/backends/webgpu/shaders/composite_transparent_f16.wgsl create mode 100644 src/model/effects/compositor_matte.ts create mode 100644 src/model/effects/compositor_transparent.ts create mode 100644 tests/effects/composite_alpha.test.ts diff --git a/README.md b/README.md index a9f776c..18205ea 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,22 @@ new EffectsPipeline(stream, { background: { video: 'https://example.com/bg.mp4' // solid color — hex string or [r, g, b] floats in [0, 1] new EffectsPipeline(stream, { background: { color: '#00b050' } }) // greenscreen new EffectsPipeline(stream, { background: { color: [0, 0.7, 0.3] } }) + +// isolate the subject on a TRANSPARENT background — the matte becomes the +// canvas alpha channel, so the subject can be composited over anything (e.g. +// a WebGL scene) with no keying. Premultiplied output. +new EffectsPipeline(stream, { background: 'transparent' }) + +// render the raw alpha matte as a white silhouette — a debug view and a +// reusable mask you can composite against your own full-resolution source. +new EffectsPipeline(stream, { background: 'matte' }) ``` +> `transparent` / `matte` only carry alpha on a canvas surface (the output +> `MediaStream`'s video track is always opaque). Route them to a page-owned +> canvas via `attachPreview(canvas)` + `setPreview({ background: 'transparent' })`, +> whose context is configured with premultiplied alpha. + Swap at runtime: ```ts diff --git a/src/model/backend.ts b/src/model/backend.ts index e5b6d33..3e5b328 100644 --- a/src/model/backend.ts +++ b/src/model/backend.ts @@ -222,6 +222,13 @@ export interface Backend { // by RenderOp when the renderer is disabled (true GPU-level passthrough // — input frame in, same frame on the canvas). CompositePassthrough: (image: Tensor, target?: RenderTarget) => Presenter; + // Transparent: composites image over nothing, using alpha as the canvas + // alpha channel — the subject is isolated so whatever sits behind the + // canvas shows through. Premultiplied output. + CompositeTransparent: (image: Tensor, alpha: Tensor, target?: RenderTarget) => Presenter; + // Matte: renders the raw 1-channel alpha as a premultiplied white + // silhouette (rgb = a, alpha = a). Debug view + reusable mask. + CompositeMatte: (alpha: Tensor, target?: RenderTarget) => Presenter; }; // Register an additional output canvas under `name` so presenters can target diff --git a/src/model/backends/webgl/index.ts b/src/model/backends/webgl/index.ts index 6bd1c8a..e9f83eb 100644 --- a/src/model/backends/webgl/index.ts +++ b/src/model/backends/webgl/index.ts @@ -31,6 +31,8 @@ import { CompositeSolidWebGL } from '~/model/backends/webgl/ops/composite_solid. import { CompositeImageWebGL } from '~/model/backends/webgl/ops/composite_image.ts' import { CompositeImageBilinearWebGL } from '~/model/backends/webgl/ops/composite_image_bilinear.ts' import { CompositePassthroughWebGL } from '~/model/backends/webgl/ops/composite_passthrough.ts' +import { CompositeTransparentWebGL } from '~/model/backends/webgl/ops/composite_transparent.ts' +import { CompositeMatteWebGL } from '~/model/backends/webgl/ops/composite_matte.ts' import { InputWebGL } from '~/model/backends/webgl/ops/input.ts' export interface WebGLBackendOptions { @@ -111,6 +113,10 @@ export class WebGLBackend implements Backend { new CompositeImageBilinearWebGL(this, image, alpha, bg), CompositePassthrough: (image) => new CompositePassthroughWebGL(this, image), + CompositeTransparent: (image, alpha) => + new CompositeTransparentWebGL(this, image, alpha), + CompositeMatte: (alpha) => + new CompositeMatteWebGL(this, alpha), } } diff --git a/src/model/backends/webgl/ops/composite_matte.ts b/src/model/backends/webgl/ops/composite_matte.ts new file mode 100644 index 0000000..0a43a6f --- /dev/null +++ b/src/model/backends/webgl/ops/composite_matte.ts @@ -0,0 +1,62 @@ +import type { Tensor } from '~/model/backend.ts' +import type { WebGLBackend } from '~/model/backends/webgl/index.ts' +import type { WebGLTensor } from '~/model/backends/webgl/base_webgl_op.ts' +import compositeMatteSrc from '~/model/backends/webgl/shaders/composite_matte.glsl' + +const QUAD_VERT = `#version 300 es +const vec2 VERTS[6] = vec2[6]( + vec2(-1.0,-1.0), vec2(1.0,-1.0), vec2(-1.0,1.0), + vec2(-1.0,1.0), vec2(1.0,-1.0), vec2(1.0,1.0) +); +void main() { gl_Position = vec4(VERTS[gl_VertexID], 0.0, 1.0); }` + +// Renders the raw 1-channel alpha matte as a premultiplied white silhouette to +// the canvas (default framebuffer). Alpha only — no image, no background. Not a +// WebGLOp — produces no Tensor output, lives at the boundary between the model +// graph and the display surface. +// +// Caller invariants: +// - canvas.width === alpha.w, canvas.height === alpha.h (no resampling here) +export class CompositeMatteWebGL { + private readonly program: WebGLProgram + private readonly alphaTex: WebGLTexture + + constructor( + private readonly backend: WebGLBackend, + alpha: Tensor, + ) { + this.alphaTex = (alpha as WebGLTensor).texture + + const gl = backend.gl + + const vert = gl.createShader(gl.VERTEX_SHADER)! + gl.shaderSource(vert, QUAD_VERT) + gl.compileShader(vert) + + const frag = gl.createShader(gl.FRAGMENT_SHADER)! + gl.shaderSource(frag, compositeMatteSrc) + gl.compileShader(frag) + if (!gl.getShaderParameter(frag, gl.COMPILE_STATUS)) + throw new Error(`composite_matte GLSL compile error: ${gl.getShaderInfoLog(frag)}`) + + this.program = gl.createProgram()! + gl.attachShader(this.program, vert) + gl.attachShader(this.program, frag) + gl.linkProgram(this.program) + if (!gl.getProgramParameter(this.program, gl.LINK_STATUS)) + throw new Error(`composite_matte GLSL link error: ${gl.getProgramInfoLog(this.program)}`) + } + + run(): void { + const gl = this.backend.gl + gl.useProgram(this.program) + + gl.activeTexture(gl.TEXTURE0) + gl.bindTexture(gl.TEXTURE_2D, this.alphaTex) + gl.uniform1i(gl.getUniformLocation(this.program, 'u_alpha'), 0) + + this.backend.bindDisplayFramebuffer() + gl.bindVertexArray(null) + gl.drawArrays(gl.TRIANGLES, 0, 6) + } +} diff --git a/src/model/backends/webgl/ops/composite_transparent.ts b/src/model/backends/webgl/ops/composite_transparent.ts new file mode 100644 index 0000000..fe7c49f --- /dev/null +++ b/src/model/backends/webgl/ops/composite_transparent.ts @@ -0,0 +1,80 @@ +import type { Tensor } from '~/model/backend.ts' +import type { WebGLBackend } from '~/model/backends/webgl/index.ts' +import type { WebGLTensor } from '~/model/backends/webgl/base_webgl_op.ts' +import compositeTransparentSrc from '~/model/backends/webgl/shaders/composite_transparent.glsl' + +const QUAD_VERT = `#version 300 es +const vec2 VERTS[6] = vec2[6]( + vec2(-1.0,-1.0), vec2(1.0,-1.0), vec2(-1.0,1.0), + vec2(-1.0,1.0), vec2(1.0,-1.0), vec2(1.0,1.0) +); +void main() { gl_Position = vec4(VERTS[gl_VertexID], 0.0, 1.0); }` + +// Composites image + alpha over TRANSPARENCY (the matte becomes the canvas +// alpha channel), writes to the canvas (default framebuffer). Like +// CompositeSolidWebGL but with no background color — the subject is isolated so +// whatever sits behind the canvas shows through. Not a WebGLOp — produces no +// Tensor output, lives at the boundary between the model graph and the display +// surface. +// +// Caller invariants: +// - image and alpha share h × w +// - canvas.width === image.w, canvas.height === image.h +// (the upscaler matches alpha to image res; this compositor does not +// resample) +export class CompositeTransparentWebGL { + private readonly program: WebGLProgram + private readonly imageTex: WebGLTexture + private readonly alphaTex: WebGLTexture + + constructor( + private readonly backend: WebGLBackend, + image: Tensor, + alpha: Tensor, + ) { + if (image.h !== alpha.h || image.w !== alpha.w) + throw new Error( + `CompositeTransparent: image (${image.h}×${image.w}) and alpha ` + + `(${alpha.h}×${alpha.w}) must match. Run the upscaler first.`, + ) + + this.imageTex = (image as WebGLTensor).texture + this.alphaTex = (alpha as WebGLTensor).texture + + const gl = backend.gl + + const vert = gl.createShader(gl.VERTEX_SHADER)! + gl.shaderSource(vert, QUAD_VERT) + gl.compileShader(vert) + + const frag = gl.createShader(gl.FRAGMENT_SHADER)! + gl.shaderSource(frag, compositeTransparentSrc) + gl.compileShader(frag) + if (!gl.getShaderParameter(frag, gl.COMPILE_STATUS)) + throw new Error(`composite_transparent GLSL compile error: ${gl.getShaderInfoLog(frag)}`) + + this.program = gl.createProgram()! + gl.attachShader(this.program, vert) + gl.attachShader(this.program, frag) + gl.linkProgram(this.program) + if (!gl.getProgramParameter(this.program, gl.LINK_STATUS)) + throw new Error(`composite_transparent GLSL link error: ${gl.getProgramInfoLog(this.program)}`) + } + + run(): void { + const gl = this.backend.gl + gl.useProgram(this.program) + + gl.activeTexture(gl.TEXTURE0) + gl.bindTexture(gl.TEXTURE_2D, this.imageTex) + gl.uniform1i(gl.getUniformLocation(this.program, 'u_image'), 0) + + gl.activeTexture(gl.TEXTURE1) + gl.bindTexture(gl.TEXTURE_2D, this.alphaTex) + gl.uniform1i(gl.getUniformLocation(this.program, 'u_alpha'), 1) + + this.backend.bindDisplayFramebuffer() + gl.bindVertexArray(null) + gl.drawArrays(gl.TRIANGLES, 0, 6) + } +} diff --git a/src/model/backends/webgl/shaders/composite_matte.glsl b/src/model/backends/webgl/shaders/composite_matte.glsl new file mode 100644 index 0000000..bf4e982 --- /dev/null +++ b/src/model/backends/webgl/shaders/composite_matte.glsl @@ -0,0 +1,23 @@ +#version 300 es +// Render the raw 1-channel alpha matte as a premultiplied white silhouette +// (rgb = a, alpha = a). Doubles as a debug view AND a reusable mask: a consumer +// can composite it against their own full-resolution source (e.g. drawImage +// with globalCompositeOperation 'destination-in'). No image input — matte only. +// +// Assumes the canvas (viewport) matches the alpha texture h×w — no resampling. + +precision highp float; + +uniform sampler2D u_alpha; // alpha as NHWC vec4 (value in .r) + +out vec4 fragColor; + +void main() { + // WebGL gl_FragCoord origin is bottom-left; tensor textures are stored + // top-down. Flip y so the displayed mask is upright. + int H = textureSize(u_alpha, 0).y; + ivec2 px = ivec2(int(gl_FragCoord.x), H - 1 - int(gl_FragCoord.y)); + float a = texelFetch(u_alpha, px, 0).r; + // Premultiplied white × matte: (a, a, a, a). + fragColor = vec4(vec3(a), a); +} diff --git a/src/model/backends/webgl/shaders/composite_transparent.glsl b/src/model/backends/webgl/shaders/composite_transparent.glsl new file mode 100644 index 0000000..ce43e21 --- /dev/null +++ b/src/model/backends/webgl/shaders/composite_transparent.glsl @@ -0,0 +1,31 @@ +#version 300 es +// Composite an RGBA image over TRANSPARENCY, gated by a 1-channel alpha mask. +// Output: premultiplied RGBA with the matte as the alpha channel, so the +// subject is isolated on a transparent background — whatever sits behind the +// canvas shows through wherever the matte is 0. Mirrors composite_solid.glsl +// but drops the background color (the "background" is nothing). +// +// Assumes image and alpha textures are the same h×w and that the canvas +// (viewport) matches that resolution — no resampling here. + +precision highp float; + +uniform sampler2D u_image; // image as NHWC vec4 (RGBA in vec4) +uniform sampler2D u_alpha; // alpha as NHWC vec4 (value in .r) + +out vec4 fragColor; + +void main() { + // WebGL gl_FragCoord origin is bottom-left, but tensor textures are stored + // top-down. Flip y when sampling so the displayed image is upright (matches + // composite_solid.glsl). + int H = textureSize(u_image, 0).y; + ivec2 px = ivec2(int(gl_FragCoord.x), H - 1 - int(gl_FragCoord.y)); + vec3 fg = texelFetch(u_image, px, 0).rgb; + float a = texelFetch(u_alpha, px, 0).r; + + // Premultiplied output: rgb·a with the matte as the alpha channel. On the + // canvas's premultiplied surface this is a correct straight-alpha subject + // over transparency. + fragColor = vec4(fg * a, a); +} diff --git a/src/model/backends/webgpu/index.ts b/src/model/backends/webgpu/index.ts index a9ec2da..3d6d609 100644 --- a/src/model/backends/webgpu/index.ts +++ b/src/model/backends/webgpu/index.ts @@ -31,6 +31,8 @@ import { CompositeSolidWebGPU } from "~/model/backends/webgpu/ops/composite_soli import { CompositeImageWebGPU } from "~/model/backends/webgpu/ops/composite_image.ts"; import { CompositeImageBilinearWebGPU } from "~/model/backends/webgpu/ops/composite_image_bilinear.ts"; import { CompositePassthroughWebGPU } from "~/model/backends/webgpu/ops/composite_passthrough.ts"; +import { CompositeTransparentWebGPU } from "~/model/backends/webgpu/ops/composite_transparent.ts"; +import { CompositeMatteWebGPU } from "~/model/backends/webgpu/ops/composite_matte.ts"; import { InputWebGPU } from "~/model/backends/webgpu/ops/input.ts"; const STORAGE = navigator.gpu ? GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST : 0; @@ -140,6 +142,24 @@ export class WebGPUBackend implements Backend { }, }; }, + CompositeTransparent: (image, alpha, target = "main") => { + const op = new CompositeTransparentWebGPU(this, image, alpha); + return { + run: () => { + op.setOutput(this.getCurrentDisplayTexture(target)); + op.run(); + }, + }; + }, + CompositeMatte: (alpha, target = "main") => { + const op = new CompositeMatteWebGPU(this, alpha); + return { + run: () => { + op.setOutput(this.getCurrentDisplayTexture(target)); + op.run(); + }, + }; + }, }; } diff --git a/src/model/backends/webgpu/ops/composite_matte.ts b/src/model/backends/webgpu/ops/composite_matte.ts new file mode 100644 index 0000000..15d3c82 --- /dev/null +++ b/src/model/backends/webgpu/ops/composite_matte.ts @@ -0,0 +1,86 @@ +import type { Tensor } from '~/model/backend.ts' +import type { WebGPUBackend } from '~/model/backends/webgpu/index.ts' +import type { WebGPUTensor } from '~/model/backends/webgpu/base_webgpu_op.ts' +import compositeMatteF32Src from '~/model/backends/webgpu/shaders/composite_matte.wgsl' +import compositeMatteF16Src from '~/model/backends/webgpu/shaders/composite_matte_f16.wgsl' + +// Renders the raw 1-channel alpha matte as a premultiplied white silhouette to +// a canvas swapchain texture. Alpha only — no image, no background. Standalone — +// not a WebGPUOp. +// +// Caller invariants: +// - alpha.c === 4 +// - canvas.width === alpha.w, canvas.height === alpha.h +// +// Per-frame contract (handled by Backend.presenters.CompositeMatte wrapper): +// compositor.setOutput(backend.getCurrentDisplayTexture(target)) +// compositor.run() +export class CompositeMatteWebGPU { + private readonly pipeline: GPURenderPipeline + private readonly bindGroup: GPUBindGroup + private readonly uniformBuffer: GPUBuffer + private outputView: GPUTextureView | null = null + + constructor( + private readonly backend: WebGPUBackend, + alpha: Tensor, + ) { + const device = backend.device + + // Params: just the alpha width (u32 = 4 bytes; round up to 16 for uniform + // buffer alignment). + this.uniformBuffer = device.createBuffer({ + size: 16, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }) + const ab = new ArrayBuffer(16) + new Uint32Array(ab, 0, 1)[0] = alpha.w + device.queue.writeBuffer(this.uniformBuffer, 0, ab) + + const src = backend.dtype === 'f16' ? compositeMatteF16Src : compositeMatteF32Src + const module = device.createShaderModule({ code: src }) + + this.pipeline = device.createRenderPipeline({ + layout: 'auto', + vertex: { module, entryPoint: 'vs' }, + fragment: { module, entryPoint: 'fs', targets: [{ format: backend.canvasFormat }] }, + primitive: { topology: 'triangle-list' }, + }) + + this.bindGroup = device.createBindGroup({ + layout: this.pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: { buffer: (alpha as WebGPUTensor).buffer } }, + { binding: 1, resource: { buffer: this.uniformBuffer } }, + ], + }) + } + + setOutput(texture: GPUTexture): void { + this.outputView = texture.createView() + } + + run(): void { + if (!this.outputView) + throw new Error('CompositeMatteWebGPU.run() called before setOutput()') + + const enc = this.backend.device.createCommandEncoder() + const pass = enc.beginRenderPass({ + colorAttachments: [{ + view: this.outputView, + clearValue: [0, 0, 0, 0], + loadOp: 'clear', + storeOp: 'store', + }], + }) + pass.setPipeline(this.pipeline) + pass.setBindGroup(0, this.bindGroup) + pass.draw(6) + pass.end() + this.backend.device.queue.submit([enc.finish()]) + + // GPUTexture is invalidated after the next browser paint — force the + // caller to set it again before the next frame. + this.outputView = null + } +} diff --git a/src/model/backends/webgpu/ops/composite_transparent.ts b/src/model/backends/webgpu/ops/composite_transparent.ts new file mode 100644 index 0000000..c6b4be0 --- /dev/null +++ b/src/model/backends/webgpu/ops/composite_transparent.ts @@ -0,0 +1,97 @@ +import type { Tensor } from '~/model/backend.ts' +import type { WebGPUBackend } from '~/model/backends/webgpu/index.ts' +import type { WebGPUTensor } from '~/model/backends/webgpu/base_webgpu_op.ts' +import compositeTransparentF32Src from '~/model/backends/webgpu/shaders/composite_transparent.wgsl' +import compositeTransparentF16Src from '~/model/backends/webgpu/shaders/composite_transparent_f16.wgsl' + +// Composites image + alpha over TRANSPARENCY (the matte becomes the canvas +// alpha) and writes the result to a canvas swapchain texture. Like +// CompositeSolidWebGPU but with no background — the subject is isolated so +// whatever sits behind the canvas shows through. Standalone — not a WebGPUOp. +// +// Caller invariants: +// - image and alpha are same h × w, c === 4 +// - canvas.width === image.w, canvas.height === image.h +// +// Per-frame contract (handled by Backend.presenters.CompositeTransparent wrapper): +// compositor.setOutput(backend.getCurrentDisplayTexture(target)) +// compositor.run() +export class CompositeTransparentWebGPU { + private readonly pipeline: GPURenderPipeline + private readonly bindGroup: GPUBindGroup + private readonly uniformBuffer: GPUBuffer + private outputView: GPUTextureView | null = null + + constructor( + private readonly backend: WebGPUBackend, + image: Tensor, + alpha: Tensor, + ) { + if (image.h !== alpha.h || image.w !== alpha.w) + throw new Error( + `CompositeTransparent: image (${image.h}×${image.w}) and alpha ` + + `(${alpha.h}×${alpha.w}) must match. Run the upscaler first.`, + ) + + const device = backend.device + + // Params: just the image width (u32 = 4 bytes; round up to 16 for uniform + // buffer alignment). + this.uniformBuffer = device.createBuffer({ + size: 16, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }) + const ab = new ArrayBuffer(16) + new Uint32Array(ab, 0, 1)[0] = image.w + device.queue.writeBuffer(this.uniformBuffer, 0, ab) + + const src = backend.dtype === 'f16' ? compositeTransparentF16Src : compositeTransparentF32Src + const module = device.createShaderModule({ code: src }) + + this.pipeline = device.createRenderPipeline({ + layout: 'auto', + vertex: { module, entryPoint: 'vs' }, + fragment: { module, entryPoint: 'fs', targets: [{ format: backend.canvasFormat }] }, + primitive: { topology: 'triangle-list' }, + }) + + this.bindGroup = device.createBindGroup({ + layout: this.pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: { buffer: (image as WebGPUTensor).buffer } }, + { binding: 1, resource: { buffer: (alpha as WebGPUTensor).buffer } }, + { binding: 2, resource: { buffer: this.uniformBuffer } }, + ], + }) + } + + setOutput(texture: GPUTexture): void { + this.outputView = texture.createView() + } + + run(): void { + if (!this.outputView) + throw new Error('CompositeTransparentWebGPU.run() called before setOutput()') + + const enc = this.backend.device.createCommandEncoder() + const pass = enc.beginRenderPass({ + colorAttachments: [{ + view: this.outputView, + // Transparent clear: pixels the full-screen quad doesn't touch (none, + // here) stay clear rather than opaque black. + clearValue: [0, 0, 0, 0], + loadOp: 'clear', + storeOp: 'store', + }], + }) + pass.setPipeline(this.pipeline) + pass.setBindGroup(0, this.bindGroup) + pass.draw(6) + pass.end() + this.backend.device.queue.submit([enc.finish()]) + + // GPUTexture is invalidated after the next browser paint — force the + // caller to set it again before the next frame. + this.outputView = null + } +} diff --git a/src/model/backends/webgpu/shaders/composite_matte.wgsl b/src/model/backends/webgpu/shaders/composite_matte.wgsl new file mode 100644 index 0000000..00f296a --- /dev/null +++ b/src/model/backends/webgpu/shaders/composite_matte.wgsl @@ -0,0 +1,39 @@ +// Render the raw 1-channel alpha matte as a premultiplied white silhouette +// (rgb = a, alpha = a) to the canvas swapchain. Doubles as a debug view AND a +// reusable mask a consumer can composite against their own source. Alpha only — +// no image, no background. +// +// Caller invariants (matched in CompositeMatteWebGPU): +// - alpha is an NHWC vec4 storage buffer +// - canvas.width === alpha.w, canvas.height === alpha.h (no resampling) + +struct VertexOut { + @builtin(position) pos: vec4, +}; + +@vertex +fn vs(@builtin(vertex_index) vi: u32) -> VertexOut { + let verts = array, 6>( + vec2(-1.0, -1.0), vec2( 1.0, -1.0), vec2(-1.0, 1.0), + vec2(-1.0, 1.0), vec2( 1.0, -1.0), vec2( 1.0, 1.0), + ); + var out: VertexOut; + out.pos = vec4(verts[vi], 0.0, 1.0); + return out; +} + +struct Params { + width: u32, +}; + +@group(0) @binding(0) var alpha: array>; +@group(0) @binding(1) var params: Params; + +@fragment +fn fs(in: VertexOut) -> @location(0) vec4 { + let x = u32(in.pos.x); + let y = u32(in.pos.y); + let i = y * params.width + x; + let a = alpha[i].r; + return vec4(a, a, a, a); +} diff --git a/src/model/backends/webgpu/shaders/composite_matte_f16.wgsl b/src/model/backends/webgpu/shaders/composite_matte_f16.wgsl new file mode 100644 index 0000000..21704d0 --- /dev/null +++ b/src/model/backends/webgpu/shaders/composite_matte_f16.wgsl @@ -0,0 +1,40 @@ +enable f16; + +// Render the raw 1-channel alpha matte as a premultiplied white silhouette +// (rgb = a, alpha = a) to the canvas swapchain. f16 variant: alpha is stored as +// f16 and promotes to f32 on read. Alpha only — no image, no background. +// +// Caller invariants (matched in CompositeMatteWebGPU): +// - alpha is an NHWC vec4 storage buffer +// - canvas.width === alpha.w, canvas.height === alpha.h (no resampling) + +struct VertexOut { + @builtin(position) pos: vec4, +}; + +@vertex +fn vs(@builtin(vertex_index) vi: u32) -> VertexOut { + let verts = array, 6>( + vec2(-1.0, -1.0), vec2( 1.0, -1.0), vec2(-1.0, 1.0), + vec2(-1.0, 1.0), vec2( 1.0, -1.0), vec2( 1.0, 1.0), + ); + var out: VertexOut; + out.pos = vec4(verts[vi], 0.0, 1.0); + return out; +} + +struct Params { + width: u32, +}; + +@group(0) @binding(0) var alpha: array>; +@group(0) @binding(1) var params: Params; + +@fragment +fn fs(in: VertexOut) -> @location(0) vec4 { + let x = u32(in.pos.x); + let y = u32(in.pos.y); + let i = y * params.width + x; + let a = f32(alpha[i].r); + return vec4(a, a, a, a); +} diff --git a/src/model/backends/webgpu/shaders/composite_transparent.wgsl b/src/model/backends/webgpu/shaders/composite_transparent.wgsl new file mode 100644 index 0000000..9b47e7d --- /dev/null +++ b/src/model/backends/webgpu/shaders/composite_transparent.wgsl @@ -0,0 +1,44 @@ +// Composite an RGBA image over TRANSPARENCY, gated by a 1-ch alpha. The matte +// becomes the canvas alpha channel, so the subject is isolated on a transparent +// background (premultiplied output) — whatever sits behind the canvas shows +// through wherever the matte is 0. Mirrors composite_solid.wgsl without the +// background color. +// +// Caller invariants (matched in CompositeTransparentWebGPU): +// - image and alpha are NHWC vec4 storage buffers, same h × w +// - canvas.width === image.w, canvas.height === image.h (no resampling) + +struct VertexOut { + @builtin(position) pos: vec4, +}; + +@vertex +fn vs(@builtin(vertex_index) vi: u32) -> VertexOut { + let verts = array, 6>( + vec2(-1.0, -1.0), vec2( 1.0, -1.0), vec2(-1.0, 1.0), + vec2(-1.0, 1.0), vec2( 1.0, -1.0), vec2( 1.0, 1.0), + ); + var out: VertexOut; + out.pos = vec4(verts[vi], 0.0, 1.0); + return out; +} + +struct Params { + width: u32, // image width in pixels (= canvas width) +}; + +@group(0) @binding(0) var image: array>; +@group(0) @binding(1) var alpha: array>; +@group(0) @binding(2) var params: Params; + +@fragment +fn fs(in: VertexOut) -> @location(0) vec4 { + let x = u32(in.pos.x); + let y = u32(in.pos.y); + let i = y * params.width + x; + + let fg = image[i].rgb; + let a = alpha[i].r; + // Premultiplied subject over transparency: rgb·a, matte as alpha. + return vec4(fg * a, a); +} diff --git a/src/model/backends/webgpu/shaders/composite_transparent_f16.wgsl b/src/model/backends/webgpu/shaders/composite_transparent_f16.wgsl new file mode 100644 index 0000000..06745f2 --- /dev/null +++ b/src/model/backends/webgpu/shaders/composite_transparent_f16.wgsl @@ -0,0 +1,44 @@ +enable f16; + +// Composite an RGBA image over TRANSPARENCY, gated by a 1-ch alpha. The matte +// becomes the canvas alpha channel, so the subject is isolated on a transparent +// background (premultiplied output). f16 variant: image and alpha are stored as +// f16 and promote to f32 on read; the fragment writes f32 to the swapchain. +// +// Caller invariants (matched in CompositeTransparentWebGPU): +// - image and alpha are NHWC vec4 storage buffers, same h × w +// - canvas.width === image.w, canvas.height === image.h (no resampling) + +struct VertexOut { + @builtin(position) pos: vec4, +}; + +@vertex +fn vs(@builtin(vertex_index) vi: u32) -> VertexOut { + let verts = array, 6>( + vec2(-1.0, -1.0), vec2( 1.0, -1.0), vec2(-1.0, 1.0), + vec2(-1.0, 1.0), vec2( 1.0, -1.0), vec2( 1.0, 1.0), + ); + var out: VertexOut; + out.pos = vec4(verts[vi], 0.0, 1.0); + return out; +} + +struct Params { + width: u32, +}; + +@group(0) @binding(0) var image: array>; +@group(0) @binding(1) var alpha: array>; +@group(0) @binding(2) var params: Params; + +@fragment +fn fs(in: VertexOut) -> @location(0) vec4 { + let x = u32(in.pos.x); + let y = u32(in.pos.y); + let i = y * params.width + x; + + let fg = vec3(image[i].rgb); + let a = f32(alpha[i].r); + return vec4(fg * a, a); +} diff --git a/src/model/effects/compositor_matte.ts b/src/model/effects/compositor_matte.ts new file mode 100644 index 0000000..c91912f --- /dev/null +++ b/src/model/effects/compositor_matte.ts @@ -0,0 +1,24 @@ +import type { Backend, Tensor, Presenter, RenderTarget } from '~/model/backend.ts' + +// Renders the raw 1-channel alpha matte as a premultiplied white silhouette to +// the backend's canvas — a debug view and a reusable mask (composite it against +// your own full-resolution source). Alpha only — no image, no background. +// Backend-agnostic — dispatches via the `Backend` interface. +// +// Caller invariants (enforced inside the per-backend op): +// - canvas.width === alpha.w, canvas.height === alpha.h (no resampling) +export class CompositorMatte { + private readonly presenter: Presenter + + constructor( + backend: Backend, + alpha: Tensor, + target: RenderTarget = 'main', + ) { + this.presenter = backend.presenters.CompositeMatte(alpha, target) + } + + run(): void { + this.presenter.run() + } +} diff --git a/src/model/effects/compositor_transparent.ts b/src/model/effects/compositor_transparent.ts new file mode 100644 index 0000000..3583e77 --- /dev/null +++ b/src/model/effects/compositor_transparent.ts @@ -0,0 +1,27 @@ +import type { Backend, Tensor, Presenter, RenderTarget } from '~/model/backend.ts' + +// Composites an RGBA image over TRANSPARENCY using a 1-channel alpha mask, +// rendering to the backend's canvas. The matte becomes the canvas alpha channel +// so the subject is isolated on a transparent background (whatever sits behind +// the canvas shows through). Backend-agnostic — dispatches via the `Backend` +// interface, mirroring CompositorSolid. +// +// Caller invariants (enforced inside the per-backend op): +// - image and alpha are the same h × w (run the upscaler first if needed) +// - canvas.width === image.w, canvas.height === image.h (no resampling) +export class CompositorTransparent { + private readonly presenter: Presenter + + constructor( + backend: Backend, + image: Tensor, + alpha: Tensor, + target: RenderTarget = 'main', + ) { + this.presenter = backend.presenters.CompositeTransparent(image, alpha, target) + } + + run(): void { + this.presenter.run() + } +} diff --git a/src/model/render_op.ts b/src/model/render_op.ts index 05d041c..964673c 100644 --- a/src/model/render_op.ts +++ b/src/model/render_op.ts @@ -4,6 +4,8 @@ import { BicubicUpscaler } from '~/model/effects/upscale_bicubic.ts' import { CompositorSolid } from '~/model/effects/compositor_solid.ts' import { CompositorImage } from '~/model/effects/compositor_image.ts' import { CompositorBlur } from '~/model/effects/compositor_blur.ts' +import { CompositorTransparent } from '~/model/effects/compositor_transparent.ts' +import { CompositorMatte } from '~/model/effects/compositor_matte.ts' export type UpscalerMode = 'bilinear' | 'bicubic' @@ -11,6 +13,10 @@ export type BackgroundConfig = | { mode: 'solid'; color: [number, number, number] } | { mode: 'image'; image: Tensor } | { mode: 'blur'; sigma: number } + // Isolate the subject on a transparent background (matte → canvas alpha). + | { mode: 'transparent' } + // Render the raw alpha matte as a white silhouette. + | { mode: 'matte' } // What compositeTo() can render: an effect (BackgroundConfig) or raw // passthrough (input image straight to the target, no alpha/bg). The renderer @@ -209,9 +215,11 @@ export class RenderOp { if (!this.upscaler) throw new Error('RenderOp.compositeTo (effect spec) called before attachNetwork') const alpha = this.upscaler.output switch (spec.mode) { - case 'solid': return new CompositorSolid(backend, image, alpha, spec.color, target) - case 'image': return new CompositorImage(backend, image, alpha, spec.image, target) - case 'blur': return new CompositorBlur(backend, image, alpha, spec.sigma, target) + case 'solid': return new CompositorSolid(backend, image, alpha, spec.color, target) + case 'image': return new CompositorImage(backend, image, alpha, spec.image, target) + case 'blur': return new CompositorBlur(backend, image, alpha, spec.sigma, target) + case 'transparent': return new CompositorTransparent(backend, image, alpha, target) + case 'matte': return new CompositorMatte(backend, alpha, target) } } } @@ -224,6 +232,9 @@ function sameSpec(a: CompositeSpec, b: CompositeSpec): boolean { if (a.mode !== b.mode) return false switch (b.mode) { case 'passthrough': return true + // No parameters — mode equality (checked above) is sufficient. + case 'transparent': return true + case 'matte': return true case 'solid': { const ac = (a as { color: [number, number, number] }).color return ac[0] === b.color[0] && ac[1] === b.color[1] && ac[2] === b.color[2] diff --git a/src/pipeline/background.ts b/src/pipeline/background.ts index 3fdc532..f92566f 100644 --- a/src/pipeline/background.ts +++ b/src/pipeline/background.ts @@ -21,6 +21,8 @@ const SIGMA_HIGH = 16 export type BackgroundInput = | 'blur' | 'none' + | 'transparent' // isolate subject on transparency (matte → alpha) + | 'matte' // render the raw alpha matte (white silhouette) | string // URL → image | ImageBitmap | HTMLImageElement @@ -54,6 +56,8 @@ export type VideoInput = export type Background = | { kind: 'none' } + | { kind: 'transparent' } // subject isolated on transparency + | { kind: 'matte' } // raw alpha matte (white silhouette) | { kind: 'color'; rgb: [number, number, number] } // floats in [0, 1] — shader-ready | { kind: 'blur'; sigma: number } | { kind: 'image'; bitmap: ImageBitmap } @@ -78,6 +82,8 @@ export interface NormalizedBackground { export async function normalizeBackground(input: BackgroundInput): Promise { if (typeof input === 'string') { if (input === 'none') return { background: { kind: 'none' } } + if (input === 'transparent') return { background: { kind: 'transparent' } } + if (input === 'matte') return { background: { kind: 'matte' } } if (input === 'blur') return { background: { kind: 'blur', sigma: SIGMA_MEDIUM } } return { background: { kind: 'image', bitmap: await loadImageFromUrl(input) } } } diff --git a/src/pipeline/worker/renderer.ts b/src/pipeline/worker/renderer.ts index be8602c..ffff6cf 100644 --- a/src/pipeline/worker/renderer.ts +++ b/src/pipeline/worker/renderer.ts @@ -488,6 +488,10 @@ export class Renderer { switch (bg.kind) { case 'none': return { mode: 'solid', color: [0, 0, 0] } + case 'transparent': + return { mode: 'transparent' } + case 'matte': + return { mode: 'matte' } case 'color': return { mode: 'solid', color: bg.rgb } case 'blur': diff --git a/tests/effects/composite_alpha.test.ts b/tests/effects/composite_alpha.test.ts new file mode 100644 index 0000000..11cc72b --- /dev/null +++ b/tests/effects/composite_alpha.test.ts @@ -0,0 +1,79 @@ +import { describe, it, expect } from 'vitest' +import { WebGLBackend } from '~/model/backends/webgl/index' +import { WebGPUBackend } from '~/model/backends/webgpu/index' + +// The transparent + matte compositors render straight to the canvas (no Tensor +// output), so — unlike the op tests — we assert on canvas pixels. WebGL is the +// correctness path (gl.readPixels off the default framebuffer). WebGPU gets a +// compile-and-run smoke (its swapchain isn't COPY_SRC-configured, so pixel +// readback would need backend changes; the shader compiling + a full frame +// submitting is what we're guarding). +// +// Two horizontal pixels: x=0 is fully opaque foreground, x=1 is fully keyed +// out (alpha 0). Both backends' canvases are premultiplied, so the expected +// output is premultiplied: fg·a in rgb, matte in a. + +const RED_THEN_BLUE = new Float32Array([ + 1, 0, 0, 1, // px0: red, image alpha unused by the compositor + 0, 0, 1, 1, // px1: blue +]) +const ALPHA_1_THEN_0 = new Float32Array([ + 1, 0, 0, 0, // px0: matte = 1 (kept) + 0, 0, 0, 0, // px1: matte = 0 (transparent) +]) + +function readWebGLCanvas(backend: WebGLBackend, w: number, h: number): Uint8Array { + const gl = backend.gl + const buf = new Uint8Array(w * h * 4) + gl.bindFramebuffer(gl.FRAMEBUFFER, null) // default framebuffer === canvas + gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, buf) + return buf +} + +describe('CompositeTransparent (WebGL)', () => { + it('writes premultiplied subject with the matte as alpha', () => { + const backend = WebGLBackend.create({ canvas: new OffscreenCanvas(2, 1) }) + const image = backend.tensor(1, 2, 4, RED_THEN_BLUE) + const alpha = backend.tensor(1, 2, 4, ALPHA_1_THEN_0) + + backend.presenters.CompositeTransparent(image, alpha).run() + const px = readWebGLCanvas(backend, 2, 1) + backend.destroy() + + // px0: red × 1 → (255,0,0,255). px1: keyed out → (0,0,0,0). + expect(Array.from(px.slice(0, 4))).toEqual([255, 0, 0, 255]) + expect(Array.from(px.slice(4, 8))).toEqual([0, 0, 0, 0]) + }) +}) + +describe('CompositeMatte (WebGL)', () => { + it('writes a premultiplied white silhouette (rgb = a, alpha = a)', () => { + const backend = WebGLBackend.create({ canvas: new OffscreenCanvas(2, 1) }) + const alpha = backend.tensor(1, 2, 4, ALPHA_1_THEN_0) + + backend.presenters.CompositeMatte(alpha).run() + const px = readWebGLCanvas(backend, 2, 1) + backend.destroy() + + expect(Array.from(px.slice(0, 4))).toEqual([255, 255, 255, 255]) + expect(Array.from(px.slice(4, 8))).toEqual([0, 0, 0, 0]) + }) +}) + +describe('CompositeTransparent / CompositeMatte (WebGPU)', () => { + it('compiles and submits a frame', async () => { + if (!navigator.gpu || !(await navigator.gpu.requestAdapter().catch(() => null))) { + // No WebGPU in this browser — the WebGL cases above cover correctness. + return + } + const backend = await WebGPUBackend.create({ canvas: new OffscreenCanvas(2, 1) }) + const image = backend.tensor(1, 2, 4, RED_THEN_BLUE) + const alpha = backend.tensor(1, 2, 4, ALPHA_1_THEN_0) + + // Throws if the WGSL fails to compile or the pass fails to submit. + expect(() => backend.presenters.CompositeTransparent(image, alpha).run()).not.toThrow() + expect(() => backend.presenters.CompositeMatte(alpha).run()).not.toThrow() + await backend.sync() + backend.destroy() + }) +})