engine-candidate: check stable-diffusion.cpp builds (engine=sd-cpp)

.github/engine-candidate/sd-cpp.json

@@ -0,0 +1,105 @@
+{
+  "engine": "sd-cpp",
+  "kind": "sdcpp",
+  "contract": "test/fixtures/sdcpp/engine-contract.json",
+  "default_repo": "AtomicBot-ai/stable-diffusion.cpp",
+  "tag_pattern": "^master-\\d+-[0-9a-f]{7}(-a[0-9a-f]{7})?$",
+  "pins": "sdcpp-manifest",
+  "pin_repo": "AtomicBot-ai/atomic-chat-conf",
+  "mirror": false,
+  "candidate_manifests": {
+    "conf": "backends/sdcpp-manifest.staging.json"
+  },
+  "upstream": {
+    "repo": "leejet/stable-diffusion.cpp",
+    "branch": "master"
+  },
+  "live_models": [
+    {
+      "id": "diffusion",
+      "url": "https://huggingface.co/unsloth/FLUX.2-klein-4B-GGUF/resolve/0084d1df98e2e2137fe776d55170bc4792ec1d66/flux-2-klein-4b-Q4_K_M.gguf",
+      "sha256": "0b25d143c8469b342bc5af3bce92b783bf6b0636d285f7b2f75e38af63af9a15",
+      "bytes": 2604311104
+    },
+    {
+      "id": "llm",
+      "url": "https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/a06e946bb6b655725eafa393f4a9745d460374c9/Qwen3-4B-Instruct-2507-Q4_K_M.gguf",
+      "sha256": "3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597",
+      "bytes": 2497281120
+    },
+    {
+      "id": "vae",
+      "url": "https://huggingface.co/unsloth/FLUX.2-VAE/resolve/bda21a63a9d498e7e718221c75f0a7fed443369f/split_files/vae/flux2-vae.safetensors",
+      "sha256": "d64f3a68e1cc4f9f4e29b6e0da38a0204fe9a49f2d4053f0ec1fa1ca02f9c4b5",
+      "bytes": 336213556
+    }
+  ],
+  "live_env": {
+    "ATOMIC_LIVE_SD_ENGINE": "{engine_dir}",
+    "ATOMIC_LIVE_SD_TAG": "{tag}",
+    "ATOMIC_LIVE_SD_MANIFEST": "{manifest}",
+    "ATOMIC_LIVE_SD_MODEL": "{model:diffusion}",
+    "ATOMIC_LIVE_SD_LLM": "{model:llm}",
+    "ATOMIC_LIVE_SD_VAE": "{model:vae}",
+    "ATOMIC_LIVE_SD_VAE_FORMAT": "flux2",
+    "ATOMIC_LIVE_SD_FAMILY": "flux.2-klein",
+    "ATOMIC_LIVE_SD_BACKEND": "cpu"
+  },
+  "platforms": [
+    {
+      "platform": "linux-x64",
+      "runner": "ubuntu-latest"
+    },
+    {
+      "platform": "linux-arm64",
+      "runner": "ubuntu-24.04-arm"
+    },
+    {
+      "platform": "macos-arm64",
+      "runner": "macos-26",
+      "core_runner": "macos-14"
+    }
+  ],
+  "live_timeout_minutes": 90,
+  "live_files": ["test/live/diffusion.test.ts", "test/live/diffusion-install.test.ts"],
+  "required_live_tests": [
+    {
+      "file": "test/live/diffusion-install.test.ts",
+      "name": "a real stable-diffusion.cpp build installed from its manifest is picked for this host, downloaded, checked, unpacked and probed"
+    },
+    {
+      "file": "test/live/diffusion-install.test.ts",
+      "name": "a real stable-diffusion.cpp build installed from its manifest names the commit of its tag"
+    },
+    {
+      "file": "test/live/diffusion-install.test.ts",
+      "name": "a real stable-diffusion.cpp build installed from its manifest is refused when its sha256 does not match the manifest"
+    },
+    {
+      "file": "test/live/diffusion.test.ts",
+      "name": "a real stable-diffusion.cpp engine answers --help with every flag the core passes"
+    },
+    {
+      "file": "test/live/diffusion.test.ts",
+      "name": "a real stable-diffusion.cpp engine installs the tree, loads the model, generates, cancels and unloads"
+    }
+  ],
+  "core_suites": ["unit", "build", "build_bin", "e2e"],
+  "protected_paths": [
+    ".github/engine-candidate/",
+    ".github/workflows/engine-candidate.yml",
+    "scripts/engine-candidate.mjs",
+    "scripts/engine-surface-diff.mjs",
+    "scripts/fetch-live-fixtures.mjs",
+    "scripts/lib/",
+    "src/diffusion/args.test.ts",
+    "test/contract/engine-pin.test.ts",
+    "test/contract/engine-results.test.ts",
+    "test/contract/engine-surface.test.ts",
+    "test/contract/sdcpp-*",
+    "test/fixtures/sdcpp/",
+    "test/live/diffusion*",
+    "vitest.config.ts"
+  ],
+  "not_verified": ["metal", "cuda", "vulkan", "rocm", "windows"]
+}

.github/workflows/engine-candidate.yml

@@ -1,22 +1,23 @@
 name: engine-candidate
 run-name: engine-candidate ${{ inputs.engine }} ${{ inputs.tag || 'newest' }} [cid:${{ inputs.candidate_id }}] core@${{ inputs.core_ref }}
 
-# Is a llama.cpp build compatible with a core: upstream (`llamacpp-upstream`) or PrismML's fork
-# (`atomic-prism`)? Started only by hand or by the update orchestrator through the API: no schedule,
-# no push or pull_request trigger, never called from ci.yml (ADR
-# 2026-10-15-check-an-engine-candidate-only-when-asked). Each engine's tags, repo, pin source,
-# platforms and live tests are in .github/engine-candidate/<engine>.json. The same check on one
-# machine is `npm run engine:check`; docs/live-tests.md, "Checking a llama.cpp build", has both.
+# Is an engine build compatible with a core: upstream llama.cpp (`llamacpp-upstream`), PrismML's fork
+# (`atomic-prism`) or stable-diffusion.cpp (`sd-cpp`)? Started only by hand or by the update
+# orchestrator through the API: no schedule, no push or pull_request trigger, never called from
+# ci.yml (ADR 2026-10-15-check-an-engine-candidate-only-when-asked). Each engine's tags, repo, pin
+# source, platforms and live tests are in .github/engine-candidate/<engine>.json, and the core
+# suites run on that engine's platforms only. The same check on one machine is `npm run
+# engine:check`; docs/live-tests.md, "Checking a llama.cpp build", has both.
 on:
   workflow_dispatch:
     inputs:
       engine:
         description: Engine to check
         type: choice
-        options: [llamacpp-upstream, atomic-prism]
+        options: [llamacpp-upstream, atomic-prism, sd-cpp]
         default: llamacpp-upstream
       tag:
-        description: Candidate tag (b11617, prism-b10787-73bfe6c); empty takes the newest one with every archive up
+        description: Candidate tag (b11617, prism-b10787-73bfe6c, master-975-95b3090); empty takes the newest one with every archive up
         type: string
         default: ''
       repo:
@@ -36,11 +37,11 @@ on:
         type: string
         default: ''
       conf_ref:
-        description: atomic-chat-conf ref for the release lists and the mirror whitelist
+        description: atomic-chat-conf ref for the release lists, manifests and the mirror whitelist
         type: string
         default: main
       suites:
-        description: all = core suites on 5 OS and the live matrix; live = the live matrix only
+        description: all = core suites on the engine's platforms and the live matrix; live = the live matrix only
         type: choice
         options: [all, live]
         default: all
@@ -62,6 +63,9 @@ jobs:
       core_sha: ${{ steps.resolve.outputs.core_sha }}
       complete: ${{ steps.resolve.outputs.complete }}
       live_matrix: ${{ steps.resolve.outputs.live_matrix }}
+      core_matrix: ${{ steps.resolve.outputs.core_matrix }}
+      live_timeout_minutes: ${{ steps.resolve.outputs.live_timeout_minutes }}
+      live_models_key: ${{ steps.resolve.outputs.live_models_key }}
     steps:
       - uses: actions/checkout@v7
         with:
@@ -126,24 +130,18 @@ jobs:
             ${{ runner.temp }}/surface/watched.patch
           if-no-files-found: ignore
 
-  # The core at core_ref on its own: the same suites ci.yml runs, as results rather than gates.
+  # The core at core_ref on its own: the same suites ci.yml runs, as results rather than gates, on
+  # the platforms of the engine's config (resolve's core_matrix: all five for llama.cpp). Windows
+  # spawns processes slowly: as in ci.yml its unit and contract files are split over two jobs
+  # (`--shard`), or the managed-engine tests time out; the first shard also builds and runs e2e, and
+  # results.json merges the two into one cell.
   core:
     needs: resolve
     if: inputs.suites == 'all' && needs.resolve.outputs.complete == 'true'
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - { platform: linux-x64, os: ubuntu-latest }
-          - { platform: linux-arm64, os: ubuntu-24.04-arm }
-          - { platform: macos-arm64, os: macos-14 }
-          # Windows spawns processes slowly: as in ci.yml its unit and contract files are split over
-          # two jobs (`--shard`), or the managed-engine tests time out; the first shard also builds
-          # and runs e2e, and results.json merges the two into one cell.
-          - { platform: windows-x64, os: windows-2022, shard: 1/2, part: '-1' }
-          - { platform: windows-x64, os: windows-2022, shard: 2/2, part: '-2' }
-          - { platform: windows-arm64, os: windows-11-arm, shard: 1/2, part: '-1' }
-          - { platform: windows-arm64, os: windows-11-arm, shard: 2/2, part: '-2' }
+        include: ${{ fromJSON(needs.resolve.outputs.core_matrix || '[]') }}
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v7
@@ -212,6 +210,8 @@ jobs:
     needs: resolve
     if: needs.resolve.outputs.complete == 'true'
     name: live ${{ matrix.platform }} ${{ matrix.role }} ${{ matrix.tag }}
+    # GitHub's 6 hours unless the engine sets its own (sd.cpp: an image model on a CPU runner).
+    timeout-minutes: ${{ fromJSON(needs.resolve.outputs.live_timeout_minutes || '360') }}
     strategy:
       fail-fast: false
       matrix:
@@ -238,8 +238,16 @@ jobs:
           path: ${{ runner.temp }}/live
           key: live-${{ inputs.engine }}-${{ matrix.platform }}-${{ matrix.repo }}-${{ matrix.tag }}
           restore-keys: live-${{ inputs.engine }}-${{ matrix.platform }}-
+      # An engine's several model files (`live_models`: sd.cpp's 5.4 GB), apart from its builds: one
+      # entry keyed by their sha256, on a path Linux and macOS spell alike, so every platform shares it
+      # within the 10 GB the repository may cache.
+      - if: needs.resolve.outputs.live_models_key != ''
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/atomic-live-models
+          key: live-models-${{ needs.resolve.outputs.live_models_key }}
       - id: fetch
-        name: Fetch the build and the engine's GGUF
+        name: Fetch the build and the engine's models
         shell: bash
         env:
           GITHUB_TOKEN: ${{ github.token }}
@@ -249,8 +257,9 @@ jobs:
           PLATFORM: ${{ matrix.platform }}
           ROLE: ${{ matrix.role }}
         run: |
-          node scripts/fetch-live-fixtures.mjs --out "$RUNNER_TEMP/live" --tag "$TAG" --repo "$REPO" --engine "$ENGINE"
-          echo "llama.cpp \`$TAG\` from $REPO, $ROLE on $PLATFORM" >> "$GITHUB_STEP_SUMMARY"
+          node scripts/fetch-live-fixtures.mjs --out "$RUNNER_TEMP/live" --tag "$TAG" --repo "$REPO" --engine "$ENGINE" \
+            --models "$HOME/.cache/atomic-live-models"
+          echo "$ENGINE \`$TAG\` from $REPO, $ROLE on $PLATFORM" >> "$GITHUB_STEP_SUMMARY"
       - name: Live tests
         if: steps.fetch.outcome == 'success'
         shell: bash

docs/decisions/2026-10-21-check-sdcpp-builds-with-the-engine-candidate.md

@@ -0,0 +1,86 @@
+---
+date: 2026-10-21
+title: "Check stable-diffusion.cpp builds with the engine candidate workflow"
+---
+
+# 2026-10-21 — Check stable-diffusion.cpp builds with the engine candidate workflow
+
+- **Context:** the engine candidate check
+  ([2026-10-15](2026-10-15-check-an-engine-candidate-only-when-asked.md),
+  [2026-10-20](2026-10-20-check-prismml-builds-with-the-engine-candidate.md)) covered llama.cpp builds
+  only. The engine update pipeline (atomic-hatchet-pipeline) now updates stable-diffusion.cpp too: the
+  Atomic fork (`AtomicBot-ai/stable-diffusion.cpp`) builds each leejet tag `master-<N>-<sha7>`, conf
+  signs it into a release `master-<N>-<sha7>-a<rev>` and stages it in `backends/sdcpp-manifest.staging.json`,
+  and approval moves it into `backends/sdcpp-manifest.json`, which every core installs from (no
+  `min_core_version`). The pipeline reads only `results.json`, and needs one for both runs: the fork's
+  build at `core_ref=main`, and the signed mirror at the core release, the workflow started from that
+  release's ref. Nothing in the llama.cpp check fits sd.cpp as it is: other binaries, another help
+  format, archives named by the commit rather than the tag, a manifest instead of a release list, and a
+  model that is several files.
+- **Decision:** `engine-candidate.yml` takes `engine=sd-cpp`. `.github/engine-candidate/sd-cpp.json`
+  (`kind: sdcpp`) holds what differs, read by the same scripts:
+  - Tags `master-<N>-<sha7>(-a<rev>)`, as `SDCPP_TAG_RE` in `src/engine-builds/manifest.ts`. The
+    candidate's `commit` is the leejet tag's (`git ls-remote`); `results.json` adds `upstream_tag`
+    and `atomic_rev` to the candidate, and `manifest` on conf.
+  - **Complete** means every archive conf's mirror requires and the CPU archive of each live platform
+    is up exactly once with GitHub's digest; on conf, also a manifest naming the tag (staging, or the
+    live one when that is the one naming it) whose sizes and sha256 equal the release's and whose
+    `download_base` is conf. A staging manifest that moved on to another tag leaves the run
+    `incomplete`, never checks something other than what would be approved.
+  - **The pin** is the live manifest's `tag_name` on each platform whose CPU archive it pins
+    (`resolveSdcppPins`, held to `parseSdcppManifest` and `sdcppHostChoice` by
+    `test/contract/engine-pin.test.ts`); its archives come from conf (`pin_repo`), the only place a
+    `-a<rev>` tag exists.
+  - **The contract** (`test/fixtures/sdcpp/engine-contract.json`) is written by hand and held to the
+    code by `test/contract/sdcpp-engine-contract.test.ts`: the flags of `required-flags.txt` plus `-v`
+    (`break`), `-M` (`warn`: emitted only when opted in, and sd-server does not have it); `sd-server
+    --help`, `--version` and `sd-cli --help`; the probe markers of `install.ts`; the routes and the
+    progress, out-of-memory, abort and GPU-fault text core reads; the archives by backend, after conf's
+    `mirror-sdcpp.yml` and the backend ladder; and the leejet sources they live in.
+  - **The surface** (`scripts/lib/sdcpp-surface.mjs`) writes the same `atomic-engine-surface-report`
+    v1 with sd.cpp's checks. Flags are matched as whole aliases, so `--vae-format` never stands for
+    `--vae`; an exit code is a `warn` (the probe reads output, not exit codes) and a probe binary
+    without a marker a `break`; `--version` has to name the tag's commit (the fork's builds say
+    `version unknown`, an `info`). The history is leejet's between the two tags, cloned from leejet and
+    shallowed by the dates of leejet's releases; the git cache is now one folder per upstream repo.
+  - **Live** runs on linux-x64, linux-arm64 and macos-arm64 (no Windows: the image test skips win32,
+    and every run would stay `incomplete`), on the CPU archives, pin and candidate. FLUX.2 Klein 4B
+    Q4_K_M with Qwen3-4B-Instruct-2507 Q4_K_M as `--llm` and the FLUX.2 VAE — conf's catalog files, 5.4
+    GB, pinned by Hugging Face revision and sha256 (`live_models`) — in a cache entry of their own keyed
+    by those hashes, on a path Linux and macOS share, so one copy fits GitHub's 10 GB. SD-Turbo, the
+    fork's own smoke model, cannot be used: core passes `--diffusion-model`, and sd.cpp then prefixes
+    the checkpoint's VAE and CLIP tensors with `model.diffusion_model.` and fails to tell its version
+    (seen on master-883: `get sd version from file failed`). `ATOMIC_DIFFUSION_BACKEND=cpu`, a new
+    narrow switch in `DiffusionService`, starts every load with `--backend cpu`: the live run sets it,
+    because a macOS runner's paravirtual GPU crashes ggml's Metal kernels. The new
+    `test/live/diffusion-install.test.ts` installs the release through a manifest of conf's shape
+    (`ATOMIC_SDCPP_MANIFEST_URL`): pick, download, sha256, unpack, probe, a tampered sha256 refused,
+    and `--version` naming the tag's commit. macOS runs on `macos-26`: the fork's macOS archive is built
+    with `minos 26.0`.
+  - The core suites run on the engine's platforms (`core_matrix` from resolve, `core_runner` where it
+    differs from the live runner); the pipeline keeps `suites=all`. For llama.cpp the matrix is the
+    same seven jobs as before. The live job's timeout comes from the config (`live_timeout_minutes`:
+    90 for sd.cpp; GitHub's 360 otherwise). Required live tests may name `platforms`.
+- **Consequences:** a check of an sd.cpp build is three core jobs and six live jobs; the first run
+  downloads 5.4 GB per live job, later ones restore it. On this machine (macOS arm64, M4 Pro, CPU
+  backend) `npm run engine:check -- --engine sd-cpp --tag master-883-137f740` passes against the pin
+  `master-883-137f740-a36f1b1a` in 9 minutes, about 4 minutes of them per generation test; a 3- or
+  4-core runner is expected to take several times longer, inside the 90-minute timeout. A core release older than this one has no
+  `.github/engine-candidate/sd-cpp.json`, so the pipeline holds an sd.cpp rollout until a core release
+  carries it. Metal, CUDA, Vulkan, ROCm and Windows are not run (`not_verified`). The macOS runner has 7
+  GB of memory for 5.4 GB of weights; if it pages too much, the live timeout says so. `-M` and three
+  deprecated flags core still emits (`--qwen2vl`, `--clip-on-cpu`, `--vae-on-cpu`) make every surface
+  report a `warn` until core moves off them. The `generating video:` line the tracker waits for is not
+  printed by leejet at master-883; the report lists it as text in neither tree.
+- **Owner:** team
+- **Links:** `.github/engine-candidate/sd-cpp.json`, `.github/workflows/engine-candidate.yml`,
+  `scripts/lib/{sdcpp-surface,sdcpp-fixtures,engine-io,engine-pin,engine-results,live-fixtures}.mjs`,
+  `scripts/{engine-candidate,engine-surface-diff,fetch-live-fixtures}.mjs`,
+  `test/fixtures/sdcpp/engine-contract.json`, `test/contract/sdcpp-{engine-contract,surface}.test.ts`,
+  `test/live/diffusion{,-install}.test.ts`, `src/diffusion/{args,service}.ts`, `docs/live-tests.md`
+  ("stable-diffusion.cpp builds"); atomic-chat-conf `mirror-sdcpp.yml` / `sdcpp-approve.yml`;
+  atomic-hatchet-pipeline `docs/engine-updater.md`.
+
+<!--
+Refines: 2026-10-15-check-an-engine-candidate-only-when-asked.md (sd.cpp is now covered)
+-->

docs/decisions/INDEX.md

@@ -11,6 +11,7 @@ add a new one that says which record it supersedes.
 
 ## Records
 
+- **2026-10-21** — [Check stable-diffusion.cpp builds with the engine candidate workflow](2026-10-21-check-sdcpp-builds-with-the-engine-candidate.md) — `engine=sd-cpp` (`kind: sdcpp`): fork builds and conf's signed mirror checked against the staging manifest, pin from conf's sd.cpp manifest, sd.cpp's own surface checks in the same report schema, live on three CPU platforms with FLUX.2 Klein and a manifest install; `ATOMIC_DIFFUSION_BACKEND=cpu`; core suites on the engine's platforms.
 - **2026-10-20** — [Check PrismML builds with the engine candidate workflow, as upstream ones are checked](2026-10-20-check-prismml-builds-with-the-engine-candidate.md) — `engine=atomic-prism`: tags, repo, branch, pin source (conf's PrismML manifest), no mirror and the Bonsai live test are data in `.github/engine-candidate/atomic-prism.json`; refines the 2026-10-15 candidate check.
 - **2026-10-19** — [Process identities do not depend on who reads them](2026-10-19-process-identities-do-not-depend-on-the-reader.md) — `darwin-utc:`/`win32-utc:` start identities (`ps` under `LC_ALL=C TZ=UTC`, UTC ticks); a legacy `darwin:`/`win32:` difference is `unknown`, never `mismatch`; the instance lock decides by `owner_started_at` first; download sidecars are not listed as diffusion model files.
 - **2026-10-15** — [Foundry Local runs in a hidden host mode of the core binary, one process per model](2026-10-15-foundry-local-runs-in-a-host-mode-of-the-core-binary.md) — `foundry-host probe|prepare|serve` on `bun:ffi` (the one `bun:*` file in `src/`, allow-listed), started with `process.execPath`; sessions behind the core's gateway, forwarded under `/catalogs/local`; `SpawnSpec.stdin: 'pipe'` and the gateway's `basePath` added; Windows arm64, notarized macOS and NPU still gate the release.