diff --git a/.github/engine-candidate/sd-cpp.json b/.github/engine-candidate/sd-cpp.json new file mode 100644 index 00000000..ac71733c --- /dev/null +++ b/.github/engine-candidate/sd-cpp.json @@ -0,0 +1,105 @@ +{ + "engine": "sd-cpp", + "kind": "sdcpp", + "contract": "test/fixtures/sdcpp/engine-contract.json", + "default_repo": "AtomicBot-ai/stable-diffusion.cpp", + "tag_pattern": "^master-\\d+-[0-9a-f]{7}(-a[0-9a-f]{7})?$", + "pins": "sdcpp-manifest", + "pin_repo": "AtomicBot-ai/atomic-chat-conf", + "mirror": false, + "candidate_manifests": { + "conf": "backends/sdcpp-manifest.staging.json" + }, + "upstream": { + "repo": "leejet/stable-diffusion.cpp", + "branch": "master" + }, + "live_models": [ + { + "id": "diffusion", + "url": "https://huggingface.co/unsloth/FLUX.2-klein-4B-GGUF/resolve/0084d1df98e2e2137fe776d55170bc4792ec1d66/flux-2-klein-4b-Q4_K_M.gguf", + "sha256": "0b25d143c8469b342bc5af3bce92b783bf6b0636d285f7b2f75e38af63af9a15", + "bytes": 2604311104 + }, + { + "id": "llm", + "url": "https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/a06e946bb6b655725eafa393f4a9745d460374c9/Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + "sha256": "3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597", + "bytes": 2497281120 + }, + { + "id": "vae", + "url": "https://huggingface.co/unsloth/FLUX.2-VAE/resolve/bda21a63a9d498e7e718221c75f0a7fed443369f/split_files/vae/flux2-vae.safetensors", + "sha256": "d64f3a68e1cc4f9f4e29b6e0da38a0204fe9a49f2d4053f0ec1fa1ca02f9c4b5", + "bytes": 336213556 + } + ], + "live_env": { + "ATOMIC_LIVE_SD_ENGINE": "{engine_dir}", + "ATOMIC_LIVE_SD_TAG": "{tag}", + "ATOMIC_LIVE_SD_MANIFEST": "{manifest}", + "ATOMIC_LIVE_SD_MODEL": "{model:diffusion}", + "ATOMIC_LIVE_SD_LLM": "{model:llm}", + "ATOMIC_LIVE_SD_VAE": "{model:vae}", + "ATOMIC_LIVE_SD_VAE_FORMAT": "flux2", + "ATOMIC_LIVE_SD_FAMILY": "flux.2-klein", + "ATOMIC_LIVE_SD_BACKEND": "cpu" + }, + "platforms": [ + { + "platform": "linux-x64", + "runner": "ubuntu-latest" + }, + { + "platform": "linux-arm64", + "runner": "ubuntu-24.04-arm" + }, + { + "platform": "macos-arm64", + "runner": "macos-26", + "core_runner": "macos-14" + } + ], + "live_timeout_minutes": 90, + "live_files": ["test/live/diffusion.test.ts", "test/live/diffusion-install.test.ts"], + "required_live_tests": [ + { + "file": "test/live/diffusion-install.test.ts", + "name": "a real stable-diffusion.cpp build installed from its manifest is picked for this host, downloaded, checked, unpacked and probed" + }, + { + "file": "test/live/diffusion-install.test.ts", + "name": "a real stable-diffusion.cpp build installed from its manifest names the commit of its tag" + }, + { + "file": "test/live/diffusion-install.test.ts", + "name": "a real stable-diffusion.cpp build installed from its manifest is refused when its sha256 does not match the manifest" + }, + { + "file": "test/live/diffusion.test.ts", + "name": "a real stable-diffusion.cpp engine answers --help with every flag the core passes" + }, + { + "file": "test/live/diffusion.test.ts", + "name": "a real stable-diffusion.cpp engine installs the tree, loads the model, generates, cancels and unloads" + } + ], + "core_suites": ["unit", "build", "build_bin", "e2e"], + "protected_paths": [ + ".github/engine-candidate/", + ".github/workflows/engine-candidate.yml", + "scripts/engine-candidate.mjs", + "scripts/engine-surface-diff.mjs", + "scripts/fetch-live-fixtures.mjs", + "scripts/lib/", + "src/diffusion/args.test.ts", + "test/contract/engine-pin.test.ts", + "test/contract/engine-results.test.ts", + "test/contract/engine-surface.test.ts", + "test/contract/sdcpp-*", + "test/fixtures/sdcpp/", + "test/live/diffusion*", + "vitest.config.ts" + ], + "not_verified": ["metal", "cuda", "vulkan", "rocm", "windows"] +} diff --git a/.github/workflows/engine-candidate.yml b/.github/workflows/engine-candidate.yml index 7af54fce..5a5b0272 100644 --- a/.github/workflows/engine-candidate.yml +++ b/.github/workflows/engine-candidate.yml @@ -1,22 +1,23 @@ name: engine-candidate run-name: engine-candidate ${{ inputs.engine }} ${{ inputs.tag || 'newest' }} [cid:${{ inputs.candidate_id }}] core@${{ inputs.core_ref }} -# Is a llama.cpp build compatible with a core: upstream (`llamacpp-upstream`) or PrismML's fork -# (`atomic-prism`)? Started only by hand or by the update orchestrator through the API: no schedule, -# no push or pull_request trigger, never called from ci.yml (ADR -# 2026-10-15-check-an-engine-candidate-only-when-asked). Each engine's tags, repo, pin source, -# platforms and live tests are in .github/engine-candidate/.json. The same check on one -# machine is `npm run engine:check`; docs/live-tests.md, "Checking a llama.cpp build", has both. +# Is an engine build compatible with a core: upstream llama.cpp (`llamacpp-upstream`), PrismML's fork +# (`atomic-prism`) or stable-diffusion.cpp (`sd-cpp`)? Started only by hand or by the update +# orchestrator through the API: no schedule, no push or pull_request trigger, never called from +# ci.yml (ADR 2026-10-15-check-an-engine-candidate-only-when-asked). Each engine's tags, repo, pin +# source, platforms and live tests are in .github/engine-candidate/.json, and the core +# suites run on that engine's platforms only. The same check on one machine is `npm run +# engine:check`; docs/live-tests.md, "Checking a llama.cpp build", has both. on: workflow_dispatch: inputs: engine: description: Engine to check type: choice - options: [llamacpp-upstream, atomic-prism] + options: [llamacpp-upstream, atomic-prism, sd-cpp] default: llamacpp-upstream tag: - description: Candidate tag (b11617, prism-b10787-73bfe6c); empty takes the newest one with every archive up + description: Candidate tag (b11617, prism-b10787-73bfe6c, master-975-95b3090); empty takes the newest one with every archive up type: string default: '' repo: @@ -36,11 +37,11 @@ on: type: string default: '' conf_ref: - description: atomic-chat-conf ref for the release lists and the mirror whitelist + description: atomic-chat-conf ref for the release lists, manifests and the mirror whitelist type: string default: main suites: - description: all = core suites on 5 OS and the live matrix; live = the live matrix only + description: all = core suites on the engine's platforms and the live matrix; live = the live matrix only type: choice options: [all, live] default: all @@ -62,6 +63,9 @@ jobs: core_sha: ${{ steps.resolve.outputs.core_sha }} complete: ${{ steps.resolve.outputs.complete }} live_matrix: ${{ steps.resolve.outputs.live_matrix }} + core_matrix: ${{ steps.resolve.outputs.core_matrix }} + live_timeout_minutes: ${{ steps.resolve.outputs.live_timeout_minutes }} + live_models_key: ${{ steps.resolve.outputs.live_models_key }} steps: - uses: actions/checkout@v7 with: @@ -126,24 +130,18 @@ jobs: ${{ runner.temp }}/surface/watched.patch if-no-files-found: ignore - # The core at core_ref on its own: the same suites ci.yml runs, as results rather than gates. + # The core at core_ref on its own: the same suites ci.yml runs, as results rather than gates, on + # the platforms of the engine's config (resolve's core_matrix: all five for llama.cpp). Windows + # spawns processes slowly: as in ci.yml its unit and contract files are split over two jobs + # (`--shard`), or the managed-engine tests time out; the first shard also builds and runs e2e, and + # results.json merges the two into one cell. core: needs: resolve if: inputs.suites == 'all' && needs.resolve.outputs.complete == 'true' strategy: fail-fast: false matrix: - include: - - { platform: linux-x64, os: ubuntu-latest } - - { platform: linux-arm64, os: ubuntu-24.04-arm } - - { platform: macos-arm64, os: macos-14 } - # Windows spawns processes slowly: as in ci.yml its unit and contract files are split over - # two jobs (`--shard`), or the managed-engine tests time out; the first shard also builds - # and runs e2e, and results.json merges the two into one cell. - - { platform: windows-x64, os: windows-2022, shard: 1/2, part: '-1' } - - { platform: windows-x64, os: windows-2022, shard: 2/2, part: '-2' } - - { platform: windows-arm64, os: windows-11-arm, shard: 1/2, part: '-1' } - - { platform: windows-arm64, os: windows-11-arm, shard: 2/2, part: '-2' } + include: ${{ fromJSON(needs.resolve.outputs.core_matrix || '[]') }} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v7 @@ -212,6 +210,8 @@ jobs: needs: resolve if: needs.resolve.outputs.complete == 'true' name: live ${{ matrix.platform }} ${{ matrix.role }} ${{ matrix.tag }} + # GitHub's 6 hours unless the engine sets its own (sd.cpp: an image model on a CPU runner). + timeout-minutes: ${{ fromJSON(needs.resolve.outputs.live_timeout_minutes || '360') }} strategy: fail-fast: false matrix: @@ -238,8 +238,16 @@ jobs: path: ${{ runner.temp }}/live key: live-${{ inputs.engine }}-${{ matrix.platform }}-${{ matrix.repo }}-${{ matrix.tag }} restore-keys: live-${{ inputs.engine }}-${{ matrix.platform }}- + # An engine's several model files (`live_models`: sd.cpp's 5.4 GB), apart from its builds: one + # entry keyed by their sha256, on a path Linux and macOS spell alike, so every platform shares it + # within the 10 GB the repository may cache. + - if: needs.resolve.outputs.live_models_key != '' + uses: actions/cache@v4 + with: + path: ~/.cache/atomic-live-models + key: live-models-${{ needs.resolve.outputs.live_models_key }} - id: fetch - name: Fetch the build and the engine's GGUF + name: Fetch the build and the engine's models shell: bash env: GITHUB_TOKEN: ${{ github.token }} @@ -249,8 +257,9 @@ jobs: PLATFORM: ${{ matrix.platform }} ROLE: ${{ matrix.role }} run: | - node scripts/fetch-live-fixtures.mjs --out "$RUNNER_TEMP/live" --tag "$TAG" --repo "$REPO" --engine "$ENGINE" - echo "llama.cpp \`$TAG\` from $REPO, $ROLE on $PLATFORM" >> "$GITHUB_STEP_SUMMARY" + node scripts/fetch-live-fixtures.mjs --out "$RUNNER_TEMP/live" --tag "$TAG" --repo "$REPO" --engine "$ENGINE" \ + --models "$HOME/.cache/atomic-live-models" + echo "$ENGINE \`$TAG\` from $REPO, $ROLE on $PLATFORM" >> "$GITHUB_STEP_SUMMARY" - name: Live tests if: steps.fetch.outcome == 'success' shell: bash diff --git a/docs/decisions/2026-10-21-check-sdcpp-builds-with-the-engine-candidate.md b/docs/decisions/2026-10-21-check-sdcpp-builds-with-the-engine-candidate.md new file mode 100644 index 00000000..c2c00014 --- /dev/null +++ b/docs/decisions/2026-10-21-check-sdcpp-builds-with-the-engine-candidate.md @@ -0,0 +1,86 @@ +--- +date: 2026-10-21 +title: "Check stable-diffusion.cpp builds with the engine candidate workflow" +--- + +# 2026-10-21 — Check stable-diffusion.cpp builds with the engine candidate workflow + +- **Context:** the engine candidate check + ([2026-10-15](2026-10-15-check-an-engine-candidate-only-when-asked.md), + [2026-10-20](2026-10-20-check-prismml-builds-with-the-engine-candidate.md)) covered llama.cpp builds + only. The engine update pipeline (atomic-hatchet-pipeline) now updates stable-diffusion.cpp too: the + Atomic fork (`AtomicBot-ai/stable-diffusion.cpp`) builds each leejet tag `master--`, conf + signs it into a release `master---a` and stages it in `backends/sdcpp-manifest.staging.json`, + and approval moves it into `backends/sdcpp-manifest.json`, which every core installs from (no + `min_core_version`). The pipeline reads only `results.json`, and needs one for both runs: the fork's + build at `core_ref=main`, and the signed mirror at the core release, the workflow started from that + release's ref. Nothing in the llama.cpp check fits sd.cpp as it is: other binaries, another help + format, archives named by the commit rather than the tag, a manifest instead of a release list, and a + model that is several files. +- **Decision:** `engine-candidate.yml` takes `engine=sd-cpp`. `.github/engine-candidate/sd-cpp.json` + (`kind: sdcpp`) holds what differs, read by the same scripts: + - Tags `master--(-a)`, as `SDCPP_TAG_RE` in `src/engine-builds/manifest.ts`. The + candidate's `commit` is the leejet tag's (`git ls-remote`); `results.json` adds `upstream_tag` + and `atomic_rev` to the candidate, and `manifest` on conf. + - **Complete** means every archive conf's mirror requires and the CPU archive of each live platform + is up exactly once with GitHub's digest; on conf, also a manifest naming the tag (staging, or the + live one when that is the one naming it) whose sizes and sha256 equal the release's and whose + `download_base` is conf. A staging manifest that moved on to another tag leaves the run + `incomplete`, never checks something other than what would be approved. + - **The pin** is the live manifest's `tag_name` on each platform whose CPU archive it pins + (`resolveSdcppPins`, held to `parseSdcppManifest` and `sdcppHostChoice` by + `test/contract/engine-pin.test.ts`); its archives come from conf (`pin_repo`), the only place a + `-a` tag exists. + - **The contract** (`test/fixtures/sdcpp/engine-contract.json`) is written by hand and held to the + code by `test/contract/sdcpp-engine-contract.test.ts`: the flags of `required-flags.txt` plus `-v` + (`break`), `-M` (`warn`: emitted only when opted in, and sd-server does not have it); `sd-server + --help`, `--version` and `sd-cli --help`; the probe markers of `install.ts`; the routes and the + progress, out-of-memory, abort and GPU-fault text core reads; the archives by backend, after conf's + `mirror-sdcpp.yml` and the backend ladder; and the leejet sources they live in. + - **The surface** (`scripts/lib/sdcpp-surface.mjs`) writes the same `atomic-engine-surface-report` + v1 with sd.cpp's checks. Flags are matched as whole aliases, so `--vae-format` never stands for + `--vae`; an exit code is a `warn` (the probe reads output, not exit codes) and a probe binary + without a marker a `break`; `--version` has to name the tag's commit (the fork's builds say + `version unknown`, an `info`). The history is leejet's between the two tags, cloned from leejet and + shallowed by the dates of leejet's releases; the git cache is now one folder per upstream repo. + - **Live** runs on linux-x64, linux-arm64 and macos-arm64 (no Windows: the image test skips win32, + and every run would stay `incomplete`), on the CPU archives, pin and candidate. FLUX.2 Klein 4B + Q4_K_M with Qwen3-4B-Instruct-2507 Q4_K_M as `--llm` and the FLUX.2 VAE — conf's catalog files, 5.4 + GB, pinned by Hugging Face revision and sha256 (`live_models`) — in a cache entry of their own keyed + by those hashes, on a path Linux and macOS share, so one copy fits GitHub's 10 GB. SD-Turbo, the + fork's own smoke model, cannot be used: core passes `--diffusion-model`, and sd.cpp then prefixes + the checkpoint's VAE and CLIP tensors with `model.diffusion_model.` and fails to tell its version + (seen on master-883: `get sd version from file failed`). `ATOMIC_DIFFUSION_BACKEND=cpu`, a new + narrow switch in `DiffusionService`, starts every load with `--backend cpu`: the live run sets it, + because a macOS runner's paravirtual GPU crashes ggml's Metal kernels. The new + `test/live/diffusion-install.test.ts` installs the release through a manifest of conf's shape + (`ATOMIC_SDCPP_MANIFEST_URL`): pick, download, sha256, unpack, probe, a tampered sha256 refused, + and `--version` naming the tag's commit. macOS runs on `macos-26`: the fork's macOS archive is built + with `minos 26.0`. + - The core suites run on the engine's platforms (`core_matrix` from resolve, `core_runner` where it + differs from the live runner); the pipeline keeps `suites=all`. For llama.cpp the matrix is the + same seven jobs as before. The live job's timeout comes from the config (`live_timeout_minutes`: + 90 for sd.cpp; GitHub's 360 otherwise). Required live tests may name `platforms`. +- **Consequences:** a check of an sd.cpp build is three core jobs and six live jobs; the first run + downloads 5.4 GB per live job, later ones restore it. On this machine (macOS arm64, M4 Pro, CPU + backend) `npm run engine:check -- --engine sd-cpp --tag master-883-137f740` passes against the pin + `master-883-137f740-a36f1b1a` in 9 minutes, about 4 minutes of them per generation test; a 3- or + 4-core runner is expected to take several times longer, inside the 90-minute timeout. A core release older than this one has no + `.github/engine-candidate/sd-cpp.json`, so the pipeline holds an sd.cpp rollout until a core release + carries it. Metal, CUDA, Vulkan, ROCm and Windows are not run (`not_verified`). The macOS runner has 7 + GB of memory for 5.4 GB of weights; if it pages too much, the live timeout says so. `-M` and three + deprecated flags core still emits (`--qwen2vl`, `--clip-on-cpu`, `--vae-on-cpu`) make every surface + report a `warn` until core moves off them. The `generating video:` line the tracker waits for is not + printed by leejet at master-883; the report lists it as text in neither tree. +- **Owner:** team +- **Links:** `.github/engine-candidate/sd-cpp.json`, `.github/workflows/engine-candidate.yml`, + `scripts/lib/{sdcpp-surface,sdcpp-fixtures,engine-io,engine-pin,engine-results,live-fixtures}.mjs`, + `scripts/{engine-candidate,engine-surface-diff,fetch-live-fixtures}.mjs`, + `test/fixtures/sdcpp/engine-contract.json`, `test/contract/sdcpp-{engine-contract,surface}.test.ts`, + `test/live/diffusion{,-install}.test.ts`, `src/diffusion/{args,service}.ts`, `docs/live-tests.md` + ("stable-diffusion.cpp builds"); atomic-chat-conf `mirror-sdcpp.yml` / `sdcpp-approve.yml`; + atomic-hatchet-pipeline `docs/engine-updater.md`. + + diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index c169fda6..59648872 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -11,6 +11,7 @@ add a new one that says which record it supersedes. ## Records +- **2026-10-21** — [Check stable-diffusion.cpp builds with the engine candidate workflow](2026-10-21-check-sdcpp-builds-with-the-engine-candidate.md) — `engine=sd-cpp` (`kind: sdcpp`): fork builds and conf's signed mirror checked against the staging manifest, pin from conf's sd.cpp manifest, sd.cpp's own surface checks in the same report schema, live on three CPU platforms with FLUX.2 Klein and a manifest install; `ATOMIC_DIFFUSION_BACKEND=cpu`; core suites on the engine's platforms. - **2026-10-20** — [Check PrismML builds with the engine candidate workflow, as upstream ones are checked](2026-10-20-check-prismml-builds-with-the-engine-candidate.md) — `engine=atomic-prism`: tags, repo, branch, pin source (conf's PrismML manifest), no mirror and the Bonsai live test are data in `.github/engine-candidate/atomic-prism.json`; refines the 2026-10-15 candidate check. - **2026-10-19** — [Process identities do not depend on who reads them](2026-10-19-process-identities-do-not-depend-on-the-reader.md) — `darwin-utc:`/`win32-utc:` start identities (`ps` under `LC_ALL=C TZ=UTC`, UTC ticks); a legacy `darwin:`/`win32:` difference is `unknown`, never `mismatch`; the instance lock decides by `owner_started_at` first; download sidecars are not listed as diffusion model files. - **2026-10-15** — [Foundry Local runs in a hidden host mode of the core binary, one process per model](2026-10-15-foundry-local-runs-in-a-host-mode-of-the-core-binary.md) — `foundry-host probe|prepare|serve` on `bun:ffi` (the one `bun:*` file in `src/`, allow-listed), started with `process.execPath`; sessions behind the core's gateway, forwarded under `/catalogs/local`; `SpawnSpec.stdin: 'pipe'` and the gateway's `basePath` added; Windows arm64, notarized macOS and NPU still gate the release. diff --git a/docs/live-tests.md b/docs/live-tests.md index 53512097..2a59976d 100644 --- a/docs/live-tests.md +++ b/docs/live-tests.md @@ -659,7 +659,8 @@ the update orchestrator through the API — never on a schedule, a push or a pul core installs and what the mirror requires, and the upstream history between the two commits (watched paths, commits announcing a removal or a deprecation). `report.json` / `report.md`; severities `break` (a launch fails or a feature is lost), `warn`, `info`, `error` (a check could not run). -- **core** (`suites=all`) — unit, contract and e2e suites and both builds on five OS, at `core_ref`. +- **core** (`suites=all`) — unit, contract and e2e suites and both builds at `core_ref`, on the platforms + of the engine's config (five OS for llama.cpp). - **live** — `test/live/llamacpp.test.ts` on five OS, for the candidate and for the pin, each with a real `llama-server` installed under its real tag. - **report** — `results.json` / `results.md`: `pass`, `fail` or `incomplete`, and why. @@ -677,6 +678,7 @@ npm run engine:check -- --repo AtomicBot-ai/atomic-chat-conf --tag b11463 # th npm run engine:check -- --bin ~/src/llama.cpp/build/bin/llama-server # any local build npm run engine:check -- --tag b10875 --pin b10874 --suites live # a backtest npm run engine:check -- --engine atomic-prism --tag prism-b10787-73bfe6c # a PrismML build +npm run engine:check -- --engine sd-cpp --tag master-883-137f740 # a stable-diffusion.cpp build ``` Without `--tag` or `--bin` it takes the newest complete b-tag of `--repo` (default `ggml-org/llama.cpp`). @@ -711,8 +713,8 @@ gh run download -n engine-candidate-results | input | default | | |---|---|---| -| `engine` | `llamacpp-upstream` | `.github/engine-candidate/.json` names the contract, the live files, the required live tests and the protected paths; `atomic-prism` checks PrismML's fork (below) | -| `tag` | newest complete | the candidate tag: `b11617`, or `prism-b10787-73bfe6c` for `atomic-prism` | +| `engine` | `llamacpp-upstream` | `.github/engine-candidate/.json` names the contract, the live files, the required live tests and the protected paths; `atomic-prism` checks PrismML's fork, `sd-cpp` stable-diffusion.cpp (below) | +| `tag` | newest complete | the candidate tag: `b11617`, `prism-b10787-73bfe6c` for `atomic-prism`, `master-975-95b3090` or `master-975-95b3090-a` for `sd-cpp` | | `repo` | the engine's | any repo with ggml-org's asset names; `AtomicBot-ai/atomic-chat-conf` tests the signed mirror | | `candidate_id` | `manual` | `[A-Za-z0-9._:-]{1,80}`, shown as `[cid:]` in the run title so a caller can find its run | | `core_ref` | `main` | the core to check, e.g. an agent's fix branch | @@ -754,3 +756,47 @@ The engine update pipeline (atomic-hatchet-pipeline) dispatches it with `repo=Pr again with `core_ref` set to the latest core release before conf's `prism-approve.yml` approves the build. A core release older than the PrismML check has no `.github/engine-candidate/atomic-prism.json`: the pipeline then stops before approving and waits for the next release. + +### stable-diffusion.cpp builds (`sd-cpp`) + +The check covers the image engine too ([ADR 2026-10-21](decisions/2026-10-21-check-sdcpp-builds-with-the-engine-candidate.md)). +`.github/engine-candidate/sd-cpp.json` (`kind: sdcpp`) and `scripts/lib/sdcpp-{surface,fixtures}.mjs` hold what +differs: + +- **Two kinds of candidate.** The Atomic fork's build of a leejet tag (`master--`, from + `AtomicBot-ai/stable-diffusion.cpp`, the default `repo`) and conf's signed mirror of it + (`master---a`, `repo=AtomicBot-ai/atomic-chat-conf`). The fork's is complete once every + archive conf's mirror requires and the CPU archive of each live platform is up once with its digest; + conf's also needs `backends/sdcpp-manifest.staging.json` at the run's conf commit (or the live manifest, + when that is the one naming the tag) to name the tag and pin exactly the release's sizes and sha256, + downloading from conf. +- **The pin** is the live manifest's `tag_name` (`backends/sdcpp-manifest.json`, the build every core + installs), fetched from conf's releases. +- **The surface** reads `sd-server --help` (flags matched as whole aliases), `sd-server --version` (the + commit has to be the tag's), the install probe's `sd-cli --help` (one of `stable-diffusion.cpp`, + `--cfg-scale`), the archives by backend, and leejet's history between the two tags: the watched + sources, the routes and log text core reads, and commits announcing a removal. The contract is + `test/fixtures/sdcpp/engine-contract.json`. +- **Live** is `test/live/diffusion.test.ts` and `test/live/diffusion-install.test.ts` on linux-x64, + linux-arm64 and macos-arm64 (`macos-26`: the archive needs macOS 26), CPU archives only, with FLUX.2 + Klein 4B Q4_K_M, Qwen3-4B-Instruct-2507 Q4_K_M and the FLUX.2 VAE (`live_models`, 5.4 GB, pinned by + revision and sha256, cached under `~/.cache/atomic-live-models` by their hashes). Every load runs on + the CPU backend (`ATOMIC_LIVE_SD_BACKEND=cpu` → `ATOMIC_DIFFUSION_BACKEND=cpu`). The install test + installs the release through a manifest `fetch-live-fixtures` writes from it + (`ATOMIC_SDCPP_MANIFEST_URL`), and refuses a tampered sha256. + +```bash +gh workflow run engine-candidate.yml -f engine=sd-cpp -f tag=master-883-137f740 \ + -f candidate_id=sdcpp-smoke -f core_ref=main +# conf's signed mirror on a core release, as the pipeline checks it before approval: +gh workflow run engine-candidate.yml --ref v0.18.7 -f engine=sd-cpp -f tag=master-883-137f740-a36f1b1a \ + -f repo=AtomicBot-ai/atomic-chat-conf -f candidate_id=sdcpp-verify -f core_ref=v0.18.7 +``` + +`node scripts/fetch-live-fixtures.mjs --engine sd-cpp --tag [--repo …] --out .live-sd [--models ]` +fetches the archive into `.live-sd/engine-/` and the models into `` (default `.live-sd/models`), +and writes `manifest.json` and `live.json` (the values the config's `live_env` fills in). The engine update +pipeline dispatches the fork's build with `core_ref=main`, then conf's mirror from the ref of a core +release; a core release without `.github/engine-candidate/sd-cpp.json` cannot check sd.cpp, and the +pipeline holds the rollout until one does. + diff --git a/docs/testing-critical-flows.md b/docs/testing-critical-flows.md index 61ae2bdf..b0a7fe6f 100644 --- a/docs/testing-critical-flows.md +++ b/docs/testing-critical-flows.md @@ -38,7 +38,7 @@ Line coverage alone never raises a grade. A PR may not lower a grade. | Server gates: host, api key, CORS, preflight | Strong (contract) | `test/contract/proxy-http.test.ts` (default and trusted hosts, `*`, missing Host, IPv6 loopback refused, Bearer/`X-Api-Key`, whitelisted docs, `/configs` hidden, origin reflection, preflight method/header/host rules), `src/server/public/gates.test.ts`, `http.test.ts` (control listener guards), `test/e2e/public-server.test.ts` (key, trusted hosts, missing Host, hidden `/configs`, 405, preflight and origin reflection configured through control on the compiled binary; Swagger assets served byte-identical from the binary) | maintained | | Dynamic trusted hosts: the tunnel name and the accepted socket's address (stage 7c) | Strong (core) | `src/server/public/dynamic-hosts.test.ts` (the five `dynamic_hosts.rs` tests against the core's `isValidHost`; `*`, URLs, ports and paths can never become the tunnel name; every socket-address spelling: mapped, hex-mapped, zone id, bracketed IPv6, loopback and unspecified in both families), `test/contract/dynamic-hosts.test.ts` on a real listener under Node and Bun (403 → 200 with the origin reflected → 403; another `*.trycloudflare.com` stays a stranger; the same keep-alive connection is refused before and accepted after; preflight; a listener on `0.0.0.0` trusts the dialled LAN address, refuses a rebinding name on that socket and the LAN literal over loopback; a loopback bind gains nothing), `src/core/public-server.test.ts` (a real core, and a repeated start stays idempotent), the `proxy-http` set unchanged | a phone on the same Wi-Fi reaching a running app | | Remote access: the Cloudflare quick tunnel (stage 7d) | Strong (core, fake cloudflared and a stand-in edge), Missing (live) | `src/remote-access/manager.test.ts` (the app's 19 state-machine tests plus: a stuck supervisor loses the state, a second stop joins the first, a process that came up after shutdown is not kept, the process-exit guard), `process.test.ts` on a real child (URL + registration, URL only, silence, early exit, exit after ready, SIGTERM ignored then killed, an exit that cannot be confirmed, Windows goes straight to the kill, a program that cannot start, real argv and no `TUNNEL_*`), `cloudflared-{args,output}.test.ts` (tables), `probe.test.ts` (ours / foreign / unreachable, the marker against the real public server, edge by SNI with certificate verification on the tunnel name, 1033 until the tunnel is there, two silent rounds give up the edge, hostname fallback, abort), `journal.test.ts` (write-then-enrich, never resurrected after a clear, identity *and* name before a kill, start-time fallback, a real child ended and a real non-tunnel spared), `spawner.test.ts`, `wiring.test.ts` (test hooks), `src/core/remote-access.test.ts` on a real core (Host gate 403 → 401 with the key → 403 as the tunnel comes and goes, tunnel ended before `server:stopped`, an unchanged start leaves it alone, HTTP/2 retry with real processes, shutdown kills it, no binary), `src/core/public-server.test.ts`, `routes/remote-access.test.ts`, `control-client.test.ts`; `test/runtime-compat/address-pin.test.ts` under Node and Bun (a Node edge process reports the SNI the client really sent); `test/e2e/remote-access.test.ts` on the compiled app-core binary (refusal, online through the stand-in edge, journal, stop, ordering over SSE, HTTP/2 retry with the real 15 s, missing binary, a crash's journal consumed by the next owner, shutdown), plus stage 7q: a tunnel that exits after coming up reported as `exited` with its name no longer trusted and the public server untouched, a tunnel that ignores SIGTERM killed after the 5 s grace before `stop` answers `off`, and the journal Atomic Chat 2.0.40 left at the data root consumed once with the process it named spared | a real `cloudflared` against Cloudflare from a running app; a phone opening the URL | -| Image generation on stable-diffusion.cpp (stages 7e–7l) | Strong (core, fake engine, compiled binary, real engine), Missing (desktop) | pure ports with the plugin's `#[test]` tables: `src/diffusion/{args,progress,tracker,workflow,parse,validate,errors,recipe}.test.ts` (`args.test.ts` holds the argv to `test/fixtures/sdcpp/required-flags.txt`); disk: `png.test.ts` (fixtures from an independent encoder, Pillow-checked; damaged and unsupported files refused), `gallery.test.ts` (foreign PNGs never listed or deleted, flags file, atomic writes, export), `install.test.ts` (finalize probes a real script, owned trees only, model files inside the root), `containment.test.ts` (symlinks out of and into the root); process and jobs: `server-process.test.ts` on the fake engine (journalled before ready, capabilities, early exit with marked lines first, OOM, timeout kills, foreign listener stopped, abandoned load stopped), `jobs.test.ts` against an HTTP stub (the `jobs.rs` runner tests: completion with outputs and every transition, failures explained by the output, queue full, forgotten job, cancel ladder incl. the second grace, cancel wins over a waiting respawn, crash mid-job with the spec kept, ggml-abort CPU retry, transport errors, ceiling), `session.test.ts`, `idle.test.ts`, `service.test.ts` (the real fake engine: finalize → load → generate with progress → gallery → cancel → respawn → idle unload → shutdown, child dead each time); wiring: `routes/diffusion.test.ts` (every route, envelopes, the 64 MiB job body, 400 with the field named), `control-client.test.ts`, `src/core/create.test.ts` (a real owner: refused foreign data folder, finalize → load → generate over the control API, `processes.json` entry under `diffusion`, no chat session, events on the bus, engine output kept out of the log, the server dead after `shutdown`, then `POST /v1/images/generations` on the public listener: a PNG back, the gallery grown, the image model absent from `/v1/models`, an abandoned request's job cancelled); app v2.0.42 additions (7k–7l): `compat.test.ts` (build ≥ 883 for Qwen Image 2.1 / Krea 2 Turbo, odd tags fail closed, install selection never switches backends), `workflow.test.ts` / `validate.test.ts` (per-family workflows, Qwen Image 2.1's references only with `llmVision`, Qwen-Image on Metal ≤ 1 MP), `session.test.ts` (a stale spec refused before spawn, `activateInstall`), `service.test.ts` on the fake engine (a Qwen Image 2.1 load blocked on 849 then served by an 883 tree, `llmVision` checked and protected, an engine finalized during a running job cancels it and unloads the old server), `jobs.test.ts` (a blank frame fails the batch before anything is saved, a GPU fault in the attempt's lines retires the server and one printed before the job does not, a respawn reads the spec under the lock), `gallery.test.ts` / `png.test.ts` (blank detection, hidden items, verdicts kept per file), `server-process.test.ts` (`GGML_METAL_TENSOR_DISABLE` reaches the child only on an M5 with Metal); facade: `server/public/images-params.test.ts` (the `images_route.rs` tables), `images.test.ts` (503 before a model, 400 before that, the OpenAI body with the atomic extras, refusals to start, failed jobs, client gone → cancel, 504 with cancel) ; `test/e2e/diffusion.test.ts` on the compiled binary (finalize → load → events over SSE → gallery, flags and export → the OpenAI facade → unload with the journal cleared; a cancel the engine ignores stops the process and the next job respawns it; a crashed owner's `sd-server` is reaped by the next owner); the 7k–7l behaviour on the compiled binary (stage 7o): `test/e2e/diffusion-engine.test.ts` (Qwen Image 2.1 refused on build 849 with `ENGINE_UPDATE_REQUIRED`, nothing spawned, `load-blocked` on the stream, then served by an 883 tree of the same backend; the projector as `--llm_vision` right after `--llm` in the real argv, `reference`/`edit` offered only with it, the projector undeletable while loaded, an edit without it named `SIDE_FILE_MISSING`; a finalized newer engine unloads the resident model with `engine-updated`, the old tree removable only then and its empty tag folder gone, an unowned tree and a path outside the root refused) and `test/e2e/diffusion-failures.test.ts` (a blank frame ends the batch with `INVALID_OUTPUT`, nothing on disk or in the gallery, the same process serving the next seed and the facade answering 500; a GPU fault printed during the job retires the still-running server with `gpu-fault`, the model kept, the next job on a fresh pid; a ggml unsupported-op abort retried once on `--backend cpu` — two pids, the second with `cpuFallback` in the recipe and the status); the rest of the failure modes and the facade on the compiled binary (stage 7p): `diffusion-engine.test.ts` (Qwen-Image past 1 MP refused on a Metal engine before anything runs and the M5 switch absent from the engine's environment on a non-M5, `MODEL_MISSING` / `SIDE_FILE_MISSING` naming the file before any spawn, an early exit read as `OUT_OF_MEMORY` and a foreign listener as `MODEL_LOAD_FAILED` with both engines gone, an unload during a load answering `CANCELLED` with the child gone first), `diffusion-failures.test.ts` (an engine dying mid-job explained by its output, `crashed` then a respawn; a full queue as `QUEUE_FULL` and a failed job as `OUT_OF_MEMORY` with the engine left running, the facade answering 429 `busy` and 500 `insufficient_memory`), `diffusion.test.ts` (the output folder moved with the next picture under it, gallery items deleted with their thumbnails and a foreign PNG never listed or touched, model files listed, protected while loaded and deleted with their empty family folder after; a generation body over 64 MiB refused with the envelope and one below it read whole; the idle unload on the next tick with the spec forgotten), `test/e2e/images-api.test.ts` (OpenAI's envelope: `response_format` refused before the model check, 503 `model_not_loaded`, a broken body and a missing prompt, `size` and `n` named, 429 `busy` behind an Images-page job; a client that hangs up cancels the job with the engine kept — the binary-level case that found Bun's silence, ADR 2026-09-22; the key and Host gates in front, the request reported as `atomic-diffusion` on `images/generations` and the prompt absent from the API events with or without the inspector), `public-server.test.ts` (`/openapi.json` from the binary documents `POST /images/generations`); `test/live/diffusion.test.ts` on a real engine and FLUX.2 Klein (`--help` holds every flag of `required-flags.txt`, load, a 256² generation with parsed sampling progress, PNG + thumbnail + recipe, the facade, a hard cancel, respawn, unload; passed 2026-09-18 on `master-849-d04e895` and again 2026-09-21 on `master-883-137f740`, the build app v2.0.42 ships, on macOS arm64 in ~5 min) | desktop acceptance from the Images page on the compiled binary and `test/live/diffusion.test.ts` on a real engine (7j), desktop acceptance from the Images page | +| Image generation on stable-diffusion.cpp (stages 7e–7l) | Strong (core, fake engine, compiled binary, real engine), Missing (desktop) | pure ports with the plugin's `#[test]` tables: `src/diffusion/{args,progress,tracker,workflow,parse,validate,errors,recipe}.test.ts` (`args.test.ts` holds the argv to `test/fixtures/sdcpp/required-flags.txt`); disk: `png.test.ts` (fixtures from an independent encoder, Pillow-checked; damaged and unsupported files refused), `gallery.test.ts` (foreign PNGs never listed or deleted, flags file, atomic writes, export), `install.test.ts` (finalize probes a real script, owned trees only, model files inside the root), `containment.test.ts` (symlinks out of and into the root); process and jobs: `server-process.test.ts` on the fake engine (journalled before ready, capabilities, early exit with marked lines first, OOM, timeout kills, foreign listener stopped, abandoned load stopped), `jobs.test.ts` against an HTTP stub (the `jobs.rs` runner tests: completion with outputs and every transition, failures explained by the output, queue full, forgotten job, cancel ladder incl. the second grace, cancel wins over a waiting respawn, crash mid-job with the spec kept, ggml-abort CPU retry, transport errors, ceiling), `session.test.ts`, `idle.test.ts`, `service.test.ts` (the real fake engine: finalize → load → generate with progress → gallery → cancel → respawn → idle unload → shutdown, child dead each time); wiring: `routes/diffusion.test.ts` (every route, envelopes, the 64 MiB job body, 400 with the field named), `control-client.test.ts`, `src/core/create.test.ts` (a real owner: refused foreign data folder, finalize → load → generate over the control API, `processes.json` entry under `diffusion`, no chat session, events on the bus, engine output kept out of the log, the server dead after `shutdown`, then `POST /v1/images/generations` on the public listener: a PNG back, the gallery grown, the image model absent from `/v1/models`, an abandoned request's job cancelled); app v2.0.42 additions (7k–7l): `compat.test.ts` (build ≥ 883 for Qwen Image 2.1 / Krea 2 Turbo, odd tags fail closed, install selection never switches backends), `workflow.test.ts` / `validate.test.ts` (per-family workflows, Qwen Image 2.1's references only with `llmVision`, Qwen-Image on Metal ≤ 1 MP), `session.test.ts` (a stale spec refused before spawn, `activateInstall`), `service.test.ts` on the fake engine (a Qwen Image 2.1 load blocked on 849 then served by an 883 tree, `llmVision` checked and protected, an engine finalized during a running job cancels it and unloads the old server), `jobs.test.ts` (a blank frame fails the batch before anything is saved, a GPU fault in the attempt's lines retires the server and one printed before the job does not, a respawn reads the spec under the lock), `gallery.test.ts` / `png.test.ts` (blank detection, hidden items, verdicts kept per file), `server-process.test.ts` (`GGML_METAL_TENSOR_DISABLE` reaches the child only on an M5 with Metal); facade: `server/public/images-params.test.ts` (the `images_route.rs` tables), `images.test.ts` (503 before a model, 400 before that, the OpenAI body with the atomic extras, refusals to start, failed jobs, client gone → cancel, 504 with cancel) ; `test/e2e/diffusion.test.ts` on the compiled binary (finalize → load → events over SSE → gallery, flags and export → the OpenAI facade → unload with the journal cleared; a cancel the engine ignores stops the process and the next job respawns it; a crashed owner's `sd-server` is reaped by the next owner); the 7k–7l behaviour on the compiled binary (stage 7o): `test/e2e/diffusion-engine.test.ts` (Qwen Image 2.1 refused on build 849 with `ENGINE_UPDATE_REQUIRED`, nothing spawned, `load-blocked` on the stream, then served by an 883 tree of the same backend; the projector as `--llm_vision` right after `--llm` in the real argv, `reference`/`edit` offered only with it, the projector undeletable while loaded, an edit without it named `SIDE_FILE_MISSING`; a finalized newer engine unloads the resident model with `engine-updated`, the old tree removable only then and its empty tag folder gone, an unowned tree and a path outside the root refused) and `test/e2e/diffusion-failures.test.ts` (a blank frame ends the batch with `INVALID_OUTPUT`, nothing on disk or in the gallery, the same process serving the next seed and the facade answering 500; a GPU fault printed during the job retires the still-running server with `gpu-fault`, the model kept, the next job on a fresh pid; a ggml unsupported-op abort retried once on `--backend cpu` — two pids, the second with `cpuFallback` in the recipe and the status); the rest of the failure modes and the facade on the compiled binary (stage 7p): `diffusion-engine.test.ts` (Qwen-Image past 1 MP refused on a Metal engine before anything runs and the M5 switch absent from the engine's environment on a non-M5, `MODEL_MISSING` / `SIDE_FILE_MISSING` naming the file before any spawn, an early exit read as `OUT_OF_MEMORY` and a foreign listener as `MODEL_LOAD_FAILED` with both engines gone, an unload during a load answering `CANCELLED` with the child gone first), `diffusion-failures.test.ts` (an engine dying mid-job explained by its output, `crashed` then a respawn; a full queue as `QUEUE_FULL` and a failed job as `OUT_OF_MEMORY` with the engine left running, the facade answering 429 `busy` and 500 `insufficient_memory`), `diffusion.test.ts` (the output folder moved with the next picture under it, gallery items deleted with their thumbnails and a foreign PNG never listed or touched, model files listed, protected while loaded and deleted with their empty family folder after; a generation body over 64 MiB refused with the envelope and one below it read whole; the idle unload on the next tick with the spec forgotten), `test/e2e/images-api.test.ts` (OpenAI's envelope: `response_format` refused before the model check, 503 `model_not_loaded`, a broken body and a missing prompt, `size` and `n` named, 429 `busy` behind an Images-page job; a client that hangs up cancels the job with the engine kept — the binary-level case that found Bun's silence, ADR 2026-09-22; the key and Host gates in front, the request reported as `atomic-diffusion` on `images/generations` and the prompt absent from the API events with or without the inspector), `public-server.test.ts` (`/openapi.json` from the binary documents `POST /images/generations`); `test/live/diffusion.test.ts` on a real engine and FLUX.2 Klein (`--help` holds every flag of `required-flags.txt`, load, a 256² generation with parsed sampling progress, PNG + thumbnail + recipe, the facade, a hard cancel, respawn, unload; passed 2026-09-18 on `master-849-d04e895` and again 2026-09-21 on `master-883-137f740`, the build app v2.0.42 ships, on macOS arm64 in ~5 min); `test/live/diffusion-install.test.ts` (a real build installed from a conf-shaped manifest: picked for the host, downloaded, sha256-checked, unpacked, probed; a tampered sha256 refused; `--version` naming the tag's commit); both run per build by `engine-candidate.yml` with `engine=sd-cpp` on linux-x64, linux-arm64 and macos-arm64 (CPU archives), with the surface held to `test/fixtures/sdcpp/engine-contract.json` (`test/contract/sdcpp-{engine-contract,surface}.test.ts`) | desktop acceptance from the Images page on the compiled binary and `test/live/diffusion.test.ts` on a real engine (7j), desktop acceptance from the Images page | | Video generation on stable-diffusion.cpp (stage 9a–9f) | Strong (core, fake engine, compiled binary), Missing (real engine, desktop) | pure: `src/diffusion/{parse,validate,args,workflow}.test.ts` (the video parsers, the frame lattice `k·step+offset` and the nearest-point snap, the fixed rate, `--audio-vae`/`--embeddings-connectors` in order, `-M vid_gen` behind its switch, the `vid_gen` body with `custom_sigmas` at the schedule's step count, image-to-video refused), `server-process.test.ts` (capabilities per mode, a mode the spec needs but the engine lacks kills the server: `MODEL_INCOMPATIBLE`), `session.test.ts` (video capabilities, image capabilities refused on a video model), `state.test.ts` (kind-aware records); the seam: `image-job.test.ts`, `video-job.test.ts` (decode, the WebM preflight, the recipe, the events, the plan's decode tiling in the body and the threshold again on the CPU fallback), `jobs.test.ts` (the image suite unchanged after the refactor; a video job through the HTTP stub on its own events with one clip, the wrong modality both ways, the preflight, a native cancel; the estimate on the first event and a failed one that does not stop the job; the plan's decode tiling in the body and the decode's tiles on the progress; the heartbeat through a ten-second step with `elapsedMs` from the job's start, images still on change only); the estimate: `video-estimate.test.ts` (latent tokens per family, the tiled decode and its peak and work per layout, the decode tiling planned by memory — one graph on 24 GB, three strips on 18 GB, the finest on 16 GB, none under `model` offload, on the CPU fallback or on unknown memory —, LTX-2 on a 16 GB Mac `exceeds` with `seconds: null`, memory growing with frames and size, the 80 %/100 % lines, the three pools and both sides of a discrete GPU, the speed table and unknown hardware, CFG passes, ×2/×3 ranges, the history band), `video-tiling.test.ts` (sd.cpp's tile count ported, tile counts to sides and back, the work of a layout, the body's tile counts under both spellings), `video-history.test.ts` (which clips count, a past clip re-planned at its own size, three clips at twice the heuristic double the middle, the median and its clamp, the cache), `video-eta.test.ts` (every ETA and fraction scenario of the progress spec, both slowdown rules and their stickiness; a tiled decode counted down per tile, stretched, restarted on a retried pass, and a stalled tile), `tracker.test.ts` (a lost tile pass ended by `generating video:`; the decode's tiles counted, a retried pass counted from zero); disk: `video-recipe.test.ts` (every key, `null`s, strict on the way back), `video-gallery.test.ts` (EBML/DocType sniff, save writes the sidecar first, foreign `.webm` never listed or deleted, the poster kept verbatim and refused when not a PNG, flags, delete, export); service: `service.test.ts` on the fake in `vid_gen` mode (`estimateVideo`: the job's refusals, an answer while a clip runs with no event, history after the first clip and gone after its delete, no hardware facts refused while the clip still runs; load with the side files, a clip into `/videos` with its sidecar, the poster, the gallery, `cancelVideoJob`, the `VideosBackend` view, a video model on an image-only engine refused with the child dead, a build without WebM refused before submit; the decode tiled by the memory of a 16 GB Mac — one graph for 1024²×9, three strips for 2048²×17 — and its tiles reported); routes: `routes/diffusion-video.test.ts` (the estimate route too), `control-client.test.ts`; facade: `videos-params.test.ts` (seconds → frames, the video object from a job and from a clip, the path matcher, the listing query), `videos.test.ts` on a real listener (503/400/405, the queued object, poll, content and poster streamed, 404s, DELETE cancels or removes, the list merge and cursor, polls not reported); owner: `src/core/create.test.ts` (`/v1/videos` queue → poll → content equals the fixture → gallery → delete; the video model absent from `/v1/models`; only the video events); binary: `test/e2e/video.test.ts` (argv, `videoOutputDir`, the events over SSE, the clip and sidecar on disk, the poster, gallery ops, a cancel the engine ignores stops it and the next clip respawns, wrong modality at load and at generate, a missing side file before any spawn, no-WebM before submit and an unreported build's own 400; the estimate route and its refusal, `job.estimate` on the first event, the heartbeat through a 3 s step, `slowdown` when a step drags past three medians and 20 s; one graph for a clip past the old threshold that fits, and a tiled decode counted down tile by tile), `test/e2e/videos-api.test.ts` (the OpenAI envelope, queue/poll/download/list/DELETE, a finished clip answered after a restart, key and Host gates, reported as `atomic-diffusion` on `videos` without the prompt); live: `test/live/diffusion.test.ts` video block, not run yet (the estimate's coefficients are unmeasured, ADR 2026-09-28); desktop: the app's Video page, not yet | | Remote media: cloud and self-hosted providers' image and video models through the core (ADR 2026-10-13) | Strong (core, fake providers, compiled binary), Missing (live providers, desktop) | adapters: one `src/media/adapters/.test.ts` per provider API on a fake fetch (request shape, auth header, body, success parsing, error mapping, live listing; video submit/poll/download/cancel; key only on the provider origin, redirects off it bare), `registry.test.ts` (one adapter per dialect); pure: `src/media/{catalog,http,mask}.test.ts`, `remote-media.test.ts` (public params, inline images only, alpha masks), `parse.test.ts` (`target`/`remote`); I/O: `catalog-service.test.ts` (live → memory → disk → baseline, newest `catalog_version`, the `file://` hook), `remote-store.test.ts`; service: `src/media/service.test.ts` (catalog and registration adapters, refusals, live listing merged or failed), `src/diffusion/remote-job.test.ts` on the real `DiffusionService` (shared table and events, no busy slot, gallery PNG/JPEG/MP4, provider errors and `diffusion:error`, cancel without touching `sd-server`, the cap, a submit never retried, transient polls retried, a clip resumed after a restart without a second submit, a resumed clip past its ceiling timed out, shutdown keeps the stored job, the facades' `runImageJob`/`runVideoJob`); registry: `cloud/registry.test.ts` (media out of the key binding, absent keeps, `null` clears, other providers untouched), `router/resolve.test.ts`; routes: `routes/media.test.ts`, `control-client.test.ts`, `server/public/{images,videos}.test.ts` (remote generations and edits, 415/404/400, MP4 content), `src/core/create.test.ts` (owner: catalog and a live listing over control); binary: `test/e2e/remote-media.test.ts` (a self-hosted OpenAI-compatible stub registered over control: an image job over control into `/images`, `/v1/images/generations` by `/`, absent from `/v1/models`, a clip over `/v1/videos` saved as MP4, the key sent to the provider; an unconnected provider refused) | live runs against each provider (`test/live`), the app's Images/Video pages | | Decision model: engine gate, supervision, fail-open calls, public passthrough (ADR 2026-09-30) | Partial (core, fake engine), Missing (compiled binary, real engine, desktop) | pure: `src/decision/{engine-version,engine-candidates,args,readiness,outcome,backoff,request}.test.ts`, `src/settings/decision.test.ts`, `src/models/gguf/classify.test.ts` (`isDecisionGguf` before `isEmbeddingGguf`); process: `src/decision/process.test.ts` on the fake `llama-server` in decision mode (readiness chain with the key, `/health` polled while loading, a slow `/props` and a 5xx `/v1/models` waited through instead of refused, API version 2 and a missing capability refused with the child killed, an older build's exit with its own lines, startup timeout, abort, a failed journal write), `engine.test.ts` (gate order, a dev build found after the newer tags failed, the probe cache per mtime, a build rejected at readiness skipped until its file changes or `forgetRejected`, the "refused at readiness" message instead of the install advice, an explicit engine, the real `-h` of the fake); service: `service.test.ts` (scores and answers through a real fake process, the 500 ms budget kept against a 2 s engine, the engine's refusal with its reason, the uncalibrated router, every refusal state, restart with backoff and giving up, a SIGKILLed process brought back, settings followed, a start in flight kept across a non-launch settings write and restarted by a launch one, the idle timer re-armed, idle unload held by a call in flight, a start aborted by unload, nothing left after shutdown, `not_calibrated` without a round trip and from a 501, an unserializable body, router scores out of order, a retry after an install and the rate-limited, quiet retry of `unsupported` (no transient `starting`), the refused builds forgotten on load, install and launch settings but not on a retry, the next engine after a readiness refusal, a public request waiting through a restart and released when the client leaves), `wiring.test.ts` (settings → installed pack → journal entry for the process's life; `backend:download-finished` for `llamacpp` bringing an `unsupported` module up while another provider or a failed install starts nothing; a refused build retried by an explicit load but not by a call; every build refused → the readiness message; a slow `/props` → ready on the first build); surface: `routes/decision.test.ts`, `server/public/decision.test.ts`; `test/contract/decision.test.ts` (the DECISION.md examples through the core's readers; public `/v1/systemone` and `/v1/router/score` to the real module and the fake engine byte for byte by SHA-256, the engine's `MALFORMED_JSON`, 503 when off; an upstream decision GGUF on the fake in upstream mode: the newest upstream pack at the floor picked over an older one and the fork, no `--decision`, `-ub` for laya, router 501 `UNSUPPORTED_ENDPOINT`, "Update llama.cpp to b11370" below the floor); upstream: `src/decision/{upstream-version,model-facts}.test.ts`, the upstream cases of `engine`, `engine-candidates`, `args`, `readiness`, `process` and `service` tests (floor by type and projector, the projector checked and the spec skipped, the router refused without a request, an install of the other provider ignored) | an e2e on the compiled binary; a live test against a real fork build (1.7.0) and a laya GGUF, and against upstream b11370+ with an upstream decision GGUF; the app's consumer | diff --git a/scripts/engine-candidate.mjs b/scripts/engine-candidate.mjs index df78b63e..eda638a1 100644 --- a/scripts/engine-candidate.mjs +++ b/scripts/engine-candidate.mjs @@ -1,9 +1,11 @@ #!/usr/bin/env node -// Is a llama.cpp build compatible with this core? The steps of -// .github/workflows/engine-candidate.yml, and the same check on this machine alone. --engine picks -// .github/engine-candidate/.json: upstream llama.cpp (`llamacpp-upstream`, b-tags from -// ggml-org, pinned by conf's release list) or PrismML's fork (`atomic-prism`, prism-b- tags -// from PrismML-Eng/llama.cpp, pinned by conf's PrismML manifest, live tests on a Bonsai GGUF). +// Is an engine build compatible with this core? The steps of .github/workflows/engine-candidate.yml, +// and the same check on this machine alone. --engine picks .github/engine-candidate/.json: +// upstream llama.cpp (`llamacpp-upstream`, b-tags from ggml-org, pinned by conf's release list), +// PrismML's fork (`atomic-prism`, prism-b- tags from PrismML-Eng/llama.cpp, pinned by conf's +// PrismML manifest, live tests on a Bonsai GGUF) or stable-diffusion.cpp (`sd-cpp`, master-- +// builds of the Atomic fork and their signed mirror master---a in conf, pinned by conf's +// sd.cpp manifest, live tests on FLUX.2 Klein). // // resolve fix core and conf to one commit each, pick the candidate (the newest complete tag of // --repo unless --tag), check its archives are all up, compute the pin per platform and @@ -17,7 +19,8 @@ // local all of it here, for this host only: `npm run engine:check -- --tag b11617`, // `-- --repo AtomicBot-ai/atomic-chat-conf --tag b11463`, `-- --bin `, // `-- --tag b10875 --pin b10874 --suites live`, -// `-- --engine atomic-prism --tag prism-b10787-73bfe6c`. Writes results.json, results.md, +// `-- --engine atomic-prism --tag prism-b10787-73bfe6c`, +// `-- --engine sd-cpp --tag master-883-137f740`. Writes results.json, results.md, // report.json and report.md into --out (default .engine-check/); exit 0 only on pass. // // CI passes its inputs as env (ENGINE, TAG, REPO, CANDIDATE_ID, CORE_REF, PIN, CONF_REF, SUITES); @@ -48,11 +51,15 @@ import { buildResults, CANDIDATE_ID_RE, coreCell, + coreMatrix, liveCell, renderResultsMarkdown, + requiredLiveTests, RESOLVE_SCHEMA, } from './lib/engine-results.mjs' import { + CONF_FILES, + CONF_REPO, coreIdentity, engineConfig, engineContract, @@ -66,7 +73,11 @@ import { ROOT, run, sha256File, + upstreamTagCommit, } from './lib/engine-io.mjs' +import { sdcppTagParts } from './lib/engine-pin.mjs' +import { expandLiveEnv, liveModelsKey } from './lib/live-fixtures.mjs' +import { sdcppCompleteness, sdcppCoreAssets } from './lib/sdcpp-surface.mjs' const readJson = (path) => JSON.parse(readFileSync(path, 'utf8')) const readJsonIf = (path) => (path && existsSync(path) ? readJson(path) : undefined) @@ -100,6 +111,64 @@ function protectedChanges(config, sha) { return diff.status === 0 ? diff.stdout.split('\n').filter(Boolean).sort() : null } +/** + * An sd.cpp release as the candidate (`--tag`, else the newest complete one of `repo`): complete when + * every archive core and conf's mirror need is up with its digest, and on conf's signed mirror when + * the staging manifest (`candidate_manifests.conf`, or the live manifest when that is the one naming + * the tag) pins exactly those bytes. `commit` is the leejet tag's, from `git ls-remote`. + */ +async function resolveSdcppCandidate({ config, contract, repo, tag, conf, keys, errors }) { + const isTag = (v) => engineTagRe(config).test(v ?? '') + const staging = config.candidate_manifests?.conf + const manifestFor = (t) => { + if (repo !== CONF_REPO || !staging) return { path: null, manifest: undefined } + // Staging names the candidate on its way to approval; the live manifest, a build already out. + const path = [staging, CONF_FILES.sdcpp].find((p) => conf.docs?.[p]?.tag_name === t) ?? staging + return { path, manifest: conf.docs?.[path] ?? null } + } + const completeness = (release) => + sdcppCompleteness({ + contract, + tag: release.tag_name, + assets: release.assets ?? [], + platforms: keys, + manifest: manifestFor(release.tag_name).manifest, + downloadBase: `https://github.com/${repo}/releases/download`, + }) + let release + if (tag) { + release = await githubRelease(repo, tag) + if (!release) throw new Error(`${repo} has no release ${tag}`) + } else { + const releases = (await githubReleases(repo)).filter((r) => !r.draft && isTag(r.tag_name)) + release = releases.find((r) => completeness(r).complete) + if (!release) throw new Error(`no complete tag among the ${releases.length} newest releases of ${repo}`) + } + const parts = sdcppTagParts(release.tag_name) + const upstreamRepo = contract.upstream?.repo + const commit = upstreamTagCommit(upstreamRepo, parts.upstream_tag) + if (!commit) errors.push(`${upstreamRepo} has no tag ${parts.upstream_tag} (or git ls-remote failed)`) + else if (!commit.startsWith(parts.sha7)) + errors.push(`${upstreamRepo} ${parts.upstream_tag} is ${commit}, not the ${parts.sha7} its name says`) + const manifest = manifestFor(release.tag_name).path + return { + repo, + tag: release.tag_name, + source: 'release', + commit, + upstream_tag: parts.upstream_tag, + atomic_rev: parts.atomic_rev, + published_at: release.published_at ?? null, + ...(manifest ? { manifest } : {}), + ...completeness(release), + core_assets: sdcppCoreAssets( + contract, + release.tag_name, + (release.assets ?? []).map((a) => a.name) + ), + } +} + /** * Everything one run is about, fixed once: core and conf commits, the candidate and whether its * archives are all up, the pin per platform, and the live cells to run. `bin` is a local binary @@ -141,7 +210,10 @@ async function resolveRun({ const keys = platforms.map((p) => p.platform) let candidate - if (bin) { + if (config.kind === 'sdcpp') { + if (bin) throw new Error(`${engine} is checked from a release (--tag), not a local binary`) + candidate = await resolveSdcppCandidate({ config, contract, repo, tag, conf, keys, errors }) + } else if (bin) { const out = run(bin, ['--version'], { cwd: dirname(bin), timeout: 60_000 }) const output = `${out.stdout}\n${out.stderr}` const build = parseBinaryVersion(output) @@ -209,14 +281,16 @@ async function resolveRun({ pins.per_platform['linux-x64']?.tag ?? keys.map((k) => pins.per_platform[k]?.tag).find(Boolean) ?? null // The pin's archives from the candidate's repo when it has them (the mirror's signed bytes against // the mirror's), else from the engine's own repo. + // sd.cpp's pin lives in conf only (`pin_repo`): the fork has no `-a` tags. let pinRepo = null if (pinTag) pinRepo = - candidate.repo && + config.pin_repo ?? + (candidate.repo && candidate.repo !== defaultRepo && (await githubRelease(candidate.repo, pinTag).catch(() => null)) ? candidate.repo - : defaultRepo + : defaultRepo) const liveMatrix = [] for (const { platform, runner } of platforms) { @@ -294,6 +368,10 @@ async function cmdResolve(argv) { core_sha: resolved.core.sha ?? '', complete: String(resolved.candidate.complete), live_matrix: JSON.stringify(resolved.live_matrix), + core_matrix: JSON.stringify(coreMatrix(config.platforms)), + // GitHub's own default when the engine sets none. + live_timeout_minutes: String(config.live_timeout_minutes ?? 360), + live_models_key: liveModelsKey(config.live_models), }) console.log( `candidate ${resolved.candidate.repo}@${resolved.candidate.tag} (${resolved.candidate.complete ? 'complete' : `incomplete: ${JSON.stringify(resolved.candidate.missing)}`}), ` + @@ -305,10 +383,20 @@ async function cmdResolve(argv) { /** Where the live contract probe of a run writes observed.json: beside its Vitest report. */ const observedDir = (report) => join(dirname(report), `${basename(report, '.json')}.observed`) -/** Run the engine's live files against a fetched build; the report lands at `report`. */ +/** + * Run the engine's live files against a fetched build; the report lands at `report`. The env is the + * config's `live_env` filled from the fetch's live.json, else `_BIN`, `_MODEL` and `_TAG`. + */ function runLive({ config, fixtures, report, tag, bin, model }) { const read = (name) => readFileSync(join(fixtures, name), 'utf8').trim() const prefix = liveEnvPrefix(config) + const engineEnv = config.live_env + ? expandLiveEnv(config.live_env, JSON.parse(read('live.json'))) + : { + [`${prefix}_BIN`]: bin ?? read('bin-path.txt'), + [`${prefix}_MODEL`]: model ?? read('model-path.txt'), + [`${prefix}_TAG`]: tag ?? read('tag.txt'), + } const vitest = join(ROOT, 'node_modules/vitest/vitest.mjs') mkdirSync(dirname(report), { recursive: true }) const r = spawnSync( @@ -330,9 +418,7 @@ function runLive({ config, fixtures, report, tag, bin, model }) { env: { ...process.env, ATOMIC_LIVE: '1', - [`${prefix}_BIN`]: bin ?? read('bin-path.txt'), - [`${prefix}_MODEL`]: model ?? read('model-path.txt'), - [`${prefix}_TAG`]: tag ?? read('tag.txt'), + ...engineEnv, // The live contract probe writes observed.json here; the cell reads it. ATOMIC_LIVE_REPORT_DIR: observedDir(report), }, @@ -389,7 +475,7 @@ function cmdCell(argv) { fixture: readJsonIf(o.fixtures && join(o.fixtures, 'fixture.json')) ?? null, vitest: readJsonIf(o.vitest[0]), observed: o.vitest[0] ? readJsonIf(join(observedDir(o.vitest[0]), 'observed.json')) : undefined, - required: config.required_live_tests, + required: requiredLiveTests(config.required_live_tests, o.platform), ...(o.fixtures && existsSync(join(o.fixtures, 'fixture.json')) ? {} : { error: 'the build was not fetched' }), @@ -606,8 +692,11 @@ async function cmdLocal(argv) { tag: m.tag, repo: m.repo, out: join(cache, m.repo.replace('/', '__'), m.tag, m.platform), - model: model === undefined, + // An engine with several model files names them in each build's live.json; they are shared + // and reused while their sha256 matches. + model: config.live_models ? true : model === undefined, engine: o.engine, + ...(config.live_models ? { models: join(cache, 'models') } : {}), }) fixtures = build.dir model ??= build.model @@ -626,7 +715,7 @@ async function cmdLocal(argv) { fixture: readJsonIf(fixtures && join(fixtures, 'fixture.json')) ?? null, vitest: readJsonIf(report), observed: readJsonIf(join(observedDir(report), 'observed.json')), - required: config.required_live_tests, + required: requiredLiveTests(config.required_live_tests, m.platform), ...(error ? { error } : {}), }) writeJson(join(out, 'cells', `live-${m.platform}-${m.role}.json`), cell) diff --git a/scripts/engine-surface-diff.mjs b/scripts/engine-surface-diff.mjs index df3a5aa6..a150b276 100644 --- a/scripts/engine-surface-diff.mjs +++ b/scripts/engine-surface-diff.mjs @@ -14,8 +14,11 @@ // // --engine picks .github/engine-candidate/.json: its contract, tag shape, default repo and // pin source (`atomic-prism`: PrismML-Eng/llama.cpp, its `prism` branch for the history, conf's -// PrismML manifest for the pin, and no mirror). --repo: any repo publishing ggml-org's asset names -// (AtomicBot-ai/atomic-chat-conf for the signed mirror). --pin auto takes conf's list for the engine +// PrismML manifest for the pin, and no mirror; `sd-cpp`: stable-diffusion.cpp's own checks in +// scripts/lib/sdcpp-surface.mjs — `sd-server --help` and `--version`, the install probe's +// `sd-cli --help`, the archives by backend — with the pin from conf's sd.cpp manifest and leejet's +// history between the two tags the builds are cut from). --repo: any repo publishing the engine's +// asset names (AtomicBot-ai/atomic-chat-conf for the signed mirror). --pin auto takes conf's list for the engine // (the newest approved release this core version runs, on linux-x64); the pin's binary comes from // --pin-repo, by default the candidate's repo when it has the tag, else the engine's. A local --candidate-bin is tagged b from its --version; // when its commit is not in ggml-org/llama.cpp (a fork, a local branch) the upstream checks say @@ -29,6 +32,7 @@ import { parseArgs } from 'node:util' import { buildReport, countStrings, + invocationKey, MIRROR_WHITELIST, parseBinaryVersion, parseGitLog, @@ -54,7 +58,10 @@ import { run, runInvocations, sha256File, + upstreamTagCommit, } from './lib/engine-io.mjs' +import { sdcppTagBuild, sdcppTagParts } from './lib/engine-pin.mjs' +import { buildSdcppReport, notInArchive, parseSdVersion } from './lib/sdcpp-surface.mjs' const SEVERITY_ORDER = ['warn', 'error', 'break'] @@ -87,6 +94,8 @@ export async function main(argv = process.argv.slice(2)) { if (!opts.inputs && !opts.candidate === !opts['candidate-bin']) throw new Error('give one of --candidate or --candidate-bin (or --inputs)') if (opts.candidate && !isTag(opts.candidate)) throw new Error(`--candidate must match ${tagRe}`) + if (config.kind === 'sdcpp' && (opts['candidate-bin'] || opts['pin-bin'])) + throw new Error(`${opts.engine} is compared from releases (--candidate, --pin), not local binaries`) if (!['auto', 'none'].includes(opts.pin) && !isTag(opts.pin)) throw new Error(`--pin is auto, none or a tag matching ${tagRe}`) if (opts['fail-on'] && !SEVERITY_ORDER.includes(opts['fail-on'])) @@ -108,7 +117,7 @@ export async function main(argv = process.argv.slice(2)) { if (inputs.candidate?.help_text) writeFileSync(join(out, 'help-candidate.txt'), inputs.candidate.help_text) if (inputs.pin?.help_text) writeFileSync(join(out, 'help-pin.txt'), inputs.pin.help_text) - const report = buildReport(contract, inputs, { + const report = (config.kind === 'sdcpp' ? buildSdcppReport : buildReport)(contract, inputs, { now: new Date().toISOString(), contractSha256: createHash('sha256').update(contractText).digest('hex'), }) @@ -144,12 +153,22 @@ async function gather(contract, opts, config) { const capture = (side, bin) => { const invocations = runInvocations(bin, contract.invocations ?? []) - const output = (flag) => invocations.find((i) => i.argv.join(' ') === flag)?.output - side.help_text = output('-h') - side.version_output = output('--version') + const ran = (key) => invocations.find((i) => invocationKey(i) === key) + const output = (key) => ran(key)?.output side.invocations = invocations.map(({ output: text, ...rest }) => rest.status === 0 ? rest : { ...rest, output: text.slice(-2000) } ) + if (config.kind === 'sdcpp') { + side.help_text = output('sd-server --help') + const cli = ran('sd-cli --help') + if (cli && !notInArchive(cli)) side.cli_help_text = cli.output + side.version_output = output('sd-server --version') + side.build = sdcppTagBuild(side.tag ?? '') + side.commit = parseSdVersion(contract, side.version_output ?? '')?.commit ?? null + return + } + side.help_text = output('-h') + side.version_output = output('--version') side.build = side.version_output === undefined ? null : (parseBinaryVersion(side.version_output) ?? null) side.commit = side.version_output === undefined ? null : (versionCommit(side.version_output) ?? null) } @@ -168,8 +187,19 @@ async function gather(contract, opts, config) { return side } + const upstreamRepo = contract.upstream?.repo ?? config.default_repo const releaseSide = async (tag, repo) => { const side = { tag, repo, source: 'release', platform } + // sd.cpp's builds are cut from a leejet tag: its commit and date place them in leejet's history. + const parts = config.kind === 'sdcpp' ? sdcppTagParts(tag) : null + if (parts) { + const upstreamRelease = await githubRelease(upstreamRepo, parts.upstream_tag).catch(() => null) + side.upstream = { + tag: parts.upstream_tag, + commit: upstreamTagCommit(upstreamRepo, parts.upstream_tag), + published_at: upstreamRelease?.published_at ?? null, + } + } try { const release = await githubRelease(repo, tag) if (!release) side.assets_error = `${repo} has no release ${tag}` @@ -189,6 +219,7 @@ async function gather(contract, opts, config) { repo, out: join(cache, repo.replace('/', '__'), tag, platform), model: false, + ...(config.kind === 'sdcpp' ? { engine: config.engine } : {}), }) const asset = build.fixture.asset side.binary = { name: asset.name, sha256: asset.sha256, digest_verified: asset.digest_verified } @@ -225,6 +256,7 @@ async function gather(contract, opts, config) { const candidateRepo = candidate.repo ?? config.default_repo const repo = opts['pin-repo'] ?? + config.pin_repo ?? (candidateRepo !== config.default_repo && (await githubRelease(candidateRepo, tag).catch(() => null)) ? candidateRepo : config.default_repo) @@ -232,10 +264,15 @@ async function gather(contract, opts, config) { } } - const upstreamRepo = contract.upstream?.repo ?? config.default_repo const { upstream, patch } = opts['skip-upstream'] ? { upstream: { available: false, reason: 'skipped (--skip-upstream)' } } - : await upstreamDiff(contract, candidate, pin, upstreamRepo, join(cache, 'llama.cpp.git')) + : await upstreamDiff( + contract, + candidate, + pin, + upstreamRepo, + join(cache, `${upstreamRepo.replace('/', '__')}.git`) + ) return { inputs: { @@ -253,9 +290,12 @@ async function gather(contract, opts, config) { } } -/** A side's upstream commit: the release's own when it is an upstream release, else what --version names. */ +/** + * A side's upstream commit: the leejet tag's for sd.cpp, the release's own when it is an upstream + * release, else what --version names. + */ const commitOf = (side, upstreamRepo) => - (side.repo === upstreamRepo ? side.release?.commit : null) ?? side.commit ?? null + side.upstream?.commit ?? (side.repo === upstreamRepo ? side.release?.commit : null) ?? side.commit ?? null async function upstreamDiff(contract, candidate, pin, upstreamRepo, gitDir) { if (!pin) return { upstream: { available: false, reason: 'no pin to compare with' } } @@ -284,15 +324,20 @@ async function upstreamDiff(contract, candidate, pin, upstreamRepo, gitDir) { */ function gitDiff(contract, { base, head, candidate, pin, upstreamRepo, gitDir }) { const watched = (contract.upstream?.watched_paths ?? []).map((w) => w.path) - // History from two days before the older release: enough for `log base..head` either way round. - const dates = [pin, candidate].map((s) => Date.parse(s.release?.published_at ?? '')).filter((d) => d > 0) + // History from two days before the older release (leejet's, for a build cut from one): enough for + // `log base..head` either way round. + const dates = [pin, candidate] + .map((s) => Date.parse(s.upstream?.published_at ?? s.release?.published_at ?? '')) + .filter((d) => d > 0) const since = new Date(dates.length > 0 ? Math.min(...dates) : Date.now() - 90 * 86_400_000) since.setUTCDate(since.getUTCDate() - 2) // The branch builds are cut from: `master` upstream, `prism` on PrismML's fork. const branch = contract.upstream?.branch ?? 'master' const refs = [`+refs/heads/${branch}:refs/remotes/origin/${branch}`] - for (const side of [pin, candidate]) - if (side.repo === upstreamRepo && side.tag) refs.push(`+refs/tags/${side.tag}:refs/tags/${side.tag}`) + for (const side of [pin, candidate]) { + const tag = side.upstream?.tag ?? (side.repo === upstreamRepo ? side.tag : null) + if (tag) refs.push(`+refs/tags/${tag}:refs/tags/${tag}`) + } const fetch = () => git(gitDir, [ 'fetch', diff --git a/scripts/fetch-live-fixtures.mjs b/scripts/fetch-live-fixtures.mjs index 9a9da716..41e490b8 100644 --- a/scripts/fetch-live-fixtures.mjs +++ b/scripts/fetch-live-fixtures.mjs @@ -13,6 +13,11 @@ // Env: ATOMIC_LIVE_RELEASE_TAG, ATOMIC_LIVE_RELEASE_REPO, ATOMIC_LIVE_ASSET (exact asset name), // ATOMIC_LIVE_MODEL_URL (+ ATOMIC_LIVE_MODEL_SHA256), GITHUB_TOKEN / GH_TOKEN for the API. // +// --engine sd-cpp (a config of `kind: sdcpp`) fetches stable-diffusion.cpp instead: this host's CPU +// archive of the release, unpacked alone, the config's `live_models` into --models (default +// /models; shared between builds), and manifest.json and live.json for the live env +// (scripts/lib/sdcpp-fixtures.mjs). +// // Writes into : bin-path.txt and model-path.txt (exported as ATOMIC_LIVE_UPSTREAM_BIN / // ATOMIC_LIVE_UPSTREAM_MODEL, or the engine's own prefix such as ATOMIC_LIVE_PRISM_*), tag.txt // (ATOMIC_LIVE_UPSTREAM_TAG) and fixture.json, which records @@ -35,6 +40,8 @@ import { import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { assetPatternFor, DEFAULT_RELEASE_REPO, digestMatches, pickRelease } from './lib/live-fixtures.mjs' +import { engineContract, hostPlatform } from './lib/engine-io.mjs' +import { fetchSdcppFixtures } from './lib/sdcpp-fixtures.mjs' const arg = (name) => { const index = process.argv.indexOf(name) @@ -68,6 +75,22 @@ const CAPTURE = process.argv.includes('--capture') mkdirSync(OUT, { recursive: true }) +if (engineConfig?.kind === 'sdcpp') { + if (!TAG) throw new Error('--engine sd-cpp needs --tag') + await fetchSdcppFixtures({ + out: OUT, + tag: TAG, + repo: arg('--repo') ?? engineConfig.default_repo, + config: engineConfig, + contract: engineContract(engineConfig), + modelsDir: resolve(arg('--models') ?? join(OUT, 'models')), + skipModel: SKIP_MODEL, + token: TOKEN, + platform: hostPlatform(), + }) + process.exit(0) +} + async function json(url) { const headers = { 'user-agent': 'atomic-chat-core-live-fixtures', 'accept': 'application/vnd.github+json' } if (TOKEN) headers.authorization = `Bearer ${TOKEN}` diff --git a/scripts/lib/engine-io.mjs b/scripts/lib/engine-io.mjs index b3b67d7d..83996450 100644 --- a/scripts/lib/engine-io.mjs +++ b/scripts/lib/engine-io.mjs @@ -1,13 +1,13 @@ // The I/O half of the engine candidate scripts (scripts/engine-surface-diff.mjs, -// scripts/engine-candidate.mjs): the GitHub API, conf at one commit, this core's identity, and a -// llama.cpp build fetched and run. The decisions live in engine-pin.mjs, engine-surface.mjs and +// scripts/engine-candidate.mjs): the GitHub API, conf at one commit, this core's identity, upstream +// tags, and an engine build fetched and run. The decisions live in engine-pin.mjs, engine-surface.mjs and // engine-results.mjs, which take what this returns as data. import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' import { existsSync, readFileSync } from 'node:fs' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' -import { resolvePins, resolvePrismPins } from './engine-pin.mjs' +import { resolvePins, resolvePrismPins, resolveSdcppPins } from './engine-pin.mjs' export const ROOT = fileURLToPath(new URL('../..', import.meta.url)) export const CONF_REPO = 'AtomicBot-ai/atomic-chat-conf' @@ -16,6 +16,8 @@ export const CONF_FILES = { manifest: 'backends/manifest.json', mirror: '.github/scripts/mirror.mjs', prism: 'backends/atomic-prism-manifest.json', + sdcpp: 'backends/sdcpp-manifest.json', + sdcppStaging: 'backends/sdcpp-manifest.staging.json', } const TOKEN = () => process.env.GITHUB_TOKEN || process.env.GH_TOKEN @@ -47,9 +49,9 @@ export const engineTagRe = (config) => new RegExp(config.tag_pattern ?? '^b\\d+$ /** The engine's pin per platform, from the conf file core reads it from (`pins` in the config). */ export function enginePins(config, conf, coreVersion) { - return config.pins === 'prism-manifest' - ? resolvePrismPins({ manifest: conf.prism, coreVersion }) - : resolvePins({ releases: conf.releases, manifest: conf.manifest, coreVersion }) + if (config.pins === 'prism-manifest') return resolvePrismPins({ manifest: conf.prism, coreVersion }) + if (config.pins === 'sdcpp-manifest') return resolveSdcppPins({ manifest: conf.sdcpp, coreVersion }) + return resolvePins({ releases: conf.releases, manifest: conf.manifest, coreVersion }) } /** `GET https://api.github.com` as JSON; `null` on a 404 when `allow404`. */ @@ -96,10 +98,11 @@ export function git(cwd, args, options = {}) { } /** - * conf's release list, legacy manifest, mirror script and PrismML manifest at one commit: a local - * checkout (`dir`), + * conf's release list, legacy manifest, mirror script, PrismML manifest and both sd.cpp manifests + * (the live one and the staging one a candidate waits in) at one commit: a local checkout (`dir`), * or a ref of AtomicBot-ai/atomic-chat-conf resolved to its sha first, so one run reads one version. - * A file that cannot be read is `undefined` and named in `errors`. + * A file that cannot be read is `undefined` and named in `errors`; `docs` holds each JSON file by + * path. */ export async function loadConf({ dir, ref }) { const conf = { ref: dir ? null : ref, sha: null, errors: [] } @@ -137,9 +140,35 @@ export async function loadConf({ dir, ref }) { conf.manifest = await json(CONF_FILES.manifest) conf.mirrorSource = await read(CONF_FILES.mirror) conf.prism = await json(CONF_FILES.prism) + conf.sdcpp = await json(CONF_FILES.sdcpp) + conf.sdcppStaging = await json(CONF_FILES.sdcppStaging) + conf.docs = { + [CONF_FILES.releases]: conf.releases, + [CONF_FILES.manifest]: conf.manifest, + [CONF_FILES.prism]: conf.prism, + [CONF_FILES.sdcpp]: conf.sdcpp, + [CONF_FILES.sdcppStaging]: conf.sdcppStaging, + } return conf } +/** + * The commit `tag` points to in `repo` (`git ls-remote`, the peeled commit of an annotated tag); + * `null` when the repo has no such tag or cannot be reached. + */ +export function upstreamTagCommit(repo, tag) { + const url = `https://github.com/${repo}.git` + const r = run('git', ['ls-remote', '--tags', url, `refs/tags/${tag}`, `refs/tags/${tag}^{}`], { + timeout: 60_000, + }) + if (r.status !== 0) return null + const lines = r.stdout + .split('\n') + .map((l) => l.split('\t')) + .filter(([sha]) => /^[0-9a-f]{40}$/.test(sha ?? '')) + return (lines.find(([, ref]) => ref?.endsWith('^{}')) ?? lines[0])?.[0] ?? null +} + /** This checkout: package version and commit (`null` outside git). */ export function coreIdentity() { const version = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')).version @@ -158,12 +187,14 @@ export const sha256File = (path) => createHash('sha256').update(readFileSync(pat /** * This host's archive of `repo`@`tag`, unpacked under `out` by scripts/fetch-live-fixtures.mjs * (which checks it against the release digest and reuses what is already there), with the GGUF - * `engine`'s live tests take unless `model` is false. + * `engine`'s live tests take unless `model` is false. `models`: where an engine with several model + * files (`live_models`) keeps them, shared between builds. */ -export function fetchBuild({ tag, repo, out, model = true, engine }) { +export function fetchBuild({ tag, repo, out, model = true, engine, models }) { const args = [join(ROOT, 'scripts/fetch-live-fixtures.mjs'), '--out', out, '--tag', tag, '--repo', repo] if (engine) args.push('--engine', engine) if (!model) args.push('--skip-model') + if (models) args.push('--models', models) const r = spawnSync(process.execPath, args, { stdio: ['ignore', 'inherit', 'inherit'] }) if (r.status !== 0) throw new Error(`fetch-live-fixtures ${repo}@${tag} exited ${r.status ?? r.signal}`) const read = (name) => readFileSync(join(out, name), 'utf8').trim() @@ -177,12 +208,15 @@ export function fetchBuild({ tag, repo, out, model = true, engine }) { /** * Run each of `invocations` (the contract's `-h`, `--version`, `--list-devices`) beside the binary's - * libraries, as core does; the output is stdout, a newline, stderr — the shape `runHelp` reads. + * libraries, as core does; the output is stdout, a newline, stderr — the shape `runHelp` reads. An + * invocation naming a `bin` (sd.cpp's `sd-cli`) runs that executable from `bin`'s folder. */ export function runInvocations(bin, invocations, timeoutMs = 60_000) { - return invocations.map(({ argv }) => { - const r = run(bin, argv, { cwd: dirname(bin), timeout: timeoutMs }) + return invocations.map(({ argv, bin: name }) => { + const exe = name ? join(dirname(bin), process.platform === 'win32' ? `${name}.exe` : name) : bin + const r = run(exe, argv, { cwd: dirname(bin), timeout: timeoutMs }) return { + ...(name ? { bin: name } : {}), argv, status: r.status, ...(r.signal ? { signal: r.signal } : {}), diff --git a/scripts/lib/engine-pin.d.mts b/scripts/lib/engine-pin.d.mts index 3863260e..d75f163d 100644 --- a/scripts/lib/engine-pin.d.mts +++ b/scripts/lib/engine-pin.d.mts @@ -60,7 +60,7 @@ export interface Pins { * Which conf file decided: the release list when this core reads it, else the legacy manifest; * the PrismML manifest for `atomic-prism`. */ - source: 'list' | 'legacy' | 'prism-manifest' | 'none' + source: 'list' | 'legacy' | 'prism-manifest' | 'sdcpp-manifest' | 'none' core_version: string per_platform: Record list: { readable: boolean; per_platform: Record | null } @@ -107,3 +107,29 @@ export declare function prismCatalogToBackends( options: { coreVersion: string; allowCandidates: boolean } ): Array<{ version: string; backend: string }> export declare function resolvePrismPins(input: { manifest: unknown; coreVersion: string }): Pins + +export declare const SDCPP_PIN_PLATFORMS: readonly PinPlatform[] + +export interface SdcppTagParts { + build: number + /** The upstream commit the tag names. */ + sha7: string + /** `master--`: the tag without the fork's `-a`. */ + upstream_tag: string + /** The Atomic fork's rev of a conf mirror tag; `null` on a bare upstream tag. */ + atomic_rev: string | null +} + +export declare function sdcppTagParts(tag: string): SdcppTagParts | null +export declare function sdcppTagBuild(tag: string): number | null + +export interface SdcppPinManifest { + updated_at?: string + tag_name: string + upstream_repo?: string + download_base?: string + assets: Array<{ backend: string; name: string; sha256?: string; size?: number; companion?: true }> +} + +export declare function parseSdcppPinManifest(value: unknown): SdcppPinManifest | null +export declare function resolveSdcppPins(input: { manifest: unknown; coreVersion: string }): Pins diff --git a/scripts/lib/engine-pin.mjs b/scripts/lib/engine-pin.mjs index c7d5da63..4ca13673 100644 --- a/scripts/lib/engine-pin.mjs +++ b/scripts/lib/engine-pin.mjs @@ -235,11 +235,12 @@ export function prismTagBuild(tag) { } /** - * Build number of a tag of any engine the check knows: `b11617` or `prism-b10787-73bfe6c`. PrismML's - * number is the upstream build the fork is cut from, which is what `--version` prints. + * Build number of a tag of any engine the check knows: `b11617`, `prism-b10787-73bfe6c` or + * `master-883-137f740`. PrismML's number is the upstream build the fork is cut from, which is what + * `--version` prints; sd.cpp's is the commit count of upstream's master. */ export function engineTagBuild(tag) { - return upstreamTagBuild(tag) ?? prismTagBuild(tag) + return upstreamTagBuild(tag) ?? prismTagBuild(tag) ?? sdcppTagBuild(tag) } function parsePrismAsset(raw) { @@ -318,3 +319,101 @@ export function resolvePrismPins({ manifest, coreVersion }) { legacy: { tag: null }, } } + +// --- stable-diffusion.cpp (sd-cpp) ------------------------------------------------------------------ +// +// conf's `backends/sdcpp-manifest.json` names one build, `tag_name`, for every core (no +// `min_core_version`). The pin of a platform is that tag when the manifest pins the platform's CPU +// archive with a sha256 and a size (`assetInstallable`); `parseSdcppPinManifest` mirrors +// `parseSdcppManifest` in src/engine-builds/manifest.ts. + +/** Where sd.cpp ships a build core installs (src/engine-builds/select.ts), with its CPU archive. */ +export const SDCPP_PIN_PLATFORMS = [ + { platform: 'linux-x64', os: 'linux', arch: 'x64', backend: 'linux-cpu-x64' }, + { platform: 'linux-arm64', os: 'linux', arch: 'arm64', backend: 'linux-cpu-arm64' }, + { platform: 'macos-arm64', os: 'macos', arch: 'arm64', backend: 'macos-arm64' }, + { platform: 'windows-x64', os: 'windows', arch: 'x64', backend: 'win-cpu-x64' }, + { platform: 'windows-arm64', os: 'windows', arch: 'arm64', backend: 'win-cpu-arm64' }, +] + +const SDCPP_TAG_RE = /^master-(\d+)-([0-9a-f]{7})(?:-a([0-9a-f]{7}))?$/ + +/** + * `master-883-137f740-a36f1b1a` → its build (883), upstream commit (`137f740`), the upstream tag + * (`master-883-137f740`) and the Atomic fork's rev (`36f1b1a`, `null` on a bare upstream tag). + * `null` for any other shape. + */ +export function sdcppTagParts(tag) { + const match = SDCPP_TAG_RE.exec(tag) + if (!match) return null + const [, build, sha7, rev] = match + return { build: Number(build), sha7, upstream_tag: `master-${build}-${sha7}`, atomic_rev: rev ?? null } +} + +/** Build number of an sd.cpp tag (`master-883-137f740` → 883), `null` for any other shape. */ +export function sdcppTagBuild(tag) { + return sdcppTagParts(tag)?.build ?? null +} + +const SDCPP_BACKEND_RE = /^[a-z0-9.-]+$/ +const SDCPP_ASSET_NAME_RE = /^[A-Za-z0-9._-]+\.(zip|tar\.gz|tgz)$/ + +/** conf's sd.cpp manifest as the core reads it, or `null` for anything it would refuse. */ +export function parseSdcppPinManifest(value) { + if (!isRecord(value)) return null + const tag = value.tag_name + if (typeof tag !== 'string' || !SDCPP_TAG_RE.test(tag)) return null + if (!Array.isArray(value.assets)) return null + const assets = [] + const seen = new Set() + for (const raw of value.assets) { + if (!isRecord(raw)) continue + const { backend, name, sha256, size } = raw + if (typeof backend !== 'string' || !SDCPP_BACKEND_RE.test(backend)) continue + if (typeof name !== 'string' || !SDCPP_ASSET_NAME_RE.test(name)) continue + if (seen.has(backend)) continue + seen.add(backend) + assets.push({ + backend, + name, + ...(typeof sha256 === 'string' && /^[0-9a-f]{64}$/.test(sha256) ? { sha256 } : {}), + ...(typeof size === 'number' && Number.isSafeInteger(size) && size > 0 ? { size } : {}), + ...(raw.companion === true ? { companion: true } : {}), + }) + } + if (assets.length === 0) return null + const manifest = { tag_name: tag, assets } + if (typeof value.updated_at === 'string') manifest.updated_at = value.updated_at + if ( + typeof value.upstream_repo === 'string' && + /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(value.upstream_repo) + ) + manifest.upstream_repo = value.upstream_repo + if (typeof value.download_base === 'string' && /^https:\/\/\S+$/.test(value.download_base)) + manifest.download_base = value.download_base.replace(/\/+$/, '') + return manifest +} + +/** + * The sd.cpp pin on every platform core ships it for, from conf's manifest (raw JSON, may be + * `undefined`). The same shape as `resolvePins`; there is no release list and no legacy file, and + * `coreVersion` decides nothing: every core gets the manifest's build. + */ +export function resolveSdcppPins({ manifest, coreVersion }) { + const parsed = manifest === undefined ? null : parseSdcppPinManifest(manifest) + const pins = Object.fromEntries( + SDCPP_PIN_PLATFORMS.map((p) => { + const asset = parsed?.assets.find((a) => a.backend === p.backend) + return [p.platform, asset?.sha256 && asset.size ? parsed.tag_name : null] + }) + ) + return { + source: parsed ? 'sdcpp-manifest' : 'none', + core_version: coreVersion, + per_platform: Object.fromEntries( + SDCPP_PIN_PLATFORMS.map((p) => [p.platform, { backend: p.backend, tag: pins[p.platform] }]) + ), + list: { readable: parsed !== null, per_platform: parsed ? pins : null }, + legacy: { tag: null }, + } +} diff --git a/scripts/lib/engine-results.d.mts b/scripts/lib/engine-results.d.mts index 527e7cd8..8a18d99d 100644 --- a/scripts/lib/engine-results.d.mts +++ b/scripts/lib/engine-results.d.mts @@ -5,11 +5,15 @@ export declare const CANDIDATE_ID_RE: RegExp export interface EngineCandidateConfig { engine: string + /** `sdcpp`: stable-diffusion.cpp's resolve, surface and fetch; absent: llama.cpp's. */ + kind?: string contract: string default_repo: string - platforms: Array<{ platform: string; runner: string }> + /** `core_runner`: where the core suites run when the live runner is another. */ + platforms: Array<{ platform: string; runner: string; core_runner?: string }> live_files: string[] - required_live_tests: Array<{ file: string; name: string }> + /** `platforms`: the platforms the test runs on; absent, every one. */ + required_live_tests: Array<{ file: string; name: string; platforms?: string[] }> core_suites: string[] protected_paths: string[] not_verified: string[] @@ -153,6 +157,13 @@ export interface Results { } export declare function vitestOutcomes(report: VitestReport | undefined): TestOutcome[] +export declare function coreMatrix( + platforms: EngineCandidateConfig['platforms'] +): Array<{ platform: string; os: string; shard?: string; part?: string }> +export declare function requiredLiveTests( + required: EngineCandidateConfig['required_live_tests'] | undefined, + platform: string +): EngineCandidateConfig['required_live_tests'] export declare function liveCell(input: { platform: string os?: string diff --git a/scripts/lib/engine-results.mjs b/scripts/lib/engine-results.mjs index ebde9dfa..b0f1fa0d 100644 --- a/scripts/lib/engine-results.mjs +++ b/scripts/lib/engine-results.mjs @@ -16,6 +16,24 @@ export const CANDIDATE_ID_RE = /^[A-Za-z0-9._:-]{1,80}$/ const OUTCOMES = { passed: 'passed', failed: 'failed' } +/** + * The jobs of the core suites for the engine's platforms (`platforms` of its config): one per platform + * on its runner (`core_runner` when the live runner is another), Windows split in two shards as in + * ci.yml. What the workflow's `core` job runs, so a platform the engine is not checked on costs nothing. + */ +export function coreMatrix(platforms) { + return platforms.flatMap(({ platform, runner, core_runner: coreRunner }) => { + const os = coreRunner ?? runner + return platform.startsWith('windows-') + ? [1, 2].map((k) => ({ platform, os, shard: `${k}/2`, part: `-${k}` })) + : [{ platform, os }] + }) +} + +/** The live tests a cell on `platform` has to pass: those without `platforms`, or naming it. */ +export const requiredLiveTests = (required, platform) => + (required ?? []).filter((r) => !r.platforms || r.platforms.includes(platform)) + /** A Vitest JSON report → `[{file, name, status}]`, `name` being the test's full name. */ export function vitestOutcomes(report) { return (report?.testResults ?? []).flatMap((file) => { diff --git a/scripts/lib/engine-surface.d.mts b/scripts/lib/engine-surface.d.mts index ac8bd784..3af24024 100644 --- a/scripts/lib/engine-surface.d.mts +++ b/scripts/lib/engine-surface.d.mts @@ -18,7 +18,8 @@ export interface SurfaceContract extends ContractLike { schema?: string schema_version?: number engine?: string - invocations?: Array<{ argv: string[]; source: string; expect?: string }> + /** `bin`: the executable beside the server that runs it (sd.cpp's `sd-cli`); absent, the server. */ + invocations?: Array<{ bin?: string; argv: string[]; source: string; expect?: string }> assets?: { core_backends: Array<{ platform: string; backend: string; cpu: boolean; asset: string }> } upstream?: { repo?: string @@ -41,6 +42,7 @@ export interface SurfaceContract extends ContractLike { } export interface Invocation { + bin?: string argv: string[] /** Exit code, `null` when the process died or did not start. */ status: number | null @@ -61,7 +63,11 @@ export interface SurfaceSide { binary?: { name?: string; path?: string; sha256?: string; digest_verified?: boolean | null } | null release?: { published_at?: string | null; commit?: string | null } | null help_text?: string + /** sd.cpp: what `sd-cli --help` printed, the install probe's output. */ + cli_help_text?: string version_output?: string + /** sd.cpp: the leejet tag the build is cut from, its commit and when leejet published it. */ + upstream?: { tag: string; commit: string | null; published_at: string | null } | null invocations?: Invocation[] /** The release's archive names; absent for a local binary. */ assets?: string[] | null @@ -159,6 +165,17 @@ export declare function candidateCompleteness( missing: Array<{ kind: 'mirror'; pattern: string } | { kind: 'core'; platform: string; backend: string }> } export declare function checkVersion(input: { tag?: string | null; output: string }): Finding[] +export declare function finding( + check: string, + subject: string | undefined, + severity: Severity, + message: string, + evidence?: unknown +): Finding +export declare function invocationKey(invocation: { bin?: string; argv: readonly string[] }): string +export declare function statusOf(findings: readonly Finding[]): 'pass' | 'warn' | 'break' | 'error' +export declare function countsOf(findings: readonly Finding[]): Record +export declare function sideSummary(side: SurfaceSide | null | undefined): Record | null export declare function checkInvocations( contract: SurfaceContract, invocations: readonly Invocation[] diff --git a/scripts/lib/engine-surface.mjs b/scripts/lib/engine-surface.mjs index 4253ae50..e5c9333f 100644 --- a/scripts/lib/engine-surface.mjs +++ b/scripts/lib/engine-surface.mjs @@ -116,7 +116,8 @@ export function versionCommit(output) { ) } -const finding = (check, subject, severity, message, evidence) => ({ +/** One finding: `id` is `check/subject`, stable between runs. */ +export const finding = (check, subject, severity, message, evidence) => ({ id: subject === undefined ? check : `${check}/${subject}`, check, severity, @@ -288,12 +289,15 @@ export function checkVersion({ tag, output }) { return [] } +/** What names an invocation in findings: its argv, after the executable when it names one (`sd-cli --help`). */ +export const invocationKey = ({ bin, argv }) => [...(bin ? [bin] : []), ...argv].join(' ') + /** Every invocation the contract lists (`-h`, `--version`, `--list-devices`) has to exit 0. */ export function checkInvocations(contract, invocations) { const findings = [] - for (const { argv, source } of contract.invocations ?? []) { - const key = argv.join(' ') - const ran = invocations.find((i) => i.argv.join(' ') === key) + for (const { bin, argv, source } of contract.invocations ?? []) { + const key = invocationKey({ bin, argv }) + const ran = invocations.find((i) => invocationKey(i) === key) if (!ran) findings.push(finding('invocations.exit', key, 'error', `${key} was not run`)) else if (ran.status !== 0) findings.push( @@ -485,21 +489,21 @@ export function checkStrings(contract, upstream) { const RANK = { break: 0, error: 1, warn: 2, info: 3 } /** The worst severity decides: break, then error (incomplete), then warn; info alone is a pass. */ -function statusOf(findings) { +export function statusOf(findings) { if (findings.some((f) => f.severity === 'break')) return 'break' if (findings.some((f) => f.severity === 'error')) return 'error' if (findings.some((f) => f.severity === 'warn')) return 'warn' return 'pass' } -const countsOf = (findings) => ({ +export const countsOf = (findings) => ({ break: findings.filter((f) => f.severity === 'break').length, warn: findings.filter((f) => f.severity === 'warn').length, info: findings.filter((f) => f.severity === 'info').length, error: findings.filter((f) => f.severity === 'error').length, }) -function sideSummary(side) { +export function sideSummary(side) { if (!side) return null return { tag: side.tag ?? null, @@ -638,12 +642,15 @@ export function buildReport(contract, inputs, { now, contractSha256 } = {}) { const cell = (v) => (v === null || v === undefined || v === '' ? '—' : String(v).replace(/\|/g, '\\|')) const code = (v) => (v === null || v === undefined || v === '' ? '—' : `\`${v}\``) +/** The engine as report.md names it: upstream llama.cpp and PrismML's fork both run llama.cpp. */ +const ENGINE_TITLES = { 'sd-cpp': 'stable-diffusion.cpp' } + /** `report.md`: the same report for a human, rendered from the JSON alone. */ export function renderMarkdown(report) { const { candidate, pin, pin_resolution: res, upstream, core, conf } = report.inputs const lines = [] lines.push( - `# llama.cpp surface: ${candidate?.tag ?? 'local build'} against ${pin?.tag ?? 'no pin'} — ${report.status.toUpperCase()}`, + `# ${ENGINE_TITLES[report.engine] ?? 'llama.cpp'} surface: ${candidate?.tag ?? 'local build'} against ${pin?.tag ?? 'no pin'} — ${report.status.toUpperCase()}`, '' ) const c = report.counts @@ -659,7 +666,7 @@ export function renderMarkdown(report) { const facts = [] if (res) facts.push( - `Pin: ${res.source === 'override' ? 'given' : `${res.source === 'list' ? 'the release list' : res.source === 'legacy' ? 'the legacy manifest' : res.source}`}` + + `Pin: ${res.source === 'override' ? 'given' : `${res.source === 'list' ? 'the release list' : res.source === 'legacy' ? 'the legacy manifest' : res.source === 'sdcpp-manifest' ? "conf's sd.cpp manifest" : res.source}`}` + (res.legacy?.tag ? `; legacy manifest names \`${res.legacy.tag}\`` : '') + (conf?.sha ? ` (conf \`${conf.sha.slice(0, 10)}\`)` : '') ) diff --git a/scripts/lib/live-fixtures.d.mts b/scripts/lib/live-fixtures.d.mts index 7311c4ff..d9441734 100644 --- a/scripts/lib/live-fixtures.d.mts +++ b/scripts/lib/live-fixtures.d.mts @@ -27,3 +27,18 @@ export declare function pickRelease( ): { release: R; asset: NonNullable[number] } | undefined export declare function digestMatches(digest: unknown, sha256: string): boolean | undefined + +/** What a fetch put on disk for the live run (`live.json`). */ +export interface LiveValues { + engine_dir?: string + tag?: string + manifest?: string + models?: Record +} + +export declare function expandLiveEnv( + template: Readonly>, + values: LiveValues +): Record + +export declare function liveModelsKey(models: ReadonlyArray<{ sha256: string }> | undefined): string diff --git a/scripts/lib/live-fixtures.mjs b/scripts/lib/live-fixtures.mjs index 0d4b562a..ffc0917c 100644 --- a/scripts/lib/live-fixtures.mjs +++ b/scripts/lib/live-fixtures.mjs @@ -1,6 +1,8 @@ // Pure helpers for `scripts/fetch-live-fixtures.mjs`: which release asset a host runs, which release -// to take from a newest-first listing, and whether downloaded bytes match the asset's digest. -// Kept free of I/O so `test/contract/live-fixtures.test.ts` can pin them. +// to take from a newest-first listing, whether downloaded bytes match the asset's digest, and the env +// and model cache key of an engine's live run. Kept free of I/O so +// `test/contract/live-fixtures.test.ts` can pin them. +import { createHash } from 'node:crypto' /** Default source of the binaries: ggml-org's releases. Any repo with the same asset names works. */ export const DEFAULT_RELEASE_REPO = 'ggml-org/llama.cpp' @@ -45,3 +47,34 @@ export function digestMatches(digest, sha256) { if (typeof digest !== 'string' || !digest.startsWith('sha256:')) return undefined return digest.slice('sha256:'.length).toLowerCase() === sha256.toLowerCase() } + +/** + * The env an engine's live files read, from its config's `live_env`: each value is literal text or a + * placeholder for what the fetch put on disk (`live.json`): `{engine_dir}`, `{tag}`, `{manifest}`, + * `{model:}`. A placeholder the fetch did not fill throws: a live run never starts half set up. + */ +export function expandLiveEnv(template, values) { + const lookup = (name) => { + const [kind, id] = name.split(':') + const value = kind === 'model' ? values.models?.[id] : id === undefined ? values[kind] : undefined + if (typeof value !== 'string' || value === '') + throw new Error(`the fetch left no {${name}} for the live env`) + return value + } + return Object.fromEntries( + Object.entries(template).map(([key, value]) => [ + key, + value.replace(/\{([a-z_]+(?::[A-Za-z0-9_.-]+)?)\}/g, (_, name) => lookup(name)), + ]) + ) +} + +/** + * The cache key of an engine's live models (`live_models`): their sha256 hashes, so a new model is a + * new key and the same models share one entry across platforms. Empty when the engine names none. + */ +export function liveModelsKey(models) { + if (!Array.isArray(models) || models.length === 0) return '' + const hashes = models.map((m) => m.sha256).sort() + return createHash('sha256').update(hashes.join('\n')).digest('hex').slice(0, 16) +} diff --git a/scripts/lib/sdcpp-fixtures.d.mts b/scripts/lib/sdcpp-fixtures.d.mts new file mode 100644 index 00000000..c17ae5f6 --- /dev/null +++ b/scripts/lib/sdcpp-fixtures.d.mts @@ -0,0 +1,42 @@ +import type { ReleaseAssetInfo, SdcppContract } from './sdcpp-surface.mjs' + +export interface SdcppManifestDoc { + updated_at?: string + upstream_repo: string + tag_name: string + download_base: string + assets: Array<{ backend: string; name: string; sha256?: string; size?: number; companion?: true }> +} + +export interface LiveModel { + id: string + url: string + sha256: string + bytes?: number +} + +export declare function sdcppHostArchive( + contract: SdcppContract, + tag: string, + assets: readonly A[], + platform: string +): { backend: string; asset: A } +export declare function sdcppInstallManifest(input: { + contract: SdcppContract + tag: string + repo: string + assets: readonly ReleaseAssetInfo[] + updatedAt?: string +}): SdcppManifestDoc +export declare function liveModelPath(modelsDir: string, model: LiveModel): string +export declare function fetchSdcppFixtures(input: { + out: string + tag: string + repo: string + config: { live_models?: LiveModel[] } + contract: SdcppContract + modelsDir: string + skipModel: boolean + token?: string + platform: string +}): Promise diff --git a/scripts/lib/sdcpp-fixtures.mjs b/scripts/lib/sdcpp-fixtures.mjs new file mode 100644 index 00000000..6496be93 --- /dev/null +++ b/scripts/lib/sdcpp-fixtures.mjs @@ -0,0 +1,245 @@ +// What sd.cpp's live tests run on (`node scripts/fetch-live-fixtures.mjs --engine sd-cpp`): this host's +// CPU archive of a release, unpacked alone into its own folder (the live test copies +// ATOMIC_LIVE_SD_ENGINE whole into its data folder, so no model may sit there), the model files of +// the config's `live_models` checked against their sha256 in a folder of their own, shared between +// builds and cached apart from them, and a manifest of the release for the install test +// (ATOMIC_SDCPP_MANIFEST_URL). The manifest and the archive choice are pure; `fetchSdcppFixtures` +// does the downloads. +import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { createReadStream, createWriteStream, existsSync, mkdirSync, renameSync, rmSync } from 'node:fs' +import { chmod, readdir, stat, writeFile } from 'node:fs/promises' +import { basename, dirname, join } from 'node:path' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' +import { sdcppTagParts } from './engine-pin.mjs' +import { sdcppBackendArchives } from './sdcpp-surface.mjs' +import { digestMatches } from './live-fixtures.mjs' + +/** `sha256:` → ``; `undefined` for anything else. */ +const digestHex = (digest) => (/^sha256:[0-9a-f]{64}$/.test(digest ?? '') ? digest.slice(7) : undefined) + +/** + * The CPU archive of `platform` (`linux-x64`, `macos-arm64`, …) in a release's `assets` + * (`{name, size, digest, browser_download_url}`): the backend every host of that platform falls back + * to. Throws when the contract has none for the platform or the release has none or two. + */ +export function sdcppHostArchive(contract, tag, assets, platform) { + const backend = (contract.assets?.core_backends ?? []).find((b) => b.platform === platform && b.cpu) + if (!backend) throw new Error(`sd.cpp ships no CPU build for ${platform}`) + const names = sdcppBackendArchives( + contract, + tag, + assets.map((a) => a.name) + )[backend.backend] + if (names.length !== 1) + throw new Error( + `${tag} has ${names.length} ${backend.backend} archives${names.length ? `: ${names.join(', ')}` : ''}` + ) + return { backend: backend.backend, asset: assets.find((a) => a.name === names[0]) } +} + +/** + * conf's sd.cpp manifest for `repo`@`tag`, as conf's mirror would write it: every archive a contract + * backend takes, pinned by its size and the sha256 of the release's digest, downloaded from the + * release itself (`download_base`). What the install test points ATOMIC_SDCPP_MANIFEST_URL at, so + * the core picks, downloads, checks and probes the candidate as it would for a user. + */ +export function sdcppInstallManifest({ contract, tag, repo, assets, updatedAt }) { + const archives = sdcppBackendArchives( + contract, + tag, + assets.map((a) => a.name) + ) + const entries = [] + for (const b of contract.assets?.core_backends ?? []) { + const names = archives[b.backend] + if (names.length !== 1) continue + const asset = assets.find((a) => a.name === names[0]) + entries.push({ + backend: b.backend, + name: asset.name, + ...(digestHex(asset.digest) ? { sha256: digestHex(asset.digest) } : {}), + ...(asset.size ? { size: asset.size } : {}), + ...(b.companion ? { companion: true } : {}), + }) + } + return { + ...(updatedAt ? { updated_at: updatedAt } : {}), + upstream_repo: repo, + tag_name: tag, + download_base: `https://github.com/${repo}/releases/download`, + assets: entries, + } +} + +/** Where a live model lives under the models folder: by its sha256, so two models never collide. */ +export const liveModelPath = (modelsDir, model) => + join(modelsDir, model.sha256.slice(0, 16), basename(new URL(model.url).pathname)) + +async function sha256Of(path) { + const hash = createHash('sha256') + await pipeline(createReadStream(path), hash) + return hash.digest('hex') +} + +async function download(url, dest, token) { + const headers = { 'user-agent': 'atomic-chat-core-live-fixtures' } + if (token && new URL(url).hostname === 'api.github.com') headers.authorization = `Bearer ${token}` + const res = await fetch(url, { redirect: 'follow', headers }) + if (!res.ok || !res.body) throw new Error(`${url} -> ${res.status}`) + mkdirSync(dirname(dest), { recursive: true }) + const part = `${dest}.part` + await pipeline(Readable.fromWeb(res.body), createWriteStream(part)) + renameSync(part, dest) +} + +/** `path` when it is there with `sha256`; otherwise downloaded from `url` and checked. */ +async function fetchChecked(url, path, sha256, token) { + if (existsSync(path) && (await sha256Of(path)) === sha256) return { reused: true } + console.log(`downloading ${url}`) + await download(url, path, token) + const actual = await sha256Of(path) + if (actual !== sha256) { + rmSync(path, { force: true }) + throw new Error(`${url}: sha256 ${actual} does not match ${sha256}`) + } + return { reused: false } +} + +function extract(archive, dir) { + rmSync(dir, { recursive: true, force: true }) + mkdirSync(dir, { recursive: true }) + const result = archive.endsWith('.zip') + ? process.platform === 'win32' + ? spawnSync( + 'powershell.exe', + [ + '-NoProfile', + '-Command', + `Expand-Archive -LiteralPath '${archive}' -DestinationPath '${dir}' -Force`, + ], + { stdio: 'inherit' } + ) + : spawnSync('unzip', ['-o', '-q', archive, '-d', dir], { stdio: 'inherit' }) + : spawnSync('tar', ['-xzf', archive, '-C', dir], { stdio: 'inherit' }) + if (result.status !== 0) throw new Error(`failed to extract ${archive}`) +} + +async function findFile(dir, wanted) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name) + if (entry.isDirectory()) { + const found = await findFile(path, wanted) + if (found) return found + } else if (entry.name === wanted) return path + } + return undefined +} + +/** + * Everything one live cell of sd.cpp needs, under `out`: the archive (reused while it matches the + * release digest), its tree unpacked into `engine-/`, the models under `modelsDir` (skipped with + * `skipModel`), `manifest.json`, and the files the live step and the cell read: `bin-path.txt`, + * `model-path.txt`, `tag.txt`, `live.json` (the values `live_env` names) and `fixture.json`. + */ +export async function fetchSdcppFixtures({ + out, + tag, + repo, + config, + contract, + modelsDir, + skipModel, + token, + platform, +}) { + const api = `https://api.github.com/repos/${repo}/releases/tags/${encodeURIComponent(tag)}` + const headers = { 'user-agent': 'atomic-chat-core-live-fixtures', 'accept': 'application/vnd.github+json' } + if (token) headers.authorization = `Bearer ${token}` + const res = await fetch(api, { headers }) + if (!res.ok) throw new Error(`${api} -> ${res.status}`) + const release = await res.json() + if (!sdcppTagParts(release.tag_name)) throw new Error(`${release.tag_name} is not an sd.cpp tag`) + const { backend, asset } = sdcppHostArchive(contract, release.tag_name, release.assets ?? [], platform) + console.log(`using sd.cpp ${release.tag_name} (${backend}) from ${repo}`) + + mkdirSync(out, { recursive: true }) + const archive = join(out, asset.name) + if (!existsSync(archive) || digestMatches(asset.digest, await sha256Of(archive)) === false) { + console.log(`downloading ${asset.name}`) + await download(asset.browser_download_url, archive, token) + } + const archiveSha256 = await sha256Of(archive) + const verified = digestMatches(asset.digest, archiveSha256) + if (verified === false) + throw new Error(`${asset.name}: sha256 ${archiveSha256} does not match ${asset.digest}`) + + const exe = (name) => (process.platform === 'win32' ? `${name}.exe` : name) + const unpacked = join(out, `engine-${release.tag_name}`) + extract(archive, unpacked) + const server = await findFile(unpacked, exe('sd-server')) + if (!server) throw new Error(`no ${exe('sd-server')} inside ${asset.name}`) + const engineDir = dirname(server) + if (process.platform !== 'win32') + for (const name of ['sd-server', 'sd-cli']) + if (existsSync(join(engineDir, name))) await chmod(join(engineDir, name), 0o755) + const version = spawnSync(server, ['--version'], { cwd: engineDir, encoding: 'utf8', timeout: 60_000 }) + const versionOutput = `${version.stdout ?? ''}\n${version.stderr ?? ''}`.trim() + + const models = {} + const records = [] + if (!skipModel) + for (const model of config.live_models ?? []) { + const path = liveModelPath(modelsDir, model) + const { reused } = await fetchChecked(model.url, path, model.sha256, token) + models[model.id] = path + records.push({ + id: model.id, + url: model.url, + sha256: model.sha256, + bytes: (await stat(path)).size, + reused, + }) + } + + const manifest = join(out, 'manifest.json') + await writeFile( + manifest, + `${JSON.stringify(sdcppInstallManifest({ contract, tag: release.tag_name, repo, assets: release.assets ?? [], updatedAt: release.published_at }), null, 2)}\n` + ) + const main = records[0] + await writeFile(join(out, 'bin-path.txt'), server) + if (main) await writeFile(join(out, 'model-path.txt'), models[main.id]) + await writeFile(join(out, 'tag.txt'), release.tag_name) + await writeFile( + join(out, 'live.json'), + `${JSON.stringify({ engine_dir: engineDir, tag: release.tag_name, manifest, models }, null, 2)}\n` + ) + await writeFile( + join(out, 'fixture.json'), + `${JSON.stringify( + { + repo, + tag: release.tag_name, + prerelease: release.prerelease === true, + platform: process.platform, + arch: process.arch, + backend, + asset: { + name: asset.name, + size: asset.size, + sha256: archiveSha256, + digest_verified: verified ?? null, + }, + model: main ? { url: main.url, sha256: main.sha256, verified: true } : null, + models: records, + version_output: versionOutput, + }, + null, + 2 + )}\n` + ) + console.log(`ATOMIC_LIVE_SD_ENGINE=${engineDir}`) + for (const [id, path] of Object.entries(models)) console.log(`model ${id}: ${path}`) +} diff --git a/scripts/lib/sdcpp-surface.d.mts b/scripts/lib/sdcpp-surface.d.mts new file mode 100644 index 00000000..35914aab --- /dev/null +++ b/scripts/lib/sdcpp-surface.d.mts @@ -0,0 +1,125 @@ +import type { Finding, Severity } from './llama-help.mjs' +import type { Invocation, SurfaceInputs, SurfaceReport } from './engine-surface.mjs' + +/** The sd.cpp engine contract as the scripts read it (test/fixtures/sdcpp/engine-contract.json). */ +export interface SdcppContract { + schema?: string + schema_version?: number + engine?: string + invocations?: Array<{ + bin: string + argv: string[] + source: string + expect?: string + severity?: Severity + optional?: boolean + }> + probe?: { bins: string[]; argv: string[]; markers: string[] } + version?: { pattern: string } + flags: Array<{ flag: string; severity?: Severity; source?: string; note?: string }> + strings?: Array<{ + id: string + text: string + channel?: string + source?: string | null + upstream_text?: string | null + severity_if_lost?: Severity + }> + assets?: { + core_backends: Array<{ + platform: string + backend: string + cpu: boolean + required: boolean + companion?: boolean + asset: string + }> + } + upstream?: { + repo?: string + branch?: string + watched_paths: Array<{ path: string; why: string }> + release_note_keywords: Array<{ pattern: string; severity: 'warn' | 'info' }> + } +} + +export interface SdHelpOption { + aliases: string[] + hint: string + help: string + deprecated: boolean +} + +export interface ReleaseAssetInfo { + name: string + size?: number + digest?: string | null +} + +export type SdcppMissing = + | { kind: 'tag'; tag: string } + | { kind: 'core'; platform: string; backend: string } + | { kind: 'ambiguous'; backend: string; names: string[] } + | { kind: 'digest'; backend: string; name: string } + | { kind: 'manifest'; reason: string } + | { kind: 'manifest-asset'; backend: string; name?: string; reason: string } + +export declare const SDCPP_CHECKS: ReadonlyArray<{ id: string; title: string }> +export declare const MIN_SD_HELP_OPTIONS: number +export declare function parseSdHelp(text: string): SdHelpOption[] +export declare function findSdOption(options: readonly SdHelpOption[], flag: string): SdHelpOption | undefined +export declare function checkSdHelp(contract: SdcppContract, options: readonly SdHelpOption[]): Finding[] +export declare function diffSdHelp( + pinOptions: readonly SdHelpOption[], + candidateOptions: readonly SdHelpOption[], + contract: SdcppContract +): Finding[] +export declare function notInArchive(ran: Invocation): boolean +export declare function checkSdInvocations( + contract: SdcppContract, + invocations: readonly Invocation[] +): Finding[] +export declare function checkSdProbe( + contract: SdcppContract, + texts: Readonly> +): Finding[] +export declare function parseSdVersion( + contract: SdcppContract, + output: string +): { version: string; commit: string } | undefined +export declare function checkSdVersion( + contract: SdcppContract, + input: { tag?: string | null; output: string } +): Finding[] +export declare function sdAssetRe(pattern: string, sha7: string): RegExp +export declare function sdcppBackendArchives( + contract: SdcppContract, + tag: string, + assets: readonly string[] +): Record +export declare function sdcppCoreAssets( + contract: SdcppContract, + tag: string, + assets: readonly string[] +): string[] +export declare function checkSdAssets( + contract: SdcppContract, + input: { + candidate: { tag: string; assets: readonly string[] } + pin: { tag: string; assets: readonly string[] } | null + } +): Finding[] +export declare function sdcppCompleteness(input: { + contract: SdcppContract + tag: string + assets: readonly ReleaseAssetInfo[] + platforms?: readonly string[] + /** conf's staging manifest at the run's commit; `null` when conf has none; absent off conf. */ + manifest?: unknown + downloadBase?: string +}): { complete: boolean; missing: SdcppMissing[] } +export declare function buildSdcppReport( + contract: SdcppContract, + inputs: SurfaceInputs, + options?: { now?: string; contractSha256?: string } +): SurfaceReport diff --git a/scripts/lib/sdcpp-surface.mjs b/scripts/lib/sdcpp-surface.mjs new file mode 100644 index 00000000..2f78f9b0 --- /dev/null +++ b/scripts/lib/sdcpp-surface.mjs @@ -0,0 +1,552 @@ +// The surface diff of a stable-diffusion.cpp candidate (engine `sd-cpp`): what can be checked without +// a model — `sd-server --help` and `--version`, the install probe's `sd-cli --help`, the release's +// archives, and leejet's history between the pin and the candidate — held to the engine contract +// (test/fixtures/sdcpp/engine-contract.json). The report is llama.cpp's (`atomic-engine-surface-report` +// v1, engine-surface.mjs) with sd.cpp's own checks. Also what `resolve` asks of a release before it is +// tested: every archive up, and on conf's signed mirror a staging manifest naming exactly those +// bytes. Pure; scripts/engine-surface-diff.mjs and scripts/engine-candidate.mjs do the I/O. +import { + checkReleaseNotes, + checkStrings, + checkWatched, + countsOf, + finding, + invocationKey, + sideSummary, + statusOf, + SURFACE_REPORT_SCHEMA, +} from './engine-surface.mjs' +import { parseSdcppPinManifest, sdcppTagParts } from './engine-pin.mjs' + +/** Every check of an sd.cpp report, in report order. */ +export const SDCPP_CHECKS = [ + { id: 'help.parse', title: '`sd-server --help` reads as a help screen' }, + { id: 'help.required_flags', title: 'sd-server knows every flag core emits' }, + { id: 'help.deprecated', title: 'none of them is deprecated' }, + { id: 'help.diff', title: '`--help` changes since the pin' }, + { id: 'invocations.exit', title: 'the invocations core makes exit 0' }, + { id: 'install.probe', title: 'the install probe recognises the build' }, + { id: 'version.output', title: '`--version` names the commit of the tag' }, + { id: 'assets.core_backends', title: 'an archive for every backend core installs' }, + { id: 'upstream.watched_paths', title: 'changes to the sources core depends on' }, + { id: 'upstream.contract_strings', title: 'the strings and routes core uses are still there' }, + { id: 'upstream.release_notes', title: 'commits announcing a removal or rename' }, +] + +const HINTS = { + 'help.parse': 'Look at help-candidate.txt; the reader is parseSdHelp in scripts/lib/sdcpp-surface.mjs.', + 'help.required_flags': + 'buildServerArgs (src/diffusion/args.ts) emits it: follow the rename there, in required-flags.txt and the contract.', + 'help.deprecated': 'Plan the replacement in buildServerArgs before upstream removes the flag.', + 'invocations.exit': 'Read what the binary printed; core runs --help at install.', + 'install.probe': + 'prepareSdcppTree (src/diffusion/install.ts) refuses a tree whose --help shows none of the markers.', + 'version.output': 'The archive may not be the build of its tag: compare the release with its build run.', + 'assets.core_backends': + 'Archive names: conf mirror-sdcpp.yml (MATRIX), backend ids: src/engine-builds/select.ts; update both and the contract.', + 'upstream.watched_paths': 'Update the contract’s upstream.watched_paths.', + 'upstream.contract_strings': 'Core matches this text or calls this route; find what upstream has now.', + 'upstream.release_notes': 'Read the commit: it may remove or rename something core uses.', +} + +/** Fewer options than this means the output is not a help screen we understand: fail closed. */ +export const MIN_SD_HELP_OPTIONS = 40 + +const ALIAS = /^-{1,2}[A-Za-z0-9][\w-]*$/ +const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g') +/** An option starts on an indented line with its first alias; help text wraps onto deeper ones. */ +const OPTION_LINE = /^ {1,8}-{1,2}[A-Za-z0-9]/ + +/** + * Options of an sd.cpp `--help` screen (examples/common/common.cpp): under each `… Options:` header, + * ` -l, --listen-ip server listen ip …`, the help wrapping onto lines indented past the + * options' column. Aliases are exact tokens, so `--vae` is not found in `--vae-format`. + */ +export function parseSdHelp(text) { + const options = [] + let current + for (const line of text.replace(/\r\n/g, '\n').replace(ANSI, '').split('\n')) { + if (OPTION_LINE.test(line)) { + const aliases = [] + let end = 0 + for (const match of line.matchAll(/[^\s,]+/g)) { + if (!ALIAS.test(match[0])) break + aliases.push(match[0]) + end = match.index + match[0].length + } + const rest = line.slice(end).trim() + const hint = /^<[^>]*>/.exec(rest)?.[0] ?? '' + current = { aliases, hint, help: rest.slice(hint.length).trim() } + options.push(current) + } else if (current && /^\s+\S/.test(line)) { + current.help = `${current.help}\n${line.trim()}`.trim() + } else current = undefined + } + return options.map((o) => ({ ...o, deprecated: /deprecated/i.test(o.help) })) +} + +/** The option a flag spells, if the build knows it. */ +export const findSdOption = (options, flag) => options.find((o) => o.aliases.includes(flag)) + +/** + * The build's `--help` against the flags core emits (the contract's `flags`): one it does not know + * fails the launch (`break`), or is a `warn` for a flag core emits only when opted in (`-M`); one it + * marks deprecated is a `warn`. + */ +export function checkSdHelp(contract, options) { + if (options.length < MIN_SD_HELP_OPTIONS) + return [ + finding( + 'help.parse', + undefined, + 'error', + `read ${options.length} options, expected ${MIN_SD_HELP_OPTIONS}+` + ), + ] + const findings = [] + for (const f of contract.flags) { + const option = findSdOption(options, f.flag) + const severity = f.severity ?? 'break' + if (!option) + findings.push( + finding( + 'help.required_flags', + f.flag, + severity, + severity === 'break' + ? `sd-server does not know ${f.flag}, which core emits: the launch fails` + : `sd-server does not know ${f.flag}${f.note ? ` (${f.note})` : ''}`, + { source: f.source } + ) + ) + else if (option.deprecated) + findings.push( + finding( + 'help.deprecated', + f.flag, + 'warn', + `sd-server marks ${f.flag} deprecated; core still emits it`, + { + help: option.help, + } + ) + ) + } + return findings +} + +/** + * What changed between the pin's and the candidate's `--help`, for the flags core never emits (a lost + * or deprecated flag core emits is already its own finding). + */ +export function diffSdHelp(pinOptions, candidateOptions, contract) { + const pin = new Set(pinOptions.flatMap((o) => o.aliases)) + const candidate = new Set(candidateOptions.flatMap((o) => o.aliases)) + const emitted = new Set(contract.flags.map((f) => f.flag)) + const findings = [] + const removed = [...pin].filter((f) => !candidate.has(f) && !emitted.has(f)).sort() + if (removed.length > 0) + findings.push( + finding('help.diff', 'removed', 'info', `${removed.length} flag(s) core never emits are gone`, { + flags: removed, + }) + ) + const added = [...candidate].filter((f) => !pin.has(f)).sort() + if (added.length > 0) + findings.push(finding('help.diff', 'added', 'info', `${added.length} new flag(s)`, { flags: added })) + const wasDeprecated = new Set(pinOptions.filter((o) => o.deprecated).flatMap((o) => o.aliases)) + const deprecated = candidateOptions + .filter((o) => o.deprecated) + .flatMap((o) => o.aliases) + .filter((f) => pin.has(f) && !wasDeprecated.has(f) && !emitted.has(f)) + .sort() + if (deprecated.length > 0) + findings.push( + finding( + 'help.diff', + 'deprecated', + 'info', + `${deprecated.length} flag(s) core never emits are newly deprecated`, + { flags: deprecated } + ) + ) + return findings +} + +/** The executable of an invocation was not there to run (`spawnSync … ENOENT`). */ +export const notInArchive = (ran) => ran.status === null && /ENOENT/.test(ran.error ?? '') + +/** + * The contract's invocations, by their exit: each has its own severity (core reads the output of + * `--help`, not its exit code). A missing executable is `break` unless the invocation is `optional` + * (an archive without `sd-cli` is probed through `sd-server`). + */ +export function checkSdInvocations(contract, invocations) { + const findings = [] + for (const inv of contract.invocations ?? []) { + const key = invocationKey(inv) + const ran = invocations.find((i) => invocationKey(i) === key) + if (!ran) findings.push(finding('invocations.exit', key, 'error', `${key} was not run`)) + else if (notInArchive(ran)) + findings.push( + inv.optional + ? finding('invocations.exit', key, 'info', `the archive has no ${inv.bin}`) + : finding('invocations.exit', key, 'break', `the archive has no ${inv.bin}: core cannot run it`) + ) + else if (ran.status !== 0) + findings.push( + finding( + 'invocations.exit', + key, + inv.severity ?? 'break', + `${key} ${ran.status === null ? `died (${ran.signal ?? ran.error ?? 'no status'})` : `exits ${ran.status}`}; core runs it (${inv.source})`, + { output: (ran.output ?? '').trim().slice(-400) } + ) + ) + } + return findings +} + +/** + * The install probe as core runs it: the first of the contract's `probe.bins` the archive has + * (`sd-cli`, else `sd-server`), its `--help` holding one of the markers. `texts` maps each binary to + * what it printed, `undefined` when the archive does not have it. + */ +export function checkSdProbe(contract, texts) { + const probe = contract.probe + if (!probe) return [] + const bin = probe.bins.find((b) => texts[b] !== undefined) + if (!bin) return [finding('install.probe', undefined, 'error', `none of ${probe.bins.join(', ')} ran`)] + const lower = texts[bin].toLowerCase() + if (probe.markers.some((m) => lower.includes(m.toLowerCase()))) return [] + return [ + finding( + 'install.probe', + bin, + 'break', + `${bin} ${probe.argv.join(' ')} prints none of ${probe.markers.map((m) => `"${m}"`).join(', ')}: the archive installs on no host`, + { output: texts[bin].trim().slice(0, 400) } + ), + ] +} + +/** `{version, commit}` from `sd-server --version`, as the contract's `version.pattern` reads it. */ +export function parseSdVersion(contract, output) { + const match = new RegExp(contract.version?.pattern ?? '$^').exec(output) + return match ? { version: match[1], commit: match[2] } : undefined +} + +/** + * `--version` ties the binary to its tag: its commit has to be the tag's `` (a conf mirror tag + * names the same upstream commit). Core does not read the version, so an unreadable line is a warn. + */ +export function checkSdVersion(contract, { tag, output }) { + const parsed = parseSdVersion(contract, output) + const evidence = { output: output.trim().slice(0, 400) } + if (!parsed) + return [ + finding( + 'version.output', + undefined, + 'warn', + '--version prints no "stable-diffusion.cpp version …, commit …" line: the build cannot be tied to its tag', + evidence + ), + ] + const parts = tag ? sdcppTagParts(tag) : null + const findings = [] + if (!/^[0-9a-f]{7,40}$/.test(parsed.commit)) + findings.push( + finding('version.output', 'commit', 'warn', `--version names no commit (${parsed.commit})`, evidence) + ) + else if (parts && !parsed.commit.startsWith(parts.sha7) && !parts.sha7.startsWith(parsed.commit)) + findings.push( + finding( + 'version.output', + 'commit', + 'break', + `--version names commit ${parsed.commit}, the tag ${parts.sha7}: the archive is not the build of ${tag}`, + evidence + ) + ) + if (parsed.version === 'unknown') + findings.push( + finding('version.output', 'version', 'info', 'built from a checkout without tags: version unknown') + ) + else if (parts && parsed.version.replace(/\+$/, '') !== parts.upstream_tag) + findings.push( + finding( + 'version.output', + 'version', + 'info', + `--version says ${parsed.version}, the tag names ${parts.upstream_tag}` + ) + ) + return findings +} + +const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +/** A contract archive pattern for the build of `sha7`. */ +export const sdAssetRe = (pattern, sha7) => new RegExp(`^${pattern.replace('{sha7}', escapeRegExp(sha7))}$`) +/** `sd-master--bin-` → ``; any other name as it is. */ +const binRest = (name, sha7) => + new RegExp(`^sd-master-${escapeRegExp(sha7)}-bin-(.+)$`).exec(name)?.[1] ?? name +const ARCHIVE = /\.(zip|tar\.gz|tgz)$/ + +/** The archives of `assets` (names) each contract backend takes, for the build `tag` names. */ +export function sdcppBackendArchives(contract, tag, assets) { + const sha7 = sdcppTagParts(tag)?.sha7 ?? '\0' + return Object.fromEntries( + (contract.assets?.core_backends ?? []).map((b) => [ + b.backend, + assets.filter((n) => sdAssetRe(b.asset, sha7).test(n)).sort(), + ]) + ) +} + +/** The archives of a release core installs: what a passing live run does not cover is listed as unverified. */ +export function sdcppCoreAssets(contract, tag, assets) { + return Object.values(sdcppBackendArchives(contract, tag, assets)) + .flat() + .sort() +} + +/** + * The release's archives against the backends core installs (`assets.core_backends`). A missing CPU + * archive leaves its platform without sd.cpp, a missing `required` one stops conf's mirror (both + * `break`); two archives for one backend stop the mirror too. `pin` is the pin's archive list: a + * backend the pin had and the candidate lacks is a loss (`warn`). + */ +export function checkSdAssets(contract, { candidate, pin }) { + const backends = contract.assets?.core_backends ?? [] + if (backends.length === 0) + return [finding('assets.core_backends', undefined, 'error', 'the contract lists no core backends')] + const findings = [] + const now = sdcppBackendArchives(contract, candidate.tag, candidate.assets) + const before = pin ? sdcppBackendArchives(contract, pin.tag, pin.assets) : null + const sha7 = sdcppTagParts(candidate.tag)?.sha7 ?? '' + const pinSha7 = pin ? (sdcppTagParts(pin.tag)?.sha7 ?? '') : '' + for (const b of backends) { + const names = now[b.backend] + const had = before?.[b.backend] ?? null + const subject = `${b.platform}/${b.backend}` + if (names.length === 0) { + const missing = `no ${b.backend} archive in ${candidate.tag}` + const [severity, message] = b.cpu + ? ['break', `${missing}: ${b.platform} gets no sd.cpp`] + : b.required + ? ['break', `${missing}, which conf's mirror requires: conf cannot mirror the tag`] + : had && had.length > 0 + ? ['warn', `${missing}, which ${pin.tag} had: core stops offering it on ${b.platform}`] + : had + ? ['info', `${missing}, nor in ${pin.tag}`] + : ['info', `${missing}: core does not offer it on ${b.platform}`] + findings.push(finding('assets.core_backends', subject, severity, message, { pattern: b.asset })) + } else if (names.length > 1) + findings.push( + finding( + 'assets.core_backends', + subject, + 'break', + `${names.length} archives match ${b.backend}: conf's mirror refuses the tag`, + { archives: names } + ) + ) + else if (had?.length === 1 && binRest(had[0], pinSha7) !== binRest(names[0], sha7)) + findings.push( + finding( + 'assets.core_backends', + subject, + 'info', + `${b.backend} moved: ${binRest(had[0], pinSha7)} → ${binRest(names[0], sha7)}` + ) + ) + } + const consumed = new Set(Object.values(now).flat()) + const unused = candidate.assets.filter((n) => ARCHIVE.test(n) && !consumed.has(n)).sort() + if (unused.length > 0) + findings.push( + finding( + 'assets.core_backends', + 'unconsumed', + 'info', + `${unused.length} archive(s) no backend core installs takes`, + { archives: unused } + ) + ) + return findings +} + +/** + * Whether a release can be tested: every archive conf's mirror requires and the CPU archive of each + * of `platforms` is up exactly once, with the digest GitHub computes once an upload finishes. On + * conf's signed mirror (`manifest`: the staging manifest at the run's conf commit) the manifest has + * to name this tag, point its downloads at `downloadBase`, and pin every archive it lists with the + * size and sha256 the release has. `assets` are the release's (`{name, size, digest}`). + */ +export function sdcppCompleteness({ contract, tag, assets, platforms, manifest, downloadBase }) { + const missing = [] + if (!sdcppTagParts(tag)) return { complete: false, missing: [{ kind: 'tag', tag }] } + const wanted = (contract.assets?.core_backends ?? []).filter( + (b) => b.required || (b.cpu && (platforms ?? []).includes(b.platform)) + ) + const archives = sdcppBackendArchives( + contract, + tag, + assets.map((a) => a.name) + ) + for (const b of wanted) { + const names = archives[b.backend] + if (names.length === 0) missing.push({ kind: 'core', platform: b.platform, backend: b.backend }) + else if (names.length > 1) missing.push({ kind: 'ambiguous', backend: b.backend, names }) + else if (!/^sha256:[0-9a-f]{64}$/.test(assets.find((a) => a.name === names[0])?.digest ?? '')) + missing.push({ kind: 'digest', backend: b.backend, name: names[0] }) + } + if (manifest !== undefined) { + const parsed = manifest === null ? null : parseSdcppPinManifest(manifest) + if (!parsed) missing.push({ kind: 'manifest', reason: 'conf has no staging manifest core reads' }) + else if (parsed.tag_name !== tag) + missing.push({ kind: 'manifest', reason: `the staging manifest names ${parsed.tag_name}` }) + else { + if (parsed.download_base !== downloadBase) + missing.push({ + kind: 'manifest', + reason: `the staging manifest downloads from ${parsed.download_base ?? 'upstream'}, not ${downloadBase}`, + }) + for (const entry of parsed.assets) { + const asset = assets.find((a) => a.name === entry.name) + const reason = !asset + ? 'not in the release' + : asset.size !== entry.size + ? `size ${asset.size} in the release, ${entry.size ?? 'none'} in the manifest` + : asset.digest !== `sha256:${entry.sha256}` + ? `digest ${asset.digest ?? 'none'} in the release, sha256 ${entry.sha256 ?? 'none'} in the manifest` + : null + if (reason) missing.push({ kind: 'manifest-asset', backend: entry.backend, name: entry.name, reason }) + } + for (const b of wanted) + if (!parsed.assets.some((a) => a.backend === b.backend)) + missing.push({ kind: 'manifest-asset', backend: b.backend, reason: 'not in the staging manifest' }) + } + } + return { complete: missing.length === 0, missing } +} + +const RANK = { break: 0, error: 1, warn: 2, info: 3 } + +/** A side as the report shows it, with the leejet tag it is cut from. */ +const summary = (side) => side && { ...sideSummary(side), upstream: side.upstream ?? null } + +/** + * The report for one sd.cpp candidate, from what scripts/engine-surface-diff.mjs gathered: both + * sides' `--help` (`help_text`, and `cli_help_text` for `sd-cli`), `--version`, the invocations and + * the archive lists, the pin's resolution, and the upstream history between the two commits. + */ +export function buildSdcppReport(contract, inputs, { now, contractSha256 } = {}) { + const findings = [] + const skipped = {} + const candidate = inputs.candidate + const pin = inputs.pin ?? null + + if (candidate.capture_error) { + for (const check of ['help.parse', 'invocations.exit', 'install.probe', 'version.output']) + findings.push( + finding(check, undefined, 'error', `the candidate was not run: ${candidate.capture_error}`) + ) + skipped['help.diff'] = 'the candidate was not run' + } else { + const options = parseSdHelp(candidate.help_text ?? '') + findings.push(...checkSdHelp(contract, options)) + const pinOptions = pin?.help_text === undefined ? undefined : parseSdHelp(pin.help_text) + if (!pin) skipped['help.diff'] = 'no pin' + else if (pin.capture_error) + findings.push(finding('help.diff', undefined, 'error', `the pin was not run: ${pin.capture_error}`)) + else if (!pinOptions || pinOptions.length < MIN_SD_HELP_OPTIONS) + findings.push( + finding('help.diff', undefined, 'error', 'the pin’s --help does not read as a help screen') + ) + else if (options.length >= MIN_SD_HELP_OPTIONS) + findings.push(...diffSdHelp(pinOptions, options, contract)) + findings.push(...checkSdInvocations(contract, candidate.invocations ?? [])) + findings.push( + ...checkSdProbe(contract, { 'sd-cli': candidate.cli_help_text, 'sd-server': candidate.help_text }) + ) + findings.push(...checkSdVersion(contract, { tag: candidate.tag, output: candidate.version_output ?? '' })) + } + + if (!Array.isArray(candidate.assets)) { + if (candidate.assets_error) + findings.push( + finding('assets.core_backends', undefined, 'error', `no archive list: ${candidate.assets_error}`) + ) + else skipped['assets.core_backends'] = 'a local binary has no release' + } else + findings.push( + ...checkSdAssets(contract, { + candidate: { tag: candidate.tag, assets: candidate.assets }, + pin: pin && Array.isArray(pin.assets) ? { tag: pin.tag, assets: pin.assets } : null, + }) + ) + + const upstream = inputs.upstream ?? { available: false, reason: 'not compared' } + if (upstream.available) { + findings.push(...checkWatched(contract, upstream)) + findings.push(...(checkStrings(contract, upstream) ?? [])) + findings.push(...checkReleaseNotes(contract, upstream)) + } else + for (const check of ['upstream.watched_paths', 'upstream.release_notes', 'upstream.contract_strings']) + findings.push(finding(check, 'unavailable', 'info', `upstream diff unavailable: ${upstream.reason}`)) + + for (const f of findings) { + const hint = HINTS[f.check] + if (hint && f.severity !== 'info') f.hint = hint + } + findings.sort((a, b) => RANK[a.severity] - RANK[b.severity] || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) + + const checks = SDCPP_CHECKS.map(({ id, title }) => { + const own = findings.filter((f) => f.check === id) + const reason = skipped[id] + return { + id, + title, + status: reason !== undefined && own.length === 0 ? 'skipped' : statusOf(own), + counts: countsOf(own), + ...(reason !== undefined && own.length === 0 ? { skipped: reason } : {}), + } + }) + + return { + schema: SURFACE_REPORT_SCHEMA, + schema_version: 1, + generated_at: now ?? null, + engine: contract.engine, + inputs: { + contract: { + schema: contract.schema, + schema_version: contract.schema_version, + sha256: contractSha256 ?? null, + }, + core: inputs.core ?? null, + conf: inputs.conf ?? null, + candidate: summary(candidate), + pin: summary(pin), + pin_resolution: inputs.pin_resolution ?? null, + mirror_whitelist: null, + upstream: upstream.available + ? { + available: true, + method: upstream.method, + base: upstream.base, + head: upstream.head, + commits: upstream.commits.length, + truncated: upstream.truncated === true, + } + : { available: false, reason: upstream.reason }, + same_as_pin: !!pin && !!candidate.tag && candidate.tag === pin.tag, + }, + status: statusOf(findings), + counts: countsOf(findings), + checks, + findings, + } +} diff --git a/src/diffusion/args.test.ts b/src/diffusion/args.test.ts index d5aa27dd..fc52660c 100644 --- a/src/diffusion/args.test.ts +++ b/src/diffusion/args.test.ts @@ -22,9 +22,11 @@ import { videoModeFlags, commandSummaryForLog, cpuBackendExtraArgs, + DIFFUSION_BACKEND_ENV, hostEnv, isGgmlUnsupportedOpAbort, isM5Brand, + loadExtraArgs, METAL_TE_GPU_ENV, metalTextEncoderFlags, offloadFlags, @@ -224,6 +226,14 @@ describe('the CPU-backend recovery', () => { expect(withoutDeviceBackendFlags(['-x', '--backend'])).toEqual(['-x']) }) + it('starts a load on the CPU only under ATOMIC_DIFFUSION_BACKEND=cpu', () => { + expect(DIFFUSION_BACKEND_ENV).toBe('ATOMIC_DIFFUSION_BACKEND') + for (const value of ['cpu', 'CPU', ' cpu ']) + expect(loadExtraArgs(value), value).toEqual(['--backend', 'cpu']) + for (const value of [undefined, '', 'metal', 'cuda0', 'cpu,metal']) + expect(loadExtraArgs(value), String(value)).toEqual([]) + }) + it('needs both markers of the ggml abort signature', () => { expect( isGgmlUnsupportedOpAbort("ggml_metal_op_encode_impl: error: unsupported op 'RMS_NORM'\nGGML_ABORT ...") diff --git a/src/diffusion/args.ts b/src/diffusion/args.ts index a3cb745d..0943569f 100644 --- a/src/diffusion/args.ts +++ b/src/diffusion/args.ts @@ -123,6 +123,18 @@ export function cpuBackendExtraArgs(extraArgs: readonly string[]): string[] { return [...withoutDeviceBackendFlags(extraArgs), '--backend', 'cpu'] } +/** + * `cpu` starts every load on the CPU backend. The engine candidate check sets it through the live + * test: a macOS CI runner's GPU is a paravirtual device that ggml's Metal kernels crash on. No + * other value does anything. + */ +export const DIFFUSION_BACKEND_ENV = 'ATOMIC_DIFFUSION_BACKEND' + +/** The `extraArgs` a load starts with: the CPU backend under `ATOMIC_DIFFUSION_BACKEND=cpu`, else none. */ +export function loadExtraArgs(envValue: string | undefined): string[] { + return (envValue ?? '').trim().toLowerCase() === 'cpu' ? cpuBackendExtraArgs([]) : [] +} + const GGML_UNSUPPORTED_OP_MARKERS = ['unsupported op', 'ggml_abort'] /** diff --git a/src/diffusion/service.test.ts b/src/diffusion/service.test.ts index d47c7b39..6488ee0c 100644 --- a/src/diffusion/service.test.ts +++ b/src/diffusion/service.test.ts @@ -69,7 +69,12 @@ const MAC_16: SystemInfo = { } function harness( - options: { idleTickMs?: number; claimGpu?: GpuClaimHook; systemInfo?: SystemInfo } = {} + options: { + idleTickMs?: number + claimGpu?: GpuClaimHook + systemInfo?: SystemInfo + env?: NodeJS.ProcessEnv + } = {} ): Harness { const events: Harness['events'] = [] const journal: Harness['journal'] = [] @@ -91,6 +96,7 @@ function harness( idleTickMs: options.idleTickMs ?? 50, ...(options.claimGpu ? { claimGpu: options.claimGpu } : {}), ...(options.systemInfo ? { systemInfo: async () => options.systemInfo as SystemInfo } : {}), + ...(options.env ? { env: options.env } : {}), }) service.start() services.push(service) @@ -390,6 +396,22 @@ describe.skipIf(!posix)('the engine and the model', () => { await h.service.unloadModel() }) + it('starts every load on the CPU backend under ATOMIC_DIFFUSION_BACKEND=cpu, and only then', async () => { + for (const [env, extraArgs] of [ + [{ ...process.env, ATOMIC_DIFFUSION_BACKEND: 'cpu' }, ['--backend', 'cpu']], + [{ ...process.env, ATOMIC_DIFFUSION_BACKEND: '' }, []], + ] as const) { + const h = harness({ env }) + await h.service.configure({ dataFolder }) + await installFakeSdEngine(layout, { tag: 'master-883-137f740', backendId: 'fake-cpu' }) + const loaded = await h.service.loadModel(await loadRequest()) + expect(h.service.state.spec?.extraArgs).toEqual(extraArgs) + // A chosen backend is not a recovery: the job record says nothing fell back. + expect(loaded.cpuFallback).toBe(false) + await h.service.unloadModel() + } + }) + it('reports a load that fails, and stops one that is abandoned', async () => { const h = harness() await h.service.configure({ dataFolder }) diff --git a/src/diffusion/service.ts b/src/diffusion/service.ts index a05de384..b1d8919f 100644 --- a/src/diffusion/service.ts +++ b/src/diffusion/service.ts @@ -36,7 +36,7 @@ import type { EngineHost } from '../engine-builds/service.js' import type { ImagesBackend, VideosBackend } from '../server/index.js' import { selectModelInstall } from './compat.js' import { samePath } from './containment.js' -import { VIDEO_VAE_TILING_PIXEL_FRAMES } from './args.js' +import { DIFFUSION_BACKEND_ENV, loadExtraArgs, VIDEO_VAE_TILING_PIXEL_FRAMES } from './args.js' import { diffusionError, errorBody, ioError, modelNotLoadedError } from './errors.js' import { Gallery } from './gallery.js' import { createSdHttpClient } from './http.js' @@ -143,6 +143,7 @@ export class DiffusionService { private readonly history: VideoHistory private readonly systemInfo: (() => Promise) | undefined private readonly platform: NodeJS.Platform + private readonly env: NodeJS.ProcessEnv private readonly dataFolder: string private readonly idleTickMs: number | undefined private idle: { stop(): void } | undefined @@ -166,6 +167,7 @@ export class DiffusionService { const http = options.http ?? createSdHttpClient() const log = options.log this.platform = platform + this.env = options.env ?? process.env this.dataFolder = options.dataFolder this.idleTickMs = options.idleTickMs this.state = new DiffusionState(options.paths, now) @@ -409,7 +411,7 @@ export class DiffusionService { ? { offloadFallback: request.offloadFallback } : {}), ...(request.threads !== undefined ? { threads: request.threads } : {}), - extraArgs: [], + extraArgs: loadExtraArgs(this.env[DIFFUSION_BACKEND_ENV]), startupTimeoutMs: (request.startupTimeoutSecs !== undefined && request.startupTimeoutSecs > 0 ? request.startupTimeoutSecs diff --git a/test/contract/engine-pin.test.ts b/test/contract/engine-pin.test.ts index 3cf153ca..7713b855 100644 --- a/test/contract/engine-pin.test.ts +++ b/test/contract/engine-pin.test.ts @@ -3,7 +3,8 @@ * the one this core takes today. It re-implements the TypeScript catalog in plain JS for the * scripts, so it is held here to the same answers as `upstreamRemoteBackends` — on the bundled * baseline and on lists with a withdrawn release, a release for newer cores, and candidates only — - * and, for PrismML, as `prismCatalogToBackends`. + * and, for PrismML, as `prismCatalogToBackends`; for sd.cpp, as `parseSdcppManifest` and the host + * ladder of `sdcppHostChoice`. */ import { describe, expect, it } from 'vitest' import { @@ -11,10 +12,15 @@ import { PIN_PLATFORMS, PRISM_PIN_PLATFORMS, parsePrismManifest as parsePrismMjs, + parseSdcppPinManifest, parseUpstreamReleases as parseMjs, prismCatalogToBackends as prismBackendsMjs, resolvePins, resolvePrismPins, + resolveSdcppPins, + SDCPP_PIN_PLATFORMS, + sdcppTagBuild, + sdcppTagParts, upstreamCatalogFromLegacyManifest as legacyMjs, upstreamRemoteBackends as remoteMjs, } from '../../scripts/lib/engine-pin.mjs' @@ -28,7 +34,10 @@ import { upstreamCatalogFromLegacyManifest, upstreamRemoteBackends, } from '../../src/backend/index.js' +import { assetInstallable, parseSdcppManifest, sdcppHostChoice } from '../../src/engine-builds/index.js' import baseline from '../../src/backend/catalog/upstream-releases-baseline.json' with { type: 'json' } +import sdcppLive from '../fixtures/sdcpp/conf/sdcpp-manifest.json' with { type: 'json' } +import sdcppStaging from '../fixtures/sdcpp/conf/sdcpp-manifest.staging.json' with { type: 'json' } import prismBaseline from '../../src/backend/catalog/prism-manifest-baseline.json' with { type: 'json' } const sha = (n: number) => n.toString(16).padStart(64, '0') @@ -282,3 +291,72 @@ describe('engine-pin.mjs reads the PrismML manifest as the core does', () => { expect(engineTagBuild('prism-b10787')).toBeNull() }) }) + +describe('engine-pin.mjs reads the sd.cpp manifest as the core does', () => { + const manifests: Record = { + 'conf live (master-883-137f740-a36f1b1a)': sdcppLive, + 'conf staging (master-883-137f740)': sdcppStaging, + 'a rebuild tag with its own rev': { ...sdcppLive, tag_name: 'master-975-95b3090-a0123456' }, + 'no sha256 on the CPU archives': { + ...sdcppLive, + assets: sdcppLive.assets.map(({ sha256: _sha256, ...a }) => a), + }, + 'an asset name with a path in it, and a backend listed twice': { + tag_name: 'master-883-137f740', + assets: [ + { backend: 'linux-cpu-x64', name: '../x.zip', sha256: sha(1), size: 1 }, + { backend: 'linux-cpu-x64', name: 'a.zip', sha256: sha(2), size: 2 }, + { backend: 'linux-cpu-x64', name: 'b.zip', sha256: sha(3), size: 3 }, + ], + }, + 'a tag without its upstream commit': { ...sdcppLive, tag_name: 'master-883' }, + 'a rev of eight characters': { ...sdcppLive, tag_name: 'master-883-137f740-a36f1b1aa' }, + 'no assets': { tag_name: 'master-883-137f740', assets: [] }, + 'not an object': 'master-883-137f740', + } + const ts = (raw: unknown) => { + try { + return parseSdcppManifest(raw) + } catch { + return null + } + } + + for (const [name, raw] of Object.entries(manifests)) + it(`${name}: the same document, the pin the core would install on each platform`, () => { + const core = ts(raw) + const mjs = parseSdcppPinManifest(raw) + expect(mjs).toEqual(core) + const pins = resolveSdcppPins({ manifest: raw, coreVersion: '0.18.6' }) + expect(pins.source).toBe(core ? 'sdcpp-manifest' : 'none') + for (const p of SDCPP_PIN_PLATFORMS) { + // A CPU-only host: the bottom rung of the ladder, which is the archive the check runs. + const choice = core + ? sdcppHostChoice({ os: p.os, arch: p.arch, features: {}, gpus: [] }, core, new Set()) + : null + const archive = core?.assets.find((a) => a.backend === choice?.backend_id) + expect(pins.per_platform[p.platform], p.platform).toEqual({ + backend: p.backend, + tag: archive && assetInstallable(archive) ? core?.tag_name : null, + }) + if (archive) expect(choice?.backend_id).toBe(p.backend) + } + }) + + it('splits a tag into its build, the upstream commit and tag, and the fork rev', () => { + expect(sdcppTagParts('master-883-137f740-a36f1b1a')).toEqual({ + build: 883, + sha7: '137f740', + upstream_tag: 'master-883-137f740', + atomic_rev: '36f1b1a', + }) + expect(sdcppTagParts('master-975-95b3090')).toMatchObject({ + upstream_tag: 'master-975-95b3090', + atomic_rev: null, + }) + for (const tag of ['master-883', 'master-883-137F740', 'b11617', 'master-883-137f740-atomic']) + expect(sdcppTagParts(tag), tag).toBeNull() + expect(sdcppTagBuild('master-883-137f740-a36f1b1a')).toBe(883) + expect(engineTagBuild('master-975-95b3090')).toBe(975) + }) +}) diff --git a/test/contract/engine-results.test.ts b/test/contract/engine-results.test.ts index 892fb2a9..5688c42c 100644 --- a/test/contract/engine-results.test.ts +++ b/test/contract/engine-results.test.ts @@ -11,9 +11,11 @@ import { buildResults, CANDIDATE_ID_RE, coreCell, + coreMatrix, mergeCoreShards, liveCell, renderResultsMarkdown, + requiredLiveTests, vitestOutcomes, } from '../../scripts/lib/engine-results.mjs' import type { @@ -401,6 +403,27 @@ describe('the llamacpp-upstream configuration', () => { expect(existsSync(new URL(path, root)), path).toBe(true) }) + it('runs the core suites on the five platforms, Windows in two shards, as the workflow always did', () => { + expect(coreMatrix(config.platforms)).toEqual([ + { platform: 'linux-x64', os: 'ubuntu-latest' }, + { platform: 'linux-arm64', os: 'ubuntu-24.04-arm' }, + { platform: 'macos-arm64', os: 'macos-14' }, + { platform: 'windows-x64', os: 'windows-2022', shard: '1/2', part: '-1' }, + { platform: 'windows-x64', os: 'windows-2022', shard: '2/2', part: '-2' }, + { platform: 'windows-arm64', os: 'windows-11-arm', shard: '1/2', part: '-1' }, + { platform: 'windows-arm64', os: 'windows-11-arm', shard: '2/2', part: '-2' }, + ]) + // Every required test applies on every platform unless it names its own. + for (const platform of PLATFORMS) + expect(requiredLiveTests(config.required_live_tests, platform)).toEqual(config.required_live_tests) + const only = [ + { file: 'a', name: 'everywhere' }, + { file: 'a', name: 'macOS only', platforms: ['macos-arm64'] }, + ] + expect(requiredLiveTests(only, 'linux-x64').map((t) => t.name)).toEqual(['everywhere']) + expect(requiredLiveTests(only, 'macos-arm64').map((t) => t.name)).toEqual(['everywhere', 'macOS only']) + }) + it('takes candidate ids an orchestrator can find in run-name', () => { expect(CANDIDATE_ID_RE.test('conf-b11617')).toBe(true) expect(CANDIDATE_ID_RE.test('hatchet:run.42_1')).toBe(true) diff --git a/test/contract/engine-surface.test.ts b/test/contract/engine-surface.test.ts index d90e7bfd..d9ffd3c2 100644 --- a/test/contract/engine-surface.test.ts +++ b/test/contract/engine-surface.test.ts @@ -17,6 +17,7 @@ import { checkStrings, checkVersion, countStrings, + invocationKey, MIRROR_WHITELIST, parseBinaryVersion, parseGitLog, @@ -230,6 +231,15 @@ describe('--version and the invocations', () => { expect(ids(findings, 'break')).toEqual(['invocations.exit/--version']) expect(ids(findings, 'error')).toEqual(['invocations.exit/--list-devices']) }) + + it('names an invocation by its executable only when it runs another than the server', () => { + expect(invocationKey({ argv: ['-h'] })).toBe('-h') + expect(invocationKey({ bin: 'sd-cli', argv: ['--help'] })).toBe('sd-cli --help') + // An sd-cli run never stands in for the server's own -h. + expect(ids(checkInvocations(contract, [{ bin: 'sd-cli', argv: ['-h'], status: 0 }]), 'error')).toContain( + 'invocations.exit/-h' + ) + }) }) describe('H2: the release notes between the pin and the candidate', () => { diff --git a/test/contract/live-fixtures.test.ts b/test/contract/live-fixtures.test.ts index d2e5072b..eeac7f0f 100644 --- a/test/contract/live-fixtures.test.ts +++ b/test/contract/live-fixtures.test.ts @@ -1,10 +1,17 @@ /** * The release-picking rules of `scripts/fetch-live-fixtures.mjs`, which decide what llama.cpp build * the live suites run: the platform's CPU asset, the newest release that has it (ggml-org's builds - * are prereleases and its `latest` carries no binaries), and a digest check on the bytes. + * are prereleases and its `latest` carries no binaries), and a digest check on the bytes; and for an + * engine with several model files (sd.cpp), the live env its config describes and the models' cache key. */ import { describe, expect, it } from 'vitest' -import { assetPatternFor, digestMatches, pickRelease } from '../../scripts/lib/live-fixtures.mjs' +import { + assetPatternFor, + digestMatches, + expandLiveEnv, + liveModelsKey, + pickRelease, +} from '../../scripts/lib/live-fixtures.mjs' const names = (tag: string) => [ `llama-${tag}-bin-macos-arm64.tar.gz`, @@ -99,3 +106,28 @@ describe('digestMatches', () => { expect(digestMatches(`md5:${hex}`, hex)).toBeUndefined() }) }) + +describe('the live env of an engine with several files', () => { + const values = { engine_dir: '/live/engine', tag: 'master-883-137f740', models: { llm: '/m/llm.gguf' } } + + it('fills the placeholders from what the fetch left, and keeps literal text', () => { + expect( + expandLiveEnv({ E: '{engine_dir}', T: 'tag={tag}', L: '{model:llm}', F: 'flux2', B: 'cpu' }, values) + ).toEqual({ E: '/live/engine', T: 'tag=master-883-137f740', L: '/m/llm.gguf', F: 'flux2', B: 'cpu' }) + }) + + it('throws on a placeholder the fetch did not fill rather than start a half-set-up run', () => { + expect(() => expandLiveEnv({ M: '{model:vae}' }, values)).toThrow('{model:vae}') + expect(() => expandLiveEnv({ M: '{manifest}' }, values)).toThrow('{manifest}') + }) + + it('keys the model cache by the models, whatever their order, and has no key without models', () => { + const a = { sha256: 'a'.repeat(64) } + const b = { sha256: 'b'.repeat(64) } + expect(liveModelsKey([a, b])).toBe(liveModelsKey([b, a])) + expect(liveModelsKey([a, b])).not.toBe(liveModelsKey([a])) + expect(liveModelsKey([a])).toMatch(/^[0-9a-f]{16}$/) + expect(liveModelsKey(undefined)).toBe('') + expect(liveModelsKey([])).toBe('') + }) +}) diff --git a/test/contract/sdcpp-engine-contract.test.ts b/test/contract/sdcpp-engine-contract.test.ts new file mode 100644 index 00000000..cd154af3 --- /dev/null +++ b/test/contract/sdcpp-engine-contract.test.ts @@ -0,0 +1,230 @@ +/** + * The stable-diffusion.cpp engine contract (test/fixtures/sdcpp/engine-contract.json) is written by + * hand, so each part is held here to the code it describes: the flags to `buildServerArgs` and + * required-flags.txt, the invocations and markers to the install probe, the routes to the HTTP + * client's paths, the strings to the output parsers, and the archives to conf's release names and + * the backend ladder of src/engine-builds/select.ts. A change on either side fails here until the + * other follows. + */ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { + CAPABILITIES_PATH, + classifyExit, + cliBinaryName, + GpuFaultWatch, + IMG_GEN_PATH, + isGgmlUnsupportedOpAbort, + isProgressRedraw, + JOBS_PATH, + parseStepLine, + parseTileAnnouncement, + probeOutputIsSdcpp, + READY_PATH, + serverBinaryName, + VID_GEN_PATH, +} from '../../src/diffusion/index.js' +import { diffusionBackendLadder } from '../../src/engine-builds/index.js' +import type { DiffusionBackendSelectionInput } from '../../src/engine-builds/index.js' +import { parseSdVersion, sdcppBackendArchives } from '../../scripts/lib/sdcpp-surface.mjs' +import type { SdcppContract } from '../../scripts/lib/sdcpp-surface.mjs' + +interface Contract { + engine: string + invocations: Array<{ bin: string; argv: string[]; source: string; severity: string; optional?: boolean }> + probe: { bins: string[]; argv: string[]; markers: string[] } + version: { pattern: string } + flags: Array<{ flag: string; severity: string; source: string }> + strings: Array<{ id: string; text: string; channel: string; upstream_text: string | null }> + assets: { + core_backends: Array<{ + platform: string + backend: string + cpu: boolean + required: boolean + companion?: boolean + asset: string + }> + } + upstream: { repo: string; watched_paths: Array<{ path: string }> } +} + +const fixture = (path: string) => readFileSync(new URL(`../fixtures/sdcpp/${path}`, import.meta.url), 'utf8') +const contract = JSON.parse(fixture('engine-contract.json')) as Contract & SdcppContract +const release = (tag: string) => + JSON.parse(fixture(`releases/${tag}.json`)) as { tag: string; assets: Array<{ name: string }> } + +describe('the sd.cpp engine contract: flags', () => { + const requiredFlags = fixture('required-flags.txt') + .split('\n') + .map((line) => line.trim()) + .filter((line) => line !== '' && !line.startsWith('#')) + + it('breaks on exactly the flags of required-flags.txt and their -v, and only warns about -M', () => { + const breaks = contract.flags.filter((f) => f.severity === 'break').map((f) => f.flag) + expect(new Set(breaks)).toEqual(new Set([...requiredFlags, '-v'])) + expect(breaks).toHaveLength(requiredFlags.length + 1) + expect(contract.flags.filter((f) => f.severity !== 'break')).toEqual([ + expect.objectContaining({ flag: '-M', severity: 'warn' }), + ]) + }) + + it('knows every flag literal of the argv builder', () => { + const source = readFileSync('src/diffusion/args.ts', 'utf8') + const universe = new Set(contract.flags.map((f) => f.flag)) + const literals = [...source.matchAll(/'(--?[A-Za-z][\w-]*)'/g)].map((m) => m[1] as string) + // commandSummaryForLog reads these from an argv someone else built; buildServerArgs never emits them. + const readOnly = new Set(['--width', '--height', '--steps', '--seed']) + expect(literals.length).toBeGreaterThan(20) + expect(literals.filter((flag) => !universe.has(flag) && !readOnly.has(flag))).toEqual([]) + }) +}) + +describe('the sd.cpp engine contract: invocations and the install probe', () => { + it('runs the binaries the install probe looks for, by their names on a POSIX host', () => { + expect(contract.probe.bins).toEqual([cliBinaryName('linux'), serverBinaryName('linux')]) + expect(contract.invocations.map((i) => `${i.bin} ${i.argv.join(' ')}`)).toEqual([ + 'sd-server --help', + 'sd-server --version', + 'sd-cli --help', + ]) + // The probe reads markers, never the exit code: no invocation may fail the check on its exit. + expect(contract.invocations.every((i) => i.severity === 'warn')).toBe(true) + expect(contract.invocations.find((i) => i.bin === 'sd-cli')?.optional).toBe(true) + }) + + it('names markers each of which alone passes the probe, as the real help screens hold them', () => { + for (const marker of contract.probe.markers) + expect(probeOutputIsSdcpp(`x ${marker} y`), marker).toBe(true) + expect(probeOutputIsSdcpp('usage: llama-server [options]')).toBe(false) + for (const bin of ['sd-server', 'sd-cli']) { + const help = fixture(`help/${bin}-master-883-137f740.txt`) + expect(probeOutputIsSdcpp(help), bin).toBe(true) + } + }) + + it('reads the version line the fork’s builds print', () => { + expect(parseSdVersion(contract, fixture('help/sd-server-version-master-883-137f740.txt'))).toEqual({ + version: 'unknown', + commit: '137f740', + }) + expect( + parseSdVersion(contract, 'stable-diffusion.cpp version master-975-95b3090, commit 95b3090') + ).toEqual({ version: 'master-975-95b3090', commit: '95b3090' }) + }) +}) + +describe('the sd.cpp engine contract: routes and strings', () => { + const byId = (id: string) => contract.strings.find((s) => s.id === id) + + it('names the routes the HTTP client calls', () => { + const routes = Object.fromEntries( + contract.strings.filter((s) => s.channel === 'http-route').map((s) => [s.id, s.text]) + ) + expect(routes).toEqual({ + 'route.ready': READY_PATH, + 'route.capabilities': CAPABILITIES_PATH, + 'route.img_gen': IMG_GEN_PATH, + 'route.vid_gen': VID_GEN_PATH, + 'route.job': `${JOBS_PATH}/{id}`, + 'route.cancel': `${JOBS_PATH}/{id}/cancel`, + }) + }) + + it('lists the progress text the parsers read', () => { + expect(isProgressRedraw(`|====> | 3/9 - 1.20${byId('progress.rate.s-it')?.text}`)).toBe(true) + expect(isProgressRedraw(`|====> | 3/9 - 1.20${byId('progress.rate.it-s')?.text}`)).toBe(true) + expect(isProgressRedraw(`|====> | 3/9 - 637.50M${byId('progress.rate.bytes')?.text}`)).toBe(true) + expect(parseStepLine('|====> | 12/28 - 3.52s/it')).toEqual([12, 28]) + expect(parseTileAnnouncement(byId('progress.tiles')?.text.replace('', '9') ?? '')).toBe(9) + }) + + it('lists every out-of-memory text classifyExit reads, and only those', () => { + const oom = contract.strings.filter((s) => s.id.startsWith('oom.')) + expect(oom.length).toBe(7) + for (const s of oom) expect(classifyExit(`... ${s.text} ...`, 1), s.id).toBe('OUT_OF_MEMORY') + expect(classifyExit('segmentation fault', 139)).toBe('ENGINE_CRASHED') + }) + + it('lists the abort signature and the GPU faults the recovery and the watch read', () => { + const text = (prefix: string) => + contract.strings + .filter((s) => s.id.startsWith(prefix)) + .map((s) => s.text) + .join('\n') + expect(isGgmlUnsupportedOpAbort(text('abort.'))).toBe(true) + for (const id of ['gpu-fault.address', 'gpu-fault.error-state']) { + const watch = new GpuFaultWatch() + watch.onLine(byId(id)?.text ?? '') + expect(watch.tripped, id).toBe(true) + } + const watch = new GpuFaultWatch() + for (const id of ['gpu-fault.command-buffer', 'gpu-fault.pagefault']) watch.onLine(byId(id)?.text ?? '') + expect(watch.tripped).toBe(true) + }) + + it('searches leejet’s sources only for sd.cpp’s own text, in files it watches', () => { + const own = contract.strings.filter((s) => s.upstream_text !== null).map((s) => s.id) + expect(own.every((id) => /^(route|progress|oom\.model-manager)/.test(id))).toBe(true) + const watched = contract.upstream.watched_paths.map((w) => w.path) + for (const path of [ + 'examples/server/routes_sdcpp.cpp', + 'examples/server/routes_openai.cpp', + 'src/core/util.cpp', + 'src/runtime/tiling.cpp', + 'src/pipeline/image.cpp', + 'src/model_manager.cpp', + ]) + expect(watched).toContain(path) + }) +}) + +describe('the sd.cpp engine contract: archives', () => { + const backends = contract.assets.core_backends + + it.each(['master-883-137f740', 'master-883-137f740-a36f1b1a'])( + '%s: each backend takes exactly one archive of the release', + (tag) => { + const archives = sdcppBackendArchives( + contract, + tag, + release(tag).assets.map((a) => a.name) + ) + for (const b of backends) expect(archives[b.backend], b.backend).toHaveLength(1) + } + ) + + it('lists every backend the host ladder can pick from a release, and nothing it never picks', () => { + const available = backends.map((b) => b.backend) + const hosts: Array> = [] + for (const os of ['macos', 'windows', 'linux'] as const) + for (const arch of ['arm64', 'x64'] as const) + for (const features of [{}, { cuda12: true, cuda13: true, vulkan: true, rocm: true }]) + hosts.push({ os, arch, features, gpus: [{ totalMemoryMib: 8192 }] }) + const picked = new Set(hosts.flatMap((host) => diffusionBackendLadder({ ...host, available }))) + const companions = backends.filter((b) => b.companion).map((b) => b.backend) + expect([...picked, ...companions].sort()).toEqual([...available].sort()) + }) + + it('marks one CPU archive per platform, the bottom of its ladder', () => { + const cpu = backends.filter((b) => b.cpu) + expect(cpu.map((b) => b.platform).sort()).toEqual([ + 'linux-arm64', + 'linux-x64', + 'macos-arm64', + 'windows-arm64', + 'windows-x64', + ]) + for (const b of cpu) { + const [os, arch] = b.platform.split('-') as ['macos' | 'windows' | 'linux', 'arm64' | 'x64'] + const ladder = diffusionBackendLadder({ + os, + arch, + features: {}, + gpus: [], + available: backends.map((x) => x.backend), + }) + expect(ladder.at(-1), b.platform).toBe(b.backend) + } + }) +}) diff --git a/test/contract/sdcpp-surface.test.ts b/test/contract/sdcpp-surface.test.ts new file mode 100644 index 00000000..5e8cd034 --- /dev/null +++ b/test/contract/sdcpp-surface.test.ts @@ -0,0 +1,439 @@ +/** + * The stable-diffusion.cpp side of the engine candidate check: the surface diff of a build + * (scripts/lib/sdcpp-surface.mjs) on the real `--help` and archives of master-883-137f740 and on + * synthetic changes of each kind it has to catch, what `resolve` takes for a complete release (the + * fork's, and conf's signed mirror with its staging manifest), the manifest the install test points + * the core at (scripts/lib/sdcpp-fixtures.mjs), and the configuration + * (`.github/engine-candidate/sd-cpp.json`) the workflow and `npm run engine:check` read. + */ +import { existsSync, readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + buildSdcppReport, + checkSdAssets, + checkSdHelp, + checkSdInvocations, + checkSdProbe, + checkSdVersion, + diffSdHelp, + findSdOption, + parseSdHelp, + sdcppCompleteness, + sdcppCoreAssets, +} from '../../scripts/lib/sdcpp-surface.mjs' +import type { ReleaseAssetInfo, SdcppContract } from '../../scripts/lib/sdcpp-surface.mjs' +import { liveModelPath, sdcppHostArchive, sdcppInstallManifest } from '../../scripts/lib/sdcpp-fixtures.mjs' +import { coreMatrix, requiredLiveTests } from '../../scripts/lib/engine-results.mjs' +import type { EngineCandidateConfig } from '../../scripts/lib/engine-results.mjs' +import { expandLiveEnv, liveModelsKey } from '../../scripts/lib/live-fixtures.mjs' +import { SDCPP_PIN_PLATFORMS } from '../../scripts/lib/engine-pin.mjs' +import { parseSdcppManifest, sdcppAssetUrl, sdcppHostChoice } from '../../src/engine-builds/index.js' + +const root = new URL('../../', import.meta.url) +const fixture = (path: string) => readFileSync(new URL(`../fixtures/sdcpp/${path}`, import.meta.url), 'utf8') +const contract = JSON.parse(fixture('engine-contract.json')) as SdcppContract +const release = (tag: string) => + JSON.parse(fixture(`releases/${tag}.json`)) as { tag: string; assets: Array> } +const FORK = release('master-883-137f740') +const MIRROR = release('master-883-137f740-a36f1b1a') +const names = (r: { assets: Array<{ name: string }> }) => r.assets.map((a) => a.name) +const SERVER_HELP = fixture('help/sd-server-master-883-137f740.txt') +const CLI_HELP = fixture('help/sd-cli-master-883-137f740.txt') +const VERSION = fixture('help/sd-server-version-master-883-137f740.txt') +const ids = (findings: Array<{ id: string; severity: string }>, severity: string) => + findings.filter((f) => f.severity === severity).map((f) => f.id) +/** `--help` without the option lines that spell `flag`. */ +const without = (help: string, flag: string) => + help + .split('\n') + .filter((line) => !new RegExp(`^ +(-\\S+, )*${flag.replace(/[-]/g, '\\-')}[ ,]`).test(line)) + .join('\n') + +describe('reading sd.cpp’s --help', () => { + const options = parseSdHelp(SERVER_HELP) + + it('reads every option of a real screen, by its exact aliases', () => { + expect(options.length).toBeGreaterThan(150) + expect(findSdOption(options, '-v')?.aliases).toEqual(['-v', '--verbose']) + expect(findSdOption(options, '--vae')?.hint).toBe('') + expect(findSdOption(options, '--vae-format')?.help).toMatch(/^VAE latent format override/) + // Help wraps onto the deeper lines that follow. + expect(findSdOption(options, '--llm')?.help).toContain('mistral-small3.2 for flux2') + expect(findSdOption(options, '--qwen2vl')?.deprecated).toBe(true) + expect(findSdOption(options, '-M')).toBeUndefined() + expect(findSdOption(parseSdHelp(CLI_HELP), '-M')?.aliases).toEqual(['-M', '--mode']) + }) + + it('takes a flag core emits as missing when only a longer one starts with it', () => { + const help = without(SERVER_HELP, '--vae') + expect(help).toContain('--vae-format') + expect(findSdOption(parseSdHelp(help), '--vae')).toBeUndefined() + expect(ids(checkSdHelp(contract, parseSdHelp(help)), 'break')).toEqual(['help.required_flags/--vae']) + }) + + it('on master-883: three flags core emits are deprecated, -M is sd-cli’s only', () => { + const findings = checkSdHelp(contract, options) + expect(ids(findings, 'break')).toEqual([]) + expect(ids(findings, 'warn').sort()).toEqual([ + 'help.deprecated/--clip-on-cpu', + 'help.deprecated/--qwen2vl', + 'help.deprecated/--vae-on-cpu', + 'help.required_flags/-M', + ]) + }) + + it('fails closed on something that is not a help screen', () => { + expect(ids(checkSdHelp(contract, parseSdHelp('error: unknown option')), 'error')).toEqual(['help.parse']) + }) + + it('lists what changed since the pin among the flags core never emits', () => { + const pin = parseSdHelp(without(SERVER_HELP, '--tokenizer')) + const candidate = parseSdHelp(without(without(SERVER_HELP, '--rpc-servers'), '--vae')) + const findings = diffSdHelp(pin, candidate, contract) + expect(findings.map((f) => [f.id, f.evidence])).toEqual([ + ['help.diff/removed', { flags: ['--rpc-servers'] }], + ['help.diff/added', { flags: ['--tokenizer'] }], + ]) + }) +}) + +describe('the invocations, the install probe and --version', () => { + const ran = (over: Record = {}) => [ + { bin: 'sd-server', argv: ['--help'], status: 0 }, + { bin: 'sd-server', argv: ['--version'], status: 0 }, + { bin: 'sd-cli', argv: ['--help'], status: 0, ...over }, + ] + + it('is quiet on a build whose binaries all answer', () => { + expect(checkSdInvocations(contract, ran())).toEqual([]) + expect(checkSdProbe(contract, { 'sd-cli': CLI_HELP, 'sd-server': SERVER_HELP })).toEqual([]) + }) + + it('takes an archive without sd-cli as probed through sd-server, and one without sd-server as broken', () => { + const noCli = ran({ status: null, error: 'spawnSync /x/sd-cli ENOENT' }) + expect(checkSdInvocations(contract, noCli).map((f) => [f.id, f.severity])).toEqual([ + ['invocations.exit/sd-cli --help', 'info'], + ]) + expect(checkSdProbe(contract, { 'sd-cli': undefined, 'sd-server': SERVER_HELP })).toEqual([]) + const noServer = ran().map((r) => + r.bin === 'sd-server' ? { ...r, status: null, error: 'spawnSync /x/sd-server ENOENT' } : r + ) + expect(ids(checkSdInvocations(contract, noServer), 'break')).toEqual([ + 'invocations.exit/sd-server --help', + 'invocations.exit/sd-server --version', + ]) + }) + + it('warns about an exit core does not read, and breaks when the probe binary shows no marker', () => { + expect(ids(checkSdInvocations(contract, ran({ status: 1, output: 'boom' })), 'warn')).toEqual([ + 'invocations.exit/sd-cli --help', + ]) + // sd-cli is there, so core probes it, whatever sd-server prints. + expect( + ids(checkSdProbe(contract, { 'sd-cli': 'usage: other-tool', 'sd-server': SERVER_HELP }), 'break') + ).toEqual(['install.probe/sd-cli']) + }) + + it('ties --version to the tag’s commit', () => { + expect(checkSdVersion(contract, { tag: 'master-883-137f740-a36f1b1a', output: VERSION })).toEqual([ + expect.objectContaining({ id: 'version.output/version', severity: 'info' }), + ]) + expect(ids(checkSdVersion(contract, { tag: 'master-975-95b3090', output: VERSION }), 'break')).toEqual([ + 'version.output/commit', + ]) + expect( + checkSdVersion(contract, { + tag: 'master-975-95b3090', + output: 'stable-diffusion.cpp version master-975-95b3090+, commit 95b3090', + }) + ).toEqual([]) + expect(ids(checkSdVersion(contract, { tag: 'master-975-95b3090', output: 'v1.0' }), 'warn')).toEqual([ + 'version.output', + ]) + }) +}) + +describe('the archives', () => { + const check = (candidate: string[], pin: string[] | null = names(MIRROR)) => + checkSdAssets(contract, { + candidate: { tag: 'master-975-95b3090', assets: candidate }, + pin: pin && { tag: MIRROR.tag, assets: pin }, + }) + /** The 883 fork release renamed to the 975 commit. */ + const fresh = names(FORK).map((n) => n.replace('137f740', '95b3090')) + + it('master-883 against its signed mirror: nothing lost, the archives no backend takes listed once', () => { + const findings = checkSdAssets(contract, { + candidate: { tag: FORK.tag, assets: names(FORK) }, + pin: { tag: MIRROR.tag, assets: names(MIRROR) }, + }) + expect(findings.map((f) => [f.id, f.severity])).toEqual([['assets.core_backends/unconsumed', 'info']]) + expect(sdcppCoreAssets(contract, FORK.tag, names(FORK))).toHaveLength(11) + }) + + it('breaks on a lost CPU or mirror-required archive, warns on a lost optional one the pin had', () => { + const drop = (part: string) => fresh.filter((n) => !n.includes(part)) + expect(ids(check(drop('aarch64.zip')), 'break')).toEqual([ + 'assets.core_backends/linux-arm64/linux-cpu-arm64', + ]) + expect(ids(check(drop('win-vulkan-x64')), 'break')).toEqual([ + 'assets.core_backends/windows-x64/win-vulkan-x64', + ]) + expect(ids(check(drop('aarch64-cuda13')), 'warn')).toEqual([ + 'assets.core_backends/linux-arm64/linux-cuda13-arm64', + ]) + expect(ids(check(drop('aarch64-cuda13'), drop('aarch64-cuda13')), 'warn')).toEqual([]) + }) + + it('breaks on two archives for one backend, and reports a moved one', () => { + const twoMacs = [...fresh, 'sd-master-95b3090-bin-Darwin-macOS-27.0-arm64.zip'] + expect(ids(check(twoMacs), 'break')).toEqual(['assets.core_backends/macos-arm64/macos-arm64']) + const moved = fresh.map((n) => n.replace('macOS-26.6.2', 'macOS-27.0')) + expect(check(moved).find((f) => f.id === 'assets.core_backends/macos-arm64/macos-arm64')?.message).toBe( + 'macos-arm64 moved: Darwin-macOS-26.6.2-arm64.zip → Darwin-macOS-27.0-arm64.zip' + ) + }) +}) + +describe('a complete release', () => { + const LIVE = ['linux-x64', 'linux-arm64', 'macos-arm64'] + const BASE = 'https://github.com/AtomicBot-ai/atomic-chat-conf/releases/download' + const live = JSON.parse(fixture('conf/sdcpp-manifest.json')) as { + tag_name: string + assets: Array> + } + + it('on the fork: every archive the mirror and the live platforms need, up with its digest', () => { + expect(sdcppCompleteness({ contract, tag: FORK.tag, assets: FORK.assets, platforms: LIVE })).toEqual({ + complete: true, + missing: [], + }) + const uploading = FORK.assets.map((a) => (a.name.endsWith('aarch64.zip') ? { ...a, digest: null } : a)) + expect( + sdcppCompleteness({ contract, tag: FORK.tag, assets: uploading, platforms: LIVE }).missing + ).toEqual([ + { + kind: 'digest', + backend: 'linux-cpu-arm64', + name: 'sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64.zip', + }, + ]) + // linux-cpu-arm64 is optional for the mirror: without a live run on arm64 Linux it is not waited for. + const noArm = FORK.assets.filter((a) => !a.name.endsWith('aarch64.zip')) + expect( + sdcppCompleteness({ contract, tag: FORK.tag, assets: noArm, platforms: ['linux-x64'] }).complete + ).toBe(true) + expect(sdcppCompleteness({ contract, tag: FORK.tag, assets: noArm, platforms: LIVE }).missing).toEqual([ + { kind: 'core', platform: 'linux-arm64', backend: 'linux-cpu-arm64' }, + ]) + }) + + it('on conf: the manifest names the tag and pins exactly the bytes of the release', () => { + const complete = (manifest: unknown, assets = MIRROR.assets) => + sdcppCompleteness({ contract, tag: MIRROR.tag, assets, platforms: LIVE, manifest, downloadBase: BASE }) + expect(complete(live)).toEqual({ complete: true, missing: [] }) + expect(complete({ ...live, tag_name: 'master-975-95b3090-a0123456' }).missing).toEqual([ + { kind: 'manifest', reason: 'the staging manifest names master-975-95b3090-a0123456' }, + ]) + expect(complete(null).missing).toEqual([ + { kind: 'manifest', reason: 'conf has no staging manifest core reads' }, + ]) + expect(complete({ ...live, download_base: 'https://example.com/x' }).missing).toEqual([ + expect.objectContaining({ + kind: 'manifest', + reason: expect.stringContaining('downloads from https://example.com/x'), + }), + ]) + const resigned = MIRROR.assets.map((a) => + a.name.includes('Darwin') ? { ...a, size: a.size + 1, digest: `sha256:${'0'.repeat(64)}` } : a + ) + expect(complete(live, resigned).missing).toEqual([ + { + kind: 'manifest-asset', + backend: 'macos-arm64', + name: 'sd-master-137f740-bin-Darwin-macOS-26.6.2-arm64.zip', + reason: 'size 34338632 in the release, 34338631 in the manifest', + }, + ]) + }) +}) + +describe('the manifest the install test installs from', () => { + it('is one the core reads, downloading each archive from the release with its size and sha256', () => { + for (const r of [FORK, MIRROR]) { + const repo = r === FORK ? 'AtomicBot-ai/stable-diffusion.cpp' : 'AtomicBot-ai/atomic-chat-conf' + const manifest = parseSdcppManifest( + sdcppInstallManifest({ contract, tag: r.tag, repo, assets: r.assets }) + ) + expect(manifest.tag_name).toBe(r.tag) + expect(manifest.assets).toHaveLength(11) + for (const p of SDCPP_PIN_PLATFORMS) { + const choice = sdcppHostChoice( + { os: p.os, arch: p.arch, features: {}, gpus: [] }, + manifest, + new Set() + ) + const asset = manifest.assets.find((a) => a.backend === choice.backend_id) + expect(choice.backend_id, p.platform).toBe(p.backend) + const source = r.assets.find((a) => a.name === asset?.name) + expect(asset).toMatchObject({ size: source?.size, sha256: source?.digest?.slice('sha256:'.length) }) + expect(sdcppAssetUrl(manifest, asset as NonNullable)).toBe( + `https://github.com/${repo}/releases/download/${r.tag}/${asset?.name}` + ) + } + expect(manifest.assets.find((a) => a.backend === 'win-cudart-cu12')?.companion).toBe(true) + } + }) + + it('takes this host’s CPU archive for the live run', () => { + expect(sdcppHostArchive(contract, FORK.tag, FORK.assets, 'linux-arm64').asset.name).toBe( + 'sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64.zip' + ) + expect(() => sdcppHostArchive(contract, FORK.tag, [], 'macos-arm64')).toThrow(/0 macos-arm64 archives/) + expect(() => sdcppHostArchive(contract, FORK.tag, FORK.assets, 'macos-x64')).toThrow(/no CPU build/) + }) +}) + +describe('the sd-cpp configuration', () => { + const config = JSON.parse( + readFileSync(new URL('.github/engine-candidate/sd-cpp.json', root), 'utf8') + ) as EngineCandidateConfig & { + tag_pattern: string + live_models: Array<{ id: string; url: string; sha256: string }> + live_env: Record + } + + it('reads the tags conf’s manifest takes, the fork’s and the signed mirror’s', () => { + const re = new RegExp(config.tag_pattern) + for (const tag of ['master-883-137f740', 'master-883-137f740-a36f1b1a']) { + expect(re.test(tag), tag).toBe(true) + expect(() => + parseSdcppManifest({ ...JSON.parse(fixture('conf/sdcpp-manifest.json')), tag_name: tag }) + ).not.toThrow() + } + for (const tag of ['b11617', 'master-883', 'master-883-137f740-atomic']) + expect(re.test(tag), tag).toBe(false) + }) + + it('names a required live test only if the live file has it', () => { + for (const { file, name } of config.required_live_tests) { + const text = readFileSync(new URL(file, root), 'utf8') + const suites = [...text.matchAll(/describe(?:\.skipIf\([^)]*\))?\(\s*'([^']+)'/g)].map((m) => m[1]) + const tests = [...text.matchAll(/\bit\(\s*'([^']+)'/g)].map((m) => m[1]) + expect( + suites.flatMap((s) => tests.map((t) => `${s} ${t}`)), + `${file} has no test named "${name}"` + ).toContain(name) + expect(config.live_files).toContain(file) + } + }) + + it('runs live on platforms core ships sd.cpp for, the core suites on the same platforms', () => { + const shipped = SDCPP_PIN_PLATFORMS.map((p) => p.platform) + for (const { platform } of config.platforms) expect(shipped).toContain(platform) + expect(coreMatrix(config.platforms)).toEqual([ + { platform: 'linux-x64', os: 'ubuntu-latest' }, + { platform: 'linux-arm64', os: 'ubuntu-24.04-arm' }, + { platform: 'macos-arm64', os: 'macos-14' }, + ]) + for (const { platform } of config.platforms) + expect(requiredLiveTests(config.required_live_tests, platform)).toEqual(config.required_live_tests) + }) + + it('fills its live env from what the fetch leaves, and keys the model cache by the models', () => { + const models = Object.fromEntries(config.live_models.map((m) => [m.id, liveModelPath('/models', m)])) + const env = expandLiveEnv(config.live_env, { engine_dir: '/e', tag: 't', manifest: '/m.json', models }) + expect(env).toMatchObject({ + ATOMIC_LIVE_SD_ENGINE: '/e', + // The host's separators: the fetch writes these paths for the host it runs on. + ATOMIC_LIVE_SD_MODEL: join('/models', '0b25d143c8469b34', 'flux-2-klein-4b-Q4_K_M.gguf'), + ATOMIC_LIVE_SD_VAE_FORMAT: 'flux2', + ATOMIC_LIVE_SD_BACKEND: 'cpu', + }) + expect(liveModelsKey(config.live_models)).toMatch(/^[0-9a-f]{16}$/) + }) + + it('protects paths that exist', () => { + for (const path of [...config.protected_paths, ...config.live_files, config.contract]) { + if (!path.endsWith('*')) { + expect(existsSync(new URL(path, root)), path).toBe(true) + continue + } + const dir = path.slice(0, path.lastIndexOf('/') + 1) + const prefix = path.slice(dir.length, -1) + expect( + readdirSync(new URL(dir, root)).some((name) => name.startsWith(prefix)), + path + ).toBe(true) + } + }) +}) + +describe('the sd.cpp surface report', () => { + const side = (tag: string, repo: string, assets: string[]) => ({ + tag, + repo, + source: 'release' as const, + platform: 'linux-x64', + help_text: SERVER_HELP, + cli_help_text: CLI_HELP, + version_output: VERSION, + invocations: [ + { bin: 'sd-server', argv: ['--help'], status: 0 }, + { bin: 'sd-server', argv: ['--version'], status: 0 }, + { bin: 'sd-cli', argv: ['--help'], status: 0 }, + ], + assets, + upstream: { + tag: 'master-883-137f740', + commit: '137f7409bbfb98c70a350a57d6a135487080db96', + published_at: null, + }, + }) + + it('master-883 against its signed mirror: a warn for the deprecated flags, nothing else to fix', () => { + const report = buildSdcppReport( + contract, + { + candidate: side(FORK.tag, 'AtomicBot-ai/stable-diffusion.cpp', names(FORK)), + pin: side(MIRROR.tag, 'AtomicBot-ai/atomic-chat-conf', names(MIRROR)), + upstream: { available: false, reason: 'not compared' }, + }, + { now: '2026-10-21T00:00:00Z' } + ) + expect(report.schema).toBe('atomic-engine-surface-report') + expect(report.engine).toBe('sd-cpp') + expect(report.status).toBe('warn') + expect(report.counts).toEqual({ break: 0, warn: 4, info: 5, error: 0 }) + expect(report.checks.map((c) => `${c.id}:${c.status}`)).toEqual([ + 'help.parse:pass', + 'help.required_flags:warn', + 'help.deprecated:warn', + 'help.diff:pass', + 'invocations.exit:pass', + 'install.probe:pass', + 'version.output:pass', + 'assets.core_backends:pass', + 'upstream.watched_paths:pass', + 'upstream.contract_strings:pass', + 'upstream.release_notes:pass', + ]) + expect(report.inputs.candidate).toMatchObject({ tag: FORK.tag, upstream: { tag: 'master-883-137f740' } }) + expect(report.inputs.same_as_pin).toBe(false) + }) + + it('is an error, not a pass, when the candidate could not be run', () => { + const report = buildSdcppReport(contract, { + candidate: { tag: FORK.tag, source: 'release', capture_error: 'fetch failed', assets: names(FORK) }, + pin: null, + }) + expect(report.status).toBe('error') + expect(ids(report.findings, 'error')).toEqual([ + 'help.parse', + 'install.probe', + 'invocations.exit', + 'version.output', + ]) + }) +}) diff --git a/test/fixtures/sdcpp/conf/sdcpp-manifest.json b/test/fixtures/sdcpp/conf/sdcpp-manifest.json new file mode 100644 index 00000000..951d2637 --- /dev/null +++ b/test/fixtures/sdcpp/conf/sdcpp-manifest.json @@ -0,0 +1,94 @@ +{ + "$schema": "./sdcpp-schema.json", + "updated_at": "2026-10-07T10:57:22Z", + "upstream_repo": "AtomicBot-ai/stable-diffusion.cpp", + "tag_name": "master-883-137f740-a36f1b1a", + "download_base": "https://github.com/AtomicBot-ai/atomic-chat-conf/releases/download", + "assets": [ + { + "backend": "linux-cpu-arm64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64.zip", + "sha256": "2fbfb1ef7c758ceedcf1ac51cd8f80107c076c8a13160603fd7a65ac2f215efe", + "size": 20947364 + }, + { + "backend": "linux-cpu-x64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64.zip", + "sha256": "dc1af0b244302cd083d69d974424ba5f2dbe8f1b7f3b04be3db8103477dae47c", + "size": 24520937 + }, + { + "backend": "linux-cuda12-x64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-cuda12.zip", + "sha256": "d3dbc188ebab9e46485d3eec878612d2b62b49e799386124cddf5ad2be3b8fdd", + "size": 718645269 + }, + { + "backend": "linux-cuda13-arm64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64-cuda13.zip", + "sha256": "1314b7154b5ffa12add7a5ddf9aa8e928f185109040eebf5060f73c75d83a940", + "size": 603693280 + }, + { + "backend": "linux-rocm-7.14-x64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.14.0.zip", + "sha256": "c8940e3c0416199b24626b76fd2273a87ac4030b14080af8f23d87783a80e4e6", + "size": 256666973 + }, + { + "backend": "linux-vulkan-x64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-vulkan.zip", + "sha256": "0972d8e978462c38f38371cd8f2376ebb5183fd869c22234a79bed6cc0e0c1bf", + "size": 37401766 + }, + { + "backend": "macos-arm64", + "name": "sd-master-137f740-bin-Darwin-macOS-26.6.2-arm64.zip", + "sha256": "56a8e614bec7a9659875020e2d3d78fcd5fc53765044ff38c868bfaf3ff3e395", + "size": 34338631 + }, + { + "backend": "win-cpu-arm64", + "name": "sd-master-137f740-bin-win-cpu-arm64.zip", + "sha256": "093259059dd18870395e869745b492533c38df95f96b804727624240b01f830a", + "size": 16092295 + }, + { + "backend": "win-cpu-x64", + "name": "sd-master-137f740-bin-win-cpu-x64.zip", + "sha256": "df9aae5591e33548a428ae822797eca209a651d41163a05fa8738684e0b6b46b", + "size": 17659940 + }, + { + "backend": "win-cuda12-x64", + "name": "sd-master-137f740-bin-win-cuda12-x64.zip", + "sha256": "836823a26c0721c90f0dd88c2bbc931399c24790d4f3333a2487ec4c48c6c074", + "size": 331362223 + }, + { + "backend": "win-cuda13-arm64", + "name": "sd-master-137f740-bin-win-cuda13-arm64.zip", + "sha256": "9e8c3aad7dea5d7db2999504573b7703db6c06b7ef2dc2203c73404156e647f9", + "size": 208687376 + }, + { + "backend": "win-cuda13-x64", + "name": "sd-master-137f740-bin-win-cuda13-x64.zip", + "sha256": "5ce539316bf831404f6142dcaa895ddd5c5398c61bac4aba987c16033ed388fc", + "size": 530924340 + }, + { + "backend": "win-cudart-cu12", + "name": "cudart-sd-bin-win-cu12-x64.zip", + "sha256": "fe20366827d357c00797eebb58244dddab7fd9a348d70090c3871004c320f38d", + "size": 563452046, + "companion": true + }, + { + "backend": "win-vulkan-x64", + "name": "sd-master-137f740-bin-win-vulkan-x64.zip", + "sha256": "a61d12deac8a39d60ab41afe6717c1f770e53da3aa0af8b1567d404c27cc2eaa", + "size": 32417764 + } + ] +} diff --git a/test/fixtures/sdcpp/conf/sdcpp-manifest.staging.json b/test/fixtures/sdcpp/conf/sdcpp-manifest.staging.json new file mode 100644 index 00000000..82c3b595 --- /dev/null +++ b/test/fixtures/sdcpp/conf/sdcpp-manifest.staging.json @@ -0,0 +1,94 @@ +{ + "$schema": "./sdcpp-schema.json", + "updated_at": "2026-10-07T10:25:13Z", + "upstream_repo": "AtomicBot-ai/stable-diffusion.cpp", + "tag_name": "master-883-137f740", + "download_base": "https://github.com/AtomicBot-ai/atomic-chat-conf/releases/download", + "assets": [ + { + "backend": "linux-cpu-arm64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64.zip", + "sha256": "2fbfb1ef7c758ceedcf1ac51cd8f80107c076c8a13160603fd7a65ac2f215efe", + "size": 20947364 + }, + { + "backend": "linux-cpu-x64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64.zip", + "sha256": "dc1af0b244302cd083d69d974424ba5f2dbe8f1b7f3b04be3db8103477dae47c", + "size": 24520937 + }, + { + "backend": "linux-cuda12-x64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-cuda12.zip", + "sha256": "d3dbc188ebab9e46485d3eec878612d2b62b49e799386124cddf5ad2be3b8fdd", + "size": 718645269 + }, + { + "backend": "linux-cuda13-arm64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64-cuda13.zip", + "sha256": "1314b7154b5ffa12add7a5ddf9aa8e928f185109040eebf5060f73c75d83a940", + "size": 603693280 + }, + { + "backend": "linux-rocm-7.14-x64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.14.0.zip", + "sha256": "c8940e3c0416199b24626b76fd2273a87ac4030b14080af8f23d87783a80e4e6", + "size": 256666973 + }, + { + "backend": "linux-vulkan-x64", + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-vulkan.zip", + "sha256": "0972d8e978462c38f38371cd8f2376ebb5183fd869c22234a79bed6cc0e0c1bf", + "size": 37401766 + }, + { + "backend": "macos-arm64", + "name": "sd-master-137f740-bin-Darwin-macOS-26.6.2-arm64.zip", + "sha256": "bfd00c4d1175926238be236d55ee7c312c188f2e0e409ad0acf18cbe3d15854c", + "size": 34338613 + }, + { + "backend": "win-cpu-arm64", + "name": "sd-master-137f740-bin-win-cpu-arm64.zip", + "sha256": "c67b295389656e5c07e4876d585ef9d41bb289f3cd140e34dc8820672e541c4a", + "size": 16092292 + }, + { + "backend": "win-cpu-x64", + "name": "sd-master-137f740-bin-win-cpu-x64.zip", + "sha256": "1d99a22c259906650d6d519dfddfbb9b24f24632b6d2e108b16207a2fa591f6c", + "size": 17659955 + }, + { + "backend": "win-cuda12-x64", + "name": "sd-master-137f740-bin-win-cuda12-x64.zip", + "sha256": "5aee9f217ab45d6e46e1697d066e17a81713f62bb6a71d807527b45bc945de2a", + "size": 331362241 + }, + { + "backend": "win-cuda13-arm64", + "name": "sd-master-137f740-bin-win-cuda13-arm64.zip", + "sha256": "9ea76ff2d83e7cb69e3a26df03c0595a8af1fc63b0c284bc726a7666f070420d", + "size": 208687381 + }, + { + "backend": "win-cuda13-x64", + "name": "sd-master-137f740-bin-win-cuda13-x64.zip", + "sha256": "d0b1b2698c1c0f5390671b15dcd88015e7dc0f23759b721a34b4dbdbf29a6eff", + "size": 530924361 + }, + { + "backend": "win-cudart-cu12", + "name": "cudart-sd-bin-win-cu12-x64.zip", + "sha256": "fe20366827d357c00797eebb58244dddab7fd9a348d70090c3871004c320f38d", + "size": 563452046, + "companion": true + }, + { + "backend": "win-vulkan-x64", + "name": "sd-master-137f740-bin-win-vulkan-x64.zip", + "sha256": "e1ca3f97ba3f55faabba098a3f24aae5e7fd9865c53c07780b962f4a167c39e1", + "size": 32417753 + } + ] +} diff --git a/test/fixtures/sdcpp/engine-contract.json b/test/fixtures/sdcpp/engine-contract.json new file mode 100644 index 00000000..2bd0aee7 --- /dev/null +++ b/test/fixtures/sdcpp/engine-contract.json @@ -0,0 +1,405 @@ +{ + "schema": "atomic-engine-contract", + "schema_version": 1, + "engine": "sd-cpp", + "generated_by": "by hand; test/contract/sdcpp-engine-contract.test.ts holds every part to src/diffusion and src/engine-builds", + "invocations": [ + { + "bin": "sd-server", + "argv": ["--help"], + "source": "src/diffusion/install.ts", + "expect": "exit-0", + "severity": "warn", + "note": "The install probe when the archive has no sd-cli; it reads the output, not the exit code." + }, + { + "bin": "sd-server", + "argv": ["--version"], + "source": "scripts/lib/sdcpp-surface.mjs", + "expect": "exit-0", + "severity": "warn", + "note": "Core never runs it; the check reads the commit the binary was built from." + }, + { + "bin": "sd-cli", + "argv": ["--help"], + "source": "src/diffusion/install.ts", + "expect": "exit-0", + "severity": "warn", + "optional": true, + "note": "The install probe (prepareSdcppTree); an archive without sd-cli is probed through sd-server." + } + ], + "probe": { + "bins": ["sd-cli", "sd-server"], + "argv": ["--help"], + "markers": ["stable-diffusion.cpp", "--cfg-scale"], + "source": "src/diffusion/install.ts", + "note": "probeOutputIsSdcpp: any marker, case-insensitive. Without one the archive installs nowhere." + }, + "version": { + "pattern": "stable-diffusion\\.cpp version (\\S+), commit (\\S+)", + "source": "examples/common/common.cpp (version_string)", + "note": "`version` is `git describe` of the build's checkout: `unknown` when it had no tags, as in the fork's builds." + }, + "flags": [ + { "flag": "--diffusion-model", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--vae", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--vae-format", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--audio-vae", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--embeddings-connectors", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--clip_l", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--t5xxl", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--llm", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--llm_vision", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--qwen2vl", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--clip-on-cpu", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--vae-on-cpu", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--vae-tiling", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--offload-to-cpu", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--diffusion-fa", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--diffusion-conv-direct", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--listen-ip", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--listen-port", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--lora-model-dir", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--hires-upscalers-dir", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--embd-dir", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--threads", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--backend", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "--verbose", "severity": "break", "source": "src/diffusion/args.ts" }, + { "flag": "-v", "severity": "break", "source": "src/diffusion/args.ts" }, + { + "flag": "-M", + "severity": "warn", + "source": "src/diffusion/args.ts", + "note": "Emitted only under ATOMIC_DIFFUSION_VID_GEN_MODE_FLAG, until a live run shows sd-server wants it; sd-cli has it." + } + ], + "strings": [ + { + "id": "route.ready", + "text": "/v1/models", + "channel": "http-route", + "source": "src/diffusion/server-process.ts", + "upstream_text": "\"/v1/models\"", + "severity_if_lost": "break", + "note": "GET; a 200 means the model is loaded (sd-server binds after loading)." + }, + { + "id": "route.capabilities", + "text": "/sdcpp/v1/capabilities", + "channel": "http-route", + "source": "src/diffusion/server-process.ts", + "upstream_text": "\"/sdcpp/v1/capabilities\"", + "severity_if_lost": "break", + "note": "GET; a 404 means the port belongs to some other server." + }, + { + "id": "route.img_gen", + "text": "/sdcpp/v1/img_gen", + "channel": "http-route", + "source": "src/diffusion/jobs.ts", + "upstream_text": "\"/sdcpp/v1/img_gen\"", + "severity_if_lost": "break" + }, + { + "id": "route.vid_gen", + "text": "/sdcpp/v1/vid_gen", + "channel": "http-route", + "source": "src/diffusion/jobs.ts", + "upstream_text": "\"/sdcpp/v1/vid_gen\"", + "severity_if_lost": "break", + "note": "The live run on CI has no video model: only this text search sees the route." + }, + { + "id": "route.job", + "text": "/sdcpp/v1/jobs/{id}", + "channel": "http-route", + "source": "src/diffusion/jobs.ts", + "upstream_text": "/sdcpp/v1/jobs/(", + "severity_if_lost": "break", + "note": "GET, polled; upstream registers it (and the cancel route) as a regex." + }, + { + "id": "route.cancel", + "text": "/sdcpp/v1/jobs/{id}/cancel", + "channel": "http-route", + "source": "src/diffusion/jobs.ts", + "upstream_text": ")/cancel)", + "severity_if_lost": "break" + }, + { + "id": "progress.step", + "text": "/ - ", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": "%i/%i - %s", + "severity_if_lost": "warn", + "note": "parseStepLine: the first N/M of a redraw (print_progress_line in src/core/util.cpp)." + }, + { + "id": "progress.rate.s-it", + "text": "s/it", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": "\"s/it\"", + "severity_if_lost": "warn", + "note": "isProgressRedraw: a redraw ends with a rate." + }, + { + "id": "progress.rate.it-s", + "text": "it/s", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": "\"it/s\"", + "severity_if_lost": "warn" + }, + { + "id": "progress.rate.bytes", + "text": "B/s", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": "MB/s", + "severity_if_lost": "info", + "note": "The tensor loader's redraw; only keeps it out of the step bar." + }, + { + "id": "progress.tiles", + "text": "processing tiles", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": "processing %i tiles", + "severity_if_lost": "warn", + "note": "parseTileAnnouncement: without it the tiles of a VAE pass read as sampling steps." + }, + { + "id": "progress.generating-image", + "text": "generating image:", + "channel": "log", + "source": "src/diffusion/tracker.ts", + "upstream_text": "generating image:", + "severity_if_lost": "warn" + }, + { + "id": "progress.generating-video", + "text": "generating video:", + "channel": "log", + "source": "src/diffusion/tracker.ts", + "upstream_text": "generating video:", + "severity_if_lost": "warn", + "note": "Not printed at master-883 (video.cpp logs `generate_video …`): reported as text in neither tree." + }, + { + "id": "oom.model-manager", + "text": "cannot make enough memory available", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": "cannot make enough memory available", + "severity_if_lost": "warn", + "note": "classifyExit: OUT_OF_MEMORY instead of ENGINE_CRASHED, and the offload fallback." + }, + { + "id": "oom.out-of-memory", + "text": "out of memory", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null, + "note": "ggml and the GPU APIs, outside sd.cpp's tree." + }, + { + "id": "oom.failed-to-allocate", + "text": "failed to allocate", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null + }, + { + "id": "oom.cuda-error-memory-allocation", + "text": "cudaErrorMemoryAllocation", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null + }, + { + "id": "oom.cuda-error-out-of-memory", + "text": "CUDA_ERROR_OUT_OF_MEMORY", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null + }, + { + "id": "oom.vulkan-out-of-device-memory", + "text": "ErrorOutOfDeviceMemory", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null + }, + { + "id": "oom.insufficient-memory", + "text": "insufficient memory", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null + }, + { + "id": "abort.unsupported-op", + "text": "unsupported op", + "channel": "log", + "source": "src/diffusion/args.ts", + "upstream_text": null, + "note": "With ggml_abort: the one death retried on the CPU backend (isGgmlUnsupportedOpAbort)." + }, + { + "id": "abort.ggml-abort", + "text": "ggml_abort", + "channel": "log", + "source": "src/diffusion/args.ts", + "upstream_text": null + }, + { + "id": "gpu-fault.address", + "text": "gpu address fault", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null, + "note": "GpuFaultWatch: ggml-metal's words." + }, + { + "id": "gpu-fault.error-state", + "text": "backend is in error state", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null + }, + { + "id": "gpu-fault.command-buffer", + "text": "command buffer", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null + }, + { + "id": "gpu-fault.pagefault", + "text": "pagefault", + "channel": "log", + "source": "src/diffusion/progress.ts", + "upstream_text": null + } + ], + "assets": { + "source": "conf .github/workflows/mirror-sdcpp.yml (MATRIX) and src/engine-builds/select.ts (diffusionBackendLadder)", + "note": "{sha7} is the upstream commit of the tag. `required`: conf's mirror refuses a release without it. `cpu`: the build every host of the platform falls back to. Mirrored archives core never selects (linux-cuda12-x64, win-cuda13-x64, linux-rocm) are not listed.", + "core_backends": [ + { + "platform": "macos-arm64", + "backend": "macos-arm64", + "cpu": true, + "required": true, + "asset": "sd-master-{sha7}-bin-Darwin-macOS-[\\d.]+-arm64\\.zip" + }, + { + "platform": "windows-x64", + "backend": "win-cuda12-x64", + "cpu": false, + "required": true, + "asset": "sd-master-{sha7}-bin-win-cuda12-x64\\.zip" + }, + { + "platform": "windows-x64", + "backend": "win-cudart-cu12", + "cpu": false, + "required": false, + "companion": true, + "asset": "cudart-sd-bin-win-cu12-x64\\.zip" + }, + { + "platform": "windows-x64", + "backend": "win-vulkan-x64", + "cpu": false, + "required": true, + "asset": "sd-master-{sha7}-bin-win-vulkan-x64\\.zip" + }, + { + "platform": "windows-x64", + "backend": "win-cpu-x64", + "cpu": true, + "required": true, + "asset": "sd-master-{sha7}-bin-win-cpu-x64\\.zip" + }, + { + "platform": "windows-arm64", + "backend": "win-cuda13-arm64", + "cpu": false, + "required": false, + "asset": "sd-master-{sha7}-bin-win-cuda13-arm64\\.zip" + }, + { + "platform": "windows-arm64", + "backend": "win-cpu-arm64", + "cpu": true, + "required": false, + "asset": "sd-master-{sha7}-bin-win-cpu-arm64\\.zip" + }, + { + "platform": "linux-x64", + "backend": "linux-vulkan-x64", + "cpu": false, + "required": true, + "asset": "sd-master-{sha7}-bin-Linux-Ubuntu-[\\d.]+-x86_64-vulkan\\.zip" + }, + { + "platform": "linux-x64", + "backend": "linux-cpu-x64", + "cpu": true, + "required": true, + "asset": "sd-master-{sha7}-bin-Linux-Ubuntu-[\\d.]+-x86_64\\.zip" + }, + { + "platform": "linux-arm64", + "backend": "linux-cuda13-arm64", + "cpu": false, + "required": false, + "asset": "sd-master-{sha7}-bin-Linux-Ubuntu-[\\d.]+-aarch64-cuda13\\.zip" + }, + { + "platform": "linux-arm64", + "backend": "linux-cpu-arm64", + "cpu": true, + "required": false, + "asset": "sd-master-{sha7}-bin-Linux-Ubuntu-[\\d.]+-aarch64\\.zip" + } + ] + }, + "upstream": { + "repo": "leejet/stable-diffusion.cpp", + "branch": "master", + "watched_paths": [ + { "path": "examples/common/common.cpp", "why": "every flag, --help, --version and the request JSON core sends" }, + { "path": "examples/common/common.h", "why": "the option and request structs behind the flags" }, + { "path": "examples/server/main.cpp", "why": "sd-server's startup: model load before bind, --version" }, + { "path": "examples/server/routes_sdcpp.cpp", "why": "/sdcpp/v1/* routes, capabilities, job submit and cancel" }, + { "path": "examples/server/routes_openai.cpp", "why": "GET /v1/models, core's readiness poll" }, + { "path": "examples/server/routes.h", "why": "which route families sd-server registers" }, + { "path": "examples/server/async_jobs.cpp", "why": "job states and result fields core polls for" }, + { "path": "examples/server/async_jobs.h", "why": "the job record" }, + { "path": "examples/server/runtime.cpp", "why": "the resident context, outputs and their mime types" }, + { "path": "examples/server/runtime.h", "why": "the server runtime" }, + { "path": "examples/cli/main.cpp", "why": "sd-cli --help, the install probe" }, + { "path": "src/core/util.cpp", "why": "the progress line core parses" }, + { "path": "src/pipeline/request.cpp", "why": "custom_sigmas and the steps they run" }, + { "path": "src/pipeline/diffusion_engine.cpp", "why": "how --diffusion-model, --vae and --llm are loaded" }, + { "path": "src/pipeline/image.cpp", "why": "`generating image:`" }, + { "path": "src/pipeline/video.cpp", "why": "video generation and its log lines" }, + { "path": "src/runtime/tiling.cpp", "why": "`processing N tiles`" }, + { "path": "src/model_manager.cpp", "why": "`cannot make enough memory available`" }, + { "path": "CMakeLists.txt", "why": "the version and commit --version prints, build options" } + ], + "release_note_keywords": [ + { "pattern": "deprecat", "severity": "warn" }, + { "pattern": "remove", "severity": "warn" }, + { "pattern": "rename", "severity": "warn" }, + { "pattern": "breaking", "severity": "warn" }, + { "pattern": "\\b(server|sd-server|api|cli|args?)\\b", "severity": "info" } + ] + } +} diff --git a/test/fixtures/sdcpp/help/sd-cli-master-883-137f740.txt b/test/fixtures/sdcpp/help/sd-cli-master-883-137f740.txt new file mode 100644 index 00000000..9cc1ae49 --- /dev/null +++ b/test/fixtures/sdcpp/help/sd-cli-master-883-137f740.txt @@ -0,0 +1,278 @@ +stable-diffusion.cpp version unknown, commit 137f740 +Usage: ./sd-cli [options] + +CLI Options: + -o, --output path to write result image to. you can use printf-style %d format specifiers for image + sequences (default: ./output.png) (eg. output_%03d.png). Single-file video outputs + support .avi, .webm, and animated .webp + --image path to the image to inspect (for metadata mode) + --metadata-format metadata output format, one of [text, json] (default: text) + --preview-path path to write preview image to (default: ./preview.png). For image generation, the + filename can have %03d placeholder for sequential numbering. Multi-frame previews + support .avi, .webm, and animated .webp + --imat-out compute the imatrix for this run and save it to the provided path + --preview-interval preview interval: in each sampling pass, positive N updates every Nth denoiser step and + -N previews only completed logical step N; 0 previews the final completed step of the + first pass (base-resolution or high-noise). Default: 1 + --output-begin-idx starting index for output image sequence, must be non-negative (default 0 if specified + %d in output path, 1 otherwise) + --compression-quality compression quality of video and JPEG / WebP images (90 by default) + --canny apply canny preprocessor (edge detection) + --convert-name convert tensor name (for convert mode) + --color colors the logging tags according to level + --taesd-preview-only prevents usage of taesd for decoding the final image. (for use with --preview tae) + --preview-noisy enables previewing noisy inputs of the models rather than the denoised outputs + --metadata-raw include raw hex previews for unparsed metadata payloads + --metadata-brief truncate long metadata text values in text output + --metadata-all include structural/container entries such as IHDR, IDAT, and non-metadata JPEG segments + -M, --mode run mode, one of [img_gen, adetailer, vid_gen, upscale, convert, metadata], default: + img_gen + --preview preview method. must be one of the following [none, proj, tae, vae] (default is none) + -h, --help show this help message and exit + --imat-in load an imatrix file for quantization or continued collection; can be specified + multiple times + --log-level minimum log level, one of [debug, verbose, info, warn, error] (default: info) + -v, --verbose equivalent to --log-level verbose + +Context Options: + -m, --model path to full model + --clip_l path to the clip-l text encoder + --clip_g path to the clip-g text encoder + --clip_vision path to the clip-vision encoder + --t5xxl path to the t5xxl text encoder + --llm path to the llm text encoder. For example: (qwenvl2.5 for qwen-image, + mistral-small3.2 for flux2, ...) + --tokenizer tokenizer.json path, or comma-separated main=FILE,clip-l=FILE,clip-g=FILE + assignments; required for PiD and Lens + --llm_vision path to the llm vit + --qwen2vl alias of --llm. Deprecated. + --qwen2vl_vision alias of --llm_vision. Deprecated. + --diffusion-model path to the standalone diffusion model + --high-noise-diffusion-model path to the standalone high noise diffusion model + --uncond-diffusion-model path to the standalone unconditional diffusion model, currently used by + Ideogram4 CFG + --embeddings-connectors path to LTXAV embeddings connectors + --vae path to standalone vae model + --vae-format VAE latent format override: auto, flux, sd3, flux2, or wan (default: auto) + --audio-vae path to standalone LTX audio vae model + --audio-encoder path to wav2vec2 audio encoder model (Wan2.2 S2V) + --taesd path to taesd. Using Tiny AutoEncoder for fast decoding (low quality) + --tae alias of --taesd + --control-net path to control net model + --ip-adapter path to IP-Adapter model (requires --clip_vision) + --motion-module path to AnimateDiff motion module (SD 1.5); enables video generation on + --video-frames > 1 + --embd-dir embeddings directory + --lora-model-dir lora model directory + --hires-upscalers-dir highres fix upscaler model directory + --tensor-type-rules weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0") + --model-args extra model args, key=value list. Supports chroma_use_dit_mask, + chroma_use_t5_mask, chroma_t5_mask_pad, qwen_image_zero_cond_t + --photo-maker path to PHOTOMAKER model + --pulid-weights path to PuLID Flux weights + --upscale-model path to esrgan model. + --backend runtime backend assignment, e.g. cpu or clip=cpu,vae=cuda0,diffusion=vulkan0 + --params-backend parameter backend assignment, e.g. disk, cpu, or diffusion=disk,clip=cpu + --split-mode weight distribution for modules assigned multiple devices (--backend + "diffusion=cuda0&cuda1"): layer (whole transformer blocks per device, + default) or row (matmul rows split across devices, CUDA only). Accepts a + single mode or per-module assignments, e.g. row or diffusion=row,te=layer + --rpc-servers comma-separated list of RPC servers to connect to for offloading, in the + format host:port, e.g. localhost:50052,192.168.1.3:50052 + --max-vram optional per-device budget in GiB for managed weights and runner buffers + during automatic graph-cut execution. Accepts a single value or assignments + by backend/device, e.g. 6 or cuda0=6,vulkan0=4. 0 uses live free VRAM without + an explicit budget; a negative value reserves that much free VRAM + -t, --threads number of threads to use during computation (default: -1). If threads <= 0, + then threads will be set to the number of CPU physical cores + --disable-prefetch disable asynchronous next-segment weight prefetch (defaults to false) + --disable-segmented-compute force monolithic graph execution even when automatic graph cutting is needed + (defaults to false) + --eager-load load all params into the params backend at model-load time instead of lazily + on first use (defaults to false) + --force-sdxl-vae-conv-scale force use of conv scale on sdxl vae + --offload-to-cpu place the weights in RAM to save VRAM, and automatically load them into VRAM + when needed + --mmap whether to memory-map model + --control-net-cpu deprecated; use --backend controlnet=cpu + --clip-on-cpu deprecated; use --backend te=cpu + --vae-on-cpu deprecated; use --backend vae=cpu + --fa use flash attention + --diffusion-fa use flash attention in the diffusion model only + --diffusion-conv-direct use ggml_conv2d_direct in the diffusion model + --vae-conv-direct use direct 2D and 3D convolutions in the vae model + --linear-scale linear input scale override (float, default: 0 = model default, 1 = no + scaling) + --attn-scale flash-attention K/V scale override (float, default: 0 = model default, 1 = no + scaling); requires --fa or --diffusion-fa + --auto-fit on|off (default: on). Preserve --backend (otherwise select one GPU) and place + weights on the compute GPU, RAM, another GPU, or disk in that order, + according to available memory (--max-vram limits GPU budgets). Disabled by + explicit --params-backend; uses automatic graph segmentation when needed + --type weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, + q4_K). If not specified, the default is the type of the weight file + --rng RNG, one of [std_default, cuda, cpu], default: cuda(sd-webui), cpu(comfyui) + --sampler-rng sampler RNG, one of [std_default, cuda, cpu]. If not specified, use --rng + --prediction prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow, + sefi_flow] + --lora-apply-mode the way to apply LoRA, one of [auto, immediately, at_runtime], default is + auto. In auto mode, if the model weights contain any quantized parameters, + the at_runtime mode will be used; otherwise, immediately will be used.The + immediately mode may have precision and compatibility issues with quantized + parameters, but it usually offers faster inference speed and, in some cases, + lower memory usage. The at_runtime mode, on the other hand, is exactly the + opposite. + --list-devices list available ggml backend devices (one 'namedescription' per line) and + exit; the names are the device names accepted by --backend and + --params-backend + +Generation Options: + -p, --prompt the prompt to render + -n, --negative-prompt the negative prompt (default: "") + --ad-model path to a converted YOLOv8 detection model for ADetailer + --ad-prompt ADetailer prompt; empty inherits the main prompt, supports [PROMPT], [SEP], + and [SKIP] + --ad-negative-prompt ADetailer negative prompt; empty inherits the main negative prompt, supports + [PROMPT] and [SEP] + --extra-ad-args extra ADetailer args, key=value list. Supports input_size, confidence, nms, + max_detections, mask_k_largest, mask_min_ratio, mask_max_ratio, dilate_erode, + x_offset, y_offset, mask_mode, merge_masks, invert_mask, mask_blur, + inpaint_padding, inpaint_width, inpaint_height, denoising_strength, steps, + cfg_scale, sample_method, scheduler, sort_by + -i, --init-img path to the init image + --end-img path to the end image, required by flf2v + --mask path to the mask image + --control-image path to control image, control net + --ip-adapter-image path to the IP-Adapter reference image + --control-video path to control video frames, It must be a directory path. The video frames + inside should be stored as images in lexicographical (character) order. For + example, if the control video path is `frames`, the directory contain images + such as 00.png, 01.png, ... etc. + --pm-id-images-dir path to PHOTOMAKER input id images dir + --pm-id-embed-path path to PHOTOMAKER v2 id embed + --pulid-id-embedding path to PuLID id embedding + --hires-upscaler highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent + (nearest-exact), Latent (antialiased), Latent (bicubic), Latent (bicubic + antialiased), or a model name under --hires-upscalers-dir (default: Latent) + --extra-sample-args extra sampler/scheduler/guidance args, key=value list. CFG supports + guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, + apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports + noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, + max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge + supports gamma; beta scheduler supports alpha, beta; logit_normal supports + mu, std, logsnr_min, logsnr_max, resolution_aware; lms supports + lms_max_order, lms_shift, lms_divisions; noise-injecting samplers support + noise_sampler with value iid (default except for dpm++2m_sde_bt) or + brownian_tree; brownian_tree_rng supports cpu (default), cuda, std_default or + sampler_rng + --extra-tiling-args extra VAE tiling args, key=value list. Supported video VAEs accept + temporal_tile_frames/temporal_tile_size (default: 4), temporal_tile_overlap + (default: 1) + --ref-image-args Key-value list to set up the way the reference images are processed (empty = + auto-detect from model weigths) + -H, --height image height, in pixel space (default: 512) + -W, --width image width, in pixel space (default: 512) + --steps number of sample steps (default: 20) + --high-noise-steps (high noise) number of sample steps (default: -1 = auto) + --clip-skip ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer + (default: -1). <= 0 represents unspecified, will be 1 for SD1.x, 2 for SD2.x + -b, --batch-count batch count + --qwen-image-layers number of Qwen Image Layered layers; latent/output count is layers + 1 + (default: 3) + --video-frames video frames (default: 1) + --fps fps (default: 24) + --timestep-shift shift timestep for NitroFusion models (default: 0). recommended N for + NitroSD-Realism around 250 and 500 for NitroSD-Vibrant + --upscale-repeats Run the ESRGAN upscaler this many times (default: 1) + --upscale-tile-size tile size for ESRGAN upscaling (default: 128) + --hires-width highres fix target width, 0 to use --hires-scale (default: 0) + --hires-height highres fix target height, 0 to use --hires-scale (default: 0) + --hires-steps highres fix second pass sample steps, 0 to reuse --steps (default: 0) + --hires-upscale-tile-size highres fix upscaler tile size, reserved for model-backed upscalers (default: + 128) + --cfg-scale unconditional guidance scale: (default: 7.0) + --img-cfg-scale image guidance scale for inpaint or image edit models: (default: same as + --cfg-scale) + --guidance distilled guidance scale for models with guidance input (default: 3.5) + --slg-scale skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means + disabled, a value of 2.5 is nice for sd3.5 medium + --skip-layer-start SLG enabling point (default: 0.01) + --skip-layer-end SLG disabling point (default: 0.2) + --eta noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and + res_2s; 1 for euler_a, er_sde, dpm++2s_a, dpm++2m_sde and dpm++2m_sde_bt) + --flow-shift shift value for Flow models like SD3.x or WAN (default: auto) + --high-noise-cfg-scale (high noise) unconditional guidance scale: (default: 7.0) + --high-noise-img-cfg-scale (high noise) image guidance scale for inpaint or image edit models (default: + same as --cfg-scale) + --high-noise-guidance (high noise) distilled guidance scale for models with guidance input + (default: 3.5) + --high-noise-slg-scale (high noise) skip layer guidance (SLG) scale, only for DiT models: (default: + 0) + --high-noise-skip-layer-start (high noise) SLG enabling point (default: 0.01) + --high-noise-skip-layer-end (high noise) SLG disabling point (default: 0.2) + --high-noise-eta (high noise) noise multiplier (default: 0 for ddim_trailing, tcd, + res_multistep and res_2s; 1 for euler_a, er_sde, dpm++2s_a, dpm++2m_sde and + dpm++2m_sde_bt) + --strength strength for noising/unnoising (default: 0.75) + --pm-style-strength + --pulid-id-weight strength of PuLID identity injection + --control-strength strength to apply Control Net (default: 0.9). 1.0 corresponds to full + destruction of information in init image + --ip-adapter-strength strength to apply IP-Adapter (default: 1.0) + --moe-boundary timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if + `--high-noise-steps` is set to -1 + --vace-strength wan vace strength + --vae-tile-overlap tile overlap for vae tiling, in fraction of tile size (default: 0.5) + --hires-scale highres fix scale when target size is not set (default: 2.0) + --hires-denoising-strength highres fix second pass denoising strength (default: 0.7) + --increase-ref-index automatically increase the indices of references images based on the order + they are listed (starting with 1). + --disable-auto-resize-ref-image disable auto resize of ref images + --circular enable circular padding on both axes for tileable output + --circularx enable circular padding on x-axis (width) only + --circulary enable circular padding on y-axis (height) only + --disable-image-metadata do not embed generation metadata on image files + --vae-tiling process vae in tiles to reduce memory usage + --temporal-tiling enable temporal tiling for supported video VAE decode + --hires enable highres fix + -s, --seed RNG seed (default: 42, use random seed for < 0) + --sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, + dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, + er_sde, euler_cfg_pp, euler_a_cfg_pp, euler_ge, dpm++2m_sde, dpm++2m_sde_bt, + lms], default: euler for Flux/SD3/Wan, euler_a otherwise + --high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, + dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, + res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp, euler_ge, dpm++2m_sde, + dpm++2m_sde_bt, lms], default: euler for Flux/SD3/Wan, euler_a otherwise + --scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, + sgm_uniform, simple, smoothstep, kl_optimal, lcm, bong_tangent, ltx2, + logit_normal, flux2, flux, beta], alias: normal=discrete, default: + model-specific + --sigmas custom sigma values for the sampler, comma-separated (e.g., + "14.61,7.8,3.5,0.0"). + --hires-sigmas custom sigma values for the highres fix second pass, comma-separated (e.g., + "0.85,0.725,0.421875,0.0"). + --skip-layers layers to skip for SLG steps (default: [7,8,9]) + --high-noise-skip-layers (high noise) layers to skip for SLG steps (default: [7,8,9]) + -r, --ref-image reference image for Flux Kontext or MiniMax-H3 Ref2VA (can be used multiple + times) + --ref-video MiniMax-H3 Ref2VA reference video frame directory at 24 fps (can be used + multiple times) + --ref-video-audio WAV soundtrack paired by index with --ref-video (can be used multiple times) + --ref-audio standalone WAV reference for MiniMax-H3 Ref2VA (can be used multiple times) + --audio driving audio track (Wan2.2 S2V; can be used once) + --cache-mode caching method: 'easycache' (DiT), 'ucache' (UNET), + 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT + Chebyshev+Taylor forecasting) + --cache-option named cache params (key=value format, comma-separated). easycache/ucache: + threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit: + Fn=,Bn=,threshold=,warmup=; spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=. + Examples: "threshold=0.25" or "threshold=1.5,reset=0" + --scm-mask SCM steps mask for cache-dit: comma-separated 0/1 (e.g., + "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache + --scm-policy SCM policy: 'dynamic' (default) or 'static' + --vae-tile-size tile size for vae tiling, format [X]x[Y] (default: 32x32) + --vae-relative-tile-size relative tile size for vae tiling, format [X]x[Y], in fraction of image size + if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size) + --prompt-file path to the file containing the prompt to render + --negative-prompt-file path to the file containing the negative prompt diff --git a/test/fixtures/sdcpp/help/sd-server-master-883-137f740.txt b/test/fixtures/sdcpp/help/sd-server-master-883-137f740.txt new file mode 100644 index 00000000..efdb1f04 --- /dev/null +++ b/test/fixtures/sdcpp/help/sd-server-master-883-137f740.txt @@ -0,0 +1,254 @@ +stable-diffusion.cpp version unknown, commit 137f740 +Usage: ./sd-server [options] + +Svr Options: + -l, --listen-ip server listen ip (default: 127.0.0.1) + --serve-html-path path to HTML file to serve at root (optional) + --listen-port server listen port (default: 1234) + --color colors the logging tags according to level + -h, --help show this help message and exit + --log-level minimum log level, one of [debug, verbose, info, warn, error] (default: info) + -v, --verbose equivalent to --log-level verbose + +Context Options: + -m, --model path to full model + --clip_l path to the clip-l text encoder + --clip_g path to the clip-g text encoder + --clip_vision path to the clip-vision encoder + --t5xxl path to the t5xxl text encoder + --llm path to the llm text encoder. For example: (qwenvl2.5 for qwen-image, + mistral-small3.2 for flux2, ...) + --tokenizer tokenizer.json path, or comma-separated main=FILE,clip-l=FILE,clip-g=FILE + assignments; required for PiD and Lens + --llm_vision path to the llm vit + --qwen2vl alias of --llm. Deprecated. + --qwen2vl_vision alias of --llm_vision. Deprecated. + --diffusion-model path to the standalone diffusion model + --high-noise-diffusion-model path to the standalone high noise diffusion model + --uncond-diffusion-model path to the standalone unconditional diffusion model, currently used by + Ideogram4 CFG + --embeddings-connectors path to LTXAV embeddings connectors + --vae path to standalone vae model + --vae-format VAE latent format override: auto, flux, sd3, flux2, or wan (default: auto) + --audio-vae path to standalone LTX audio vae model + --audio-encoder path to wav2vec2 audio encoder model (Wan2.2 S2V) + --taesd path to taesd. Using Tiny AutoEncoder for fast decoding (low quality) + --tae alias of --taesd + --control-net path to control net model + --ip-adapter path to IP-Adapter model (requires --clip_vision) + --motion-module path to AnimateDiff motion module (SD 1.5); enables video generation on + --video-frames > 1 + --embd-dir embeddings directory + --lora-model-dir lora model directory + --hires-upscalers-dir highres fix upscaler model directory + --tensor-type-rules weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0") + --model-args extra model args, key=value list. Supports chroma_use_dit_mask, + chroma_use_t5_mask, chroma_t5_mask_pad, qwen_image_zero_cond_t + --photo-maker path to PHOTOMAKER model + --pulid-weights path to PuLID Flux weights + --upscale-model path to esrgan model. + --backend runtime backend assignment, e.g. cpu or clip=cpu,vae=cuda0,diffusion=vulkan0 + --params-backend parameter backend assignment, e.g. disk, cpu, or diffusion=disk,clip=cpu + --split-mode weight distribution for modules assigned multiple devices (--backend + "diffusion=cuda0&cuda1"): layer (whole transformer blocks per device, + default) or row (matmul rows split across devices, CUDA only). Accepts a + single mode or per-module assignments, e.g. row or diffusion=row,te=layer + --rpc-servers comma-separated list of RPC servers to connect to for offloading, in the + format host:port, e.g. localhost:50052,192.168.1.3:50052 + --max-vram optional per-device budget in GiB for managed weights and runner buffers + during automatic graph-cut execution. Accepts a single value or assignments + by backend/device, e.g. 6 or cuda0=6,vulkan0=4. 0 uses live free VRAM without + an explicit budget; a negative value reserves that much free VRAM + -t, --threads number of threads to use during computation (default: -1). If threads <= 0, + then threads will be set to the number of CPU physical cores + --disable-prefetch disable asynchronous next-segment weight prefetch (defaults to false) + --disable-segmented-compute force monolithic graph execution even when automatic graph cutting is needed + (defaults to false) + --eager-load load all params into the params backend at model-load time instead of lazily + on first use (defaults to false) + --force-sdxl-vae-conv-scale force use of conv scale on sdxl vae + --offload-to-cpu place the weights in RAM to save VRAM, and automatically load them into VRAM + when needed + --mmap whether to memory-map model + --control-net-cpu deprecated; use --backend controlnet=cpu + --clip-on-cpu deprecated; use --backend te=cpu + --vae-on-cpu deprecated; use --backend vae=cpu + --fa use flash attention + --diffusion-fa use flash attention in the diffusion model only + --diffusion-conv-direct use ggml_conv2d_direct in the diffusion model + --vae-conv-direct use direct 2D and 3D convolutions in the vae model + --linear-scale linear input scale override (float, default: 0 = model default, 1 = no + scaling) + --attn-scale flash-attention K/V scale override (float, default: 0 = model default, 1 = no + scaling); requires --fa or --diffusion-fa + --auto-fit on|off (default: on). Preserve --backend (otherwise select one GPU) and place + weights on the compute GPU, RAM, another GPU, or disk in that order, + according to available memory (--max-vram limits GPU budgets). Disabled by + explicit --params-backend; uses automatic graph segmentation when needed + --type weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, + q4_K). If not specified, the default is the type of the weight file + --rng RNG, one of [std_default, cuda, cpu], default: cuda(sd-webui), cpu(comfyui) + --sampler-rng sampler RNG, one of [std_default, cuda, cpu]. If not specified, use --rng + --prediction prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow, + sefi_flow] + --lora-apply-mode the way to apply LoRA, one of [auto, immediately, at_runtime], default is + auto. In auto mode, if the model weights contain any quantized parameters, + the at_runtime mode will be used; otherwise, immediately will be used.The + immediately mode may have precision and compatibility issues with quantized + parameters, but it usually offers faster inference speed and, in some cases, + lower memory usage. The at_runtime mode, on the other hand, is exactly the + opposite. + --list-devices list available ggml backend devices (one 'namedescription' per line) and + exit; the names are the device names accepted by --backend and + --params-backend + +Default Generation Options: + -p, --prompt the prompt to render + -n, --negative-prompt the negative prompt (default: "") + --ad-model path to a converted YOLOv8 detection model for ADetailer + --ad-prompt ADetailer prompt; empty inherits the main prompt, supports [PROMPT], [SEP], + and [SKIP] + --ad-negative-prompt ADetailer negative prompt; empty inherits the main negative prompt, supports + [PROMPT] and [SEP] + --extra-ad-args extra ADetailer args, key=value list. Supports input_size, confidence, nms, + max_detections, mask_k_largest, mask_min_ratio, mask_max_ratio, dilate_erode, + x_offset, y_offset, mask_mode, merge_masks, invert_mask, mask_blur, + inpaint_padding, inpaint_width, inpaint_height, denoising_strength, steps, + cfg_scale, sample_method, scheduler, sort_by + -i, --init-img path to the init image + --end-img path to the end image, required by flf2v + --mask path to the mask image + --control-image path to control image, control net + --ip-adapter-image path to the IP-Adapter reference image + --control-video path to control video frames, It must be a directory path. The video frames + inside should be stored as images in lexicographical (character) order. For + example, if the control video path is `frames`, the directory contain images + such as 00.png, 01.png, ... etc. + --pm-id-images-dir path to PHOTOMAKER input id images dir + --pm-id-embed-path path to PHOTOMAKER v2 id embed + --pulid-id-embedding path to PuLID id embedding + --hires-upscaler highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent + (nearest-exact), Latent (antialiased), Latent (bicubic), Latent (bicubic + antialiased), or a model name under --hires-upscalers-dir (default: Latent) + --extra-sample-args extra sampler/scheduler/guidance args, key=value list. CFG supports + guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, + apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports + noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, + max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge + supports gamma; beta scheduler supports alpha, beta; logit_normal supports + mu, std, logsnr_min, logsnr_max, resolution_aware; lms supports + lms_max_order, lms_shift, lms_divisions; noise-injecting samplers support + noise_sampler with value iid (default except for dpm++2m_sde_bt) or + brownian_tree; brownian_tree_rng supports cpu (default), cuda, std_default or + sampler_rng + --extra-tiling-args extra VAE tiling args, key=value list. Supported video VAEs accept + temporal_tile_frames/temporal_tile_size (default: 4), temporal_tile_overlap + (default: 1) + --ref-image-args Key-value list to set up the way the reference images are processed (empty = + auto-detect from model weigths) + -H, --height image height, in pixel space (default: 512) + -W, --width image width, in pixel space (default: 512) + --steps number of sample steps (default: 20) + --high-noise-steps (high noise) number of sample steps (default: -1 = auto) + --clip-skip ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer + (default: -1). <= 0 represents unspecified, will be 1 for SD1.x, 2 for SD2.x + -b, --batch-count batch count + --qwen-image-layers number of Qwen Image Layered layers; latent/output count is layers + 1 + (default: 3) + --video-frames video frames (default: 1) + --fps fps (default: 24) + --timestep-shift shift timestep for NitroFusion models (default: 0). recommended N for + NitroSD-Realism around 250 and 500 for NitroSD-Vibrant + --upscale-repeats Run the ESRGAN upscaler this many times (default: 1) + --upscale-tile-size tile size for ESRGAN upscaling (default: 128) + --hires-width highres fix target width, 0 to use --hires-scale (default: 0) + --hires-height highres fix target height, 0 to use --hires-scale (default: 0) + --hires-steps highres fix second pass sample steps, 0 to reuse --steps (default: 0) + --hires-upscale-tile-size highres fix upscaler tile size, reserved for model-backed upscalers (default: + 128) + --cfg-scale unconditional guidance scale: (default: 7.0) + --img-cfg-scale image guidance scale for inpaint or image edit models: (default: same as + --cfg-scale) + --guidance distilled guidance scale for models with guidance input (default: 3.5) + --slg-scale skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means + disabled, a value of 2.5 is nice for sd3.5 medium + --skip-layer-start SLG enabling point (default: 0.01) + --skip-layer-end SLG disabling point (default: 0.2) + --eta noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and + res_2s; 1 for euler_a, er_sde, dpm++2s_a, dpm++2m_sde and dpm++2m_sde_bt) + --flow-shift shift value for Flow models like SD3.x or WAN (default: auto) + --high-noise-cfg-scale (high noise) unconditional guidance scale: (default: 7.0) + --high-noise-img-cfg-scale (high noise) image guidance scale for inpaint or image edit models (default: + same as --cfg-scale) + --high-noise-guidance (high noise) distilled guidance scale for models with guidance input + (default: 3.5) + --high-noise-slg-scale (high noise) skip layer guidance (SLG) scale, only for DiT models: (default: + 0) + --high-noise-skip-layer-start (high noise) SLG enabling point (default: 0.01) + --high-noise-skip-layer-end (high noise) SLG disabling point (default: 0.2) + --high-noise-eta (high noise) noise multiplier (default: 0 for ddim_trailing, tcd, + res_multistep and res_2s; 1 for euler_a, er_sde, dpm++2s_a, dpm++2m_sde and + dpm++2m_sde_bt) + --strength strength for noising/unnoising (default: 0.75) + --pm-style-strength + --pulid-id-weight strength of PuLID identity injection + --control-strength strength to apply Control Net (default: 0.9). 1.0 corresponds to full + destruction of information in init image + --ip-adapter-strength strength to apply IP-Adapter (default: 1.0) + --moe-boundary timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if + `--high-noise-steps` is set to -1 + --vace-strength wan vace strength + --vae-tile-overlap tile overlap for vae tiling, in fraction of tile size (default: 0.5) + --hires-scale highres fix scale when target size is not set (default: 2.0) + --hires-denoising-strength highres fix second pass denoising strength (default: 0.7) + --increase-ref-index automatically increase the indices of references images based on the order + they are listed (starting with 1). + --disable-auto-resize-ref-image disable auto resize of ref images + --circular enable circular padding on both axes for tileable output + --circularx enable circular padding on x-axis (width) only + --circulary enable circular padding on y-axis (height) only + --disable-image-metadata do not embed generation metadata on image files + --vae-tiling process vae in tiles to reduce memory usage + --temporal-tiling enable temporal tiling for supported video VAE decode + --hires enable highres fix + -s, --seed RNG seed (default: 42, use random seed for < 0) + --sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, + dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, + er_sde, euler_cfg_pp, euler_a_cfg_pp, euler_ge, dpm++2m_sde, dpm++2m_sde_bt, + lms], default: euler for Flux/SD3/Wan, euler_a otherwise + --high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, + dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, + res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp, euler_ge, dpm++2m_sde, + dpm++2m_sde_bt, lms], default: euler for Flux/SD3/Wan, euler_a otherwise + --scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, + sgm_uniform, simple, smoothstep, kl_optimal, lcm, bong_tangent, ltx2, + logit_normal, flux2, flux, beta], alias: normal=discrete, default: + model-specific + --sigmas custom sigma values for the sampler, comma-separated (e.g., + "14.61,7.8,3.5,0.0"). + --hires-sigmas custom sigma values for the highres fix second pass, comma-separated (e.g., + "0.85,0.725,0.421875,0.0"). + --skip-layers layers to skip for SLG steps (default: [7,8,9]) + --high-noise-skip-layers (high noise) layers to skip for SLG steps (default: [7,8,9]) + -r, --ref-image reference image for Flux Kontext or MiniMax-H3 Ref2VA (can be used multiple + times) + --ref-video MiniMax-H3 Ref2VA reference video frame directory at 24 fps (can be used + multiple times) + --ref-video-audio WAV soundtrack paired by index with --ref-video (can be used multiple times) + --ref-audio standalone WAV reference for MiniMax-H3 Ref2VA (can be used multiple times) + --audio driving audio track (Wan2.2 S2V; can be used once) + --cache-mode caching method: 'easycache' (DiT), 'ucache' (UNET), + 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT + Chebyshev+Taylor forecasting) + --cache-option named cache params (key=value format, comma-separated). easycache/ucache: + threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit: + Fn=,Bn=,threshold=,warmup=; spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=. + Examples: "threshold=0.25" or "threshold=1.5,reset=0" + --scm-mask SCM steps mask for cache-dit: comma-separated 0/1 (e.g., + "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache + --scm-policy SCM policy: 'dynamic' (default) or 'static' + --vae-tile-size tile size for vae tiling, format [X]x[Y] (default: 32x32) + --vae-relative-tile-size relative tile size for vae tiling, format [X]x[Y], in fraction of image size + if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size) + --prompt-file path to the file containing the prompt to render + --negative-prompt-file path to the file containing the negative prompt diff --git a/test/fixtures/sdcpp/help/sd-server-version-master-883-137f740.txt b/test/fixtures/sdcpp/help/sd-server-version-master-883-137f740.txt new file mode 100644 index 00000000..2bc92164 --- /dev/null +++ b/test/fixtures/sdcpp/help/sd-server-version-master-883-137f740.txt @@ -0,0 +1 @@ +stable-diffusion.cpp version unknown, commit 137f740 diff --git a/test/fixtures/sdcpp/releases/master-883-137f740-a36f1b1a.json b/test/fixtures/sdcpp/releases/master-883-137f740-a36f1b1a.json new file mode 100644 index 00000000..334ddab3 --- /dev/null +++ b/test/fixtures/sdcpp/releases/master-883-137f740-a36f1b1a.json @@ -0,0 +1,78 @@ +{ + "repo": "AtomicBot-ai/atomic-chat-conf", + "tag": "master-883-137f740-a36f1b1a", + "published_at": "2026-10-07T10:57:19Z", + "target_commitish": "main", + "assets": [ + { + "name": "cudart-sd-bin-win-cu12-x64.zip", + "size": 563452046, + "digest": "sha256:fe20366827d357c00797eebb58244dddab7fd9a348d70090c3871004c320f38d" + }, + { + "name": "sd-master-137f740-bin-Darwin-macOS-26.6.2-arm64.zip", + "size": 34338631, + "digest": "sha256:56a8e614bec7a9659875020e2d3d78fcd5fc53765044ff38c868bfaf3ff3e395" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64-cuda13.zip", + "size": 603693280, + "digest": "sha256:1314b7154b5ffa12add7a5ddf9aa8e928f185109040eebf5060f73c75d83a940" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64.zip", + "size": 20947364, + "digest": "sha256:2fbfb1ef7c758ceedcf1ac51cd8f80107c076c8a13160603fd7a65ac2f215efe" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-cuda12.zip", + "size": 718645269, + "digest": "sha256:d3dbc188ebab9e46485d3eec878612d2b62b49e799386124cddf5ad2be3b8fdd" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.14.0.zip", + "size": 256666973, + "digest": "sha256:c8940e3c0416199b24626b76fd2273a87ac4030b14080af8f23d87783a80e4e6" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-vulkan.zip", + "size": 37401766, + "digest": "sha256:0972d8e978462c38f38371cd8f2376ebb5183fd869c22234a79bed6cc0e0c1bf" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64.zip", + "size": 24520937, + "digest": "sha256:dc1af0b244302cd083d69d974424ba5f2dbe8f1b7f3b04be3db8103477dae47c" + }, + { + "name": "sd-master-137f740-bin-win-cpu-arm64.zip", + "size": 16092295, + "digest": "sha256:093259059dd18870395e869745b492533c38df95f96b804727624240b01f830a" + }, + { + "name": "sd-master-137f740-bin-win-cpu-x64.zip", + "size": 17659940, + "digest": "sha256:df9aae5591e33548a428ae822797eca209a651d41163a05fa8738684e0b6b46b" + }, + { + "name": "sd-master-137f740-bin-win-cuda12-x64.zip", + "size": 331362223, + "digest": "sha256:836823a26c0721c90f0dd88c2bbc931399c24790d4f3333a2487ec4c48c6c074" + }, + { + "name": "sd-master-137f740-bin-win-cuda13-arm64.zip", + "size": 208687376, + "digest": "sha256:9e8c3aad7dea5d7db2999504573b7703db6c06b7ef2dc2203c73404156e647f9" + }, + { + "name": "sd-master-137f740-bin-win-cuda13-x64.zip", + "size": 530924340, + "digest": "sha256:5ce539316bf831404f6142dcaa895ddd5c5398c61bac4aba987c16033ed388fc" + }, + { + "name": "sd-master-137f740-bin-win-vulkan-x64.zip", + "size": 32417764, + "digest": "sha256:a61d12deac8a39d60ab41afe6717c1f770e53da3aa0af8b1567d404c27cc2eaa" + } + ] +} diff --git a/test/fixtures/sdcpp/releases/master-883-137f740.json b/test/fixtures/sdcpp/releases/master-883-137f740.json new file mode 100644 index 00000000..7b00047e --- /dev/null +++ b/test/fixtures/sdcpp/releases/master-883-137f740.json @@ -0,0 +1,103 @@ +{ + "repo": "AtomicBot-ai/stable-diffusion.cpp", + "tag": "master-883-137f740", + "published_at": "2026-10-06T13:11:28Z", + "target_commitish": "atomic", + "assets": [ + { + "name": "SHA256SUMS", + "size": 1710, + "digest": "sha256:5e46b5b783d19676755cdbe0e5c2d614f73ec3c2771eed58a1627c035d59c3bd" + }, + { + "name": "cudart-sd-bin-win-cu12-x64.zip", + "size": 563452046, + "digest": "sha256:fe20366827d357c00797eebb58244dddab7fd9a348d70090c3871004c320f38d" + }, + { + "name": "sd-master-137f740-bin-Darwin-macOS-26.6.2-arm64.zip", + "size": 33966114, + "digest": "sha256:e55bd434c6c2032ac29cffc7ea7c716a94dccc7e888f57b21a84a92526829b17" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64-cuda13.zip", + "size": 603693280, + "digest": "sha256:1314b7154b5ffa12add7a5ddf9aa8e928f185109040eebf5060f73c75d83a940" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-aarch64.zip", + "size": 20947364, + "digest": "sha256:2fbfb1ef7c758ceedcf1ac51cd8f80107c076c8a13160603fd7a65ac2f215efe" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-cuda12.zip", + "size": 718645269, + "digest": "sha256:d3dbc188ebab9e46485d3eec878612d2b62b49e799386124cddf5ad2be3b8fdd" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.14.0.zip", + "size": 256666973, + "digest": "sha256:c8940e3c0416199b24626b76fd2273a87ac4030b14080af8f23d87783a80e4e6" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64-vulkan.zip", + "size": 37401766, + "digest": "sha256:0972d8e978462c38f38371cd8f2376ebb5183fd869c22234a79bed6cc0e0c1bf" + }, + { + "name": "sd-master-137f740-bin-Linux-Ubuntu-24.04-x86_64.zip", + "size": 24520937, + "digest": "sha256:dc1af0b244302cd083d69d974424ba5f2dbe8f1b7f3b04be3db8103477dae47c" + }, + { + "name": "sd-master-137f740-bin-win-cpu-arm64.zip", + "size": 15901594, + "digest": "sha256:9caae9257a861fd591c409d597b54395b73d1e3fbf80065adce9c59867e4c9aa" + }, + { + "name": "sd-master-137f740-bin-win-cpu-x64.zip", + "size": 17372415, + "digest": "sha256:36e84e42d790b22c9b178b5310af1f531f00a12f00dc1d8a940a6394780bfc20" + }, + { + "name": "sd-master-137f740-bin-win-cuda12-x64.zip", + "size": 330580342, + "digest": "sha256:e703c5e5f083474fa8773dd180a93796ddeee3172d1ca5c2c21e0ecb7eb489ec" + }, + { + "name": "sd-master-137f740-bin-win-cuda13-arm64.zip", + "size": 207636824, + "digest": "sha256:e360efe22804fe13e55cfb39a8b15cd07a7d055ec1b9d9ca79e1240daf9ad82d" + }, + { + "name": "sd-master-137f740-bin-win-cuda13-x64.zip", + "size": 528997393, + "digest": "sha256:cb82a19a142b3d0226a2ef561a276295e82ae2b66577fc9f27b933cdc7418a8c" + }, + { + "name": "sd-master-137f740-bin-win-rocm-7.14.0-x64.zip", + "size": 185430115, + "digest": "sha256:8bff9761671f9970de4ed219ca666fa0a1f6b039c0e0d18107a4b4453bf87be7" + }, + { + "name": "sd-master-137f740-bin-win-vulkan-x64.zip", + "size": 31794911, + "digest": "sha256:f0aa3d0d03c08289d549b3f37a1d52c38295e1af4d56021aa7969047bec7bdbc" + }, + { + "name": "verify-spark.cmd", + "size": 785, + "digest": "sha256:28266739fa7babf874a0e872bd2b64527cea7597af3d5ac4611c2f3b927738b3" + }, + { + "name": "verify-spark.ps1", + "size": 11668, + "digest": "sha256:eaabd6450fa64651b326c29aab46822b799a6e7ce48673051c2e8d3c5dd9c261" + }, + { + "name": "verify-spark.sh", + "size": 9834, + "digest": "sha256:522f78fe0d336f25b99952a511e47f75fa040e8892794fd58e68b42b8695d0b0" + } + ] +} diff --git a/test/live/README.md b/test/live/README.md index de3435b8..ec9683f6 100644 --- a/test/live/README.md +++ b/test/live/README.md @@ -67,7 +67,8 @@ Stage 5 providers, each opt-in on its own: `ATOMIC_LIVE_SD_LLM_VISION` (Qwen Image 2.1's Qwen3-VL projector; with it and `ATOMIC_LIVE_SD_FAMILY=qwen-image-2.1` or `qwen-image-2.1-turbo` a `reference` generation from the first output runs as well), `ATOMIC_LIVE_SD_FAMILY` (default `flux.2-klein`; `qwen-image-2.1-turbo` runs its 8-step schedule as `custom_sigmas`), `ATOMIC_LIVE_SD_TAG` (the engine's release tag, default - `master-883-137f740`, the build app v2.0.42 ships). Holds `sd-server --help` to `test/fixtures/sdcpp/required-flags.txt`, + `master-883-137f740`, the build app v2.0.42 ships), `ATOMIC_LIVE_SD_BACKEND=cpu` (every load on the CPU backend, as on + CI runners). Holds `sd-server --help` to the `break` flags of `test/fixtures/sdcpp/engine-contract.json`, alias by alias, then finalize → load → a 256² generation with parsed step progress, PNG, thumbnail and recipe → the OpenAI facade → a hard cancel of a long job → respawn → unload. About five minutes with FLUX.2 Klein Q4_K_M on an M-series Mac. The video block of the same file runs when `ATOMIC_LIVE_SD_VIDEO_MODEL` names a video model (LTX-2.3 distilled by @@ -76,6 +77,11 @@ Stage 5 providers, each opt-in on its own: Wan; `ATOMIC_LIVE_SD_VIDEO_MODE_FLAG=1` also passes `-M vid_gen`): load → a nine-frame clip with parsed step progress, the WebM and its sidecar → `/v1/videos` queue, poll and content → a hard cancel → respawn → unload. It prints the engine's capabilities and the clip's numbers, which is the evidence the ADR of 2026-09-23 leaves open. + `test/live/diffusion-install.test.ts` takes `ATOMIC_LIVE_SD_MANIFEST` (a conf-shaped sd.cpp manifest, a path or an + `https://` URL) and installs its build the way a user gets it: picked for the host, downloaded, checked, unpacked and + probed; a tampered sha256 refused; `--version` naming the tag's commit. `node scripts/fetch-live-fixtures.mjs --engine + sd-cpp --tag --out .live-sd` fetches a release, its manifest and FLUX.2 Klein for both files, and + `npm run engine:check -- --engine sd-cpp --tag ` runs the whole check. Managed TensorRT-LLM install on Linux (`test/live/managed-install.test.ts`, task 2.18): `ATOMIC_LIVE=1` **and** `ATOMIC_LIVE_MANAGED=1`, on a throwaway Linux VM with an NVIDIA GPU, driver ≥ the descriptor's minimum and passwordless diff --git a/test/live/diffusion-install.test.ts b/test/live/diffusion-install.test.ts new file mode 100644 index 00000000..4796b14c --- /dev/null +++ b/test/live/diffusion-install.test.ts @@ -0,0 +1,149 @@ +/** + * Live checks of a real stable-diffusion.cpp build installed the way a user gets it: from a manifest + * of conf's shape (ATOMIC_SDCPP_MANIFEST_URL), the core picks this host's archive, downloads it, + * checks its size and sha256, unpacks it and probes it, then the build answers `--version` with the + * commit its tag names. The engine candidate check points the manifest at the release under test + * (scripts/lib/sdcpp-fixtures.mjs writes it); by hand, conf's own manifest works as well. + * + * Opt in with: + * ATOMIC_LIVE=1 + * ATOMIC_LIVE_SD_MANIFEST=/path/to/manifest.json (or an https:// URL, such as conf's + * backends/sdcpp-manifest.json on raw.githubusercontent.com) + * + * Downloads the archive twice (once refused): a few tens of MB on a CPU host. Nothing is written + * outside the temporary data folders. + */ +import { spawnSync } from 'node:child_process' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { CoreClient } from '../../src/client/index.js' +import { AtomicCore } from '../../src/core/index.js' +import { readInstallRecord, serverBinaryName } from '../../src/diffusion/install.js' +import { parseSdcppManifest } from '../../src/engine-builds/index.js' +import type { SdcppManifest } from '../../src/engine-builds/index.js' +import { makeTmpDataFolder } from '../helpers/tmp-data-folder.js' +import type { TmpDataFolder } from '../helpers/tmp-data-folder.js' + +const MANIFEST = process.env['ATOMIC_LIVE_SD_MANIFEST'] ?? '' +const ENABLED = process.env['ATOMIC_LIVE'] === '1' && MANIFEST !== '' +const manifestUrl = /^https?:\/\//.test(MANIFEST) ? MANIFEST : pathToFileURL(MANIFEST).href + +async function readManifest(): Promise { + const text = manifestUrl.startsWith('file:') + ? await readFile(MANIFEST, 'utf8') + : await (await fetch(manifestUrl)).text() + return parseSdcppManifest(JSON.parse(text)) +} + +/** A core on its own data folder, reading sd.cpp's manifest from `url`. */ +async function coreReading(url: string, prefix: string) { + const data = await makeTmpDataFolder(prefix) + const core = await AtomicCore.create({ + dataFolder: data.root, + controlPort: 0, + env: { ...process.env, ATOMIC_SDCPP_MANIFEST_URL: url }, + }) + return { data, core, client: new CoreClient({ baseUrl: core.control.url, token: core.controlToken }) } +} + +describe.skipIf(!ENABLED)('a real stable-diffusion.cpp build installed from its manifest', () => { + let manifest: SdcppManifest + let data: TmpDataFolder + let core: AtomicCore + let client: CoreClient + + beforeAll(async () => { + manifest = await readManifest() + ;({ data, core, client } = await coreReading(manifestUrl, 'atomic-core-live-sd-install-')) + // As the app does at startup: image generation then reports the engine it can load from. + await client.configureDiffusion({ dataFolder: data.root, idleUnloadSecs: 0 }) + }, 120_000) + + afterAll(async () => { + await core?.shutdown() + await data?.cleanup() + }) + + it( + 'is refused when its sha256 does not match the manifest', + async () => { + // The same manifest with other bytes pinned: the download is checked and nothing is installed. + const dir = await mkdtemp(join(tmpdir(), 'atomic-core-live-sd-tampered-manifest-')) + const tampered = join(dir, 'manifest.json') + await writeFile( + tampered, + JSON.stringify({ + ...manifest, + assets: manifest.assets.map((a) => ({ ...a, sha256: 'f'.repeat(64) })), + }) + ) + const other = await coreReading(pathToFileURL(tampered).href, 'atomic-core-live-sd-tampered-') + try { + await expect( + other.client.installEngineBuild('sd-cpp', { task_id: 'live-sd-tampered' }) + ).rejects.toMatchObject({ + code: 'ENGINE_INSTALL_FAILED', + message: 'The downloaded engine archive does not match the manifest.', + }) + expect((await other.client.engineBuildCatalog('sd-cpp')).installed).toEqual([]) + } finally { + await other.core.shutdown() + await other.data.cleanup() + await rm(dir, { recursive: true, force: true }) + } + }, + 10 * 60_000 + ) + + it( + 'is picked for this host, downloaded, checked, unpacked and probed', + async () => { + const catalog = await client.engineBuildCatalog('sd-cpp', { force: true }) + expect(catalog.manifest_error).toBeNull() + expect(catalog.manifest?.tag).toBe(manifest.tag_name) + const backendId = catalog.host_backend_id + expect(backendId, catalog.host_reason ?? '').not.toBeNull() + + // The probe (`sd-cli --help` naming stable-diffusion.cpp) runs inside the install. + const result = await client.installEngineBuild('sd-cpp', { task_id: 'live-sd-install' }) + expect(result).toMatchObject({ + installed: true, + build: { tag: manifest.tag_name, backend_id: backendId, origin: 'downloaded' }, + }) + expect(result.failed_backend_ids ?? []).toEqual([]) + const after = await client.engineBuildCatalog('sd-cpp') + expect(after.active).toMatchObject({ tag: manifest.tag_name, backend_id: backendId, active: true }) + + const dir = join(data.layout.diffusion.backendsDir, manifest.tag_name, backendId as string) + const record = await readInstallRecord(dir) + expect(record).toMatchObject({ tag: manifest.tag_name, backendId, engine: 'sd-cpp' }) + expect(record?.sha256).toBe(manifest.assets.find((a) => a.backend === backendId)?.sha256) + expect((await client.diffusionStatus()).install).toMatchObject({ state: 'installed', backendId, dir }) + + // Asked again, nothing is downloaded. + expect(await client.installEngineBuild('sd-cpp', { task_id: 'live-sd-install-2' })).toMatchObject({ + installed: false, + reason: 'already-installed', + }) + }, + 10 * 60_000 + ) + + it('names the commit of its tag', async () => { + const active = (await client.engineBuildCatalog('sd-cpp')).active + expect(active).not.toBeNull() + const dir = join(data.layout.diffusion.backendsDir, active?.tag as string, active?.backend_id as string) + const out = spawnSync(join(dir, serverBinaryName(process.platform)), ['--version'], { + cwd: dir, + encoding: 'utf8', + timeout: 120_000, + }) + const text = `${out.stdout}\n${out.stderr}` + expect(out.status, text).toBe(0) + const sha7 = /^master-\d+-([0-9a-f]{7})/.exec(manifest.tag_name)?.[1] + expect(text).toMatch(new RegExp(`stable-diffusion\\.cpp version \\S+, commit ${sha7}`)) + }) +}) diff --git a/test/live/diffusion.test.ts b/test/live/diffusion.test.ts index 5867df3c..4a6f7e60 100644 --- a/test/live/diffusion.test.ts +++ b/test/live/diffusion.test.ts @@ -17,6 +17,9 @@ * ATOMIC_LIVE_SD_FAMILY=flux.2-klein (optional; default flux.2-klein, 4 steps, cfg 1; * qwen-image-2.1-turbo runs its own 8-step schedule) * ATOMIC_LIVE_SD_TAG=master-883-137f740 (optional; the engine's release tag, as the app records it) + * ATOMIC_LIVE_SD_BACKEND=cpu (optional; every load on the CPU backend, through + * ATOMIC_DIFFUSION_BACKEND: a macOS CI runner's + * paravirtual GPU crashes ggml's Metal kernels) * * The video block runs when a video model is named as well (LTX-2.3 distilled by default): * ATOMIC_LIVE_SD_VIDEO_MODEL=/path/to/ltx-2.3-22b-distilled-Q4_K_M.gguf @@ -35,6 +38,8 @@ import { cp, mkdir, readFile, stat } from 'node:fs/promises' import { join } from 'node:path' import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { prepareSdcppTree, writeInstallRecord } from '../../src/diffusion/install.js' +import { findSdOption, parseSdHelp } from '../../scripts/lib/sdcpp-surface.mjs' +import type { SdcppContract } from '../../scripts/lib/sdcpp-surface.mjs' import { CoreClient } from '../../src/client/index.js' import type { CoreEvents, ImageJob, VideoJob } from '../../src/contracts/index.js' import { AtomicCore } from '../../src/core/index.js' @@ -79,6 +84,9 @@ const ENABLED = process.env['ATOMIC_LIVE'] === '1' && ENGINE !== '' && MODEL !== '' && process.platform !== 'win32' /** The build app v2.0.42 ships, and the oldest that runs Qwen Image 2.1 and Krea 2 Turbo. */ const TAG = process.env['ATOMIC_LIVE_SD_TAG'] ?? 'master-883-137f740' +const BACKEND = process.env['ATOMIC_LIVE_SD_BACKEND'] ?? '' +/** The env the core runs with: the backend override when one is asked for. */ +const CORE_ENV = BACKEND ? { env: { ...process.env, ATOMIC_DIFFUSION_BACKEND: BACKEND } } : {} const VIDEO_MODEL = process.env['ATOMIC_LIVE_SD_VIDEO_MODEL'] ?? '' const VIDEO_VAE = process.env['ATOMIC_LIVE_SD_VIDEO_VAE'] ?? '' @@ -113,7 +121,7 @@ describe.skipIf(!ENABLED)('a real stable-diffusion.cpp engine', () => { engineDir = join(data.layout.diffusion.backendsDir, TAG, 'live') await mkdir(engineDir, { recursive: true }) await cp(ENGINE, engineDir, { recursive: true }) - core = await AtomicCore.create({ dataFolder: data.root, controlPort: 0 }) + core = await AtomicCore.create({ dataFolder: data.root, controlPort: 0, ...CORE_ENV }) client = new CoreClient({ baseUrl: core.control.url, token: core.controlToken }) for (const name of ['diffusion:state', 'diffusion:job', 'diffusion:progress', 'diffusion:error'] as const) core.events.on(name, (payload) => events.push({ name, payload })) @@ -127,14 +135,13 @@ describe.skipIf(!ENABLED)('a real stable-diffusion.cpp engine', () => { it('answers --help with every flag the core passes', async () => { const help = spawnSync(join(engineDir, 'sd-server'), ['--help'], { encoding: 'utf8', timeout: 120_000 }) - const text = `${help.stdout}${help.stderr}` - const required = ( - await readFile(new URL('../fixtures/sdcpp/required-flags.txt', import.meta.url), 'utf8') - ) - .split('\n') - .map((line) => line.trim()) - .filter((line) => line !== '' && !line.startsWith('#')) - const missing = required.filter((flag) => !text.includes(flag)) + // Options by their exact aliases, as the surface diff reads them: `--vae-format` is not `--vae`. + const options = parseSdHelp(`${help.stdout}${help.stderr}`) + const contract = JSON.parse( + await readFile(new URL('../fixtures/sdcpp/engine-contract.json', import.meta.url), 'utf8') + ) as SdcppContract + const required = contract.flags.filter((f) => f.severity === 'break').map((f) => f.flag) + const missing = required.filter((flag) => !findSdOption(options, flag)) expect(missing, 'flags the pinned engine no longer knows').toEqual([]) }, 130_000) @@ -223,7 +230,8 @@ describe.skipIf(!ENABLED)('a real stable-diffusion.cpp engine', () => { const edited = await untilJob(referenced.jobId, ['completed', 'failed', 'cancelled'], 15 * 60_000) expect(edited.state, JSON.stringify(edited.error)).toBe('completed') expect(edited.outputs[0]?.recipe.workflow).toBe('reference') - } else { + } else if (FAMILY === 'qwen-image-2.1' || FAMILY === 'qwen-image-2.1-turbo') { + // Without its vision projector Qwen Image 2.1 only creates; FLUX.2 Klein edits without one. expect(capabilities.workflows).not.toContain('edit') } @@ -318,7 +326,7 @@ describe.skipIf(!VIDEO_ENABLED)('a real stable-diffusion.cpp engine serving vide await cp(ENGINE, dir, { recursive: true }) if (process.env['ATOMIC_LIVE_SD_VIDEO_MODE_FLAG'] === '1') process.env['ATOMIC_DIFFUSION_VID_GEN_MODE_FLAG'] = '1' - videoCore = await AtomicCore.create({ dataFolder: videoData.root, controlPort: 0 }) + videoCore = await AtomicCore.create({ dataFolder: videoData.root, controlPort: 0, ...CORE_ENV }) videoClient = new CoreClient({ baseUrl: videoCore.control.url, token: videoCore.controlToken }) for (const name of [ 'diffusion:state',