From 5d3fb37cd501eecc679aca6462c8b594e5caae89 Mon Sep 17 00:00:00 2001 From: Brent Shulman Date: Mon, 5 Oct 2026 18:31:05 -0400 Subject: [PATCH 1/7] feat(webapp,cli): let self-hosted instances require deploy base images Adds DEPLOY_BASE_IMAGES / DEPLOY_BUILD_BASE_IMAGES (runtime=image csv) to the webapp. The deployment initialize response carries the images for the deploy's runtime, and the CLI rewrites the Containerfile to build on them. --- .changeset/instance-deploy-base-images.md | 6 +++ apps/webapp/app/env.server.ts | 2 + apps/webapp/app/routes/api.v1.deployments.ts | 6 +++ apps/webapp/app/v3/deployBaseImages.server.ts | 45 +++++++++++++++++++ apps/webapp/test/deployBaseImages.test.ts | 31 +++++++++++++ docs/self-hosting/env/webapp.mdx | 2 + docs/self-hosting/overview.mdx | 22 +++++++++ packages/cli-v3/src/build/buildWorker.ts | 2 +- packages/cli-v3/src/commands/deploy.ts | 11 ++++- packages/cli-v3/src/deploy/buildImage.test.ts | 35 +++++++++++++++ packages/cli-v3/src/deploy/buildImage.ts | 4 +- packages/core/src/v3/schemas/api.ts | 7 +++ packages/core/src/v3/schemas/build.ts | 2 + 13 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 .changeset/instance-deploy-base-images.md create mode 100644 apps/webapp/app/v3/deployBaseImages.server.ts create mode 100644 apps/webapp/test/deployBaseImages.test.ts diff --git a/.changeset/instance-deploy-base-images.md b/.changeset/instance-deploy-base-images.md new file mode 100644 index 00000000000..16a9390c955 --- /dev/null +++ b/.changeset/instance-deploy-base-images.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Self-hosted instances can require custom base images for deploys, such as FIPS-validated or hardened Node images, with the new `DEPLOY_BASE_IMAGES` webapp setting. The CLI builds on the base images the instance specifies. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index be0106debbf..2697b880e14 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -901,6 +901,8 @@ const EnvironmentSchema = z ), DEPLOY_IMAGE_PLATFORM: z.string().default("linux/amd64"), + DEPLOY_BASE_IMAGES: z.string().optional(), // csv of runtime=image, for example: "node-26=registry.example.com/node-fips:26@sha256:..." + DEPLOY_BUILD_BASE_IMAGES: z.string().optional(), // csv of runtime=image for the build stage DEPLOY_TIMEOUT_MS: z.coerce .number() .int() diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 9b008d5ee75..232b0f63b72 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -9,6 +9,8 @@ import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; +import { env } from "~/env.server"; +import { resolveDeployBaseImages } from "~/v3/deployBaseImages.server"; import { InitializeDeploymentService } from "~/v3/services/initializeDeployment.server"; export async function action({ request, params }: ActionFunctionArgs) { @@ -60,6 +62,10 @@ export async function action({ request, params }: ActionFunctionArgs) { ? { externalBuildData: result.deployment .externalBuildData as InitializeDeploymentResponseBody["externalBuildData"], + baseImages: resolveDeployBaseImages(result.deployment.runtime, { + base: env.DEPLOY_BASE_IMAGES, + buildBase: env.DEPLOY_BUILD_BASE_IMAGES, + }), eventStream: result.eventStream, canceledDeployments: result.canceledDeployments, } diff --git a/apps/webapp/app/v3/deployBaseImages.server.ts b/apps/webapp/app/v3/deployBaseImages.server.ts new file mode 100644 index 00000000000..a5b9a0e66ed --- /dev/null +++ b/apps/webapp/app/v3/deployBaseImages.server.ts @@ -0,0 +1,45 @@ +type BaseImages = { base?: string; buildBase?: string }; + +/** Base images the operator requires for a runtime, from `runtime=image` csv env vars. */ +export function resolveDeployBaseImages( + runtime: string | null | undefined, + config: { base?: string; buildBase?: string } +): BaseImages | undefined { + if (!runtime) { + return undefined; + } + + const base = parseImageMap(config.base)[runtime]; + const buildBase = parseImageMap(config.buildBase)[runtime]; + + if (!base && !buildBase) { + return undefined; + } + + return { + ...(base ? { base } : {}), + ...(buildBase ? { buildBase } : {}), + }; +} + +function parseImageMap(value: string | undefined): Record { + if (!value) { + return {}; + } + + return Object.fromEntries( + value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean) + .flatMap((entry) => { + const separator = entry.indexOf("="); + if (separator <= 0) { + return []; + } + const runtime = entry.slice(0, separator).trim(); + const image = entry.slice(separator + 1).trim(); + return image ? [[runtime, image] as const] : []; + }) + ); +} diff --git a/apps/webapp/test/deployBaseImages.test.ts b/apps/webapp/test/deployBaseImages.test.ts new file mode 100644 index 00000000000..ca92b2345de --- /dev/null +++ b/apps/webapp/test/deployBaseImages.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { resolveDeployBaseImages } from "~/v3/deployBaseImages.server"; + +describe("resolveDeployBaseImages", () => { + it("returns undefined when nothing is configured", () => { + expect(resolveDeployBaseImages("node-26", {})).toBeUndefined(); + }); + + it("returns the images configured for the runtime", () => { + expect( + resolveDeployBaseImages("node-26", { + base: "node-24=acme/node-fips:24@sha256:aaa, node-26=acme/node-fips:26@sha256:bbb", + buildBase: "node-26=acme/node:26-dev@sha256:ccc", + }) + ).toEqual({ base: "acme/node-fips:26@sha256:bbb", buildBase: "acme/node:26-dev@sha256:ccc" }); + }); + + it("returns undefined for runtimes without an entry", () => { + expect(resolveDeployBaseImages("bun", { base: "node-26=acme/node-fips:26" })).toBeUndefined(); + }); + + it("returns undefined when the deployment has no runtime", () => { + expect(resolveDeployBaseImages(null, { base: "node-26=acme/node-fips:26" })).toBeUndefined(); + }); + + it("skips malformed entries", () => { + expect( + resolveDeployBaseImages("node-26", { base: "garbage,=nope,node-26=,node-26=acme/node:26" }) + ).toEqual({ base: "acme/node:26" }); + }); +}); diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 506e95880f7..2a9564b201f 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -97,6 +97,8 @@ mode: "wide" | `DEPLOY_REGISTRY_NAMESPACE` | No | trigger | Deploy registry namespace. | | `DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY` | No | — | Raw IAM policy JSON applied via SetRepositoryPolicy to every ECR repo created by the webapp. Use to grant cross-account pull access to EKS workers when the ECR account is separate from the cluster account. | | `DEPLOY_IMAGE_PLATFORM` | No | linux/amd64 | Deploy image platform, same values as docker `--platform` flag. | +| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on, per runtime, as `runtime=image` csv, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Use for FIPS-validated or hardened images. See [custom base images](/self-hosting/overview#custom-base-images). | +| `DEPLOY_BUILD_BASE_IMAGES` | No | — | Build-stage toolchain images per runtime, same format as `DEPLOY_BASE_IMAGES`. Defaults to the published `-build` images. | | `DEPLOY_TIMEOUT_MS` | No | 480000 (8m) | Deploy timeout (ms). | | `DEPLOY_QUEUE_TIMEOUT_MS` | No | 900000 (15m) | Deploy queue timeout (ms). | | **Object store (S3)** | | | | diff --git a/docs/self-hosting/overview.mdx b/docs/self-hosting/overview.mdx index 0b2192a166b..229fe845629 100644 --- a/docs/self-hosting/overview.mdx +++ b/docs/self-hosting/overview.mdx @@ -100,6 +100,28 @@ All fields are optional. Partial overrides are supported: } ``` +## Custom base images + +Deploys build on the published `triggerdotdev/node` and `triggerdotdev/bun` Debian images. To require a different base for every deploy to your instance, such as a FIPS-validated or hardened Node image, set `DEPLOY_BASE_IMAGES` on the webapp (and optionally `DEPLOY_BUILD_BASE_IMAGES` for the build stage): + +```bash +DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:..." +``` + +The CLI builds with these images for any runtime that has an entry. Runtimes without one keep the published images. With the Helm chart, set them through `webapp.extraEnvVars`. + +You own a custom base image. It must provide: + +- `node` (or `bun`) on `PATH` at the runtime's major version +- `busybox`, `ca-certificates`, `dumb-init`, `git` and `openssl` +- a `node` user +- glibc, so native modules built in the build stage load at runtime + + + `image.pkgs` and build extensions that run `apt-get` (such as `aptGet` and `playwright`) assume a + Debian base. On other distributions, install those packages in your base image instead. + + ## Community support It's dangerous to go alone! Join the self-hosting channel on our [Discord server](https://discord.gg/NQTxt5NA7s). diff --git a/packages/cli-v3/src/build/buildWorker.ts b/packages/cli-v3/src/build/buildWorker.ts index 6ad0da8ba07..299b4fc5d76 100644 --- a/packages/cli-v3/src/build/buildWorker.ts +++ b/packages/cli-v3/src/build/buildWorker.ts @@ -276,7 +276,7 @@ async function readProjectPackageJson(packageJsonPath: string) { return packageJson; } -async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) { +export async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) { if (!buildManifest.runControllerEntryPoint || !buildManifest.indexControllerEntryPoint) { throw new Error("Something went wrong with the build. Aborting deployment. [code 7789]"); } diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index 9dd74ca7eaf..51b7751d04d 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -22,7 +22,7 @@ import { x } from "tinyexec"; import { z } from "zod"; import chalk from "chalk"; import type { CliApiClient } from "../apiClient.js"; -import { buildWorker } from "../build/buildWorker.js"; +import { buildWorker, writeContainerfile } from "../build/buildWorker.js"; import { resolveAlwaysExternal } from "../build/externals.js"; import { createContextArchive, getArchiveSize } from "../deploy/archiveContext.js"; import { createBundleArchive } from "../deploy/bundleArchive.js"; @@ -638,6 +638,15 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { warnAboutCanceledDeployments(deployment.canceledDeployments, options.externalId); + if (deployment.baseImages) { + logger.debug("Using base images required by the server", deployment.baseImages); + + await writeContainerfile(destination.path, { + ...buildManifest, + image: { ...buildManifest.image, ...deployment.baseImages }, + }); + } + // When `externalBuildData` is not present the deployment implicitly goes into the local build path // which is used in self-hosted setups. There are a few subtle differences between local builds for the cloud // and local builds for self-hosted setups. We need to make the separation of the two paths clearer to avoid confusion. diff --git a/packages/cli-v3/src/deploy/buildImage.test.ts b/packages/cli-v3/src/deploy/buildImage.test.ts index 128b2261733..00409e3f519 100644 --- a/packages/cli-v3/src/deploy/buildImage.test.ts +++ b/packages/cli-v3/src/deploy/buildImage.test.ts @@ -233,4 +233,39 @@ describe("generateContainerfile", () => { expect(excludeCopy).toBeGreaterThan(codeStage); } ); + + it.each(["node", "bun"] as BuildRuntime[])( + "uses the configured base and build images on %s", + async (runtime) => { + const containerfile = await generateContainerfile({ + runtime, + build: {}, + image: { + base: "acme/node-fips:26@sha256:abc", + buildBase: "acme/node:26-dev@sha256:def", + }, + indexScript: "index.js", + entrypoint: "entrypoint.js", + }); + + expect(containerfile).toContain("FROM acme/node-fips:26@sha256:abc AS base"); + expect(containerfile).toContain("FROM acme/node:26-dev@sha256:def AS build"); + expect(containerfile).toContain("FROM base AS final"); + expect(containerfile).not.toContain(BASE_IMAGE[runtime]); + expect(containerfile).not.toContain(BUILD_IMAGE[runtime]); + } + ); + + it("keeps the published build image when only the base is overridden", async () => { + const containerfile = await generateContainerfile({ + runtime: "node-26", + build: {}, + image: { base: "acme/node-fips:26@sha256:abc" }, + indexScript: "index.js", + entrypoint: "entrypoint.js", + }); + + expect(containerfile).toContain("FROM acme/node-fips:26@sha256:abc AS base"); + expect(containerfile).toContain(`FROM ${BUILD_IMAGE["node-26"]} AS build`); + }); }); diff --git a/packages/cli-v3/src/deploy/buildImage.ts b/packages/cli-v3/src/deploy/buildImage.ts index 47d0c84d63e..3ed093ef610 100644 --- a/packages/cli-v3/src/deploy/buildImage.ts +++ b/packages/cli-v3/src/deploy/buildImage.ts @@ -808,14 +808,14 @@ RUN apt-get update && \\ apt-get install -y --no-install-recommends ${TOOLCHAIN_PACKAGES} && \\ apt-get clean && \\ rm -rf /var/lib/apt/lists/*` - : `FROM ${BUILD_IMAGE[options.runtime]} AS build + : `FROM ${options.image?.buildBase ?? BUILD_IMAGE[options.runtime]} AS build ENV DEBIAN_FRONTEND=noninteractive${ userPackages.length > 0 ? `\n\n${aptInstall(userPackages, { repair: false })}` : "" }`; return { - baseImage: BASE_IMAGE[options.runtime], + baseImage: options.image?.base ?? BASE_IMAGE[options.runtime], buildStage, customization, buildArgs, diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 3a6c46621a8..fb1092af254 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -849,6 +849,13 @@ export const InitializeDeploymentResponseBody = z.object({ outcome: z.enum(["created", "existing"]).optional(), isPromoted: z.boolean().optional(), externalBuildData: ExternalBuildData.optional().nullable(), + /** Base images the instance operator requires for this deployment's runtime */ + baseImages: z + .object({ + base: z.string().optional(), + buildBase: z.string().optional(), + }) + .optional(), canceledDeployments: z.array(z.object({ version: z.string(), shortCode: z.string() })).optional(), eventStream: z .object({ diff --git a/packages/core/src/v3/schemas/build.ts b/packages/core/src/v3/schemas/build.ts index f31f0882b8f..b966313ae7e 100644 --- a/packages/core/src/v3/schemas/build.ts +++ b/packages/core/src/v3/schemas/build.ts @@ -85,6 +85,8 @@ export const BuildManifest = z.object({ .object({ pkgs: z.array(z.string()).optional(), instructions: z.array(z.string()).optional(), + base: z.string().optional(), + buildBase: z.string().optional(), }) .optional(), otelImportHook: z From 2a0c5ba3c45559e784ab4d5c433069058fc42f5a Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:28:46 +0100 Subject: [PATCH 2/7] feat(cli,webapp): harden instance deploy base images Validate DEPLOY_BASE_IMAGES and DEPLOY_BUILD_BASE_IMAGES when the webapp starts: entries must name a known runtime and a digest-pinned image, with no duplicate runtimes. Invalid values fail startup instead of silently falling back to the published images. Honor the build-stage image when build extensions add image instructions: the build stage is created from the configured build image and the instructions are replayed on it, instead of being derived from the base with a toolchain install. Apply the server's base images on --from-bundle deploys by regenerating the bundle's Containerfile, and print the images in the deploy output. Docs: split the Node and Bun base image requirements, describe the validation rules and the paths the setting applies to. --- apps/webapp/app/env.server.ts | 11 ++- apps/webapp/app/v3/deployBaseImages.server.ts | 91 +++++++++++++------ apps/webapp/test/deployBaseImages.test.ts | 89 ++++++++++++++---- docs/self-hosting/env/webapp.mdx | 4 +- docs/self-hosting/overview.mdx | 19 +++- packages/cli-v3/src/commands/deploy.ts | 57 ++++++++++-- packages/cli-v3/src/deploy/buildImage.test.ts | 45 +++++++++ packages/cli-v3/src/deploy/buildImage.ts | 22 +++-- packages/core/src/v3/schemas/api.ts | 1 - 9 files changed, 266 insertions(+), 73 deletions(-) diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 2697b880e14..bd95c5bd326 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -3,6 +3,7 @@ import { MachinePresetName } from "@trigger.dev/core/v3"; import { parseNaturalLanguageDurationInMs } from "@trigger.dev/core/v3/isomorphic"; import { BoolEnv } from "./utils/boolEnv"; import { isValidDatabaseUrl } from "./utils/db"; +import { parseDeployBaseImages } from "~/v3/deployBaseImages.server"; import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; import { isValidRegex } from "./utils/regex"; import { isValidDuration } from "./services/realtime/duration.server"; @@ -901,8 +902,14 @@ const EnvironmentSchema = z ), DEPLOY_IMAGE_PLATFORM: z.string().default("linux/amd64"), - DEPLOY_BASE_IMAGES: z.string().optional(), // csv of runtime=image, for example: "node-26=registry.example.com/node-fips:26@sha256:..." - DEPLOY_BUILD_BASE_IMAGES: z.string().optional(), // csv of runtime=image for the build stage + DEPLOY_BASE_IMAGES: z + .string() + .optional() + .transform((v) => parseDeployBaseImages(v, "DEPLOY_BASE_IMAGES")), + DEPLOY_BUILD_BASE_IMAGES: z + .string() + .optional() + .transform((v) => parseDeployBaseImages(v, "DEPLOY_BUILD_BASE_IMAGES")), DEPLOY_TIMEOUT_MS: z.coerce .number() .int() diff --git a/apps/webapp/app/v3/deployBaseImages.server.ts b/apps/webapp/app/v3/deployBaseImages.server.ts index a5b9a0e66ed..06c38e153e3 100644 --- a/apps/webapp/app/v3/deployBaseImages.server.ts +++ b/apps/webapp/app/v3/deployBaseImages.server.ts @@ -1,16 +1,71 @@ -type BaseImages = { base?: string; buildBase?: string }; +import { BuildRuntime } from "@trigger.dev/core/v3"; + +type BaseImageMap = Partial>; + +const DIGEST_PINNED = /@sha256:[a-f0-9]{64}$/; + +function invalidSegment(envVarName: string, segment: string, reason: string): Error { + return new Error(`${envVarName}: ${reason} in "${segment}"`); +} + +export function parseDeployBaseImages(value: string | undefined, envVarName: string): BaseImageMap { + const result: BaseImageMap = {}; + + if (!value) { + return result; + } + + for (const segment of value.split(",").map((s) => s.trim())) { + if (!segment) { + continue; + } + + const separator = segment.indexOf("="); + if (separator === -1) { + throw invalidSegment(envVarName, segment, "expected runtime=image"); + } + + const runtimeName = segment.slice(0, separator).trim(); + const image = segment.slice(separator + 1).trim(); + + const runtime = BuildRuntime.safeParse(runtimeName); + if (!runtime.success) { + throw invalidSegment( + envVarName, + segment, + `unknown runtime "${runtimeName}" (expected one of ${BuildRuntime.options.join(", ")})` + ); + } + + if (!image) { + throw invalidSegment(envVarName, segment, "missing image"); + } + + if (!DIGEST_PINNED.test(image)) { + throw invalidSegment(envVarName, segment, "image must be pinned by digest (@sha256:<64 hex chars>)"); + } + + if (runtime.data in result) { + throw invalidSegment(envVarName, segment, `duplicate runtime "${runtimeName}"`); + } + + result[runtime.data] = image; + } + + return result; +} -/** Base images the operator requires for a runtime, from `runtime=image` csv env vars. */ export function resolveDeployBaseImages( runtime: string | null | undefined, - config: { base?: string; buildBase?: string } -): BaseImages | undefined { - if (!runtime) { + config: { base: BaseImageMap; buildBase: BaseImageMap } +): { base?: string; buildBase?: string } | undefined { + const parsedRuntime = BuildRuntime.safeParse(runtime); + if (!parsedRuntime.success) { return undefined; } - const base = parseImageMap(config.base)[runtime]; - const buildBase = parseImageMap(config.buildBase)[runtime]; + const base = config.base[parsedRuntime.data]; + const buildBase = config.buildBase[parsedRuntime.data]; if (!base && !buildBase) { return undefined; @@ -21,25 +76,3 @@ export function resolveDeployBaseImages( ...(buildBase ? { buildBase } : {}), }; } - -function parseImageMap(value: string | undefined): Record { - if (!value) { - return {}; - } - - return Object.fromEntries( - value - .split(",") - .map((entry) => entry.trim()) - .filter(Boolean) - .flatMap((entry) => { - const separator = entry.indexOf("="); - if (separator <= 0) { - return []; - } - const runtime = entry.slice(0, separator).trim(); - const image = entry.slice(separator + 1).trim(); - return image ? [[runtime, image] as const] : []; - }) - ); -} diff --git a/apps/webapp/test/deployBaseImages.test.ts b/apps/webapp/test/deployBaseImages.test.ts index ca92b2345de..28037b3ee72 100644 --- a/apps/webapp/test/deployBaseImages.test.ts +++ b/apps/webapp/test/deployBaseImages.test.ts @@ -1,31 +1,86 @@ import { describe, expect, it } from "vitest"; -import { resolveDeployBaseImages } from "~/v3/deployBaseImages.server"; +import { parseDeployBaseImages, resolveDeployBaseImages } from "~/v3/deployBaseImages.server"; -describe("resolveDeployBaseImages", () => { - it("returns undefined when nothing is configured", () => { - expect(resolveDeployBaseImages("node-26", {})).toBeUndefined(); +const digestA = `sha256:${"a".repeat(64)}`; +const digestB = `sha256:${"b".repeat(64)}`; +const digestC = `sha256:${"c".repeat(64)}`; + +describe("parseDeployBaseImages", () => { + it("returns an empty map for undefined and empty values", () => { + expect(parseDeployBaseImages(undefined, "DEPLOY_BASE_IMAGES")).toEqual({}); + expect(parseDeployBaseImages("", "DEPLOY_BASE_IMAGES")).toEqual({}); + expect(parseDeployBaseImages(" , ,", "DEPLOY_BASE_IMAGES")).toEqual({}); }); - it("returns the images configured for the runtime", () => { + it("parses multiple entries and trims whitespace", () => { expect( - resolveDeployBaseImages("node-26", { - base: "node-24=acme/node-fips:24@sha256:aaa, node-26=acme/node-fips:26@sha256:bbb", - buildBase: "node-26=acme/node:26-dev@sha256:ccc", - }) - ).toEqual({ base: "acme/node-fips:26@sha256:bbb", buildBase: "acme/node:26-dev@sha256:ccc" }); + parseDeployBaseImages( + ` node-24 = acme/node-fips:24@${digestA} , bun=acme/bun:1@${digestB},`, + "DEPLOY_BASE_IMAGES" + ) + ).toEqual({ + "node-24": `acme/node-fips:24@${digestA}`, + bun: `acme/bun:1@${digestB}`, + }); + }); + + it.each([ + ["missing =", "garbage"], + ["unknown runtime", `node-23=acme/node:23@${digestA}`], + ["empty image", "node-24="], + ["missing digest", "node-24=acme/node:24"], + ["duplicate runtime", `node-24=acme/a@${digestA},node-24=acme/b@${digestB}`], + ])("throws naming the env var and segment: %s", (_name, value) => { + const segments = value.split(","); + const offending = segments[segments.length - 1]!; + + let error: Error | undefined; + try { + parseDeployBaseImages(`node-22=acme/ok@${digestC},${value}`, "DEPLOY_BUILD_BASE_IMAGES"); + } catch (e) { + error = e as Error; + } + + expect(error).toBeInstanceOf(Error); + expect(error?.message).toContain("DEPLOY_BUILD_BASE_IMAGES"); + expect(error?.message).toContain(offending); }); +}); + +describe("resolveDeployBaseImages", () => { + const base = { "node-26": `acme/node-fips:26@${digestA}` } as const; + const buildBase = { "node-26": `acme/node:26-dev@${digestB}` } as const; - it("returns undefined for runtimes without an entry", () => { - expect(resolveDeployBaseImages("bun", { base: "node-26=acme/node-fips:26" })).toBeUndefined(); + it("returns undefined for an unknown runtime", () => { + expect(resolveDeployBaseImages("node-23", { base, buildBase })).toBeUndefined(); }); it("returns undefined when the deployment has no runtime", () => { - expect(resolveDeployBaseImages(null, { base: "node-26=acme/node-fips:26" })).toBeUndefined(); + expect(resolveDeployBaseImages(null, { base, buildBase })).toBeUndefined(); + expect(resolveDeployBaseImages(undefined, { base, buildBase })).toBeUndefined(); }); - it("skips malformed entries", () => { - expect( - resolveDeployBaseImages("node-26", { base: "garbage,=nope,node-26=,node-26=acme/node:26" }) - ).toEqual({ base: "acme/node:26" }); + it("returns undefined when the runtime has no entries", () => { + expect(resolveDeployBaseImages("bun", { base, buildBase })).toBeUndefined(); + expect(resolveDeployBaseImages("node-26", { base: {}, buildBase: {} })).toBeUndefined(); + }); + + it("returns both images", () => { + expect(resolveDeployBaseImages("node-26", { base, buildBase })).toEqual({ + base: base["node-26"], + buildBase: buildBase["node-26"], + }); + }); + + it("returns only the base image", () => { + expect(resolveDeployBaseImages("node-26", { base, buildBase: {} })).toEqual({ + base: base["node-26"], + }); + }); + + it("returns only the build base image", () => { + expect(resolveDeployBaseImages("node-26", { base: {}, buildBase })).toEqual({ + buildBase: buildBase["node-26"], + }); }); }); diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 2a9564b201f..0eadafe4042 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -97,8 +97,8 @@ mode: "wide" | `DEPLOY_REGISTRY_NAMESPACE` | No | trigger | Deploy registry namespace. | | `DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY` | No | — | Raw IAM policy JSON applied via SetRepositoryPolicy to every ECR repo created by the webapp. Use to grant cross-account pull access to EKS workers when the ECR account is separate from the cluster account. | | `DEPLOY_IMAGE_PLATFORM` | No | linux/amd64 | Deploy image platform, same values as docker `--platform` flag. | -| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on, per runtime, as `runtime=image` csv, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Use for FIPS-validated or hardened images. See [custom base images](/self-hosting/overview#custom-base-images). | -| `DEPLOY_BUILD_BASE_IMAGES` | No | — | Build-stage toolchain images per runtime, same format as `DEPLOY_BASE_IMAGES`. Defaults to the published `-build` images. | +| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on. Comma-separated `runtime=image@sha256:` entries, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Runtimes: `node`, `node-22`, `node-24`, `node-26`, `bun`. The digest is required. An invalid value prevents the webapp from starting. See [custom base images](/self-hosting/overview#custom-base-images). | +| `DEPLOY_BUILD_BASE_IMAGES` | No | — | Build-stage images per runtime. Same format and validation as `DEPLOY_BASE_IMAGES`. Defaults to the published `-build` images. | | `DEPLOY_TIMEOUT_MS` | No | 480000 (8m) | Deploy timeout (ms). | | `DEPLOY_QUEUE_TIMEOUT_MS` | No | 900000 (15m) | Deploy queue timeout (ms). | | **Object store (S3)** | | | | diff --git a/docs/self-hosting/overview.mdx b/docs/self-hosting/overview.mdx index 229fe845629..7af3bec2552 100644 --- a/docs/self-hosting/overview.mdx +++ b/docs/self-hosting/overview.mdx @@ -105,18 +105,29 @@ All fields are optional. Partial overrides are supported: Deploys build on the published `triggerdotdev/node` and `triggerdotdev/bun` Debian images. To require a different base for every deploy to your instance, such as a FIPS-validated or hardened Node image, set `DEPLOY_BASE_IMAGES` on the webapp (and optionally `DEPLOY_BUILD_BASE_IMAGES` for the build stage): ```bash -DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:..." +DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:<64-character-digest>" ``` -The CLI builds with these images for any runtime that has an entry. Runtimes without one keep the published images. With the Helm chart, set them through `webapp.extraEnvVars`. +Entries are comma-separated `runtime=image@sha256:`. The runtimes are `node`, `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. With the Helm chart, set them through `webapp.extraEnvVars`. -You own a custom base image. It must provide: +The CLI builds with these images for any runtime that has an entry. Runtimes without one keep the published images. It applies to deploys built with the CLI's local build path, which is what self-hosted instances use. `--from-bundle` deploys regenerate the bundle's Containerfile with these images. This is a self-hosting setting and does not apply to Trigger.dev Cloud. -- `node` (or `bun`) on `PATH` at the runtime's major version +You own a custom base image. A Node image must provide: + +- `node` on `PATH` at the runtime's major version - `busybox`, `ca-certificates`, `dumb-init`, `git` and `openssl` - a `node` user - glibc, so native modules built in the build stage load at runtime +A Bun image must provide: + +- `bun` and `node` on `PATH`, because the final stage starts the app with `dumb-init node` +- `busybox`, `ca-certificates`, `dumb-init`, `git` and `openssl` +- a `bun` user +- glibc, so native modules built in the build stage load at runtime + +A `DEPLOY_BUILD_BASE_IMAGES` image must contain everything the base image provides, plus the toolchain the published build images include: `python3`, `make` and `g++`. When a project's build extensions add image instructions, those instructions are replayed on the build-stage image, so it must be able to run them. Without a `DEPLOY_BUILD_BASE_IMAGES` entry, the build stage is created from your base image and the toolchain is installed with `apt-get`, so the base image must be Debian-based for those projects. + `image.pkgs` and build extensions that run `apt-get` (such as `aptGet` and `playwright`) assume a Debian base. On other distributions, install those packages in your base image instead. diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index 51b7751d04d..3f2156b69a1 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -638,14 +638,12 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { warnAboutCanceledDeployments(deployment.canceledDeployments, options.externalId); - if (deployment.baseImages) { - logger.debug("Using base images required by the server", deployment.baseImages); - - await writeContainerfile(destination.path, { - ...buildManifest, - image: { ...buildManifest.image, ...deployment.baseImages }, - }); - } + await applyServerBaseImages({ + baseImages: deployment.baseImages, + outputPath: destination.path, + buildManifest, + options, + }); // When `externalBuildData` is not present the deployment implicitly goes into the local build path // which is used in self-hosted setups. There are a few subtle differences between local builds for the cloud @@ -1211,6 +1209,42 @@ function buildDeploymentLinks({ }; } +async function applyServerBaseImages({ + baseImages, + outputPath, + buildManifest, + options, +}: { + baseImages: InitializeDeploymentResponseBody["baseImages"]; + outputPath: string; + buildManifest: BuildManifest; + options: DeployCommandOptions; +}) { + if (!baseImages) { + return; + } + + logger.debug("Using base images required by the server", baseImages); + + const required = [ + baseImages.base ? `base ${baseImages.base}` : undefined, + baseImages.buildBase ? `build ${baseImages.buildBase}` : undefined, + ].filter(Boolean); + + const message = `Building on base images required by this instance: ${required.join(", ")}`; + + if (options.plain) { + console.log(message); + } else { + log.info(message); + } + + await writeContainerfile(outputPath, { + ...buildManifest, + image: { ...buildManifest.image, ...baseImages }, + }); +} + function warnAboutSkippedBuild(externalId: string | undefined, isPromoted: boolean | undefined) { prettyWarning( "Environment variables were not synced because nothing was built.", @@ -2274,6 +2308,13 @@ async function handleFromBundleDeploy({ existingDeploymentId ); + await applyServerBaseImages({ + baseImages: deployment.baseImages, + outputPath: bundlePath, + buildManifest: bundleManifest, + options, + }); + // Fail fast if we know local builds will fail const buildxResult = await x("docker", ["buildx", "version"]); diff --git a/packages/cli-v3/src/deploy/buildImage.test.ts b/packages/cli-v3/src/deploy/buildImage.test.ts index 00409e3f519..9611138209d 100644 --- a/packages/cli-v3/src/deploy/buildImage.test.ts +++ b/packages/cli-v3/src/deploy/buildImage.test.ts @@ -268,4 +268,49 @@ describe("generateContainerfile", () => { expect(containerfile).toContain("FROM acme/node-fips:26@sha256:abc AS base"); expect(containerfile).toContain(`FROM ${BUILD_IMAGE["node-26"]} AS build`); }); + + it("builds on the configured build image and replays instructions there", async () => { + const containerfile = await generateContainerfile({ + runtime: "node-26", + build: {}, + image: { + base: "acme/node-fips:26@sha256:abc", + buildBase: "acme/node:26-dev@sha256:def", + pkgs: ["jq"], + instructions: ["RUN echo first > /etc/first", "RUN echo second > /etc/second"], + }, + indexScript: "index.js", + entrypoint: "entrypoint.js", + }); + + const buildStage = containerfile.slice(containerfile.indexOf("AS build")); + + expect(containerfile).toContain("FROM acme/node-fips:26@sha256:abc AS base"); + expect(containerfile).toContain("FROM acme/node:26-dev@sha256:def AS build"); + expect(containerfile).not.toContain("FROM base AS build"); + expect(containerfile).not.toContain(TOOLCHAIN_PACKAGES); + expect(buildStage).toContain("apt-get install -y --no-install-recommends --allow-downgrades jq"); + expect(buildStage).toContain("RUN echo first > /etc/first"); + expect(buildStage).toContain("RUN echo second > /etc/second"); + expect(containerfile.indexOf("RUN echo first > /etc/first")).toBeLessThan( + containerfile.indexOf("AS build") + ); + }); + + it("builds from the base stage when instructions have no configured build image", async () => { + const containerfile = await generateContainerfile({ + runtime: "node-26", + build: {}, + image: { + base: "acme/node-fips:26@sha256:abc", + instructions: ["RUN echo custom > /etc/marker"], + }, + indexScript: "index.js", + entrypoint: "entrypoint.js", + }); + + expect(containerfile).toContain("FROM acme/node-fips:26@sha256:abc AS base"); + expect(containerfile).toContain("FROM base AS build"); + expect(containerfile).toContain(TOOLCHAIN_PACKAGES); + }); }); diff --git a/packages/cli-v3/src/deploy/buildImage.ts b/packages/cli-v3/src/deploy/buildImage.ts index 3ed093ef610..63360b81e9d 100644 --- a/packages/cli-v3/src/deploy/buildImage.ts +++ b/packages/cli-v3/src/deploy/buildImage.ts @@ -799,20 +799,22 @@ const parseGenerateOptions = (options: GenerateContainerfileOptions) => { .filter(Boolean) .join("\n\n"); - // Instructions run once (FROM base) since their downloads are unbounded; - // package-only projects keep the prebuilt toolchain and repeat the small install - const buildStage = baseInstructions - ? `FROM base AS build + const prebuiltBuildStage = `FROM ${options.image?.buildBase ?? BUILD_IMAGE[options.runtime]} AS build + +ENV DEBIAN_FRONTEND=noninteractive${ + userPackages.length > 0 ? `\n\n${aptInstall(userPackages, { repair: false })}` : "" + }`; + + const buildStage = !baseInstructions // FROM base runs instructions once: unbounded downloads + ? prebuiltBuildStage + : options.image?.buildBase + ? `${prebuiltBuildStage}\n\n${baseInstructions}` + : `FROM base AS build RUN apt-get update && \\ apt-get install -y --no-install-recommends ${TOOLCHAIN_PACKAGES} && \\ apt-get clean && \\ - rm -rf /var/lib/apt/lists/*` - : `FROM ${options.image?.buildBase ?? BUILD_IMAGE[options.runtime]} AS build - -ENV DEBIAN_FRONTEND=noninteractive${ - userPackages.length > 0 ? `\n\n${aptInstall(userPackages, { repair: false })}` : "" - }`; + rm -rf /var/lib/apt/lists/*`; return { baseImage: options.image?.base ?? BASE_IMAGE[options.runtime], diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index fb1092af254..a7dec4bae40 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -849,7 +849,6 @@ export const InitializeDeploymentResponseBody = z.object({ outcome: z.enum(["created", "existing"]).optional(), isPromoted: z.boolean().optional(), externalBuildData: ExternalBuildData.optional().nullable(), - /** Base images the instance operator requires for this deployment's runtime */ baseImages: z .object({ base: z.string().optional(), From d5c12391c20c5f5d0c66e57b80827fc663a29c6d Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:38:46 +0100 Subject: [PATCH 3/7] feat(cli,webapp): enforce instance base images end to end Reject initialize-deployment requests from CLIs that cannot apply the instance's base images, so the setting is enforced rather than advisory. The CLI declares support on the paths that can honour it, and fails with a clear error on --native-build and --local-bundle, which cannot. Return the base images on the get-deployment response as well, so deploys that attach to an existing deployment build on them too. Validate image refs with one shared schema in core on both the server and the CLI: a single token pinned by digest. Reject the runtime alias node in favour of the concrete runtime keys, and report every invalid env entry in one error at startup. Build the custom build stage with the same customization block as the base stage, so instructions and package installs run in the same order. --- .changeset/instance-deploy-base-images.md | 2 +- apps/webapp/app/env.server.ts | 16 ++++- .../api.v1.deployments.$deploymentId.ts | 6 ++ apps/webapp/app/v3/deployBaseImages.server.ts | 52 ++++++++------ .../services/initializeDeployment.server.ts | 14 ++++ apps/webapp/test/deployBaseImages.test.ts | 69 ++++++++++++++----- docs/self-hosting/env/webapp.mdx | 2 +- docs/self-hosting/overview.mdx | 4 +- packages/cli-v3/src/commands/deploy.ts | 16 +++++ packages/cli-v3/src/deploy/buildImage.test.ts | 17 ++++- packages/cli-v3/src/deploy/buildImage.ts | 2 +- packages/core/src/v3/schemas/api.ts | 18 +++-- 12 files changed, 165 insertions(+), 53 deletions(-) diff --git a/.changeset/instance-deploy-base-images.md b/.changeset/instance-deploy-base-images.md index 16a9390c955..4baf5db50d5 100644 --- a/.changeset/instance-deploy-base-images.md +++ b/.changeset/instance-deploy-base-images.md @@ -3,4 +3,4 @@ "trigger.dev": patch --- -Self-hosted instances can require custom base images for deploys, such as FIPS-validated or hardened Node images, with the new `DEPLOY_BASE_IMAGES` webapp setting. The CLI builds on the base images the instance specifies. +Self-hosted instances can require custom deploy base images per runtime via the new `DEPLOY_BASE_IMAGES` and `DEPLOY_BUILD_BASE_IMAGES` webapp settings. The CLI builds on the images the instance specifies, and older CLIs are rejected with an upgrade message. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index bd95c5bd326..57d7ae3f6c0 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -15,6 +15,18 @@ function durationString() { return z.string().refine(isValidDuration, "must be a duration like 7d, 30d, 365d, 1h, 1y"); } +const parseDeployBaseImagesEnv = ( + value: string | undefined, + envVarName: string, + ctx: z.RefinementCtx +) => { + const { images, errors } = parseDeployBaseImages(value, envVarName); + for (const message of errors) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message }); + } + return errors.length > 0 ? z.NEVER : images; +}; + // Parses a CSV of machine preset names (e.g. "small-1x,small-2x") into a // non-empty array of MachinePresetName. Used by COMPUTE_TEMPLATE_MACHINE_PRESETS // and its _REQUIRED variant. Adds zod issues for empty input or unknown names. @@ -905,11 +917,11 @@ const EnvironmentSchema = z DEPLOY_BASE_IMAGES: z .string() .optional() - .transform((v) => parseDeployBaseImages(v, "DEPLOY_BASE_IMAGES")), + .transform((v, ctx) => parseDeployBaseImagesEnv(v, "DEPLOY_BASE_IMAGES", ctx)), DEPLOY_BUILD_BASE_IMAGES: z .string() .optional() - .transform((v) => parseDeployBaseImages(v, "DEPLOY_BUILD_BASE_IMAGES")), + .transform((v, ctx) => parseDeployBaseImagesEnv(v, "DEPLOY_BUILD_BASE_IMAGES", ctx)), DEPLOY_TIMEOUT_MS: z.coerce .number() .int() diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts index e63bb9542d3..c5fa3d7224e 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts @@ -2,6 +2,8 @@ import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; import { type GetDeploymentResponseBody } from "@trigger.dev/core/v3"; import { z } from "zod"; import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { resolveDeployBaseImages } from "~/v3/deployBaseImages.server"; import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -65,6 +67,10 @@ export async function loader({ request, params }: LoaderFunctionArgs) { externalId: deployment.externalId ?? undefined, externalBuildData: deployment.externalBuildData as GetDeploymentResponseBody["externalBuildData"], + baseImages: resolveDeployBaseImages(deployment.runtime, { + base: env.DEPLOY_BASE_IMAGES, + buildBase: env.DEPLOY_BUILD_BASE_IMAGES, + }), errorData: deployment.errorData as GetDeploymentResponseBody["errorData"], canceledReason: deployment.canceledReason, worker: deployment.worker diff --git a/apps/webapp/app/v3/deployBaseImages.server.ts b/apps/webapp/app/v3/deployBaseImages.server.ts index 06c38e153e3..8c9c25cc53a 100644 --- a/apps/webapp/app/v3/deployBaseImages.server.ts +++ b/apps/webapp/app/v3/deployBaseImages.server.ts @@ -1,18 +1,18 @@ -import { BuildRuntime } from "@trigger.dev/core/v3"; +import { BuildRuntime, DeployBaseImageRef } from "@trigger.dev/core/v3"; type BaseImageMap = Partial>; -const DIGEST_PINNED = /@sha256:[a-f0-9]{64}$/; +export type ParsedDeployBaseImages = { images: BaseImageMap; errors: string[] }; -function invalidSegment(envVarName: string, segment: string, reason: string): Error { - return new Error(`${envVarName}: ${reason} in "${segment}"`); -} - -export function parseDeployBaseImages(value: string | undefined, envVarName: string): BaseImageMap { - const result: BaseImageMap = {}; +export function parseDeployBaseImages( + value: string | undefined, + envVarName: string +): ParsedDeployBaseImages { + const images: BaseImageMap = {}; + const errors: string[] = []; if (!value) { - return result; + return { images, errors }; } for (const segment of value.split(",").map((s) => s.trim())) { @@ -20,39 +20,47 @@ export function parseDeployBaseImages(value: string | undefined, envVarName: str continue; } + const fail = (reason: string) => errors.push(`${envVarName}: ${reason} in "${segment}"`); + const separator = segment.indexOf("="); if (separator === -1) { - throw invalidSegment(envVarName, segment, "expected runtime=image"); + fail("expected runtime=image"); + continue; } const runtimeName = segment.slice(0, separator).trim(); const image = segment.slice(separator + 1).trim(); + if (runtimeName === "node") { + fail('runtime "node" is an alias; use the concrete runtime (node-22, node-24, node-26)'); + continue; + } + const runtime = BuildRuntime.safeParse(runtimeName); if (!runtime.success) { - throw invalidSegment( - envVarName, - segment, - `unknown runtime "${runtimeName}" (expected one of ${BuildRuntime.options.join(", ")})` - ); + fail(`unknown runtime "${runtimeName}" (expected one of ${BuildRuntime.options.join(", ")})`); + continue; } if (!image) { - throw invalidSegment(envVarName, segment, "missing image"); + fail("missing image"); + continue; } - if (!DIGEST_PINNED.test(image)) { - throw invalidSegment(envVarName, segment, "image must be pinned by digest (@sha256:<64 hex chars>)"); + if (!DeployBaseImageRef.safeParse(image).success) { + fail("image must be image@sha256:<64 hex chars> with no whitespace before the digest"); + continue; } - if (runtime.data in result) { - throw invalidSegment(envVarName, segment, `duplicate runtime "${runtimeName}"`); + if (runtime.data in images) { + fail(`duplicate runtime "${runtimeName}"`); + continue; } - result[runtime.data] = image; + images[runtime.data] = image; } - return result; + return { images, errors }; } export function resolveDeployBaseImages( diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index 31e8b6546ab..c40da1e0a97 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -14,6 +14,7 @@ import { generateFriendlyId } from "../friendlyIdentifiers"; import { createRemoteImageBuild, remoteBuildsEnabled } from "../remoteImageBuilder.server"; import { BaseService, ServiceValidationError } from "./baseService.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; +import { resolveDeployBaseImages } from "../deployBaseImages.server"; import { getDeploymentImageRef } from "../getDeploymentImageRef.server"; import { tryCatch } from "@trigger.dev/core"; import { getRegistryConfig } from "../registryConfig.server"; @@ -147,6 +148,19 @@ export class InitializeDeploymentService extends BaseService { throw new ServiceValidationError("UNMANAGED deployments are not supported"); } + if ( + resolveDeployBaseImages(runtime, { + base: env.DEPLOY_BASE_IMAGES, + buildBase: env.DEPLOY_BUILD_BASE_IMAGES, + }) && + payload.supportsInstanceBaseImages !== true + ) { + throw new ServiceValidationError( + "This instance requires custom deploy base images, which this version of the CLI cannot apply. Upgrade the trigger.dev CLI and deploy again.", + 400 + ); + } + // Upgrade the project to engine "V2" if it's not already. This should cover cases where people deploy to V2 without running dev first. if (payload.type === "MANAGED" && environment.project.engine === "V1") { await this._prisma.project.update({ diff --git a/apps/webapp/test/deployBaseImages.test.ts b/apps/webapp/test/deployBaseImages.test.ts index 28037b3ee72..fcd947bce8a 100644 --- a/apps/webapp/test/deployBaseImages.test.ts +++ b/apps/webapp/test/deployBaseImages.test.ts @@ -5,11 +5,14 @@ const digestA = `sha256:${"a".repeat(64)}`; const digestB = `sha256:${"b".repeat(64)}`; const digestC = `sha256:${"c".repeat(64)}`; +const hex64 = "a".repeat(64); + describe("parseDeployBaseImages", () => { it("returns an empty map for undefined and empty values", () => { - expect(parseDeployBaseImages(undefined, "DEPLOY_BASE_IMAGES")).toEqual({}); - expect(parseDeployBaseImages("", "DEPLOY_BASE_IMAGES")).toEqual({}); - expect(parseDeployBaseImages(" , ,", "DEPLOY_BASE_IMAGES")).toEqual({}); + const empty = { images: {}, errors: [] }; + expect(parseDeployBaseImages(undefined, "DEPLOY_BASE_IMAGES")).toEqual(empty); + expect(parseDeployBaseImages("", "DEPLOY_BASE_IMAGES")).toEqual(empty); + expect(parseDeployBaseImages(" , ,", "DEPLOY_BASE_IMAGES")).toEqual(empty); }); it("parses multiple entries and trims whitespace", () => { @@ -19,31 +22,61 @@ describe("parseDeployBaseImages", () => { "DEPLOY_BASE_IMAGES" ) ).toEqual({ - "node-24": `acme/node-fips:24@${digestA}`, - bun: `acme/bun:1@${digestB}`, + images: { + "node-24": `acme/node-fips:24@${digestA}`, + bun: `acme/bun:1@${digestB}`, + }, + errors: [], + }); + }); + + it("accepts a registry with a port and a tag before the digest", () => { + const image = `registry.example.com:5000/ns/img:tag@sha256:${hex64}`; + expect(parseDeployBaseImages(`node-24=${image}`, "DEPLOY_BASE_IMAGES")).toEqual({ + images: { "node-24": image }, + errors: [], }); }); it.each([ ["missing =", "garbage"], ["unknown runtime", `node-23=acme/node:23@${digestA}`], + ["node alias", `node=acme/node:24@${digestA}`], ["empty image", "node-24="], ["missing digest", "node-24=acme/node:24"], + ["flag before image", `node-24=--platform=linux/arm64 acme/node@sha256:${hex64}`], + ["bare digest", `node-24=@sha256:${hex64}`], + ["newline in image", `node-24=acme/node\nx@sha256:${hex64}`], ["duplicate runtime", `node-24=acme/a@${digestA},node-24=acme/b@${digestB}`], - ])("throws naming the env var and segment: %s", (_name, value) => { + ])("reports an error naming the env var and segment: %s", (_name, value) => { const segments = value.split(","); - const offending = segments[segments.length - 1]!; - - let error: Error | undefined; - try { - parseDeployBaseImages(`node-22=acme/ok@${digestC},${value}`, "DEPLOY_BUILD_BASE_IMAGES"); - } catch (e) { - error = e as Error; - } - - expect(error).toBeInstanceOf(Error); - expect(error?.message).toContain("DEPLOY_BUILD_BASE_IMAGES"); - expect(error?.message).toContain(offending); + const offending = segments[segments.length - 1]!.trim(); + + const { images, errors } = parseDeployBaseImages( + `node-22=acme/ok@${digestC},${value}`, + "DEPLOY_BUILD_BASE_IMAGES" + ); + + expect(images["node-22"]).toBe(`acme/ok@${digestC}`); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("DEPLOY_BUILD_BASE_IMAGES"); + expect(errors[0]).toContain(offending); + }); + + it("explains that node is an alias", () => { + const { errors } = parseDeployBaseImages(`node=acme/node@${digestA}`, "DEPLOY_BASE_IMAGES"); + expect(errors[0]).toContain('runtime "node" is an alias; use the concrete runtime'); + }); + + it("reports every bad segment", () => { + const { errors } = parseDeployBaseImages( + `garbage,node-23=acme/node@${digestA},bun=acme/bun`, + "DEPLOY_BASE_IMAGES" + ); + expect(errors).toHaveLength(3); + expect(errors[0]).toContain("garbage"); + expect(errors[1]).toContain("node-23"); + expect(errors[2]).toContain("bun=acme/bun"); }); }); diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 0eadafe4042..0761ace385f 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -97,7 +97,7 @@ mode: "wide" | `DEPLOY_REGISTRY_NAMESPACE` | No | trigger | Deploy registry namespace. | | `DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY` | No | — | Raw IAM policy JSON applied via SetRepositoryPolicy to every ECR repo created by the webapp. Use to grant cross-account pull access to EKS workers when the ECR account is separate from the cluster account. | | `DEPLOY_IMAGE_PLATFORM` | No | linux/amd64 | Deploy image platform, same values as docker `--platform` flag. | -| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on. Comma-separated `runtime=image@sha256:` entries, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Runtimes: `node`, `node-22`, `node-24`, `node-26`, `bun`. The digest is required. An invalid value prevents the webapp from starting. See [custom base images](/self-hosting/overview#custom-base-images). | +| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on. Comma-separated `runtime=image@sha256:` entries, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Runtimes: `node-22`, `node-24`, `node-26`, `bun`. The digest is required. Projects with `runtime: "node"` resolve to the current default Node runtime (`node-24` today), so set that key for them. Deploys from CLI versions that cannot apply the images are rejected with an upgrade message, and deploys using `--native-build` or `--local-bundle` fail when base images are configured. An invalid value prevents the webapp from starting. See [custom base images](/self-hosting/overview#custom-base-images). | | `DEPLOY_BUILD_BASE_IMAGES` | No | — | Build-stage images per runtime. Same format and validation as `DEPLOY_BASE_IMAGES`. Defaults to the published `-build` images. | | `DEPLOY_TIMEOUT_MS` | No | 480000 (8m) | Deploy timeout (ms). | | `DEPLOY_QUEUE_TIMEOUT_MS` | No | 900000 (15m) | Deploy queue timeout (ms). | diff --git a/docs/self-hosting/overview.mdx b/docs/self-hosting/overview.mdx index 7af3bec2552..11806ed586b 100644 --- a/docs/self-hosting/overview.mdx +++ b/docs/self-hosting/overview.mdx @@ -108,7 +108,9 @@ Deploys build on the published `triggerdotdev/node` and `triggerdotdev/bun` Debi DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:<64-character-digest>" ``` -Entries are comma-separated `runtime=image@sha256:`. The runtimes are `node`, `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. With the Helm chart, set them through `webapp.extraEnvVars`. +Entries are comma-separated `runtime=image@sha256:`. The runtimes are `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. With the Helm chart, set them through `webapp.extraEnvVars`. + +Projects with `runtime: "node"` in their config resolve to the current default Node runtime (`node-24` today), so set that key for them. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade. Deploys using `--native-build` or `--local-bundle` fail with an error when base images are configured, since those paths cannot apply them. The CLI builds with these images for any runtime that has an entry. Runtimes without one keep the published images. It applies to deploys built with the CLI's local build path, which is what self-hosted instances use. `--from-bundle` deploys regenerate the bundle's Containerfile with these images. This is a self-hosting setting and does not apply to Trigger.dev Cloud. diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index 3f2156b69a1..887e084d9ad 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -591,6 +591,7 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { triggeredVia: getTriggeredVia(), externalId: options.externalId, force: options.force, + supportsInstanceBaseImages: true, }, envVars.TRIGGER_EXISTING_DEPLOYMENT_ID ); @@ -1497,6 +1498,13 @@ async function handleNativeBuildServerDeploy({ const deployment = initializeDeploymentResult.data; + if (deployment.baseImages) { + $deploymentSpinner.stop("Failed to initialize deployment"); + throw new Error( + "This instance requires custom deploy base images, which cannot be applied with --native-build. Deploy without that flag." + ); + } + const rawDeploymentLink = `${dashboardUrl}/projects/v3/${config.project}/deployments/${deployment.shortCode}`; const rawTestLink = `${dashboardUrl}/projects/v3/${config.project}/test?environment=${ options.env === "prod" ? "prod" : "stg" @@ -1830,6 +1838,13 @@ async function handleLocalBundleDeploy({ const deployment = initializeDeploymentResult.data; + if (deployment.baseImages) { + $deploymentSpinner.stop("Failed to initialize deployment"); + throw new Error( + "This instance requires custom deploy base images, which cannot be applied with --local-bundle. Deploy without that flag." + ); + } + const rawDeploymentLink = `${dashboardUrl}/projects/v3/${config.project}/deployments/${deployment.shortCode}`; const rawTestLink = `${dashboardUrl}/projects/v3/${config.project}/test?environment=${ options.env === "prod" ? "prod" : "stg" @@ -2304,6 +2319,7 @@ async function handleFromBundleDeploy({ isLocalBuild: true, isNativeBuild: false, triggeredVia: getTriggeredVia(), + supportsInstanceBaseImages: true, }, existingDeploymentId ); diff --git a/packages/cli-v3/src/deploy/buildImage.test.ts b/packages/cli-v3/src/deploy/buildImage.test.ts index 9611138209d..e771258bb3c 100644 --- a/packages/cli-v3/src/deploy/buildImage.test.ts +++ b/packages/cli-v3/src/deploy/buildImage.test.ts @@ -289,12 +289,27 @@ describe("generateContainerfile", () => { expect(containerfile).toContain("FROM acme/node:26-dev@sha256:def AS build"); expect(containerfile).not.toContain("FROM base AS build"); expect(containerfile).not.toContain(TOOLCHAIN_PACKAGES); - expect(buildStage).toContain("apt-get install -y --no-install-recommends --allow-downgrades jq"); + expect(buildStage).toContain( + "apt-get install -y --no-install-recommends --allow-downgrades jq" + ); expect(buildStage).toContain("RUN echo first > /etc/first"); expect(buildStage).toContain("RUN echo second > /etc/second"); expect(containerfile.indexOf("RUN echo first > /etc/first")).toBeLessThan( containerfile.indexOf("AS build") ); + + const buildFrom = "FROM acme/node:26-dev@sha256:def AS build"; + const baseEnv = "ENV DEBIAN_FRONTEND=noninteractive\n\n"; + const baseStart = containerfile.indexOf(baseEnv) + baseEnv.length; + const baseCustomization = containerfile.slice( + baseStart, + containerfile.indexOf(buildFrom) - "\n\n".length + ); + + expect(buildStage.indexOf("RUN echo second > /etc/second")).toBeLessThan( + buildStage.indexOf("apt-get install") + ); + expect(containerfile).toContain(`${buildFrom}\n\n${baseEnv}${baseCustomization}\n\n`); }); it("builds from the base stage when instructions have no configured build image", async () => { diff --git a/packages/cli-v3/src/deploy/buildImage.ts b/packages/cli-v3/src/deploy/buildImage.ts index 63360b81e9d..7998b7f83ac 100644 --- a/packages/cli-v3/src/deploy/buildImage.ts +++ b/packages/cli-v3/src/deploy/buildImage.ts @@ -808,7 +808,7 @@ ENV DEBIAN_FRONTEND=noninteractive${ const buildStage = !baseInstructions // FROM base runs instructions once: unbounded downloads ? prebuiltBuildStage : options.image?.buildBase - ? `${prebuiltBuildStage}\n\n${baseInstructions}` + ? `FROM ${options.image.buildBase} AS build\n\nENV DEBIAN_FRONTEND=noninteractive\n\n${customization}` : `FROM base AS build RUN apt-get update && \\ diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index a7dec4bae40..d0f699a56fd 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -838,6 +838,15 @@ export const CreateArtifactResponseBody = z.object({ export type CreateArtifactResponseBody = z.infer; +export const DeployBaseImageRef = z.string().regex(/^[^\s@]+@sha256:[a-f0-9]{64}$/); + +export const DeployBaseImages = z.object({ + base: DeployBaseImageRef.optional(), + buildBase: DeployBaseImageRef.optional(), +}); + +export type DeployBaseImages = z.infer; + export const InitializeDeploymentResponseBody = z.object({ id: z.string(), contentHash: z.string(), @@ -849,12 +858,7 @@ export const InitializeDeploymentResponseBody = z.object({ outcome: z.enum(["created", "existing"]).optional(), isPromoted: z.boolean().optional(), externalBuildData: ExternalBuildData.optional().nullable(), - baseImages: z - .object({ - base: z.string().optional(), - buildBase: z.string().optional(), - }) - .optional(), + baseImages: DeployBaseImages.optional(), canceledDeployments: z.array(z.object({ version: z.string(), shortCode: z.string() })).optional(), eventStream: z .object({ @@ -883,6 +887,7 @@ const InitializeDeploymentRequestBodyBase = z.object({ buildId: z.string().optional(), externalId: ExternalDeploymentId, force: z.boolean().optional(), + supportsInstanceBaseImages: z.boolean().optional(), }); type BaseOutput = z.output; @@ -1033,6 +1038,7 @@ export const GetDeploymentResponseBody = z.object({ */ externalId: z.string().optional(), externalBuildData: ExternalBuildData.optional().nullable(), + baseImages: DeployBaseImages.optional(), errorData: DeploymentErrorData.nullish(), canceledReason: z.string().nullish(), worker: z From 701591d2a7365ba36801c5acee301ea7369ed215 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:43:53 +0100 Subject: [PATCH 4/7] fix(webapp,cli): reject native builds server-side when base images are set Native builds and local bundles cannot apply the instance's base images and never declared support, so the server was rejecting them with the message meant for outdated CLIs. Reject them with their own message on the server and drop the CLI-side checks that could never run. Docs: describe what the build stage uses without a build image entry, note that --from-bundle rewrites the Containerfile inside the bundle directory, and shorten the env table rows. --- .../services/initializeDeployment.server.ts | 20 ++++++++++++------- docs/self-hosting/env/webapp.mdx | 4 ++-- docs/self-hosting/overview.mdx | 6 +++--- packages/cli-v3/src/commands/deploy.ts | 14 ------------- 4 files changed, 18 insertions(+), 26 deletions(-) diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index c40da1e0a97..4ee0784c17f 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -148,13 +148,19 @@ export class InitializeDeploymentService extends BaseService { throw new ServiceValidationError("UNMANAGED deployments are not supported"); } - if ( - resolveDeployBaseImages(runtime, { - base: env.DEPLOY_BASE_IMAGES, - buildBase: env.DEPLOY_BUILD_BASE_IMAGES, - }) && - payload.supportsInstanceBaseImages !== true - ) { + const requiredBaseImages = resolveDeployBaseImages(runtime, { + base: env.DEPLOY_BASE_IMAGES, + buildBase: env.DEPLOY_BUILD_BASE_IMAGES, + }); + + if (requiredBaseImages && payload.isNativeBuild) { + throw new ServiceValidationError( + "This instance requires custom deploy base images, which native builds cannot apply. Deploy without --native-build or --local-bundle.", + 400 + ); + } + + if (requiredBaseImages && payload.supportsInstanceBaseImages !== true) { throw new ServiceValidationError( "This instance requires custom deploy base images, which this version of the CLI cannot apply. Upgrade the trigger.dev CLI and deploy again.", 400 diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 0761ace385f..164cefb1f14 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -97,8 +97,8 @@ mode: "wide" | `DEPLOY_REGISTRY_NAMESPACE` | No | trigger | Deploy registry namespace. | | `DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY` | No | — | Raw IAM policy JSON applied via SetRepositoryPolicy to every ECR repo created by the webapp. Use to grant cross-account pull access to EKS workers when the ECR account is separate from the cluster account. | | `DEPLOY_IMAGE_PLATFORM` | No | linux/amd64 | Deploy image platform, same values as docker `--platform` flag. | -| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on. Comma-separated `runtime=image@sha256:` entries, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Runtimes: `node-22`, `node-24`, `node-26`, `bun`. The digest is required. Projects with `runtime: "node"` resolve to the current default Node runtime (`node-24` today), so set that key for them. Deploys from CLI versions that cannot apply the images are rejected with an upgrade message, and deploys using `--native-build` or `--local-bundle` fail when base images are configured. An invalid value prevents the webapp from starting. See [custom base images](/self-hosting/overview#custom-base-images). | -| `DEPLOY_BUILD_BASE_IMAGES` | No | — | Build-stage images per runtime. Same format and validation as `DEPLOY_BASE_IMAGES`. Defaults to the published `-build` images. | +| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on. Comma-separated `runtime=image@sha256:` entries, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Runtimes: `node-22`, `node-24`, `node-26`, `bun`. The digest is required. An invalid value prevents the webapp from starting. See [custom base images](/self-hosting/overview#custom-base-images). | +| `DEPLOY_BUILD_BASE_IMAGES` | No | — | Build-stage images per runtime, same format and validation. Without an entry the build stage uses the published build image, or the base image plus a toolchain install when build extensions add image instructions. | | `DEPLOY_TIMEOUT_MS` | No | 480000 (8m) | Deploy timeout (ms). | | `DEPLOY_QUEUE_TIMEOUT_MS` | No | 900000 (15m) | Deploy queue timeout (ms). | | **Object store (S3)** | | | | diff --git a/docs/self-hosting/overview.mdx b/docs/self-hosting/overview.mdx index 11806ed586b..5133d1f719f 100644 --- a/docs/self-hosting/overview.mdx +++ b/docs/self-hosting/overview.mdx @@ -110,9 +110,9 @@ DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:<64-charact Entries are comma-separated `runtime=image@sha256:`. The runtimes are `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. With the Helm chart, set them through `webapp.extraEnvVars`. -Projects with `runtime: "node"` in their config resolve to the current default Node runtime (`node-24` today), so set that key for them. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade. Deploys using `--native-build` or `--local-bundle` fail with an error when base images are configured, since those paths cannot apply them. +Projects with `runtime: "node"` in their config resolve to the current default Node runtime (`node-24` today), so set that key for them. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade. Deploys using `--native-build` or `--local-bundle` are rejected when base images are configured, since those paths cannot apply them. -The CLI builds with these images for any runtime that has an entry. Runtimes without one keep the published images. It applies to deploys built with the CLI's local build path, which is what self-hosted instances use. `--from-bundle` deploys regenerate the bundle's Containerfile with these images. This is a self-hosting setting and does not apply to Trigger.dev Cloud. +The CLI builds with these images for any runtime that has an entry. Runtimes without one keep the published images. It applies to deploys built with the CLI's local build path, which is what self-hosted instances use. `--from-bundle` deploys regenerate the Containerfile inside the bundle directory with these images. This is a self-hosting setting and does not apply to Trigger.dev Cloud. You own a custom base image. A Node image must provide: @@ -128,7 +128,7 @@ A Bun image must provide: - a `bun` user - glibc, so native modules built in the build stage load at runtime -A `DEPLOY_BUILD_BASE_IMAGES` image must contain everything the base image provides, plus the toolchain the published build images include: `python3`, `make` and `g++`. When a project's build extensions add image instructions, those instructions are replayed on the build-stage image, so it must be able to run them. Without a `DEPLOY_BUILD_BASE_IMAGES` entry, the build stage is created from your base image and the toolchain is installed with `apt-get`, so the base image must be Debian-based for those projects. +A `DEPLOY_BUILD_BASE_IMAGES` image must contain everything the base image provides, plus the toolchain the published build images include: `python3`, `make` and `g++`. When a project's build extensions add image instructions, those instructions are replayed on the build-stage image, so it must be able to run them. Without a `DEPLOY_BUILD_BASE_IMAGES` entry, the build stage uses the published build image. The exception is a project whose build extensions add image instructions: its build stage is created from your base image and the toolchain is installed with `apt-get`, so the base image must be Debian-based for those projects. To avoid the published build image entirely, set both `DEPLOY_BASE_IMAGES` and `DEPLOY_BUILD_BASE_IMAGES`. `image.pkgs` and build extensions that run `apt-get` (such as `aptGet` and `playwright`) assume a diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index 887e084d9ad..0334da53a88 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -1498,13 +1498,6 @@ async function handleNativeBuildServerDeploy({ const deployment = initializeDeploymentResult.data; - if (deployment.baseImages) { - $deploymentSpinner.stop("Failed to initialize deployment"); - throw new Error( - "This instance requires custom deploy base images, which cannot be applied with --native-build. Deploy without that flag." - ); - } - const rawDeploymentLink = `${dashboardUrl}/projects/v3/${config.project}/deployments/${deployment.shortCode}`; const rawTestLink = `${dashboardUrl}/projects/v3/${config.project}/test?environment=${ options.env === "prod" ? "prod" : "stg" @@ -1838,13 +1831,6 @@ async function handleLocalBundleDeploy({ const deployment = initializeDeploymentResult.data; - if (deployment.baseImages) { - $deploymentSpinner.stop("Failed to initialize deployment"); - throw new Error( - "This instance requires custom deploy base images, which cannot be applied with --local-bundle. Deploy without that flag." - ); - } - const rawDeploymentLink = `${dashboardUrl}/projects/v3/${config.project}/deployments/${deployment.shortCode}`; const rawTestLink = `${dashboardUrl}/projects/v3/${config.project}/test?environment=${ options.env === "prod" ? "prod" : "stg" From fbfdb575914430f55c62e0d7068154cb3c5ddb45 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:25:43 +0100 Subject: [PATCH 5/7] refactor(cli,webapp): simplify instance base image handling --- apps/webapp/app/v3/deployBaseImages.server.ts | 13 ++---------- apps/webapp/test/deployBaseImages.test.ts | 15 +++++-------- packages/cli-v3/src/commands/deploy.ts | 2 -- packages/cli-v3/src/deploy/buildImage.test.ts | 5 ----- packages/cli-v3/src/deploy/buildImage.ts | 21 ++++++++----------- 5 files changed, 16 insertions(+), 40 deletions(-) diff --git a/apps/webapp/app/v3/deployBaseImages.server.ts b/apps/webapp/app/v3/deployBaseImages.server.ts index 8c9c25cc53a..6f98784de25 100644 --- a/apps/webapp/app/v3/deployBaseImages.server.ts +++ b/apps/webapp/app/v3/deployBaseImages.server.ts @@ -2,12 +2,10 @@ import { BuildRuntime, DeployBaseImageRef } from "@trigger.dev/core/v3"; type BaseImageMap = Partial>; -export type ParsedDeployBaseImages = { images: BaseImageMap; errors: string[] }; - export function parseDeployBaseImages( value: string | undefined, envVarName: string -): ParsedDeployBaseImages { +): { images: BaseImageMap; errors: string[] } { const images: BaseImageMap = {}; const errors: string[] = []; @@ -75,12 +73,5 @@ export function resolveDeployBaseImages( const base = config.base[parsedRuntime.data]; const buildBase = config.buildBase[parsedRuntime.data]; - if (!base && !buildBase) { - return undefined; - } - - return { - ...(base ? { base } : {}), - ...(buildBase ? { buildBase } : {}), - }; + return base || buildBase ? { base, buildBase } : undefined; } diff --git a/apps/webapp/test/deployBaseImages.test.ts b/apps/webapp/test/deployBaseImages.test.ts index fcd947bce8a..e60b53d4b5a 100644 --- a/apps/webapp/test/deployBaseImages.test.ts +++ b/apps/webapp/test/deployBaseImages.test.ts @@ -5,8 +5,6 @@ const digestA = `sha256:${"a".repeat(64)}`; const digestB = `sha256:${"b".repeat(64)}`; const digestC = `sha256:${"c".repeat(64)}`; -const hex64 = "a".repeat(64); - describe("parseDeployBaseImages", () => { it("returns an empty map for undefined and empty values", () => { const empty = { images: {}, errors: [] }; @@ -31,7 +29,7 @@ describe("parseDeployBaseImages", () => { }); it("accepts a registry with a port and a tag before the digest", () => { - const image = `registry.example.com:5000/ns/img:tag@sha256:${hex64}`; + const image = `registry.example.com:5000/ns/img:tag@${digestA}`; expect(parseDeployBaseImages(`node-24=${image}`, "DEPLOY_BASE_IMAGES")).toEqual({ images: { "node-24": image }, errors: [], @@ -44,9 +42,9 @@ describe("parseDeployBaseImages", () => { ["node alias", `node=acme/node:24@${digestA}`], ["empty image", "node-24="], ["missing digest", "node-24=acme/node:24"], - ["flag before image", `node-24=--platform=linux/arm64 acme/node@sha256:${hex64}`], - ["bare digest", `node-24=@sha256:${hex64}`], - ["newline in image", `node-24=acme/node\nx@sha256:${hex64}`], + ["flag before image", `node-24=--platform=linux/arm64 acme/node@${digestA}`], + ["bare digest", `node-24=@${digestA}`], + ["newline in image", `node-24=acme/node\nx@${digestA}`], ["duplicate runtime", `node-24=acme/a@${digestA},node-24=acme/b@${digestB}`], ])("reports an error naming the env var and segment: %s", (_name, value) => { const segments = value.split(","); @@ -84,11 +82,8 @@ describe("resolveDeployBaseImages", () => { const base = { "node-26": `acme/node-fips:26@${digestA}` } as const; const buildBase = { "node-26": `acme/node:26-dev@${digestB}` } as const; - it("returns undefined for an unknown runtime", () => { + it("returns undefined for a missing or unknown runtime", () => { expect(resolveDeployBaseImages("node-23", { base, buildBase })).toBeUndefined(); - }); - - it("returns undefined when the deployment has no runtime", () => { expect(resolveDeployBaseImages(null, { base, buildBase })).toBeUndefined(); expect(resolveDeployBaseImages(undefined, { base, buildBase })).toBeUndefined(); }); diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index 0334da53a88..406698db56b 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -1225,8 +1225,6 @@ async function applyServerBaseImages({ return; } - logger.debug("Using base images required by the server", baseImages); - const required = [ baseImages.base ? `base ${baseImages.base}` : undefined, baseImages.buildBase ? `build ${baseImages.buildBase}` : undefined, diff --git a/packages/cli-v3/src/deploy/buildImage.test.ts b/packages/cli-v3/src/deploy/buildImage.test.ts index e771258bb3c..061c6013d6c 100644 --- a/packages/cli-v3/src/deploy/buildImage.test.ts +++ b/packages/cli-v3/src/deploy/buildImage.test.ts @@ -292,8 +292,6 @@ describe("generateContainerfile", () => { expect(buildStage).toContain( "apt-get install -y --no-install-recommends --allow-downgrades jq" ); - expect(buildStage).toContain("RUN echo first > /etc/first"); - expect(buildStage).toContain("RUN echo second > /etc/second"); expect(containerfile.indexOf("RUN echo first > /etc/first")).toBeLessThan( containerfile.indexOf("AS build") ); @@ -306,9 +304,6 @@ describe("generateContainerfile", () => { containerfile.indexOf(buildFrom) - "\n\n".length ); - expect(buildStage.indexOf("RUN echo second > /etc/second")).toBeLessThan( - buildStage.indexOf("apt-get install") - ); expect(containerfile).toContain(`${buildFrom}\n\n${baseEnv}${baseCustomization}\n\n`); }); diff --git a/packages/cli-v3/src/deploy/buildImage.ts b/packages/cli-v3/src/deploy/buildImage.ts index 7998b7f83ac..ac41a18d1d7 100644 --- a/packages/cli-v3/src/deploy/buildImage.ts +++ b/packages/cli-v3/src/deploy/buildImage.ts @@ -799,22 +799,19 @@ const parseGenerateOptions = (options: GenerateContainerfileOptions) => { .filter(Boolean) .join("\n\n"); - const prebuiltBuildStage = `FROM ${options.image?.buildBase ?? BUILD_IMAGE[options.runtime]} AS build - -ENV DEBIAN_FRONTEND=noninteractive${ - userPackages.length > 0 ? `\n\n${aptInstall(userPackages, { repair: false })}` : "" - }`; - - const buildStage = !baseInstructions // FROM base runs instructions once: unbounded downloads - ? prebuiltBuildStage - : options.image?.buildBase - ? `FROM ${options.image.buildBase} AS build\n\nENV DEBIAN_FRONTEND=noninteractive\n\n${customization}` - : `FROM base AS build + // Instructions run once (FROM base) since their downloads are unbounded; + // package-only projects keep the prebuilt toolchain and repeat the small install + const buildStage = + baseInstructions && !options.image?.buildBase + ? `FROM base AS build RUN apt-get update && \\ apt-get install -y --no-install-recommends ${TOOLCHAIN_PACKAGES} && \\ apt-get clean && \\ - rm -rf /var/lib/apt/lists/*`; + rm -rf /var/lib/apt/lists/*` + : `FROM ${options.image?.buildBase ?? BUILD_IMAGE[options.runtime]} AS build + +ENV DEBIAN_FRONTEND=noninteractive${customization ? `\n\n${customization}` : ""}`; return { baseImage: options.image?.base ?? BASE_IMAGE[options.runtime], From a6bd65af8235cedf0e2589a2b574820997d4c566 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:25:48 +0100 Subject: [PATCH 6/7] docs: trim custom base images section --- docs/self-hosting/env/webapp.mdx | 4 ++-- docs/self-hosting/overview.mdx | 21 ++++++--------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index d508c6db1f6..786ec339e9b 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -98,8 +98,8 @@ mode: "wide" | `DEPLOY_REGISTRY_NAMESPACE` | No | trigger | Deploy registry namespace. | | `DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY` | No | — | Raw IAM policy JSON applied via SetRepositoryPolicy to every ECR repo created by the webapp. Use to grant cross-account pull access to EKS workers when the ECR account is separate from the cluster account. | | `DEPLOY_IMAGE_PLATFORM` | No | linux/amd64 | Deploy image platform, same values as docker `--platform` flag. | -| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on. Comma-separated `runtime=image@sha256:` entries, e.g. `node-26=registry.example.com/node-fips:26@sha256:...`. Runtimes: `node-22`, `node-24`, `node-26`, `bun`. The digest is required. An invalid value prevents the webapp from starting. See [custom base images](/self-hosting/overview#custom-base-images). | -| `DEPLOY_BUILD_BASE_IMAGES` | No | — | Build-stage images per runtime, same format and validation. Without an entry the build stage uses the published build image, or the base image plus a toolchain install when build extensions add image instructions. | +| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on, as comma-separated `runtime=image@sha256:`. See [custom base images](/self-hosting/overview#custom-base-images). | +| `DEPLOY_BUILD_BASE_IMAGES` | No | — | Build-stage images, in the same format as `DEPLOY_BASE_IMAGES`. See [custom base images](/self-hosting/overview#custom-base-images). | | `DEPLOY_TIMEOUT_MS` | No | 480000 (8m) | Deploy timeout (ms). | | `DEPLOY_QUEUE_TIMEOUT_MS` | No | 900000 (15m) | Deploy queue timeout (ms). | | **Object store (S3)** | | | | diff --git a/docs/self-hosting/overview.mdx b/docs/self-hosting/overview.mdx index 5133d1f719f..f592bc2f787 100644 --- a/docs/self-hosting/overview.mdx +++ b/docs/self-hosting/overview.mdx @@ -108,27 +108,18 @@ Deploys build on the published `triggerdotdev/node` and `triggerdotdev/bun` Debi DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:<64-character-digest>" ``` -Entries are comma-separated `runtime=image@sha256:`. The runtimes are `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. With the Helm chart, set them through `webapp.extraEnvVars`. +Entries are comma-separated `runtime=image@sha256:`. The runtimes are `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. Projects with `runtime: "node"` in their config resolve to the current default Node runtime (`node-24` today), so set that key for them. With the Helm chart, set the variables through `webapp.extraEnvVars`. -Projects with `runtime: "node"` in their config resolve to the current default Node runtime (`node-24` today), so set that key for them. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade. Deploys using `--native-build` or `--local-bundle` are rejected when base images are configured, since those paths cannot apply them. +Runtimes with an entry build on that image; the others keep the published images. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade, and so are `--native-build` and `--local-bundle` deploys. This is a self-hosting setting and does not apply to Trigger.dev Cloud. -The CLI builds with these images for any runtime that has an entry. Runtimes without one keep the published images. It applies to deploys built with the CLI's local build path, which is what self-hosted instances use. `--from-bundle` deploys regenerate the Containerfile inside the bundle directory with these images. This is a self-hosting setting and does not apply to Trigger.dev Cloud. +You own a custom base image. It must provide: -You own a custom base image. A Node image must provide: - -- `node` on `PATH` at the runtime's major version -- `busybox`, `ca-certificates`, `dumb-init`, `git` and `openssl` -- a `node` user -- glibc, so native modules built in the build stage load at runtime - -A Bun image must provide: - -- `bun` and `node` on `PATH`, because the final stage starts the app with `dumb-init node` +- `node` on `PATH` at the runtime's major version (for `bun`, both `bun` and `node`, because the final stage starts the app with `dumb-init node`) - `busybox`, `ca-certificates`, `dumb-init`, `git` and `openssl` -- a `bun` user +- a `node` user (a `bun` user for the Bun runtime) - glibc, so native modules built in the build stage load at runtime -A `DEPLOY_BUILD_BASE_IMAGES` image must contain everything the base image provides, plus the toolchain the published build images include: `python3`, `make` and `g++`. When a project's build extensions add image instructions, those instructions are replayed on the build-stage image, so it must be able to run them. Without a `DEPLOY_BUILD_BASE_IMAGES` entry, the build stage uses the published build image. The exception is a project whose build extensions add image instructions: its build stage is created from your base image and the toolchain is installed with `apt-get`, so the base image must be Debian-based for those projects. To avoid the published build image entirely, set both `DEPLOY_BASE_IMAGES` and `DEPLOY_BUILD_BASE_IMAGES`. +A `DEPLOY_BUILD_BASE_IMAGES` image needs everything the base image provides, plus `python3`, `make` and `g++`. Build extensions that add image instructions are replayed on it. Without an entry, the build stage uses the published build image, except for projects whose build extensions add image instructions: those build from your base image and install the toolchain with `apt-get`, so that base must be Debian-based. To avoid the published images entirely, set both variables. `image.pkgs` and build extensions that run `apt-get` (such as `aptGet` and `playwright`) assume a From 0549f557310a2dd7f229a5af9696ea5ace0c5ce0 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:04:39 +0100 Subject: [PATCH 7/7] fix(cli): reject from-bundle deploys that need instance base images --- docs/self-hosting/overview.mdx | 2 +- packages/cli-v3/src/commands/deploy.ts | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/self-hosting/overview.mdx b/docs/self-hosting/overview.mdx index f592bc2f787..2f7479f2309 100644 --- a/docs/self-hosting/overview.mdx +++ b/docs/self-hosting/overview.mdx @@ -110,7 +110,7 @@ DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:<64-charact Entries are comma-separated `runtime=image@sha256:`. The runtimes are `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. Projects with `runtime: "node"` in their config resolve to the current default Node runtime (`node-24` today), so set that key for them. With the Helm chart, set the variables through `webapp.extraEnvVars`. -Runtimes with an entry build on that image; the others keep the published images. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade, and so are `--native-build` and `--local-bundle` deploys. This is a self-hosting setting and does not apply to Trigger.dev Cloud. +Runtimes with an entry build on that image; the others keep the published images. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade, and so are `--native-build`, `--local-bundle` and `--from-bundle` deploys. This is a self-hosting setting and does not apply to Trigger.dev Cloud. You own a custom base image. It must provide: diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index 406698db56b..ce70f4408df 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -2308,12 +2308,16 @@ async function handleFromBundleDeploy({ existingDeploymentId ); - await applyServerBaseImages({ - baseImages: deployment.baseImages, - outputPath: bundlePath, - buildManifest: bundleManifest, - options, - }); + if (deployment.baseImages) { + const message = + "This instance requires custom deploy base images, which --from-bundle deploys cannot apply. Deploy without --from-bundle."; + + await projectClient.client.failDeployment(deployment.id, { + error: { name: "BuildError", message }, + }); + + throw new Error(message); + } // Fail fast if we know local builds will fail const buildxResult = await x("docker", ["buildx", "version"]);