feat(server): build releases in kubernetes without the docker binary

.github/workflows/tests.yml

@@ -142,6 +142,113 @@ jobs:
           name: e2e_coverage
           path: tests_coverage
 
+  e2e_buildkitd_driver_forwarding:
+    name: End-to-end tests (buildkitd driver forwarding)
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    # The mirror image of e2e_buildx_config, for a driver that cannot succeed
+    # without a cluster. The configuration names the kubernetes buildkitd driver
+    # and a namespace; TRDL_BUILDX_DRIVER only spells out the fallback the code
+    # would take anyway, so that the path this job forbids is written down.
+    # The release therefore has to FAIL, and with the driver's own error naming
+    # that namespace: if the driver stops being forwarded the build falls back to
+    # the docker CLI and succeeds, and if the OPTIONS stop being forwarded the
+    # namespace is the kubeconfig's instead. The kubeconfig below points at a
+    # closed port precisely so the driver gets past client configuration and
+    # reaches pod creation, which is where the namespace becomes observable.
+    env:
+      TRDL_BUILDX_DRIVER: docker-container
+      TRDL_TEST_BUILDKITD_DRIVER: kubernetes
+      TRDL_TEST_BUILDKITD_DRIVER_OPTS_NAMESPACE: namespace=trdl-forwarding-guard
+      KUBECONFIG: /tmp/guard-kubeconfig.yaml
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: e2e/go.mod
+
+      - name: Install Task
+        uses: go-task/setup-task@v2
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up prebuilt trdl test binary
+        run: |
+          task --yes client:build-with-coverage
+          echo TRDL_TEST_BINARY_PATH=$GITHUB_WORKSPACE/bin/coverage/trdl >> $GITHUB_ENV
+          echo TRDL_TEST_COVERAGE_DIR=$GITHUB_WORKSPACE/tests_coverage/e2e >> $GITHUB_ENV
+
+      - name: Set up git config
+        run: task --yes ci:setup:git-config
+
+      - name: Prepare environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gpg
+          task --yes server:deps:install:c
+
+      - name: Install 3p-git-signatures
+        run: task --yes ci:install:3p-git-signatures
+
+      - name: Install ginkgo
+        run: task --yes -p deps:install:ginkgo
+
+      - name: Write a kubeconfig pointing at a closed port
+        run: |
+          # Its namespace is deliberately NOT the configured one: the driver must
+          # take the namespace from buildkitd_driver_opts, and this is what makes
+          # the difference observable in the error.
+          cat > /tmp/guard-kubeconfig.yaml <<'EOF'
+          apiVersion: v1
+          kind: Config
+          clusters:
+            - name: unreachable
+              cluster:
+                server: https://127.0.0.1:1
+          contexts:
+            - name: unreachable
+              context:
+                cluster: unreachable
+                user: none
+                namespace: kubeconfig-namespace
+          current-context: unreachable
+          users:
+            - name: none
+              user: {}
+          EOF
+
+      - name: Setup vault
+        run: |
+          task --yes server:setup-vault-local
+          echo "$HOME/bin" >> $GITHUB_PATH
+
+      - name: Test
+        run: |
+          set -o pipefail
+          task --yes e2e:test:e2e paths='./tests/flow_vault' labelFilter='buildkitd-driver-forwarding' 2>&1 | tee /tmp/guard.log
+
+      - name: Assert the guard actually ran
+        run: |
+          # A green ginkgo run proves nothing on its own: it exits 0 when the
+          # label filter selects no spec at all, and at --vv it prints a skipped
+          # spec's name too, so neither the exit code nor the spec's name shows
+          # that this guard executed.
+          # No anchor on the first: ginkgo colours that line, so it does not
+          # start with "Ran".
+          grep -qE 'Ran [1-9][0-9]* of [0-9]+ Specs' /tmp/guard.log
+          # Only the guard itself can put this in the log — it is the release
+          # output the spec asserts on, and no other spec produces it.
+          grep -q 'unable to create builder pod trdl-forwarding-guard/' /tmp/guard.log
+
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: e2e_coverage_buildkitd_driver_forwarding
+          path: tests_coverage
+
   e2e_buildx_config:
     name: End-to-end tests (buildx driver from configure)
     runs-on: ubuntu-22.04
@@ -275,6 +382,7 @@ jobs:
       - e2e_tests
       - e2e_buildkit
       - e2e_buildx_config
+      - e2e_buildkitd_driver_forwarding
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
@@ -299,6 +407,7 @@ jobs:
       - e2e_tests
       - e2e_buildkit
       - e2e_buildx_config
+      - e2e_buildkitd_driver_forwarding
       - upload_coverage
     uses: werf/common-ci/.github/workflows/notification.yml@main
     secrets:

.github/workflows/tests_buildx_kubernetes_driver.yaml

@@ -1,15 +1,22 @@
-name: Tests. Buildx kubernetes driver
+name: Tests. In-cluster Kubernetes builders
 
-# Opt-in: the buildx kubernetes driver needs a cluster, so this is not part of
-# the required checks. It runs the flow_vault suite — the real Vault plugin,
-# a real release build and buildkit secret mounts — against in-cluster BuildKit
-# pods instead of the default docker-container builder.
+# Opt-in: both jobs need a cluster, so this is not part of the required checks.
+# Each runs the flow_vault suite — the real Vault plugin, a real release build
+# and buildkit secret mounts — against in-cluster BuildKit pods instead of the
+# default docker-container builder: one through the buildx kubernetes driver and
+# the docker CLI, one through the plugin's own kubernetes buildkitd driver.
 on:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 env:
   TASK_X_REMOTE_TASKFILES: 1
   KIND_VERSION: v0.32.0
+  # Pinned so a mutable release asset cannot become whatever it likes before it is
+  # installed as root; from the release's own kind-linux-amd64.sha256sum.
+  KIND_SHA256: 50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
 
 jobs:
   e2e_tests_buildx_kubernetes:
@@ -22,6 +29,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v6
+        with:
+          persist-credentials: false
 
       - name: Set up Go
         uses: actions/setup-go@v6
@@ -62,6 +71,7 @@ jobs:
       - name: Set up kind cluster
         run: |
           curl -fsSLo /tmp/kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64
+          echo "${KIND_SHA256}  /tmp/kind" | sha256sum --check --strict
           sudo install -m 0755 /tmp/kind /usr/local/bin/kind
           kind create cluster --name trdl-buildx
           kubectl cluster-info --context kind-trdl-buildx
@@ -80,3 +90,97 @@ jobs:
         with:
           name: e2e_coverage_buildx_kubernetes
           path: tests_coverage
+
+  e2e_tests_buildkitd_kubernetes:
+    name: End-to-end tests (kubernetes buildkitd driver)
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    env:
+      # The plugin provisions the builder itself, configured per project rather
+      # than through the environment. TRDL_BUILDX_DRIVER names a driver that
+      # cannot work, so the release passes only while the docker CLI path stays
+      # unused: a regression routing back to it fails on this value.
+      TRDL_BUILDX_DRIVER: this-driver-does-not-exist
+      TRDL_TEST_BUILDKITD_DRIVER: kubernetes
+      TRDL_TEST_BUILDKITD_DRIVER_OPTS_NAMESPACE: namespace=default
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: e2e/go.mod
+
+      - name: Install Task
+        uses: go-task/setup-task@v2
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up prebuilt trdl test binary
+        run: |
+          task --yes client:build-with-coverage
+          echo TRDL_TEST_BINARY_PATH=$GITHUB_WORKSPACE/bin/coverage/trdl >> $GITHUB_ENV
+          echo TRDL_TEST_COVERAGE_DIR=$GITHUB_WORKSPACE/tests_coverage/e2e >> $GITHUB_ENV
+
+      - name: Set up git config
+        run: task --yes ci:setup:git-config
+
+      - name: Prepare environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gpg
+          task --yes server:deps:install:c
+
+      - name: Install 3p-git-signatures
+        run: task --yes ci:install:3p-git-signatures
+
+      - name: Install ginkgo
+        run: task --yes -p deps:install:ginkgo
+
+      - name: Setup vault
+        run: |
+          task --yes server:setup-vault-local
+          echo "$HOME/bin" >> $GITHUB_PATH
+
+      - name: Set up kind cluster
+        run: |
+          curl -fsSLo /tmp/kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64
+          echo "${KIND_SHA256}  /tmp/kind" | sha256sum --check --strict
+          sudo install -m 0755 /tmp/kind /usr/local/bin/kind
+          kind create cluster --name trdl-buildkitd
+          kubectl cluster-info --context kind-trdl-buildkitd
+
+      - name: Test
+        # Only the complete-cycle spec: this job sets TRDL_TEST_BUILDKITD_DRIVER,
+        # which also registers the negative forwarding guard, and the suite's
+        # AfterEach removes the plugin binary BeforeSuite built once, so a second
+        # spec would find nothing to enable.
+        run: task --yes e2e:test:e2e paths='./tests/flow_vault' labelFilter='flow'
+
+      - name: Collect builder diagnostics
+        if: failure()
+        run: kubectl get pods --all-namespaces || true
+
+      - name: Assert no builder pod was left behind
+        if: always()
+        run: |
+          # An empty pod list only means "nothing left behind" if the query itself
+          # ran. Probe the API first, so an unreachable cluster fails this step
+          # instead of passing it silently.
+          kubectl get namespace default > /dev/null
+          kubectl get pods -n default --no-headers > /tmp/pods-after.txt
+          echo "pods still in the namespace: $(wc -l < /tmp/pods-after.txt)"
+
+          if grep '^trdl-builder-' /tmp/pods-after.txt; then
+            echo "the release left a builder pod behind"
+            exit 1
+          fi
+
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: e2e_coverage_buildkitd_kubernetes
+          path: tests_coverage

docs/_includes/reference/vault_plugin/configure.md

@@ -9,9 +9,11 @@ Configure the plugin.
 
 ### Parameters
 
-* `buildkitd_address` (string, optional) — An address of a running buildkitd (unix://, tcp://, docker-container:// or kube-pod:// scheme) to build release artifacts with the BuildKit client; the docker CLI is used if not set. Build secrets are sent to that daemon, and tcp:// is neither encrypted nor authenticated, so securing the channel and isolating the daemon is the administrator's responsibility.
-* `buildx_driver` (string, optional) — The buildx driver to build release artifacts with: docker-container (used by default) or kubernetes. Takes precedence over the TRDL_BUILDX_DRIVER environment variable, and cannot be combined with buildkitd_address.
-* `buildx_driver_opts` (array, optional) — The buildx driver options, one --driver-opt per element (e.g. namespace=trdl-build), passed through as is. Take precedence over the TRDL_BUILDX_DRIVER_OPTS_* environment variables, and cannot be combined with buildkitd_address.
+* `buildkitd_address` (string, optional) — An address of a running buildkitd (unix://, tcp://, docker-container:// or kube-pod:// scheme) to build release artifacts with the BuildKit client; the docker CLI is used only when neither this nor buildkitd_driver is set. Build secrets are sent to that daemon, and tcp:// is neither encrypted nor authenticated, so securing the channel and isolating the daemon is the administrator's responsibility.
+* `buildkitd_driver` (string, optional) — Provision an ephemeral buildkitd per build instead of using the docker CLI: kubernetes runs it as a pod and needs no docker binary next to the plugin. Cannot be combined with buildkitd_address, buildx_driver or buildx_driver_opts. A TRDL_BUILDKITD_ADDRESS set on the process wins over a stored driver, and the build reports the driver as unused.
+* `buildkitd_driver_opts` (array, optional) — The buildkitd driver options, one name=value pair per element (e.g. namespace=trdl-build); they require buildkitd_driver to be set. The kubernetes driver accepts annotations, deadline, image, labels, limits.cpu, limits.ephemeral-storage, limits.memory, namespace, nodeselector, requests.cpu, requests.ephemeral-storage, requests.memory, rootless, serviceaccount and timeout; anything else is rejected.
+* `buildx_driver` (string, optional) — The buildx driver to build release artifacts with: docker-container (used by default) or kubernetes. Takes precedence over the TRDL_BUILDX_DRIVER environment variable, and cannot be combined with buildkitd_address or buildkitd_driver.
+* `buildx_driver_opts` (array, optional) — The buildx driver options, one --driver-opt per element (e.g. namespace=trdl-build), passed through as is. Take precedence over the TRDL_BUILDX_DRIVER_OPTS_* environment variables, and cannot be combined with buildkitd_address or buildkitd_driver.
 * `git_repo_url` (string, required) — URL of the Git repository.
 * `git_trdl_channels_branch` (string, optional) — A special Git branch to store the trdl channels configuration file.
 * `git_trdl_channels_path` (string, optional) — A path in the Git repository to the trdl channels configuration file (trdl_channels.yaml is used by default).