diff --git a/ci/fly-bats.yml b/ci/fly-bats.yml new file mode 100644 index 00000000000..86e4708312a --- /dev/null +++ b/ci/fly-bats.yml @@ -0,0 +1,116 @@ +--- +# Standalone pipeline for running the BOSH Acceptance Test suite (BATs) +# against a specific branch. +# +# Intended to be driven by `bundle exec rake fly:bats` from src/, which +# pushes the current branch and sets this pipeline automatically. +# +# Manual usage: +# fly -t bosh set-pipeline -p bats-local \ +# -c ci/fly-bats.yml \ +# --var bosh_repo=https://github.com/cloudfoundry/bosh.git \ +# --var bosh_branch=my-feature-branch \ +# --var env_name=bats-local \ +# --var stemcell_name=bosh-google-kvm-ubuntu-noble \ +# --var deploy_args="-o bosh-deployment/external-ip-not-recommended.yml" \ +# --var bat_rspec_flags="" +# fly -t bosh unpause-pipeline -p bats-local +# fly -t bosh trigger-job -j bats-local/bats -w + +resources: + - name: bosh + type: git + source: + uri: ((bosh_repo)) + branch: ((bosh_branch)) + + # bosh-ci is the same repo as bosh but filtered to ci/ paths so that + # scripts under ci/bats/ are available at bosh-ci/ in the task workspace. + - name: bosh-ci + type: git + source: + uri: ((bosh_repo)) + branch: ((bosh_branch)) + paths: [ci] + + - name: bosh-cli + type: github-release + source: + owner: cloudfoundry + repository: bosh-cli + access_token: ((github_public_repo_token)) + + - name: stemcell + type: bosh-io-stemcell + source: + name: bosh-google-kvm-ubuntu-noble + + - name: bats + type: git + source: + uri: https://github.com/cloudfoundry/bosh-acceptance-tests.git + branch: master + + - name: bosh-deployment + type: git + source: + uri: https://github.com/cloudfoundry/bosh-deployment + branch: master + + - name: integration-image + type: registry-image + source: + repository: ghcr.io/cloudfoundry/bosh/integration + tag: main + username: ((github_read_write_packages.username)) + password: ((github_read_write_packages.password)) + +jobs: + - name: bats + serial: true + plan: + - do: + - in_parallel: + - get: bosh + - get: bosh-ci + - get: bosh-cli + params: + globs: [bosh-cli-*-linux-amd64] + - get: stemcell + - get: bats + - get: bosh-deployment + - get: integration-image + + - task: make-candidate + image: integration-image + file: bosh-ci/ci/tasks/make-candidate.yml + + - task: compile-bosh-release + file: bosh-deployment/ci/tasks/shared/bosh-agent-compile.yml + + - task: run-bats + image: integration-image + input_mapping: + bosh-release: compiled-release + config: + platform: linux + inputs: + - name: bosh + - name: bosh-ci + - name: bosh-cli + - name: bosh-deployment + - name: stemcell + - name: bats + - name: bosh-release + caches: + - path: cache-dot-bosh-dir + params: + BAT_INFRASTRUCTURE: gcp + GCP_JSON_KEY: ((gcp_json_key)) + GCP_PROJECT_ID: ((gcp_project_id)) + STEMCELL_NAME: ((stemcell_name)) + ENV_NAME: ((env_name)) + DEPLOY_ARGS: ((deploy_args)) + BAT_RSPEC_FLAGS: ((bat_rspec_flags)) + run: + path: bosh/ci/tasks/run-bats-pipeline.sh diff --git a/ci/tasks/run-bats-pipeline.sh b/ci/tasks/run-bats-pipeline.sh new file mode 100755 index 00000000000..b0fef1516e5 --- /dev/null +++ b/ci/tasks/run-bats-pipeline.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# run-bats-pipeline.sh +# +# Chains together the full BATs pipeline in a single Concourse task: +# terraform apply → deploy-director → prepare-bats-config → run-bats +# terraform destroy ← destroy-director ← (EXIT trap, always runs) +# +# Required env vars (set via run-bats-pipeline.yml params): +# GCP_JSON_KEY – GCP service account JSON (resolved from Concourse creds) +# GCP_PROJECT_ID – GCP project ID +# ENV_NAME – Unique terraform env name, e.g. "bats-local" +# BAT_INFRASTRUCTURE – "gcp" +# STEMCELL_NAME – Stemcell name for bats-config.yml +# DEPLOY_ARGS – Extra ops-file args for bosh create-env +# BAT_RSPEC_FLAGS – Extra flags appended to the BAT run (optional) + +set -eu + +ROOT_DIR="$PWD" +TERRAFORM_DIR="${ROOT_DIR}/bosh-ci/ci/bats/iaas/gcp/terraform" +TERRAFORM_VERSION="1.9.8" + +# ── Shared working directories ─────────────────────────────────────────────── +mkdir -p director-state bats-config environment +# cache-dot-bosh-dir is provided as a Concourse cache volume; create if absent +mkdir -p cache-dot-bosh-dir/.bosh + +# prepare-bats-config.sh expects its terraform metadata at terraform/metadata +# but deploy-director.sh expects it at environment/metadata. +# Symlink terraform/ → environment/ so both scripts find what they need. +ln -sf "${ROOT_DIR}/environment" "${ROOT_DIR}/terraform" + +# ── Install terraform ──────────────────────────────────────────────────────── +if ! command -v terraform &>/dev/null; then + echo "--- Installing terraform ${TERRAFORM_VERSION} ---" + curl -sSL \ + "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" \ + -o /tmp/terraform.zip + unzip -qo /tmp/terraform.zip -d /usr/local/bin terraform + chmod +x /usr/local/bin/terraform +fi + +# ── GCP credentials file (used by the GCS backend and the Google provider) ── +GCP_CREDS_FILE="$(mktemp /tmp/gcp-creds-XXXXXX.json)" +echo "${GCP_JSON_KEY}" > "${GCP_CREDS_FILE}" +chmod 600 "${GCP_CREDS_FILE}" + +# ── Provision GCP environment via terraform ────────────────────────────────── +echo "--- Provisioning GCP environment (env: ${ENV_NAME}) ---" +pushd "${TERRAFORM_DIR}" >/dev/null + +terraform init \ + -input=false \ + -reconfigure \ + -backend-config="bucket=bosh-director-pipeline" \ + -backend-config="prefix=bats-terraform/${ENV_NAME}" \ + -backend-config="credentials=${GCP_CREDS_FILE}" + +terraform apply \ + -input=false \ + -auto-approve \ + -var "project_id=${GCP_PROJECT_ID}" \ + -var "gcp_credentials_json=${GCP_JSON_KEY}" \ + -var "name=${ENV_NAME}" + +# Convert terraform outputs to the flat metadata JSON consumed by director-vars +# and prepare-bats-config.sh. +terraform output -json \ + | jq 'with_entries(.value = .value.value)' \ + > "${ROOT_DIR}/environment/metadata" + +popd >/dev/null + +# ── Teardown trap (always runs on EXIT) ───────────────────────────────────── +function collect_director_diagnostics { + # Only collect when we have a deployed director and bosh-cli is available. + [[ -f director-state/director-creds.yml ]] || return 0 + [[ -f "${ROOT_DIR}/environment/metadata" ]] || return 0 + command -v bosh-cli &>/dev/null || return 0 + + local director_ip + director_ip="$(jq -r '.director_public_ip // empty' "${ROOT_DIR}/environment/metadata")" + [[ -n "${director_ip}" ]] || return 0 + + echo "--- Collecting director diagnostics (IP: ${director_ip}) ---" + + # Extract jumpbox SSH private key from the vars-store. + local jumpbox_key_file + jumpbox_key_file="$(mktemp /tmp/jumpbox-key-XXXXXX)" + bosh-cli interpolate director-state/director-creds.yml \ + --path /jumpbox_ssh/private_key > "${jumpbox_key_file}" 2>/dev/null || { rm -f "${jumpbox_key_file}"; return 0; } + chmod 600 "${jumpbox_key_file}" + + ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -i "${jumpbox_key_file}" \ + "jumpbox@${director_ip}" \ + 'echo "=== monit status ===" && sudo /var/vcap/bosh/bin/monit status; + echo "=== bosh_nats_sync log (last 100 lines) ===" && sudo tail -100 /var/vcap/sys/log/nats/bosh-nats-sync.log 2>/dev/null || true; + echo "=== bosh_nats_sync bpm stdout ===" && sudo cat /var/vcap/sys/log/bpm/nats/bosh_nats_sync.stdout.log 2>/dev/null || true; + echo "=== bosh_nats_sync bpm stderr ===" && sudo cat /var/vcap/sys/log/bpm/nats/bosh_nats_sync.stderr.log 2>/dev/null || true; + echo "=== nats log (last 50 lines) ===" && sudo tail -50 /var/vcap/sys/log/nats/nats.log 2>/dev/null || true; + echo "=== nats bpm stdout (last 50 lines) ===" && sudo tail -50 /var/vcap/sys/log/bpm/nats/nats.stdout.log 2>/dev/null || true; + echo "=== health_monitor log (last 100 lines) ===" && sudo tail -100 /var/vcap/sys/log/health_monitor/health_monitor.log 2>/dev/null || true; + echo "=== health_monitor bpm stdout ===" && sudo tail -50 /var/vcap/sys/log/bpm/health_monitor/health_monitor.stdout.log 2>/dev/null || true; + echo "=== health_monitor bpm stderr ===" && sudo tail -50 /var/vcap/sys/log/bpm/health_monitor/health_monitor.stderr.log 2>/dev/null || true' \ + 2>&1 || echo "(SSH diagnostics failed — VM may not be reachable)" + + rm -f "${jumpbox_key_file}" +} + +function teardown { + local exit_code=$? + set +e + + # Always collect diagnostics – on success this helps correlate logs with + # passing runs; on failure it captures the state at the point of failure. + collect_director_diagnostics + + echo "--- Tearing down BOSH director ---" + if [[ -f director-state/director-state.json ]]; then + # destroy-director.sh expects bosh-cli/bosh-cli-* to exist; restore it + # because deploy-director.sh already moved the original binary away. + cp /usr/local/bin/bosh-cli bosh-cli/bosh-cli-restore 2>/dev/null || true + bosh-ci/ci/bats/tasks/destroy-director.sh || true + fi + + echo "--- Destroying GCP environment (env: ${ENV_NAME}) ---" + pushd "${TERRAFORM_DIR}" >/dev/null + terraform destroy \ + -input=false \ + -auto-approve \ + -var "project_id=${GCP_PROJECT_ID}" \ + -var "gcp_credentials_json=${GCP_JSON_KEY}" \ + -var "name=${ENV_NAME}" || true + popd >/dev/null + + rm -f "${GCP_CREDS_FILE}" + + exit "${exit_code}" +} +trap teardown EXIT + +# ── Deploy BOSH director ───────────────────────────────────────────────────── +echo "--- Deploying BOSH director ---" +# deploy-director.sh moves bosh-cli/bosh-cli-* to /usr/local/bin/bosh-cli. +# After this call bosh-cli is installed system-wide as 'bosh-cli'. +bosh-ci/ci/bats/tasks/deploy-director.sh + +# ── Prepare BATs config ────────────────────────────────────────────────────── +echo "--- Preparing BATs config ---" +bosh-ci/ci/bats/iaas/gcp/prepare-bats-config.sh + +# ── Run BATs ───────────────────────────────────────────────────────────────── +echo "--- Running BATs ---" +# Source the environment file that prepare-bats-config.sh wrote; this exports +# BOSH_ENVIRONMENT, BOSH_CLIENT, BOSH_CLIENT_SECRET, BOSH_CA_CERT, +# BOSH_ALL_PROXY, and the default BAT_RSPEC_FLAGS. +# shellcheck source=/dev/null +source bats-config/bats.env + +# Allow the caller to append extra RSpec flags (e.g. "--tag wip"). +if [[ -n "${BAT_RSPEC_FLAGS:-}" ]]; then + export BAT_RSPEC_FLAGS="${BAT_RSPEC_FLAGS}" +fi + +bats/ci/tasks/run-bats.sh diff --git a/src/tasks/fly.rake b/src/tasks/fly.rake index ad745ba7632..e280c63a307 100644 --- a/src/tasks/fly.rake +++ b/src/tasks/fly.rake @@ -1,3 +1,5 @@ +require 'shellwords' + namespace :fly do desc 'Fly unit specs' task :unit do @@ -9,6 +11,53 @@ namespace :fly do COVERAGE: ENV.fetch('COVERAGE', false)) end + desc 'Run BATs (BOSH Acceptance Tests) against the current branch via Concourse' + # + # Sets the ci/fly-bats.yml pipeline on Concourse, then triggers and watches + # the bats job. The current branch is pushed to origin automatically so + # Concourse can fetch it. + # + # GCP credentials are resolved from the Concourse credential store + # (((gcp_json_key)) and ((gcp_project_id))). + # + # Useful env vars: + # BATS_ENV_NAME – terraform env name, must be unique per concurrent run + # (default: "bats-local") + # STEMCELL_NAME – GCP stemcell name override + # DEPLOY_ARGS – extra ops-files passed to bosh create-env + # BAT_RSPEC_FLAGS – extra flags appended to the RSpec BATs run + task :bats do + env_name = ENV.fetch('BATS_ENV_NAME', 'bats-local') + + branch = `git -C .. rev-parse --abbrev-ref HEAD`.strip + repo = `git -C .. remote get-url origin`.strip + .sub(/\Agit@github\.com:/, 'https://github.com/') + .sub(/\.git\z/, '.git') + + # Push the current branch so Concourse can check it out. + sh "git -C .. push origin HEAD" + + # ── Set the pipeline ───────────────────────────────────────────────────── + sh [ + "fly #{concourse_target}", + 'set-pipeline', + '--non-interactive', + '--pipeline bats-local', + '--config ../ci/fly-bats.yml', + "--var bosh_repo=#{Shellwords.escape(repo)}", + "--var bosh_branch=#{Shellwords.escape(branch)}", + "--var env_name=#{Shellwords.escape(env_name)}", + "--var stemcell_name=#{Shellwords.escape(ENV.fetch('STEMCELL_NAME', 'bosh-google-kvm-ubuntu-noble'))}", + "--var deploy_args=#{Shellwords.escape(ENV.fetch('DEPLOY_ARGS', '-o bosh-deployment/external-ip-not-recommended.yml'))}", + "--var bat_rspec_flags=#{Shellwords.escape(ENV.fetch('BAT_RSPEC_FLAGS', ''))}", + ].compact.join(' ') + + sh "fly #{concourse_target} unpause-pipeline --pipeline bats-local" + + # ── Trigger and stream the job output ──────────────────────────────────── + sh "fly #{concourse_target} trigger-job --job bats-local/bats --watch" + end + desc 'Fly integration specs' task :integration, [:cli_dir] do |_, args| db, db_version = fetch_db_and_version('postgresql')