ci-linux-e2e.yml

name: ci-linux-e2e

# The e2e suite (tests/e2e/run_all.sh) split out of ci-linux.yml so it runs in
# PARALLEL with the build/unit/toolchain jobs instead of tacked on after them,
# and SHARDED across two runners on top of that. Both workflows share the same
# cache lineage (mcpp sandbox + xlings + target/), so each shard restores a warm
# build and the only added wall-clock vs. the inline version is one extra warm
# `mcpp build` per runner.
#
# Paired workflows: ci-linux.yml (build + unit + toolchain legs + integration),
# ci-macos.yml / ci-macos-e2e.yml, ci-windows.yml / ci-windows-e2e.yml.

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  workflow_dispatch:

concurrency:
  group: ci-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  e2e:
    name: e2e ${{ matrix.shard }}/2 (linux x86_64, self-host)
    runs-on: ubuntu-24.04
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2]
    env:
      # Round-robin slice of tests/e2e (see run_all.sh). Two shards halve the
      # suite's wall-clock; the split is computed on the full file list, so a
      # test's shard does not move when host capabilities differ.
      E2E_SHARD: ${{ matrix.shard }}/2
      MCPP_HOME: /home/runner/.mcpp
      # NOTE: do NOT force MCPP_VERBOSE here. The e2e suite includes tests that
      # assert mcpp's DEFAULT (quiet) output — e.g. 48_build_error_output and
      # 53_namespaced_cache_label — which forced verbose would break. Verbose is
      # set only in the fresh-install workflows (cold bootstrap, no such asserts).
      # A specific test that needs verbose passes `--verbose` itself.
    steps:
      # `submodules: recursive` so tests/e2e/233_bench_matrix.sh can check that
      # each `hub`/`body` in bench/matrix.json exists in the tree it names.
      # Without the trees that check reads "submodule not initialised" and
      # reports nothing, which is how a hub path written for the CURRENT
      # layout stayed in matrix.json while the workload it named is a
      # HISTORICAL mcpp -- three cells reporting `skipped` on every bench run
      # and the job still green (#599).
      #
      # It is not free and it is not expensive: the three pinned workloads are
      # 725 + 701 + 806 tracked files, under 10 MB of source in total, and
      # nothing here builds them. Both shards carry it because run_all.sh
      # slices the file list round-robin, so which shard holds 233 moves when
      # a test is added.
      - uses: actions/checkout@v4
        with:
          submodules: recursive

      # Same cache lineage as ci-linux.yml so this job lands on a warm
      # toolchain/sandbox instead of re-installing it.
      - uses: ./.github/actions/bootstrap-mcpp

      - name: Configure mirror + Build mcpp from source (self-host)
        run: |
          export MCPP_VENDORED_XLINGS="$XLINGS_BIN"
          "$XLINGS_BIN" config --mirror GLOBAL 2>/dev/null || true
          "$MCPP" self config --mirror GLOBAL 2>/dev/null || true
          "$MCPP" build

      - name: E2E suite
        # Per-test 600s timeout lives in tests/e2e/run_all.sh and identifies
        # WHICH test hung; this caps the whole suite so a hang fails fast.
        timeout-minutes: 25
        run: |
          # Point the e2e runner at the freshly-built binary, not the
          # bootstrap one. Tests cd into mktemp -d, so $MCPP must be
          # absolute or the relative path breaks under the temp cwd.
          # The RELEASED mcpp this job bootstrapped from, kept for e2e 252: the
          # claim that an older client can still build against a package the PR
          # produces is only worth making if something checks it against a real
          # old binary. Captured before $MCPP is repointed at the fresh build.
          export MCPP_BOOT="$MCPP"
          MCPP=$(realpath "$(find target -type f -name mcpp -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2)")
          test -x "$MCPP"
          export MCPP
          # Tests that set MCPP_HOME to a fresh tmpdir need an xlings to
          # bootstrap from; surface the xlings binary installed above.
          export MCPP_VENDORED_XLINGS="$XLINGS_BIN"
          test -x "$MCPP_VENDORED_XLINGS"
          # GitHub-hosted runners are outside CN; keep CI toolchain downloads on
          # the global mirror while mcpp's default remains CN for fresh local
          # sandboxes. E2E tests with their own MCPP_HOME read this variable.
          export MCPP_E2E_TOOLCHAIN_MIRROR=GLOBAL
          "$MCPP" self config --mirror "$MCPP_E2E_TOOLCHAIN_MIRROR"
          "$MCPP" self config
          # Pin the global default so test 28 (default-toolchain path) gets a
          # deterministic GNU answer instead of an auto-install pick.
          #
          # Installed explicitly, not assumed. `toolchain default` requires the
          # toolchain to be present, and nothing in this job puts it there:
          # building mcpp above uses whatever first-run picks (llvm). It worked
          # only because the sandbox cache happened to carry a gcc some earlier
          # run had installed -- so the moment the cache key changed (an xlings
          # pin bump moves the whole prefix), every e2e shard failed with
          # `gcc@16.1.0 is not installed` before running a single test.
          # Warm runs re-install nothing; this costs a lookup.
          "$MCPP" toolchain install gcc 16.1.0
          "$MCPP" toolchain default gcc@16.1.0
          # Warm musl once so fresh-home e2e tests inherit the payload.
          "$MCPP" toolchain install gcc 16.1.0-musl
          bash tests/e2e/run_all.sh

  # ──────────────────────────────────────────────────────────────────
  # Bare metal: the one chain the sharded suite above cannot be trusted
  # to exercise.
  #
  # tests/e2e/130_freestanding_riscv_build_and_run.sh declares
  # `# requires: qemu-riscv`, which is legitimately absent on the macOS and
  # Windows runners — so it must be a SOFT token, and a soft token means the
  # test skips in silence on a Linux runner that lost qemu too. That is the
  # exact shape this repository has been burned by twice (65_* never ran at
  # all; ten pack e2e skipped on two platforms), and no token can tell the two
  # cases apart.
  #
  # So the guard lives here, where it can be exact: install qemu, run the one
  # test, and assert its PASS line appeared. A skip fails this job.
  # ──────────────────────────────────────────────────────────────────
  baremetal:
    name: bare-metal e2e (riscv64-none-elf + cortex-m, qemu)
    runs-on: ubuntu-24.04
    timeout-minutes: 40
    env:
      MCPP_HOME: /home/runner/.mcpp
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/bootstrap-mcpp

      - name: Build mcpp from source (self-host)
        run: |
          export MCPP_VENDORED_XLINGS="$XLINGS_BIN"
          "$XLINGS_BIN" config --mirror GLOBAL 2>/dev/null || true
          "$MCPP" self config --mirror GLOBAL 2>/dev/null || true
          "$MCPP" build

      - name: Install the emulator (xim:qemu-riscv)
        run: |
          # BOTH homes. The shim on PATH dispatches against whichever home
          # owns it, and `mcpp run` runs the runner through that shim — so an
          # emulator installed only in the ambient xlings home answers
          # "xlings: 'qemu-system-riscv64' is not installed" when mcpp asks.
          # Measured: the job installed it once, the shim resolved, and the
          # run still failed.
          "$XLINGS_BIN" install xim:qemu-riscv -y
          XLINGS_HOME="${MCPP_HOME:-$HOME/.mcpp}/registry" \
            "$XLINGS_BIN" install xim:qemu-riscv -y
          # Assert it is reachable AND runnable BEFORE the tests. Without this
          # the capability probe simply would not add `qemu-riscv` and the
          # tests would skip — which is what this job exists to prevent.
          command -v qemu-system-riscv64
          qemu-system-riscv64 --version | head -1
          # The target sysroot, into the home MCPP uses. Test 131's BSP
          # declares it as an `[xlings] deps` entry and finds it through
          # `xpkg_dir`; installed into the ambient xlings home instead, the
          # test would SKIP and the seam would go unexercised.
          XLINGS_HOME="${MCPP_HOME:-$HOME/.mcpp}/registry" \
            "$XLINGS_BIN" install xim:picolibc-riscv -y
          test -d "${MCPP_HOME:-$HOME/.mcpp}/registry/data/xpkgs/xim-x-picolibc-riscv"
          # The M-profile emulator, in BOTH homes for the reason above.
          # `xim:qemu-arm` carries `qemu-system-arm` as well as
          # `qemu-system-aarch64`; test 332 addresses it by absolute path out of
          # the payload, so what matters is that the payload EXISTS in the home
          # mcpp uses rather than that a shim resolves.
          "$XLINGS_BIN" install xim:qemu-arm -y
          XLINGS_HOME="${MCPP_HOME:-$HOME/.mcpp}/registry" \
            "$XLINGS_BIN" install xim:qemu-arm -y
          # Reachable AND runnable before the tests, so that a missing emulator
          # fails this step rather than silently skipping test 332.
          ls "${MCPP_HOME:-$HOME/.mcpp}"/registry/data/xpkgs/xim-x-qemu-arm/*/bin/qemu-system-arm \
            | sort -V | tail -1 | xargs -I{} {} --version | head -1

      - name: Bare-metal e2e
        timeout-minutes: 25
        run: |
          MCPP=$(realpath "$(find target -type f -name mcpp -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2)")
          test -x "$MCPP"
          export MCPP
          export MCPP_VENDORED_XLINGS="$XLINGS_BIN"
          export MCPP_E2E_TOOLCHAIN_MIRROR=GLOBAL
          "$MCPP" self config --mirror "$MCPP_E2E_TOOLCHAIN_MIRROR"
          # llvm is the toolchain a freestanding target pins; install it
          # explicitly rather than relying on whatever the sandbox cache holds.
          "$MCPP" toolchain install llvm 22.1.8
          # Run the two scripts DIRECTLY rather than through run_all.sh.
          # They are standalone (they take $MCPP and nothing else), run_all.sh
          # accepts no filter — it would run the whole 250-test suite here for
          # two tests — and, more to the point, run_all.sh exits 0 on a skip.
          # Invoked directly, a skip is visible: the script either prints its
          # PASS line or it does not.
          for t in tests/e2e/130_freestanding_riscv_build_and_run.sh \
                   tests/e2e/131_freestanding_bsp_supplies_everything.sh \
                   tests/e2e/132_freestanding_test_and_artifacts.sh \
                   tests/e2e/133_freestanding_std_subset.sh \
                   tests/e2e/332_cortex_m_builds_and_boots.sh \
                   tests/e2e/336_armv7a_builds_and_boots.sh \
                   tests/e2e/338_cortex_m_picolibc_sysroot.sh; do
            echo "=== $t ==="
            bash "$t" 2>&1 | tee "$(basename "$t").log"
            rc=${PIPESTATUS[0]}
            [ "$rc" = "0" ] || { echo "$t failed (exit $rc)"; exit 1; }
          done
          # The assertion this job exists for: both tests RAN. Each has an
          # early `exit 0` for a missing capability, so a zero exit code alone
          # does not distinguish "passed" from "skipped".
          grep -q 'PASS: freestanding riscv64 build + run' \
            130_freestanding_riscv_build_and_run.sh.log || {
            echo "130 (engine chain) skipped on the runner that must run it"; exit 1; }
          grep -q 'PASS: BSP supplies the sysroot' \
            131_freestanding_bsp_supplies_everything.sh.log || {
            echo "131 (ecosystem chain) skipped on the runner that must run it"; exit 1; }
          grep -q 'PASS: bare-metal mcpp test names its failure' \
            132_freestanding_test_and_artifacts.sh.log || {
            echo "132 (test + artifacts) skipped on the runner that must run it"; exit 1; }
          grep -q 'PASS: the freestanding std subset' \
            133_freestanding_std_subset.sh.log || {
            echo "133 (std subset) skipped on the runner that must run it"; exit 1; }
          # 332 declares `# requires: qemu-arm`, which no sharded runner has
          # — so on the shards it exits 0 without running. This job is the only
          # place its PASS line can be demanded.
          grep -q 'PASS: cortex-m rows build, boot' \
            332_cortex_m_builds_and_boots.sh.log || {
            echo "332 (cortex-m) skipped on the runner that must run it"; exit 1; }
          # And a count, because four `grep -q` calls that each matched say
          # nothing about how many rows the script actually booted: a fixture
          # that stopped iterating would still print its PASS line.
          booted=$(grep -c 'booted on ' 332_cortex_m_builds_and_boots.sh.log || true)
          [ "$booted" = "4" ] || {
            echo "332 booted $booted rows, expected 4"; exit 1; }
          # 336 IS HERE FOR THE SAME REASON, AND ADDING IT TO THE LOOP ABOVE
          # WITHOUT ADDING IT HERE WOULD HAVE BEEN THE SAME DEFECT: it declares
          # `# requires: qemu-arm`, so on a shard it exits 0 having run nothing.
          grep -q 'PASS: armv7-a rows build, boot' \
            336_armv7a_builds_and_boots.sh.log || {
            echo "336 (armv7-a) skipped on the runner that must run it"; exit 1; }
          a32=$(grep -c 'booted on virt' 336_armv7a_builds_and_boots.sh.log || true)
          [ "$a32" = "2" ] || {
            echo "336 booted $a32 rows, expected 2"; exit 1; }
          # 338 SKIPS UNTIL `xim:picolibc-arm` IS PUBLISHED, and a skip here
          # is legitimate rather than a defect — the payload is a separate
          # release. So its PASS line is NOT demanded; what IS demanded is that
          # the script either passed or said why, which is what distinguishes a
          # skip from a silent zero-exit.
          grep -qE 'PASS: a Cortex-M project opts into picolibc|SKIP: picolibc-arm is not installed' \
            338_cortex_m_picolibc_sysroot.sh.log || {
            cat 338_cortex_m_picolibc_sysroot.sh.log
            echo "338 neither passed nor reported why it did not run"; exit 1; }

  # ──────────────────────────────────────────────────────────────────
  # Android: the rows no sharded runner can reach.
  #
  # `run_all.sh` adds the `android-ndk` capability only when the NDK payload
  # is already installed, and no other job installs it, so every
  # `# requires: android-ndk` test skipped on every CI runner while reporting
  # green. This job installs the payload, runs those tests directly and demands
  # each PASS line, then runs `mcpp test` on an API 34 emulator through
  # `adb-run` (the configuration measured on mcpp#635), which is the only
  # place a test program's loading on the device is observed.
  # ──────────────────────────────────────────────────────────────────
  android:
    name: android e2e (NDK rows + x86_64 emulator)
    runs-on: ubuntu-24.04
    timeout-minutes: 90
    env:
      MCPP_HOME: /home/runner/.mcpp
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/bootstrap-mcpp

      - name: Build mcpp from source (self-host)
        run: |
          export MCPP_VENDORED_XLINGS="$XLINGS_BIN"
          "$XLINGS_BIN" config --mirror GLOBAL 2>/dev/null || true
          "$MCPP" self config --mirror GLOBAL 2>/dev/null || true
          "$MCPP" build

      - name: Install the NDK (xim:android-ndk)
        timeout-minutes: 20
        run: |
          XLINGS_HOME="$MCPP_HOME/registry" "$XLINGS_BIN" install xim:android-ndk@30.0.16248370 -y
          # Present before the tests, so a missing payload fails this step
          # rather than an assertion deep inside a test.
          ls "$MCPP_HOME"/registry/data/xpkgs/xim-x-android-ndk/*/toolchains/llvm/prebuilt/*/bin/clang++

      - name: Android e2e (no device)
        timeout-minutes: 40
        run: |
          MCPP=$(realpath "$(find target -type f -name mcpp -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2)")
          test -x "$MCPP"
          export MCPP
          export MCPP_VENDORED_XLINGS="$XLINGS_BIN"
          export MCPP_E2E_TOOLCHAIN_MIRROR=GLOBAL
          "$MCPP" self config --mirror "$MCPP_E2E_TOOLCHAIN_MIRROR"
          "$MCPP" toolchain install gcc 16.1.0
          "$MCPP" toolchain default gcc@16.1.0
          # Run directly, not through run_all.sh, so a skip cannot pass as a
          # green: each script either prints its PASS line or this step fails.
          declare -A pass=(
            [652b_an_application_on_android_is_a_shared_library]='652b: kind = "app" on Android is a shared library OK'
            [664_a_universal_apk_is_two_legs_in_one_tree]='664: the universal APK is the library route applied to an app OK'
            [667_an_android_pack_stages_its_closure]='667: an Android pack stages its closure OK'
            [675_android_test_programs_carry_their_cxx_runtime]='PASS: 675_android_test_programs_carry_their_cxx_runtime'
            [680_a_dependency_floor_on_the_android_api_level]='PASS: 680_a_dependency_floor_on_the_android_api_level'
          )
          for t in "${!pass[@]}"; do
            echo "=== $t ==="
            bash "tests/e2e/$t.sh" 2>&1 | tee "$t.log"
            rc=${PIPESTATUS[0]}
            [ "$rc" = "0" ] || { echo "$t failed (exit $rc)"; exit 1; }
            grep -qF "${pass[$t]}" "$t.log" || { echo "$t did not print its PASS line"; exit 1; }
          done
          echo "MCPP=$MCPP" >> "$GITHUB_ENV"

      - name: Enable KVM
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm

      - name: Compile the device tests before the emulator starts
        timeout-minutes: 20
        run: bash .github/tools/android_emulator_test.sh prebuild "$RUNNER_TEMP/droidtest"

      - name: mcpp test on an API 34 emulator
        timeout-minutes: 30
        uses: reactivecircus/android-emulator-runner@v2
        with:
          api-level: 34
          arch: x86_64
          target: google_apis
          force-avd-creation: false
          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
          disable-animations: true
          script: bash .github/tools/android_emulator_test.sh run "$RUNNER_TEMP/droidtest"

  # ──────────────────────────────────────────────────────────────────
  # Hermetic (no host toolchain): the ONLY environment class that
  # faithfully reproduces issue #195. Standard runners ship gcc +
  # libc6-dev, so a sandbox toolchain that leaks to the host's CRT
  # still links "green" there; this container has no compiler and no
  # host Scrt1.o, so any leak fails loudly. Builds PR code with the
  # bootstrap mcpp, then runs the llvm flow end-to-end.
  # ──────────────────────────────────────────────────────────────────
  hermetic:
    name: hermetic e2e (no host toolchain, container)
    runs-on: ubuntu-24.04
    container: debian:stable-slim
    timeout-minutes: 60
    env:
      XLINGS_NON_INTERACTIVE: '1'
    steps:
      - name: Install base utilities (NO compiler)
        run: |
          apt-get update -qq
          apt-get install -y -qq curl ca-certificates git xz-utils unzip
          # The whole point of this job: no host toolchain, no host CRT.
          ! command -v gcc
          ! command -v cc
          test ! -e /usr/lib/x86_64-linux-gnu/Scrt1.o
          test ! -e /usr/lib/gcc

      - uses: actions/checkout@v4

      # Payload cache (downloads only — the container still has no host
      # toolchain, which is the property under test).
      - name: Cache mcpp sandbox payloads
        uses: actions/cache@v4
        with:
          path: ~/.mcpp
          key: mcpp-hermetic-${{ hashFiles('mcpp.toml') }}
          restore-keys: |
            mcpp-hermetic-

      - name: Bootstrap xlings + released mcpp
        run: |
          curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s v2026.9.16.1
          export PATH="$HOME/.xlings/subos/current/bin:$PATH"
          xlings update
          xlings install mcpp -y -g
          MCPP_BOOT="$HOME/.xlings/subos/current/bin/mcpp"
          "$MCPP_BOOT" --version
          "$MCPP_BOOT" self config --mirror GLOBAL
          echo "MCPP_BOOT=$MCPP_BOOT" >> "$GITHUB_ENV"

      - name: Build PR mcpp from source (sandbox gcc only)
        run: |
          "$MCPP_BOOT" build
          MCPP=$(realpath "$(find target -type f -name mcpp -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2)")
          test -x "$MCPP"
          "$MCPP" --version
          echo "MCPP=$MCPP" >> "$GITHUB_ENV"

      - name: "issue #195 reproduction: manifest llvm toolchain, fresh"
        run: |
          cd "$(mktemp -d)"
          "$MCPP" new hello195
          cd hello195
          printf '\n[toolchain]\nlinux = "llvm@22.1.8"\n' >> mcpp.toml
          printf 'import std;\nint main() { std::println("hello {}", 195); return 0; }\n' > src/main.cpp
          "$MCPP" run

      - name: Hermetic llvm e2e subset
        run: |
          export PATH="$HOME/.xlings/subos/current/bin:$PATH"
          export MCPP
          bash tests/e2e/86_llvm_hermetic_link.sh
          bash tests/e2e/37_llvm_import_std.sh

      # The scripts whose C++ runtime is a package (`llvm.libcxx`) declare
      # `# requires: llvm`, which no shard has, and `run_all.sh` exits 0 when it
      # skips. They run here, where llvm is installed, and each is held to the
      # line it prints only when it ran to the end, and to the line of the step
      # that runs a program (#641).
      - name: "C++ runtime e2e that needs llvm (663, 690, 696, 700)"
        run: |
          set -o pipefail
          export PATH="$HOME/.xlings/subos/current/bin:$PATH"
          export MCPP
          run_and_assert() {   # $1 = script; the remaining arguments are lines it must print
            local script="$1"; shift
            local log="$RUNNER_TEMP/$(basename "$script").log"
            bash "$script" 2>&1 | tee "$log"
            for line in "$@"; do
              grep -qxF "$line" "$log" || { echo "::error::$script did not print: $line"; exit 1; }
            done
          }
          run_and_assert tests/e2e/663_a_graph_libcxx_over_the_payloads_c_library.sh \
            "ok: llvm.libcxx supplies the C++ layer over the payload's C library, and the program runs"
          run_and_assert tests/e2e/690_a_shared_library_over_a_graph_cxx_runtime.sh \
            'ok: linkage = "static" builds and runs' \
            'ok: cxx_runtime = { shared = "self-contained" } links a private copy and runs' \
            "PASS: 690 a shared library over a graph C++ runtime is refused or carries a stated private copy"
          run_and_assert tests/e2e/696_a_cxx_layer_provider_keeps_its_own_standard.sh \
            "PASS: a C++-layer provider compiles its implementation units at its own standard"
          # #646 F3a: the default llvm shape of a program over a C++ shared library
          # aborted with std::bad_cast before its programs took the library's contract.
          run_and_assert tests/e2e/700_a_program_over_a_cxx_shared_library_has_one_cxx_runtime.sh \
            "ok: a program over a C++ shared library runs on one C++ runtime" \
            "ok: a stated self-contained program over a coupled C++ shared library is refused" \
            "PASS: 700 a program over a C++ shared library has one C++ runtime"