ci.yml

# libdb continuous integration.
#
# Builds the living fork (master) and PRs across as many of the platforms,
# compilers, and configure options that Berkeley DB and GitHub-hosted runners
# both support. Experimental/best-effort jobs are marked continue-on-error so
# they inform without gating.

name: CI

on:
  push:
    branches: [master]
  pull_request:
  schedule:
    # Nightly (03:17 UTC) exercises the full TCL suite via the tcl-tests job
    # below; the heavy scheduled-only jobs live in ci-extended.yml.
    - cron: '17 3 * * *'
  workflow_dispatch:
    inputs:
      full_tcl:
        description: 'Run the full TCL regression suite (long)'
        type: boolean
        default: false

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  # ----------------------------------------------------------------------------
  # POSIX builds: Linux + macOS, gcc + clang, across configure variants.
  # ----------------------------------------------------------------------------
  posix:
    name: ${{ matrix.os }} ${{ matrix.cc }} ${{ matrix.config }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, ubuntu-22.04, macos-latest, macos-14]
        cc: [gcc, clang]
        config: [default, debug, cxx, sql, no-crypto, smallbuild]
        exclude:
          # Reduce fan-out: only exercise the full config set on ubuntu-latest.
          - { os: ubuntu-22.04, config: cxx }
          - { os: ubuntu-22.04, config: sql }
          - { os: ubuntu-22.04, config: no-crypto }
          - { os: ubuntu-22.04, config: smallbuild }
          - { os: macos-14, config: cxx }
          - { os: macos-14, config: sql }
          - { os: macos-14, config: no-crypto }
          - { os: macos-14, config: smallbuild }
          # BDB's C++ headers don't compile against Xcode's libc++ <atomic>.
          - { os: macos-latest, config: cxx }
    env:
      CC: ${{ matrix.cc }}
    steps:
      - uses: actions/checkout@v4

      - name: Map config to configure flags
        id: cfg
        run: |
          case "${{ matrix.config }}" in
            default)    flags="" ;;
            debug)      flags="--enable-debug --enable-diagnostic" ;;
            cxx)        flags="--enable-cxx" ;;
            sql)        flags="--enable-sql" ;;
            no-crypto)  flags="--disable-cryptography" ;;
            smallbuild) flags="--enable-smallbuild" ;;
          esac
          echo "flags=$flags" >> "$GITHUB_OUTPUT"

      - name: Configure
        working-directory: build_unix
        run: ../dist/configure ${{ steps.cfg.outputs.flags }}

      - name: Build
        working-directory: build_unix
        run: make -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)

      - name: Smoke test (C API)
        working-directory: build_unix
        run: |
          # ex_access exercises open/put/get/cursor on a btree.
          make ex_access 2>/dev/null || true
          ls -l libdb-*.a 2>/dev/null || ls -l .libs/libdb-*.* 2>/dev/null || true

      # The test/db/ regression runners guard specific fixed bugs (the #139 hash
      # comparator, recd compact/handler recovery, and the on-disk upgrade path).
      # They were previously orphaned: no CI job and no make target invoked them,
      # so a regression would have gone unnoticed. They all follow the same
      # BUILD=${BUILD:-.} convention and expect to run from the build directory.
      #
      # The runner list used to be an inline `for r in ...` loop here.  It now
      # lives in test/MANIFEST and test/db/run_all.sh READS IT FROM THERE, so
      # the two cannot drift: adding a runner to one list and forgetting the
      # other is the shape that made aio_concurrent_sync and mvcc_purge_visible
      # never run.  run_all.sh also emits a RESULT verdict per runner.
      - name: Regression runners (test/db)
        if: matrix.config == 'default'
        working-directory: build_unix
        run: sh ../test/db/run_all.sh

      - name: Manifest gate (assert every test/db runner produced a verdict)
        if: always() && matrix.config == 'default'
        run: ./test/check_manifest.sh --tier db

  # ----------------------------------------------------------------------------
  # 32-bit build on Linux (pointer-size / portability coverage).
  # ----------------------------------------------------------------------------
  linux-32bit:
    name: ubuntu 32-bit
    runs-on: ubuntu-latest
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
      - name: Install 32-bit toolchain
        run: |
          sudo dpkg --add-architecture i386
          sudo apt-get update
          sudo apt-get install -y gcc-multilib g++-multilib
      - name: Configure (32-bit)
        working-directory: build_unix
        run: ../dist/configure --build=i686-pc-linux-gnu --with-mutex=POSIX/pthreads "CFLAGS=-m32" "LDFLAGS=-m32"
      - name: Build
        working-directory: build_unix
        run: make -j$(nproc)

  # ----------------------------------------------------------------------------
  # AddressSanitizer / UBSan build + smoke (catches memory/UB regressions).
  # ----------------------------------------------------------------------------
  sanitizers:
    name: clang asan+ubsan
    runs-on: ubuntu-latest
    continue-on-error: true
    env:
      CC: clang
    steps:
      - uses: actions/checkout@v4
      - name: Configure (sanitizers)
        working-directory: build_unix
        run: ../dist/configure --enable-debug "CFLAGS=-fsanitize=address,undefined -fno-omit-frame-pointer -g"
      - name: Build
        working-directory: build_unix
        run: make -j$(nproc)

  # ----------------------------------------------------------------------------
  # ThreadSanitizer build + concurrency SMOKE (SSI / lock / mutex TCL subset).
  # Critical for the ongoing SSI/MVCC/atomics work; best-effort so it informs
  # without gating. NOT the full suite -- TSan slowdown would time out.
  # ----------------------------------------------------------------------------
  tsan:
    name: clang tsan (concurrency smoke)
    runs-on: ubuntu-latest
    continue-on-error: true
    env:
      CC: clang
      # halt_on_error keeps a clean failure signal; second_deadlock_stack aids
      # triage of any lock-order report from the SSI paths.
      TSAN_OPTIONS: halt_on_error=1:second_deadlock_stack=1
    steps:
      - uses: actions/checkout@v4
      - name: Install Tcl
        run: sudo apt-get update && sudo apt-get install -y tcl-dev tcl
      - name: Configure (tsan + test + tcl)
        working-directory: build_unix
        run: >
          ../dist/configure --enable-debug --enable-test --with-tcl=/usr/lib/tcl8.6
          "CFLAGS=-fsanitize=thread -fno-omit-frame-pointer -g"
          "LDFLAGS=-fsanitize=thread"
      - name: Build
        working-directory: build_unix
        run: make -j$(nproc)
      - name: Concurrency smoke subset
        working-directory: build_unix
        run: |
          cat > /tmp/tsan.tcl <<'TCL'
          source ../test/tcl/test.tcl
          foreach t {mut001 lock001 lock002 ssi001 ssi002 ssi009} {
            source ../test/tcl/$t.tcl
            if {[catch {eval $t} res]} { puts "FAIL $t: $res"; exit 1 }
            puts "PASS $t"
          }
          TCL
          # Generous per-test slowdown under TSan, but bounded so a hang fails
          # loudly instead of eating the whole runner budget.
          timeout 1800 tclsh /tmp/tsan.tcl 2>&1 | tee /tmp/tsan.out
          ! grep -qE "^FAIL|data race|WARNING: ThreadSanitizer" /tmp/tsan.out

  # ----------------------------------------------------------------------------
  # Compiler-warnings gate: -Wall -Wextra with gcc + clang. Advisory
  # (continue-on-error) and NOT -Werror -- this is pre-2000 K&R-era C and the
  # autoconf build already suppresses two whole warning classes
  # (-Wno-deprecated-non-prototype / -Wno-knr-promoted-parameter). We surface
  # the count so new warnings in touched code are visible without blocking PRs.
  # ----------------------------------------------------------------------------
  warnings:
    name: warnings gate ${{ matrix.cc }}
    runs-on: ubuntu-latest
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        cc: [gcc, clang]
    env:
      CC: ${{ matrix.cc }}
    steps:
      - uses: actions/checkout@v4
      - name: Configure (-Wall -Wextra)
        working-directory: build_unix
        run: ../dist/configure --enable-debug "CFLAGS=-Wall -Wextra -g -O1"
      - name: Build and count warnings
        working-directory: build_unix
        run: |
          set -o pipefail
          make -j$(nproc) 2>&1 | tee /tmp/build.log
          n=$(grep -c 'warning:' /tmp/build.log || true)
          echo "::notice title=Compiler warnings (${{ matrix.cc }})::$n warning line(s)"
          echo "Top warning kinds:"
          grep 'warning:' /tmp/build.log \
            | grep -oE '\-W[a-z0-9-]+' | sort | uniq -c | sort -rn | head -20 || true

  # ----------------------------------------------------------------------------
  # Mutex backend variants (Linux only). --with-mutex overrides autodetection,
  # so each entry forces a genuinely different mutex implementation:
  #   POSIX/pthreads          -> HAVE_MUTEX_PTHREADS (blocking pthread mutexes)
  #   POSIX/pthreads/library  -> same, linked explicitly against -lpthread
  #   x86_64/gcc-assembly     -> HAVE_MUTEX_X86_64_GCC_ASSEMBLY (test-and-set)
  # posix-forced uses --enable-posixmutexes (intra-process pthread fast path).
  # All names verified against dist/aclocal/mutex.m4 + a local configure run.
  # ----------------------------------------------------------------------------
  mutex-backends:
    name: mutex ${{ matrix.name }}
    runs-on: ubuntu-latest
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        include:
          - { name: pthreads,         flags: '--with-mutex=POSIX/pthreads' }
          - { name: pthreads-library, flags: '--with-mutex=POSIX/pthreads/library' }
          - { name: tas-x86_64,       flags: '--with-mutex=x86_64/gcc-assembly' }
          - { name: posix-forced,     flags: '--enable-posixmutexes' }
    steps:
      - uses: actions/checkout@v4
      - name: Configure (${{ matrix.name }})
        working-directory: build_unix
        run: ../dist/configure --enable-debug ${{ matrix.flags }}
      - name: Build
        working-directory: build_unix
        run: make -j$(nproc)
      - name: Smoke (build ex_access)
        working-directory: build_unix
        run: make ex_access 2>/dev/null || true

  # ----------------------------------------------------------------------------
  # TCL test suite: targeted subset on every push/PR (fast PR signal); the full
  # run is manual (workflow_dispatch full_tcl=true). The generous nightly full
  # suite lives in ci-extended.yml so it never gates PRs.
  # ----------------------------------------------------------------------------
  tcl-tests:
    name: tcl tests (${{ github.event.inputs.full_tcl == 'true' && 'full' || 'targeted' }})
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install Tcl
        run: sudo apt-get update && sudo apt-get install -y tcl-dev tcl
      - name: Configure (test + tcl)
        working-directory: build_unix
        run: ../dist/configure --enable-debug --enable-test --with-tcl=/usr/lib/tcl8.6
      - name: Build
        working-directory: build_unix
        run: make -j$(nproc)
      - name: Run tests
        working-directory: build_unix
        run: |
          if [ "${{ github.event.inputs.full_tcl }}" = "true" ]; then
            cat > /tmp/run.tcl <<'TCL'
          source ../test/tcl/test.tcl
          run_std
          TCL
            timeout 3600 tclsh /tmp/run.tcl 2>&1 | tee /tmp/out.txt
            ! grep -qE "^FAIL|FAIL:" /tmp/out.txt
          else
            # The targeted subset's test list lives in test/MANIFEST, and
            # run_targeted.sh reads it from there and emits a RESULT verdict per
            # test.  It used to be an inline heredoc that exited on the first
            # failure, so every test after that one produced neither a pass nor
            # a fail -- indistinguishable from never having been listed, which
            # is exactly how `ssi` went unsourced and the job exited 0 having
            # tested nothing.
            sh ../test/tcl/run_targeted.sh
          fi

      - name: Manifest gate (assert every targeted TCL test produced a verdict)
        if: always() && github.event.inputs.full_tcl != 'true'
        run: ./test/check_manifest.sh --tier tcl

  # ----------------------------------------------------------------------------
  # Deterministic Simulation Testing (DST): build --enable-dst, run one capstone
  # per fault class + the multi-process failchk pilot, and enforce the
  # zero-overhead-when-off gate (an --enable-dst-off build must contain 0
  # __db_sim_* symbols). The full 41-scenario sweep + planted-bug harness is a
  # local/nightly concern; this is the fast per-push regression signal that the
  # DST layer still builds, runs, and stays compiled-out in production builds.
  # ----------------------------------------------------------------------------
  dst:
    name: dst (fault-class smoke + off-build gate)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Off-build zero-symbol gate
        working-directory: build_unix
        run: |
          # A normal (--enable-dst absent) debug build must compile the DST
          # hooks and every planted-bug/buggify site out entirely.
          ../dist/configure --enable-debug
          make -j$(nproc) libdb.a
          n=$(nm .libs/*.o 2>/dev/null | grep -c '__db_sim_' || true)
          echo "OFF-build __db_sim_* symbols: $n"
          test "$n" -eq 0 || { echo 'FAIL: DST symbols leaked into a non-DST build'; exit 1; }
          make realclean >/dev/null 2>&1 || make clean >/dev/null 2>&1
      - name: Configure + build (--enable-dst)
        working-directory: build_unix
        run: |
          ../dist/configure --enable-debug --enable-dst
          make -j$(nproc)
      # The capstone list lives in test/MANIFEST and run_capstones.sh reads it
      # from there, so the workflow and the expectation cannot drift.  It also
      # runs ALL of them: the old `set -e` loop stopped at the first failure, so
      # the rest produced no signal -- indistinguishable from never listed.
      - name: Fault-class smoke (one capstone per class)
        working-directory: build_unix
        run: sh ../test/sim/run_capstones.sh

      - name: Manifest gate (assert every DST capstone produced a verdict)
        if: always()
        run: ./test/check_manifest.sh --tier sim
      # The multi-process failchk pilot spawns + kill -9's real processes
      # sharing a real region. That is inherently timing-sensitive on shared
      # CI runners (a survivor can stall waiting on a dead peer's slot), so it
      # is best-effort here -- the authoritative sweep runs locally/nightly.
      # We still BUILD it every push (a compile break is a hard signal) and
      # smoke ONE seed with a tight timeout so a hang informs without gating.
      - name: Multi-process failchk pilot (build + 1-seed smoke, best-effort)
        continue-on-error: true
        working-directory: build_unix
        run: |
          make mp_failchk_pilot
          SEEDS=0x1 timeout 90 bash ../test/sim/mp-failchk.sh || \
            echo '::warning title=mp-failchk::pilot smoke did not pass cleanly on this runner (best-effort; run locally for the authoritative sweep)'

  # ----------------------------------------------------------------------------
  # Windows build via the bundled Visual Studio solution (best-effort).
  # ----------------------------------------------------------------------------
  windows:
    name: windows msbuild
    runs-on: windows-latest
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
      - uses: microsoft/setup-msbuild@v2
      - name: Build Berkeley DB library
        shell: cmd
        working-directory: build_windows
        # Use the VS2010 (.vcxproj) solution -- the legacy Berkeley_DB.sln uses
        # VS2008 .vcproj files MSBuild can no longer load (MSB4025). Build only
        # the `db` library project and retarget to the runner's toolset/SDK.
        run: msbuild Berkeley_DB_vs2010.sln /m /t:db /p:Configuration=Release /p:Platform=x64 /p:PlatformToolset=v143 /p:WindowsTargetPlatformVersion=10.0

  # ----------------------------------------------------------------------------
  # Windows ARM64: configuration drift check, plus a best-effort cross-build.
  #
  # The drift check always runs -- it needs only Python, and it is the part that
  # actually guards against a project being added without its ARM64
  # configurations.
  #
  # The cross-build is conditional because the hosted runner image does not ship
  # the MSVC ARM64 cross tools.  (It has VS2026/v180 with the v143 *x64* tools --
  # which is why the x64 job above passes with /p:PlatformToolset=v143 -- but no
  # v143 ARM64, so pinning that toolset fails with MSB8020.)  Rather than pin a
  # toolset that may not exist, probe for an ARM64 cl.exe and let msbuild use the
  # installed default; if there is no ARM64 compiler at all, warn and skip.  When
  # the runner image gains the component this starts verifying for real with no
  # change here.
  #
  # Even when it does run, this can only cross-compile and inspect: GitHub has no
  # Windows-on-ARM runner, so no ARM64 binary is ever executed.  A green result
  # means "still builds, still emits AA64", never that libdb works on ARM64.
  # ----------------------------------------------------------------------------
  windows-arm64:
    name: windows msbuild (ARM64 cross)
    runs-on: windows-latest
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Check the ARM64 configurations are in sync
        run: python dist/win_arm64_configs.py --check
      - uses: microsoft/setup-msbuild@v2
      - name: Look for an ARM64 cross-compiler
        id: arm64tools
        shell: pwsh
        run: |
          $glob = 'C:\Program Files*\Microsoft Visual Studio\*\*\VC\Tools\MSVC\*\bin\Host*\arm64\cl.exe'
          $cl = Get-ChildItem -Path $glob -ErrorAction SilentlyContinue | Select-Object -First 1
          if ($cl) {
            Add-Content $env:GITHUB_OUTPUT 'found=true'
            "ARM64 cross-compiler: $($cl.FullName)"
          } else {
            Add-Content $env:GITHUB_OUTPUT 'found=false'
            "::warning title=arm64-toolset::This runner has no MSVC ARM64 cross-compiler, so the ARM64 cross-build was skipped. The ARM64 configurations were still checked for drift. Build them locally with the VC.Tools.ARM64 component -- see docs_src/guides/installation/win_build_arm64.md."
          }
      - name: Cross-build the library and utilities for ARM64
        if: steps.arm64tools.outputs.found == 'true'
        shell: cmd
        working-directory: build_windows
        # No /p:PlatformToolset here on purpose: pin nothing and let msbuild use
        # whichever toolset the installed VS owns, since that is the install the
        # ARM64 compiler we just found belongs to.
        run: msbuild Berkeley_DB_vs2010.sln /m /t:db;db_archive;db_checkpoint;db_deadlock;db_dump;db_hotbackup;db_load;db_log_verify;db_printlog;db_recover;db_replicate;db_stat;db_tuner;db_upgrade;db_verify /p:Configuration=Release /p:Platform=ARM64 /p:WindowsTargetPlatformVersion=10.0
      - name: Verify every artifact is a genuine ARM64 (AA64) image
        if: steps.arm64tools.outputs.found == 'true'
        shell: pwsh
        working-directory: build_windows/ARM64/Release
        # Read the PE header directly rather than shelling out to dumpbin, so
        # this does not depend on the VC tools being on PATH.  IMAGE_FILE_MACHINE_ARM64
        # is 0xAA64 and lives 4 bytes past the PE signature at e_lfanew.
        run: |
          $bad = 0
          Get-ChildItem -Include *.dll,*.exe -Path * | ForEach-Object {
            $b = [IO.File]::ReadAllBytes($_.FullName)
            $pe = [BitConverter]::ToInt32($b, 0x3C)
            $machine = [BitConverter]::ToUInt16($b, $pe + 4)
            $hex = '0x{0:X4}' -f $machine
            if ($machine -eq 0xAA64) { "OK   $($_.Name)  $hex (ARM64)" }
            else { "FAIL $($_.Name)  $hex (expected 0xAA64)"; $bad++ }
          }
          if ($bad -gt 0) { throw "$bad artifact(s) are not ARM64" }

  # ----------------------------------------------------------------------------
  # Meson/Ninja build of the core library (parallel build system).
  # ----------------------------------------------------------------------------
  meson:
    name: meson/ninja ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Install Meson + Ninja
        run: python -m pip install --upgrade meson ninja
      - name: Configure (Ninja backend)
        run: meson setup build-meson
      - name: Build
        run: ninja -C build-meson
      - name: Smoke test (link + db_version)
        run: |
          cat > /tmp/smoke.c <<'EOF'
          #include <stdio.h>
          #include "db.h"
          int main(void){int a,b,c;char *s=db_version(&a,&b,&c);
            printf("libdb %s (%d.%d.%d)\n",s,a,b,c);return 0;}
          EOF
          cc /tmp/smoke.c -Ibuild-meson/dist -Lbuild-meson/dist -ldb -o /tmp/smoke
          if [ "$RUNNER_OS" = "macOS" ]; then export DYLD_LIBRARY_PATH=build-meson/dist; else export LD_LIBRARY_PATH=build-meson/dist; fi
          /tmp/smoke

  # ----------------------------------------------------------------------------
  # Nix flake: build the default package via `nix build`.
  # ----------------------------------------------------------------------------
  nix-flake:
    name: nix flake build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install Nix (flakes enabled)
        uses: cachix/install-nix-action@v27
        with:
          extra_nix_config: |
            experimental-features = nix-command flakes
      - name: Build the flake's default package
        run: nix build .#libdb-meson --print-build-logs
      - name: Show result
        run: ls -l result/lib