fix: verify ambiguous commit ownership by identity and intent

.github/workflows/nightly_run.yml

@@ -45,6 +45,55 @@ jobs:
           # to avoid OOM issues (these tests use >5GiB memory each)
           cargo test --release --package lance-encoding -- --ignored jumbo --test-threads=1
 
+  commit-conformance-fault-sweep:
+    if: github.repository == 'lance-format/lance'
+    timeout-minutes: 180
+    runs-on: ubuntu-24.04-8x
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
+      - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+        with:
+          workspaces: python
+      - name: Set up uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - name: Install build dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build Lance
+        run: make install
+      - name: Start S3-compatible services
+        working-directory: .
+        run: docker compose -f docker-compose.yml --profile commit-conformance up -d --wait
+      - name: Run local commit conformance sweep
+        env:
+          LANCE_COMMIT_CONFORMANCE_TRACE_DIR: target/commit-conformance-traces
+        run: uv run pytest --run-integration -m "not real_s3" -q python/tests/test_commit_conformance.py
+      - name: Upload commit conformance traces
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: nightly-commit-conformance-traces
+          path: python/target/commit-conformance-traces
+          if-no-files-found: ignore
+          retention-days: 14
+      - name: Stop S3-compatible services
+        if: always()
+        working-directory: .
+        run: docker compose -f docker-compose.yml --profile commit-conformance down -v --remove-orphans --timeout 0
+
   # Cross-version index maintenance-sequence search (see python/tests/compat/compat_sequence.py).
   # Ages an index under the latest release of each of the two previous majors and exercises it
   # under this commit, searching op sequences for panics or correctness divergence. The search is

.github/workflows/python.yml

@@ -18,6 +18,10 @@ on:
       - .github/workflows/build_linux_wheel/**
       - .github/workflows/build_mac_wheel/**
       - .github/workflows/run_tests/**
+      - .github/workflows/run_integtests/**
+      - .github/workflows/nightly_run.yml
+      - ci/run_real_s3_commit_conformance.sh
+      - docker-compose.yml
 
 permissions:
   contents: read
@@ -320,3 +324,11 @@ jobs:
           pip install ray[data]
           pip install torch --index-url https://download.pytorch.org/whl/cpu
       - uses: ./.github/workflows/run_integtests
+      - name: Upload commit conformance traces
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: commit-conformance-traces
+          path: python/target/commit-conformance-traces
+          if-no-files-found: ignore
+          retention-days: 7

.github/workflows/run_integtests/action.yml

@@ -9,10 +9,10 @@ runs:
     shell: bash
     run: |
       pip3 install $(ls target/wheels/pylance-*.whl)[tests,ray]
-  - name: Start localstack
+  - name: Start S3-compatible services
     shell: bash
     run: |
-      docker compose -f docker-compose.yml up -d --wait
+      docker compose -f docker-compose.yml --profile commit-conformance up -d --wait
   - name: Run python tests
     shell: bash
     working-directory: python

ci/run_real_s3_commit_conformance.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+set -euo pipefail
+
+if [[ -z "${LANCE_CONFORMANCE_REAL_S3_BUCKET:-}" ]]; then
+  echo "LANCE_CONFORMANCE_REAL_S3_BUCKET must name an existing test bucket" >&2
+  exit 2
+fi
+
+for command in flock getent iptables setsid sudo; do
+  if ! command -v "${command}" >/dev/null 2>&1; then
+    echo "${command} is required by the fail-closed real-S3 runner" >&2
+    exit 2
+  fi
+done
+sudo -n true
+
+region="${LANCE_CONFORMANCE_REAL_S3_REGION:-us-east-2}"
+proxy_port="${LANCE_CONFORMANCE_PROXY_PORT:-18080}"
+s3_host="s3.${region}.amazonaws.com"
+test_user_id="$(id -u)"
+nat_chain="LANCE_CC_S3"
+guard_chain="LANCE_CC_S3_GUARD"
+hosts_marker="# lance-commit-conformance"
+lock_file="/tmp/lance-commit-conformance.lock"
+test_pid=""
+
+exec 9>"${lock_file}"
+if ! flock -n 9; then
+  echo "another real-S3 commit conformance run holds ${lock_file}" >&2
+  exit 2
+fi
+
+remove_stale_network_state() {
+  sudo iptables -t nat -D OUTPUT \
+    -p tcp -d 127.0.0.1/32 --dport 80 \
+    -m owner --uid-owner "${test_user_id}" -j "${nat_chain}" \
+    2>/dev/null || true
+  sudo iptables -D OUTPUT \
+    -p tcp -d 127.0.0.1/32 --dport 80 \
+    -m owner --uid-owner "${test_user_id}" -j "${guard_chain}" \
+    2>/dev/null || true
+  sudo iptables -t nat -F "${nat_chain}" 2>/dev/null || true
+  sudo iptables -t nat -X "${nat_chain}" 2>/dev/null || true
+  sudo iptables -F "${guard_chain}" 2>/dev/null || true
+  sudo iptables -X "${guard_chain}" 2>/dev/null || true
+  sudo sed -i "\|${hosts_marker}$|d" /etc/hosts
+}
+
+cleanup() {
+  status=$?
+  trap - EXIT INT TERM
+  set +e
+  if [[ -n "${test_pid}" ]] && kill -0 "${test_pid}" 2>/dev/null; then
+    kill -TERM -- "-${test_pid}" 2>/dev/null || true
+    for _ in {1..20}; do
+      kill -0 "${test_pid}" 2>/dev/null || break
+      sleep 0.1
+    done
+    kill -KILL -- "-${test_pid}" 2>/dev/null || true
+    wait "${test_pid}" 2>/dev/null || true
+  fi
+  remove_stale_network_state
+  if grep -Fq "${hosts_marker}" /etc/hosts; then
+    echo "failed to remove the real-S3 /etc/hosts isolation entry" >&2
+    status=1
+  fi
+  exit "${status}"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+# Recover exact state left by a previously killed runner before resolving the
+# real upstream address. The generated /etc/hosts entry maps only the regional
+# S3 hostname and is tagged with hosts_marker.
+remove_stale_network_state
+upstream_ip="$(getent ahostsv4 "${s3_host}" | awk 'NR == 1 { print $1 }')"
+if [[ -z "${upstream_ip}" || "${upstream_ip}" == "127.0.0.1" ]]; then
+  echo "could not resolve a non-loopback IPv4 address for ${s3_host}" >&2
+  exit 2
+fi
+
+# Resolve the test client's S3 hostname to loopback. Its only port-80 route is
+# redirected to the fault proxy. The filter guard rejects the request if the
+# NAT redirect disappears, so signed plaintext cannot escape to real S3.
+printf '127.0.0.1 %s %s\n' "${s3_host}" "${hosts_marker}" | sudo tee -a /etc/hosts >/dev/null
+sudo iptables -t nat -N "${nat_chain}"
+sudo iptables -t nat -A "${nat_chain}" \
+  -p tcp -j REDIRECT --to-ports "${proxy_port}"
+sudo iptables -t nat -I OUTPUT 1 \
+  -p tcp -d 127.0.0.1/32 --dport 80 \
+  -m owner --uid-owner "${test_user_id}" -j "${nat_chain}"
+sudo iptables -N "${guard_chain}"
+sudo iptables -A "${guard_chain}" -j REJECT
+sudo iptables -I OUTPUT 1 \
+  -p tcp -d 127.0.0.1/32 --dport 80 \
+  -m owner --uid-owner "${test_user_id}" -j "${guard_chain}"
+
+cd "$(dirname "$0")/../python"
+export LANCE_COMMIT_CONFORMANCE_TRACE_DIR="${LANCE_COMMIT_CONFORMANCE_TRACE_DIR:-target/commit-conformance-traces}"
+export LANCE_CONFORMANCE_REAL_S3_ISOLATED=1
+export LANCE_CONFORMANCE_REAL_S3_UPSTREAM_IP="${upstream_ip}"
+setsid uv run --frozen pytest --run-integration -m "recurring and real_s3" -q \
+  python/tests/test_commit_conformance.py &
+test_pid=$!
+wait "${test_pid}"
+test_pid=""

docker-compose.yml

@@ -1,9 +1,8 @@
-version: "3.9"
 services:
   localstack:
     image: localstack/localstack:4.0
     ports:
-      - 4566:4566
+      - 127.0.0.1:4566:4566
     environment:
       - SERVICES=s3,dynamodb,kms
       - DOCKER_HOST=unix:///var/run/docker.sock
@@ -15,3 +14,17 @@ services:
       interval: 5s
       retries: 3
       start_period: 10s
+  minio:
+    image: minio/minio:RELEASE.2025-09-07T16-13-09Z
+    profiles: ["commit-conformance"]
+    command: server /data
+    ports:
+      - 127.0.0.1:9000:9000
+    environment:
+      - MINIO_ROOT_USER=ACCESS_KEY
+      - MINIO_ROOT_PASSWORD=SECRET_KEY
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 5s
+      retries: 3
+      start_period: 10s

python/DEVELOPMENT.md

@@ -267,19 +267,45 @@ sophisticated enough to represent asynchronous parallel work.
 As a result, a single instrumented async method may appear as many different
 spans in the UI.
 
-## Running S3 Integration tests
+## Running S3 integration tests
 
-The integration tests run against local minio and local dynamodb. To start the
-services, run
+The integration tests run against MinIO and LocalStack DynamoDB. The standard
+entry point starts both services, runs the integration tests, and tears the
+services down:
 
 ```shell
-docker compose up
+uv run make integtest
 ```
 
-Then you can run the tests with
+The PR commit-conformance matrix covers append, compaction, add-columns, and
+create-index through both conditional object-store and DynamoDB external
+metadata commits. Each operation is checked under lost commit responses, lost
+verification responses, and a deterministic foreign-writer race. The oracle
+checks the complete visible version history, the latest-opened version, unique
+transaction identities, and the correspondence between successful writer
+results and persisted transactions. Request traces include executable fault
+plans and are written to `target/commit-conformance-traces`.
+
+The complete fault sweep is marked `recurring` and runs nightly. To run it
+locally against already-started services:
+
+```shell
+uv run pytest --run-integration -m "recurring and not real_s3" \
+  python/tests/test_commit_conformance.py
+```
+
+Real S3 response-loss validation requires a Linux host with AWS credentials and
+an existing test bucket. Always use the runner: the tests refuse to send real-S3
+traffic unless it has installed the fail-closed network guard. The runner maps
+only the regional S3 hostname to loopback, redirects that loopback traffic to
+the fault proxy, and rejects it if the redirect is missing. The proxy connects
+to a pre-resolved S3 address over HTTPS while preserving TLS hostname
+verification. A lock and exact cleanup markers make concurrent or stale runs
+detectable.
 
 ```shell
-pytest --run-integration python/tests/test_s3_ddb.py
+LANCE_CONFORMANCE_REAL_S3_BUCKET=my-test-bucket \
+  ../ci/run_real_s3_commit_conformance.sh
 ```
 
 ## Building wheels locally

python/Makefile

@@ -4,6 +4,8 @@ PYTHON ?=
 PYTEST_ARGS ?= -vvv -s -m "not recurring"
 KEEP_COMPOSE ?= 0
 COMPOSE_FILE ?= ../docker-compose.yml
+COMPOSE_PROFILE ?= commit-conformance
+COMMIT_CONFORMANCE_TRACE_DIR ?= target/commit-conformance-traces
 UV_SYNC = uv sync
 UV_RUN = uv run --frozen
 
@@ -35,14 +37,14 @@ build: ## Build the local Rust extension with maturin
 test: ## Run Python tests except recurring tests
 	pytest $(PYTEST_ARGS) python/tests
 
-integtest: ## Start LocalStack and run integration tests
+integtest: ## Start S3-compatible services and run integration tests
 	@if [ "$(KEEP_COMPOSE)" = "1" ]; then \
-		docker compose -f $(COMPOSE_FILE) up -d --wait && \
-		pytest --run-integration $(PYTEST_ARGS) python/tests/test_s3_ddb.py python/tests/test_namespace_integration.py; \
+		docker compose -f $(COMPOSE_FILE) --profile $(COMPOSE_PROFILE) up -d --wait && \
+		LANCE_COMMIT_CONFORMANCE_TRACE_DIR=$(COMMIT_CONFORMANCE_TRACE_DIR) pytest --run-integration $(PYTEST_ARGS) python/tests/test_s3_ddb.py python/tests/test_commit_conformance.py python/tests/test_namespace_integration.py; \
 	else \
-		trap 'docker compose -f $(COMPOSE_FILE) down -v --remove-orphans --timeout 0 >/dev/null 2>&1 || true' EXIT; \
-		docker compose -f $(COMPOSE_FILE) up -d --wait && \
-		pytest --run-integration $(PYTEST_ARGS) python/tests/test_s3_ddb.py python/tests/test_namespace_integration.py; \
+		trap 'docker compose -f $(COMPOSE_FILE) --profile $(COMPOSE_PROFILE) down -v --remove-orphans --timeout 0 >/dev/null 2>&1 || true' EXIT; \
+		docker compose -f $(COMPOSE_FILE) --profile $(COMPOSE_PROFILE) up -d --wait && \
+		LANCE_COMMIT_CONFORMANCE_TRACE_DIR=$(COMMIT_CONFORMANCE_TRACE_DIR) pytest --run-integration $(PYTEST_ARGS) python/tests/test_s3_ddb.py python/tests/test_commit_conformance.py python/tests/test_namespace_integration.py; \
 	fi
 
 doctest: ## Run Python doctests

python/pyproject.toml

@@ -115,7 +115,11 @@ include = [
     "python/lance/util.py",
     "python/lance/arrow.py",
     "python/tests/test_arrow.py",
+    "python/tests/_commit_conformance.py",
+    "python/tests/_commit_fault_proxy.py",
+    "python/tests/test_commit_conformance.py",
 ]
+extraPaths = ["python/tests"]
 # Dependencies like pyarrow make this difficult to enforce strictly.
 reportMissingTypeStubs = "warning"
 reportImportCycles = "error"
@@ -132,6 +136,7 @@ markers = [
     "slow",
     "torch: tests which rely on pytorch being installed",
     "recurring: marks tests as recurring tests",
+    "real_s3: tests that require the fail-closed real-AWS S3 runner",
 ]
 filterwarnings = [
     'error::FutureWarning',