fix(fts): re-evaluate overlay-stale rows without segment scans

.github/workflows/rust.yml

@@ -282,7 +282,41 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev qemu-user
+          sudo apt install -y \
+            protobuf-compiler \
+            libssl-dev \
+            ninja-build \
+            pkg-config \
+            libglib2.0-dev \
+            python3-venv \
+            curl \
+            xz-utils
+      - name: Install QEMU 8.2.10
+        run: |
+          QEMU_VERSION=8.2.10
+          QEMU_ARCHIVE="qemu-${QEMU_VERSION}.tar.xz"
+          QEMU_SOURCE_DIR="${RUNNER_TEMP}/qemu-${QEMU_VERSION}"
+
+          # Ubuntu 24.04 ships QEMU 8.2.2, which is affected by
+          # https://gitlab.com/qemu-project/qemu/-/issues/2170.
+          curl --fail --location --silent --show-error \
+            "https://download.qemu.org/${QEMU_ARCHIVE}" \
+            --output "${RUNNER_TEMP}/${QEMU_ARCHIVE}"
+          echo "37b4a643da8ed6015eef35f5d7f06e7259d9c95359965a0a98e9667c621ab2bb  ${RUNNER_TEMP}/${QEMU_ARCHIVE}" \
+            | sha256sum --check -
+          tar --extract \
+            --file "${RUNNER_TEMP}/${QEMU_ARCHIVE}" \
+            --directory "${RUNNER_TEMP}"
+          mkdir "${QEMU_SOURCE_DIR}/build"
+          (
+            cd "${QEMU_SOURCE_DIR}/build"
+            ../configure --target-list=x86_64-linux-user --disable-docs
+            ninja qemu-x86_64
+          )
+          sudo install -m 0755 \
+            "${QEMU_SOURCE_DIR}/build/qemu-x86_64" \
+            /usr/local/bin/qemu-x86_64
+          qemu-x86_64 --version
       - name: Build lance-linalg lib tests in release mode
         run: |
           cargo test --release -p lance-linalg --lib --no-run

rust/examples/src/hnsw.rs

@@ -129,6 +129,7 @@ async fn main() {
             lower_bound: None,
             upper_bound: None,
             dist_q_c: 0.0,
+            use_acorn: false,
         };
         let results: HashSet<u32> = hnsw
             .search_basic(q.clone(), k, &params, None, vector_store.as_ref())

rust/lance/src/dataset/overlay.rs

@@ -132,35 +132,8 @@ pub fn overlaid_fragments(fragments: &[Fragment]) -> HashMap<u32, &Fragment> {
         .collect()
 }
 
-/// Insert into `stale` the ids of fragments covered by `segment` whose index entries may be
-/// stale because an overlay committed after the segment was built touches a field the segment
-/// indexes. Field-aware and version-gated via [`overlay_exclusion_offsets`].
-///
-/// `overlaid_frags` holds only the fragments that actually carry overlays (rare), so the loop is
-/// `O(overlaid_frags)` rather than `O(fragments the segment covers)`.
-pub fn collect_overlay_stale_frags(
-    segment: &IndexMetadata,
-    overlaid_frags: &HashMap<u32, &Fragment>,
-    stale: &mut RoaringBitmap,
-    schema: &Schema,
-) -> Result<()> {
-    let coverage = segment.fragment_bitmap.as_ref();
-    for (&frag_id, fragment) in overlaid_frags {
-        if stale.contains(frag_id) || !covers_fragment(coverage, frag_id) {
-            continue;
-        }
-        if !stale_offsets_for_fragment(fragment, &segment.fields, segment.dataset_version, schema)?
-            .is_empty()
-        {
-            stale.insert(frag_id);
-        }
-    }
-    Ok(())
-}
-
-/// Like [`collect_overlay_stale_frags`] but with row-level granularity: instead of marking the
-/// whole fragment stale, it computes exactly which row offsets within each covered fragment are
-/// stale and accumulates them into `stale` (fragment_id → stale row offsets).
+/// Compute exactly which row offsets within each covered fragment are stale and accumulate them
+/// into `stale` (fragment_id → stale row offsets).
 ///
 /// Used by the scalar and vector paths to block only the affected rows from index results and
 /// re-evaluate only those rows on the flat path, keeping overhead proportional to the number of
@@ -1350,45 +1323,6 @@ mod tests {
         fragment
     }
 
-    #[test]
-    fn test_collect_frags_missing_bitmap_covers_all() {
-        let schema = flat_test_schema();
-        // A segment with no fragment_bitmap (legacy index predating bitmap tracking) must treat
-        // every overlaid fragment as covered so stale rows can't leak past the index unmasked.
-        let fragment = fragment_with_overlay(3, dense_overlay(vec![3], [1, 2], 9));
-        let overlaid: HashMap<u32, &Fragment> = HashMap::from([(3u32, &fragment)]);
-
-        let mut stale = RoaringBitmap::new();
-        collect_overlay_stale_frags(&segment(vec![3], 1, None), &overlaid, &mut stale, &schema)
-            .unwrap();
-        assert_eq!(stale, bitmap([3]), "missing bitmap must cover fragment 3");
-
-        // A present bitmap that excludes fragment 3 leaves it untouched.
-        let mut stale = RoaringBitmap::new();
-        collect_overlay_stale_frags(
-            &segment(vec![3], 1, Some(bitmap([0]))),
-            &overlaid,
-            &mut stale,
-            &schema,
-        )
-        .unwrap();
-        assert!(
-            stale.is_empty(),
-            "fragment absent from bitmap is not covered"
-        );
-
-        // A present bitmap that includes fragment 3 marks it stale.
-        let mut stale = RoaringBitmap::new();
-        collect_overlay_stale_frags(
-            &segment(vec![3], 1, Some(bitmap([3]))),
-            &overlaid,
-            &mut stale,
-            &schema,
-        )
-        .unwrap();
-        assert_eq!(stale, bitmap([3]));
-    }
-
     #[test]
     fn test_collect_rows_missing_bitmap_covers_all() {
         let schema = flat_test_schema();
@@ -1423,5 +1357,16 @@ mod tests {
             stale.is_empty(),
             "fragment absent from bitmap contributes no rows"
         );
+
+        // A present bitmap that includes fragment 3 contributes its stale rows.
+        let mut stale = HashMap::new();
+        collect_overlay_stale_rows_for_segment(
+            &segment(vec![3], 1, Some(bitmap([3]))),
+            &overlaid,
+            &mut stale,
+            &schema,
+        )
+        .unwrap();
+        assert_eq!(stale.get(&3), Some(&bitmap([1, 2])));
     }
 }