feat: compute table size including blobs

java/lance-jni/src/fragment.rs

@@ -576,7 +576,7 @@ fn inner_encode_row_ids(env: &mut JNIEnv, row_ids: &JLongArray) -> Result<String
 
 const DATA_FILE_CLASS: &str = "org/lance/fragment/DataFile";
 const DATA_FILE_CONSTRUCTOR_SIG: &str =
-    "(Ljava/lang/String;[I[IIILjava/lang/Long;Ljava/lang/Integer;)V";
+    "(Ljava/lang/String;[I[IIILjava/lang/Long;Ljava/lang/Integer;[J)V";
 const DELETE_FILE_CLASS: &str = "org/lance/fragment/DeletionFile";
 const DELETE_FILE_CONSTRUCTOR_SIG: &str =
     "(JJLjava/lang/Long;Lorg/lance/fragment/DeletionFileType;Ljava/lang/Integer;)V";
@@ -635,6 +635,13 @@ impl IntoJava for &DataFile {
             None => JObject::null(),
         };
         let base_id = convert_to_java_integer(env, self.base_id)?;
+        let blob_bytes = JLance(
+            self.blob_bytes
+                .iter()
+                .map(|v| *v as i64)
+                .collect::<Vec<i64>>(),
+        )
+        .into_java(env)?;
         Ok(env.new_object(
             DATA_FILE_CLASS,
             DATA_FILE_CONSTRUCTOR_SIG,
@@ -646,6 +653,7 @@ impl IntoJava for &DataFile {
                 JValueGen::Int(self.file_minor_version as i32),
                 JValueGen::Object(&file_size_bytes),
                 JValueGen::Object(&base_id),
+                JValueGen::Object(&blob_bytes),
             ],
         )?)
     }
@@ -904,6 +912,12 @@ impl FromJObjectWithEnv<DataFile> for JObject<'_> {
         let file_size_bytes =
             file_size_bytes.map_or(Default::default(), |r| CachedFileSize::new(r as u64));
         let base_id = get_base_id(env, self)?;
+        let blob_bytes_obj = env.call_method(self, "getBlobBytes", "()[J", &[])?.l()?;
+        let blob_bytes: Vec<u64> = if blob_bytes_obj.is_null() {
+            Vec::new()
+        } else {
+            JLongArray::from(blob_bytes_obj).extract_object(env)?
+        };
         Ok(DataFile {
             path,
             fields: fields.into(),
@@ -912,6 +926,7 @@ impl FromJObjectWithEnv<DataFile> for JObject<'_> {
             file_minor_version,
             file_size_bytes,
             base_id,
+            blob_bytes: Arc::from(blob_bytes),
         })
     }
 }

java/lance-jni/src/traits.rs

@@ -309,6 +309,15 @@ impl FromJObjectWithEnv<Vec<u32>> for JLongArray<'_> {
     }
 }
 
+impl FromJObjectWithEnv<Vec<u64>> for JLongArray<'_> {
+    fn extract_object(&self, env: &mut JNIEnv<'_>) -> Result<Vec<u64>> {
+        let len = env.get_array_length(self)?;
+        let mut ret: Vec<i64> = vec![0; len as usize];
+        env.get_long_array_region(self, 0, ret.as_mut_slice())?;
+        Ok(ret.into_iter().map(|val| val as u64).collect())
+    }
+}
+
 impl FromJObjectWithEnv<i32> for JObject<'_> {
     fn extract_object(&self, env: &mut JNIEnv<'_>) -> Result<i32> {
         let ret = env.call_method(self, "intValue", "()I", &[])?.i()?;

java/src/main/java/org/lance/fragment/DataFile.java

@@ -29,6 +29,7 @@ public class DataFile implements Serializable {
   private final int fileMinorVersion;
   private final Long fileSizeBytes;
   private final Integer baseId;
+  private final long[] blobBytes;
 
   public DataFile(
       String path,
@@ -38,13 +39,28 @@ public DataFile(
       int fileMinorVersion,
       Long fileSizeBytes,
       Integer baseId) {
+    this(
+        path, fields, columnIndices, fileMajorVersion, fileMinorVersion, fileSizeBytes, baseId,
+        null);
+  }
+
+  public DataFile(
+      String path,
+      int[] fields,
+      int[] columnIndices,
+      int fileMajorVersion,
+      int fileMinorVersion,
+      Long fileSizeBytes,
+      Integer baseId,
+      long[] blobBytes) {
     this.path = path;
     this.fields = fields;
     this.columnIndices = columnIndices;
     this.fileMajorVersion = fileMajorVersion;
     this.fileMinorVersion = fileMinorVersion;
     this.fileSizeBytes = fileSizeBytes;
     this.baseId = baseId;
+    this.blobBytes = blobBytes == null ? new long[0] : blobBytes;
   }
 
   public String getPath() {
@@ -75,6 +91,16 @@ public Optional<Integer> getBaseId() {
     return Optional.ofNullable(baseId);
   }
 
+  /**
+   * Returns the total size in bytes of the blob payloads backing each field of this file.
+   *
+   * <p>An empty array means blob payload sizes were not recorded (unknown). When non-empty, the
+   * array has exactly one entry per entry in {@link #getFields()}.
+   */
+  public long[] getBlobBytes() {
+    return blobBytes;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
@@ -85,7 +111,8 @@ public boolean equals(Object o) {
         && Objects.equals(path, that.path)
         && Arrays.equals(fields, that.fields)
         && Arrays.equals(columnIndices, that.columnIndices)
-        && Objects.equals(fileSizeBytes, that.fileSizeBytes);
+        && Objects.equals(fileSizeBytes, that.fileSizeBytes)
+        && Arrays.equals(blobBytes, that.blobBytes);
   }
 
   @Override
@@ -98,6 +125,7 @@ public String toString() {
         .add("fileMinorVersion", fileMinorVersion)
         .add("fileSizeBytes", fileSizeBytes)
         .add("baseId", baseId)
+        .add("blobBytes", blobBytes)
         .toString();
   }
 }

protos/table.proto

@@ -445,6 +445,21 @@ message DataFile {
   // The base path index of the data file. Used when the file is imported or referred from another dataset.
   // Lance use it as key of the base_paths field in Manifest to determine the actual base path of the data file.
   optional uint32 base_id = 7;
+
+  // Blob payload bytes stored by this data file for each field in `fields`,
+  // outside the field's regular column pages. This covers blob v2 payloads
+  // written inline (out-of-line buffers inside this file) and payloads spilled
+  // to packed or dedicated sidecar `.blob` files. Bytes of external blobs
+  // (foreign URIs not owned by Lance) are never included.
+  //
+  // Recording the sizes here allows reporting of blob storage sizes without
+  // extra IO, mirroring `IndexFile.size_bytes`.
+  //
+  // When present, there must be one entry per entry in `fields` (zero for
+  // non-blob fields). When empty, blob payload sizes were not recorded —
+  // either the file was written before this field existed or the writer could
+  // not know them (e.g. externally created data files).
+  repeated uint64 blob_bytes = 8;
 } // DataFile
 
 // An overlay file supplies new values for a subset of (row offset, field) cells

python/python/lance/fragment.py

@@ -225,6 +225,10 @@ class DataFile:
         The minor version of the data storage format.
     file_size_bytes : Optional[int]
         The size of the data file in bytes, if available.
+    blob_bytes : List[int]
+        The blob payload bytes stored by this data file, one entry per entry
+        in `fields` (0 for non-blob fields). Empty if blob payload sizes were
+        not recorded.
     """
 
     _path: str
@@ -234,6 +238,7 @@ class DataFile:
     file_minor_version: int = 0
     file_size_bytes: Optional[int] = None
     base_id: Optional[int] = None
+    blob_bytes: List[int] = field(default_factory=list)
 
     def __init__(
         self,
@@ -244,6 +249,7 @@ def __init__(
         file_minor_version: int = 0,
         file_size_bytes: Optional[int] = None,
         base_id: Optional[int] = None,
+        blob_bytes: List[int] = None,
     ):
         # TODO: only we eliminate the path method, we can remove this
         self._path = path
@@ -253,6 +259,7 @@ def __init__(
         self.file_minor_version = file_minor_version
         self.file_size_bytes = file_size_bytes
         self.base_id = base_id
+        self.blob_bytes = blob_bytes or []
 
     def __repr__(self):
         # pretend we have a 'path' attribute

python/src/fragment.rs

@@ -894,6 +894,12 @@ impl FromPyObject<'_, '_> for PyLance<DataFile> {
         let file_size_bytes = CachedFileSize::new(file_size_bytes.unwrap_or(0));
         let fields: Vec<i32> = ob.getattr("fields")?.extract()?;
         let column_indices: Vec<i32> = ob.getattr("column_indices")?.extract()?;
+        // Older DataFile objects may not have a blob_bytes attribute; treat
+        // that the same as "blob payload sizes not recorded" (empty).
+        let blob_bytes: Vec<u64> = ob
+            .getattr("blob_bytes")
+            .and_then(|v| v.extract())
+            .unwrap_or_default();
         Ok(Self(DataFile {
             path: ob.getattr("path")?.extract()?,
             fields: fields.into(),
@@ -902,6 +908,7 @@ impl FromPyObject<'_, '_> for PyLance<DataFile> {
             file_minor_version: ob.getattr("file_minor_version")?.extract()?,
             file_size_bytes,
             base_id: ob.getattr("base_id")?.extract()?,
+            blob_bytes: Arc::from(blob_bytes),
         }))
     }
 }
@@ -926,6 +933,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&DataFile> {
             self.0.file_minor_version,
             file_size_bytes,
             self.0.base_id,
+            self.0.blob_bytes.to_vec(),
         ))
     }
 }

rust/lance-table/benches/manifest_intern.rs

@@ -58,6 +58,7 @@ fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec<pb::DataFragment>
                 file_minor_version: 0,
                 file_size_bytes: 0,
                 base_id: None,
+                blob_bytes: vec![],
             }],
             overlays: vec![],
             deletion_file: None,
@@ -135,6 +136,7 @@ fn make_diverse_pb_fragments(
                     file_minor_version: 0,
                     file_size_bytes: 0,
                     base_id: None,
+                    blob_bytes: vec![],
                 }],
                 overlays: vec![],
                 deletion_file: None,