From 47800f3cc2dcf3200faafd529cf407640291f3b6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 16 Jul 2026 19:06:10 +0800 Subject: [PATCH] chore(deps): upgrade OpenDAL to 0.58 --- Cargo.lock | 124 ++++++++-------- Cargo.toml | 4 +- java/lance-jni/Cargo.lock | 132 +++++++++++------- python/Cargo.lock | 132 +++++++++++------- .../src/object_store/dynamic_opendal.rs | 4 +- .../src/object_store/providers/aws.rs | 3 +- .../src/object_store/providers/azure.rs | 29 ++-- .../src/object_store/providers/gcp.rs | 3 +- .../src/object_store/providers/goosefs.rs | 8 +- .../src/object_store/providers/huggingface.rs | 8 +- .../src/object_store/providers/oss.rs | 3 +- .../src/object_store/providers/tencent.rs | 3 +- .../src/object_store/providers/tos.rs | 3 +- rust/lance-io/tests/goosefs_integration.rs | 2 +- 14 files changed, 258 insertions(+), 200 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7873595a4ef..a690c6070e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1673,15 +1673,6 @@ dependencies = [ "spin 0.10.0", ] -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -5971,9 +5962,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -6034,12 +6025,13 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "77d02c6564e376d3670aaf66ad886cd34f83c0aca407b6364777c624a63e0e2d" dependencies = [ "ctor 1.0.7", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -6057,24 +6049,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "f8564bd76b75d2aea59178cb5b6e9f770be1da73b1bca062720ec151cc56215a" dependencies = [ "anyhow", "base64 0.22.1", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -6083,11 +6073,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b7fd001b204df76be5d2b3f7a81c48b5cf25b28e2cfb3b8a819bb89cdc0ea3a" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "215341e41220815d7945e5b936323ead2248354d2d69573a1cd9d1a30aa09440" dependencies = [ "futures", "http 1.4.2", @@ -6097,9 +6101,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "ed46bc88c1ea9b961c342a796ec790ee5a0ac5a02bf465aea6791684d72ca291" dependencies = [ "log", "opendal-core", @@ -6107,9 +6111,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "f2df70875ab7fd6f80720d4787c49c70883cef0d81bfae947ecba88b8d1cd62e" dependencies = [ "backon", "log", @@ -6118,9 +6122,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "18f1bb30ab396a617ef884863b27ae91fbe966e312ce69876fa41584ab0fc5c6" dependencies = [ "opendal-core", "tokio", @@ -6128,9 +6132,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "d2ca33907a9a1e91e779b753826d96fa96d7798203d63673d1a025f90c138aeb" dependencies = [ "base64 0.22.1", "bytes", @@ -6138,7 +6142,7 @@ dependencies = [ "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6149,17 +6153,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "91b03a371e8266b6995884399dff28b28b71fbb00f5ff8dbb98eb0d7ccb2a839" dependencies = [ "base64 0.22.1", "bytes", "http 1.4.2", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6169,9 +6174,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "f7c2419a8fd504849580943560066f8a68a188763606bc6fa3052f2b4a612972" dependencies = [ "http 1.4.2", "opendal-core", @@ -6179,15 +6184,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "2d2466f1da8abaccc5ef26a19bf0bcc3d9429a45803dbab72ee62ba37e6c1f42" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -6196,9 +6201,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "7cbde2f65ee021ff8849e9ea864aa5cf9ef6a59f4c7a71bc774a8d89ceaa7eef" dependencies = [ "async-trait", "bytes", @@ -6206,7 +6211,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -6217,9 +6222,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "873b66b0c6f4d738293be192e307809cc91476b8456660f59de1a17fdef7b380" dependencies = [ "bytes", "goosefs-sdk", @@ -6231,9 +6236,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "34d3eb9c8b28b59b6c166bb6f2136b830e5aa3a5b55c3d0a80a8d6010fa9c037" dependencies = [ "bytes", "hf-xet", @@ -6241,22 +6246,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "74f73f24e7b5b00b1fbd7b598c877893a517d5c8466b2f61cd00a255fd21a972" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -6265,18 +6269,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "750d9cc8588c19b27c4c2c5d06d7eb6f2bff62549c48652f1f9a7603cd872377" dependencies = [ "base64 0.22.1", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -6286,14 +6290,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "438500cd4218f9e210b332c0861c92187fc3fcb05c86e46f2096b98d5057720b" dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -7015,6 +7019,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" diff --git a/Cargo.toml b/Cargo.toml index 8133dd306db..f30ce858036 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -175,8 +175,8 @@ moka = { version = "0.12", features = ["future", "sync"] } ndarray = { version = "0.16.1", features = ["matrixmultiply-threading"] } num-traits = "0.2" object_store = { version = "0.13.2" } -opendal = { version = "0.57" } -object_store_opendal = { version = "0.57" } +opendal = { version = "0.58" } +object_store_opendal = { version = "0.58" } pin-project = "1.0" path_abs = "0.5" pprof = { version = "0.15.0", features = ["flamegraph"] } diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index cd7ba798a10..65de7ab32c9 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -1361,12 +1361,13 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -4274,7 +4275,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -4797,9 +4798,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -4832,12 +4833,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "77d02c6564e376d3670aaf66ad886cd34f83c0aca407b6364777c624a63e0e2d" dependencies = [ "ctor 1.0.7", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -4855,24 +4857,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "f8564bd76b75d2aea59178cb5b6e9f770be1da73b1bca062720ec151cc56215a" dependencies = [ "anyhow", "base64", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -4881,11 +4881,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b7fd001b204df76be5d2b3f7a81c48b5cf25b28e2cfb3b8a819bb89cdc0ea3a" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "215341e41220815d7945e5b936323ead2248354d2d69573a1cd9d1a30aa09440" dependencies = [ "futures", "http 1.4.2", @@ -4895,9 +4909,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "ed46bc88c1ea9b961c342a796ec790ee5a0ac5a02bf465aea6791684d72ca291" dependencies = [ "log", "opendal-core", @@ -4905,9 +4919,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "f2df70875ab7fd6f80720d4787c49c70883cef0d81bfae947ecba88b8d1cd62e" dependencies = [ "backon", "log", @@ -4916,9 +4930,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "18f1bb30ab396a617ef884863b27ae91fbe966e312ce69876fa41584ab0fc5c6" dependencies = [ "opendal-core", "tokio", @@ -4926,9 +4940,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "d2ca33907a9a1e91e779b753826d96fa96d7798203d63673d1a025f90c138aeb" dependencies = [ "base64", "bytes", @@ -4936,7 +4950,7 @@ dependencies = [ "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -4947,17 +4961,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "91b03a371e8266b6995884399dff28b28b71fbb00f5ff8dbb98eb0d7ccb2a839" dependencies = [ "base64", "bytes", "http 1.4.2", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -4967,9 +4982,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "f7c2419a8fd504849580943560066f8a68a188763606bc6fa3052f2b4a612972" dependencies = [ "http 1.4.2", "opendal-core", @@ -4977,15 +4992,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "2d2466f1da8abaccc5ef26a19bf0bcc3d9429a45803dbab72ee62ba37e6c1f42" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -4994,9 +5009,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "7cbde2f65ee021ff8849e9ea864aa5cf9ef6a59f4c7a71bc774a8d89ceaa7eef" dependencies = [ "async-trait", "bytes", @@ -5004,7 +5019,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -5015,9 +5030,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "873b66b0c6f4d738293be192e307809cc91476b8456660f59de1a17fdef7b380" dependencies = [ "bytes", "goosefs-sdk", @@ -5029,9 +5044,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "34d3eb9c8b28b59b6c166bb6f2136b830e5aa3a5b55c3d0a80a8d6010fa9c037" dependencies = [ "bytes", "hf-xet", @@ -5039,22 +5054,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "74f73f24e7b5b00b1fbd7b598c877893a517d5c8466b2f61cd00a255fd21a972" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5063,18 +5077,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "750d9cc8588c19b27c4c2c5d06d7eb6f2bff62549c48652f1f9a7603cd872377" dependencies = [ "base64", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5084,14 +5098,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "438500cd4218f9e210b332c0861c92187fc3fcb05c86e46f2096b98d5057720b" dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -5516,6 +5530,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6726,6 +6750,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" diff --git a/python/Cargo.lock b/python/Cargo.lock index e9a1f66ebde..6d558baacb1 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -1481,12 +1481,13 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -4569,7 +4570,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -5224,9 +5225,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -5259,12 +5260,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "77d02c6564e376d3670aaf66ad886cd34f83c0aca407b6364777c624a63e0e2d" dependencies = [ "ctor 1.0.7", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -5282,24 +5284,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "f8564bd76b75d2aea59178cb5b6e9f770be1da73b1bca062720ec151cc56215a" dependencies = [ "anyhow", "base64", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -5308,11 +5308,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b7fd001b204df76be5d2b3f7a81c48b5cf25b28e2cfb3b8a819bb89cdc0ea3a" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "215341e41220815d7945e5b936323ead2248354d2d69573a1cd9d1a30aa09440" dependencies = [ "futures", "http 1.4.2", @@ -5322,9 +5336,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "ed46bc88c1ea9b961c342a796ec790ee5a0ac5a02bf465aea6791684d72ca291" dependencies = [ "log", "opendal-core", @@ -5332,9 +5346,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "f2df70875ab7fd6f80720d4787c49c70883cef0d81bfae947ecba88b8d1cd62e" dependencies = [ "backon", "log", @@ -5343,9 +5357,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "18f1bb30ab396a617ef884863b27ae91fbe966e312ce69876fa41584ab0fc5c6" dependencies = [ "opendal-core", "tokio", @@ -5353,9 +5367,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "d2ca33907a9a1e91e779b753826d96fa96d7798203d63673d1a025f90c138aeb" dependencies = [ "base64", "bytes", @@ -5363,7 +5377,7 @@ dependencies = [ "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5374,17 +5388,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "91b03a371e8266b6995884399dff28b28b71fbb00f5ff8dbb98eb0d7ccb2a839" dependencies = [ "base64", "bytes", "http 1.4.2", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5394,9 +5409,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "f7c2419a8fd504849580943560066f8a68a188763606bc6fa3052f2b4a612972" dependencies = [ "http 1.4.2", "opendal-core", @@ -5404,15 +5419,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "2d2466f1da8abaccc5ef26a19bf0bcc3d9429a45803dbab72ee62ba37e6c1f42" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -5421,9 +5436,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "7cbde2f65ee021ff8849e9ea864aa5cf9ef6a59f4c7a71bc774a8d89ceaa7eef" dependencies = [ "async-trait", "bytes", @@ -5431,7 +5446,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -5442,9 +5457,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "873b66b0c6f4d738293be192e307809cc91476b8456660f59de1a17fdef7b380" dependencies = [ "bytes", "goosefs-sdk", @@ -5456,9 +5471,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "34d3eb9c8b28b59b6c166bb6f2136b830e5aa3a5b55c3d0a80a8d6010fa9c037" dependencies = [ "bytes", "hf-xet", @@ -5466,22 +5481,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "74f73f24e7b5b00b1fbd7b598c877893a517d5c8466b2f61cd00a255fd21a972" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5490,18 +5504,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "750d9cc8588c19b27c4c2c5d06d7eb6f2bff62549c48652f1f9a7603cd872377" dependencies = [ "base64", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5511,14 +5525,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "438500cd4218f9e210b332c0861c92187fc3fcb05c86e46f2096b98d5057720b" dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -6192,6 +6206,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -7509,6 +7533,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" diff --git a/rust/lance-io/src/object_store/dynamic_opendal.rs b/rust/lance-io/src/object_store/dynamic_opendal.rs index 3367eacd507..ce9809be1d7 100644 --- a/rust/lance-io/src/object_store/dynamic_opendal.rs +++ b/rust/lance-io/src/object_store/dynamic_opendal.rs @@ -278,7 +278,7 @@ mod tests { "Failed to create memory operator: {e:?}" )) })?; - Ok(OpendalStore::new(operator.finish())) + Ok(OpendalStore::new(operator)) }, ); @@ -316,7 +316,7 @@ mod tests { "Failed to create memory operator: {e:?}" )) })?; - Ok(OpendalStore::new(operator.finish())) + Ok(OpendalStore::new(operator)) }, ) .with_protected_keys(["bucket", "root"]); diff --git a/rust/lance-io/src/object_store/providers/aws.rs b/rust/lance-io/src/object_store/providers/aws.rs index 8cad196f087..e3079e5bb0c 100644 --- a/rust/lance-io/src/object_store/providers/aws.rs +++ b/rust/lance-io/src/object_store/providers/aws.rs @@ -129,8 +129,7 @@ impl AwsStoreProvider { } let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create S3 operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create S3 operator: {:?}", e)))?; Ok(Arc::new(OpendalStore::new(operator)) as Arc) } diff --git a/rust/lance-io/src/object_store/providers/azure.rs b/rust/lance-io/src/object_store/providers/azure.rs index 9de866bff91..3a337e0fac0 100644 --- a/rust/lance-io/src/object_store/providers/azure.rs +++ b/rust/lance-io/src/object_store/providers/azure.rs @@ -106,14 +106,9 @@ impl AzureBlobStoreProvider { config_map.insert("root".to_string(), format!("/{}", prefix)); } - Operator::from_iter::(config_map) - .map_err(|e| { - Error::invalid_input(format!( - "Failed to create Azure Blob operator: {:?}", - e - )) - }) - .map(|b| b.finish()) + Operator::from_iter::(config_map).map_err(|e| { + Error::invalid_input(format!("Failed to create Azure Blob operator: {:?}", e)) + }) } "abfss" => { let filesystem = base_path.username(); @@ -139,14 +134,12 @@ impl AzureBlobStoreProvider { config_map.insert("root".to_string(), format!("/{}", root_path)); } - Operator::from_iter::(config_map) - .map_err(|e| { - Error::invalid_input(format!( - "Failed to create Azure DFS (ADLS Gen2) operator: {:?}", - e - )) - }) - .map(|b| b.finish()) + Operator::from_iter::(config_map).map_err(|e| { + Error::invalid_input(format!( + "Failed to create Azure DFS (ADLS Gen2) operator: {:?}", + e + )) + }) } _ => Err(Error::invalid_input(format!( "Unsupported Azure scheme: {}", @@ -651,8 +644,8 @@ mod tests { let abfss_operator = AzureBlobStoreProvider::build_opendal_operator(&abfss_url, &common_opts).unwrap(); - let azblob_cap = az_operator.info().native_capability(); - let azdls_cap = abfss_operator.info().native_capability(); + let azblob_cap = az_operator.info().capability(); + let azdls_cap = abfss_operator.info().capability(); // Both support basic operations assert!(azblob_cap.read); diff --git a/rust/lance-io/src/object_store/providers/gcp.rs b/rust/lance-io/src/object_store/providers/gcp.rs index 35f79b66804..4720b7f3c3c 100644 --- a/rust/lance-io/src/object_store/providers/gcp.rs +++ b/rust/lance-io/src/object_store/providers/gcp.rs @@ -49,8 +49,7 @@ impl GcsStoreProvider { } let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create GCS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create GCS operator: {:?}", e)))?; Ok(Arc::new(OpendalStore::new(operator)) as Arc) } diff --git a/rust/lance-io/src/object_store/providers/goosefs.rs b/rust/lance-io/src/object_store/providers/goosefs.rs index 5d2a648522b..fe3002bccc0 100644 --- a/rust/lance-io/src/object_store/providers/goosefs.rs +++ b/rust/lance-io/src/object_store/providers/goosefs.rs @@ -171,11 +171,9 @@ impl ObjectStoreProvider for GooseFsStoreProvider { } // Create OpenDAL Operator with GooseFS service - let operator = Operator::from_iter::(config_map) - .map_err(|e| { - Error::invalid_input(format!("Failed to create GooseFS operator: {:?}", e)) - })? - .finish(); + let operator = Operator::from_iter::(config_map).map_err(|e| { + Error::invalid_input(format!("Failed to create GooseFS operator: {:?}", e)) + })?; // Wrap as object_store::ObjectStore via OpendalStore bridge let opendal_store = Arc::new(OpendalStore::new(operator)); diff --git a/rust/lance-io/src/object_store/providers/huggingface.rs b/rust/lance-io/src/object_store/providers/huggingface.rs index cfef0440068..cda56e36fbe 100644 --- a/rust/lance-io/src/object_store/providers/huggingface.rs +++ b/rust/lance-io/src/object_store/providers/huggingface.rs @@ -160,11 +160,9 @@ fn build_hf_store(config_map: HashMap) -> Result { builder = builder.download_mode(download_mode); } - let operator = Operator::new(builder) - .map_err(|e| { - Error::invalid_input(format!("Failed to create Huggingface operator: {:?}", e)) - })? - .finish(); + let operator = Operator::new(builder).map_err(|e| { + Error::invalid_input(format!("Failed to create Huggingface operator: {:?}", e)) + })?; Ok(OpendalStore::new(operator)) } diff --git a/rust/lance-io/src/object_store/providers/oss.rs b/rust/lance-io/src/object_store/providers/oss.rs index b84afb8ed1f..3d116e2e3cc 100644 --- a/rust/lance-io/src/object_store/providers/oss.rs +++ b/rust/lance-io/src/object_store/providers/oss.rs @@ -95,8 +95,7 @@ impl OssStoreProvider { fn build_oss_store(config_map: HashMap) -> Result { let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create OSS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create OSS operator: {:?}", e)))?; Ok(OpendalStore::new(operator)) } diff --git a/rust/lance-io/src/object_store/providers/tencent.rs b/rust/lance-io/src/object_store/providers/tencent.rs index 5fa885ea5a9..d29d5a6ad62 100644 --- a/rust/lance-io/src/object_store/providers/tencent.rs +++ b/rust/lance-io/src/object_store/providers/tencent.rs @@ -80,8 +80,7 @@ impl ObjectStoreProvider for TencentStoreProvider { } let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create COS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create COS operator: {:?}", e)))?; let opendal_store = Arc::new(OpendalStore::new(operator)); diff --git a/rust/lance-io/src/object_store/providers/tos.rs b/rust/lance-io/src/object_store/providers/tos.rs index 923186484c6..7dee659f5f9 100644 --- a/rust/lance-io/src/object_store/providers/tos.rs +++ b/rust/lance-io/src/object_store/providers/tos.rs @@ -101,8 +101,7 @@ impl TosStoreProvider { fn build_tos_store(config_map: HashMap) -> Result { let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create TOS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create TOS operator: {:?}", e)))?; Ok(OpendalStore::new(operator)) } diff --git a/rust/lance-io/tests/goosefs_integration.rs b/rust/lance-io/tests/goosefs_integration.rs index fd9015bcda0..d3f9021f929 100644 --- a/rust/lance-io/tests/goosefs_integration.rs +++ b/rust/lance-io/tests/goosefs_integration.rs @@ -25,7 +25,7 @@ fn get_operator() -> Operator { cfg.insert("master_addr".to_string(), addr); cfg.insert("root".to_string(), "/lance-test/opendal".to_string()); cfg.insert("auth_type".to_string(), auth_type); - Operator::from_iter::(cfg).unwrap().finish() + Operator::from_iter::(cfg).unwrap() } // ============================================================