Implemented by SDK streams that can unblock a concurrent {@code read()} or {@code write()}. + * Discovered via {@code instanceof}. + * + *
Contract: + * + *
All methods are called with the stream's internal lock held. Implementations do not need + * their own synchronisation. + */ +@Internal +public interface InputStreamExtension { + + /** Read-only view of stream state passed to extension callbacks. */ + @Internal + interface StreamContext { + + /** Returns the current byte position. */ + long getPos(); + + /** Returns the total content length in bytes. */ + long getContentLength(); + } + + /** + * Opens streams at the current {@linkplain StreamContext#getPos() read position}. + * + *
Called during lazy initialization (first read) and stream recovery (seek beyond + * threshold). The previous streams are closed by the base class before this call. The base + * class manages the lifecycle of the returned streams (reads, closes, reopens on seek). + * + *
If the underlying source cannot do range reads (e.g., encrypted streams opened at offset + * 0), open at offset 0 and skip forward to {@link StreamContext#getPos()} after wrapping. + * + *
On failure, close any streams opened during this call before rethrowing. + * + * @param ctx read-only view of the stream state at the time of opening + * @return the raw and wrapped stream pair + * @throws IOException if opening fails + */ + RawAndWrappedInputStreams openStream(StreamContext ctx) throws IOException; + + /** + * Returns the default extension that opens the raw stream via {@code opener} and wraps it with + * a {@link java.io.BufferedInputStream}. + * + * @param opener opens a raw stream at a given byte position + * @param readBufferSize the buffer size for the {@link java.io.BufferedInputStream} + */ + static InputStreamExtension buffering( + final InputStreamOpener opener, final int readBufferSize) { + return new BufferingInputStreamExtension(opener, readBufferSize); + } +} diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/InputStreamOpener.java b/flink-core/src/main/java/org/apache/flink/core/fs/InputStreamOpener.java new file mode 100644 index 00000000000000..d8705b3d04f448 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/core/fs/InputStreamOpener.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.core.fs; + +import org.apache.flink.annotation.Experimental; +import org.apache.flink.annotation.Internal; + +import javax.annotation.concurrent.NotThreadSafe; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Cloud-agnostic opener that returns an input stream starting at the position described by a {@link + * ReadContext}. + * + *
The implementation captures all cloud-specific state (client, path, bucket) in its closure, + * exposing only the read context to the caller. + * + * @see ReadContext + */ +@Internal +@Experimental +@NotThreadSafe +@FunctionalInterface +public interface InputStreamOpener { + + /** + * Opens an input stream starting at the position indicated by {@code ctx}. + * + * @param ctx context for this read, including the byte offset to start from + * @return an input stream positioned at {@code ctx.getPos()} + * @throws IOException if the stream cannot be opened + */ + InputStream open(ReadContext ctx) throws IOException; +} diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/ObjectStorageFileSystem.java b/flink-core/src/main/java/org/apache/flink/core/fs/ObjectStorageFileSystem.java new file mode 100644 index 00000000000000..530b2c1940b7c9 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/core/fs/ObjectStorageFileSystem.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.core.fs; + +import org.apache.flink.annotation.Experimental; +import org.apache.flink.annotation.Internal; +import org.apache.flink.util.CloseableIterator; + +import java.io.IOException; + +/** + * Optional capability for object storage filesystems that support rich metadata access and + * ETag-guarded atomic renames. + * + *
Implementations expose per-object metadata (e.g., encryption headers) and a conditional + * rename primitive used by re-encryption workflows. Discovered via {@code instanceof}. + * + * @see RichFileStatus + */ +@Internal +@Experimental +public interface ObjectStorageFileSystem { + + /** + * Lazily enumerates all file paths under {@code prefix}, recursively. + * + *
The returned iterator must be closed to release any paging cursors or SDK connections.
+ *
+ * @param prefix the path prefix to enumerate; must not be {@code null}
+ * @return a closeable, lazy iterator over all file paths under {@code prefix}
+ * @throws IOException if enumeration cannot be started
+ */
+ CloseableIterator Performs one HEAD-equivalent request per call.
+ *
+ * @param path the path to query; must not be {@code null}
+ * @return file status enriched with user metadata and ETag
+ * @throws IOException if the status cannot be fetched
+ */
+ RichFileStatus getRichFileStatus(Path path) throws IOException;
+
+ /**
+ * ETag-guarded atomic rename: moves {@code src} to {@code dst} only if the ETags match.
+ *
+ * The move succeeds only when the source ETag matches {@code srcETag} and, if {@code
+ * dstETag} is non-empty, the destination ETag matches {@code dstETag}. An empty {@code dstETag}
+ * asserts that the destination does not exist.
+ *
+ * @param src source path; must not be {@code null}
+ * @param dst destination path; must not be {@code null}
+ * @param srcETag expected ETag of the source object; must not be {@code null}
+ * @param dstETag expected ETag of the destination object, or {@code ""} to assert non-existence
+ * @return {@code true} if the move succeeded; {@code false} if any ETag guard fired
+ * @throws IOException if the storage operation itself failed and the source state is unknown
+ */
+ boolean moveVerified(Path src, Path dst, String srcETag, String dstETag) throws IOException;
+}
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/OutputStreamOpener.java b/flink-core/src/main/java/org/apache/flink/core/fs/OutputStreamOpener.java
new file mode 100644
index 00000000000000..69b047a97b1358
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/OutputStreamOpener.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.fs;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.annotation.Internal;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Cloud-agnostic opener that creates an output stream for a write operation.
+ *
+ * The implementation captures all cloud-specific state (client, path, bucket) in its closure.
+ * The {@link WriteContext} provides encryption metadata to attach to the cloud object before the
+ * stream is opened.
+ */
+@Internal
+@Experimental
+@NotThreadSafe
+@FunctionalInterface
+public interface OutputStreamOpener {
+
+ /**
+ * Opens an output stream for the write operation described by {@code ctx}.
+ *
+ * @param ctx context for this write, including metadata to attach to the cloud object
+ * @return an output stream for writing the file content
+ * @throws IOException if the stream cannot be opened
+ */
+ OutputStream open(WriteContext ctx) throws IOException;
+}
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/RawAndWrappedInputStreams.java b/flink-core/src/main/java/org/apache/flink/core/fs/RawAndWrappedInputStreams.java
new file mode 100644
index 00000000000000..d021801deb1b23
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/RawAndWrappedInputStreams.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.fs;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.util.Preconditions;
+
+import java.io.InputStream;
+
+/**
+ * A pair of streams returned by {@link
+ * InputStreamExtension#openStream(InputStreamExtension.StreamContext)}.
+ */
+@Internal
+public final class RawAndWrappedInputStreams {
+
+ private final InputStream sdk;
+ private final InputStream wrapped;
+
+ /**
+ * Creates a new stream pair.
+ *
+ * @param sdk the raw SDK stream
+ * @param wrapped the wrapped stream used for reads
+ */
+ public RawAndWrappedInputStreams(final InputStream sdk, final InputStream wrapped) {
+ this.sdk = Preconditions.checkNotNull(sdk, "sdk");
+ this.wrapped = Preconditions.checkNotNull(wrapped, "wrapped");
+ }
+
+ /** Returns the raw SDK stream. */
+ public InputStream sdk() {
+ return sdk;
+ }
+
+ /** Returns the wrapped stream used for reads. */
+ public InputStream wrapped() {
+ return wrapped;
+ }
+}
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/ReadContext.java b/flink-core/src/main/java/org/apache/flink/core/fs/ReadContext.java
new file mode 100644
index 00000000000000..3d38b950b83c32
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/ReadContext.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.fs;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * Context passed to {@link InputStreamOpener} describing the desired read position.
+ *
+ * @see InputStreamOpener
+ */
+@Internal
+@Experimental
+@Immutable
+public interface ReadContext {
+
+ /**
+ * Returns the byte offset at which the stream should start.
+ *
+ * @return byte offset; must be ≥ 0
+ */
+ long getPos();
+
+ /**
+ * Creates a {@link ReadContext} for the given byte position.
+ *
+ * @param pos byte offset; must be ≥ 0
+ * @return a {@link ReadContext} that returns {@code pos}
+ * @throws IllegalArgumentException if {@code pos} is negative
+ */
+ static ReadContext of(final long pos) {
+ Preconditions.checkArgument(pos >= 0, "pos must be >= 0");
+ return () -> pos;
+ }
+}
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/RichFileStatus.java b/flink-core/src/main/java/org/apache/flink/core/fs/RichFileStatus.java
new file mode 100644
index 00000000000000..8837f6303e7da8
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/RichFileStatus.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.fs;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.annotation.Internal;
+
+import java.util.Map;
+
+/**
+ * A {@link FileStatus} enriched with cloud-object user metadata and an ETag.
+ *
+ * Returned by {@link ObjectStorageFileSystem#getRichFileStatus(Path)} and used in
+ * re-encryption workflows to read encryption headers and obtain a stable ETag before a conditional
+ * move.
+ *
+ * @see ObjectStorageFileSystem
+ */
+@Internal
+@Experimental
+public interface RichFileStatus extends FileStatus {
+
+ /**
+ * Returns user-defined blob metadata (e.g., Azure {@code PathProperties.getMetadata()}).
+ *
+ * @return an unmodifiable map of metadata key-value pairs; never {@code null}
+ */
+ Map Used for {@link ObjectStorageFileSystem#moveVerified} to guard against concurrent
+ * modifications.
+ *
+ * @return the ETag string; never {@code null}
+ */
+ String getETag();
+}
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/WriteContext.java b/flink-core/src/main/java/org/apache/flink/core/fs/WriteContext.java
new file mode 100644
index 00000000000000..d07837bb4fc305
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/WriteContext.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.fs;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.concurrent.Immutable;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Context passed to {@link OutputStreamOpener} describing the write operation, including encryption
+ * metadata to be persisted with the cloud object.
+ *
+ * @see OutputStreamOpener
+ */
+@Internal
+@Experimental
+@Immutable
+public interface WriteContext {
+
+ /**
+ * Returns the metadata to attach to the cloud object (e.g., encryption headers).
+ *
+ * @return key-value metadata map; never {@code null}
+ */
+ Map