feat(oci): expose streaming and object storage response headers

Author: Bill Leoutsakos

apps/sim/lib/internal/oci/client.server.test.ts

@@ -708,26 +708,233 @@ describe('credential-bound OCI client', () => {
     expect(JSON.stringify(failure)).not.toContain('authorization')
   })
 
-  it('returns only selected safe headers and bounded Uint8Array bodies', async () => {
+  it.each([true, false])('keeps additional response headers opt-in: %s', async (requested) => {
+    const additionalHeaders = {
+      'opc-next-cursor': 'opaque/next+cursor==%2F',
+      'content-length': '3',
+      'last-modified': 'Sat, 05 Sep 2026 12:00:00 GMT',
+      'content-md5': 'content-md5==',
+      'opc-content-md5': 'opc-content-md5==',
+      'opc-multipart-md5': 'multipart-md5==',
+      'content-encoding': 'identity',
+      'content-language': 'en',
+      'content-disposition': 'attachment; filename="report.csv"',
+      'cache-control': 'private, max-age=60',
+      'storage-tier': 'Archive',
+      'archival-state': 'Restored',
+      'time-of-archival': '2026-09-06T12:00:00Z',
+      'version-id': 'opaque-version-id',
+      'is-delete-marker': 'false',
+    }
+    const defaultHeaders = {
+      'content-type': 'application/octet-stream',
+      etag: 'etag-1',
+      'opc-request-id': 'request-1',
+    }
     mocks.secureFetch.mockResolvedValueOnce(
       secureResponse({
         status: 200,
         body: new Uint8Array([1, 2, 3]),
-        headers: { etag: 'etag-1', 'x-provider-secret': 'hidden' },
+        headers: { ...additionalHeaders, ...defaultHeaders, 'x-provider-secret': 'hidden' },
       })
     )
     const { client, endpoint } = await createPreparedClient()
     const result = await client.request({
       endpoint,
       method: 'GET',
       encodedPath: '/v1/test',
-      responseHeaders: ['etag'],
+      responseHeaders: requested
+        ? ['ETAG', ...Object.keys(additionalHeaders).map((name) => name.toUpperCase())]
+        : undefined,
       timeoutMs: 10_000,
       maxResponseBytes: 3,
     })
     expect([...result.body]).toEqual([1, 2, 3])
-    expect(result.headers.etag).toBe('etag-1')
-    expect(result.headers).not.toHaveProperty('x-provider-secret')
+    expect(result.headers).toEqual({
+      ...defaultHeaders,
+      ...(requested ? additionalHeaders : {}),
+    })
+    expect(Object.isFrozen(result.headers)).toBe(true)
+  })
+
+  it('retains an opaque next cursor when the message batch is empty', async () => {
+    const cursor = 'opaque/next+cursor==%2F'
+    mocks.secureFetch.mockResolvedValueOnce(
+      secureResponse({
+        body: '[]',
+        headers: { 'opc-next-cursor': cursor, 'opc-next-page': 'not-a-message-cursor' },
+      })
+    )
+    const { client, endpoint } = await createPreparedClient()
+    const result = await client.request({
+      endpoint,
+      method: 'GET',
+      encodedPath: '/v1/messages',
+      responseHeaders: ['opc-next-cursor'],
+      timeoutMs: 10_000,
+      maxResponseBytes: 1024,
+    })
+    expect(new TextDecoder().decode(result.body)).toBe('[]')
+    expect(result.headers).toEqual({ 'opc-next-cursor': cursor })
+  })
+
+  it('projects HEAD metadata without applying the body limit to the object size', async () => {
+    mocks.secureFetch.mockResolvedValueOnce(
+      secureResponse({
+        headers: { 'content-length': '1099511627776', 'opc-meta-source': 'head metadata' },
+      })
+    )
+    const { client, endpoint } = await createPreparedClient()
+    const result = await client.request({
+      endpoint,
+      method: 'HEAD',
+      encodedPath: '/v1/object',
+      responseHeaders: ['content-length', 'opc-meta-*'],
+      timeoutMs: 10_000,
+      maxResponseBytes: 1,
+    })
+    expect(result.body.byteLength).toBe(0)
+    expect(result.headers).toEqual({
+      'content-length': '1099511627776',
+      'opc-meta-source': 'head metadata',
+    })
+  })
+
+  it.each([true, false])(
+    'projects only explicitly requested object metadata: %s',
+    async (requested) => {
+      mocks.secureFetch.mockResolvedValueOnce(
+        secureResponse({
+          headers: {
+            'OPC-Meta-Source': 'Mixed CASE / café',
+            'opc-meta-empty': '',
+            'opc-meta-': 'missing suffix',
+            'x-opc-meta-secret': 'excluded',
+            'set-cookie': 'excluded=secret',
+            authorization: 'Bearer excluded-secret',
+          },
+        })
+      )
+      const { client, endpoint } = await createPreparedClient()
+      const result = await client.request({
+        endpoint,
+        method: 'GET',
+        encodedPath: '/v1/object',
+        responseHeaders: requested ? ['OPC-META-*', 'opc-meta-*'] : [],
+        timeoutMs: 10_000,
+        maxResponseBytes: 1024,
+      })
+      expect(result.headers).toEqual(
+        requested
+          ? {
+              'opc-meta-source': 'Mixed CASE / café',
+              'opc-meta-empty': '',
+            }
+          : {}
+      )
+      expect(Object.isFrozen(result.headers)).toBe(true)
+    }
+  )
+
+  it.each([4096, 4097])('bounds projected object metadata entries: %i', async (count) => {
+    const headers: Record<string, string> = {}
+    for (let index = 0; index < count; index += 1) {
+      headers[`opc-meta-${index}`] = ''
+    }
+    mocks.secureFetch.mockResolvedValueOnce(secureResponse({ headers }))
+    const { client, endpoint } = await createPreparedClient()
+    const pending = client.request({
+      endpoint,
+      method: 'GET',
+      encodedPath: '/v1/object',
+      responseHeaders: ['opc-meta-*'],
+      timeoutMs: 10_000,
+      maxResponseBytes: 1024,
+    })
+    if (count === 4096) {
+      const result = await pending
+      expect(result.headers).toEqual(headers)
+      expect(Object.isFrozen(result.headers)).toBe(true)
+    } else {
+      await expect(pending).rejects.toMatchObject({
+        code: 'response_too_large',
+        message: 'OCI response exceeded the configured limit',
+      })
+    }
+    expect(mocks.secureFetch).toHaveBeenCalledOnce()
+  })
+
+  it.each([65536, 65537])('bounds metadata names and values by UTF-8 bytes: %i', async (bytes) => {
+    const name = 'opc-meta-test'
+    const valueBytes = bytes - Buffer.byteLength(name, 'utf8')
+    const value = 'é'.repeat(Math.floor(valueBytes / 2)) + 'x'.repeat(valueBytes % 2)
+    expect(Buffer.byteLength(name + value, 'utf8')).toBe(bytes)
+    mocks.secureFetch.mockResolvedValueOnce(secureResponse({ headers: { [name]: value } }))
+    const { client, endpoint } = await createPreparedClient()
+    const pending = client.request({
+      endpoint,
+      method: 'GET',
+      encodedPath: '/v1/object',
+      responseHeaders: ['opc-meta-*'],
+      timeoutMs: 10_000,
+      maxResponseBytes: 1024,
+    })
+    if (bytes === 65536) {
+      expect((await pending).headers).toEqual({ [name]: value })
+    } else {
+      await expect(pending).rejects.toMatchObject({
+        code: 'response_too_large',
+        message: 'OCI response exceeded the configured limit',
+      })
+    }
+  })
+
+  it('applies the metadata byte limit across entries only when requested', async () => {
+    const headers = {
+      'opc-meta-first': 'a'.repeat(40_000),
+      'opc-meta-second': 'b'.repeat(40_000),
+    }
+    mocks.secureFetch.mockResolvedValue(secureResponse({ headers }))
+    const { client, endpoint } = await createPreparedClient()
+    const request = {
+      endpoint,
+      method: 'GET' as const,
+      encodedPath: '/v1/object',
+      timeoutMs: 10_000,
+      maxResponseBytes: 1024,
+    }
+    expect((await client.request(request)).headers).toEqual({})
+    mocks.secureFetch.mockResolvedValueOnce(secureResponse({ headers }))
+    await expect(
+      client.request({ ...request, responseHeaders: ['opc-meta-*'] })
+    ).rejects.toMatchObject({
+      code: 'response_too_large',
+      message: 'OCI response exceeded the configured limit',
+    })
+  })
+
+  it.each([
+    '*',
+    'opc-*',
+    'opc-meta-*suffix',
+    'opc-meta-',
+    'opc-meta-name',
+    'set-cookie',
+    'x-provider-secret',
+  ])('rejects unsupported header selectors before DNS or transport: %s', async (name) => {
+    const { client, endpoint } = await createPreparedClient()
+    await expect(
+      client.request({
+        endpoint,
+        method: 'GET',
+        encodedPath: '/v1/object',
+        responseHeaders: [name],
+        timeoutMs: 10_000,
+        maxResponseBytes: 1024,
+      })
+    ).rejects.toMatchObject({ code: 'invalid_request' })
+    expect(mocks.validateUrl).not.toHaveBeenCalled()
+    expect(mocks.secureFetch).not.toHaveBeenCalled()
   })
 
   it('cancels and classifies a success body beyond the operation limit', async () => {

apps/sim/lib/internal/oci/client.server.ts

@@ -67,6 +67,7 @@ interface OciRequestBase {
   readonly headers?: Readonly<Record<string, string>>
   readonly timeoutMs: number
   readonly maxResponseBytes: number
+  /** Additional allowlisted headers; `opc-meta-*` selects bounded object metadata. */
   readonly responseHeaders?: readonly string[]
   readonly signal?: AbortSignal
 }
@@ -165,14 +166,33 @@ const SIGNING_CONTROLLED_HEADERS: ReadonlySet<string> = new Set([
   'x-content-sha256',
 ])
 const RESPONSE_HEADER_ALLOWLIST: ReadonlySet<string> = new Set([
+  'archival-state',
+  'cache-control',
+  'content-disposition',
+  'content-encoding',
+  'content-language',
+  'content-length',
+  'content-md5',
   'content-type',
   'etag',
+  'is-delete-marker',
+  'last-modified',
   'location',
+  'opc-content-md5',
+  'opc-multipart-md5',
+  'opc-next-cursor',
   'opc-next-page',
   'opc-request-id',
   'opc-work-request-id',
   'retry-after',
+  'storage-tier',
+  'time-of-archival',
+  'version-id',
 ])
+const OBJECT_METADATA_HEADER_PREFIX = 'opc-meta-'
+const OBJECT_METADATA_HEADER_SELECTOR = 'opc-meta-*'
+const MAX_OBJECT_METADATA_HEADERS = 4096
+const MAX_OBJECT_METADATA_HEADER_BYTES = 64 * 1024
 const RETRYABLE_STATUSES: ReadonlySet<number> = new Set([429, 500, 502, 503, 504])
 const RETRYABLE_TRANSPORT_CODES: ReadonlySet<string> = new Set([
   'ECONNRESET',
@@ -526,7 +546,11 @@ function validateRequest(request: OciRequest): {
     throw new OciClientError('invalid_request')
   }
   for (const name of request.responseHeaders ?? []) {
-    if (typeof name !== 'string' || !RESPONSE_HEADER_ALLOWLIST.has(name.toLowerCase())) {
+    if (
+      typeof name !== 'string' ||
+      (!RESPONSE_HEADER_ALLOWLIST.has(name.toLowerCase()) &&
+        name.toLowerCase() !== OBJECT_METADATA_HEADER_SELECTOR)
+    ) {
       throw new OciClientError('invalid_request')
     }
   }
@@ -586,13 +610,36 @@ function selectedResponseHeaders(
   response: SecureFetchResponse,
   requested: readonly string[]
 ): Readonly<Record<string, string>> {
-  const selected = new Set(['content-type', 'etag', 'opc-request-id', ...requested.map(String)])
+  const selected = new Set([
+    'content-type',
+    'etag',
+    'opc-request-id',
+    ...requested.map((name) => name.toLowerCase()),
+  ])
   const result: Record<string, string> = {}
   for (const name of selected) {
-    const normalized = name.toLowerCase()
-    if (!RESPONSE_HEADER_ALLOWLIST.has(normalized)) continue
-    const value = response.headers.get(normalized)
-    if (value !== null) result[normalized] = value
+    if (!RESPONSE_HEADER_ALLOWLIST.has(name)) continue
+    const value = response.headers.get(name)
+    if (value !== null) result[name] = value
+  }
+  if (selected.has(OBJECT_METADATA_HEADER_SELECTOR)) {
+    let count = 0
+    let bytes = 0
+    for (const [name, value] of response.headers) {
+      const normalized = name.toLowerCase()
+      if (
+        !normalized.startsWith(OBJECT_METADATA_HEADER_PREFIX) ||
+        normalized.length === OBJECT_METADATA_HEADER_PREFIX.length
+      ) {
+        continue
+      }
+      count += 1
+      bytes += Buffer.byteLength(normalized, 'utf8') + Buffer.byteLength(value, 'utf8')
+      if (count > MAX_OBJECT_METADATA_HEADERS || bytes > MAX_OBJECT_METADATA_HEADER_BYTES) {
+        throw new OciClientError('response_too_large')
+      }
+      result[normalized] = value
+    }
   }
   return Object.freeze(result)
 }