Skip to content

Commit 7b59ece

Browse files
Bill LeoutsakosBill Leoutsakos
authored andcommitted
feat(oci-document): add native Document Understanding integration
1 parent 3fa59e7 commit 7b59ece

47 files changed

Lines changed: 2857 additions & 7 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

apps/docs/components/ui/icon-mapping.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
482482
notion: NotionIcon,
483483
notion_v2: NotionIcon,
484484
obsidian: ObsidianIcon,
485+
oci_document_understanding: NetSuiteIcon,
485486
okta: OktaIcon,
486487
onedrive: MicrosoftOneDriveIcon,
487488
onepassword: OnePasswordIcon,

apps/docs/content/docs/integrations/meta.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@
185185
"notion",
186186
"notion-service-account",
187187
"obsidian",
188+
"oci_document_understanding",
188189
"okta",
189190
"onedrive",
190191
"onepassword",

apps/docs/content/docs/integrations/oci_document_understanding.mdx

Lines changed: 337 additions & 0 deletions
Large diffs are not rendered by default.

apps/sim/blocks/blocks/oci_document_understanding.ts

Lines changed: 174 additions & 0 deletions
Large diffs are not rendered by default.

apps/sim/blocks/registry-maps.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ import { MothershipBlock } from '@/blocks/blocks/mothership'
238238
import { MSSQLBlock, MSSQLBlockMeta } from '@/blocks/blocks/mssql'
239239
import { MySQLBlock, MySQLBlockMeta } from '@/blocks/blocks/mysql'
240240
import { Neo4jBlock, Neo4jBlockMeta } from '@/blocks/blocks/neo4j'
241+
import { OciDocumentUnderstandingBlock, OciDocumentUnderstandingBlockMeta } from '@/blocks/blocks/oci_document_understanding'
241242
import { NetSuiteBlock, NetSuiteBlockMeta } from '@/blocks/blocks/netsuite'
242243
import { NeverBounceBlock, NeverBounceBlockMeta } from '@/blocks/blocks/neverbounce'
243244
import { NewRelicBlock, NewRelicBlockMeta } from '@/blocks/blocks/new_relic'
@@ -595,6 +596,7 @@ export const BLOCK_REGISTRY: Record<string, BlockConfig> = {
595596
notion: NotionBlock,
596597
notion_v2: NotionV2Block,
597598
obsidian: ObsidianBlock,
599+
oci_document_understanding: OciDocumentUnderstandingBlock,
598600
okta: OktaBlock,
599601
onedrive: OneDriveBlock,
600602
onepassword: OnePasswordBlock,
@@ -920,6 +922,7 @@ export const BLOCK_META_REGISTRY: Record<string, BlockMeta> = {
920922
notion: NotionBlockMeta,
921923
notion_v2: NotionV2BlockMeta,
922924
obsidian: ObsidianBlockMeta,
925+
oci_document_understanding: OciDocumentUnderstandingBlockMeta,
923926
okta: OktaBlockMeta,
924927
onedrive: OneDriveBlockMeta,
925928
onepassword: OnePasswordBlockMeta,

apps/sim/lib/copilot/generated/docs-manifest.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ export const DOCS_MANIFEST: readonly string[] = [
243243
'integrations/notion-service-account.mdx',
244244
'integrations/notion.mdx',
245245
'integrations/obsidian.mdx',
246+
'integrations/oci_document_understanding.mdx',
246247
'integrations/okta.mdx',
247248
'integrations/onedrive.mdx',
248249
'integrations/onepassword.mdx',

apps/sim/lib/integrations/icon-mapping.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
465465
notion: NotionIcon,
466466
notion_v2: NotionIcon,
467467
obsidian: ObsidianIcon,
468+
oci_document_understanding: NetSuiteIcon,
468469
okta: OktaIcon,
469470
onedrive: MicrosoftOneDriveIcon,
470471
onepassword: OnePasswordIcon,
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { createOciClient, type OciAuthenticatedResponse } from '@/lib/internal/oci/client.server'
2+
import { createOciStaticEndpointPolicy } from '@/lib/internal/oci/endpoints'
3+
import { DocumentOperationError } from '@/lib/internal/oci-document-understanding/errors'
4+
import { isDocumentJsonWithinLimit } from '@/tools/oci_document_understanding/shared'
5+
6+
const documentPolicy = createOciStaticEndpointPolicy({ serviceId: 'oci_document_understanding', serviceName: 'document.aiservice', hostnameTemplate: 'regional-oci' })
7+
const storagePolicy = createOciStaticEndpointPolicy({ serviceId: 'oci_document_understanding', serviceName: 'objectstorage', hostnameTemplate: 'regional' })
8+
9+
export async function prepareDocumentClient(input: { credentialId: string; region?: string }, workspaceId: string) {
10+
if (!workspaceId) throw new DocumentOperationError('Workspace context is required', 403)
11+
const client = await createOciClient({ credentialId: input.credentialId, region: input.region, workspaceId, serviceId: 'oci_document_understanding' })
12+
return { client, endpoint: await client.prepareStaticEndpoint(documentPolicy), storage: await client.prepareStaticEndpoint(storagePolicy) }
13+
}
14+
15+
export type PreparedDocumentClient = Awaited<ReturnType<typeof prepareDocumentClient>>
16+
17+
export function documentPath(value: string) {
18+
return encodeURIComponent(value).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)
19+
}
20+
21+
export function documentJsonBody(value: unknown, limit: number) {
22+
if (!isDocumentJsonWithinLimit(value, limit)) throw new DocumentOperationError('Document request exceeds its byte limit', 413)
23+
return new Uint8Array(Buffer.from(JSON.stringify(value), 'utf8'))
24+
}
25+
26+
export function parseDocumentJson(response: OciAuthenticatedResponse): unknown {
27+
try { return JSON.parse(Buffer.from(response.body).toString('utf8')) } catch {
28+
throw new DocumentOperationError('Unexpected Document Understanding JSON response', 502)
29+
}
30+
}
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/**
2+
* @vitest-environment node
3+
*/
4+
import { beforeEach, describe, expect, it, vi } from 'vitest'
5+
6+
const { authorize, principal, provenance, safeKey, safeContributor, download, openPdf, destroyPdf, imageMetadata } = vi.hoisted(() => ({
7+
authorize: vi.fn(), principal: vi.fn(), provenance: vi.fn(), safeKey: vi.fn(),
8+
safeContributor: vi.fn(), download: vi.fn(), openPdf: vi.fn(), destroyPdf: vi.fn(), imageMetadata: vi.fn(),
9+
}))
10+
vi.mock('@/lib/execution/payloads/materialization.server', () => ({ assertUserFileContentAccess: authorize }))
11+
vi.mock('@/lib/internal/principals/executor', () => ({ createExecutorPrincipalFromExecutionContext: principal }))
12+
vi.mock('@sim/auth/principal', () => ({ resolvePrincipalSubject: () => ({ kind: 'sim_user', userId: 'actor-1' }) }))
13+
vi.mock('@/lib/execution/model-input-provenance', () => ({ validateOpaqueModelInputProvenance: provenance }))
14+
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({
15+
isModelSafeWorkspaceFileKey: safeKey,
16+
isOpaqueWorkspaceFileEgressSafe: safeContributor,
17+
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: 'Unsafe workspace file',
18+
}))
19+
vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadServableFileFromStorage: download }))
20+
vi.mock('@/lib/file-parsers/pdfjs-server', () => ({ openPdfDocument: openPdf }))
21+
vi.mock('sharp', () => ({ default: () => ({ metadata: imageMetadata }) }))
22+
vi.mock('@/lib/workspace-files/application/authorization', () => ({ WORKSPACE_FILES_DELEGATION_AUDIENCE: 'workspace-files' }))
23+
24+
import { prepareDocumentSource, validateDocumentBytes } from '@/lib/internal/oci-document-understanding/document-input'
25+
import { type AnalysisInput, documentInputSchema } from '@/lib/internal/oci-document-understanding/schema'
26+
import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types'
27+
28+
const file = {
29+
id: 'file-1', name: 'invoice.pdf', key: 'workspace/workspace-1/file-1',
30+
url: 'https://untrusted.example/ignored', size: 12, type: 'application/pdf',
31+
}
32+
const call: InternalToolOperationCall = {
33+
toolId: 'oci_document_understanding_analyze_document', requestId: 'request-1', headers: new Headers(),
34+
context: { workspaceId: 'workspace-1', workflowId: 'workflow-1', executionId: 'execution-1', userId: 'owner-not-actor' },
35+
}
36+
function input(values: Record<string, unknown> = {}): AnalysisInput {
37+
const parsed = documentInputSchema.parse({
38+
operation: 'analyze_document', credentialId: 'authorized', source: 'file', file,
39+
features: [{ featureType: 'TEXT_EXTRACTION' }], ...values,
40+
})
41+
if (parsed.operation !== 'analyze_document' && parsed.operation !== 'create_processor_job') throw new Error('Expected analysis')
42+
return parsed
43+
}
44+
45+
describe('authorized document inputs', () => {
46+
beforeEach(() => {
47+
vi.clearAllMocks()
48+
principal.mockResolvedValue({ kind: 'session', userId: 'actor-1', sessionId: 'session-1' })
49+
provenance.mockReturnValue({ success: true })
50+
authorize.mockResolvedValue(undefined)
51+
safeKey.mockResolvedValue(true)
52+
safeContributor.mockResolvedValue(true)
53+
download.mockResolvedValue({ buffer: Buffer.from('%PDF-synthetic') })
54+
openPdf.mockResolvedValue({ numPages: 1, destroy: destroyPdf })
55+
imageMetadata.mockResolvedValue({ format: 'png', width: 100, height: 100 })
56+
})
57+
58+
it('authorizes the stored file with the acting principal and bounds the shared download', async () => {
59+
const result = await prepareDocumentSource(input(), call)
60+
expect(authorize).toHaveBeenCalledWith(file, expect.objectContaining({ userId: 'actor-1', workspaceId: 'workspace-1' }))
61+
expect(safeKey).toHaveBeenCalledWith(file.key, { workspaceId: 'workspace-1', actorUserId: 'actor-1' })
62+
expect(download).toHaveBeenCalledWith(file, 'request-1', expect.anything(), expect.objectContaining({ maxBytes: 8_000_000, filePrincipal: expect.objectContaining({ userId: 'actor-1' }) }))
63+
expect(result).toEqual({ source: 'INLINE', data: Buffer.from('%PDF-synthetic').toString('base64') })
64+
expect(destroyPdf).toHaveBeenCalledOnce()
65+
})
66+
67+
it('rejects opaque secret provenance before touching a file or Oracle object', async () => {
68+
provenance.mockReturnValue({ success: false, status: 400, error: 'Unsafe model input' })
69+
await expect(prepareDocumentSource(input(), call)).rejects.toThrow('Unsafe model input')
70+
expect(authorize).not.toHaveBeenCalled()
71+
expect(download).not.toHaveBeenCalled()
72+
})
73+
74+
it.each(['authorization', 'provenance'])('denies %s failures before reading bytes', async (kind) => {
75+
if (kind === 'authorization') authorize.mockRejectedValue(new Error('private detail'))
76+
else safeKey.mockResolvedValue(false)
77+
await expect(prepareDocumentSource(input(), call)).rejects.toThrow(kind === 'authorization' ? 'File is not available' : 'Unsafe workspace file')
78+
expect(download).not.toHaveBeenCalled()
79+
})
80+
81+
it('rejects unsafe contributing files after authorized materialization', async () => {
82+
const contributor = { kind: 'workspace_file', fileId: 'contributor-1' }
83+
download.mockResolvedValue({ buffer: Buffer.from('%PDF-synthetic'), contributingFiles: [contributor] })
84+
safeContributor.mockResolvedValue(false)
85+
await expect(prepareDocumentSource(input(), call)).rejects.toThrow('Unsafe workspace file')
86+
expect(safeContributor).toHaveBeenCalledWith('workspace-1', contributor)
87+
expect(openPdf).not.toHaveBeenCalled()
88+
})
89+
90+
it('uses Oracle namespace/bucket/object locations without treating them as Sim files', async () => {
91+
const objects = [{ namespaceName: 'namespace', bucketName: 'bucket', objectName: 'exact/a b.pdf', pageRange: ['1-3'] }]
92+
const sync = await prepareDocumentSource(input({ source: 'objectStorage', file: undefined, objects }), call)
93+
expect(sync).toEqual({ source: 'OBJECT_STORAGE', ...objects[0] })
94+
const batch = await prepareDocumentSource(input({
95+
operation: 'create_processor_job', source: 'objectStorage', file: undefined, objects,
96+
compartmentId: 'compartment-1', outputLocation: { namespaceName: 'namespace', bucketName: 'results', prefix: 'docs' },
97+
}), call)
98+
expect(batch).toEqual({ sourceType: 'OBJECT_STORAGE_LOCATIONS', objectLocations: objects })
99+
expect(provenance).toHaveBeenCalledTimes(2)
100+
expect(download).not.toHaveBeenCalled()
101+
expect(authorize).not.toHaveBeenCalled()
102+
})
103+
104+
it('rejects URL-only and inline-base64 inputs at the boundary', () => {
105+
expect(() => input({ file: { url: 'https://example.com/private.pdf' } })).toThrow()
106+
expect(() => input({ file: { ...file, base64: 'raw-document' } })).toThrow()
107+
expect(() => input({ file: { ...file, providerFileId: 'file-external' } })).toThrow()
108+
})
109+
110+
it('enforces actual byte and page limits rather than trusting file metadata', async () => {
111+
await expect(validateDocumentBytes(Buffer.alloc(8_000_001))).rejects.toThrow('8,000,000')
112+
openPdf.mockResolvedValue({ numPages: 6, destroy: destroyPdf })
113+
await expect(validateDocumentBytes(Buffer.from('%PDF-synthetic'))).rejects.toThrow('five pages')
114+
expect(destroyPdf).toHaveBeenCalledOnce()
115+
imageMetadata.mockResolvedValue({ format: 'tiff', pages: 6, width: 100, height: 100 })
116+
await expect(validateDocumentBytes(Buffer.from('synthetic TIFF'))).rejects.toThrow('five pages')
117+
imageMetadata.mockResolvedValue({ format: 'png', width: 10001, height: 100 })
118+
await expect(validateDocumentBytes(Buffer.from('synthetic PNG'))).rejects.toThrow('pixels')
119+
imageMetadata.mockResolvedValue({ format: 'webp', width: 100, height: 100 })
120+
await expect(validateDocumentBytes(Buffer.from('synthetic WebP'))).rejects.toThrow('Only JPEG')
121+
})
122+
})
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import { resolvePrincipalSubject } from '@sim/auth/principal'
2+
import { createLogger } from '@sim/logger'
3+
import { assertUserFileContentAccess } from '@/lib/execution/payloads/materialization.server'
4+
import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance'
5+
import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server'
6+
import { DocumentOperationError } from '@/lib/internal/oci-document-understanding/errors'
7+
import { type AnalysisInput, DOCUMENT_INLINE_BYTES } from '@/lib/internal/oci-document-understanding/schema'
8+
import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor'
9+
import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types'
10+
import {
11+
isModelSafeWorkspaceFileKey,
12+
isOpaqueWorkspaceFileEgressSafe,
13+
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
14+
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
15+
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
16+
import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization'
17+
18+
const logger = createLogger('OciDocumentInput')
19+
20+
export async function validateDocumentBytes(buffer: Buffer, signal?: AbortSignal) {
21+
if (buffer.length === 0 || buffer.length > DOCUMENT_INLINE_BYTES) {
22+
throw new DocumentOperationError('Inline documents must contain at most 8,000,000 bytes', 413)
23+
}
24+
signal?.throwIfAborted()
25+
if (buffer.subarray(0, 5).toString('ascii') === '%PDF-') {
26+
const pdf = await openPdfDocument(new Uint8Array(buffer), signal)
27+
try {
28+
if (pdf.numPages > 5) throw new DocumentOperationError('Inline documents must have at most five pages')
29+
} finally {
30+
await pdf.destroy()
31+
}
32+
return
33+
}
34+
const { default: sharp } = await import('sharp')
35+
const metadata = await sharp(buffer, { limitInputPixels: 100_000_000 }).metadata()
36+
signal?.throwIfAborted()
37+
if (!['jpeg', 'png', 'tiff'].includes(metadata.format ?? '')) {
38+
throw new DocumentOperationError('Only JPEG, PNG, PDF and TIFF documents are supported')
39+
}
40+
if ((metadata.pages ?? 1) > 5) throw new DocumentOperationError('Inline documents must have at most five pages')
41+
const height = metadata.pageHeight ?? metadata.height ?? 0
42+
const width = metadata.width ?? 0
43+
if (width < 32 || height < 32 || width > 10000 || height > 10000) {
44+
throw new DocumentOperationError('Document images must be between 32 and 10,000 pixels in each dimension')
45+
}
46+
}
47+
48+
export async function prepareDocumentSource(input: AnalysisInput, request: InternalToolOperationCall) {
49+
const provenance = validateOpaqueModelInputProvenance({ headers: request.headers, payload: input, isInternalRequest: true })
50+
if (!provenance.success) throw new DocumentOperationError(provenance.error, provenance.status)
51+
if (input.source === 'objectStorage') {
52+
return input.operation === 'analyze_document'
53+
? { source: 'OBJECT_STORAGE', ...input.objects![0] }
54+
: { sourceType: 'OBJECT_STORAGE_LOCATIONS', objectLocations: input.objects! }
55+
}
56+
const file = input.file!
57+
const workspaceId = request.context.workspaceId!
58+
const principal = await createExecutorPrincipalFromExecutionContext({
59+
context: request.context,
60+
audience: WORKSPACE_FILES_DELEGATION_AUDIENCE,
61+
})
62+
const subject = resolvePrincipalSubject(principal)
63+
const userId = subject?.kind === 'sim_user' ? subject.userId : undefined
64+
try {
65+
await assertUserFileContentAccess(file, { ...request.context, principal, userId, requestId: request.requestId, logger })
66+
} catch {
67+
throw new DocumentOperationError('File is not available in this execution', 404)
68+
}
69+
request.signal?.throwIfAborted()
70+
if (!(await isModelSafeWorkspaceFileKey(file.key, { workspaceId, actorUserId: userId }))) {
71+
throw new DocumentOperationError(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE)
72+
}
73+
const servable = await downloadServableFileFromStorage(file, request.requestId, logger, {
74+
maxBytes: DOCUMENT_INLINE_BYTES, signal: request.signal, filePrincipal: principal,
75+
})
76+
for (const contributor of servable.contributingFiles ?? []) {
77+
request.signal?.throwIfAborted()
78+
if (!(await isOpaqueWorkspaceFileEgressSafe(workspaceId, contributor))) {
79+
throw new DocumentOperationError(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE)
80+
}
81+
}
82+
await validateDocumentBytes(servable.buffer, request.signal)
83+
request.signal?.throwIfAborted()
84+
return {
85+
...(input.operation === 'analyze_document' ? { source: 'INLINE' } : { sourceType: 'INLINE_DOCUMENT_CONTENT' }),
86+
data: servable.buffer.toString('base64'),
87+
...(input.pageRange ? { pageRange: input.pageRange } : {}),
88+
}
89+
}

0 commit comments

Comments
 (0)