diff --git a/.agents/upstream-review.md b/.agents/upstream-review.md index f487fdac3..a1b546247 100644 --- a/.agents/upstream-review.md +++ b/.agents/upstream-review.md @@ -1,8 +1,8 @@ --- remote: t3code-upstream branch: main -reviewed-through: "a43f9b45ae85caf37e0be8270ad3d27365ece2bd" -reviewed-through-date: "2026-09-13" +reviewed-through: "bbedad0278bbf753503184c00e0c09a0eab6679c" +reviewed-through-date: "2026-09-14" --- # T3 upstream decision index @@ -138,6 +138,12 @@ The maintainer explicitly reopened the four remaining functional exceptions on 2 All three sources in the range `d1d15c67f4a5fb82fd8d5e01e5e3b288296789c3..a43f9b45ae85caf37e0be8270ad3d27365ece2bd` are accounted for in [#536](https://github.com/pylon-code/pylon/issues/536). This advances the decision cursor without merging upstream ancestry. The opening trigger audit found WATCH-1 still open/unmerged; DEF-7, WATCH-2 and WATCH-3 retain their earliest 2026-11-01 checks. +## Completed cycle through `bbedad0278` + +| Group / bounded head | Sources | Outcome and remaining scope | Pylon PR / verification | +| ----------------------------------------------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Mobile off-thread JPEG rendering / `bbedad0278bbf753503184c00e0c09a0eab6679c` | `bbedad0278bbf753503184c00e0c09a0eab6679c` | Adopted: render photo library picks to a bounded JPEG (longest edge 2048px, quality 0.85) off the JS thread using `expo-image-manipulator`. Avoids UI freezes and high-resolution OOMs on mobile; preserves raw originals for small supported PNG/GIF/WebP. Preserved Pylon branding in composer documentation. | Implementation PR; independent review, 1694 mobile tests (including 46 composer file tests), mobile typecheck, and final-head CI. | + ## Deferred register No open functional deferrals remain in this register. DEF-7 and DEF-16 are completed above; remaining exact native rollback compatibility limits are documented above. Revisit a limit when the native provider exposes the missing proof or fork behavior; do not substitute relative counts. Historical reasons and revisit conditions remain in the archive and earlier cycle records. diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 50632f0f3..df96bfa18 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -91,6 +91,7 @@ "expo-glass-effect": "~57.0.1", "expo-haptics": "~57.0.2", "expo-image": "~57.0.3", + "expo-image-manipulator": "~57.0.17", "expo-image-picker": "~57.0.14", "expo-linking": "~57.0.8", "expo-network": "~57.0.1", diff --git a/apps/mobile/src/lib/composerFiles.test.ts b/apps/mobile/src/lib/composerFiles.test.ts index b38c0813c..962be3501 100644 --- a/apps/mobile/src/lib/composerFiles.test.ts +++ b/apps/mobile/src/lib/composerFiles.test.ts @@ -11,6 +11,8 @@ const mocks = vi.hoisted(() => ({ open: vi.fn(), size: vi.fn(), readBase64: vi.fn(), + manipulate: vi.fn(), + release: vi.fn(), })); vi.mock("expo-file-system", () => { @@ -80,6 +82,10 @@ vi.mock("expo-file-system", () => { vi.mock("expo-image-picker", () => ({ launchImageLibraryAsync: mocks.pickMedia })); vi.mock("expo-document-picker", () => ({ getDocumentAsync: mocks.pickFile })); +vi.mock("expo-image-manipulator", () => ({ + SaveFormat: { JPEG: "jpeg", PNG: "png", WEBP: "webp" }, + ImageManipulator: { manipulate: mocks.manipulate }, +})); vi.mock("./uuid", () => ({ uuidv4: () => "attachment-id" })); import { @@ -106,20 +112,50 @@ describe("composer file attachments", () => { }); describe("photo library image conversion", () => { - const jpeg = "/9j/2Q=="; + const rendered = { uri: "file:///cache/ImageManipulator/photo.jpg", base64: "/9j/2Q==" }; const photo: ImagePickerAsset = { uri: "file:///picker/photo.heic", type: "image", fileName: "photo.HEIC", mimeType: "image/heic", - fileSize: 20 * 1024 * 1024, - base64: jpeg, - width: 1, - height: 1, + fileSize: 4 * 1024 * 1024, + width: 4032, + height: 3024, + }; + /** + * Stands in for the native manipulator: `size` is what decoding the source yields, + * `resizes` records every requested resize, and `saved` is what saving returns. + */ + const native = { + size: { width: 1, height: 1 }, + resizes: [] as Array<{ width?: number | null; height?: number | null }>, + saved: rendered as { uri: string; base64?: string }, }; + beforeEach(() => { + native.size = { width: 1, height: 1 }; + native.resizes = []; + native.saved = rendered; + mocks.manipulate.mockReset(); + mocks.release.mockReset(); + mocks.manipulate.mockImplementation(() => { + const context = { + resize(size: { width?: number | null; height?: number | null }) { + native.resizes.push(size); + return context; + }, + renderAsync: async () => ({ + ...native.size, + release: mocks.release, + saveAsync: async () => native.saved, + }), + }; + return context; + }); + }); + it.each(["image/heic", "image/heif", undefined])( - "attaches the native JPEG conversion with matching metadata when the source MIME is %s", + "renders a %s photo to JPEG natively and previews the rendered file", async (mimeType) => { mocks.pickMedia.mockResolvedValue({ canceled: false, @@ -128,6 +164,9 @@ describe("composer file attachments", () => { const result = await pickComposerImages({ existingCount: 0 }); + expect(mocks.pickMedia).toHaveBeenCalledWith(expect.objectContaining({ base64: false })); + expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri); + expect(mocks.readBase64).not.toHaveBeenCalled(); expect(result).toEqual({ images: [ { @@ -136,8 +175,8 @@ describe("composer file attachments", () => { name: "photo.jpg", mimeType: "image/jpeg", sizeBytes: 4, - dataUrl: `data:image/jpeg;base64,${jpeg}`, - previewUri: `data:image/jpeg;base64,${jpeg}`, + dataUrl: `data:image/jpeg;base64,${rendered.base64}`, + previewUri: rendered.uri, }, ], error: null, @@ -145,11 +184,26 @@ describe("composer file attachments", () => { }, ); + it.each([ + { size: { width: 4032, height: 3024 }, resizes: [{ width: 2048 }] }, + { size: { width: 3024, height: 4032 }, resizes: [{ height: 2048 }] }, + { size: { width: 2048, height: 1536 }, resizes: [] }, + ])("bounds a $size.width x $size.height photo to a 2048 px longest edge", async (input) => { + native.size = input.size; + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] }); + + await pickComposerImages({ existingCount: 0 }); + + expect(native.resizes).toEqual(input.resizes); + // Every decoded bitmap is released, including the full-size one a resize replaces. + expect(mocks.release).toHaveBeenCalledTimes(input.resizes.length + 1); + }); + it.each([ { extension: "png", mimeType: "image/png", base64: "iVBORw0KGgo=" }, { extension: "gif", mimeType: "image/gif", base64: "R0lGODlh" }, { extension: "webp", mimeType: "image/webp", base64: "UklGRgQAAABXRUJQ" }, - ])("preserves original $extension bytes instead of the picker's JPEG", async (original) => { + ])("keeps original $extension bytes from the picker file", async (original) => { const name = `photo.${original.extension}`; mocks.pickMedia.mockResolvedValue({ canceled: false, @@ -159,6 +213,8 @@ describe("composer file attachments", () => { const result = await pickComposerImages({ existingCount: 0 }); + expect(mocks.manipulate).not.toHaveBeenCalled(); + expect(mocks.readBase64).toHaveBeenCalledWith(photo.uri); expect(result.error).toBeNull(); expect(result.images).toEqual([ expect.objectContaining({ @@ -166,37 +222,74 @@ describe("composer file attachments", () => { mimeType: original.mimeType, dataUrl: `data:${original.mimeType};base64,${original.base64}`, sizeBytes: Buffer.from(original.base64, "base64").byteLength, + previewUri: photo.uri, }), ]); }); - it("checks the converted JPEG size even when the HEIC source was smaller", async () => { - const oversized = - jpeg.slice(0, 4) + "A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4); + it("renders a supported original that exceeds the image limit instead of rejecting it", async () => { mocks.pickMedia.mockResolvedValue({ canceled: false, - assets: [{ ...photo, fileSize: 42, base64: oversized }], + assets: [{ ...photo, fileName: "photo.jpg", mimeType: "image/jpeg" }], }); + mocks.size.mockReturnValue(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1); - await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({ - images: [], - error: "'photo.HEIC' exceeds the 10 MB attachment limit.", + const result = await pickComposerImages({ existingCount: 0 }); + + expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri); + expect(result.error).toBeNull(); + expect(result.images).toEqual([ + expect.objectContaining({ name: "photo.jpg", mimeType: "image/jpeg", sizeBytes: 4 }), + ]); + }); + + it("measures the picker file instead of trusting the reported size", async () => { + // A content stream can deliver more bytes than the picker advertises; a supported + // original only skips rendering when the file itself measures within the limit. + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [{ ...photo, fileName: "photo.png", mimeType: "image/png", fileSize: 42 }], }); + mocks.size.mockReturnValue(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1); + + const result = await pickComposerImages({ existingCount: 0 }); + + expect(mocks.readBase64).not.toHaveBeenCalled(); + expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri); + expect(result.images).toEqual([expect.objectContaining({ mimeType: "image/jpeg" })]); }); - it("does not relabel unconverted HEIC bytes as JPEG", async () => { + it("renders a supported original whose size cannot be measured", async () => { mocks.pickMedia.mockResolvedValue({ canceled: false, - assets: [{ ...photo, base64: "AAAAGGZ0eXBoZWlj" }], + assets: [ + { ...photo, uri: "content://media/1", fileName: "photo.png", mimeType: "image/png" }, + ], }); const result = await pickComposerImages({ existingCount: 0 }); - expect(result.images).toEqual([]); - expect(result.error).toContain("not a supported image type"); + expect(mocks.readBase64).not.toHaveBeenCalled(); + expect(mocks.manipulate).toHaveBeenCalledWith("content://media/1"); + expect(result.images).toEqual([expect.objectContaining({ mimeType: "image/jpeg" })]); }); - it("retains a converted photo when another original cannot be read", async () => { + it("checks the rendered JPEG against the image limit", async () => { + native.saved = { + uri: rendered.uri, + base64: + rendered.base64.slice(0, 4) + + "A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4), + }; + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] }); + + await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({ + images: [], + error: "'photo.HEIC' exceeds the 10 MB attachment limit.", + }); + }); + + it("retains a rendered photo when another original cannot be read", async () => { mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [{ ...photo, fileName: "missing.gif", mimeType: "image/gif" }, photo], @@ -208,6 +301,21 @@ describe("composer file attachments", () => { expect(result.images).toEqual([expect.objectContaining({ name: "photo.jpg" })]); expect(result.error).toBe("Failed to read 'missing.gif'."); }); + + it("reports a photo the native renderer cannot decode", async () => { + mocks.manipulate.mockImplementation(() => ({ + resize: () => { + throw new Error("unreachable"); + }, + renderAsync: () => Promise.reject(new Error("corrupt")), + })); + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] }); + + await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({ + images: [], + error: "Failed to read 'photo.HEIC'.", + }); + }); }); describe("photo library videos", () => { @@ -217,10 +325,13 @@ describe("composer file attachments", () => { fileName: "photo.png", mimeType: "image/png", fileSize: 3, - base64: "YWJj", width: 1, height: 1, }; + + beforeEach(() => { + mocks.readBase64.mockResolvedValue("YWJj"); + }); const video: ImagePickerAsset = { uri: "file:///picker/clip.mov", type: "video", @@ -234,7 +345,9 @@ describe("composer file attachments", () => { it("retains mixed photos and videos, keeping video bytes in durable file storage", async () => { mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [image, video] }); - mocks.size.mockReturnValue(video.fileSize); + mocks.size.mockImplementation((uri: string) => + uri.endsWith("clip.mov") ? video.fileSize : 3, + ); const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: 50 * 1024 * 1024 }); @@ -345,7 +458,7 @@ describe("composer file attachments", () => { canceled: false, assets: [{ ...video, fileSize: reported }, image], }); - mocks.size.mockReturnValue(stored); + mocks.size.mockImplementation((uri: string) => (uri.endsWith("clip.mov") ? stored : 3)); const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: limit }); diff --git a/apps/mobile/src/lib/composerImages.ts b/apps/mobile/src/lib/composerImages.ts index 5a45825c3..86658db8d 100644 --- a/apps/mobile/src/lib/composerImages.ts +++ b/apps/mobile/src/lib/composerImages.ts @@ -307,6 +307,46 @@ export async function pickComposerFiles(input: { return { files: attachments, error }; } +/** + * Longest edge kept when a photo has to be re-encoded. Matches the web composer's + * MAX_DIMENSION so every client hands providers the same resolution. + */ +const PHOTO_MAX_EDGE = 2048; +const PHOTO_JPEG_QUALITY = 0.85; + +/** + * Renders a photo-library pick to a provider-readable JPEG. Decode, downscale, and encode run + * natively; only the bounded result crosses the bridge. Camera photos are 12-48 MP HEIC files, + * so a full-size conversion is both slow to transfer and far more than a model can use. + */ +async function renderPhotoAsJpeg(uri: string): Promise<{ base64: string; uri: string }> { + const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator"); + let image = await ImageManipulator.manipulate(uri).renderAsync(); + try { + const longestEdge = Math.max(image.width, image.height); + if (longestEdge > PHOTO_MAX_EDGE) { + const resized = await ImageManipulator.manipulate(image) + .resize( + image.width >= image.height ? { width: PHOTO_MAX_EDGE } : { height: PHOTO_MAX_EDGE }, + ) + .renderAsync(); + image.release(); + image = resized; + } + const saved = await image.saveAsync({ + format: SaveFormat.JPEG, + compress: PHOTO_JPEG_QUALITY, + base64: true, + }); + if (!saved.base64) { + throw new Error("The rendered photo has no bytes."); + } + return { base64: saved.base64, uri: saved.uri }; + } finally { + image.release(); + } +} + async function loadImagePicker() { try { return await import("expo-image-picker"); @@ -369,7 +409,10 @@ export async function pickComposerMedia(input: { mediaTypes: input.maxVideoBytes === undefined ? ["images"] : ["images", "videos"], allowsMultipleSelection: true, selectionLimit: remainingSlots, - base64: true, + // Bytes stay in the picker's file until we know how much of them we need. Asking for + // base64 here made iOS decode and re-encode every camera photo at full resolution and + // hand JS a 10 MB+ string, which stalled the composer for seconds. + base64: false, quality: 1, shouldDownloadFromNetwork: true, }); @@ -397,7 +440,7 @@ export async function pickComposerMedia(input: { error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} attachments per message.`; break; } - let mimeType = asset.mimeType?.toLowerCase(); + const mimeType = asset.mimeType?.toLowerCase(); if (asset.type === "video" || mimeType?.startsWith("video/")) { if (input.maxVideoBytes === undefined) { error = "Video attachments are unavailable here."; @@ -426,56 +469,67 @@ export async function pickComposerMedia(input: { continue; } - let base64 = asset.base64; - if (!base64) { - error = `Failed to read '${asset.fileName ?? "image"}'.`; - continue; + const name = asset.fileName?.trim() || "image"; + // The picker's reported size is a hint, not a measurement: Android content streams can + // deliver more bytes than they advertise. Only a size read from the file itself decides + // whether the original bytes are safe to load into JS. + let sourceBytes: number | null = null; + try { + const { File } = await import("expo-file-system"); + sourceBytes = new File(asset.uri).size; + } catch { + sourceBytes = null; } - - let name = asset.fileName?.trim() || "image"; - // The iOS picker returns JPEG base64 even when its metadata describes HEIC, - // PNG, or GIF. Keep supported originals so transparency and animation survive; - // use the native JPEG conversion for formats providers cannot accept. - if (base64.startsWith("/9j/")) { - if ( - mimeType && - mimeType !== "image/jpeg" && - isProviderSendTurnSupportedImageMimeType(mimeType) - ) { - try { - const { File } = await import("expo-file-system"); - base64 = await new File(asset.uri).base64(); - } catch { - error = `Failed to read '${name}'.`; - continue; - } + // Originals the provider can read and that fit the cap pass through byte for byte so + // transparency and animation survive. Everything else (HEIC/HEIF, oversized JPEGs, + // unmeasurable sources) is rendered to a bounded JPEG off the JS thread. + const originalMimeType = + mimeType !== undefined && + isProviderSendTurnSupportedImageMimeType(mimeType) && + sourceBytes !== null && + sourceBytes > 0 && + sourceBytes <= PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + ? mimeType + : null; + + let image: { base64: string; mimeType: string; name: string; previewUri: string }; + try { + if (originalMimeType !== null) { + const { File } = await import("expo-file-system"); + image = { + base64: await new File(asset.uri).base64(), + mimeType: originalMimeType, + name, + previewUri: asset.uri, + }; } else { - mimeType = "image/jpeg"; - if (!/\.jpe?g$/i.test(name)) { - name = `${name.replace(/\.[^.]+$/, "")}.jpg`; - } + const rendered = await renderPhotoAsJpeg(asset.uri); + image = { + base64: rendered.base64, + mimeType: "image/jpeg", + name: /\.jpe?g$/i.test(name) ? name : `${name.replace(/\.[^.]+$/, "")}.jpg`, + previewUri: rendered.uri, + }; } - } - if (!mimeType || !isProviderSendTurnSupportedImageMimeType(mimeType)) { - error = `'${name}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`; + } catch { + error = `Failed to read '${name}'.`; continue; } - const sizeBytes = estimateBase64ByteSize(base64); + const sizeBytes = estimateBase64ByteSize(image.base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { - error = `'${asset.fileName ?? "image"}' exceeds the 10 MB attachment limit.`; + error = `'${name}' exceeds the 10 MB attachment limit.`; continue; } - const dataUrl = `data:${mimeType};base64,${base64}`; attachments.push({ id: uuidv4(), type: "image", - name, - mimeType, + name: image.name, + mimeType: image.mimeType, sizeBytes, - dataUrl, - previewUri: mimeType === asset.mimeType?.toLowerCase() ? asset.uri : dataUrl, + dataUrl: `data:${image.mimeType};base64,${image.base64}`, + previewUri: image.previewUri, }); } diff --git a/docs/user/composer.md b/docs/user/composer.md index de3763d61..ab523424e 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -20,8 +20,8 @@ and sends once its files reach the server. A Prime Agent follow-up still waits f Retry or remove a failed upload. You can drag or paste images into the web or desktop composer. HEIC and HEIF photos are converted to -JPEG there and when selected from the iOS photo library; the image limit applies after conversion. -On mobile, tap **+** for **Photo Library** or **Choose Files**, or send photos, videos, and files to +JPEG there and when selected from the mobile photo library; photos over the image limit are also resized +to fit. On mobile, tap **+** for **Photo Library** or **Choose Files**, or send photos, videos, and files to Pylon through another app's share sheet. Select a received file on mobile to preview it, save it, or open it in another app. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ec3784504..376bb9052 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -373,6 +373,9 @@ importers: expo-image: specifier: ~57.0.3 version: 57.0.3(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) + expo-image-manipulator: + specifier: ~57.0.17 + version: 57.0.17(expo@57.0.18) expo-image-picker: specifier: ~57.0.14 version: 57.0.14(expo@57.0.18) @@ -7547,6 +7550,11 @@ packages: peerDependencies: expo: '*' + expo-image-manipulator@57.0.17: + resolution: {integrity: sha512-VpM8qAotTeSIobIAL8RAIDrZKL0jQBK/6O41NnwmdT96sd2wT263DlTbkvQ6RkcFrR9+NZiC5Um7FDBK+jFneg==} + peerDependencies: + expo: '*' + expo-image-picker@57.0.14: resolution: {integrity: sha512-NK9XBQqOtscbB/uRts1Gm7Oki6odMN3FQPWD6fSKmNUfKM69O6kcgU25lEXFTSLZ7sLl+AopHa2X0uILd8rjaQ==} peerDependencies: @@ -18217,6 +18225,11 @@ snapshots: dependencies: expo: 57.0.18(fc5a731e35a0144aab60c7305f29cbed) + expo-image-manipulator@57.0.17(expo@57.0.18): + dependencies: + expo: 57.0.18(fc5a731e35a0144aab60c7305f29cbed) + expo-image-loader: 57.0.1(expo@57.0.18) + expo-image-picker@57.0.14(expo@57.0.18): dependencies: expo: 57.0.18(fc5a731e35a0144aab60c7305f29cbed)