[4.x] Fail fast when a package exceeds the AWS Lambda size limit

docs/guides/packaging.md

@@ -79,6 +79,10 @@ package:
     - '!.git/**'
 ```
 
+### Size limit
+
+AWS Lambda limits a function's code and layers to 250 MB once unzipped. When a package built by osls is larger than that, `osls package` and `osls deploy` fail before anything is uploaded, and list the largest files and directories of the package. Use `patterns` to exclude files that are not needed at runtime.
+
 ### Artifact
 
 For complete control over the packaging process you can specify your own artifact zip file.

lib/plugins/package/lib/zip-service.js

@@ -55,6 +55,10 @@ const getArchiveEntryName = (filePath, prefix) => {
 
 const zipMtime = new Date(0);
 const maxConcurrentFileStats = 64;
+// AWS Lambda rejects packages larger than 250 MB (262144000 bytes) once unzipped
+const maxUnzippedSize = 250 * 1024 * 1024;
+const packagingDocsUrl =
+  'https://github.com/oss-serverless/osls/blob/4.x/docs/guides/packaging.md#patterns';
 
 const getFileMode = (stat) =>
   Number(stat.mode) & 0o100 || process.platform === 'win32' ? 0o100755 : 0o100644;
@@ -100,6 +104,43 @@ const assertSameFile = (entry, currentStat) => {
   }
 };
 
+// Lambda counts MB as 1,024 KB
+const formatMegabytes = (bytes) => `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+
+// Fail before zipping instead of letting the deployment fail minutes later in CloudFormation
+const assertUnzippedSizeWithinLimit = (entries, zipFileName) => {
+  const totalSize = entries.reduce((total, entry) => total + Number(entry.stat.size), 0);
+  if (totalSize < maxUnzippedSize) return;
+
+  const sizeByTopLevelPath = new Map();
+  for (const entry of entries) {
+    const separatorIndex = entry.name.indexOf('/');
+    const topLevelPath =
+      separatorIndex === -1 ? entry.name : entry.name.slice(0, separatorIndex + 1);
+    sizeByTopLevelPath.set(
+      topLevelPath,
+      (sizeByTopLevelPath.get(topLevelPath) || 0) + Number(entry.stat.size)
+    );
+  }
+  const largestPaths = Array.from(sizeByTopLevelPath)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 5);
+  const pathColumnWidth = Math.max(...largestPaths.map(([topLevelPath]) => topLevelPath.length));
+
+  throw new ServerlessError(
+    [
+      `The "${zipFileName}" package is ${formatMegabytes(totalSize)} unzipped, above the 250 MB AWS Lambda limit (function code and layers combined).`,
+      'Largest files and directories in the package:',
+      ...largestPaths.map(
+        ([topLevelPath, size]) =>
+          `  ${topLevelPath.padEnd(pathColumnWidth)}  ${formatMegabytes(size).padStart(9)}`
+      ),
+      `Exclude files that are not needed at runtime with "package.patterns": ${packagingDocsUrl}`,
+    ].join('\n'),
+    'PACKAGE_TOO_LARGE'
+  );
+};
+
 const compareUtf8 = (left, right) =>
   Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8'));
 
@@ -319,6 +360,12 @@ const zipService = {
 
           entries.sort(compareZipEntries);
 
+          // A getFileContent() override can change file sizes, the size on disk is only
+          // known to be the unzipped size when files are zipped as they are
+          if (usesDefaultGetFileContent) {
+            assertUnzippedSizeWithinLimit(entries, zipFileName);
+          }
+
           for (const entry of entries) {
             const options = {
               mode: entry.mode,

test/unit/lib/plugins/package/lib/zip-service.test.js

@@ -1406,7 +1406,8 @@ describe('zipService', () => {
 
         serverless.utils.writeFileDir(largeFilePath);
         await fs.promises.writeFile(largeFilePath, '');
-        await fs.promises.truncate(largeFilePath, 512 * 1024 * 1024);
+        // Just below the 250 MB AWS Lambda limit, larger packages are rejected
+        await fs.promises.truncate(largeFilePath, 240 * 1024 * 1024);
 
         await packagePlugin.zipFiles(['event.json'], getTestArtifactFileName('warmup'));
         forceGc();
@@ -1425,8 +1426,8 @@ describe('zipService', () => {
         }
 
         // The sampled peak includes dead stream chunks that V8 has not collected yet,
-        // and that GC lag alone can exceed 64 MiB. The cap only needs to stay far
-        // below the 512 MiB file size to prove the file was streamed, not materialized.
+        // and that GC lag alone can exceed 64 MiB. The cap only needs to stay well
+        // below the 240 MiB file size to prove the file was streamed, not materialized.
         expect(peakArrayBuffers - before.arrayBuffers).to.be.lessThan(128 * 1024 * 1024);
 
         forceGc();
@@ -1444,6 +1445,40 @@ describe('zipService', () => {
         'No files to package'
       ));
 
+    it('rejects packages above the AWS Lambda unzipped size limit before zipping them', async () => {
+      serverless.utils.writeFileSync(path.join(tmpDirPath, 'handler.js'), 'handler');
+      // Sparse files: only their size is read
+      const largeFiles = {
+        'vendor/large.bin': 200 * 1024 * 1024,
+        'node_modules/medium.bin': 60 * 1024 * 1024,
+      };
+      for (const [filePath, size] of Object.entries(largeFiles)) {
+        serverless.utils.writeFileSync(path.join(tmpDirPath, filePath), '');
+        await fs.promises.truncate(path.join(tmpDirPath, filePath), size);
+      }
+      const artifactFilePath = path.join(serverless.serviceDir, '.serverless', 'too-large.zip');
+
+      let error;
+      try {
+        await packagePlugin.zipFiles(['handler.js', ...Object.keys(largeFiles)], 'too-large.zip');
+      } catch (caughtError) {
+        error = caughtError;
+      }
+
+      expect(error).to.have.property('code', 'PACKAGE_TOO_LARGE');
+      expect(error.message).to.equal(
+        [
+          'The "too-large.zip" package is 260.0 MB unzipped, above the 250 MB AWS Lambda limit (function code and layers combined).',
+          'Largest files and directories in the package:',
+          '  vendor/         200.0 MB',
+          '  node_modules/    60.0 MB',
+          '  handler.js        0.0 MB',
+          'Exclude files that are not needed at runtime with "package.patterns": https://github.com/oss-serverless/osls/blob/4.x/docs/guides/packaging.md#patterns',
+        ].join('\n')
+      );
+      expect(fs.existsSync(artifactFilePath)).to.equal(false);
+    });
+
     it('configures metadata stat concurrency with ext/promise/limit', () => {
       let configuredLimit;
       const fakeLimit = function (limitValue, callback) {