ci(e2e): report Cypress retries in CI so flaky specs are visible

.github/workflows/ci.yaml

@@ -27,6 +27,9 @@ jobs:
     secrets: inherit
   e2e-tests:
     uses: ./.github/workflows/e2e-tests.yaml
+    permissions:
+      contents: read
+      pull-requests: write
     secrets: inherit
   ci:
     runs-on: ubuntu-slim

.github/workflows/e2e-tests.yaml

@@ -73,3 +73,119 @@ jobs:
         with:
           name: cypress-screenshots-${{ matrix.os }}-${{ matrix.containers }}
           path: cypress/screenshots
+      - name: Upload retry report
+        uses: actions/upload-artifact@v7
+        if: always() && hashFiles('cypress/reports/retries.json') != ''
+        with:
+          name: cypress-retries-${{ matrix.os }}-${{ matrix.containers }}
+          path: cypress/reports/retries.json
+
+  report-retries:
+    name: Report flaky test retries
+    needs: e2e-tests
+    if: always()
+    runs-on: ubuntu-slim
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Download retry reports
+        uses: actions/download-artifact@v8
+        with:
+          pattern: cypress-retries-*
+          path: retry-reports
+        continue-on-error: true
+      - name: Summarize retried tests
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const root = 'retry-reports';
+            const retries = [];
+
+            if (fs.existsSync(root)) {
+              for (const shard of fs.readdirSync(root)) {
+                const reportFile = path.join(root, shard, 'retries.json');
+                if (!fs.existsSync(reportFile)) continue;
+                const entries = JSON.parse(fs.readFileSync(reportFile, 'utf8'));
+                for (const entry of entries) {
+                  retries.push({ shard, ...entry });
+                }
+              }
+            }
+
+            if (retries.length === 0) {
+              await core.summary
+                .addHeading('Cypress retries', 3)
+                .addRaw('No specs needed a retry in this run.')
+                .write();
+            } else {
+              await core.summary
+                .addHeading('Cypress retries', 3)
+                .addTable([
+                  [
+                    { data: 'Shard', header: true },
+                    { data: 'Spec', header: true },
+                    { data: 'Test', header: true },
+                    { data: 'Attempts', header: true }
+                  ],
+                  ...retries.map((retry) => [
+                    retry.shard,
+                    retry.spec,
+                    retry.title,
+                    String(retry.attempts)
+                  ])
+                ])
+                .write();
+            }
+
+            // Best-effort PR comment: on forked PRs GITHUB_TOKEN is read-only,
+            // so a failure here must not fail the job. The step summary above
+            // is the primary, always-available output.
+            const pullRequest = context.payload.pull_request;
+            const isFork =
+              pullRequest &&
+              pullRequest.head.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`;
+            if (!pullRequest || isFork) return;
+
+            const marker = '<!-- cypress-retry-report -->';
+            const body =
+              retries.length === 0
+                ? `${marker}\n**No specs needed a retry in this run.**`
+                : `${marker}\n**${retries.length} Cypress test(s) were retried in this run.** See the job summary for details.`;
+
+            try {
+              const comments = await github.paginate(
+                github.rest.issues.listComments,
+                {
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: pullRequest.number
+                }
+              );
+              const existingComment = comments.find((comment) =>
+                comment.body?.includes(marker)
+              );
+
+              if (existingComment) {
+                // Keep the marker comment in sync, including clearing it back
+                // to "no retries" so a fixed run doesn't leave a stale report.
+                await github.rest.issues.updateComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: existingComment.id,
+                  body
+                });
+              } else if (retries.length > 0) {
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: pullRequest.number,
+                  body
+                });
+              }
+            } catch (error) {
+              core.warning(`Could not post the retry-report comment: ${error.message}`);
+            }

.github/workflows/tests.yaml

@@ -26,3 +26,6 @@ jobs:
         run: uv run --no-project pytest --cov=chainlit/
       - name: Run frontend tests
         run: pnpm run test
+      - name: Run cypress support unit tests
+        if: matrix.python-version == '3.13'
+        run: pnpm test:unit

.gitignore

@@ -17,6 +17,7 @@ chainlit.md
 cypress/screenshots
 cypress/videos
 cypress/downloads
+cypress/reports
 
 __pycache__
 

cypress.config.ts

@@ -1,11 +1,26 @@
 import { defineConfig } from 'cypress';
 import cypressSplit from 'cypress-split';
 import fkill from 'fkill';
+import { mkdir, writeFile } from 'fs/promises';
+import { dirname, join } from 'path';
 
+import {
+  RetriedTest,
+  collectRetriedTests
+} from './cypress/support/retryReport';
 import { runChainlit } from './cypress/support/run';
 
 export const CHAINLIT_APP_PORT = 8000;
 
+// Per-shard record of tests that failed at least once but passed on retry,
+// written after every spec so a killed job still leaves partial data behind.
+const RETRY_REPORT_PATH = join(
+  process.cwd(),
+  'cypress',
+  'reports',
+  'retries.json'
+);
+
 async function killChainlit() {
   await fkill(`:${CHAINLIT_APP_PORT}`, {
     force: true,
@@ -41,12 +56,27 @@ export default defineConfig({
       await killChainlit(); // Fallback to ensure no previous instance is running
       await runChainlit(); // Start Chainlit before running tests as Cypress require
 
+      const retriedTests: RetriedTest[] = [];
+
       on('before:spec', async (spec) => {
         await killChainlit();
         await runChainlit(spec);
       });
 
-      on('after:spec', async () => {
+      on('after:spec', async (spec, results) => {
+        // `results` is undefined under `cypress open` (no run results are
+        // collected in interactive mode), so guard before reading it.
+        if (results) {
+          retriedTests.push(
+            ...collectRetriedTests(spec.relative, results.tests)
+          );
+        }
+        await mkdir(dirname(RETRY_REPORT_PATH), { recursive: true });
+        await writeFile(
+          RETRY_REPORT_PATH,
+          JSON.stringify(retriedTests, null, 2)
+        );
+
         await killChainlit();
       });
 

cypress/support/retryReport.test.ts

@@ -0,0 +1,63 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+
+import { collectRetriedTests } from './retryReport.ts';
+
+const asTests = (tests: unknown) =>
+  tests as CypressCommandLine.RunResult['tests'];
+
+test('ignores a test that passed on its first attempt', () => {
+  const tests = asTests([
+    {
+      title: ['suite', 'stable test'],
+      state: 'passed',
+      attempts: [{ state: 'passed' }]
+    }
+  ]);
+
+  assert.deepEqual(
+    collectRetriedTests('cypress/e2e/example/spec.cy.ts', tests),
+    []
+  );
+});
+
+test('reports a test that failed then passed as retried', () => {
+  const tests = asTests([
+    {
+      title: ['suite', 'flaky test'],
+      state: 'passed',
+      attempts: [{ state: 'failed' }, { state: 'passed' }]
+    }
+  ]);
+
+  assert.deepEqual(
+    collectRetriedTests('cypress/e2e/example/spec.cy.ts', tests),
+    [
+      {
+        spec: 'cypress/e2e/example/spec.cy.ts',
+        title: 'suite > flaky test',
+        attempts: 2
+      }
+    ]
+  );
+});
+
+test('excludes a test that failed every attempt', () => {
+  const tests = asTests([
+    {
+      title: ['suite', 'broken test'],
+      state: 'failed',
+      attempts: [
+        { state: 'failed' },
+        { state: 'failed' },
+        { state: 'failed' },
+        { state: 'failed' }
+      ]
+    }
+  ]);
+
+  assert.deepEqual(
+    collectRetriedTests('cypress/e2e/example/spec.cy.ts', tests),
+    []
+  );
+});

cypress/support/retryReport.ts

@@ -0,0 +1,28 @@
+export interface RetriedTest {
+  spec: string;
+  title: string;
+  attempts: number;
+}
+
+/**
+ * A test counts as retried when at least one attempt failed but the overall
+ * test still passed - i.e. Cypress' `retries` config absorbed the failure.
+ * Tests that exhaust every retry and still fail are already visible via the
+ * job's exit status, so they are excluded here.
+ */
+export function collectRetriedTests(
+  specRelative: string,
+  tests: CypressCommandLine.RunResult['tests']
+): RetriedTest[] {
+  return tests
+    .filter(
+      (test) =>
+        test.state === 'passed' &&
+        test.attempts.some((attempt) => attempt.state === 'failed')
+    )
+    .map((test) => ({
+      spec: specRelative,
+      title: test.title.join(' > '),
+      attempts: test.attempts.length
+    }));
+}

package.json

@@ -36,7 +36,8 @@
     "check-overrides": "node scripts/check-pnpm-overrides.mjs",
     "test": "pnpm run --recursive test",
     "test:e2e": "cypress run",
-    "test:e2e:interactive": "cypress open"
+    "test:e2e:interactive": "cypress open",
+    "test:unit": "node --test cypress/support/*.test.ts"
   },
   "pnpm": {
     "overrides": {

tsconfig.json

@@ -3,7 +3,9 @@
     "lib": ["ESNext", "dom"],
     "types": ["cypress", "cypress-plugin-steps", "node"],
     "baseUrl": ".",
-    "esModuleInterop": true
+    "esModuleInterop": true,
+    "allowImportingTsExtensions": true,
+    "noEmit": true
   },
   "include": ["cypress/**/*.ts", "cypress.config.ts"]
 }