diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 199a730044..ca57e20db6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,6 +27,9 @@ jobs: secrets: inherit e2e-tests: uses: ./.github/workflows/e2e-tests.yaml + permissions: + contents: read + pull-requests: write secrets: inherit ci: runs-on: ubuntu-slim diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 6bd0ad4312..b402e8012f 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -73,3 +73,119 @@ jobs: with: name: cypress-screenshots-${{ matrix.os }}-${{ matrix.containers }} path: cypress/screenshots + - name: Upload retry report + uses: actions/upload-artifact@v7 + if: always() && hashFiles('cypress/reports/retries.json') != '' + with: + name: cypress-retries-${{ matrix.os }}-${{ matrix.containers }} + path: cypress/reports/retries.json + + report-retries: + name: Report flaky test retries + needs: e2e-tests + if: always() + runs-on: ubuntu-slim + permissions: + contents: read + pull-requests: write + steps: + - name: Download retry reports + uses: actions/download-artifact@v8 + with: + pattern: cypress-retries-* + path: retry-reports + continue-on-error: true + - name: Summarize retried tests + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + const root = 'retry-reports'; + const retries = []; + + if (fs.existsSync(root)) { + for (const shard of fs.readdirSync(root)) { + const reportFile = path.join(root, shard, 'retries.json'); + if (!fs.existsSync(reportFile)) continue; + const entries = JSON.parse(fs.readFileSync(reportFile, 'utf8')); + for (const entry of entries) { + retries.push({ shard, ...entry }); + } + } + } + + if (retries.length === 0) { + await core.summary + .addHeading('Cypress retries', 3) + .addRaw('No specs needed a retry in this run.') + .write(); + } else { + await core.summary + .addHeading('Cypress retries', 3) + .addTable([ + [ + { data: 'Shard', header: true }, + { data: 'Spec', header: true }, + { data: 'Test', header: true }, + { data: 'Attempts', header: true } + ], + ...retries.map((retry) => [ + retry.shard, + retry.spec, + retry.title, + String(retry.attempts) + ]) + ]) + .write(); + } + + // Best-effort PR comment: on forked PRs GITHUB_TOKEN is read-only, + // so a failure here must not fail the job. The step summary above + // is the primary, always-available output. + const pullRequest = context.payload.pull_request; + const isFork = + pullRequest && + pullRequest.head.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`; + if (!pullRequest || isFork) return; + + const marker = ''; + const body = + retries.length === 0 + ? `${marker}\n**No specs needed a retry in this run.**` + : `${marker}\n**${retries.length} Cypress test(s) were retried in this run.** See the job summary for details.`; + + try { + const comments = await github.paginate( + github.rest.issues.listComments, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number + } + ); + const existingComment = comments.find((comment) => + comment.body?.includes(marker) + ); + + if (existingComment) { + // Keep the marker comment in sync, including clearing it back + // to "no retries" so a fixed run doesn't leave a stale report. + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body + }); + } else if (retries.length > 0) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number, + body + }); + } + } catch (error) { + core.warning(`Could not post the retry-report comment: ${error.message}`); + } diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 826bcecaf3..5c8472c95f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -26,3 +26,6 @@ jobs: run: uv run --no-project pytest --cov=chainlit/ - name: Run frontend tests run: pnpm run test + - name: Run cypress support unit tests + if: matrix.python-version == '3.13' + run: pnpm test:unit diff --git a/.gitignore b/.gitignore index fe6668e33c..2eb8f18cbd 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ chainlit.md cypress/screenshots cypress/videos cypress/downloads +cypress/reports __pycache__ diff --git a/cypress.config.ts b/cypress.config.ts index 7f036f3b19..f3b7517100 100644 --- a/cypress.config.ts +++ b/cypress.config.ts @@ -1,11 +1,26 @@ import { defineConfig } from 'cypress'; import cypressSplit from 'cypress-split'; import fkill from 'fkill'; +import { mkdir, writeFile } from 'fs/promises'; +import { dirname, join } from 'path'; +import { + RetriedTest, + collectRetriedTests +} from './cypress/support/retryReport'; import { runChainlit } from './cypress/support/run'; export const CHAINLIT_APP_PORT = 8000; +// Per-shard record of tests that failed at least once but passed on retry, +// written after every spec so a killed job still leaves partial data behind. +const RETRY_REPORT_PATH = join( + process.cwd(), + 'cypress', + 'reports', + 'retries.json' +); + async function killChainlit() { await fkill(`:${CHAINLIT_APP_PORT}`, { force: true, @@ -41,12 +56,27 @@ export default defineConfig({ await killChainlit(); // Fallback to ensure no previous instance is running await runChainlit(); // Start Chainlit before running tests as Cypress require + const retriedTests: RetriedTest[] = []; + on('before:spec', async (spec) => { await killChainlit(); await runChainlit(spec); }); - on('after:spec', async () => { + on('after:spec', async (spec, results) => { + // `results` is undefined under `cypress open` (no run results are + // collected in interactive mode), so guard before reading it. + if (results) { + retriedTests.push( + ...collectRetriedTests(spec.relative, results.tests) + ); + } + await mkdir(dirname(RETRY_REPORT_PATH), { recursive: true }); + await writeFile( + RETRY_REPORT_PATH, + JSON.stringify(retriedTests, null, 2) + ); + await killChainlit(); }); diff --git a/cypress/support/retryReport.test.ts b/cypress/support/retryReport.test.ts new file mode 100644 index 0000000000..fa669ab8b1 --- /dev/null +++ b/cypress/support/retryReport.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { collectRetriedTests } from './retryReport.ts'; + +const asTests = (tests: unknown) => + tests as CypressCommandLine.RunResult['tests']; + +test('ignores a test that passed on its first attempt', () => { + const tests = asTests([ + { + title: ['suite', 'stable test'], + state: 'passed', + attempts: [{ state: 'passed' }] + } + ]); + + assert.deepEqual( + collectRetriedTests('cypress/e2e/example/spec.cy.ts', tests), + [] + ); +}); + +test('reports a test that failed then passed as retried', () => { + const tests = asTests([ + { + title: ['suite', 'flaky test'], + state: 'passed', + attempts: [{ state: 'failed' }, { state: 'passed' }] + } + ]); + + assert.deepEqual( + collectRetriedTests('cypress/e2e/example/spec.cy.ts', tests), + [ + { + spec: 'cypress/e2e/example/spec.cy.ts', + title: 'suite > flaky test', + attempts: 2 + } + ] + ); +}); + +test('excludes a test that failed every attempt', () => { + const tests = asTests([ + { + title: ['suite', 'broken test'], + state: 'failed', + attempts: [ + { state: 'failed' }, + { state: 'failed' }, + { state: 'failed' }, + { state: 'failed' } + ] + } + ]); + + assert.deepEqual( + collectRetriedTests('cypress/e2e/example/spec.cy.ts', tests), + [] + ); +}); diff --git a/cypress/support/retryReport.ts b/cypress/support/retryReport.ts new file mode 100644 index 0000000000..486c7829d6 --- /dev/null +++ b/cypress/support/retryReport.ts @@ -0,0 +1,28 @@ +export interface RetriedTest { + spec: string; + title: string; + attempts: number; +} + +/** + * A test counts as retried when at least one attempt failed but the overall + * test still passed - i.e. Cypress' `retries` config absorbed the failure. + * Tests that exhaust every retry and still fail are already visible via the + * job's exit status, so they are excluded here. + */ +export function collectRetriedTests( + specRelative: string, + tests: CypressCommandLine.RunResult['tests'] +): RetriedTest[] { + return tests + .filter( + (test) => + test.state === 'passed' && + test.attempts.some((attempt) => attempt.state === 'failed') + ) + .map((test) => ({ + spec: specRelative, + title: test.title.join(' > '), + attempts: test.attempts.length + })); +} diff --git a/package.json b/package.json index 929ab75064..b0d893df90 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,8 @@ "check-overrides": "node scripts/check-pnpm-overrides.mjs", "test": "pnpm run --recursive test", "test:e2e": "cypress run", - "test:e2e:interactive": "cypress open" + "test:e2e:interactive": "cypress open", + "test:unit": "node --test cypress/support/*.test.ts" }, "pnpm": { "overrides": { diff --git a/tsconfig.json b/tsconfig.json index 573e90001a..b536655124 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,9 @@ "lib": ["ESNext", "dom"], "types": ["cypress", "cypress-plugin-steps", "node"], "baseUrl": ".", - "esModuleInterop": true + "esModuleInterop": true, + "allowImportingTsExtensions": true, + "noEmit": true }, "include": ["cypress/**/*.ts", "cypress.config.ts"] }