fix(playwright-testing): detect legacy anonymous accessibility snapshots

packages/playwright-testing/docs/accessibility.md

@@ -64,7 +64,19 @@ reason and tracking link before committing it:
 When `CI` is set, a missing baseline is still written and attached to the
 report, but the test fails. Download and commit the seed rather than silently
 accepting new violations. A committed legacy `.txt` accessibility snapshot is
-still honored, but new tests should use JSON baselines.
+still honored, but new tests should use JSON baselines. Detection uses Playwright's
+anonymous text snapshot naming, including counter-dependent truncation and
+hashing, and resolves the exact path for the current project and snapshot
+suffix with `testInfo.snapshotPath()`. PNG snapshots and files belonging to
+other tests or projects do not select snapshot mode.
+
+Detection supports templates that place `{arg}` in the filename, with a
+stable parent directory (which may use `{testName}`, `{projectName}`, and
+`{ext}`). Explicitly named snapshots and templates that put `{arg}` in a
+directory are not discovered. Tests whose titles sanitize to the same filename
+have Playwright's own collision limitations. Anonymous argument generation
+mirrors Playwright's naming algorithm because no public API generates an
+arbitrary counter; runner regression tests guard against upstream changes.
 
 Entries that match current violations are reported as baselined annotations.
 New violations fail the assertion with a copy-pasteable entry, while entries

packages/playwright-testing/src/accessibility-baseline-file.test.ts

@@ -1,6 +1,8 @@
 import { describe, it, expect, beforeEach, afterEach } from "vitest";
 import { promises as fs } from "fs";
 import os from "os";
+import { execFileSync } from "child_process";
+import { createRequire } from "module";
 import path from "path";
 import {
   baselineFilePath,
@@ -16,14 +18,24 @@ function makeTestInfo(opts: {
   dir: string;
   title?: string;
   titlePath?: string[];
+  suffix?: string;
 }) {
   const title = opts.title ?? "standalone accessibility check works";
   const titlePath = opts.titlePath ?? ["file.spec.ts", title];
   return {
     testId: `${title}-${Math.random()}`,
     title,
     titlePath,
-    snapshotPath: (...segs: string[]) => path.join(opts.dir, ...segs),
+    snapshotPath: (...segs: string[]) => {
+      const name = segs.join(path.sep);
+      const ext = path.extname(name);
+      return path.join(
+        opts.dir,
+        name.slice(0, name.length - ext.length) +
+          (ext === ".txt" ? (opts.suffix ?? "-chromium-linux") : "") +
+          ext,
+      );
+    },
   };
 }
 
@@ -115,37 +127,184 @@ describe("accessibility-baseline-file", () => {
       const ti = makeTestInfo({
         dir: tmpDir,
         title: "Test error pages: ID 3303",
+        suffix: "-desktop-chrome-linux",
       });
       expect(await snapshotExists(ti)).toBe(true);
     });
 
     it("matches with different project suffixes", async () => {
       await touch("Login-Works-1-mobile-safari-darwin.txt");
-      const ti = makeTestInfo({ dir: tmpDir, title: "Login Works" });
+      const ti = makeTestInfo({
+        dir: tmpDir,
+        title: "Login Works",
+        suffix: "-mobile-safari-darwin",
+      });
       expect(await snapshotExists(ti)).toBe(true);
     });
 
+    it("does not match a numeric extension of the title or another project", async () => {
+      await touch("Login-Works-2-1-chromium-linux.txt");
+      await touch("Login-Works-1-other-project-linux.txt");
+      expect(
+        await snapshotExists(
+          makeTestInfo({ dir: tmpDir, title: "Login Works" }),
+        ),
+      ).toBe(false);
+    });
+
+    it("matches punctuation adjoining literal hyphens and trailing separators", async () => {
+      await touch(
+        "Landing-Page-Content-of-type-Landing-Page-2-bureau-of-sorcery-tablet-linux.txt",
+      );
+      await touch(
+        "admin-toolbar-hover-expansion-When-I-mouse-over-an-admin-menu-item-it-expands-consistently-3-desktop-chrome-linux.txt",
+      );
+      expect(
+        await snapshotExists(
+          makeTestInfo({
+            dir: tmpDir,
+            titlePath: [
+              "file.spec.ts",
+              "Landing Page",
+              "Content of type Landing Page!",
+            ],
+            suffix: "-bureau-of-sorcery-tablet-linux",
+          }),
+        ),
+      ).toBe(true);
+      expect(
+        await snapshotExists(
+          makeTestInfo({
+            dir: tmpDir,
+            titlePath: [
+              "file.spec.ts",
+              "admin toolbar hover expansion",
+              "When I mouse over an admin menu item it expands consistently!",
+            ],
+            suffix: "-desktop-chrome-linux",
+          }),
+        ),
+      ).toBe(true);
+    });
+
+    it.each([
+      undefined,
+      "{testDir}/snapshots/{projectName}/{arg}-{platform}{ext}",
+    ])(
+      "detects real Playwright snapshots with template %s",
+      async (template) => {
+        const require = createRequire(import.meta.url);
+        const runner = require.resolve("@playwright/test/cli");
+        const api = require.resolve("@playwright/test");
+        const helper = require.resolve("./accessibility-baseline-file.ts");
+        const cases = [
+          [
+            "News",
+            "The new Defense Against the Dark Arts teacher",
+            "4/07/26 new defense against dark arts teacher!",
+          ],
+          ["Landing Page", "Content of type Landing Page!"],
+          [
+            "admin toolbar hover expansion",
+            "When I mouse over an admin menu item it expands consistently!",
+          ],
+          ["Nested: suite-", "Inner / describe", "Punctuation!? - trailing-"],
+          ["Long " + "title!? - ".repeat(20), "Same ending"],
+          ["Long " + "title!? - ".repeat(20) + "different", "Same ending"],
+        ];
+        await fs.writeFile(
+          path.join(tmpDir, "playwright.config.cjs"),
+          `module.exports = {
+        testDir: '.', outputDir: 'output', reporter: 'line',
+        projects: [{name: 'bureau-of-sorcery-chrome'}, {name: 'tablet'}],
+        snapshotPathTemplate: ${JSON.stringify(template)}
+      };`,
+        );
+        await fs.writeFile(
+          path.join(tmpDir, "generated.spec.cjs"),
+          `
+        const {test, expect} = require(${JSON.stringify(api)});
+        const {snapshotExists} = require(${JSON.stringify(helper)});
+        const fs = require('fs');
+        const path = require('path');
+        for (const [outer, inner, title = inner] of ${JSON.stringify(cases)}) {
+          test.describe(outer, () => test.describe(inner, () => {
+            test(title, async ({}, info) => {
+              expect(await snapshotExists(info)).toBe(false);
+              for (let i = 0; i < 12; i++) expect('[]').toMatchSnapshot();
+              const dir = path.dirname(info.snapshotPath('probe.txt'));
+              const files = fs.readdirSync(dir).filter(name => name.endsWith('.txt'));
+              // Leave only the twelfth snapshot for this test: detection cannot
+              // assume that the first accessibility scan has counter one.
+              const own = files.filter(name => name.includes('-12-'));
+              for (const name of files) if (!own.includes(name)) fs.unlinkSync(path.join(dir, name));
+              expect(await snapshotExists(info)).toBe(true);
+              expect('[]').toMatchSnapshot();
+              const next = fs.readdirSync(dir).find(name => name.includes('-13-'));
+              expect(next).toBeTruthy(); // detection did not consume counters
+              fs.unlinkSync(path.join(dir, next));
+              expect(await snapshotExists({...info, titlePath: [...info.titlePath.slice(0, -1), title + ' 2'], snapshotPath: info.snapshotPath.bind(info)})).toBe(false);
+              expect(await snapshotExists({...info, titlePath: [info.titlePath[0], outer + 'different', ...info.titlePath.slice(2)], snapshotPath: info.snapshotPath.bind(info)})).toBe(false);
+              for (const name of own) fs.renameSync(path.join(dir, name), path.join(dir, name.slice(0, -4) + '.png'));
+              expect(await snapshotExists(info)).toBe(false);
+            });
+          }));
+        }
+      `,
+        );
+        execFileSync(
+          process.execPath,
+          [
+            runner,
+            "test",
+            "--config",
+            path.join(tmpDir, "playwright.config.cjs"),
+            "--update-snapshots",
+            "--workers=1",
+          ],
+          { encoding: "utf8", timeout: 30000 },
+        );
+      },
+      40000,
+    );
+
     it("does not match a title that differs only by case", async () => {
       await touch("login-works-1-desktop-chrome-linux.txt");
-      const ti = makeTestInfo({ dir: tmpDir, title: "Login Works" });
+      const ti = makeTestInfo({
+        dir: tmpDir,
+        title: "Login Works",
+        suffix: "-desktop-chrome-linux",
+      });
       expect(await snapshotExists(ti)).toBe(false);
     });
 
     it("returns false when nothing matches", async () => {
       await touch("Other-Test-1-desktop-chrome-linux.txt");
-      const ti = makeTestInfo({ dir: tmpDir, title: "Login Works" });
+      const ti = makeTestInfo({
+        dir: tmpDir,
+        title: "Login Works",
+        suffix: "-desktop-chrome-linux",
+      });
       expect(await snapshotExists(ti)).toBe(false);
     });
 
     it("ignores a matching .png without a .txt", async () => {
       await touch("Login-Works-1-desktop-chrome-linux.png");
-      const ti = makeTestInfo({ dir: tmpDir, title: "Login Works" });
+      const ti = makeTestInfo({
+        dir: tmpDir,
+        title: "Login Works",
+        suffix: "-desktop-chrome-linux",
+      });
       expect(await snapshotExists(ti)).toBe(false);
     });
 
     it("does not match a different test whose title extends this one", async () => {
       await touch("CRUD-tests-Video-Promo-1-desktop-chrome-linux.txt");
-      const ti = makeTestInfo({ dir: tmpDir, title: "CRUD tests - Video" });
+      const ti = makeTestInfo({
+        dir: tmpDir,
+        title: "CRUD tests - Video",
+        suffix: "-desktop-chrome-linux",
+      });
       expect(await snapshotExists(ti)).toBe(false);
     });
   });

packages/playwright-testing/src/accessibility-baseline-file.ts

@@ -1,5 +1,6 @@
 import { promises as fs } from "fs";
 import path from "path";
+import { createHash } from "crypto";
 import type { TestInfo } from "@playwright/test";
 import type { AccessibilityBaselineEntry } from "./accessibility-baseline.js";
 
@@ -45,9 +46,7 @@ export function resetAccessibilityScanCounts(testInfo: object): void {
 }
 
 /**
- * Slugify a test's fully qualified title the same way we use it as the
- * stem of both on-disk baseline filenames and (for existence checks) the
- * prefix of Playwright's auto-generated snapshot filenames.
+ * Slugify a test's fully qualified title for on-disk JSON baseline filenames.
  *
  * Implemented as a single-pass character scan to avoid regex-based
  * polynomial backtracking on library-supplied input (CodeQL
@@ -80,19 +79,23 @@ function slugifyTitle(testInfo: Pick<TestInfo, "titlePath" | "title">): string {
 }
 
 /**
- * Reproduce the stem Playwright uses for auto-named snapshot files: the
- * title path (minus the spec file) joined with spaces, with every run of
- * control/punctuation characters collapsed to a single hyphen. Unlike
- * `slugifyTitle()` this preserves case and does not trim hyphens, so it
- * matches committed snapshot filenames exactly. Mirrors Playwright's
- * `sanitizeForFilePath()`. Very long titles, which Playwright truncates
- * and hashes, are not handled.
+ * Playwright's anonymous snapshot argument: trim the full title INCLUDING
+ * the counter to 100 UTF-16 code units, inserting five SHA-1 hex characters
+ * in the middle, then sanitize it. There is no public API for generating an
+ * arbitrary anonymous counter; snapshotPath() resolves this explicit argument.
+ * Keep this compatibility implementation covered by real runner tests.
  */
 function playwrightSnapshotStem(
   testInfo: Pick<TestInfo, "titlePath" | "title">,
+  counter: string,
 ): string {
   const segments = testInfo.titlePath?.slice(1) ?? [];
-  const raw = segments.length > 0 ? segments.join(" ") : testInfo.title;
+  const title = segments.length > 0 ? segments.join(" ") : testInfo.title;
+  let raw = `${title} ${counter}`;
+  if (raw.length > 100) {
+    const hash = createHash("sha1").update(raw).digest("hex").slice(0, 5);
+    raw = `${raw.slice(0, 46)}-${hash}-${raw.slice(-47)}`;
+  }
   let out = "";
   let lastWasReplaced = false;
   for (let i = 0; i < raw.length; i++) {
@@ -148,29 +151,31 @@ export function baselineFilePath(
 export async function snapshotExists(
   testInfo: Pick<TestInfo, "snapshotPath" | "titlePath" | "title">,
 ): Promise<boolean> {
-  const dir = path.dirname(testInfo.snapshotPath("a11y-baseline-probe"));
+  const dir = path.dirname(testInfo.snapshotPath("a11y-baseline-probe.txt"));
   let entries: string[];
   try {
     entries = await fs.readdir(dir);
   } catch (err: any) {
     if (err?.code === "ENOENT") return false;
     throw err;
   }
-  // Playwright names snapshots `<stem>-<counter>[-<project>-<platform>].txt`.
-  // Require the counter so a title that merely extends this one (e.g.
-  // "Video" vs "Video Promo") is not mistaken for this test's snapshot.
-  const prefix = `${playwrightSnapshotStem(testInfo)}-`;
-  return entries.some((name) => {
-    if (!name.startsWith(prefix) || !name.endsWith(".txt")) return false;
-    let i = prefix.length;
-    while (
-      i < name.length &&
-      name.charCodeAt(i) >= 48 &&
-      name.charCodeAt(i) <= 57
-    )
-      i++;
-    return i > prefix.length && (name[i] === "-" || name[i] === ".");
-  });
+  // Extract possible counters, then compare the entire resolved path. A
+  // prefix match can accidentally select another test (e.g. "Video 2").
+  // Resolving via the public API honors project, suffix, and path templates.
+  for (const name of entries) {
+    if (!name.endsWith(".txt")) continue;
+    for (const match of name.matchAll(/[0-9]+/g)) {
+      const counter = match[0];
+      if (counter.startsWith("0")) continue;
+      const expected = testInfo.snapshotPath(
+        `${playwrightSnapshotStem(testInfo, counter)}.txt`,
+      );
+      if (path.resolve(expected) === path.resolve(dir, name)) {
+        if ((await fs.stat(expected)).isFile()) return true;
+      }
+    }
+  }
+  return false;
 }
 
 export async function readBaselineFile(

packages/playwright-testing/src/accessibility-baseline.test.ts

@@ -959,13 +959,15 @@ describe("accessibility baseline", () => {
     });
 
     it("still uses snapshot mode when a legacy snapshot file exists for the test", async () => {
-      mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
+      mockAnalyze.mockResolvedValue(
+        makeAxeResults({ violations: [makeViolation("image-alt", [["img"]])] }),
+      );
 
       // Pre-create a Playwright-style snapshot file in the snapshots dir.
       const fs = await import("fs");
       const path = await import("path");
       await fs.promises.writeFile(
-        path.join(tmpDir, "legacy-test-1-chromium-linux.txt"),
+        path.join(tmpDir, "legacy-test-1.txt"),
         "[]\n",
       );
 
@@ -978,6 +980,21 @@ describe("accessibility baseline", () => {
         bestPracticeMode: "off",
       });
 
+      // A retry must stay in snapshot mode without creating TODO metadata.
+      await checkAccessibility(
+        makePage() as any,
+        makeTestInfo({
+          updateSnapshots: "none",
+          snapshotsDir: tmpDir,
+          title: "legacy test",
+        }) as any,
+        { bestPracticeMode: "off" },
+      );
+      expect(
+        (await fs.promises.readdir(tmpDir)).filter((name) =>
+          name.endsWith(".json"),
+        ),
+      ).toEqual([]);
       // Snapshot mode -> toMatchSnapshot was invoked.
       expect(mockToMatchSnapshot).toHaveBeenCalled();
     });