Validate WEB_FEATURES.yml

For every pull request, verify that the `WEB_FEATURES.yml` file is
internally consistent and that the pull request does not reduce the
number of tests associated with a given classifier.

Failing for any reduction in the number of matched tests is intended to
alert contributors during file renaming operations. This failure will
need to be ignored when tests are simply deleted, and this could
potentially become a distraction for maintainers. However, test deletion
has historically been so rare that spurious failures are not expected to
be common enough to substantively interfere with routine maintenance.

Author: jugglinmike

.github/workflows/checks.yml

@@ -54,6 +54,39 @@ jobs:
             printf "::error file=%s,line=1::%s\n", substr($0, 1, i-1), substr($0, i+2);
           }'
 
+  lint-web-features:
+    name: Lint web-features
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tools/web-features/requirements.txt
+
+      - name: Validate WEB_FEATURES.yml
+        run: |
+          ./tools/web-features/lint.py --manifest WEB_FEATURES-manifest-pr.json
+
+      - name: Checkout base branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.base_ref || github.ref_name }}
+          # The `clean` option must be unset in order to retain the first
+          # manifest file.
+          clean: false
+
+      - name: Generate manifest on base branch
+        run: |
+          ./tools/web-features/lint.py --manifest WEB_FEATURES-manifest-base.json
+
+      - name: Verify absence of regressions in classification coverage
+        run: |
+          ./tools/web-features/check-coverage.py \
+            --base WEB_FEATURES-manifest-base.json \
+            --new WEB_FEATURES-manifest-pr.json
 
   build:
     name: Build generated tests

.github/workflows/test-tools.yml

@@ -50,3 +50,24 @@ jobs:
 
       - name: Test the generation tool
         run: ./tools/generation/test/run.py
+
+  web-features:
+    name: Test the web-features tooling
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+          cache: pip
+
+      - name: Install dependencies for web-features tooling
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tools/web-features/requirements.txt
+
+      - name: Test the web-features tooling
+        run: ./tools/web-features/test/run.py

tools/web-features/README.md

@@ -84,8 +84,22 @@ For details on the design considerations, please see [GitHub issue
 
 ## Tooling
 
-This directory defines a command-line utility in the form of an executable
-Python script named `lint.py`. Upon execution, the script will validate some
-basic expectations about the contents of the `WEB_FEATURES.yml` (e.g. that the
-file conforms to the schema and that every pattern impacts the set of matched
-test files).
+This directory defines two command-line utilities in the form of executable
+Python scripts. Their dependencies can be installed with the following command
+(executed from the root of this repository):
+
+    pip install -r tools/web-features/requirements.txt
+
+### `lint.py`
+
+This script validates basic expectations about the contents of the
+`WEB_FEATURES.yml` (e.g. that the file conforms to the schema and that every
+pattern impacts the set of matched test files). If invoked with the argument
+`--manifest`, this utility will write a "manifest" of all web-features and
+their associated test file names.
+
+### `check-coverage.py'
+
+This script compares two web-features manifests to identify changes that likely
+indicate an unintentional regression. Specifically, any reduction in the number
+of tests associated with a web-feature will produce an error.

tools/web-features/check-coverage.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# Copyright (C) 2025 the V8 project authors. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import argparse
+import json
+import sys
+
+def main(base_filename, new_filename):
+    with open(base_filename, 'r') as base_handle:
+        base = json.loads(base_handle.read())
+
+    assert base['version'] == 1, 'Unsupported base manifest version'
+
+    with open(new_filename, 'r') as new_handle:
+        new = json.loads(new_handle.read())
+
+    assert new['version'] == 1, 'Unsupported new manifest version'
+
+    errors = []
+    for name, tests in base['data'].items():
+        assert name in new['data']
+
+        base_count = len(tests)
+        new_count = len(new['data'][name])
+
+        if base_count > new_count:
+            errors.append(
+                f'Web feature "{name}" appears to have regressed ' \
+                f'(from {base_count} matching tests to {new_count} matching ' \
+                f'tests). Ensure the classifiers in WEB_FEATURES.yml have ' \
+                f'been updated to accommodate the changes on this branch.'
+            )
+
+    return errors
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Test262 web-features manifest regression checker'
+    )
+
+    parser.add_argument('--base', help='''A web-features manifest to use as the
+        basis for validation''')
+    parser.add_argument('--new', help='''A web-features manifest to check
+        for regressions''')
+
+    args = parser.parse_args()
+    errors = main(args.base, args.new)
+
+    if len(errors) == 0:
+        sys.exit(0)
+    else:
+        for error in errors:
+            print(error, file=sys.stderr)
+        sys.exit(1)

tools/web-features/lint.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
 
+import argparse
+import json
 import os
 import re
+import sys
 import yaml
 
 def read_features(filename):
@@ -25,7 +28,7 @@ def file_is_test(filename):
     return not (filename.endswith('FIXTURE.js') or filename.endswith('.json'))
 
 def pattern_from_path_spec(path_spec):
-    return re.compile(re.sub('\*', '.*', path_spec) + '(/|$)')
+    return re.compile(re.sub('\\*', '.*', path_spec) + '(/|$)')
 
 def get_filenames(path_spec):
     pattern = pattern_from_path_spec(path_spec)
@@ -92,23 +95,35 @@ def match(file_path, tag_specs):
                 return False
     return True
 
-def main(web_features_filename):
+def main(web_features_filename, manifest_filename):
     with open(web_features_filename, 'r') as handle:
         features = yaml.safe_load(handle)['features']
+    manifest = {'data': dict(), 'version': 1}
 
     for feature in features:
         name = feature['name']
         path_specs = feature['files']
         tag_specs = feature.get('tags', [])
 
-        tests = [*filter(
+        manifest['data'][name] = [*filter(
             lambda candidate: match(candidate, tag_specs),
             get_filenames_from_path_specs(path_specs)
         )]
 
-        print(f'{name},{len(tests)}')
-
-    print(f'{web_features_filename} is conformant')
+    if manifest_filename:
+        with open(manifest_filename, 'w') as manifest_handle:
+            manifest_handle.write(json.dumps(manifest))
 
 if __name__ == '__main__':
-    main('./WEB_FEATURES.yml')
+    parser = argparse.ArgumentParser(
+        description='Test262 web-features manifest regression checker'
+    )
+
+    parser.add_argument('--manifest', help='''The name of a JSON-formatted test
+        manifest file to create''')
+
+    args = parser.parse_args()
+
+    main('./WEB_FEATURES.yml', args.manifest)
+
+    print('./WEB_FEATURES.yml is conformant')

tools/web-features/requirements.txt

@@ -0,0 +1 @@
+PyYAML==6.0.3