Merge pull request #22459 from github/tausbn/unified-add-swift-node-type-generator

unified: Add Swift node type schema generator

Author: tausbn

unified/AGENTS.md

@@ -16,7 +16,10 @@ by Apple's swift-syntax rather than by tree-sitter.
 - `extractor/src/languages/swift/adapter.rs` converts that JSON into a yeast AST.
 
 - The raw parse tree's shape is described by `extractor/swift_node_types.yml`,
-  which is maintained by hand.
+  which is generated from swift-syntax by `swift-syntax-rs/schemagen`. Do not
+  edit it by hand; regenerate it with `scripts/regenerate-node-types.sh` after
+  changing the pinned swift-syntax version, then review the diff alongside the
+  mapping in `extractor/src/languages/swift/swift.rs`.
 
 ## AST Mapping
 - The target AST shape is described by `extractor/ast_types.yml`.

unified/extractor/src/languages/swift/adapter.rs

@@ -261,7 +261,9 @@ fn parse_range(node: &Value) -> Option<Range> {
     })
 }
 
-/// The authoritative swift-syntax input node-types schema.
+/// The authoritative swift-syntax input node-types schema, generated from
+/// swift-syntax by `swift-syntax-rs/schemagen` (run
+/// `unified/scripts/regenerate-node-types.sh` to refresh it).
 /// [`json_to_ast`] seeds every parse with the schema built from this,
 /// pre-registering every input kind and field so rule matching never references
 /// a name absent from a given file's tree.

unified/extractor/swift_node_types.yml

@@ -1,3 +1,5 @@
+# GENERATED from swift-syntax by unified/swift-syntax-rs/schemagen.
+# Do not edit; run unified/scripts/regenerate-node-types.sh instead.
 supertypes:
   decl:
     - accessorDecl

unified/scripts/regenerate-node-types.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Regenerate `extractor/swift_node_types.yml`, the schema describing the shape
+# of the trees produced by `swift_syntax_rs::parse_to_json`, from swift-syntax
+# itself.
+#
+# Run this after changing the pinned swift-syntax version, and review the diff:
+# a new or renamed node kind generally means the mapping in
+# `extractor/src/languages/swift/swift.rs` needs attention too.
+#
+# This needs a local Swift toolchain (see `swift-syntax-rs/.swift-version` for
+# the pinned version). The schema it derives from lives in `SyntaxSupport`, a
+# target of swift-syntax's separate `CodeGeneration` package: it is not a
+# product of swift-syntax, and Bazel's swift-syntax module does not export its
+# sources, so there is no way to depend on it directly.
+set -euo pipefail
+IFS=$'\n\t'
+
+root=$(cd "$(dirname "$0")/.." && pwd)
+swift_syntax_rs_dir="$root/swift-syntax-rs"
+schemagen_dir="$swift_syntax_rs_dir/schemagen"
+output="$root/extractor/swift_node_types.yml"
+
+if ! command -v swift >/dev/null 2>&1; then
+    echo "error: Swift is required; install the version pinned in $swift_syntax_rs_dir/.swift-version." >&2
+    exit 1
+fi
+
+# Codespaces sets `safe.bareRepository=explicit` through environment-based Git
+# configuration, which prevents SwiftPM from using its cached bare dependency
+# repositories. Relax only that injected setting, and only for Swift
+# subprocesses, as `swift-syntax-rs/build.rs` does for local Cargo builds.
+run_swift() {
+    if [[ ${GIT_CONFIG_KEY_0:-} == "safe.bareRepository" ]]; then
+        GIT_CONFIG_VALUE_0=all swift "$@"
+    else
+        swift "$@"
+    fi
+}
+
+echo "Resolving swift-syntax..." >&2
+(
+    cd "$schemagen_dir"
+    run_swift package resolve >&2
+)
+checkout="$schemagen_dir/.build/checkouts/swift-syntax"
+syntax_support="$checkout/CodeGeneration/Sources/SyntaxSupport"
+if [[ ! -d $syntax_support ]]; then
+    echo "error: $syntax_support not found after resolving swift-syntax." >&2
+    exit 1
+fi
+
+# Refresh rather than merge, so that sources deleted upstream do not linger.
+rm -rf "$schemagen_dir/Sources/SyntaxSupport"
+cp -R "$syntax_support" "$schemagen_dir/Sources/SyntaxSupport"
+
+echo "Generating $output..." >&2
+# Generate to a temporary file first: redirecting straight into `$output` would
+# truncate the existing schema before the build has even run, leaving nothing
+# behind if it fails.
+tmp=$(mktemp)
+trap 'rm -f "$tmp"' EXIT
+(
+    cd "$schemagen_dir"
+    run_swift run schemagen
+) > "$tmp"
+if [[ ! -s $tmp ]]; then
+    echo "error: schemagen produced no output; $output left unchanged." >&2
+    exit 1
+fi
+mv "$tmp" "$output"
+chmod 644 "$output"
+echo "Regenerated $output" >&2

unified/swift-syntax-rs/README.md

@@ -150,6 +150,20 @@ cargo test
 
 The first build compiles `swift-syntax` and can take several minutes.
 
+## Regenerating the extractor node types
+
+After updating the pinned swift-syntax version, regenerate the unified
+extractor's input schema:
+
+```sh
+../scripts/regenerate-node-types.sh
+```
+
+The script uses swift-syntax's authoritative `SyntaxSupport` definitions and
+requires the local Swift toolchain pinned by [`.swift-version`](.swift-version).
+Review the resulting `extractor/swift_node_types.yml` diff alongside the Swift
+mapping rules. See [`schemagen/README.md`](schemagen/README.md) for details.
+
 ## Building with Bazel (CI)
 
 CI builds this crate hermetically with Bazel. A Swift toolchain is downloaded
@@ -182,12 +196,15 @@ Requirements:
   swift.org. The Bazel C++ toolchain must still provide the macOS SDK, but a
   full Xcode installation is not required.
 
-The Swift compiler version is kept in sync across three places: the
+The Swift compiler version is kept in sync between the
 [`.swift-version`](.swift-version) file (read by the local `cargo`/`swift build`
-and by [swiftly](https://www.swift.org/swiftly/)), the literal `swift_version`
-pinned on `swift.toolchain(...)` in the root `MODULE.bazel` (the hermetic
-swift.org Bazel toolchain), and the `swift-syntax` release in
-`swift/Package.swift`.
+and by [swiftly](https://www.swift.org/swiftly/)) and the literal
+`swift_version` pinned on `swift.toolchain(...)` in the root `MODULE.bazel`
+(the hermetic swift.org Bazel toolchain).
+
+The swift-syntax version is independently pinned in the root `MODULE.bazel`,
+[`swift/Package.swift`](swift/Package.swift), and
+[`schemagen/Package.swift`](schemagen/Package.swift). Update all three together.
 
 (The Bazel toolchain pins a literal rather than reading `.swift-version` via
 `swift_version_file`, because the latter makes the module extension read a

unified/swift-syntax-rs/schemagen/.gitignore

@@ -0,0 +1,4 @@
+/.build
+# Copied from swift-syntax's CodeGeneration package by
+# `unified/scripts/regenerate-node-types.sh`; not ours to vendor.
+/Sources/SyntaxSupport

unified/swift-syntax-rs/schemagen/Package.resolved

@@ -0,0 +1,42 @@
+// swift-tools-version:5.9
+import PackageDescription
+
+// `schemagen` regenerates `unified/extractor/swift_node_types.yml`, the input
+// schema describing the shape of the trees produced by
+// `swift_syntax_rs::parse_to_json`. Run it through
+// `unified/scripts/regenerate-node-types.sh`, which stages the sources this
+// package needs; see `README.md` for the details.
+//
+// The tools version is deliberately older than the FFI package's: it selects
+// the Swift 5 language mode, and `SyntaxSupport` (see below) is not clean under
+// Swift 6 strict concurrency because its node tables are non-Sendable globals.
+let package = Package(
+    name: "schemagen",
+    platforms: [
+        // Matches the FFI package: swift-syntax 603 requires macOS 10.15.
+        .macOS(.v10_15),
+    ],
+    dependencies: [
+        // Keep this independent pin synchronized with the swift-syntax pins in
+        // `../swift/Package.swift` and the repository's `MODULE.bazel`.
+        .package(
+            url: "https://github.com/swiftlang/swift-syntax.git",
+            exact: "603.0.2"
+        ),
+    ],
+    targets: [
+        // `SyntaxSupport` is a target of swift-syntax's separate
+        // `CodeGeneration` package, not a product of swift-syntax itself, so it
+        // cannot be depended on directly. The regeneration script copies its
+        // sources here (the directory is git-ignored) and this target builds
+        // them as if they were our own.
+        .target(
+            name: "SyntaxSupport",
+            dependencies: [
+                .product(name: "SwiftSyntax", package: "swift-syntax"),
+                .product(name: "SwiftSyntaxBuilder", package: "swift-syntax"),
+            ]
+        ),
+        .executableTarget(name: "schemagen", dependencies: ["SyntaxSupport"]),
+    ]
+)

unified/swift-syntax-rs/schemagen/Package.swift

@@ -0,0 +1,72 @@
+# schemagen
+
+Generates [`unified/extractor/swift_node_types.yml`][schema], the schema that
+describes the shape of the trees produced by `swift_syntax_rs::parse_to_json`.
+The extractor seeds every parse with it, so rule matching never refers to a
+node kind or field that swift-syntax can produce but the schema does not know.
+
+Run it through the script, which stages the sources described below:
+
+```console
+$ unified/scripts/regenerate-node-types.sh
+```
+
+Do this after changing the pinned swift-syntax version, and read the resulting
+diff: a new or renamed node kind usually means the mapping in
+[`swift.rs`][mapping] needs attention too.
+
+This requires the local Swift toolchain pinned by
+[`.swift-version`](../.swift-version).
+
+## Why the sources are copied in
+
+The schema is derived from `SyntaxSupport`, the module that describes
+swift-syntax's own syntax tree. This is the same description swift-syntax
+generates itself from, and is therefore authoritative in a way that observing
+parser output never would be. The runtime `SwiftSyntax` module is not a
+substitute: its `SyntaxNodeStructure` exposes layout as key paths, without the
+field names, optionality, and base-kind relationships this schema records.
+
+`SyntaxSupport` is awkward to depend on, though. It is a target of
+`CodeGeneration`, a package inside the swift-syntax repository that is
+separate from swift-syntax itself, and it is not one of that package's
+products. SwiftPM can only depend on products, and Bazel's swift-syntax module
+does not export the `CodeGeneration` sources, so neither build system can
+reach it directly.
+
+The regeneration script therefore resolves this package's swift-syntax
+dependency and copies its `CodeGeneration/Sources/SyntaxSupport` sources into
+`Sources/SyntaxSupport`, where this package builds them as its own. That
+directory is git-ignored and refreshed on every run, so it always matches
+schemagen's pin rather than drifting as a stale vendored copy would.
+
+Schemagen has its own exact swift-syntax pin in `Package.swift`. Keep it
+synchronized with the SwiftPM parser pin in `../swift/Package.swift` and the
+Bazel pin in the repository's `MODULE.bazel`. The build systems resolve these
+independently, so regeneration does not itself guarantee that all three pins
+match.
+
+## What is filtered out
+
+The schema describes the JSON the extractor's adapter receives, not
+swift-syntax's tree verbatim, so `main.swift` mirrors what
+[`adapter.rs`][adapter] does:
+
+- Abstract base kinds become `supertypes:` entries rather than node kinds.
+- Collection nodes are dropped, and a collection-typed child is recorded as
+  its element kinds, because the adapter elides collections into JSON arrays.
+- `unexpectedBeforeX`, `unexpectedBetweenXAndY`, and `unexpectedAfterX`
+  error-recovery children are dropped; no rule matches them. This filters on
+  the child name: `unexpectedCodeDecl` is a real node kind and is retained.
+- Token-typed children become the synthetic `_token` kind. Only the varying
+  token kinds whose `TokenSpec` is `.other` and has no fixed text are emitted
+  as kinds of their own. These are derived from `Token.allCases` and should match
+  `VARYING_TOKEN_KINDS` in `adapter.rs`. Fixed tokens are anonymous and keyed
+  by their text, so no rule can name them.
+
+Setting `EMIT_SUPERTYPES=0` omits the `supertypes:` section, which can be useful
+when diffing two versions for kind and field changes alone.
+
+[schema]: ../../extractor/swift_node_types.yml
+[mapping]: ../../extractor/src/languages/swift/swift.rs
+[adapter]: ../../extractor/src/languages/swift/adapter.rs

unified/swift-syntax-rs/schemagen/README.md

@@ -0,0 +1,100 @@
+import Foundation
+import SyntaxSupport
+
+// Named-leaf ("varying") token kinds, mirroring the extractor adapter's
+// VARYING_TOKEN_KINDS. Fixed tokens are anonymous (keyed by text) and are not
+// matched by any rule, so they are not emitted here.
+let varyingTokens = Token.allCases.compactMap { token -> String? in
+    let spec = token.spec
+    guard spec.text == nil else { return nil }
+    // The generic `keyword` token has no `TokenSpec.text`, but each concrete
+    // keyword has a fixed spelling carried by its associated value.
+    guard case .other = spec.kind else { return nil }
+    return spec.identifier.text
+}
+
+// The yeast type references that a child maps to. A collection wrapper is
+// elided by the adapter, so a collection child maps to its element kinds.
+func typeRefs(_ child: Child) -> [String] {
+    switch child.kind {
+    case .node(let kind):
+        return [kind.rawValue]
+    case .nodeChoices(let choices, _):
+        return choices.flatMap { typeRefs($0) }
+    case .collection(let kind, _, _, _, _):
+        if let collection = SYNTAX_NODES.first(where: { $0.kind == kind })?.collectionNode {
+            let elements = collection.elementChoices.map { $0.rawValue }
+            return elements.isEmpty ? [kind.rawValue] : elements
+        }
+        return [kind.rawValue]
+    case .token:
+        return ["_token"]
+    }
+}
+
+func isMultiple(_ child: Child) -> Bool {
+    if case .collection = child.kind {
+        return true
+    }
+    return false
+}
+
+var supertypes: [String: [String]] = [:]
+var named: [(String, [Child])] = []
+
+for node in SYNTAX_NODES {
+    if node.kind.isBase {
+        continue
+    }
+    if node.base == .syntaxCollection {
+        continue
+    }
+    supertypes[node.base.rawValue, default: []].append(node.kind.rawValue)
+    named.append((node.kind.rawValue, node.layoutNode?.children ?? []))
+}
+
+var output = ""
+output += "# GENERATED from swift-syntax by unified/swift-syntax-rs/schemagen.\n"
+output += "# Do not edit; run unified/scripts/regenerate-node-types.sh instead.\n"
+let emitSupertypes = ProcessInfo.processInfo.environment["EMIT_SUPERTYPES"] != "0"
+if emitSupertypes {
+    output += "supertypes:\n"
+    for base in supertypes.keys.sorted() {
+        output += "  \(base):\n"
+        for member in supertypes[base]!.sorted() {
+            output += "    - \(member)\n"
+        }
+    }
+}
+output += "named:\n"
+for (kind, children) in named.sorted(by: { $0.0 < $1.0 }) {
+    output += "  \(kind):\n"
+    for child in children {
+        // swift-syntax error-recovery slots (`unexpectedBeforeX`,
+        // `unexpectedBetweenXAndY`, and `unexpectedAfterX`) are never matched
+        // by rules.
+        if child.name.hasPrefix("unexpected") {
+            continue
+        }
+        var key = child.name
+        if isMultiple(child) {
+            key += "*"
+        } else if child.isOptional {
+            key += "?"
+        }
+        let refs = typeRefs(child)
+        let value = refs.count == 1 ? refs[0] : "[" + refs.joined(separator: ", ") + "]"
+        output += "    \(key): \(value)\n"
+    }
+}
+
+// Synthetic leaf for token-typed fields, plus the named ("varying") token
+// kinds that are not already emitted as layout nodes (`stringSegment`, for
+// example, is both a node and a token kind and must only be emitted once).
+let namedKinds = Set(named.map { $0.0 })
+output += "  _token:\n"
+for token in varyingTokens.sorted() where !namedKinds.contains(token) {
+    output += "  \(token):\n"
+}
+
+print(output, terminator: "")