[ARH] Fix Review Hub Artifact Generation

eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml

@@ -33,10 +33,10 @@ steps:
         - bash: |
             set -Eeuo pipefail
 
-            source_repo="$(ApiReviewSourceDir)"
-            tooling_dir="$(ApiReviewToolingDir)"
-            output_dir="${{ parameters.outputDir }}"
-            state_dir="${{ parameters.workingDir }}/state"
+            source_repo="$SOURCE_REPO"
+            tooling_dir="$TOOLING_DIR"
+            output_dir="$OUTPUT_DIR"
+            state_dir="$WORKING_DIR/state"
 
             mkdir -p "$source_repo" "$output_dir" "$state_dir"
 
@@ -60,18 +60,18 @@ steps:
               printf '%s' "$matches"
             }
 
-            package_dir=$(find_package_dir "$source_repo" "${{ parameters.packageName }}")
+            package_dir=$(find_package_dir "$source_repo" "$PACKAGE_NAME")
             package_relative_path=$(realpath --relative-to="$source_repo" "$package_dir")
 
-            echo "Generating ${{ parameters.kind }} API review artifact from ${{ parameters.ref }}"
+            echo "Generating $KIND API review artifact from $REF"
             common_tasks_dir="$source_repo/scripts/devops_tasks"
             if [ ! -f "$common_tasks_dir/common_tasks.py" ]; then
               echo "Expected Python SDK common_tasks.py was not found at $common_tasks_dir/common_tasks.py" >&2
               exit 1
             fi
             install -D "$tooling_dir/eng/common/scripts/Export-APIViewMarkdown.ps1" "$source_repo/eng/common/scripts/Export-APIViewMarkdown.ps1"
             install -D "$tooling_dir/eng/scripts/extract_apiview_metadata.py" "$source_repo/eng/scripts/extract_apiview_metadata.py"
-            (cd "$source_repo" && PYTHONPATH="$common_tasks_dir${PYTHONPATH:+:$PYTHONPATH}" python -m azpysdk.main apistub --dest-dir "$package_dir" "${{ parameters.packageName }}")
+            (cd "$source_repo" && PYTHONPATH="$common_tasks_dir${PYTHONPATH:+:$PYTHONPATH}" python -m azpysdk.main apistub --dest-dir "$package_dir" "$PACKAGE_NAME")
 
             if [ ! -f "$package_dir/api.md" ]; then
               echo "Expected api.md was not generated at $package_dir/api.md" >&2
@@ -85,7 +85,19 @@ steps:
             cp "$package_dir/api.md" "$output_dir/api.md"
             cp "$package_dir/api.metadata.yml" "$output_dir/api.metadata.yml"
 
-            version=$(python -c 'import pathlib, re, sys; package_dir = pathlib.Path(sys.argv[1]); pattern = re.compile(r"^\s*VERSION\s*[:=]\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", re.MULTILINE); candidates = sorted(package_dir.rglob("*_version.py")) + sorted(package_dir.rglob("version.py")); match = next((pattern.search(candidate.read_text(encoding="utf-8")) for candidate in candidates if "_generated" not in candidate.parts and pattern.search(candidate.read_text(encoding="utf-8"))), None); print(match.group(1) if match else "")' "$package_dir")
+            version=$(sed -n 's/^packageVersion:[[:space:]]*//p' "$package_dir/api.metadata.yml")
+            if [ -z "$version" ]; then
+              echo "Expected packageVersion was not found in $package_dir/api.metadata.yml" >&2
+              exit 1
+            fi
             printf '%s' "$package_relative_path" > "$state_dir/package-relative-path.txt"
             printf '%s' "$version" > "$state_dir/version.txt"
           displayName: 'Generate ${{ parameters.kind }} Python API review bundle'
+          env:
+            SOURCE_REPO: $(ApiReviewSourceDir)
+            TOOLING_DIR: $(ApiReviewToolingDir)
+            OUTPUT_DIR: ${{ parameters.outputDir }}
+            WORKING_DIR: ${{ parameters.workingDir }}
+            PACKAGE_NAME: ${{ parameters.packageName }}
+            KIND: ${{ parameters.kind }}
+            REF: ${{ parameters.ref }}

eng/scripts/extract_apiview_metadata.py

@@ -10,13 +10,13 @@
 )
 
 
-def extract_metadata(api_markdown_path: pathlib.Path) -> Dict[str, str]:
+def extract_metadata(api_markdown_path: pathlib.Path, package_version: str) -> Dict[str, str]:
     with api_markdown_path.open(encoding="utf-8-sig", newline="") as api_markdown_file:
         file_text = api_markdown_file.read()
     line_ending = "\r\n" if "\r\n" in file_text else "\n"
     lines = re.split(r"\r?\n", file_text)
 
-    metadata: Dict[str, str] = {}
+    metadata: Dict[str, str] = {"packageVersion": package_version}
     filtered: List[str] = []
     for line in lines:
         match = _METADATA_PATTERN.match(line)
@@ -51,12 +51,13 @@ def main() -> None:
     parser = argparse.ArgumentParser(description="Extract Python APIView metadata from API markdown")
     parser.add_argument("--api-markdown-path")
     parser.add_argument("--output-path", default=".")
+    parser.add_argument("--package-version", required=True)
     args = parser.parse_args()
 
     api_markdown_path = pathlib.Path(args.api_markdown_path) if args.api_markdown_path else pathlib.Path(args.output_path) / "api.md"
     if not api_markdown_path.is_file():
         parser.error(f"API markdown file not found: {api_markdown_path}")
-    extract_metadata(api_markdown_path)
+    extract_metadata(api_markdown_path, args.package_version)
 
 
 if __name__ == "__main__":

eng/scripts/save_package_api_hash.py

@@ -46,7 +46,7 @@ def main() -> None:
             ["pwsh", str(export_script), "-TokenJsonPath", str(token_file), "-OutputPath", str(package_artifact_dir)],
             check=True,
         )
-        extract_metadata(package_artifact_dir / "api.md")
+        extract_metadata(package_artifact_dir / "api.md", package_info["Version"])
 
         update_package_info(package_artifact_dir / "api.metadata.yml", package_info_path)
 

eng/tools/azure-sdk-tools/azpysdk/apistub.py

@@ -180,6 +180,7 @@ def run(self, args: argparse.Namespace) -> int:
                 return getattr(e, "returncode", 1)
 
             generate_from_pypi = getattr(args, "generate_from_pypi", None)
+            package_version = generate_from_pypi or parsed.version
 
             if generate_from_pypi:
                 pkg_path = self.download_pypi_wheel(executable, package_name, generate_from_pypi, staging_directory)
@@ -248,7 +249,14 @@ def run(self, args: argparse.Namespace) -> int:
 
                         logger.info(f"Extracting API metadata for {package_name}")
                         metadata_result = run(
-                            [executable, metadata_script, "--output-path", out_token_path],
+                            [
+                                executable,
+                                metadata_script,
+                                "--output-path",
+                                out_token_path,
+                                "--package-version",
+                                package_version,
+                            ],
                             check=True,
                             capture_output=True,
                             text=True,

eng/tools/azure-sdk-tools/tests/test_apistub.py

@@ -1,5 +1,7 @@
 import argparse
 import os
+import pathlib
+import subprocess
 import sys
 import pytest
 
@@ -26,6 +28,31 @@ def test_generate_from_pypi_flag_sets_version(self):
         assert args.generate_from_pypi == "1.0.0"
 
 
+class TestApiViewMetadata:
+    def test_package_version_is_written(self, tmp_path):
+        api_markdown = tmp_path / "api.md"
+        api_markdown.write_text(
+            "# Package is parsed using apiview-stub-generator(version:0.3.31), Python version: 3.12.9\n" "API body\n",
+            encoding="utf-8",
+        )
+        metadata_script = pathlib.Path(__file__).parents[3] / "scripts" / "extract_apiview_metadata.py"
+
+        subprocess.run(
+            [
+                sys.executable,
+                str(metadata_script),
+                "--api-markdown-path",
+                str(api_markdown),
+                "--package-version",
+                "1.35.0",
+            ],
+            check=True,
+        )
+
+        metadata = (tmp_path / "api.metadata.yml").read_text(encoding="utf-8")
+        assert "packageVersion: 1.35.0\n" in metadata
+
+
 # ── get_package_wheel_path() ─────────────────────────────────────────────
 
 
@@ -256,8 +283,10 @@ def test_outputs_use_package_directory(
         fake_parsed = MagicMock()
         fake_parsed.folder = str(tmp_path)
         fake_parsed.name = "azure-core"
+        fake_parsed.version = "1.35.0"
 
         captured_cmds = []
+        metadata_cmd = None
 
         def fake_apistub_run(exe, cmds, **kwargs):
             captured_cmds.append(cmds)
@@ -266,10 +295,12 @@ def fake_apistub_run(exe, cmds, **kwargs):
             open(os.path.join(out_dir, "azure-core_python.json"), "w").close()
 
         def fake_pwsh(cmd, **kwargs):
+            nonlocal metadata_cmd
             output_arg = "--output-path" if "extract_apiview_metadata.py" in cmd[1] else "-OutputPath"
             out_idx = cmd.index(output_arg)
             out_dir = cmd[out_idx + 1]
             if "extract_apiview_metadata.py" in cmd[1]:
+                metadata_cmd = cmd
                 open(os.path.join(out_dir, "api.metadata.yml"), "w").close()
             else:
                 open(os.path.join(out_dir, "api.md"), "w").close()
@@ -295,6 +326,9 @@ def fake_pwsh(cmd, **kwargs):
         assert os.path.exists(os.path.join(str(tmp_path), "api.md"))
         assert os.path.exists(os.path.join(str(tmp_path), "api.metadata.yml"))
         assert os.path.exists(os.path.join(str(tmp_path), "azure-core_python.json"))
+        assert metadata_cmd is not None
+        version_idx = metadata_cmd.index("--package-version")
+        assert metadata_cmd[version_idx + 1] == "1.35.0"
 
     @patch(
         "azpysdk.apistub.REPO_ROOT", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))