diff --git a/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml b/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml index 0d1a3010c2c7..518815323753 100644 --- a/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml +++ b/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml @@ -33,10 +33,10 @@ steps: - bash: | set -Eeuo pipefail - source_repo="$(ApiReviewSourceDir)" - tooling_dir="$(ApiReviewToolingDir)" - output_dir="${{ parameters.outputDir }}" - state_dir="${{ parameters.workingDir }}/state" + source_repo="$SOURCE_REPO" + tooling_dir="$TOOLING_DIR" + output_dir="$OUTPUT_DIR" + state_dir="$WORKING_DIR/state" mkdir -p "$source_repo" "$output_dir" "$state_dir" @@ -60,10 +60,10 @@ steps: printf '%s' "$matches" } - package_dir=$(find_package_dir "$source_repo" "${{ parameters.packageName }}") + package_dir=$(find_package_dir "$source_repo" "$PACKAGE_NAME") package_relative_path=$(realpath --relative-to="$source_repo" "$package_dir") - echo "Generating ${{ parameters.kind }} API review artifact from ${{ parameters.ref }}" + echo "Generating $KIND API review artifact from $REF" common_tasks_dir="$source_repo/scripts/devops_tasks" if [ ! -f "$common_tasks_dir/common_tasks.py" ]; then echo "Expected Python SDK common_tasks.py was not found at $common_tasks_dir/common_tasks.py" >&2 @@ -71,7 +71,7 @@ steps: fi install -D "$tooling_dir/eng/common/scripts/Export-APIViewMarkdown.ps1" "$source_repo/eng/common/scripts/Export-APIViewMarkdown.ps1" install -D "$tooling_dir/eng/scripts/extract_apiview_metadata.py" "$source_repo/eng/scripts/extract_apiview_metadata.py" - (cd "$source_repo" && PYTHONPATH="$common_tasks_dir${PYTHONPATH:+:$PYTHONPATH}" python -m azpysdk.main apistub --dest-dir "$package_dir" "${{ parameters.packageName }}") + (cd "$source_repo" && PYTHONPATH="$common_tasks_dir${PYTHONPATH:+:$PYTHONPATH}" python -m azpysdk.main apistub --dest-dir "$package_dir" "$PACKAGE_NAME") if [ ! -f "$package_dir/api.md" ]; then echo "Expected api.md was not generated at $package_dir/api.md" >&2 @@ -85,7 +85,19 @@ steps: cp "$package_dir/api.md" "$output_dir/api.md" cp "$package_dir/api.metadata.yml" "$output_dir/api.metadata.yml" - version=$(python -c 'import pathlib, re, sys; package_dir = pathlib.Path(sys.argv[1]); pattern = re.compile(r"^\s*VERSION\s*[:=]\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", re.MULTILINE); candidates = sorted(package_dir.rglob("*_version.py")) + sorted(package_dir.rglob("version.py")); match = next((pattern.search(candidate.read_text(encoding="utf-8")) for candidate in candidates if "_generated" not in candidate.parts and pattern.search(candidate.read_text(encoding="utf-8"))), None); print(match.group(1) if match else "")' "$package_dir") + version=$(sed -n 's/^packageVersion:[[:space:]]*//p' "$package_dir/api.metadata.yml") + if [ -z "$version" ]; then + echo "Expected packageVersion was not found in $package_dir/api.metadata.yml" >&2 + exit 1 + fi printf '%s' "$package_relative_path" > "$state_dir/package-relative-path.txt" printf '%s' "$version" > "$state_dir/version.txt" displayName: 'Generate ${{ parameters.kind }} Python API review bundle' + env: + SOURCE_REPO: $(ApiReviewSourceDir) + TOOLING_DIR: $(ApiReviewToolingDir) + OUTPUT_DIR: ${{ parameters.outputDir }} + WORKING_DIR: ${{ parameters.workingDir }} + PACKAGE_NAME: ${{ parameters.packageName }} + KIND: ${{ parameters.kind }} + REF: ${{ parameters.ref }} diff --git a/eng/scripts/extract_apiview_metadata.py b/eng/scripts/extract_apiview_metadata.py index 36e9e2ddf32a..8bf8db71de2d 100644 --- a/eng/scripts/extract_apiview_metadata.py +++ b/eng/scripts/extract_apiview_metadata.py @@ -10,13 +10,13 @@ ) -def extract_metadata(api_markdown_path: pathlib.Path) -> Dict[str, str]: +def extract_metadata(api_markdown_path: pathlib.Path, package_version: str) -> Dict[str, str]: with api_markdown_path.open(encoding="utf-8-sig", newline="") as api_markdown_file: file_text = api_markdown_file.read() line_ending = "\r\n" if "\r\n" in file_text else "\n" lines = re.split(r"\r?\n", file_text) - metadata: Dict[str, str] = {} + metadata: Dict[str, str] = {"packageVersion": package_version} filtered: List[str] = [] for line in lines: match = _METADATA_PATTERN.match(line) @@ -51,12 +51,13 @@ def main() -> None: parser = argparse.ArgumentParser(description="Extract Python APIView metadata from API markdown") parser.add_argument("--api-markdown-path") parser.add_argument("--output-path", default=".") + parser.add_argument("--package-version", required=True) args = parser.parse_args() api_markdown_path = pathlib.Path(args.api_markdown_path) if args.api_markdown_path else pathlib.Path(args.output_path) / "api.md" if not api_markdown_path.is_file(): parser.error(f"API markdown file not found: {api_markdown_path}") - extract_metadata(api_markdown_path) + extract_metadata(api_markdown_path, args.package_version) if __name__ == "__main__": diff --git a/eng/scripts/save_package_api_hash.py b/eng/scripts/save_package_api_hash.py index d9158218b6e0..7e2306f651da 100644 --- a/eng/scripts/save_package_api_hash.py +++ b/eng/scripts/save_package_api_hash.py @@ -46,7 +46,7 @@ def main() -> None: ["pwsh", str(export_script), "-TokenJsonPath", str(token_file), "-OutputPath", str(package_artifact_dir)], check=True, ) - extract_metadata(package_artifact_dir / "api.md") + extract_metadata(package_artifact_dir / "api.md", package_info["Version"]) update_package_info(package_artifact_dir / "api.metadata.yml", package_info_path) diff --git a/eng/tools/azure-sdk-tools/azpysdk/apistub.py b/eng/tools/azure-sdk-tools/azpysdk/apistub.py index 5f71e8fd9c96..0ff625e86ea7 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/apistub.py +++ b/eng/tools/azure-sdk-tools/azpysdk/apistub.py @@ -180,6 +180,7 @@ def run(self, args: argparse.Namespace) -> int: return getattr(e, "returncode", 1) generate_from_pypi = getattr(args, "generate_from_pypi", None) + package_version = generate_from_pypi or parsed.version if generate_from_pypi: pkg_path = self.download_pypi_wheel(executable, package_name, generate_from_pypi, staging_directory) @@ -248,7 +249,14 @@ def run(self, args: argparse.Namespace) -> int: logger.info(f"Extracting API metadata for {package_name}") metadata_result = run( - [executable, metadata_script, "--output-path", out_token_path], + [ + executable, + metadata_script, + "--output-path", + out_token_path, + "--package-version", + package_version, + ], check=True, capture_output=True, text=True, diff --git a/eng/tools/azure-sdk-tools/tests/test_apistub.py b/eng/tools/azure-sdk-tools/tests/test_apistub.py index b6f5e9191603..c9639850e1c6 100644 --- a/eng/tools/azure-sdk-tools/tests/test_apistub.py +++ b/eng/tools/azure-sdk-tools/tests/test_apistub.py @@ -1,5 +1,7 @@ import argparse import os +import pathlib +import subprocess import sys import pytest @@ -26,6 +28,31 @@ def test_generate_from_pypi_flag_sets_version(self): assert args.generate_from_pypi == "1.0.0" +class TestApiViewMetadata: + def test_package_version_is_written(self, tmp_path): + api_markdown = tmp_path / "api.md" + api_markdown.write_text( + "# Package is parsed using apiview-stub-generator(version:0.3.31), Python version: 3.12.9\n" "API body\n", + encoding="utf-8", + ) + metadata_script = pathlib.Path(__file__).parents[3] / "scripts" / "extract_apiview_metadata.py" + + subprocess.run( + [ + sys.executable, + str(metadata_script), + "--api-markdown-path", + str(api_markdown), + "--package-version", + "1.35.0", + ], + check=True, + ) + + metadata = (tmp_path / "api.metadata.yml").read_text(encoding="utf-8") + assert "packageVersion: 1.35.0\n" in metadata + + # ── get_package_wheel_path() ───────────────────────────────────────────── @@ -256,8 +283,10 @@ def test_outputs_use_package_directory( fake_parsed = MagicMock() fake_parsed.folder = str(tmp_path) fake_parsed.name = "azure-core" + fake_parsed.version = "1.35.0" captured_cmds = [] + metadata_cmd = None def fake_apistub_run(exe, cmds, **kwargs): captured_cmds.append(cmds) @@ -266,10 +295,12 @@ def fake_apistub_run(exe, cmds, **kwargs): open(os.path.join(out_dir, "azure-core_python.json"), "w").close() def fake_pwsh(cmd, **kwargs): + nonlocal metadata_cmd output_arg = "--output-path" if "extract_apiview_metadata.py" in cmd[1] else "-OutputPath" out_idx = cmd.index(output_arg) out_dir = cmd[out_idx + 1] if "extract_apiview_metadata.py" in cmd[1]: + metadata_cmd = cmd open(os.path.join(out_dir, "api.metadata.yml"), "w").close() else: open(os.path.join(out_dir, "api.md"), "w").close() @@ -295,6 +326,9 @@ def fake_pwsh(cmd, **kwargs): assert os.path.exists(os.path.join(str(tmp_path), "api.md")) assert os.path.exists(os.path.join(str(tmp_path), "api.metadata.yml")) assert os.path.exists(os.path.join(str(tmp_path), "azure-core_python.json")) + assert metadata_cmd is not None + version_idx = metadata_cmd.index("--package-version") + assert metadata_cmd[version_idx + 1] == "1.35.0" @patch( "azpysdk.apistub.REPO_ROOT", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))