From 8f80aeed34c0e9a856c93011798a53d988a6fc5b Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Fri, 9 Oct 2026 15:57:51 +0200
Subject: [PATCH] =?UTF-8?q?Revert=20"Publish=20each=20test=20assembly's=20?=
=?UTF-8?q?results=20as=20its=20own=20CI=20run=20instead=20of=20mer?=
=?UTF-8?q?=E2=80=A6"?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This reverts commit b164da6a001152d431e5bf5d9172cacefaa85b6a.
---
azure-pipelines-PR.yml | 28 ++--
azure-pipelines.yml | 14 +-
eng/PublishTestResults.ps1 | 78 ----------
eng/templates/batched-test-steps.yml | 12 +-
eng/templates/publish-test-results.yml | 17 ---
eng/tests/PublishTestResults.Tests.ps1 | 164 ----------------------
eng/tests/ToolingSafetyWorkflow.Tests.ps1 | 46 ------
7 files changed, 36 insertions(+), 323 deletions(-)
delete mode 100644 eng/PublishTestResults.ps1
delete mode 100644 eng/templates/publish-test-results.yml
delete mode 100644 eng/tests/PublishTestResults.Tests.ps1
delete mode 100644 eng/tests/ToolingSafetyWorkflow.Tests.ps1
diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml
index 2d4fb2d2cf1..18d1c38f3bd 100644
--- a/azure-pipelines-PR.yml
+++ b/azure-pipelines-PR.yml
@@ -268,9 +268,15 @@ stages:
NativeToolsOnMachine: true
displayName: Build
- - template: /eng/templates/publish-test-results.yml
- parameters:
+ - task: PublishTestResults@2
+ displayName: Publish Test Results
+ inputs:
+ testResultsFormat: 'XUnit'
testRunTitle: WindowsNoRealsig_testCoreclr
+ mergeTestResults: true
+ testResultsFiles: '*.xml'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
+ condition: succeededOrFailed()
- task: PublishBuildArtifacts@1
displayName: Publish Build BinLog
@@ -368,10 +374,16 @@ stages:
NativeToolsOnMachine: true
displayName: Build and Test $(_testKind) $(transparentCompiler)
- - template: /eng/templates/publish-test-results.yml
- parameters:
- configuration: $(_configuration)
+ - task: PublishTestResults@2
+ displayName: Publish Test Results
+ inputs:
+ testResultsFormat: 'XUnit'
testRunTitle: WindowsCompressedMetadata $(_testKind) $(transparentCompiler)
+ mergeTestResults: true
+ testResultsFiles: '*.xml'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)'
+ continueOnError: true
+ condition: succeededOrFailed()
- task: PublishBuildArtifacts@1
displayName: Publish Tests BinLog
condition: always()
@@ -454,14 +466,10 @@ stages:
steps:
- checkout: self
clean: true
- - powershell: |
- $result = Invoke-Pester eng/tests/*.Tests.ps1 -PassThru
- if ($result.FailedCount -gt 0) { exit 1 }
- displayName: Pipeline regression tests
- pwsh: .\eng\MockBuild.ps1
displayName: Build with OfficialBuildId
- # Linux (split into 2 batches to limit test-process memory; batch 3 is desktop-only FSharpSuite)
+ # Linux (split into 2 batches; batch 3 is desktop-only FSharpSuite)
- job: Linux
strategy:
matrix:
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 21d5db3a7d7..1517ff30b68 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -148,11 +148,14 @@ extends:
NativeToolsOnMachine: true
FSHARP_CACHE_OVERRIDE: 256
FSharp_CacheEvictionImmediate: true
- - template: /eng/templates/publish-test-results.yml
- parameters:
- configuration: $(_BuildConfig)
- testRunTitle: Official $(_BuildConfig)
- condition: ne(variables['SkipTests'], 'true')
+ - task: PublishTestResults@2
+ displayName: Publish Test Results
+ inputs:
+ testResultsFormat: 'XUnit'
+ testResultsFiles: '*.xml'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ continueOnError: true
+ condition: ne(variables['SkipTests'], 'true')
- task: 1ES.PublishPipelineArtifact@1
displayName: Publish Test Logs
inputs:
@@ -229,3 +232,4 @@ extends:
insertTeamEmail: fsharpteam@microsoft.com
insertTeamName: 'F#'
completeInsertion: 'auto'
+
diff --git a/eng/PublishTestResults.ps1 b/eng/PublishTestResults.ps1
deleted file mode 100644
index 22cdf7f3ddb..00000000000
--- a/eng/PublishTestResults.ps1
+++ /dev/null
@@ -1,78 +0,0 @@
-param(
- [Parameter(Mandatory = $true)][string]$ResultsDirectory,
- [Parameter(Mandatory = $true)][string]$RunTitle,
- [int]$TimeoutSeconds = 300
-)
-
-Set-StrictMode -Version Latest
-$ErrorActionPreference = 'Stop'
-
-function ConvertTo-VsoValue([string]$Value) {
- $Value.Replace('%', '%AZP25').Replace("`r", '%0D').Replace("`n", '%0A').Replace(';', '%3B').Replace(']', '%5D')
-}
-
-$files = @(if (Test-Path -LiteralPath $ResultsDirectory) {
- Get-ChildItem -LiteralPath $ResultsDirectory -Filter '*.xml' -File | Sort-Object Name
-})
-if ($files.Count -eq 0) {
- Write-Warning "No test result XML files found in $ResultsDirectory."
- return
-}
-
-if (!$env:SYSTEM_ACCESSTOKEN) {
- throw 'SYSTEM_ACCESSTOKEN is required to verify published test runs.'
-}
-
-$headers = @{ Authorization = "Bearer $env:SYSTEM_ACCESSTOKEN" }
-$project = [uri]::EscapeDataString($env:SYSTEM_TEAMPROJECT)
-$build = [uri]::EscapeDataString($env:BUILD_BUILDURI)
-$runsUrl = "$env:SYSTEM_COLLECTIONURI$project/_apis/test/runs"
-$runIds = @()
-
-foreach ($file in $files) {
- $xml = [xml](Get-Content -LiteralPath $file.FullName -Raw)
- # Azure's XUnit parser coalesces results with the same case-sensitive test name.
- $testNames = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
- foreach ($test in $xml.SelectNodes('/assemblies/assembly/collection/test')) {
- $null = $testNames.Add($test.GetAttribute('name'))
- }
- $expected = $testNames.Count
- if ($expected -eq 0) {
- throw "No test cases found in $($file.FullName)."
- }
- if ($file.FullName.Contains(',')) {
- throw "The test result path cannot contain a comma: $($file.FullName)"
- }
-
- $title = "$RunTitle $($file.Name) $env:SYSTEM_JOBID.$env:SYSTEM_JOBATTEMPT"
- $commandTitle = ConvertTo-VsoValue $title
- $commandFile = ConvertTo-VsoValue $file.FullName
- Write-Host "##vso[results.publish type=XUnit;mergeResults=false;runTitle=$commandTitle;publishRunAttachments=true;]$commandFile"
-
- $deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds)
- do {
- $response = Invoke-RestMethod -Uri "${runsUrl}?buildUri=$build&`$top=10000&api-version=7.1" -Headers $headers -TimeoutSec 30
- $runs = @($response.value | Where-Object { $_.name -eq $title -or $_.name -eq "${title}_1" })
- if ($runs.Count -gt 1) {
- throw "Multiple test runs found for $($file.Name)."
- }
- if ($runs.Count -eq 1) {
- $run = Invoke-RestMethod -Uri "$runsUrl/$($runs[0].id)?api-version=7.1" -Headers $headers -TimeoutSec 30
- if ($runIds -contains $run.id) {
- throw "Test run $($run.id) was reused for $($file.Name)."
- }
- if ($run.state -eq 'Completed') {
- if ($run.totalTests -ne $expected -or $run.incompleteTests -ne 0) {
- throw "Run $($run.id) contains $($run.totalTests) tests ($($run.incompleteTests) incomplete); expected $expected from $($file.Name)."
- }
- $runIds += $run.id
- Write-Host "Verified $($file.Name): run $($run.id), $expected results, Completed."
- break
- }
- }
- if ([DateTime]::UtcNow -ge $deadline) {
- throw "Timed out waiting for a complete test run for $($file.Name)."
- }
- Start-Sleep -Seconds 5
- } while ($true)
-}
diff --git a/eng/templates/batched-test-steps.yml b/eng/templates/batched-test-steps.yml
index a661aec6893..712d5e3219f 100644
--- a/eng/templates/batched-test-steps.yml
+++ b/eng/templates/batched-test-steps.yml
@@ -33,10 +33,16 @@ steps:
env: ${{ parameters.buildEnv }}
displayName: Build / Test
-- template: /eng/templates/publish-test-results.yml
- parameters:
- configuration: ${{ parameters.configuration }}
+- task: PublishTestResults@2
+ displayName: Publish Test Results
+ inputs:
+ testResultsFormat: 'XUnit'
testRunTitle: ${{ parameters.testRunTitlePrefix }} Batch$(batchNumber)
+ mergeTestResults: true
+ testResultsFiles: '*.xml'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}'
+ continueOnError: true
+ condition: succeededOrFailed()
- ${{ if parameters.publishBinLog }}:
- task: PublishBuildArtifacts@1
diff --git a/eng/templates/publish-test-results.yml b/eng/templates/publish-test-results.yml
deleted file mode 100644
index e8799d610ee..00000000000
--- a/eng/templates/publish-test-results.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-parameters:
- configuration: 'Release'
- testRunTitle: ''
- condition: succeededOrFailed()
-
-steps:
-- task: PowerShell@2
- displayName: Publish and verify test results
- inputs:
- pwsh: true
- filePath: 'eng/PublishTestResults.ps1'
- arguments: >-
- -ResultsDirectory "$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}"
- -RunTitle "${{ parameters.testRunTitle }}"
- env:
- SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- condition: ${{ parameters.condition }}
diff --git a/eng/tests/PublishTestResults.Tests.ps1 b/eng/tests/PublishTestResults.Tests.ps1
deleted file mode 100644
index 77f7a44e5a1..00000000000
--- a/eng/tests/PublishTestResults.Tests.ps1
+++ /dev/null
@@ -1,164 +0,0 @@
-# Run with: Invoke-Pester eng/tests/PublishTestResults.Tests.ps1 -EnableExit
-
-Describe 'Per-file test result publication' {
- BeforeEach {
- $script:publication = @{ Commands = @(); Runs = @(); Mode = 'complete'; Polls = 0 }
- $script:publisher = Join-Path $PSScriptRoot '..\PublishTestResults.ps1'
- $script:results = Join-Path $TestDrive 'results'
- New-Item $results -ItemType Directory -Force | Out-Null
-
- Mock Write-Host {
- param($Object)
- $line = [string]$Object
- if ($line -like '##vso[[]results.publish*') {
- if ($publication.Runs | Where-Object state -ne 'Completed') {
- throw 'The next file was published before the previous run completed.'
- }
-
- $publication.Commands += $line
- $title, $file = @(
- [regex]::Match($line, 'runTitle=([^;]+);').Groups[1].Value
- $line.Substring($line.IndexOf(']') + 1)
- ) | ForEach-Object { $_.Replace('%0D', "`r").Replace('%0A', "`n").Replace('%3B', ';').Replace('%5D', ']').Replace('%AZP25', '%') }
- $xml = [xml][IO.File]::ReadAllText($file)
- $count = @($xml.SelectNodes('//test') | ForEach-Object { $_.GetAttribute('name') } | Select-Object -Unique).Count
- $publication.Runs += [pscustomobject]@{
- id = $(if ($publication.Mode -eq 'reused') { 1 } else { $publication.Commands.Count })
- name = $(if ($publication.Mode -eq 'suffix') { "${title}_1" } else { $title })
- state = 'InProgress'
- totalTests = $(if ($publication.Mode -eq 'truncated') { $count - 1 } else { $count })
- incompleteTests = $(if ($publication.Mode -eq 'incomplete') { 1 } else { 0 })
- }
- }
- }
- Mock Invoke-RestMethod {
- $publication.Polls++
- if ($publication.Mode -ne 'stuck' -and ($publication.Mode -ne 'delayed' -or $publication.Polls -gt 2)) {
- $publication.Runs | ForEach-Object { $_.state = 'Completed' }
- }
- if ($Uri -match '/runs/(\d+)\?') {
- return $publication.Runs | Where-Object id -eq $Matches[1] | Select-Object -Last 1
- }
- if ($publication.Mode -eq 'missing' -or ($publication.Mode -eq 'late' -and $publication.Polls -eq 1)) {
- return @{ value = @() }
- }
- if ($publication.Mode -eq 'ambiguous') {
- return @{ value = @($publication.Runs[0], @{ id = 99; name = "$($publication.Runs[0].name)_1" }) }
- }
- return @{ value = $publication.Runs }
- }
- Mock Start-Sleep {}
-
- $script:savedEnvironment = @{}
- foreach ($name in @('SYSTEM_ACCESSTOKEN', 'SYSTEM_COLLECTIONURI', 'SYSTEM_TEAMPROJECT', 'BUILD_BUILDURI', 'SYSTEM_JOBID', 'SYSTEM_JOBATTEMPT')) {
- $script:savedEnvironment[$name] = [Environment]::GetEnvironmentVariable($name)
- [Environment]::SetEnvironmentVariable($name, 'test')
- }
- $env:SYSTEM_COLLECTIONURI = 'https://dev.azure.com/example/'
-
- foreach ($count in @(117, 6175)) {
- $tests = (1..$count | ForEach-Object { "" }) -join ''
- Set-Content (Join-Path $results "$count.xml") "$tests"
- }
- }
-
- AfterEach {
- foreach ($name in $savedEnvironment.Keys) {
- [Environment]::SetEnvironmentVariable($name, $savedEnvironment[$name])
- }
- }
-
- It 'publishes and verifies a separate complete run for each XML ()' -TestCases @(
- @{ Mode = 'complete' }
- @{ Mode = 'suffix' }
- @{ Mode = 'duplicate-names' }
- @{ Mode = 'case-distinct-names' }
- ) {
- param($Mode)
- $publication.Mode = $Mode
- if ($Mode -in @('duplicate-names', 'case-distinct-names')) {
- foreach ($file in Get-ChildItem $results -Filter '*.xml') {
- $xml = [xml](Get-Content $file.FullName -Raw)
- $test = $xml.SelectSingleNode('//test').CloneNode($true)
- $collection = $xml.CreateElement('collection')
- $null = $collection.AppendChild($test)
- $null = $xml.assemblies.assembly.AppendChild($collection)
- if ($Mode -eq 'case-distinct-names') { $test.SetAttribute('name', 'case1') }
- $xml.Save($file.FullName)
- }
- }
- & $publisher -ResultsDirectory $results -RunTitle 'Linux Batch1' -TimeoutSeconds 0
- $publication.Commands.Count | Should Be 2
- @($publication.Runs.id | Select-Object -Unique).Count | Should Be 2
- $expected = if ($Mode -eq 'case-distinct-names') { '118,6176' } else { '117,6175' }
- ($publication.Runs.totalTests -join ',') | Should Be $expected
- foreach ($command in $publication.Commands) {
- $command | Should Match 'type=XUnit;mergeResults=false;'
- $command | Should Match 'publishRunAttachments=true;'
- }
- Assert-MockCalled Invoke-RestMethod -Times 2 -Exactly -Scope It -ParameterFilter { $Uri -match '/runs/\d+\?' }
- Assert-MockCalled Invoke-RestMethod -Times 1 -Exactly -Scope It -ParameterFilter { $Uri -match '/runs/1\?' }
- Assert-MockCalled Invoke-RestMethod -Times 1 -Exactly -Scope It -ParameterFilter { $Uri -match '/runs/2\?' }
- }
-
- It 'rejects publication rather than silently losing results' -TestCases @(
- @{ Mode = 'truncated'; Error = 'expected 117'; Published = 1 }
- @{ Mode = 'incomplete'; Error = '1 incomplete'; Published = 1 }
- @{ Mode = 'reused'; Error = 'reused'; Published = 2 }
- @{ Mode = 'stuck'; Error = 'Timed out'; Published = 1 }
- @{ Mode = 'missing'; Error = 'Timed out'; Published = 1 }
- @{ Mode = 'ambiguous'; Error = 'Multiple test runs'; Published = 1 }
- ) {
- param($Mode, $Error, $Published)
- $publication.Mode = $Mode
- { & $publisher -ResultsDirectory $results -RunTitle 'Linux Batch1' -TimeoutSeconds 0 } | Should Throw $Error
- $publication.Commands.Count | Should Be $Published
- }
-
- It 'waits for runs before publishing the next file' -TestCases @(
- @{ Mode = 'delayed' }
- @{ Mode = 'late' }
- ) {
- param($Mode)
- $publication.Mode = $Mode
- & $publisher -ResultsDirectory $results -RunTitle 'Linux Batch1'
- Assert-MockCalled Start-Sleep -Times 1 -Exactly -Scope It
- $publication.Commands.Count | Should Be 2
- }
-
- It 'escapes logging command properties and file names' {
- Get-ChildItem $results -Filter '*.xml' | Rename-Item -NewName { "space %;] $($_.Name)" }
- & $publisher -ResultsDirectory $results -RunTitle 'Linux %;] Batch1'
- $publication.Commands.Count | Should Be 2
- $publication.Commands[0] | Should Match 'Linux %AZP25%3B%5D Batch1'
- foreach ($command in $publication.Commands) {
- $command | Should Match ']([^]]*)space %AZP25%3B%5D (117|6175)\.xml$'
- }
- }
-
- It 'warns without publishing when the results directory is ' -TestCases @(
- @{ Directory = 'empty' }
- @{ Directory = 'missing' }
- ) {
- param($Directory)
- $path = Join-Path $TestDrive $Directory
- if ($Directory -eq 'empty') { New-Item $path -ItemType Directory | Out-Null }
- Mock Write-Warning {}
- & $publisher -ResultsDirectory $path -RunTitle 'Linux Batch1'
- Assert-MockCalled Write-Warning -Times 1 -Exactly -Scope It
- $publication.Commands.Count | Should Be 0
- }
-}
-
-Describe 'Test result publishing pipelines' {
- It 'uses isolated publication in ' -TestCases @(
- @{ File = 'azure-pipelines-PR.yml'; Calls = 2 }
- @{ File = 'azure-pipelines.yml'; Calls = 1 }
- @{ File = 'eng\templates\batched-test-steps.yml'; Calls = 1 }
- ) {
- param($File, $Calls)
- $yaml = Get-Content (Join-Path $PSScriptRoot "..\..\$File") -Raw
- ($yaml -match 'task: PublishTestResults@2') | Should Be $false
- [regex]::Matches($yaml, 'template: /eng/templates/publish-test-results.yml').Count | Should Be $Calls
- }
-}
diff --git a/eng/tests/ToolingSafetyWorkflow.Tests.ps1 b/eng/tests/ToolingSafetyWorkflow.Tests.ps1
deleted file mode 100644
index 3d4279b9e01..00000000000
--- a/eng/tests/ToolingSafetyWorkflow.Tests.ps1
+++ /dev/null
@@ -1,46 +0,0 @@
-# Run with: Invoke-Pester eng/tests/ToolingSafetyWorkflow.Tests.ps1 -EnableExit
-
-Describe 'Tooling safety workflow' {
- function Assert-ScannerBaselineHandoff($Jobs) {
- $Jobs['pre_activation'] | Should Match '(?m)^ - uses: actions/upload-artifact@[^\r\n]+\r?\n with:\r?\n name: scanner-categories\r?\n path: \$\{\{ runner\.temp \}\}/scanner-categories\.json(?:\r?\n|\z)'
- $Jobs['agent'] | Should Match '(?m)^ - uses: actions/download-artifact@[^\r\n]+\r?\n with:\r?\n name: scanner-categories\r?\n path: /tmp/gh-aw/(?:\r?\n|\z)'
- $Jobs['agent'] | Should Match "fs\.readFileSync\('/tmp/gh-aw/scanner-categories\.json', 'utf8'\)"
- }
-
- BeforeEach {
- $yaml = Get-Content (Join-Path $PSScriptRoot '..\..\.github\workflows\labelops-pr-security-scan.lock.yml') -Raw
- $jobs = @{}
- foreach ($match in [regex]::Matches($yaml, '(?ms)^ (\w+):\r?\n(.*?)(?=^ \w+:|\z)')) {
- $jobs[$match.Groups[1].Value] = $match.Groups[2].Value
- }
- $agent = $jobs['agent']
- }
-
- It 'does not check out PR code in the API-only agent' {
- ($agent -match 'uses: actions/checkout@|checkout_pr_branch\.cjs') | Should Be $false
- }
-
- It 'gates activation on the pre-activation proceed output' {
- $jobs['activation'] | Should Match '(?m)^ needs: pre_activation\r?$'
- $jobs['activation'] | Should Match "needs\.pre_activation\.result == 'success' && needs\.pre_activation\.outputs\.proceed == 'true'"
- $jobs['pre_activation'] | Should Match '(?m)^ proceed: \$\{\{ steps\.gate\.outputs\.proceed \}\}\r?$'
- }
-
- It 'hands the category baseline from pre-activation to the comment filter' {
- Assert-ScannerBaselineHandoff $jobs
- }
-
- It 'rejects a baseline handoff' -TestCases @(
- @{ Defect = 'missing upload'; Job = 'pre_activation'; From = 'uses: actions/upload-artifact@'; To = 'uses: missing@' }
- @{ Defect = 'missing download'; Job = 'agent'; From = 'uses: actions/download-artifact@'; To = 'uses: missing@' }
- @{ Defect = 'mismatched artifact name'; Job = 'agent'; From = 'name: scanner-categories'; To = 'name: other-categories' }
- @{ Defect = 'mismatched upload path'; Job = 'pre_activation'; From = 'path: ${{ runner.temp }}/scanner-categories.json'; To = 'path: ${{ runner.temp }}/other.json' }
- @{ Defect = 'mismatched download path'; Job = 'agent'; From = 'path: /tmp/gh-aw/'; To = 'path: /tmp/other/' }
- @{ Defect = 'mismatched filter input'; Job = 'agent'; From = "fs.readFileSync('/tmp/gh-aw/scanner-categories.json'"; To = "fs.readFileSync('/tmp/gh-aw/other.json'" }
- ) {
- param($Defect, $Job, $From, $To)
- $broken = $jobs.Clone()
- $broken[$Job] = $broken[$Job].Replace($From, $To)
- { Assert-ScannerBaselineHandoff $broken } | Should Throw
- }
-}