[PR2 / DO NOT MERGE until servers support abandon] Async WaitForExitAsync; grandchildren require abandon

source/Octopus.Manager.Tentacle.Tests/Builders/SetupTentacleWizardModelBuilder.cs

@@ -32,6 +32,8 @@ public SetupTentacleWizardModelBuilder()
             commandLineRunner = Substitute.For<ICommandLineRunner>();
             commandLineRunner.Execute(Arg.Any<CommandLineInvocation>(), Arg.Any<ILog>()).Returns(true);
             commandLineRunner.Execute(Arg.Any<IEnumerable<CommandLineInvocation>>(), Arg.Any<ILog>()).Returns(true);
+            commandLineRunner.ExecuteAsync(Arg.Any<CommandLineInvocation>(), Arg.Any<ILog>()).Returns(true);
+            commandLineRunner.ExecuteAsync(Arg.Any<IEnumerable<CommandLineInvocation>>(), Arg.Any<ILog>()).Returns(true);
         }
 
         public SetupTentacleWizardModelBuilder WithTelemetryService(ITelemetryService telemetryService)

source/Octopus.Manager.Tentacle/PreReq/PowerShellPrerequisite.cs

@@ -1,5 +1,7 @@
 using System;
 using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
 using Octopus.Tentacle.Diagnostics;
 using Octopus.Tentacle.Util;
 
@@ -9,31 +11,46 @@ public class PowerShellPrerequisite : IPrerequisite
     {
         public string StatusMessage => "Checking that Windows PowerShell 2.0 is installed...";
 
+        // Why this is sync: IPrerequisite.Check() is part of a sync interface used by
+        // the WPF installer's prerequisite plumbing. Making it async would mean
+        // converting the whole IPrerequisite chain, which is a wider refactor than
+        // this PR.
+        //
+        // Why blocking on the async call is safe: PreReqWindow.Start dispatches each
+        // prerequisite via DispatchHelper.Background, which queues us via
+        // ThreadPool.QueueUserWorkItem. That's a plain thread-pool worker with no
+        // SynchronizationContext, so there's nothing for the awaited continuation
+        // to wait on.
+        // See https://blog.stephencleary.com/2012/07/dont-block-on-async-code.html
         public PrerequisiteCheckResult Check()
+            => CheckAsync().GetAwaiter().GetResult();
+
+        async Task<PrerequisiteCheckResult> CheckAsync()
         {
-            return CheckPowerShellIsInstalled(out var commandLineOutput)
+            var (installed, commandLineOutput) = await CheckPowerShellIsInstalledAsync();
+            return installed
                 ? PrerequisiteCheckResult.Successful()
                 : PrerequisiteCheckResult.Failed("Windows PowerShell 2.0 or above does not appear to be installed and on the System Path on this machine. Please install Windows PowerShell or add it to the System Path then re-run this setup tool.",
                     helpLink: "http://g.octopushq.com/HowDoIInstallPowerShell",
                     helpLinkText: "Download and install Windows PowerShell",
                     commandLineOutput: commandLineOutput);
         }
 
-        static bool CheckPowerShellIsInstalled(out string commandLineOutput)
+        static async Task<(bool installed, string commandLineOutput)> CheckPowerShellIsInstalledAsync()
         {
             var stdOut = new StringWriter();
             var stdErr = new StringWriter();
 
             const string powerShellExe = "powershell.exe";
             const string arguments = "-NonInteractive -NoProfile -Command \"Write-Output $PSVersionTable.PSVersion\"";
-            commandLineOutput = $"{powerShellExe} {arguments}";
+            var commandLineOutput = $"{powerShellExe} {arguments}";
 
             // Despite our old check conforming to Microsoft's recommendations
             // for PS version checking around the 1.0/2.0 era, and extending
             // to detect 3.0, it failed to detect 4. Going the direct route:
             try
             {
-                SilentProcessRunnerExtended.ExecuteCommand(
+                await SilentProcessRunnerExtended.ExecuteCommandAsync(
                     powerShellExe,
                     arguments,
                     ".",
@@ -45,12 +62,12 @@ static bool CheckPowerShellIsInstalled(out string commandLineOutput)
 
                 commandLineOutput = $"{commandLineOutput}{Environment.NewLine}{stdOut}{Environment.NewLine}{stdErr}";
 
-                return outputText.Contains("Major");
+                return (outputText.Contains("Major"), commandLineOutput);
             }
             catch (Exception ex)
             {
                 commandLineOutput = $"{commandLineOutput}{Environment.NewLine}{ex}";
-                return false;
+                return (false, commandLineOutput);
             }
         }
     }

source/Octopus.Manager.Tentacle/TentacleConfiguration/SetupWizard/ReviewAndRunScriptTabViewModel.cs

@@ -46,7 +46,7 @@ public async Task<bool> GenerateAndExecuteScript()
             {
                 var script = GenerateScript();
                 ContributeSensitiveValues(logger);
-                success = commandLineRunner.Execute(script, logger);
+                success = await commandLineRunner.ExecuteAsync(script, logger);
             }
             catch (Exception ex)
             {

source/Octopus.Tentacle.Client.Tests/CapabilitiesResponseV2ExtensionMethodsTests.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using FluentAssertions;
+using NUnit.Framework;
+using Octopus.Tentacle.Client.Capabilities;
+using Octopus.Tentacle.Contracts.Capabilities;
+
+namespace Octopus.Tentacle.Client.Tests
+{
+    [TestFixture]
+    public class CapabilitiesResponseV2ExtensionMethodsTests
+    {
+        [Test]
+        public void HasAbandonScriptV2_WhenAdvertised_ReturnsTrue()
+        {
+            var capabilities = new CapabilitiesResponseV2(new List<string> { "IScriptServiceV2", "AbandonScriptAsync" });
+            capabilities.HasAbandonScriptV2().Should().BeTrue();
+        }
+
+        [Test]
+        public void HasAbandonScriptV2_WhenNotAdvertised_ReturnsFalse()
+        {
+            var capabilities = new CapabilitiesResponseV2(new List<string> { "IScriptServiceV2" });
+            capabilities.HasAbandonScriptV2().Should().BeFalse();
+        }
+
+        [Test]
+        public void HasAbandonScriptV2_WhenEmpty_ReturnsFalse()
+        {
+            var capabilities = new CapabilitiesResponseV2(new List<string>());
+            capabilities.HasAbandonScriptV2().Should().BeFalse();
+        }
+    }
+}

source/Octopus.Tentacle.Client.Tests/ObservingScriptOrchestratorAbandonTests.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using FluentAssertions;
+using NSubstitute;
+using NUnit.Framework;
+using Octopus.Tentacle.Client.EventDriven;
+using Octopus.Tentacle.Client.Scripts;
+using Octopus.Tentacle.Contracts;
+
+namespace Octopus.Tentacle.Client.Tests
+{
+    [TestFixture]
+    public class ObservingScriptOrchestratorAbandonTests
+    {
+        static CommandContext Context() => new(new ScriptTicket("T"), 0, ScriptServiceVersion.ScriptServiceVersion2);
+
+        static ScriptOperationExecutionResult Running()
+            => new(new ScriptStatus(ProcessState.Running, 0, new List<ProcessOutput>()), Context());
+
+        static ScriptOperationExecutionResult Complete(int exitCode)
+            => new(new ScriptStatus(ProcessState.Complete, exitCode, new List<ProcessOutput>()), Context());
+
+        static ObservingScriptOrchestrator CreateOrchestrator(IScriptExecutor executor, TimeSpan? abandonAfter)
+            => new(
+                new ImmediateBackoff(),
+                _ => { },
+                _ => Task.CompletedTask,
+                executor,
+                abandonAfter);
+
+        sealed class ImmediateBackoff : IScriptObserverBackoffStrategy
+        {
+            public TimeSpan GetBackoff(int iteration) => TimeSpan.Zero;
+        }
+
+        [Test]
+        public async Task ParamUnset_CancelsOnly_NeverAbandons()
+        {
+            var executor = Substitute.For<IScriptExecutor>();
+            executor.CancelScript(Arg.Any<CommandContext>())
+                .Returns(Running(), Complete(ScriptExitCodes.CanceledExitCode));
+
+            var orchestrator = CreateOrchestrator(executor, abandonAfter: null);
+            using var cts = new CancellationTokenSource();
+            cts.Cancel();
+
+            await orchestrator.ObserveUntilComplete(Running(), cts.Token);
+
+            await executor.DidNotReceive().AbandonScript(Arg.Any<CommandContext>());
+            await executor.Received().CancelScript(Arg.Any<CommandContext>());
+        }
+
+        [Test]
+        public async Task ThresholdCrossed_SwitchesFromCancelToAbandon()
+        {
+            var executor = Substitute.For<IScriptExecutor>();
+            // abandonAfter = Zero means the threshold is crossed on the first cancelled iteration,
+            // so the orchestrator abandons immediately. AbandonScript returns Complete to end the loop.
+            executor.AbandonScript(Arg.Any<CommandContext>())
+                .Returns(Complete(ScriptExitCodes.AbandonedExitCode));
+
+            var orchestrator = CreateOrchestrator(executor, abandonAfter: TimeSpan.Zero);
+            using var cts = new CancellationTokenSource();
+            cts.Cancel();
+
+            var result = await orchestrator.ObserveUntilComplete(Running(), cts.Token);
+
+            await executor.Received().AbandonScript(Arg.Any<CommandContext>());
+            result.ScriptStatus.ExitCode.Should().Be(ScriptExitCodes.AbandonedExitCode);
+        }
+    }
+}

source/Octopus.Tentacle.Client.Tests/ScriptServiceV2ExecutorAbandonTests.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using FluentAssertions;
+using Halibut.ServiceModel;
+using NSubstitute;
+using NUnit.Framework;
+using Octopus.Tentacle.Client;
+using Octopus.Tentacle.Client.EventDriven;
+using Octopus.Tentacle.Client.Execution;
+using Octopus.Tentacle.Client.Observability;
+using Octopus.Tentacle.Client.Retries;
+using Octopus.Tentacle.Client.Scripts;
+using Octopus.Tentacle.Contracts;
+using Octopus.Tentacle.Contracts.Capabilities;
+using Octopus.Tentacle.Contracts.ClientServices;
+using Octopus.Tentacle.Contracts.Logging;
+using Octopus.Tentacle.Contracts.Observability;
+using Octopus.Tentacle.Contracts.ScriptServiceV2;
+
+namespace Octopus.Tentacle.Client.Tests
+{
+    [TestFixture]
+    public class ScriptServiceV2ExecutorAbandonTests
+    {
+        static ScriptServiceV2Executor CreateExecutor(IAsyncClientScriptServiceV2 scriptService, IAsyncClientCapabilitiesServiceV2 capabilities)
+            => new(
+                scriptService,
+                capabilities,
+                RpcCallExecutorFactory.Create(TimeSpan.Zero, Substitute.For<ITentacleClientObserver>()),
+                ClientOperationMetricsBuilder.Start(),
+                TimeSpan.Zero,
+                new TentacleClientOptions(new RpcRetrySettings(RetriesEnabled: false, RetryDuration: TimeSpan.Zero)),
+                Substitute.For<ITentacleClientTaskLog>());
+
+        static CommandContext Context() => new(new ScriptTicket("TestTicket"), 0, ScriptServiceVersion.ScriptServiceVersion2);
+
+        [Test]
+        public async Task AbandonScript_WhenCapabilityAdvertised_CallsAbandonScriptAsync()
+        {
+            var scriptService = Substitute.For<IAsyncClientScriptServiceV2>();
+            scriptService.AbandonScriptAsync(Arg.Any<AbandonScriptCommandV2>(), Arg.Any<HalibutProxyRequestOptions>())
+                .Returns(x => Task.FromResult(new ScriptStatusResponseV2(
+                    x.Arg<AbandonScriptCommandV2>().Ticket, ProcessState.Complete,
+                    ScriptExitCodes.AbandonedExitCode, new List<ProcessOutput>(), 1)));
+
+            var capabilities = Substitute.For<IAsyncClientCapabilitiesServiceV2>();
+            capabilities.GetCapabilitiesAsync(Arg.Any<HalibutProxyRequestOptions>())
+                .Returns(Task.FromResult(new CapabilitiesResponseV2(new List<string> { "IScriptServiceV2", "AbandonScriptAsync" })));
+
+            var executor = CreateExecutor(scriptService, capabilities);
+
+            var result = await executor.AbandonScript(Context());
+
+            await scriptService.Received(1).AbandonScriptAsync(Arg.Any<AbandonScriptCommandV2>(), Arg.Any<HalibutProxyRequestOptions>());
+            await scriptService.DidNotReceive().CancelScriptAsync(Arg.Any<CancelScriptCommandV2>(), Arg.Any<HalibutProxyRequestOptions>());
+            result.ScriptStatus.ExitCode.Should().Be(ScriptExitCodes.AbandonedExitCode);
+        }
+
+        [Test]
+        public async Task AbandonScript_WhenCapabilityAbsent_FallsBackToCancelScriptAsync()
+        {
+            var scriptService = Substitute.For<IAsyncClientScriptServiceV2>();
+            scriptService.CancelScriptAsync(Arg.Any<CancelScriptCommandV2>(), Arg.Any<HalibutProxyRequestOptions>())
+                .Returns(x => Task.FromResult(new ScriptStatusResponseV2(
+                    x.Arg<CancelScriptCommandV2>().Ticket, ProcessState.Running,
+                    0, new List<ProcessOutput>(), 1)));
+
+            var capabilities = Substitute.For<IAsyncClientCapabilitiesServiceV2>();
+            capabilities.GetCapabilitiesAsync(Arg.Any<HalibutProxyRequestOptions>())
+                .Returns(Task.FromResult(new CapabilitiesResponseV2(new List<string> { "IScriptServiceV2" })));
+
+            var executor = CreateExecutor(scriptService, capabilities);
+
+            await executor.AbandonScript(Context());
+
+            await scriptService.DidNotReceive().AbandonScriptAsync(Arg.Any<AbandonScriptCommandV2>(), Arg.Any<HalibutProxyRequestOptions>());
+            await scriptService.Received(1).CancelScriptAsync(Arg.Any<CancelScriptCommandV2>(), Arg.Any<HalibutProxyRequestOptions>());
+        }
+    }
+}

source/Octopus.Tentacle.Client/Capabilities/CapabilitiesResponseV2ExtensionMethods.cs

@@ -15,5 +15,17 @@ public static bool HasScriptServiceV2(this CapabilitiesResponseV2 capabilities)
 
             return capabilities.SupportedCapabilities.Contains(nameof(IScriptServiceV2));
         }
+
+        public static bool HasAbandonScriptV2(this CapabilitiesResponseV2 capabilities)
+        {
+            if (capabilities?.SupportedCapabilities?.Any() != true)
+            {
+                return false;
+            }
+
+            // Tentacle advertises this as nameof(ScriptServiceV2.AbandonScriptAsync). Keep the
+            // literal in sync with CapabilitiesServiceV2.GetCapabilitiesAsync on the Tentacle side.
+            return capabilities.SupportedCapabilities.Contains("AbandonScriptAsync");
+        }
     }
 }

source/Octopus.Tentacle.Client/ITentacleClient.cs

@@ -8,6 +8,7 @@
 using Octopus.Tentacle.Client.Scripts.Models;
 using Octopus.Tentacle.Contracts;
 using Octopus.Tentacle.Contracts.Logging;
+using Octopus.Tentacle.Contracts.ScriptServiceV2;
 
 namespace Octopus.Tentacle.Client
 {
@@ -31,7 +32,8 @@ Task<ScriptExecutionResult> ExecuteScript(
             OnScriptStatusResponseReceived onScriptStatusResponseReceived,
             OnScriptCompleted onScriptCompleted,
             ITentacleClientTaskLog logger,
-            CancellationToken scriptExecutionCancellationToken);
+            CancellationToken scriptExecutionCancellationToken,
+            TimeSpan? abandonAfterCancellationPendingFor = null);
 
         /// <summary>
         /// Start the script.
@@ -59,6 +61,17 @@ Task<ScriptOperationExecutionResult> StartScript(ExecuteScriptCommand command,
         /// <returns>The result, which includes the CommandContext for the next command</returns>
         Task<ScriptOperationExecutionResult> CancelScript(CommandContext commandContext, ITentacleClientTaskLog logger);
 
+        /// <summary>
+        /// Abandon a running script. Signals Tentacle to release the script's isolation mutex
+        /// and clean up its workspace without waiting for the underlying process to exit.
+        /// Used as an escape hatch when CancelScript cannot terminate a stuck process.
+        /// </summary>
+        /// <param name="scriptTicket">The ticket of the script to abandon</param>
+        /// <param name="logger">Used to output user orientated log messages</param>
+        /// <param name="cancellationToken">Cancels the RPC call</param>
+        /// <returns>The current status snapshot of the script at the time abandon was processed</returns>
+        Task<ScriptStatusResponseV2> AbandonScript(ScriptTicket scriptTicket, ITentacleClientTaskLog logger, CancellationToken cancellationToken);
+
         /// <summary>
         /// Complete the script.
         /// </summary>

source/Octopus.Tentacle.Client/ScriptExecutor.cs

@@ -69,7 +69,15 @@ public async Task<ScriptOperationExecutionResult> CancelScript(CommandContext co
 
             return await scriptExecutor.CancelScript(commandContext);
         }
-
+
+        public async Task<ScriptOperationExecutionResult> AbandonScript(CommandContext commandContext)
+        {
+            var scriptExecutorFactory = CreateScriptExecutorFactory();
+            var scriptExecutor = scriptExecutorFactory.CreateScriptExecutor(commandContext.ScripServiceVersionUsed);
+
+            return await scriptExecutor.AbandonScript(commandContext);
+        }
+
         public async Task<ScriptStatus?> CompleteScript(CommandContext commandContext, CancellationToken cancellationToken)
         {
             var scriptExecutorFactory = CreateScriptExecutorFactory();

source/Octopus.Tentacle.Client/Scripts/IScriptExecutor.cs

@@ -32,6 +32,14 @@ Task<ScriptOperationExecutionResult> StartScript(ExecuteScriptCommand command,
         /// <returns>The result, which includes the CommandContext for the next command</returns>
         Task<ScriptOperationExecutionResult> CancelScript(CommandContext commandContext);
 
+        /// <summary>
+        /// Abandon the script: signal Tentacle to stop waiting and release the isolation mutex.
+        /// Returns the abandoned status when the Tentacle advertises the abandon capability;
+        /// otherwise falls back to cancelling and returns that result.
+        /// </summary>
+        /// <param name="commandContext">The CommandContext from the previous command</param>
+        Task<ScriptOperationExecutionResult> AbandonScript(CommandContext commandContext);
+
         /// <summary>
         /// Complete the script.
         /// </summary>

source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1Executor.cs

@@ -184,6 +184,12 @@ async Task<KubernetesScriptStatusResponseV1> CancelScriptAction(CancellationToke
             return Map(kubernetesScriptStatusResponseV1);
         }
 
+        public Task<ScriptOperationExecutionResult> AbandonScript(CommandContext commandContext)
+        {
+            // KubernetesScriptServiceV1 has no abandon verb; degrade to cancel.
+            return CancelScript(commandContext);
+        }
+
         public async Task<ScriptStatus?> CompleteScript(CommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken)
         {
             using var activity = TentacleClient.ActivitySource.StartActivity($"{nameof(KubernetesScriptServiceV1Executor)}.{nameof(CompleteScript)}");