From 4eb180bdfaed67fcbdb67c01b5c48d7f5fc807e2 Mon Sep 17 00:00:00 2001 From: David Justo Date: Thu, 3 Aug 2023 10:54:02 -0700 Subject: [PATCH 1/7] use grpc Url instead of baseurl --- src/AzureFunctions.PowerShell.Durable.SDK.psm1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/AzureFunctions.PowerShell.Durable.SDK.psm1 b/src/AzureFunctions.PowerShell.Durable.SDK.psm1 index bb0df9a..24ced9f 100644 --- a/src/AzureFunctions.PowerShell.Durable.SDK.psm1 +++ b/src/AzureFunctions.PowerShell.Durable.SDK.psm1 @@ -49,7 +49,7 @@ function Get-DurableStatus { $DurableClient = GetDurableClientFromModulePrivateData } - $requestUrl = "$($DurableClient.BaseUrl)/instances/$InstanceId" + $requestUrl = "$($DurableClient.rpcBaseUrl)/instances/$InstanceId" $query = @() if ($ShowHistory.IsPresent) { @@ -159,7 +159,7 @@ function Stop-DurableOrchestration { $DurableClient = GetDurableClientFromModulePrivateData } - $requestUrl = "$($DurableClient.BaseUrl)/instances/$InstanceId/terminate?reason=$([System.Web.HttpUtility]::UrlEncode($Reason))" + $requestUrl = "$($DurableClient.rpcBaseUrl)/instances/$InstanceId/terminate?reason=$([System.Web.HttpUtility]::UrlEncode($Reason))" Invoke-RestMethod -Uri $requestUrl -Method 'POST' } @@ -291,7 +291,7 @@ function GetRaiseEventUrl( [string] $TaskHubName, [string] $ConnectionName) { - $RequestUrl = $DurableClient.BaseUrl + "/instances/$InstanceId/raiseEvent/$EventName" + $RequestUrl = $DurableClient.rpcBaseUrl + "/instances/$InstanceId/raiseEvent/$EventName" $query = @() if ($null -eq $TaskHubName) { From a9cfcd64ea853e551189a8d6eb6f0517af38ea77 Mon Sep 17 00:00:00 2001 From: David Justo Date: Fri, 8 Sep 2023 09:25:29 -0700 Subject: [PATCH 2/7] Prevent races between orchestration-invoker and user code (#26) --- src/DurableEngine/SharedMemory.cs | 10 ++++++++++ src/DurableEngine/Tasks/DurableTask.cs | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/src/DurableEngine/SharedMemory.cs b/src/DurableEngine/SharedMemory.cs index 66093d8..20d02a1 100644 --- a/src/DurableEngine/SharedMemory.cs +++ b/src/DurableEngine/SharedMemory.cs @@ -65,5 +65,15 @@ public bool YieldToUserCodeThread(WaitHandle completionHandle) var shouldStop = index == 0; return shouldStop; } + + /// + /// Blocks user code thread if the orchestrator-invoker thread is currently running. + /// This guarantees that the user-code thread and the orchestration-invoker thread run one + /// at a time after this point. + /// + public void GuaranteeUserCodeTurn() + { + userCodeThreadTurn.WaitOne(); + } } } \ No newline at end of file diff --git a/src/DurableEngine/Tasks/DurableTask.cs b/src/DurableEngine/Tasks/DurableTask.cs index 1dd4b72..0e1604a 100644 --- a/src/DurableEngine/Tasks/DurableTask.cs +++ b/src/DurableEngine/Tasks/DurableTask.cs @@ -49,6 +49,10 @@ internal OrchestrationAction GetOrCreateAction() /// Function to write an exception to the pipeline. public void Execute(Action write, Action writeErr) { + // Ensure that a DurableTask in the usercode thread + // only executes while the orchestration-invoker thread is blocked. + OrchestrationContext.SharedMemory.GuaranteeUserCodeTurn(); + DurableTask task = this; if (NoWait) From 090cdcfa78da8ffa7e10a86a474ce9ae16bd19bb Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 11 Sep 2023 09:43:10 -0700 Subject: [PATCH 3/7] Remove redundant and error-prone call to `.Reset()` in our AutoResetEvents (#68) --- src/DurableEngine/OrchestrationInvoker.cs | 8 +++++++- src/DurableEngine/SharedMemory.cs | 22 ++++++---------------- src/DurableEngine/Tasks/DurableTask.cs | 4 ---- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/DurableEngine/OrchestrationInvoker.cs b/src/DurableEngine/OrchestrationInvoker.cs index ae58b15..e9841e6 100644 --- a/src/DurableEngine/OrchestrationInvoker.cs +++ b/src/DurableEngine/OrchestrationInvoker.cs @@ -76,7 +76,7 @@ internal Hashtable Invoke(IPowerShellServices powerShellServices) while (true) { // block this thread until user-code thread (the PS orchestrator) invokes a DF CmdLet or completes. - var orchestratorReturned = context.SharedMemory.YieldToUserCodeThread(orchestratorReturnedHandle); + var orchestratorReturned = context.SharedMemory.WaitForInvokerThreadTurn(orchestratorReturnedHandle); if (orchestratorReturned) { // The PS orchestrator has a return value, there's no more DF APIs to await. @@ -109,6 +109,12 @@ internal Hashtable Invoke(IPowerShellServices powerShellServices) await task.GetDTFxTask(); } // Exceptions are ignored at this point, they will be re-surfaced by the PS code if left unhandled. catch { } + + // Wake up user-code thread. For a small moment, both the user code thread and the invoker thread + // will be running at the same time. + // However, the invoker thread will block itself again at the start of the next loop until the user-code + // thread yields control. + context.SharedMemory.WakeUserCodeThread(); } }; diff --git a/src/DurableEngine/SharedMemory.cs b/src/DurableEngine/SharedMemory.cs index 20d02a1..c5012a6 100644 --- a/src/DurableEngine/SharedMemory.cs +++ b/src/DurableEngine/SharedMemory.cs @@ -40,25 +40,16 @@ public void YieldToInvokerThread() { // Wake invoker thread. invokerThreadTurn.Set(); - - // Block user-code thread. - userCodeThreadTurn.Reset(); userCodeThreadTurn.WaitOne(); } /// - /// Blocks Orchestration-invoker thread, wakes up user-code thread. - /// This is usually used after the invoker has a result for the PS orchestrator. + /// Blocks Orchestration-invoker thread until the user-code thread completes or yields. /// /// The WaitHandle tracking if the user-code thread completed. /// True if the user-code thread completed, False if it requests an API to be awaited. - public bool YieldToUserCodeThread(WaitHandle completionHandle) + public bool WaitForInvokerThreadTurn(WaitHandle completionHandle) { - // Wake user-code thread - userCodeThreadTurn.Set(); - - // Get invoker thread ready to block - invokerThreadTurn.Reset(); // Wake up when either the user-code returns, or when we're yielded-to for `await`'ing. var index = WaitHandle.WaitAny(new[] { completionHandle, invokerThreadTurn }); @@ -67,13 +58,12 @@ public bool YieldToUserCodeThread(WaitHandle completionHandle) } /// - /// Blocks user code thread if the orchestrator-invoker thread is currently running. - /// This guarantees that the user-code thread and the orchestration-invoker thread run one - /// at a time after this point. + /// Wakes up the user-code thread without blocking the invoker thread. + /// The invoker thread should block itself afterwards to prevent races. /// - public void GuaranteeUserCodeTurn() + public void WakeUserCodeThread() { - userCodeThreadTurn.WaitOne(); + userCodeThreadTurn.Set(); } } } \ No newline at end of file diff --git a/src/DurableEngine/Tasks/DurableTask.cs b/src/DurableEngine/Tasks/DurableTask.cs index 0e1604a..1dd4b72 100644 --- a/src/DurableEngine/Tasks/DurableTask.cs +++ b/src/DurableEngine/Tasks/DurableTask.cs @@ -49,10 +49,6 @@ internal OrchestrationAction GetOrCreateAction() /// Function to write an exception to the pipeline. public void Execute(Action write, Action writeErr) { - // Ensure that a DurableTask in the usercode thread - // only executes while the orchestration-invoker thread is blocked. - OrchestrationContext.SharedMemory.GuaranteeUserCodeTurn(); - DurableTask task = this; if (NoWait) From 959cf1afea08beebfc65f0761ece404506b20499 Mon Sep 17 00:00:00 2001 From: David Justo Date: Mon, 11 Sep 2023 13:55:36 -0700 Subject: [PATCH 4/7] Increase to version 1.0.2 (#69) As in title --- src/AzureFunctions.PowerShell.Durable.SDK.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AzureFunctions.PowerShell.Durable.SDK.psd1 b/src/AzureFunctions.PowerShell.Durable.SDK.psd1 index 66d6607..242356b 100644 --- a/src/AzureFunctions.PowerShell.Durable.SDK.psd1 +++ b/src/AzureFunctions.PowerShell.Durable.SDK.psd1 @@ -1,6 +1,6 @@ @{ # Version number of this module. - ModuleVersion = '1.0.1' + ModuleVersion = '1.0.2' # Supported PSEditions CompatiblePSEditions = @('Core') From 2e15342715f6776b87fff901c0cd2edcd3842453 Mon Sep 17 00:00:00 2001 From: Naiyuan Tian <110135109+nytian@users.noreply.github.com> Date: Mon, 12 Feb 2024 10:21:44 -0800 Subject: [PATCH 5/7] Add suspend-resume support for durable client (#72) As titled. --- ...AzureFunctions.PowerShell.Durable.SDK.psd1 | 6 +- ...AzureFunctions.PowerShell.Durable.SDK.psm1 | 58 +++++++++++++++++++ .../DurableClientTests.cs | 30 ++++++++++ .../DurableClientSuspending/function.json | 25 ++++++++ .../DurableClientSuspending/run.ps1 | 23 ++++++++ 5 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 test/E2E/durableApp/DurableClientSuspending/function.json create mode 100644 test/E2E/durableApp/DurableClientSuspending/run.ps1 diff --git a/src/AzureFunctions.PowerShell.Durable.SDK.psd1 b/src/AzureFunctions.PowerShell.Durable.SDK.psd1 index 242356b..b1f27ad 100644 --- a/src/AzureFunctions.PowerShell.Durable.SDK.psd1 +++ b/src/AzureFunctions.PowerShell.Durable.SDK.psd1 @@ -37,8 +37,10 @@ 'Get-DurableStatus', 'New-DurableOrchestrationCheckStatusResponse', 'Send-DurableExternalEvent', - 'Start-DurableOrchestration' - 'Stop-DurableOrchestration' + 'Start-DurableOrchestration', + 'Stop-DurableOrchestration', + 'Suspend-DurableOrchestration', + 'Resume-DurableOrchestration' ) # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. diff --git a/src/AzureFunctions.PowerShell.Durable.SDK.psm1 b/src/AzureFunctions.PowerShell.Durable.SDK.psm1 index bb0df9a..82594d1 100644 --- a/src/AzureFunctions.PowerShell.Durable.SDK.psm1 +++ b/src/AzureFunctions.PowerShell.Durable.SDK.psm1 @@ -164,6 +164,64 @@ function Stop-DurableOrchestration { Invoke-RestMethod -Uri $requestUrl -Method 'POST' } +function Suspend-DurableOrchestration { + [CmdletBinding()] + param( + [Parameter( + Mandatory = $true, + Position = 0, + ValueFromPipelineByPropertyName = $true)] + [ValidateNotNullOrEmpty()] + [string] $InstanceId, + + [Parameter( + Mandatory = $true, + Position = 1, + ValueFromPipelineByPropertyName = $true)] + [ValidateNotNullOrEmpty()] + [string] $Reason + ) + + $ErrorActionPreference = 'Stop' + + if ($null -eq $DurableClient) { + $DurableClient = GetDurableClientFromModulePrivateData + } + + $requestUrl = "$($DurableClient.BaseUrl)/instances/$InstanceId/suspend?reason=$([System.Web.HttpUtility]::UrlEncode($Reason))" + + Invoke-RestMethod -Uri $requestUrl -Method 'POST' +} + +function Resume-DurableOrchestration { + [CmdletBinding()] + param( + [Parameter( + Mandatory = $true, + Position = 0, + ValueFromPipelineByPropertyName = $true)] + [ValidateNotNullOrEmpty()] + [string] $InstanceId, + + [Parameter( + Mandatory = $true, + Position = 1, + ValueFromPipelineByPropertyName = $true)] + [ValidateNotNullOrEmpty()] + [string] $Reason + ) + + $ErrorActionPreference = 'Stop' + + if ($null -eq $DurableClient) { + $DurableClient = GetDurableClientFromModulePrivateData + } + + $requestUrl = "$($DurableClient.BaseUrl)/instances/$InstanceId/resume?reason=$([System.Web.HttpUtility]::UrlEncode($Reason))" + + Invoke-RestMethod -Uri $requestUrl -Method 'POST' +} + function IsValidUrl([uri]$Url) { $Url.IsAbsoluteUri -and ($Url.Scheme -in 'http', 'https') } diff --git a/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs b/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs index f51a73d..8c23ccc 100644 --- a/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs +++ b/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs @@ -227,5 +227,35 @@ await ValidateDurableWorkflowResults( Assert.Equal("Terminated intentionally", (string)finalStatusResponseBody.output); }); } + + [Fact] + public async Task DurableClientSuspendOrchestration() + { + var initialResponse = await Utilities.GetHttpStartResponse( + orchestratorName: "SendDurableExternalEventOrchestrator", + clientRoute: "suspendingOrchestrators"); + Assert.Equal(HttpStatusCode.Accepted, initialResponse.StatusCode); + + await ValidateDurableWorkflowResults( + initialResponse, + validateIntermediateResponse: (dynamic intermediateStatusResponseBody) => + { + Assert.Equal("Suspended", (string)intermediateStatusResponseBody.runtimeStatus); + Assert.Equal("Suspend orchestrator", (string)intermediateStatusResponseBody.output); + }); + + await ValidateDurableWorkflowResults( + initialResponse, + validateIntermediateResponse: (dynamic intermediateStatusResponseBody) => + { + Assert.Equal("Running", (string)intermediateStatusResponseBody.runtimeStatus); + }, + validateFinalResponse: (dynamic finalStatusResponseBody) => + { + Assert.Equal("Completed", (string)finalStatusResponseBody.runtimeStatus); + Assert.Equal("FirstTimeout", finalStatusResponseBody.output[0].ToString()); + Assert.Equal("SecondExternalEvent", finalStatusResponseBody.output[1].ToString()); + }); + } } } \ No newline at end of file diff --git a/test/E2E/durableApp/DurableClientSuspending/function.json b/test/E2E/durableApp/DurableClientSuspending/function.json new file mode 100644 index 0000000..59667af --- /dev/null +++ b/test/E2E/durableApp/DurableClientSuspending/function.json @@ -0,0 +1,25 @@ +{ + "bindings": [ + { + "authLevel": "function", + "name": "Request", + "type": "httpTrigger", + "direction": "in", + "route": "suspendingOrchestrators/{FunctionName}", + "methods": [ + "post", + "get" + ] + }, + { + "type": "http", + "direction": "out", + "name": "Response" + }, + { + "name": "starter", + "type": "durableClient", + "direction": "in" + } + ] +} diff --git a/test/E2E/durableApp/DurableClientSuspending/run.ps1 b/test/E2E/durableApp/DurableClientSuspending/run.ps1 new file mode 100644 index 0000000..2036eb4 --- /dev/null +++ b/test/E2E/durableApp/DurableClientSuspending/run.ps1 @@ -0,0 +1,23 @@ +param($Request, $TriggerMetadata) +$ErrorActionPreference = 'Stop' + +Write-Host "DurableClientSuspending started" + +$OrchestratorInputs = @{ FirstDuration = 5; SecondDuration = 60 } + +$FunctionName = $Request.Params.FunctionName +$InstanceId = Start-DurableOrchestration -FunctionName $FunctionName -InputObject $OrchestratorInputs +Write-Host "Started orchestration with ID = '$InstanceId'" + +Start-Sleep -Seconds 5 +Suspend-DurableOrchestration -InstanceId $InstanceId -Reason 'Suspend orchestrator' + +$SuspendResponse = New-DurableOrchestrationCheckStatusResponse -Request $Request -InstanceId $InstanceId +Push-OutputBinding -Name Response -Value $SuspendResponse + +Start-Sleep -Seconds 10 +Resume-DurableOrchestration -InstanceId $InstanceId -Reason 'Resume orchestrator' + +Send-DurableExternalEvent -InstanceId $InstanceId -EventName "SecondExternalEvent" + +Write-Host "DurableClientSuspending completed" From 7bacfe54a17325a3fbae3a4cdce97726bfb0d543 Mon Sep 17 00:00:00 2001 From: David Justo Date: Tue, 13 Feb 2024 19:03:14 -0800 Subject: [PATCH 6/7] increase version --- src/AzureFunctions.PowerShell.Durable.SDK.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AzureFunctions.PowerShell.Durable.SDK.psd1 b/src/AzureFunctions.PowerShell.Durable.SDK.psd1 index 66d6607..3152e1b 100644 --- a/src/AzureFunctions.PowerShell.Durable.SDK.psd1 +++ b/src/AzureFunctions.PowerShell.Durable.SDK.psd1 @@ -1,6 +1,6 @@ @{ # Version number of this module. - ModuleVersion = '1.0.1' + ModuleVersion = '1.1.0' # Supported PSEditions CompatiblePSEditions = @('Core') From 4ddb241c15301960288f54234374f83f36a311c3 Mon Sep 17 00:00:00 2001 From: David Justo Date: Tue, 13 Feb 2024 19:08:09 -0800 Subject: [PATCH 7/7] propagate changes to suspend/resume --- src/AzureFunctions.PowerShell.Durable.SDK.psm1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AzureFunctions.PowerShell.Durable.SDK.psm1 b/src/AzureFunctions.PowerShell.Durable.SDK.psm1 index f4fa037..1cffa0c 100644 --- a/src/AzureFunctions.PowerShell.Durable.SDK.psm1 +++ b/src/AzureFunctions.PowerShell.Durable.SDK.psm1 @@ -188,7 +188,7 @@ function Suspend-DurableOrchestration { $DurableClient = GetDurableClientFromModulePrivateData } - $requestUrl = "$($DurableClient.BaseUrl)/instances/$InstanceId/suspend?reason=$([System.Web.HttpUtility]::UrlEncode($Reason))" + $requestUrl = "$($DurableClient.rpcBaseUrl)/instances/$InstanceId/suspend?reason=$([System.Web.HttpUtility]::UrlEncode($Reason))" Invoke-RestMethod -Uri $requestUrl -Method 'POST' } @@ -217,7 +217,7 @@ function Resume-DurableOrchestration { $DurableClient = GetDurableClientFromModulePrivateData } - $requestUrl = "$($DurableClient.BaseUrl)/instances/$InstanceId/resume?reason=$([System.Web.HttpUtility]::UrlEncode($Reason))" + $requestUrl = "$($DurableClient.rpcBaseUrl)/instances/$InstanceId/resume?reason=$([System.Web.HttpUtility]::UrlEncode($Reason))" Invoke-RestMethod -Uri $requestUrl -Method 'POST' }