From 801545f289a07adb63767d3ac16ff68434ee184b Mon Sep 17 00:00:00 2001 From: David Justo Date: Tue, 12 Jul 2022 16:12:17 -0700 Subject: [PATCH] enable external SDK --- azure-pipelines-e2e-integration-tests.yml | 8 +- build.ps1 | 6 - protobuf/CODEOWNERS | 14 ++ protobuf/README.md | 3 +- protobuf/src/proto/FunctionRpc.proto | 87 ++++++--- release_notes.md | 3 - .../DependencySnapshotInstaller.cs | 6 +- src/Durable/OrchestrationInvoker.cs | 78 -------- src/Durable/PowerShellServices.cs | 74 -------- .../Actions/ActionType.cs | 0 .../Actions/CallActivityAction.cs | 0 .../Actions/CallActivityWithRetryAction.cs | 0 .../Actions/CreateDurableTimerAction.cs | 0 .../Actions/ExternalEventAction.cs | 0 .../Actions/OrchestrationAction.cs | 0 .../ActivityFailureException.cs | 0 .../Commands/GetDurableTaskResult.cs | 0 .../Commands/InvokeDurableActivityCommand.cs | 2 - .../Commands/SetDurableCustomStatusCommand.cs | 0 .../SetFunctionInvocationContextCommand.cs | 1 + ...tartDurableExternalEventListenerCommand.cs | 0 .../Commands/StartDurableTimerCommand.cs | 0 .../Commands/StopDurableTimerTaskCommand.cs | 0 .../Commands/WaitDurableTaskCommand.cs | 0 .../CurrentUtcDateTimeUpdater.cs | 0 .../DurableActivityErrorHandler.cs | 0 .../DurableTaskHandler.cs | 36 +++- src/DurableSDK/ExternalInvoker.cs | 26 +++ src/{Durable => DurableSDK}/HistoryEvent.cs | 0 .../HistoryEventType.cs | 0 src/DurableSDK/IExternalInvoker.cs | 16 ++ .../IOrchestrationInvoker.cs | 1 + .../IPowerShellServices.cs | 11 +- .../OrchestrationActionCollector.cs | 1 + .../OrchestrationBindingInfo.cs | 0 .../OrchestrationContext.cs | 0 .../OrchestrationFailureException.cs | 0 src/DurableSDK/OrchestrationInvoker.cs | 123 +++++++++++++ .../OrchestrationMessage.cs | 0 src/DurableSDK/PowerShellExtensions.cs | 69 +++++++ src/DurableSDK/PowerShellServices.cs | 168 ++++++++++++++++++ src/{Durable => DurableSDK}/RetryOptions.cs | 0 src/{Durable => DurableSDK}/RetryProcessor.cs | 0 .../Tasks/ActivityInvocationTask.cs | 26 +-- .../Tasks/DurableTask.cs | 0 .../Tasks/DurableTimerTask.cs | 0 .../Tasks/ExternalEventTask.cs | 0 .../DurableBindings.cs | 2 +- .../DurableController.cs | 50 ++---- .../DurableFunctionInfo.cs | 5 +- .../DurableFunctionInfoFactory.cs | 2 +- .../DurableFunctionType.cs | 2 +- src/FunctionInfo.cs | 2 +- ...ft.Azure.Functions.PowerShellWorker.csproj | 2 +- ...soft.Azure.Functions.PowerShellWorker.psm1 | 2 +- src/PowerShell/PowerShellManager.cs | 53 +++--- src/RequestProcessor.cs | 2 +- src/Utility/TypeExtensions.cs | 2 +- src/Utility/Utils.cs | 1 + src/resources/PowerShellWorkerStrings.resx | 2 +- ...zure.Functions.PowerShellWorker.E2E.csproj | 2 +- test/E2E/TestFunctionApp/profile.ps1 | 22 +++ .../DependencyManagementTests.cs | 8 +- .../Durable/ActivityInvocationTaskTests.cs | 53 ------ test/Unit/Durable/DurableControllerTests.cs | 112 +++++------- .../DurableFunctionInfoFactoryTests.cs | 2 +- .../Unit/Durable/OrchestrationInvokerTests.cs | 8 + ...ure.Functions.PowerShellWorker.Test.csproj | 2 +- .../Unit/PowerShell/PowerShellManagerTests.cs | 1 + tools/helper.psm1 | 24 +-- 70 files changed, 690 insertions(+), 430 deletions(-) create mode 100644 protobuf/CODEOWNERS delete mode 100644 src/Durable/OrchestrationInvoker.cs delete mode 100644 src/Durable/PowerShellServices.cs rename src/{Durable => DurableSDK}/Actions/ActionType.cs (100%) rename src/{Durable => DurableSDK}/Actions/CallActivityAction.cs (100%) rename src/{Durable => DurableSDK}/Actions/CallActivityWithRetryAction.cs (100%) rename src/{Durable => DurableSDK}/Actions/CreateDurableTimerAction.cs (100%) rename src/{Durable => DurableSDK}/Actions/ExternalEventAction.cs (100%) rename src/{Durable => DurableSDK}/Actions/OrchestrationAction.cs (100%) rename src/{Durable => DurableSDK}/ActivityFailureException.cs (100%) rename src/{Durable => DurableSDK}/Commands/GetDurableTaskResult.cs (100%) rename src/{Durable => DurableSDK}/Commands/InvokeDurableActivityCommand.cs (93%) rename src/{Durable => DurableSDK}/Commands/SetDurableCustomStatusCommand.cs (100%) rename src/{Durable => DurableSDK}/Commands/SetFunctionInvocationContextCommand.cs (97%) rename src/{Durable => DurableSDK}/Commands/StartDurableExternalEventListenerCommand.cs (100%) rename src/{Durable => DurableSDK}/Commands/StartDurableTimerCommand.cs (100%) rename src/{Durable => DurableSDK}/Commands/StopDurableTimerTaskCommand.cs (100%) rename src/{Durable => DurableSDK}/Commands/WaitDurableTaskCommand.cs (100%) rename src/{Durable => DurableSDK}/CurrentUtcDateTimeUpdater.cs (100%) rename src/{Durable => DurableSDK}/DurableActivityErrorHandler.cs (100%) rename src/{Durable => DurableSDK}/DurableTaskHandler.cs (88%) create mode 100644 src/DurableSDK/ExternalInvoker.cs rename src/{Durable => DurableSDK}/HistoryEvent.cs (100%) rename src/{Durable => DurableSDK}/HistoryEventType.cs (100%) create mode 100644 src/DurableSDK/IExternalInvoker.cs rename src/{Durable => DurableSDK}/IOrchestrationInvoker.cs (86%) rename src/{Durable => DurableSDK}/IPowerShellServices.cs (64%) rename src/{Durable => DurableSDK}/OrchestrationActionCollector.cs (98%) rename src/{Durable => DurableSDK}/OrchestrationBindingInfo.cs (100%) rename src/{Durable => DurableSDK}/OrchestrationContext.cs (100%) rename src/{Durable => DurableSDK}/OrchestrationFailureException.cs (100%) create mode 100644 src/DurableSDK/OrchestrationInvoker.cs rename src/{Durable => DurableSDK}/OrchestrationMessage.cs (100%) create mode 100644 src/DurableSDK/PowerShellExtensions.cs create mode 100644 src/DurableSDK/PowerShellServices.cs rename src/{Durable => DurableSDK}/RetryOptions.cs (100%) rename src/{Durable => DurableSDK}/RetryProcessor.cs (100%) rename src/{Durable => DurableSDK}/Tasks/ActivityInvocationTask.cs (64%) rename src/{Durable => DurableSDK}/Tasks/DurableTask.cs (100%) rename src/{Durable => DurableSDK}/Tasks/DurableTimerTask.cs (100%) rename src/{Durable => DurableSDK}/Tasks/ExternalEventTask.cs (100%) rename src/{Durable => DurableWorker}/DurableBindings.cs (95%) rename src/{Durable => DurableWorker}/DurableController.cs (64%) rename src/{Durable => DurableWorker}/DurableFunctionInfo.cs (84%) rename src/{Durable => DurableWorker}/DurableFunctionInfoFactory.cs (96%) rename src/{Durable => DurableWorker}/DurableFunctionType.cs (80%) create mode 100644 test/E2E/TestFunctionApp/profile.ps1 diff --git a/azure-pipelines-e2e-integration-tests.yml b/azure-pipelines-e2e-integration-tests.yml index 4df22ec5..65d08001 100644 --- a/azure-pipelines-e2e-integration-tests.yml +++ b/azure-pipelines-e2e-integration-tests.yml @@ -9,12 +9,14 @@ trigger: none strategy: matrix: linux: - imageName: 'ubuntu-latest' + imageName: 'MMSUbuntu20.04TLS' windows: - imageName: 'windows-latest' + imageName: 'MMS2019TLS' pool: - vmImage: $(imageName) + name: '1ES-Hosted-AzFunc' + demands: + - ImageOverride -equals $(imageName) steps: - pwsh: | diff --git a/build.ps1 b/build.ps1 index 3596634e..39fb107f 100644 --- a/build.ps1 +++ b/build.ps1 @@ -168,12 +168,6 @@ if (!$NoBuild.IsPresent) { Get-Item "$PSScriptRoot/src/Modules/PackageManagement/1.1.7.0/fullclr" -ErrorAction SilentlyContinue | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue - # TODO: Remove this once the SDK properly bundles modules - Get-WebFile -Url 'https://raw.githubusercontent.com/PowerShell/PowerShell/master/src/Modules/Windows/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1' ` - -OutFile "$PSScriptRoot/src/Modules/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1" - Get-WebFile -Url 'https://raw.githubusercontent.com/PowerShell/PowerShell/master/src/Modules/Windows/Microsoft.PowerShell.Management/Microsoft.PowerShell.Management.psd1' ` - -OutFile "$PSScriptRoot/src/Modules/Microsoft.PowerShell.Management/Microsoft.PowerShell.Management.psd1" - dotnet publish -c $Configuration "/p:BuildNumber=$BuildNumber" $PSScriptRoot if ($AddSBOM) diff --git a/protobuf/CODEOWNERS b/protobuf/CODEOWNERS new file mode 100644 index 00000000..23847f14 --- /dev/null +++ b/protobuf/CODEOWNERS @@ -0,0 +1,14 @@ +# See https://help.github.com/articles/about-codeowners/ +# for more info about CODEOWNERS file +# +# It uses the same pattern rule for gitignore file +# https://git-scm.com/docs/gitignore#_pattern_format + + + +# AZURE FUNCTIONS TEAM +# For all file changes, github would automatically +# include the following people in the PRs. +# Language owners should get notified of any new changes to the proto file. + +src/proto/FunctionRpc.proto @vrdmr @gavin-aguiar @YunchuWang @surgupta-msft @satvu @ejizba @alrod @anatolib @kaibocai @shreyas-gopalakrishna @amamounelsayed @Francisco-Gamino diff --git a/protobuf/README.md b/protobuf/README.md index 14c406e2..f6c9cc4c 100644 --- a/protobuf/README.md +++ b/protobuf/README.md @@ -42,7 +42,8 @@ From within the Azure Functions language worker repo: - Be sure to include details of the release 2. Create a release version, following semantic versioning guidelines ([semver.org](https://semver.org/)) 3. Tag the version with the pattern: `v..

-protofile` (example: `v1.1.0-protofile`) -3. Merge `dev` to `master` +4. Merge `dev` to `main` +5. Run the release you'd created ## Consuming FunctionRPC.proto *Note: Update versionNumber before running following commands* diff --git a/protobuf/src/proto/FunctionRpc.proto b/protobuf/src/proto/FunctionRpc.proto index 49face20..72f5069a 100644 --- a/protobuf/src/proto/FunctionRpc.proto +++ b/protobuf/src/proto/FunctionRpc.proto @@ -26,21 +26,18 @@ message StreamingMessage { oneof content { // Worker initiates stream - StartStream start_stream = 20; + StartStream start_stream = 20; // Host sends capabilities/init data to worker WorkerInitRequest worker_init_request = 17; // Worker responds after initializing with its capabilities & status WorkerInitResponse worker_init_response = 16; - // Worker periodically sends empty heartbeat message to host - WorkerHeartbeat worker_heartbeat = 15; - // Host sends terminate message to worker. // Worker terminates if it can, otherwise host terminates after a grace period WorkerTerminate worker_terminate = 14; - // Add any worker relevant status to response + // Host periodically sends status request to the worker WorkerStatusRequest worker_status_request = 12; WorkerStatusResponse worker_status_response = 13; @@ -49,25 +46,25 @@ message StreamingMessage { // Worker requests a desired action (restart worker, reload function) WorkerActionResponse worker_action_response = 7; - + // Host sends required metadata to worker to load function FunctionLoadRequest function_load_request = 8; // Worker responds after loading with the load result FunctionLoadResponse function_load_response = 9; - + // Host requests a given invocation InvocationRequest invocation_request = 4; // Worker responds to a given invocation InvocationResponse invocation_response = 5; - // Host sends cancel message to attempt to cancel an invocation. + // Host sends cancel message to attempt to cancel an invocation. // If an invocation is cancelled, host will receive an invocation response with status cancelled. InvocationCancel invocation_cancel = 21; // Worker logs a message back to the host RpcLog rpc_log = 2; - + FunctionEnvironmentReloadRequest function_environment_reload_request = 25; FunctionEnvironmentReloadResponse function_environment_reload_response = 26; @@ -78,14 +75,20 @@ message StreamingMessage { // Worker indexing message types FunctionsMetadataRequest functions_metadata_request = 29; - FunctionMetadataResponses function_metadata_responses = 30; + FunctionMetadataResponse function_metadata_response = 30; + + // Host sends required metadata to worker to load functions + FunctionLoadRequestCollection function_load_request_collection = 31; + + // Host gets the list of function load responses + FunctionLoadResponseCollection function_load_response_collection = 32; } } // Process.Start required info // connection details // protocol type -// protocol version +// protocol version // Worker sends the host information identifying itself message StartStream { @@ -93,7 +96,7 @@ message StartStream { string worker_id = 2; } -// Host requests the worker to initialize itself +// Host requests the worker to initialize itself message WorkerInitRequest { // version of the host sending init request string host_version = 1; @@ -107,6 +110,9 @@ message WorkerInitRequest { // Full path of worker.config.json location string worker_directory = 4; + + // base directory for function app + string function_app_directory = 5; } // Worker responds with the result of initializing itself @@ -141,11 +147,6 @@ message StatusResult { repeated RpcLog logs = 3; } -// TODO: investigate grpc heartbeat - don't limit to grpc implemention - -// Message is empty by design - Will add more fields in future if needed -message WorkerHeartbeat {} - // Warning before killing the process after grace_period // Worker self terminates ..no response on this message WorkerTerminate { @@ -176,12 +177,12 @@ message FileChangeEventRequest { // Indicates whether worker reloaded successfully or needs a restart message WorkerActionResponse { - // indicates whether a restart is needed, or reload succesfully + // indicates whether a restart is needed, or reload successfully enum Action { Restart = 0; Reload = 1; } - + // action for this response Action action = 1; @@ -189,11 +190,12 @@ message WorkerActionResponse { string reason = 2; } -// NOT USED -message WorkerStatusRequest{ +// Used by the host to determine worker health +message WorkerStatusRequest { } -// NOT USED +// Worker responds with status message +// TODO: Add any worker relevant status to response message WorkerStatusResponse { } @@ -220,7 +222,17 @@ message CloseSharedMemoryResourcesResponse { map close_map_results = 1; } -// Host tells the worker to load a Function +// Host tells the worker to load a list of Functions +message FunctionLoadRequestCollection { + repeated FunctionLoadRequest function_load_requests = 1; +} + +// Host gets the list of function load responses +message FunctionLoadResponseCollection { + repeated FunctionLoadResponse function_load_responses = 1; +} + +// Load request of a single Function message FunctionLoadRequest { // unique function identifier (avoid name collisions, facilitate reload case) string function_id = 1; @@ -252,7 +264,7 @@ message RpcFunctionMetadata { // base directory for the Function string directory = 1; - + // Script file specified string script_file = 2; @@ -273,6 +285,12 @@ message RpcFunctionMetadata { // Raw binding info repeated string raw_bindings = 10; + + // unique function identifier (avoid name collisions, facilitate reload case) + string function_id = 13; + + // A flag indicating if managed dependency is enabled or not + bool managed_dependency_enabled = 14; } // Host tells worker it is ready to receive metadata @@ -282,12 +300,15 @@ message FunctionsMetadataRequest { } // Worker sends function metadata back to host -message FunctionMetadataResponses { +message FunctionMetadataResponse { // list of function indexing responses - repeated FunctionLoadRequest function_load_requests_results = 1; + repeated RpcFunctionMetadata function_metadata_results = 1; // status of overall metadata request StatusResult result = 2; + + // if set to true then host will perform indexing + bool use_default_metadata_indexing = 3; } // Host requests worker to invoke a Function @@ -464,7 +485,7 @@ message BindingInfo { DataType data_type = 4; } -// Used to send logs back to the Host +// Used to send logs back to the Host message RpcLog { // Matching ILogger semantics // https://github.com/aspnet/Logging/blob/9506ccc3f3491488fe88010ef8b9eb64594abf95/src/Microsoft.Extensions.Logging/Logger.cs @@ -515,7 +536,7 @@ message RpcLog { map propertiesMap = 9; } -// Encapsulates an Exception +// Encapsulates an Exception message RpcException { // Source of the exception string source = 3; @@ -525,6 +546,14 @@ message RpcException { // Textual message describing the exception string message = 2; + + // Worker specifies whether exception is a user exception, + // for purpose of application insights logging. Defaults to false. + optional bool is_user_exception = 4; + + // Type of exception. If it's a user exception, the type is passed along to app insights. + // Otherwise, it's ignored for now. + optional string type = 5; } // Http cookie type. Note that only name and value are used for Http requests @@ -569,7 +598,7 @@ message RpcHttpCookie { // TODO - solidify this or remove it message RpcHttp { string method = 1; - string url = 2; + string url = 2; map headers = 3; TypedData body = 4; map params = 10; diff --git a/release_notes.md b/release_notes.md index 24d68654..e69de29b 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,3 +0,0 @@ -* Bug fix: [Context.InstanceId can now be accessed](https://github.com/Azure/azure-functions-powershell-worker/issues/727) -* Bug fix: [Data in External Events is now read and returned to orchestrator](https://github.com/Azure/azure-functions-powershell-worker/issues/68) -* New feature (external contribution): [Get-TaskResult can now be used to obtain the result of an already-completed Durable Functions Task](https://github.com/Azure/azure-functions-powershell-worker/pull/786) \ No newline at end of file diff --git a/src/DependencyManagement/DependencySnapshotInstaller.cs b/src/DependencyManagement/DependencySnapshotInstaller.cs index 5e687770..a6cb4e5b 100644 --- a/src/DependencyManagement/DependencySnapshotInstaller.cs +++ b/src/DependencyManagement/DependencySnapshotInstaller.cs @@ -9,6 +9,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement using System.Collections.Generic; using System.Management.Automation; using System.Threading; + using System.Diagnostics; using Microsoft.Azure.Functions.PowerShellWorker.Utility; @@ -129,9 +130,12 @@ private void InstallModule(DependencyInfo module, string installingPath, PowerSh { try { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + _moduleProvider.SaveModule(pwsh, module.Name, module.ExactVersion, installingPath); - var message = string.Format(PowerShellWorkerStrings.ModuleHasBeenInstalled, module.Name, module.ExactVersion); + var message = string.Format(PowerShellWorkerStrings.ModuleHasBeenInstalled, module.Name, module.ExactVersion, stopwatch.ElapsedMilliseconds); logger.Log(isUserOnlyLog: false, LogLevel.Trace, message); break; diff --git a/src/Durable/OrchestrationInvoker.cs b/src/Durable/OrchestrationInvoker.cs deleted file mode 100644 index fef557ba..00000000 --- a/src/Durable/OrchestrationInvoker.cs +++ /dev/null @@ -1,78 +0,0 @@ -// -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. -// - -namespace Microsoft.Azure.Functions.PowerShellWorker.Durable -{ - using System; - using System.Collections; - using System.Collections.Generic; - using System.Linq; - using System.Management.Automation; - - using PowerShellWorker.Utility; - using Microsoft.Azure.Functions.PowerShellWorker.Durable.Actions; - - internal class OrchestrationInvoker : IOrchestrationInvoker - { - public Hashtable Invoke(OrchestrationBindingInfo orchestrationBindingInfo, IPowerShellServices pwsh) - { - try - { - var outputBuffer = new PSDataCollection(); - var context = orchestrationBindingInfo.Context; - - // context.History should never be null when initializing CurrentUtcDateTime - var orchestrationStart = context.History.First( - e => e.EventType == HistoryEventType.OrchestratorStarted); - context.CurrentUtcDateTime = orchestrationStart.Timestamp.ToUniversalTime(); - - // Marks the first OrchestratorStarted event as processed - orchestrationStart.IsProcessed = true; - - var asyncResult = pwsh.BeginInvoke(outputBuffer); - - var (shouldStop, actions) = - orchestrationBindingInfo.Context.OrchestrationActionCollector.WaitForActions(asyncResult.AsyncWaitHandle); - - if (shouldStop) - { - // The orchestration function should be stopped and restarted - pwsh.StopInvoke(); - return CreateOrchestrationResult(isDone: false, actions, output: null, context.CustomStatus); - } - else - { - try - { - // The orchestration function completed - pwsh.EndInvoke(asyncResult); - var result = FunctionReturnValueBuilder.CreateReturnValueFromFunctionOutput(outputBuffer); - return CreateOrchestrationResult(isDone: true, actions, output: result, context.CustomStatus); - } - catch (Exception e) - { - // The orchestrator code has thrown an unhandled exception: - // this should be treated as an entire orchestration failure - throw new OrchestrationFailureException(actions, context.CustomStatus, e); - } - } - } - finally - { - pwsh.ClearStreamsAndCommands(); - } - } - - private static Hashtable CreateOrchestrationResult( - bool isDone, - List> actions, - object output, - object customStatus) - { - var orchestrationMessage = new OrchestrationMessage(isDone, actions, output, customStatus); - return new Hashtable { { AzFunctionInfo.DollarReturn, orchestrationMessage } }; - } - } -} diff --git a/src/Durable/PowerShellServices.cs b/src/Durable/PowerShellServices.cs deleted file mode 100644 index 0efb681d..00000000 --- a/src/Durable/PowerShellServices.cs +++ /dev/null @@ -1,74 +0,0 @@ -// -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. -// - -namespace Microsoft.Azure.Functions.PowerShellWorker.Durable -{ - using System; - using System.Management.Automation; - using PowerShell; - - internal class PowerShellServices : IPowerShellServices - { - private const string SetFunctionInvocationContextCommand = - "Microsoft.Azure.Functions.PowerShellWorker\\Set-FunctionInvocationContext"; - - private readonly PowerShell _pwsh; - private bool _hasSetOrchestrationContext = false; - - public PowerShellServices(PowerShell pwsh) - { - _pwsh = pwsh; - } - - public void SetDurableClient(object durableClient) - { - _pwsh.AddCommand(SetFunctionInvocationContextCommand) - .AddParameter("DurableClient", durableClient) - .InvokeAndClearCommands(); - - _hasSetOrchestrationContext = true; - } - - public void SetOrchestrationContext(OrchestrationContext orchestrationContext) - { - _pwsh.AddCommand(SetFunctionInvocationContextCommand) - .AddParameter("OrchestrationContext", orchestrationContext) - .InvokeAndClearCommands(); - - _hasSetOrchestrationContext = true; - } - - public void ClearOrchestrationContext() - { - if (_hasSetOrchestrationContext) - { - _pwsh.AddCommand(SetFunctionInvocationContextCommand) - .AddParameter("Clear", true) - .InvokeAndClearCommands(); - } - } - - public IAsyncResult BeginInvoke(PSDataCollection output) - { - return _pwsh.BeginInvoke(input: null, output); - } - - public void EndInvoke(IAsyncResult asyncResult) - { - _pwsh.EndInvoke(asyncResult); - } - - public void StopInvoke() - { - _pwsh.Stop(); - } - - public void ClearStreamsAndCommands() - { - _pwsh.Streams.ClearStreams(); - _pwsh.Commands.Clear(); - } - } -} diff --git a/src/Durable/Actions/ActionType.cs b/src/DurableSDK/Actions/ActionType.cs similarity index 100% rename from src/Durable/Actions/ActionType.cs rename to src/DurableSDK/Actions/ActionType.cs diff --git a/src/Durable/Actions/CallActivityAction.cs b/src/DurableSDK/Actions/CallActivityAction.cs similarity index 100% rename from src/Durable/Actions/CallActivityAction.cs rename to src/DurableSDK/Actions/CallActivityAction.cs diff --git a/src/Durable/Actions/CallActivityWithRetryAction.cs b/src/DurableSDK/Actions/CallActivityWithRetryAction.cs similarity index 100% rename from src/Durable/Actions/CallActivityWithRetryAction.cs rename to src/DurableSDK/Actions/CallActivityWithRetryAction.cs diff --git a/src/Durable/Actions/CreateDurableTimerAction.cs b/src/DurableSDK/Actions/CreateDurableTimerAction.cs similarity index 100% rename from src/Durable/Actions/CreateDurableTimerAction.cs rename to src/DurableSDK/Actions/CreateDurableTimerAction.cs diff --git a/src/Durable/Actions/ExternalEventAction.cs b/src/DurableSDK/Actions/ExternalEventAction.cs similarity index 100% rename from src/Durable/Actions/ExternalEventAction.cs rename to src/DurableSDK/Actions/ExternalEventAction.cs diff --git a/src/Durable/Actions/OrchestrationAction.cs b/src/DurableSDK/Actions/OrchestrationAction.cs similarity index 100% rename from src/Durable/Actions/OrchestrationAction.cs rename to src/DurableSDK/Actions/OrchestrationAction.cs diff --git a/src/Durable/ActivityFailureException.cs b/src/DurableSDK/ActivityFailureException.cs similarity index 100% rename from src/Durable/ActivityFailureException.cs rename to src/DurableSDK/ActivityFailureException.cs diff --git a/src/Durable/Commands/GetDurableTaskResult.cs b/src/DurableSDK/Commands/GetDurableTaskResult.cs similarity index 100% rename from src/Durable/Commands/GetDurableTaskResult.cs rename to src/DurableSDK/Commands/GetDurableTaskResult.cs diff --git a/src/Durable/Commands/InvokeDurableActivityCommand.cs b/src/DurableSDK/Commands/InvokeDurableActivityCommand.cs similarity index 93% rename from src/Durable/Commands/InvokeDurableActivityCommand.cs rename to src/DurableSDK/Commands/InvokeDurableActivityCommand.cs index ba9b429f..4cb8639e 100644 --- a/src/Durable/Commands/InvokeDurableActivityCommand.cs +++ b/src/DurableSDK/Commands/InvokeDurableActivityCommand.cs @@ -43,10 +43,8 @@ protected override void EndProcessing() { var privateData = (Hashtable)MyInvocation.MyCommand.Module.PrivateData; var context = (OrchestrationContext)privateData[SetFunctionInvocationContextCommand.ContextKey]; - var loadedFunctions = FunctionLoader.GetLoadedFunctions(); var task = new ActivityInvocationTask(FunctionName, Input, RetryOptions); - ActivityInvocationTask.ValidateTask(task, loadedFunctions); _durableTaskHandler.StopAndInitiateDurableTaskOrReplay( task, context, NoWait.IsPresent, diff --git a/src/Durable/Commands/SetDurableCustomStatusCommand.cs b/src/DurableSDK/Commands/SetDurableCustomStatusCommand.cs similarity index 100% rename from src/Durable/Commands/SetDurableCustomStatusCommand.cs rename to src/DurableSDK/Commands/SetDurableCustomStatusCommand.cs diff --git a/src/Durable/Commands/SetFunctionInvocationContextCommand.cs b/src/DurableSDK/Commands/SetFunctionInvocationContextCommand.cs similarity index 97% rename from src/Durable/Commands/SetFunctionInvocationContextCommand.cs rename to src/DurableSDK/Commands/SetFunctionInvocationContextCommand.cs index 943e8362..3430be16 100644 --- a/src/Durable/Commands/SetFunctionInvocationContextCommand.cs +++ b/src/DurableSDK/Commands/SetFunctionInvocationContextCommand.cs @@ -9,6 +9,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable.Commands { using System.Collections; using System.Management.Automation; + using Microsoft.PowerShell.Commands; /// /// Set the orchestration context. diff --git a/src/Durable/Commands/StartDurableExternalEventListenerCommand.cs b/src/DurableSDK/Commands/StartDurableExternalEventListenerCommand.cs similarity index 100% rename from src/Durable/Commands/StartDurableExternalEventListenerCommand.cs rename to src/DurableSDK/Commands/StartDurableExternalEventListenerCommand.cs diff --git a/src/Durable/Commands/StartDurableTimerCommand.cs b/src/DurableSDK/Commands/StartDurableTimerCommand.cs similarity index 100% rename from src/Durable/Commands/StartDurableTimerCommand.cs rename to src/DurableSDK/Commands/StartDurableTimerCommand.cs diff --git a/src/Durable/Commands/StopDurableTimerTaskCommand.cs b/src/DurableSDK/Commands/StopDurableTimerTaskCommand.cs similarity index 100% rename from src/Durable/Commands/StopDurableTimerTaskCommand.cs rename to src/DurableSDK/Commands/StopDurableTimerTaskCommand.cs diff --git a/src/Durable/Commands/WaitDurableTaskCommand.cs b/src/DurableSDK/Commands/WaitDurableTaskCommand.cs similarity index 100% rename from src/Durable/Commands/WaitDurableTaskCommand.cs rename to src/DurableSDK/Commands/WaitDurableTaskCommand.cs diff --git a/src/Durable/CurrentUtcDateTimeUpdater.cs b/src/DurableSDK/CurrentUtcDateTimeUpdater.cs similarity index 100% rename from src/Durable/CurrentUtcDateTimeUpdater.cs rename to src/DurableSDK/CurrentUtcDateTimeUpdater.cs diff --git a/src/Durable/DurableActivityErrorHandler.cs b/src/DurableSDK/DurableActivityErrorHandler.cs similarity index 100% rename from src/Durable/DurableActivityErrorHandler.cs rename to src/DurableSDK/DurableActivityErrorHandler.cs diff --git a/src/Durable/DurableTaskHandler.cs b/src/DurableSDK/DurableTaskHandler.cs similarity index 88% rename from src/Durable/DurableTaskHandler.cs rename to src/DurableSDK/DurableTaskHandler.cs index c65b1275..a35546b8 100644 --- a/src/Durable/DurableTaskHandler.cs +++ b/src/DurableSDK/DurableTaskHandler.cs @@ -6,10 +6,12 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable { using System; + using System.Collections; using System.Collections.Generic; + using System.Management.Automation; using System.Threading; using Microsoft.Azure.Functions.PowerShellWorker.Durable.Tasks; - using Utility; + using Microsoft.PowerShell.Commands; internal class DurableTaskHandler { @@ -83,7 +85,7 @@ public void StopAndInitiateDurableTaskOrReplay( retryOptions.MaxNumberOfAttempts, onSuccess: result => { - output(TypeExtensions.ConvertFromJson(result)); + output(ConvertFromJson(result)); }, onFailure); @@ -232,15 +234,41 @@ private static object GetEventResult(HistoryEvent historyEvent) if (historyEvent.EventType == HistoryEventType.TaskCompleted) { - return TypeExtensions.ConvertFromJson(historyEvent.Result); + return ConvertFromJson(historyEvent.Result); } else if (historyEvent.EventType == HistoryEventType.EventRaised) { - return TypeExtensions.ConvertFromJson(historyEvent.Input); + return ConvertFromJson(historyEvent.Input); } return null; } + public static object ConvertFromJson(string json) + { + object retObj = JsonObject.ConvertFromJson(json, returnHashtable: true, error: out _); + + if (retObj is PSObject psObj) + { + retObj = psObj.BaseObject; + } + + if (retObj is Hashtable hashtable) + { + try + { + // ConvertFromJson returns case-sensitive Hashtable by design -- JSON may contain keys that only differ in case. + // We try casting the Hashtable to a case-insensitive one, but if that fails, we keep using the original one. + retObj = new Hashtable(hashtable, StringComparer.OrdinalIgnoreCase); + } + catch + { + retObj = hashtable; + } + } + + return retObj; + } + private void InitiateAndWaitForStop(OrchestrationContext context) { context.OrchestrationActionCollector.Stop(); diff --git a/src/DurableSDK/ExternalInvoker.cs b/src/DurableSDK/ExternalInvoker.cs new file mode 100644 index 00000000..fa8a31c6 --- /dev/null +++ b/src/DurableSDK/ExternalInvoker.cs @@ -0,0 +1,26 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +{ + using System; + using System.Collections; + using System.Management.Automation; + + internal class ExternalInvoker : IExternalInvoker + { + private readonly Func _externalSDKInvokerFunction; + + public ExternalInvoker(Func invokerFunction) + { + _externalSDKInvokerFunction = invokerFunction; + } + + public Hashtable Invoke(IPowerShellServices powerShellServices) + { + return (Hashtable)_externalSDKInvokerFunction.Invoke(powerShellServices.GetPowerShell()); + } + } +} diff --git a/src/Durable/HistoryEvent.cs b/src/DurableSDK/HistoryEvent.cs similarity index 100% rename from src/Durable/HistoryEvent.cs rename to src/DurableSDK/HistoryEvent.cs diff --git a/src/Durable/HistoryEventType.cs b/src/DurableSDK/HistoryEventType.cs similarity index 100% rename from src/Durable/HistoryEventType.cs rename to src/DurableSDK/HistoryEventType.cs diff --git a/src/DurableSDK/IExternalInvoker.cs b/src/DurableSDK/IExternalInvoker.cs new file mode 100644 index 00000000..3a703f3d --- /dev/null +++ b/src/DurableSDK/IExternalInvoker.cs @@ -0,0 +1,16 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +{ + using System.Collections; + + // Represents a contract for the + internal interface IExternalInvoker + { + // Method to invoke an orchestration using the external Durable SDK + Hashtable Invoke(IPowerShellServices powerShellServices); + } +} diff --git a/src/Durable/IOrchestrationInvoker.cs b/src/DurableSDK/IOrchestrationInvoker.cs similarity index 86% rename from src/Durable/IOrchestrationInvoker.cs rename to src/DurableSDK/IOrchestrationInvoker.cs index 7e80aba3..36011b56 100644 --- a/src/Durable/IOrchestrationInvoker.cs +++ b/src/DurableSDK/IOrchestrationInvoker.cs @@ -10,5 +10,6 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable internal interface IOrchestrationInvoker { Hashtable Invoke(OrchestrationBindingInfo orchestrationBindingInfo, IPowerShellServices pwsh); + void SetExternalInvoker(IExternalInvoker externalInvoker); } } diff --git a/src/Durable/IPowerShellServices.cs b/src/DurableSDK/IPowerShellServices.cs similarity index 64% rename from src/Durable/IPowerShellServices.cs rename to src/DurableSDK/IPowerShellServices.cs index a8cf897b..cdd850bc 100644 --- a/src/Durable/IPowerShellServices.cs +++ b/src/DurableSDK/IPowerShellServices.cs @@ -5,17 +5,26 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable { + using Microsoft.Azure.WebJobs.Script.Grpc.Messages; using System; using System.Management.Automation; internal interface IPowerShellServices { + PowerShell GetPowerShell(); + + bool UseExternalDurableSDK(); + void SetDurableClient(object durableClient); - void SetOrchestrationContext(OrchestrationContext orchestrationContext); + OrchestrationBindingInfo SetOrchestrationContext(ParameterBinding context, out IExternalInvoker externalInvoker); void ClearOrchestrationContext(); + void TracePipelineObject(); + + void AddParameter(string name, object value); + IAsyncResult BeginInvoke(PSDataCollection output); void EndInvoke(IAsyncResult asyncResult); diff --git a/src/Durable/OrchestrationActionCollector.cs b/src/DurableSDK/OrchestrationActionCollector.cs similarity index 98% rename from src/Durable/OrchestrationActionCollector.cs rename to src/DurableSDK/OrchestrationActionCollector.cs index b62fbc4b..1542c2bf 100644 --- a/src/Durable/OrchestrationActionCollector.cs +++ b/src/DurableSDK/OrchestrationActionCollector.cs @@ -11,6 +11,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable using System.Threading; using Microsoft.Azure.Functions.PowerShellWorker.Durable.Actions; + using Newtonsoft.Json; internal class OrchestrationActionCollector { diff --git a/src/Durable/OrchestrationBindingInfo.cs b/src/DurableSDK/OrchestrationBindingInfo.cs similarity index 100% rename from src/Durable/OrchestrationBindingInfo.cs rename to src/DurableSDK/OrchestrationBindingInfo.cs diff --git a/src/Durable/OrchestrationContext.cs b/src/DurableSDK/OrchestrationContext.cs similarity index 100% rename from src/Durable/OrchestrationContext.cs rename to src/DurableSDK/OrchestrationContext.cs diff --git a/src/Durable/OrchestrationFailureException.cs b/src/DurableSDK/OrchestrationFailureException.cs similarity index 100% rename from src/Durable/OrchestrationFailureException.cs rename to src/DurableSDK/OrchestrationFailureException.cs diff --git a/src/DurableSDK/OrchestrationInvoker.cs b/src/DurableSDK/OrchestrationInvoker.cs new file mode 100644 index 00000000..b71116b4 --- /dev/null +++ b/src/DurableSDK/OrchestrationInvoker.cs @@ -0,0 +1,123 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + using System.Management.Automation; + + using Microsoft.Azure.Functions.PowerShellWorker.Durable.Actions; + + internal class OrchestrationInvoker : IOrchestrationInvoker + { + private IExternalInvoker _externalInvoker; + internal static string isOrchestrationFailureKey = "IsOrchestrationFailure"; + + public Hashtable Invoke( + OrchestrationBindingInfo orchestrationBindingInfo, + IPowerShellServices powerShellServices) + { + try + { + if (powerShellServices.UseExternalDurableSDK()) + { + return InvokeExternalDurableSDK(powerShellServices); + } + return InvokeInternalDurableSDK(orchestrationBindingInfo, powerShellServices); + } + catch (Exception ex) + { + ex.Data.Add(isOrchestrationFailureKey, true); + throw; + } + finally + { + powerShellServices.ClearStreamsAndCommands(); + } + } + + public Hashtable InvokeExternalDurableSDK(IPowerShellServices powerShellServices) + { + return _externalInvoker.Invoke(powerShellServices); + } + + public Hashtable InvokeInternalDurableSDK( + OrchestrationBindingInfo orchestrationBindingInfo, + IPowerShellServices powerShellServices) + { + var outputBuffer = new PSDataCollection(); + var context = orchestrationBindingInfo.Context; + + // context.History should never be null when initializing CurrentUtcDateTime + var orchestrationStart = context.History.First( + e => e.EventType == HistoryEventType.OrchestratorStarted); + context.CurrentUtcDateTime = orchestrationStart.Timestamp.ToUniversalTime(); + + // Marks the first OrchestratorStarted event as processed + orchestrationStart.IsProcessed = true; + + // Finish initializing the Function invocation + powerShellServices.AddParameter(orchestrationBindingInfo.ParameterName, context); + powerShellServices.TracePipelineObject(); + + var asyncResult = powerShellServices.BeginInvoke(outputBuffer); + + var (shouldStop, actions) = + orchestrationBindingInfo.Context.OrchestrationActionCollector.WaitForActions(asyncResult.AsyncWaitHandle); + + if (shouldStop) + { + // The orchestration function should be stopped and restarted + powerShellServices.StopInvoke(); + // return (Hashtable)orchestrationBindingInfo.Context.OrchestrationActionCollector.output; + return CreateOrchestrationResult(isDone: false, actions, output: null, context.CustomStatus); + } + else + { + try + { + // The orchestration function completed + powerShellServices.EndInvoke(asyncResult); + var result = CreateReturnValueFromFunctionOutput(outputBuffer); + return CreateOrchestrationResult(isDone: true, actions, output: result, context.CustomStatus); + } + catch (Exception e) + { + // The orchestrator code has thrown an unhandled exception: + // this should be treated as an entire orchestration failure + throw new OrchestrationFailureException(actions, context.CustomStatus, e); + } + } + } + + public static object CreateReturnValueFromFunctionOutput(IList pipelineItems) + { + if (pipelineItems == null || pipelineItems.Count <= 0) + { + return null; + } + + return pipelineItems.Count == 1 ? pipelineItems[0] : pipelineItems.ToArray(); + } + + private static Hashtable CreateOrchestrationResult( + bool isDone, + List> actions, + object output, + object customStatus) + { + var orchestrationMessage = new OrchestrationMessage(isDone, actions, output, customStatus); + return new Hashtable { { "$return", orchestrationMessage } }; + } + + public void SetExternalInvoker(IExternalInvoker externalInvoker) + { + _externalInvoker = externalInvoker; + } + } +} diff --git a/src/Durable/OrchestrationMessage.cs b/src/DurableSDK/OrchestrationMessage.cs similarity index 100% rename from src/Durable/OrchestrationMessage.cs rename to src/DurableSDK/OrchestrationMessage.cs diff --git a/src/DurableSDK/PowerShellExtensions.cs b/src/DurableSDK/PowerShellExtensions.cs new file mode 100644 index 00000000..a5225188 --- /dev/null +++ b/src/DurableSDK/PowerShellExtensions.cs @@ -0,0 +1,69 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System.Collections; +using System.Collections.ObjectModel; + +namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +{ + using System.Management.Automation; + + internal static class PowerShellExtensions + { + public static void InvokeAndClearCommands(this PowerShell pwsh) + { + try + { + pwsh.Invoke(); + } + finally + { + pwsh.Streams.ClearStreams(); + pwsh.Commands.Clear(); + } + } + + public static void InvokeAndClearCommands(this PowerShell pwsh, IEnumerable input) + { + try + { + pwsh.Invoke(input); + } + finally + { + pwsh.Streams.ClearStreams(); + pwsh.Commands.Clear(); + } + } + + public static Collection InvokeAndClearCommands(this PowerShell pwsh) + { + try + { + var result = pwsh.Invoke(); + return result; + } + finally + { + pwsh.Streams.ClearStreams(); + pwsh.Commands.Clear(); + } + } + + public static Collection InvokeAndClearCommands(this PowerShell pwsh, IEnumerable input) + { + try + { + var result = pwsh.Invoke(input); + return result; + } + finally + { + pwsh.Streams.ClearStreams(); + pwsh.Commands.Clear(); + } + } + } +} diff --git a/src/DurableSDK/PowerShellServices.cs b/src/DurableSDK/PowerShellServices.cs new file mode 100644 index 00000000..fda8d80d --- /dev/null +++ b/src/DurableSDK/PowerShellServices.cs @@ -0,0 +1,168 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +{ + using System; + using System.Collections.ObjectModel; + using System.Linq; + using System.Management.Automation; + using Microsoft.Azure.Functions.PowerShellWorker.Utility; + using Microsoft.Azure.WebJobs.Script.Grpc.Messages; + using Newtonsoft.Json; + + internal class PowerShellServices : IPowerShellServices + { + private readonly string SetFunctionInvocationContextCommand; + private const string ExternalDurableSDKName = "DurableSDK"; + private const string InternalDurableSDKName = "Microsoft.Azure.Functions.PowerShellWorker"; + + private readonly PowerShell _pwsh; + private bool _hasInitializedDurableFunction = false; + private readonly bool _useExternalDurableSDK = false; + + public PowerShellServices(PowerShell pwsh) + { + //This logic will be commented out until the external SDK is published on the PS Gallery + + // We attempt to import the external SDK upon construction of the PowerShellServices object. + // We maintain the boolean member _useExternalDurableSDK in this object rather than + // DurableController because the expected input and functionality of SetFunctionInvocationContextCommand + // may differ between the internal and external implementations. + + try + { + pwsh.AddCommand(Utils.ImportModuleCmdletInfo) + .AddParameter("Name", ExternalDurableSDKName) + .AddParameter("ErrorAction", ActionPreference.Stop) + .InvokeAndClearCommands(); + _useExternalDurableSDK = true; + } + catch (Exception e) + { + // Check to see if ExternalDurableSDK is among the modules imported or + // available to be imported: if it is, then something went wrong with + // the Import-Module statement and we should throw an Exception. + // Otherwise, we use the InternalDurableSDK + var availableModules = pwsh.AddCommand(Utils.GetModuleCmdletInfo) + .AddParameter("Name", ExternalDurableSDKName) + .InvokeAndClearCommands(); + if (availableModules.Count() > 0) + { + // TODO: evaluate if there is a better error message or exception type to be throwing. + // Ideally, this should never happen. + throw new InvalidOperationException("The external Durable SDK was detected, but unable to be imported.", e); + } + _useExternalDurableSDK = false; + } + //_useExternalDurableSDK = false; + + if (_useExternalDurableSDK) + { + SetFunctionInvocationContextCommand = $"{ExternalDurableSDKName}\\Set-FunctionInvocationContext"; + } + else + { + SetFunctionInvocationContextCommand = $"{InternalDurableSDKName}\\Set-FunctionInvocationContext"; + } + _pwsh = pwsh; + } + + public bool UseExternalDurableSDK() + { + return _useExternalDurableSDK; + } + + public PowerShell GetPowerShell() + { + return this._pwsh; + } + + public void SetDurableClient(object durableClient) + { + _pwsh.AddCommand(SetFunctionInvocationContextCommand) + .AddParameter("DurableClient", durableClient) + .InvokeAndClearCommands(); + _hasInitializedDurableFunction = true; + } + + public OrchestrationBindingInfo SetOrchestrationContext( + ParameterBinding context, + out IExternalInvoker externalInvoker) + { + externalInvoker = null; + OrchestrationBindingInfo orchestrationBindingInfo = new OrchestrationBindingInfo( + context.Name, + JsonConvert.DeserializeObject(context.Data.String)); + + if (_useExternalDurableSDK) + { + Collection> output = _pwsh.AddCommand(SetFunctionInvocationContextCommand) + // The external SetFunctionInvocationContextCommand expects a .json string to deserialize + // and writes an invoker function to the output pipeline. + .AddParameter("OrchestrationContext", context.Data.String) + .InvokeAndClearCommands>(); + if (output.Count() == 1) + { + externalInvoker = new ExternalInvoker(output[0]); + } + else + { + throw new InvalidOperationException($"Only a single output was expected for an invocation of {SetFunctionInvocationContextCommand}"); + } + } + else + { + _pwsh.AddCommand(SetFunctionInvocationContextCommand) + .AddParameter("OrchestrationContext", orchestrationBindingInfo.Context) + .InvokeAndClearCommands(); + } + _hasInitializedDurableFunction = true; + return orchestrationBindingInfo; + } + + + public void AddParameter(string name, object value) + { + _pwsh.AddParameter(name, value); + } + + public void ClearOrchestrationContext() + { + if (_hasInitializedDurableFunction) + { + _pwsh.AddCommand(SetFunctionInvocationContextCommand) + .AddParameter("Clear", true) + .InvokeAndClearCommands(); + } + } + + public void TracePipelineObject() + { + _pwsh.AddCommand("Microsoft.Azure.Functions.PowerShellWorker\\Trace-PipelineObject"); + } + + public IAsyncResult BeginInvoke(PSDataCollection output) + { + return _pwsh.BeginInvoke(input: null, output); + } + + public void EndInvoke(IAsyncResult asyncResult) + { + _pwsh.EndInvoke(asyncResult); + } + + public void StopInvoke() + { + _pwsh.Stop(); + } + + public void ClearStreamsAndCommands() + { + _pwsh.Streams.ClearStreams(); + _pwsh.Commands.Clear(); + } + } +} diff --git a/src/Durable/RetryOptions.cs b/src/DurableSDK/RetryOptions.cs similarity index 100% rename from src/Durable/RetryOptions.cs rename to src/DurableSDK/RetryOptions.cs diff --git a/src/Durable/RetryProcessor.cs b/src/DurableSDK/RetryProcessor.cs similarity index 100% rename from src/Durable/RetryProcessor.cs rename to src/DurableSDK/RetryProcessor.cs diff --git a/src/Durable/Tasks/ActivityInvocationTask.cs b/src/DurableSDK/Tasks/ActivityInvocationTask.cs similarity index 64% rename from src/Durable/Tasks/ActivityInvocationTask.cs rename to src/DurableSDK/Tasks/ActivityInvocationTask.cs index a0ba49b4..5f9f6a33 100644 --- a/src/Durable/Tasks/ActivityInvocationTask.cs +++ b/src/DurableSDK/Tasks/ActivityInvocationTask.cs @@ -7,15 +7,10 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable.Tasks { - using System; using System.Linq; - using System.Collections.Generic; - - using WebJobs.Script.Grpc.Messages; - - using Microsoft.Azure.Functions.PowerShellWorker; using Microsoft.Azure.Functions.PowerShellWorker.Durable; using Microsoft.Azure.Functions.PowerShellWorker.Durable.Actions; + using Microsoft.Azure.Functions.PowerShellWorker.DurableWorker; public class ActivityInvocationTask : DurableTask { @@ -61,24 +56,5 @@ internal override OrchestrationAction CreateOrchestrationAction() ? new CallActivityAction(FunctionName, Input) : new CallActivityWithRetryAction(FunctionName, Input, RetryOptions); } - - internal static void ValidateTask(ActivityInvocationTask task, IEnumerable loadedFunctions) - { - var functionInfo = loadedFunctions.FirstOrDefault(fi => fi.FuncName == task.FunctionName); - if (functionInfo == null) - { - var message = string.Format(PowerShellWorkerStrings.FunctionNotFound, task.FunctionName); - throw new InvalidOperationException(message); - } - - var activityTriggerBinding = functionInfo.InputBindings.FirstOrDefault( - entry => DurableBindings.IsActivityTrigger(entry.Value.Type) - && entry.Value.Direction == BindingInfo.Types.Direction.In); - if (activityTriggerBinding.Key == null) - { - var message = string.Format(PowerShellWorkerStrings.FunctionDoesNotHaveProperActivityFunctionBinding, task.FunctionName); - throw new InvalidOperationException(message); - } - } } } diff --git a/src/Durable/Tasks/DurableTask.cs b/src/DurableSDK/Tasks/DurableTask.cs similarity index 100% rename from src/Durable/Tasks/DurableTask.cs rename to src/DurableSDK/Tasks/DurableTask.cs diff --git a/src/Durable/Tasks/DurableTimerTask.cs b/src/DurableSDK/Tasks/DurableTimerTask.cs similarity index 100% rename from src/Durable/Tasks/DurableTimerTask.cs rename to src/DurableSDK/Tasks/DurableTimerTask.cs diff --git a/src/Durable/Tasks/ExternalEventTask.cs b/src/DurableSDK/Tasks/ExternalEventTask.cs similarity index 100% rename from src/Durable/Tasks/ExternalEventTask.cs rename to src/DurableSDK/Tasks/ExternalEventTask.cs diff --git a/src/Durable/DurableBindings.cs b/src/DurableWorker/DurableBindings.cs similarity index 95% rename from src/Durable/DurableBindings.cs rename to src/DurableWorker/DurableBindings.cs index 6bdf2468..d46a4adc 100644 --- a/src/Durable/DurableBindings.cs +++ b/src/DurableWorker/DurableBindings.cs @@ -3,7 +3,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // -namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +namespace Microsoft.Azure.Functions.PowerShellWorker.DurableWorker { using System; diff --git a/src/Durable/DurableController.cs b/src/DurableWorker/DurableController.cs similarity index 64% rename from src/Durable/DurableController.cs rename to src/DurableWorker/DurableController.cs index 6b1f31bb..3d756e15 100644 --- a/src/Durable/DurableController.cs +++ b/src/DurableWorker/DurableController.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable { - using System; using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; @@ -16,6 +15,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable using WebJobs.Script.Grpc.Messages; using PowerShellWorker.Utility; + using Microsoft.Azure.Functions.PowerShellWorker.DurableWorker; /// /// The main entry point for durable functions support. @@ -47,9 +47,14 @@ internal DurableController( _orchestrationInvoker = orchestrationInvoker; } - public void BeforeFunctionInvocation(IList inputData) + public string GetOrchestrationParameterName() { - // If the function is an orchestration client, then we set the DurableClient + return _orchestrationBindingInfo?.ParameterName; + } + + public void InitializeBindings(IList inputData) + { + // If the function is an durable client, then we set the DurableClient // in the module context for the 'Start-DurableOrchestration' function to use. if (_durableFunctionInfo.IsDurableClient) { @@ -58,11 +63,14 @@ public void BeforeFunctionInvocation(IList inputData) .Data.ToObject(); _powerShellServices.SetDurableClient(durableClient); + } else if (_durableFunctionInfo.IsOrchestrationFunction) { - _orchestrationBindingInfo = CreateOrchestrationBindingInfo(inputData); - _powerShellServices.SetOrchestrationContext(_orchestrationBindingInfo.Context); + _orchestrationBindingInfo = _powerShellServices.SetOrchestrationContext( + inputData[0], + out IExternalInvoker externalInvoker); + _orchestrationInvoker.SetExternalInvoker(externalInvoker); } } @@ -87,46 +95,22 @@ public bool TryGetInputBindingParameterValue(string bindingName, out object valu public void AddPipelineOutputIfNecessary(Collection pipelineItems, Hashtable result) { - var shouldAddPipelineOutput = - _durableFunctionInfo.Type == DurableFunctionType.ActivityFunction; - - if (shouldAddPipelineOutput) + + if (ShouldSuppressPipelineTraces()) { var returnValue = FunctionReturnValueBuilder.CreateReturnValueFromFunctionOutput(pipelineItems); result.Add(AzFunctionInfo.DollarReturn, returnValue); } } - public bool TryInvokeOrchestrationFunction(out Hashtable result) + public Hashtable InvokeOrchestrationFunction() { - if (!_durableFunctionInfo.IsOrchestrationFunction) - { - result = null; - return false; - } - - result = _orchestrationInvoker.Invoke(_orchestrationBindingInfo, _powerShellServices); - return true; + return _orchestrationInvoker.Invoke(_orchestrationBindingInfo, _powerShellServices); } public bool ShouldSuppressPipelineTraces() { return _durableFunctionInfo.Type == DurableFunctionType.ActivityFunction; } - - private static OrchestrationBindingInfo CreateOrchestrationBindingInfo(IList inputData) - { - // Quote from https://docs.microsoft.com/en-us/azure/azure-functions/durable-functions-bindings: - // - // "Orchestrator functions should never use any input or output bindings other than the orchestration trigger binding. - // Doing so has the potential to cause problems with the Durable Task extension because those bindings may not obey the single-threading and I/O rules." - // - // Therefore, it's by design that input data contains only one item, which is the metadata of the orchestration context. - var context = inputData[0]; - - return new OrchestrationBindingInfo( - context.Name, - JsonConvert.DeserializeObject(context.Data.String)); - } } } diff --git a/src/Durable/DurableFunctionInfo.cs b/src/DurableWorker/DurableFunctionInfo.cs similarity index 84% rename from src/Durable/DurableFunctionInfo.cs rename to src/DurableWorker/DurableFunctionInfo.cs index a245ac67..05c3650d 100644 --- a/src/Durable/DurableFunctionInfo.cs +++ b/src/DurableWorker/DurableFunctionInfo.cs @@ -3,7 +3,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // -namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +namespace Microsoft.Azure.Functions.PowerShellWorker.DurableWorker { internal class DurableFunctionInfo { @@ -13,9 +13,12 @@ public DurableFunctionInfo(DurableFunctionType type, string durableClientBinding DurableClientBindingName = durableClientBindingName; } + public bool IsActivityFunction => Type == DurableFunctionType.ActivityFunction; + public bool IsDurableClient => DurableClientBindingName != null; public bool IsOrchestrationFunction => Type == DurableFunctionType.OrchestrationFunction; + public string DurableClientBindingName { get; } diff --git a/src/Durable/DurableFunctionInfoFactory.cs b/src/DurableWorker/DurableFunctionInfoFactory.cs similarity index 96% rename from src/Durable/DurableFunctionInfoFactory.cs rename to src/DurableWorker/DurableFunctionInfoFactory.cs index 3e65b9f7..0cc08d27 100644 --- a/src/Durable/DurableFunctionInfoFactory.cs +++ b/src/DurableWorker/DurableFunctionInfoFactory.cs @@ -3,7 +3,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // -namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +namespace Microsoft.Azure.Functions.PowerShellWorker.DurableWorker { using System.Linq; diff --git a/src/Durable/DurableFunctionType.cs b/src/DurableWorker/DurableFunctionType.cs similarity index 80% rename from src/Durable/DurableFunctionType.cs rename to src/DurableWorker/DurableFunctionType.cs index 73feb504..2ecacc17 100644 --- a/src/Durable/DurableFunctionType.cs +++ b/src/DurableWorker/DurableFunctionType.cs @@ -3,7 +3,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // -namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +namespace Microsoft.Azure.Functions.PowerShellWorker.DurableWorker { internal enum DurableFunctionType { diff --git a/src/FunctionInfo.cs b/src/FunctionInfo.cs index 2aea62ec..091eecde 100644 --- a/src/FunctionInfo.cs +++ b/src/FunctionInfo.cs @@ -15,7 +15,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker { - using Durable; + using DurableWorker; /// /// This type represents the metadata of an Azure PowerShell Function. diff --git a/src/Microsoft.Azure.Functions.PowerShellWorker.csproj b/src/Microsoft.Azure.Functions.PowerShellWorker.csproj index 885ac6ec..fdfbc2c4 100644 --- a/src/Microsoft.Azure.Functions.PowerShellWorker.csproj +++ b/src/Microsoft.Azure.Functions.PowerShellWorker.csproj @@ -21,7 +21,7 @@ Licensed under the MIT license. See LICENSE file in the project root for full li - + diff --git a/src/Modules/Microsoft.Azure.Functions.PowerShellWorker/Microsoft.Azure.Functions.PowerShellWorker.psm1 b/src/Modules/Microsoft.Azure.Functions.PowerShellWorker/Microsoft.Azure.Functions.PowerShellWorker.psm1 index c87f08b6..be368a48 100644 --- a/src/Modules/Microsoft.Azure.Functions.PowerShellWorker/Microsoft.Azure.Functions.PowerShellWorker.psm1 +++ b/src/Modules/Microsoft.Azure.Functions.PowerShellWorker/Microsoft.Azure.Functions.PowerShellWorker.psm1 @@ -11,7 +11,7 @@ Set-Alias -Name Start-NewOrchestration -Value Start-DurableOrchestration function GetDurableClientFromModulePrivateData { $PrivateData = $PSCmdlet.MyInvocation.MyCommand.Module.PrivateData - if ($PrivateData -eq $null -or $PrivateData['DurableClient'] -eq $null) { + if ($null -eq $PrivateData -or $null -eq $PrivateData['DurableClient']) { throw "No binding of the type 'durableClient' was defined." } else { diff --git a/src/PowerShell/PowerShellManager.cs b/src/PowerShell/PowerShellManager.cs index 7e458914..0344c05d 100644 --- a/src/PowerShell/PowerShellManager.cs +++ b/src/PowerShell/PowerShellManager.cs @@ -15,6 +15,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.PowerShell { + using Microsoft.Azure.Functions.PowerShellWorker.DurableWorker; using System.Management.Automation; using System.Text; @@ -204,32 +205,37 @@ public Hashtable InvokeFunction( FunctionInvocationPerformanceStopwatch stopwatch) { var outputBindings = FunctionMetadata.GetOutputBindingHashtable(_pwsh.Runspace.InstanceId); - - var durableController = new DurableController(functionInfo.DurableFunctionInfo, _pwsh); + var durableFunctionsUtils = new DurableController(functionInfo.DurableFunctionInfo, _pwsh); try { - durableController.BeforeFunctionInvocation(inputData); + durableFunctionsUtils.InitializeBindings(inputData); AddEntryPointInvocationCommand(functionInfo); stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.FunctionCodeReady); - SetInputBindingParameterValues(functionInfo, inputData, durableController, triggerMetadata, traceContext, retryContext); + var orchestrationParamName = durableFunctionsUtils.GetOrchestrationParameterName(); + SetInputBindingParameterValues(functionInfo, inputData, orchestrationParamName, triggerMetadata, traceContext, retryContext); stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InputBindingValuesReady); - if (!durableController.ShouldSuppressPipelineTraces()) - { - _pwsh.AddCommand("Microsoft.Azure.Functions.PowerShellWorker\\Trace-PipelineObject"); - } - stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InvokingFunctionCode); Logger.Log(isUserOnlyLog: false, LogLevel.Trace, CreateInvocationPerformanceReportMessage(functionInfo.FuncName, stopwatch)); try { - return durableController.TryInvokeOrchestrationFunction(out var result) - ? result - : InvokeNonOrchestrationFunction(durableController, outputBindings); + if(functionInfo.DurableFunctionInfo.IsOrchestrationFunction) + { + return durableFunctionsUtils.InvokeOrchestrationFunction(); + } + else + { + var isActivityFunction = functionInfo.DurableFunctionInfo.IsActivityFunction; + if (!isActivityFunction) + { + _pwsh.AddCommand("Microsoft.Azure.Functions.PowerShellWorker\\Trace-PipelineObject"); + } + return ExecuteUserCode(isActivityFunction, outputBindings); + } } catch (RuntimeException e) { @@ -237,9 +243,9 @@ public Hashtable InvokeFunction( Logger.Log(isUserOnlyLog: true, LogLevel.Error, GetFunctionExceptionMessage(e)); throw; } - catch (OrchestrationFailureException e) + catch (Exception e) { - if (e.InnerException is IContainsErrorRecord inner) + if (e.Data.Contains(OrchestrationInvoker.isOrchestrationFailureKey) && e.InnerException is IContainsErrorRecord inner) { Logger.Log(isUserOnlyLog: true, LogLevel.Error, GetFunctionExceptionMessage(inner)); } @@ -248,7 +254,7 @@ public Hashtable InvokeFunction( } finally { - durableController.AfterFunctionInvocation(); + durableFunctionsUtils.AfterFunctionInvocation(); outputBindings.Clear(); ResetRunspace(); } @@ -257,7 +263,7 @@ public Hashtable InvokeFunction( private void SetInputBindingParameterValues( AzFunctionInfo functionInfo, IEnumerable inputData, - DurableController durableController, + string orchParamName, Hashtable triggerMetadata, TraceContext traceContext, RetryContext retryContext) @@ -266,13 +272,12 @@ private void SetInputBindingParameterValues( { if (functionInfo.FuncParameters.TryGetValue(binding.Name, out var paramInfo)) { - if (!durableController.TryGetInputBindingParameterValue(binding.Name, out var valueToUse)) + if (string.CompareOrdinal(binding.Name, orchParamName) != 0) { var bindingInfo = functionInfo.InputBindings[binding.Name]; - valueToUse = Utils.TransformInBindingValueAsNeeded(paramInfo, bindingInfo, binding.Data.ToObject()); + var valueToUse = Utils.TransformInBindingValueAsNeeded(paramInfo, bindingInfo, binding.Data.ToObject()); + _pwsh.AddParameter(binding.Name, valueToUse); } - - _pwsh.AddParameter(binding.Name, valueToUse); } } @@ -296,11 +301,15 @@ private void SetInputBindingParameterValues( /// /// Execution a function fired by a trigger or an activity function scheduled by an orchestration. /// - private Hashtable InvokeNonOrchestrationFunction(DurableController durableController, IDictionary outputBindings) + private Hashtable ExecuteUserCode(bool addPipelineOutput, IDictionary outputBindings) { var pipelineItems = _pwsh.InvokeAndClearCommands(); var result = new Hashtable(outputBindings, StringComparer.OrdinalIgnoreCase); - durableController.AddPipelineOutputIfNecessary(pipelineItems, result); + if (addPipelineOutput) + { + var returnValue = FunctionReturnValueBuilder.CreateReturnValueFromFunctionOutput(pipelineItems); + result.Add(AzFunctionInfo.DollarReturn, returnValue); + } return result; } diff --git a/src/RequestProcessor.cs b/src/RequestProcessor.cs index 89b4c7e9..c9c10d2f 100644 --- a/src/RequestProcessor.cs +++ b/src/RequestProcessor.cs @@ -13,7 +13,7 @@ using Microsoft.Azure.Functions.PowerShellWorker.PowerShell; using Microsoft.Azure.Functions.PowerShellWorker.Utility; using Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement; -using Microsoft.Azure.Functions.PowerShellWorker.Durable; +using Microsoft.Azure.Functions.PowerShellWorker.DurableWorker; using Microsoft.Azure.WebJobs.Script.Grpc.Messages; namespace Microsoft.Azure.Functions.PowerShellWorker diff --git a/src/Utility/TypeExtensions.cs b/src/Utility/TypeExtensions.cs index 2f3c4186..b3bcb9e3 100644 --- a/src/Utility/TypeExtensions.cs +++ b/src/Utility/TypeExtensions.cs @@ -142,7 +142,7 @@ public static object ConvertFromJson(string json) private static string ConvertToJson(object fromObj) { var context = new JsonObject.ConvertToJsonContext( - maxDepth: 4, + maxDepth: 10, //TODO: fix enumsAsStrings: false, compressOutput: true); diff --git a/src/Utility/Utils.cs b/src/Utility/Utils.cs index 549ba5de..21ec68b8 100644 --- a/src/Utility/Utils.cs +++ b/src/Utility/Utils.cs @@ -17,6 +17,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Utility internal class Utils { + internal readonly static CmdletInfo GetModuleCmdletInfo = new CmdletInfo("Get-Module", typeof(GetModuleCommand)); internal readonly static CmdletInfo ImportModuleCmdletInfo = new CmdletInfo("Import-Module", typeof(ImportModuleCommand)); internal readonly static CmdletInfo RemoveModuleCmdletInfo = new CmdletInfo("Remove-Module", typeof(RemoveModuleCommand)); internal readonly static CmdletInfo RemoveJobCmdletInfo = new CmdletInfo("Remove-Job", typeof(RemoveJobCommand)); diff --git a/src/resources/PowerShellWorkerStrings.resx b/src/resources/PowerShellWorkerStrings.resx index ff966b56..2d0228d7 100644 --- a/src/resources/PowerShellWorkerStrings.resx +++ b/src/resources/PowerShellWorkerStrings.resx @@ -179,7 +179,7 @@ Started installing module '{0}' version '{1}'. - Module name '{0}' version '{1}' has been installed. + Module name '{0}' version '{1}' has been installed. Module installation completed in {2} ms. The function app has existing dependencies installed. Updating the dependencies to the latest versions will be performed in the background. New function app instances will pick up any new dependencies. diff --git a/test/E2E/Azure.Functions.PowerShellWorker.E2E/Azure.Functions.PowerShellWorker.E2E/Azure.Functions.PowerShellWorker.E2E.csproj b/test/E2E/Azure.Functions.PowerShellWorker.E2E/Azure.Functions.PowerShellWorker.E2E/Azure.Functions.PowerShellWorker.E2E.csproj index dbe362e6..bda4034f 100644 --- a/test/E2E/Azure.Functions.PowerShellWorker.E2E/Azure.Functions.PowerShellWorker.E2E/Azure.Functions.PowerShellWorker.E2E.csproj +++ b/test/E2E/Azure.Functions.PowerShellWorker.E2E/Azure.Functions.PowerShellWorker.E2E/Azure.Functions.PowerShellWorker.E2E.csproj @@ -11,7 +11,7 @@ - + diff --git a/test/E2E/TestFunctionApp/profile.ps1 b/test/E2E/TestFunctionApp/profile.ps1 new file mode 100644 index 00000000..9afdf1da --- /dev/null +++ b/test/E2E/TestFunctionApp/profile.ps1 @@ -0,0 +1,22 @@ +# Azure Functions profile.ps1 +# +# This profile.ps1 will get executed every "cold start" of your Function App. +# "cold start" occurs when: +# +# * A Function App starts up for the very first time +# * A Function App starts up after being de-allocated due to inactivity +# +# You can define helper functions, run commands, or specify environment variables +# NOTE: any variables defined that are not environment variables will get reset after the first execution + +# Authenticate with Azure PowerShell using MSI. +# Remove this if you are not planning on using MSI or Azure PowerShell. +if ($env:MSI_SECRET) { + Disable-AzContextAutosave -Scope Process | Out-Null + Connect-AzAccount -Identity +} + +# Uncomment the next line to enable legacy AzureRm alias in Azure PowerShell. +# Enable-AzureRmAlias + +# You can also define functions or aliases that can be referenced in any of your PowerShell functions. \ No newline at end of file diff --git a/test/Unit/DependencyManagement/DependencyManagementTests.cs b/test/Unit/DependencyManagement/DependencyManagementTests.cs index 401334df..28334dfc 100644 --- a/test/Unit/DependencyManagement/DependencyManagementTests.cs +++ b/test/Unit/DependencyManagement/DependencyManagementTests.cs @@ -11,8 +11,10 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Test { + using System.Diagnostics; using System.Linq; using System.Management.Automation; + using System.Threading; public class DependencyManagementTests : IDisposable { @@ -481,8 +483,12 @@ public void SaveModule(PowerShell pwsh, string moduleName, string version, strin { if (SuccessfulDownload || (SaveModuleCount >= ShouldNotThrowAfterCount)) { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + Thread.Sleep(10); // wait for 10 milliseconds + // Save the module name and version for a successful download. - DownloadedModuleInfo = string.Format(PowerShellWorkerStrings.ModuleHasBeenInstalled, moduleName, version); + DownloadedModuleInfo = string.Format(PowerShellWorkerStrings.ModuleHasBeenInstalled, moduleName, version, stopwatch.ElapsedMilliseconds); return; } diff --git a/test/Unit/Durable/ActivityInvocationTaskTests.cs b/test/Unit/Durable/ActivityInvocationTaskTests.cs index 530711ca..96e27571 100644 --- a/test/Unit/Durable/ActivityInvocationTaskTests.cs +++ b/test/Unit/Durable/ActivityInvocationTaskTests.cs @@ -163,59 +163,6 @@ public void StopAndInitiateDurableTaskOrReplay_OutputsActivityInvocationTask_Whe Assert.Equal(FunctionName, allOutput.Single().FunctionName); } - [Fact] - public void ValidateTask_Throws_WhenActivityFunctionDoesNotExist() - { - var history = CreateHistory(scheduled: false, completed: false, failed: false, output: InvocationResultJson); - var orchestrationContext = new OrchestrationContext { History = history }; - - var loadedFunctions = new[] - { - DurableTestUtilities.CreateFakeAzFunctionInfo(FunctionName, "fakeTriggerBindingName", ActivityTriggerBindingType, BindingInfo.Types.Direction.In) - }; - - const string wrongFunctionName = "AnotherFunction"; - - var durableTaskHandler = new DurableTaskHandler(); - - var exception = - Assert.Throws( - () => ActivityInvocationTask.ValidateTask( - new ActivityInvocationTask(wrongFunctionName, FunctionInput), loadedFunctions)); - - Assert.Contains(wrongFunctionName, exception.Message); - Assert.DoesNotContain(ActivityTriggerBindingType, exception.Message); - - DurableTestUtilities.VerifyNoActionAdded(orchestrationContext); - } - - [Theory] - [InlineData("IncorrectBindingType", BindingInfo.Types.Direction.In)] - [InlineData(ActivityTriggerBindingType, BindingInfo.Types.Direction.Out)] - public void ValidateTask_Throws_WhenActivityFunctionHasNoProperBinding( - string bindingType, BindingInfo.Types.Direction bindingDirection) - { - var history = CreateHistory(scheduled: false, completed: false, failed: false, output: InvocationResultJson); - var orchestrationContext = new OrchestrationContext { History = history }; - - var loadedFunctions = new[] - { - DurableTestUtilities.CreateFakeAzFunctionInfo(FunctionName, "fakeTriggerBindingName", bindingType, bindingDirection) - }; - - var durableTaskHandler = new DurableTaskHandler(); - - var exception = - Assert.Throws( - () => ActivityInvocationTask.ValidateTask( - new ActivityInvocationTask(FunctionName, FunctionInput), loadedFunctions)); - - Assert.Contains(FunctionName, exception.Message); - Assert.Contains(ActivityTriggerBindingType, exception.Message); - - DurableTestUtilities.VerifyNoActionAdded(orchestrationContext); - } - [Theory] [InlineData(false)] [InlineData(true)] diff --git a/test/Unit/Durable/DurableControllerTests.cs b/test/Unit/Durable/DurableControllerTests.cs index 68531f7c..b57cb767 100644 --- a/test/Unit/Durable/DurableControllerTests.cs +++ b/test/Unit/Durable/DurableControllerTests.cs @@ -11,6 +11,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Test.Durable using System.Collections.ObjectModel; using Microsoft.Azure.WebJobs.Script.Grpc.Messages; + using Microsoft.Azure.Functions.PowerShellWorker.DurableWorker; using Microsoft.Azure.Functions.PowerShellWorker.Durable; using Microsoft.Azure.Functions.PowerShellWorker.Utility; using Newtonsoft.Json; @@ -22,9 +23,12 @@ public class DurableControllerTests { private readonly Mock _mockPowerShellServices = new Mock(MockBehavior.Strict); private readonly Mock _mockOrchestrationInvoker = new Mock(MockBehavior.Strict); + private const string _contextParameterName = "ParameterName"; + private static readonly OrchestrationContext _orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; + private static readonly OrchestrationBindingInfo _orchestrationBindingInfo = new OrchestrationBindingInfo(_contextParameterName, _orchestrationContext); [Fact] - public void BeforeFunctionInvocation_SetsDurableClient_ForDurableClientFunction() + public void InitializeBindings_SetsDurableClient_ForDurableClientFunction() { var durableController = CreateDurableController(DurableFunctionType.None, "DurableClientBindingName"); @@ -38,7 +42,7 @@ public void BeforeFunctionInvocation_SetsDurableClient_ForDurableClientFunction( _mockPowerShellServices.Setup(_ => _.SetDurableClient(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); + durableController.InitializeBindings(inputData); _mockPowerShellServices.Verify( _ => _.SetDurableClient( @@ -47,50 +51,50 @@ public void BeforeFunctionInvocation_SetsDurableClient_ForDurableClientFunction( } [Fact] - public void BeforeFunctionInvocation_SetsOrchestrationContext_ForOrchestrationFunction() + public void InitializeBindings_SetsOrchestrationContext_ForOrchestrationFunction() { var durableController = CreateDurableController(DurableFunctionType.OrchestrationFunction); - - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; var inputData = new[] { - CreateParameterBinding("ParameterName", orchestrationContext) + CreateParameterBinding("ParameterName", _orchestrationContext) }; + + _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny(), + out It.Ref.IsAny)) + .Returns(_orchestrationBindingInfo); + _mockOrchestrationInvoker.Setup(_ => _.SetExternalInvoker(It.IsAny())); - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - - durableController.BeforeFunctionInvocation(inputData); + durableController.InitializeBindings(inputData); _mockPowerShellServices.Verify( _ => _.SetOrchestrationContext( - It.Is(c => c.InstanceId == orchestrationContext.InstanceId)), + It.Is(c => c.Data.ToString().Contains(_orchestrationContext.InstanceId)), + out It.Ref.IsAny), Times.Once); } [Fact] - public void BeforeFunctionInvocation_Throws_OnOrchestrationFunctionWithoutContextParameter() + public void InitializeBindings_Throws_OnOrchestrationFunctionWithoutContextParameter() { var durableController = CreateDurableController(DurableFunctionType.OrchestrationFunction); var inputData = new ParameterBinding[0]; - Assert.ThrowsAny(() => durableController.BeforeFunctionInvocation(inputData)); + Assert.ThrowsAny(() => durableController.InitializeBindings(inputData)); } [Theory] [InlineData(DurableFunctionType.None)] [InlineData(DurableFunctionType.ActivityFunction)] - internal void BeforeFunctionInvocation_DoesNothing_ForNonOrchestrationFunction(DurableFunctionType durableFunctionType) + internal void InitializeBindings_DoesNothing_ForNonOrchestrationFunction(DurableFunctionType durableFunctionType) { var durableController = CreateDurableController(durableFunctionType); - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - var inputData = new[] { // Even if a parameter similar to orchestration context is passed: - CreateParameterBinding("ParameterName", orchestrationContext) + CreateParameterBinding("ParameterName", _orchestrationContext) }; - durableController.BeforeFunctionInvocation(inputData); + durableController.InitializeBindings(inputData); } [Theory] @@ -111,19 +115,19 @@ internal void AfterFunctionInvocation_ClearsOrchestrationContext(DurableFunction public void TryGetInputBindingParameterValue_RetrievesOrchestrationContextParameter_ForOrchestrationFunction() { var durableController = CreateDurableController(DurableFunctionType.OrchestrationFunction); - - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - const string contextParameterName = "ParameterName"; var inputData = new[] { - CreateParameterBinding(contextParameterName, orchestrationContext) + CreateParameterBinding(_contextParameterName, _orchestrationContext) }; - - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); - - Assert.True(durableController.TryGetInputBindingParameterValue(contextParameterName, out var value)); - Assert.Equal(orchestrationContext.InstanceId, ((OrchestrationContext)value).InstanceId); + _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext( + It.IsAny(), + out It.Ref.IsAny)) + .Returns(_orchestrationBindingInfo); + _mockOrchestrationInvoker.Setup(_ => _.SetExternalInvoker(It.IsAny())); + durableController.InitializeBindings(inputData); + + Assert.True(durableController.TryGetInputBindingParameterValue(_contextParameterName, out var value)); + Assert.Equal(_orchestrationContext.InstanceId, ((OrchestrationContext)value).InstanceId); } [Theory] @@ -132,70 +136,52 @@ public void TryGetInputBindingParameterValue_RetrievesOrchestrationContextParame internal void TryGetInputBindingParameterValue_RetrievesNothing_ForNonOrchestrationFunction(DurableFunctionType durableFunctionType) { var durableController = CreateDurableController(durableFunctionType); - - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - const string contextParameterName = "ParameterName"; var inputData = new[] { - CreateParameterBinding(contextParameterName, orchestrationContext) + CreateParameterBinding(_contextParameterName, _orchestrationContext) }; - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); + _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext( + It.IsAny(), + out It.Ref.IsAny)) + .Returns(_orchestrationBindingInfo); + _mockOrchestrationInvoker.Setup(_ => _.SetExternalInvoker(It.IsAny())); + durableController.InitializeBindings(inputData); - Assert.False(durableController.TryGetInputBindingParameterValue(contextParameterName, out var value)); + Assert.False(durableController.TryGetInputBindingParameterValue(_contextParameterName, out var value)); Assert.Null(value); } [Fact] public void TryInvokeOrchestrationFunction_InvokesOrchestrationFunction() { - var contextParameterName = "ParameterName"; - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - var inputData = new[] { CreateParameterBinding(contextParameterName, orchestrationContext) }; - + var inputData = new[] { CreateParameterBinding(_contextParameterName, _orchestrationContext) }; var durableController = CreateDurableController(DurableFunctionType.OrchestrationFunction); - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); + _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext( + It.IsAny(), + out It.Ref.IsAny)) + .Returns(_orchestrationBindingInfo); + _mockOrchestrationInvoker.Setup(_ => _.SetExternalInvoker(It.IsAny())); + durableController.InitializeBindings(inputData); var expectedResult = new Hashtable(); _mockOrchestrationInvoker.Setup( _ => _.Invoke(It.IsAny(), It.IsAny())) .Returns(expectedResult); - var invoked = durableController.TryInvokeOrchestrationFunction(out var actualResult); - Assert.True(invoked); + var actualResult = durableController.InvokeOrchestrationFunction(); Assert.Same(expectedResult, actualResult); _mockOrchestrationInvoker.Verify( _ => _.Invoke( It.Is( - bindingInfo => bindingInfo.Context.InstanceId == orchestrationContext.InstanceId - && bindingInfo.ParameterName == contextParameterName), + bindingInfo => bindingInfo.Context.InstanceId == _orchestrationContext.InstanceId + && bindingInfo.ParameterName == _contextParameterName), _mockPowerShellServices.Object), Times.Once); } - [Theory] - [InlineData(DurableFunctionType.None)] - [InlineData(DurableFunctionType.ActivityFunction)] - internal void TryInvokeOrchestrationFunction_DoesNotInvokeNonOrchestrationFunction(DurableFunctionType durableFunctionType) - { - var contextParameterName = "ParameterName"; - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - var inputData = new[] { CreateParameterBinding(contextParameterName, orchestrationContext) }; - - var durableController = CreateDurableController(durableFunctionType); - - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); - - var invoked = durableController.TryInvokeOrchestrationFunction(out var actualResult); - Assert.False(invoked); - Assert.Null(actualResult); - } - [Fact] public void AddPipelineOutputIfNecessary_AddsDollarReturn_ForActivityFunction() { diff --git a/test/Unit/Durable/DurableFunctionInfoFactoryTests.cs b/test/Unit/Durable/DurableFunctionInfoFactoryTests.cs index 09cea266..2c898316 100644 --- a/test/Unit/Durable/DurableFunctionInfoFactoryTests.cs +++ b/test/Unit/Durable/DurableFunctionInfoFactoryTests.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Test.Durable using Xunit; - using Microsoft.Azure.Functions.PowerShellWorker.Durable; + using Microsoft.Azure.Functions.PowerShellWorker.DurableWorker; public class DurableFunctionInfoFactoryTests { diff --git a/test/Unit/Durable/OrchestrationInvokerTests.cs b/test/Unit/Durable/OrchestrationInvokerTests.cs index 65c4b24d..2a2db694 100644 --- a/test/Unit/Durable/OrchestrationInvokerTests.cs +++ b/test/Unit/Durable/OrchestrationInvokerTests.cs @@ -34,12 +34,16 @@ public void InvocationRunsToCompletionIfNotStopped() { var invocationAsyncResult = DurableTestUtilities.CreateInvocationResult(completed: true); DurableTestUtilities.ExpectBeginInvoke(_mockPowerShellServices, invocationAsyncResult); + _mockPowerShellServices.Setup(_ => _.UseExternalDurableSDK()).Returns(false); _orchestrationInvoker.Invoke(_orchestrationBindingInfo, _mockPowerShellServices.Object); _mockPowerShellServices.Verify(_ => _.BeginInvoke(It.IsAny>()), Times.Once); _mockPowerShellServices.Verify(_ => _.EndInvoke(invocationAsyncResult), Times.Once); _mockPowerShellServices.Verify(_ => _.ClearStreamsAndCommands(), Times.Once); + _mockPowerShellServices.Verify(_ => _.TracePipelineObject(), Times.Once); + _mockPowerShellServices.Verify(_ => _.AddParameter(It.IsAny(), It.IsAny()), Times.Once); + _mockPowerShellServices.Verify(_ => _.UseExternalDurableSDK(), Times.Once); _mockPowerShellServices.VerifyNoOtherCalls(); } @@ -47,10 +51,14 @@ public void InvocationRunsToCompletionIfNotStopped() public void InvocationStopsOnStopEvent() { InvokeOrchestration(completed: false); + _mockPowerShellServices.Setup(_ => _.UseExternalDurableSDK()).Returns(false); _mockPowerShellServices.Verify(_ => _.BeginInvoke(It.IsAny>()), Times.Once); _mockPowerShellServices.Verify(_ => _.StopInvoke(), Times.Once); _mockPowerShellServices.Verify(_ => _.ClearStreamsAndCommands(), Times.Once); + _mockPowerShellServices.Verify(_ => _.TracePipelineObject(), Times.Once); + _mockPowerShellServices.Verify(_ => _.AddParameter(It.IsAny(), It.IsAny()), Times.Once); + _mockPowerShellServices.Verify(_ => _.UseExternalDurableSDK(), Times.Once); _mockPowerShellServices.VerifyNoOtherCalls(); } diff --git a/test/Unit/Microsoft.Azure.Functions.PowerShellWorker.Test.csproj b/test/Unit/Microsoft.Azure.Functions.PowerShellWorker.Test.csproj index db07a51c..ff575f15 100644 --- a/test/Unit/Microsoft.Azure.Functions.PowerShellWorker.Test.csproj +++ b/test/Unit/Microsoft.Azure.Functions.PowerShellWorker.Test.csproj @@ -10,7 +10,7 @@ - + diff --git a/test/Unit/PowerShell/PowerShellManagerTests.cs b/test/Unit/PowerShell/PowerShellManagerTests.cs index 5d4faf42..36588611 100644 --- a/test/Unit/PowerShell/PowerShellManagerTests.cs +++ b/test/Unit/PowerShell/PowerShellManagerTests.cs @@ -18,6 +18,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Test using System.Collections.ObjectModel; using System.Management.Automation; using Microsoft.Azure.Functions.PowerShellWorker.Durable; + using Microsoft.Azure.Functions.PowerShellWorker.DurableWorker; using Newtonsoft.Json; internal class TestUtils diff --git a/tools/helper.psm1 b/tools/helper.psm1 index a0b48aa1..2d967241 100644 --- a/tools/helper.psm1 +++ b/tools/helper.psm1 @@ -12,13 +12,13 @@ $DotnetSDKVersionRequirements = @{ # .NET SDK 3.1 is required by the Microsoft.ManifestTool.dll tool '3.1' = @{ - MinimalPatch = '417' - DefaultPatch = '417' + MinimalPatch = '419' + DefaultPatch = '419' } '6.0' = @{ - MinimalPatch = '201' - DefaultPatch = '201' + MinimalPatch = '300' + DefaultPatch = '300' } } @@ -119,7 +119,8 @@ function Resolve-ProtoBufToolPath $nugetPath = Get-NugetPackagesPath $toolsPath = "$RepoRoot/tools" - if (-not (Test-Path "$toolsPath/obj/project.assets.json")) { + if (-not (Test-Path "$toolsPath/obj/project.assets.json") -or + -not (Test-Path "$nugetPath/grpc.tools/$GrpcToolsVersion")) { dotnet restore $toolsPath --verbosity quiet if ($LASTEXITCODE -ne 0) { throw "Cannot resolve protobuf tools. 'dotnet restore $toolsPath' failed." @@ -164,19 +165,6 @@ function Resolve-ProtoBufToolPath } } -function Get-WebFile { - param ( - [string] $Url, - [string] $OutFile - ) - $directoryName = [System.IO.Path]::GetDirectoryName($OutFile) - if (!(Test-Path $directoryName)) { - New-Item -Type Directory $directoryName - } - Remove-Item $OutFile -ErrorAction SilentlyContinue - Invoke-RestMethod $Url -OutFile $OutFile -} - function Write-Log { param(