[WIP] Prep worker for DF external SDK

azure-pipelines-e2e-integration-tests.yml

@@ -9,12 +9,14 @@ trigger: none
 strategy:
   matrix:
     linux:
-      imageName: 'ubuntu-latest'
+      imageName: 'MMSUbuntu20.04TLS'
     windows:
-      imageName: 'windows-latest'
+      imageName: 'MMS2019TLS'
 
 pool:
-    vmImage: $(imageName)
+    name: '1ES-Hosted-AzFunc'
+    demands:
+      - ImageOverride -equals $(imageName)
 
 steps:
 - pwsh: |

build.ps1

@@ -168,12 +168,6 @@ if (!$NoBuild.IsPresent) {
     Get-Item "$PSScriptRoot/src/Modules/PackageManagement/1.1.7.0/fullclr" -ErrorAction SilentlyContinue |
         Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
 
-    # TODO: Remove this once the SDK properly bundles modules
-    Get-WebFile -Url 'https://raw.githubusercontent.com/PowerShell/PowerShell/master/src/Modules/Windows/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1' `
-        -OutFile "$PSScriptRoot/src/Modules/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1"
-    Get-WebFile -Url 'https://raw.githubusercontent.com/PowerShell/PowerShell/master/src/Modules/Windows/Microsoft.PowerShell.Management/Microsoft.PowerShell.Management.psd1' `
-        -OutFile "$PSScriptRoot/src/Modules/Microsoft.PowerShell.Management/Microsoft.PowerShell.Management.psd1" 
-
     dotnet publish -c $Configuration "/p:BuildNumber=$BuildNumber" $PSScriptRoot
 
     if ($AddSBOM)

protobuf/CODEOWNERS

@@ -0,0 +1,14 @@
+# See https://help.github.com/articles/about-codeowners/
+# for more info about CODEOWNERS file
+#
+# It uses the same pattern rule for gitignore file
+# https://git-scm.com/docs/gitignore#_pattern_format
+
+
+
+# AZURE FUNCTIONS TEAM
+# For all file changes, github would automatically
+# include the following people in the PRs.
+# Language owners should get notified of any new changes to the proto file.
+
+src/proto/FunctionRpc.proto @vrdmr @gavin-aguiar @YunchuWang @surgupta-msft @satvu @ejizba @alrod @anatolib @kaibocai @shreyas-gopalakrishna @amamounelsayed @Francisco-Gamino

protobuf/README.md

@@ -42,7 +42,8 @@ From within the Azure Functions language worker repo:
     -   Be sure to include details of the release
 2.	Create a release version, following semantic versioning guidelines ([semver.org](https://semver.org/))
 3.	Tag the version with the pattern: `v<M>.<m>.<p>-protofile` (example: `v1.1.0-protofile`)
-3.	Merge `dev` to `master`
+4.	Merge `dev` to `main`
+5.	Run the release you'd created
 
 ## Consuming FunctionRPC.proto
 *Note: Update versionNumber before running following commands*

protobuf/src/proto/FunctionRpc.proto

@@ -26,21 +26,18 @@ message StreamingMessage {
   oneof content {
 
     // Worker initiates stream
-    StartStream start_stream = 20; 
+    StartStream start_stream = 20;
 
     // Host sends capabilities/init data to worker
     WorkerInitRequest worker_init_request = 17;
     // Worker responds after initializing with its capabilities & status
     WorkerInitResponse worker_init_response = 16;
 
-    // Worker periodically sends empty heartbeat message to host
-    WorkerHeartbeat worker_heartbeat = 15;
-
     // Host sends terminate message to worker.
     // Worker terminates if it can, otherwise host terminates after a grace period
     WorkerTerminate worker_terminate = 14;
 
-    // Add any worker relevant status to response
+    // Host periodically sends status request to the worker
     WorkerStatusRequest worker_status_request = 12;
     WorkerStatusResponse worker_status_response = 13;
 
@@ -49,25 +46,25 @@ message StreamingMessage {
 
     // Worker requests a desired action (restart worker, reload function)
     WorkerActionResponse worker_action_response = 7;
-    
+
     // Host sends required metadata to worker to load function
     FunctionLoadRequest function_load_request = 8;
     // Worker responds after loading with the load result
     FunctionLoadResponse function_load_response = 9;
-    
+
     // Host requests a given invocation
     InvocationRequest invocation_request = 4;
 
     // Worker responds to a given invocation
     InvocationResponse invocation_response = 5;
 
-    // Host sends cancel message to attempt to cancel an invocation. 
+    // Host sends cancel message to attempt to cancel an invocation.
     // If an invocation is cancelled, host will receive an invocation response with status cancelled.
     InvocationCancel invocation_cancel = 21;
 
     // Worker logs a message back to the host
     RpcLog rpc_log = 2;
-    
+
     FunctionEnvironmentReloadRequest function_environment_reload_request = 25;
 
     FunctionEnvironmentReloadResponse function_environment_reload_response = 26;
@@ -78,22 +75,28 @@ message StreamingMessage {
 
     // Worker indexing message types
     FunctionsMetadataRequest functions_metadata_request = 29;
-    FunctionMetadataResponses function_metadata_responses = 30;
+    FunctionMetadataResponse function_metadata_response = 30;
+
+    // Host sends required metadata to worker to load functions
+    FunctionLoadRequestCollection function_load_request_collection = 31;
+
+    // Host gets the list of function load responses
+    FunctionLoadResponseCollection function_load_response_collection = 32;
   }
 }
 
 // Process.Start required info
 //   connection details
 //   protocol type
-//   protocol version 
+//   protocol version
 
 // Worker sends the host information identifying itself
 message StartStream {
   // id of the worker
   string worker_id = 2;
 }
 
-// Host requests the worker to initialize itself 
+// Host requests the worker to initialize itself
 message WorkerInitRequest {
   // version of the host sending init request
   string host_version = 1;
@@ -107,6 +110,9 @@ message WorkerInitRequest {
 
   // Full path of worker.config.json location
   string worker_directory = 4;
+
+  // base directory for function app
+  string function_app_directory = 5;
 }
 
 // Worker responds with the result of initializing itself
@@ -141,11 +147,6 @@ message StatusResult {
   repeated RpcLog logs = 3;
 }
 
-// TODO: investigate grpc heartbeat - don't limit to grpc implemention
-
-// Message is empty by design - Will add more fields in future if needed
-message WorkerHeartbeat {}
-
 // Warning before killing the process after grace_period
 // Worker self terminates ..no response on this
 message WorkerTerminate {
@@ -176,24 +177,25 @@ message FileChangeEventRequest {
 
 // Indicates whether worker reloaded successfully or needs a restart
 message WorkerActionResponse {
-  // indicates whether a restart is needed, or reload succesfully
+  // indicates whether a restart is needed, or reload successfully
   enum Action {
     Restart = 0;
     Reload = 1;
   }
-  
+
   // action for this response
   Action action = 1;
 
   // text reason for the response
   string reason = 2;
 }
 
-// NOT USED
-message WorkerStatusRequest{
+// Used by the host to determine worker health
+message WorkerStatusRequest {
 }
 
-// NOT USED
+// Worker responds with status message
+// TODO: Add any worker relevant status to response
 message WorkerStatusResponse {
 }
 
@@ -220,7 +222,17 @@ message CloseSharedMemoryResourcesResponse {
   map<string, bool> close_map_results = 1;
 }
 
-// Host tells the worker to load a Function
+// Host tells the worker to load a list of Functions
+message FunctionLoadRequestCollection {
+  repeated FunctionLoadRequest function_load_requests = 1;
+}
+
+// Host gets the list of function load responses
+message FunctionLoadResponseCollection {
+  repeated FunctionLoadResponse function_load_responses = 1;
+}
+
+// Load request of a single Function
 message FunctionLoadRequest {
   // unique function identifier (avoid name collisions, facilitate reload case)
   string function_id = 1;
@@ -252,7 +264,7 @@ message RpcFunctionMetadata {
 
   // base directory for the Function
   string directory = 1;
-  
+
   // Script file specified
   string script_file = 2;
 
@@ -273,6 +285,12 @@ message RpcFunctionMetadata {
 
   // Raw binding info
   repeated string raw_bindings = 10;
+
+  // unique function identifier (avoid name collisions, facilitate reload case)
+  string function_id = 13;
+
+  // A flag indicating if managed dependency is enabled or not
+  bool managed_dependency_enabled = 14;
 }
 
 // Host tells worker it is ready to receive metadata
@@ -282,12 +300,15 @@ message FunctionsMetadataRequest {
 }
 
 // Worker sends function metadata back to host
-message FunctionMetadataResponses {
+message FunctionMetadataResponse {
   // list of function indexing responses
-  repeated FunctionLoadRequest function_load_requests_results = 1;
+  repeated RpcFunctionMetadata function_metadata_results = 1;
 
   // status of overall metadata request
   StatusResult result = 2;
+
+  // if set to true then host will perform indexing
+  bool use_default_metadata_indexing = 3;
 }
 
 // Host requests worker to invoke a Function
@@ -464,7 +485,7 @@ message BindingInfo {
   DataType data_type = 4;
 }
 
-// Used to send logs back to the Host 
+// Used to send logs back to the Host
 message RpcLog {
   // Matching ILogger semantics
   // https://github.com/aspnet/Logging/blob/9506ccc3f3491488fe88010ef8b9eb64594abf95/src/Microsoft.Extensions.Logging/Logger.cs
@@ -515,7 +536,7 @@ message RpcLog {
   map<string, TypedData> propertiesMap = 9;
 }
 
-// Encapsulates an Exception 
+// Encapsulates an Exception
 message RpcException {
   // Source of the exception
   string source = 3;
@@ -525,6 +546,14 @@ message RpcException {
 
   // Textual message describing the exception
   string message = 2;
+
+  // Worker specifies whether exception is a user exception, 
+  // for purpose of application insights logging. Defaults to false. 
+  optional bool is_user_exception = 4;
+
+  // Type of exception. If it's a user exception, the type is passed along to app insights.
+  // Otherwise, it's ignored for now.
+  optional string type = 5; 
 }
 
 // Http cookie type. Note that only name and value are used for Http requests
@@ -569,7 +598,7 @@ message RpcHttpCookie {
 // TODO - solidify this or remove it
 message RpcHttp {
   string method = 1;
-  string url = 2; 
+  string url = 2;
   map<string,string> headers = 3;
   TypedData body = 4;
   map<string,string> params = 10;

release_notes.md

@@ -1,3 +0,0 @@
-* Bug fix: [Context.InstanceId can now be accessed](https://github.com/Azure/azure-functions-powershell-worker/issues/727)
-* Bug fix: [Data in External Events is now read and returned to orchestrator](https://github.com/Azure/azure-functions-powershell-worker/issues/68)
-* New feature (external contribution): [Get-TaskResult can now be used to obtain the result of an already-completed Durable Functions Task](https://github.com/Azure/azure-functions-powershell-worker/pull/786)

src/DependencyManagement/DependencySnapshotInstaller.cs

@@ -9,6 +9,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement
     using System.Collections.Generic;
     using System.Management.Automation;
     using System.Threading;
+    using System.Diagnostics;
 
     using Microsoft.Azure.Functions.PowerShellWorker.Utility;
 
@@ -129,9 +130,12 @@ private void InstallModule(DependencyInfo module, string installingPath, PowerSh
             {
                 try
                 {
+                    var stopwatch = new Stopwatch();
+                    stopwatch.Start();
+
                     _moduleProvider.SaveModule(pwsh, module.Name, module.ExactVersion, installingPath);
 
-                    var message = string.Format(PowerShellWorkerStrings.ModuleHasBeenInstalled, module.Name, module.ExactVersion);
+                    var message = string.Format(PowerShellWorkerStrings.ModuleHasBeenInstalled, module.Name, module.ExactVersion, stopwatch.ElapsedMilliseconds);
                     logger.Log(isUserOnlyLog: false, LogLevel.Trace, message);
 
                     break;