Update protobuf (Azure#528)

Author: maiqbal11

azure-pipelines.yml

@@ -5,7 +5,7 @@ trigger:
 - master
 
 variables:
-    DOTNET_VERSION: '2.2.300'
+    DOTNET_VERSION: '2.2.401'
 
 jobs:
 - job: Tests

azure_functions_worker/protos/_src/README.md

@@ -26,19 +26,30 @@ From within the Azure Functions language worker repo:
 1.	Define remote branch for cleaner git commands
     -	`git remote add proto-file https://github.com/azure/azure-functions-language-worker-protobuf.git`
     -	`git fetch proto-file`
-2.	Merge updates
-    -   `git merge -s subtree proto-file/<version branch> --squash --allow-unrelated-histories` 
-        -   You can also merge with an explicit path to subtree: `git merge -X subtree=<path in language worker repo> --squash proto-file/<version branch> --allow-unrelated-histories`
-3.	Finalize with commit
-    -	`git commit -m "Updated subtree from https://github.com/azure/azure-functions-language-worker-protobuf. Branch: <version branch>. Commit: <latest protobuf commit hash>"`
+2.	Pull a specific release tag
+    -   `git fetch proto-file refs/tags/<tag-name>`
+        -   Example: `git fetch proto-file refs/tags/v1.1.0-protofile`
+3.	Merge updates
+    -   Merge with an explicit path to subtree: `git merge -X subtree=<path in language worker repo> --squash <tag-name> --allow-unrelated-histories --strategy-option theirs`
+        -   Example: `git merge -X subtree=src/WebJobs.Script.Grpc/azure-functions-language-worker-protobuf --squash v1.1.0-protofile --allow-unrelated-histories --strategy-option theirs`
+4.	Finalize with commit
+    -	`git commit -m "Updated subtree from https://github.com/azure/azure-functions-language-worker-protobuf. Tag: <tag-name>. Commit: <commit hash>"`
     -	`git push`
-	
+
+## Releasing a Language Worker Protobuf version
+
+1.	Draft a release in the GitHub UI
+    -   Be sure to inculde details of the release
+2.	Create a release version, following semantic versioning guidelines ([semver.org](https://semver.org/))
+3.	Tag the version with the pattern: `v<M>.<m>.<p>-protofile` (example: `v1.1.0-protofile`)
+3.	Merge `dev` to `master`
+
 ## Consuming FunctionRPC.proto
 *Note: Update versionNumber before running following commands*
 
 ## CSharp
 ```
-set NUGET_PATH=%UserProfile%\.nuget\packages
+set NUGET_PATH="%UserProfile%\.nuget\packages"
 set GRPC_TOOLS_PATH=%NUGET_PATH%\grpc.tools\<versionNumber>\tools\windows_x86
 set PROTO_PATH=.\azure-functions-language-worker-protobuf\src\proto
 set PROTO=.\azure-functions-language-worker-protobuf\src\proto\FunctionRpc.proto

azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto

@@ -10,6 +10,8 @@ option go_package ="github.com/Azure/azure-functions-go-worker/internal/rpc";
 package AzureFunctionsRpcMessages;
 
 import "google/protobuf/duration.proto";
+import "identity/ClaimsIdentityRpc.proto";
+import "shared/NullableTypes.proto";
 
 // Interface exported by the server.
 service FunctionRpc {
@@ -201,6 +203,9 @@ message FunctionLoadRequest {
 
   // Metadata for the request
   RpcFunctionMetadata metadata = 2;
+
+  // A flag indicating if managed dependency is enabled or not
+  bool managed_dependency_enabled = 3;
 }
 
 // Worker tells host result of reload
@@ -211,6 +216,9 @@ message FunctionLoadResponse {
   // Result of load operation
   StatusResult result = 2;
   // TODO: return type expected?
+
+  // Result of load operation
+  bool is_dependency_downloaded = 3;
 }
 
 // Information on how a Function should be loaded and its bindings
@@ -229,6 +237,9 @@ message RpcFunctionMetadata {
 
   // Bindings info
   map<string, BindingInfo> bindings = 6;
+
+  // Is set to true for proxy
+  bool is_proxy = 7;
 }
 
 // Host requests worker to invoke a Function
@@ -278,11 +289,35 @@ message TypedData {
     bytes bytes = 3;
     bytes stream = 4;
     RpcHttp http = 5;
-	  sint64 int = 6;
+    sint64 int = 6;
     double double = 7;
+    CollectionBytes collection_bytes = 8;
+    CollectionString collection_string = 9;
+    CollectionDouble collection_double = 10;
+    CollectionSInt64 collection_sint64 = 11;
   }
 }
 
+// Used to encapsulate collection string
+message CollectionString {
+  repeated string string = 1;
+}
+
+// Used to encapsulate collection bytes
+message CollectionBytes {
+  repeated bytes bytes = 1;
+}
+
+// Used to encapsulate collection double
+message CollectionDouble {
+  repeated double double = 1;
+}
+
+// Used to encapsulate collection sint64
+message CollectionSInt64 {
+  repeated sint64 sint64 = 1;
+}
+
 // Used to describe a given binding on invocation
 message ParameterBinding {
   // Name for the binding
@@ -368,6 +403,44 @@ message RpcException {
   string message = 2;
 }
 
+// Http cookie type. Note that only name and value are used for Http requests
+message RpcHttpCookie {
+  // Enum that lets servers require that a cookie shouoldn't be sent with cross-site requests
+  enum SameSite {
+      None = 0;
+      Lax = 1;
+      Strict = 2;
+  }
+
+  // Cookie name
+  string name = 1;
+
+  // Cookie value
+  string value = 2;
+
+  // Specifies allowed hosts to receive the cookie
+  NullableString domain = 3;
+
+  // Specifies URL path that must exist in the requested URL
+  NullableString path = 4;
+
+  // Sets the cookie to expire at a specific date instead of when the client closes.
+  // It is generally recommended that you use "Max-Age" over "Expires".
+  NullableTimestamp expires = 5;
+
+  // Sets the cookie to only be sent with an encrypted request
+  NullableBool secure = 6;
+
+  // Sets the cookie to be inaccessible to JavaScript's Document.cookie API
+  NullableBool http_only = 7;
+
+  // Allows servers to assert that a cookie ought not to be sent along with cross-site requests
+  SameSite same_site = 8;
+
+  // Number of seconds until the cookie expires. A zero or negative number will expire the cookie immediately.
+  NullableDouble max_age = 9;
+}
+
 // TODO - solidify this or remove it
 message RpcHttp {
   string method = 1;
@@ -379,4 +452,6 @@ message RpcHttp {
   map<string,string> query = 15;
   bool enable_content_negotiation= 16;
   TypedData rawBody = 17;
+  repeated RpcClaimsIdentity identities = 18;
+  repeated RpcHttpCookie cookies = 19;
 }

azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+// protobuf vscode extension: https://marketplace.visualstudio.com/items?itemName=zxh404.vscode-proto3
+
+option java_package = "com.microsoft.azure.functions.rpc.messages";
+
+import "shared/NullableTypes.proto";
+
+// Light-weight representation of a .NET System.Security.Claims.ClaimsIdentity object.
+// This is the same serialization as found in EasyAuth, and needs to be kept in sync with
+// its ClaimsIdentitySlim definition, as seen in the WebJobs extension:
+// https://github.com/Azure/azure-webjobs-sdk-extensions/blob/dev/src/WebJobs.Extensions.Http/ClaimsIdentitySlim.cs
+message RpcClaimsIdentity {
+  NullableString authentication_type = 1;
+  NullableString name_claim_type = 2;
+  NullableString role_claim_type = 3;
+  repeated RpcClaim claims = 4;
+}
+
+// Light-weight representation of a .NET System.Security.Claims.Claim object.
+// This is the same serialization as found in EasyAuth, and needs to be kept in sync with
+// its ClaimSlim definition, as seen in the WebJobs extension:
+// https://github.com/Azure/azure-webjobs-sdk-extensions/blob/dev/src/WebJobs.Extensions.Http/ClaimSlim.cs
+message RpcClaim {
+  string value = 1;
+  string type = 2;
+}

azure_functions_worker/protos/_src/src/proto/shared/NullableTypes.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+// protobuf vscode extension: https://marketplace.visualstudio.com/items?itemName=zxh404.vscode-proto3
+
+option java_package = "com.microsoft.azure.functions.rpc.messages";
+
+import "google/protobuf/timestamp.proto";
+
+message NullableString {
+	oneof string {
+		string value = 1;
+	}
+}
+
+message NullableDouble {
+	oneof double {
+		double value = 1;
+	}
+}
+
+message NullableBool {
+	oneof bool {
+		bool value = 1;
+	}
+}
+
+message NullableTimestamp {
+	oneof timestamp {
+		google.protobuf.Timestamp value = 1;
+	}
+}

setup.py

@@ -8,16 +8,18 @@
 import tempfile
 import urllib.request
 import zipfile
+import re
+from distutils import dir_util
 from distutils.command import build
 
 from setuptools import setup
 from setuptools.command import develop
 
 
 # TODO: change this to something more stable when available.
-WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/1em48j4cd3odl4s9'
+WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/sfelyng3x6p5sus0'
                '/artifacts'
-               '/Functions.Binaries.2.0.12521-prerelease.no-runtime.zip')
+               '/Functions.Binaries.2.0.12642.no-runtime.zip')
 
 # Extensions necessary for non-core bindings.
 AZURE_EXTENSIONS = [
@@ -73,36 +75,74 @@ def _gen_grpc(self):
 
         proto_root_dir = root / 'azure_functions_worker' / 'protos'
         proto_src_dir = proto_root_dir / '_src' / 'src' / 'proto'
-        staging_root_dir = root / 'build' / 'protos'
+        build_dir = root / 'build'
+        staging_root_dir = build_dir / 'protos'
         staging_dir = (staging_root_dir
                        / 'azure_functions_worker' / 'protos')
-        build_dir = staging_dir / 'azure_functions_worker' / 'protos'
+        built_protos_dir = build_dir / 'built_protos'
 
-        if os.path.exists(build_dir):
-            shutil.rmtree(build_dir)
+        if os.path.exists(staging_root_dir):
+            shutil.rmtree(staging_root_dir)
 
-        shutil.copytree(proto_src_dir, build_dir)
+        if os.path.exists(built_protos_dir):
+            shutil.rmtree(built_protos_dir)
 
-        subprocess.run([
-            sys.executable, '-m', 'grpc_tools.protoc',
-            '-I', os.sep.join(('azure_functions_worker', 'protos')),
-            '--python_out', str(staging_root_dir),
-            '--grpc_python_out', str(staging_root_dir),
-            os.sep.join(('azure_functions_worker', 'protos',
-                         'azure_functions_worker', 'protos',
-                         'FunctionRpc.proto')),
-        ], check=True, stdout=sys.stdout, stderr=sys.stderr,
-            cwd=staging_root_dir)
+        shutil.copytree(proto_src_dir, staging_dir)
 
-        compiled = glob.glob(str(staging_dir / '*.py'))
+        os.makedirs(built_protos_dir)
 
-        if not compiled:
+        protos = [
+            os.sep.join(('shared', 'NullableTypes.proto')),
+            os.sep.join(('identity', 'ClaimsIdentityRpc.proto')),
+            'FunctionRpc.proto'
+        ]
+
+        for proto in protos:
+            subprocess.run([
+                sys.executable, '-m', 'grpc_tools.protoc',
+                '-I', os.sep.join(('azure_functions_worker', 'protos')),
+                '--python_out', str(built_protos_dir),
+                '--grpc_python_out', str(built_protos_dir),
+                os.sep.join(('azure_functions_worker', 'protos', proto)),
+            ], check=True, stdout=sys.stdout, stderr=sys.stderr,
+                cwd=staging_root_dir)
+
+        compiled_files = glob.glob(
+            str(built_protos_dir / '**' / '*.py'),
+            recursive=True)
+
+        if not compiled_files:
             print('grpc_tools.protoc produced no Python files',
                   file=sys.stderr)
             sys.exit(1)
 
-        for f in compiled:
-            shutil.copy(f, proto_root_dir)
+        # Needed to support absolute imports in files. See
+        # https://github.com/protocolbuffers/protobuf/issues/1491
+        self.make_absolute_imports(compiled_files)
+
+        dir_util.copy_tree(built_protos_dir, str(proto_root_dir))
+
+    def make_absolute_imports(self, compiled_files):
+        for compiled in compiled_files:
+            with open(compiled, 'r+') as f:
+                content = f.read()
+                f.seek(0)
+                # Convert lines of the form:
+                # import xxx_pb2 as xxx__pb2 to
+                # from azure_functions_worker.protos import xxx_pb2 as..
+                p1 = re.sub(
+                    r'\nimport (.*?_pb2)',
+                    r'\nfrom azure_functions_worker.protos import \g<1>',
+                    content)
+                # Convert lines of the form:
+                # from identity import xxx_pb2 as.. to
+                # from azure_functions_worker.protos.identity import xxx_pb2..
+                p2 = re.sub(
+                    r'from ([a-z]*) (import.*_pb2)',
+                    r'from azure_functions_worker.protos.\g<1> \g<2>',
+                    p1)
+                f.write(p2)
+                f.truncate()
 
 
 class build(build.build, BuildGRPC):