Update proto and add custom build steps (#601)

Author: maiqbal11

azure_functions_worker/protos/_src/README.md

@@ -1,6 +1,6 @@
 # Azure Functions Languge Worker Protobuf
 
-This repository contains the protobuf definition file which defines the gRPC service which is used between the Azure WebJobs Script host and the Azure Functions language workers. This repo is shared across many repos in many languages (for each worker) by using git commands.
+This repository contains the protobuf definition file which defines the gRPC service which is used between the [Azure Functions Host](https://github.com/Azure/azure-functions-host) and the Azure Functions language workers. This repo is shared across many repos in many languages (for each worker) by using git commands.
 
 To use this repo in Azure Functions language workers, follow steps below to add this repo as a subtree (*Adding This Repo*). If this repo is already embedded in a language worker repo, follow the steps to update the consumed file (*Pulling Updates*).
 

azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto

@@ -10,8 +10,8 @@ option go_package ="github.com/Azure/azure-functions-go-worker/internal/rpc";
 package AzureFunctionsRpcMessages;
 
 import "google/protobuf/duration.proto";
-import "azure_functions_worker/protos/ClaimsIdentityRpc.proto";
-import "azure_functions_worker/protos/NullableTypes.proto";
+import "identity/ClaimsIdentityRpc.proto";
+import "shared/NullableTypes.proto";
 
 // Interface exported by the server.
 service FunctionRpc {
@@ -189,6 +189,8 @@ message WorkerStatusResponse {
 message FunctionEnvironmentReloadRequest {
   // Environment variables from the current process
   map<string, string> environment_variables = 1;
+  // Current directory of function app
+  string function_app_directory = 2;
 }
 
 message FunctionEnvironmentReloadResponse {
@@ -255,6 +257,21 @@ message InvocationRequest {
 
   // binding metadata from trigger
   map<string, TypedData> trigger_metadata = 4;
+
+  // Populates activityId, tracestate and tags from host
+  RpcTraceContext trace_context = 5;
+}
+
+// Host sends ActivityId, traceStateString and Tags from host
+message RpcTraceContext {
+	// This corresponds to Activity.Current?.Id
+	string trace_parent = 1;
+
+	// This corresponds to Activity.Current?.TraceStateString
+	string trace_state = 2;
+
+	// This corresponds to Activity.Current?.Tags
+	map<string, string> attributes = 3;
 }
 
 // Host requests worker to cancel invocation
@@ -368,6 +385,12 @@ message RpcLog {
     None = 6;
   }
 
+  // Category of the log. Defaults to User if not specified.
+  enum RpcLogCategory {
+	User = 0;
+	System = 1;
+  }
+
   // Unique id for invocation (if exists)
   string invocation_id = 1;
 
@@ -389,6 +412,9 @@ message RpcLog {
 
   // json serialized property bag, or could use a type scheme like map<string, TypedData>
   string properties = 7;
+
+  // Category of the log. Either user(default) or system.
+  RpcLogCategory log_category = 8;
 }
 
 // Encapsulates an Exception 
@@ -399,7 +425,7 @@ message RpcException {
   // Stack trace for the exception
   string stack_trace = 1;
 
-  // Textual message describing hte exception
+  // Textual message describing the exception
   string message = 2;
 }
 

azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto

@@ -3,7 +3,7 @@ syntax = "proto3";
 
 option java_package = "com.microsoft.azure.functions.rpc.messages";
 
-import "azure_functions_worker/protos/NullableTypes.proto";
+import "shared/NullableTypes.proto";
 
 // Light-weight representation of a .NET System.Security.Claims.ClaimsIdentity object.
 // This is the same serialization as found in EasyAuth, and needs to be kept in sync with

setup.py

@@ -8,6 +8,8 @@
 import tempfile
 import urllib.request
 import zipfile
+import re
+from distutils import dir_util
 from distutils.command import build
 
 from setuptools import setup
@@ -73,56 +75,71 @@ def _gen_grpc(self):
 
         proto_root_dir = root / 'azure_functions_worker' / 'protos'
         proto_src_dir = proto_root_dir / '_src' / 'src' / 'proto'
-        staging_root_dir = root / 'build' / 'protos'
+        build_dir = root / 'build'
+        staging_root_dir = build_dir / 'protos'
         staging_dir = (staging_root_dir
                        / 'azure_functions_worker' / 'protos')
-        build_dir = staging_dir / 'azure_functions_worker' / 'protos'
-        built_protos_dir = root / 'build' / 'built_protos'
-        built_proto_files_dir = (built_protos_dir
-                                 / 'azure_functions_worker' / 'protos')
+        built_protos_dir = build_dir / 'built_protos'
 
         if os.path.exists(build_dir):
             shutil.rmtree(build_dir)
 
-        if os.path.exists(built_protos_dir):
-            shutil.rmtree(built_protos_dir)
+        shutil.copytree(proto_src_dir, staging_dir)
 
-        proto_files = glob.glob(str(proto_src_dir / '**' / '*.proto'),
-                                recursive=True)
-
-        os.makedirs(build_dir)
-        for proto_file in proto_files:
-            shutil.copy(proto_file, build_dir)
+        os.makedirs(built_protos_dir)
 
-        protos = [os.path.basename(proto_file) for proto_file in proto_files]
+        protos = [
+            os.sep.join(('shared', 'NullableTypes.proto')),
+            os.sep.join(('identity', 'ClaimsIdentityRpc.proto')),
+            'FunctionRpc.proto'
+        ]
 
-        full_protos = []
         for proto in protos:
-            full_proto = os.sep.join(
-                ('azure_functions_worker', 'protos',
-                 'azure_functions_worker', 'protos', proto)
-            )
-            full_protos.append(full_proto)
-
-        os.makedirs(built_protos_dir)
-        subprocess.run([
-            sys.executable, '-m', 'grpc_tools.protoc',
-            '-I', os.sep.join(('azure_functions_worker', 'protos')),
-            '--python_out', str(built_protos_dir),
-            '--grpc_python_out', str(built_protos_dir),
-            *full_protos
-        ], check=True, stdout=sys.stdout, stderr=sys.stderr,
-            cwd=staging_root_dir)
-
-        compiled = glob.glob(str(built_proto_files_dir / '*.py'))
-
-        if not compiled:
+            subprocess.run([
+                sys.executable, '-m', 'grpc_tools.protoc',
+                '-I', os.sep.join(('azure_functions_worker', 'protos')),
+                '--python_out', str(built_protos_dir),
+                '--grpc_python_out', str(built_protos_dir),
+                os.sep.join(('azure_functions_worker', 'protos', proto)),
+            ], check=True, stdout=sys.stdout, stderr=sys.stderr,
+                cwd=staging_root_dir)
+
+        compiled_files = glob.glob(
+            str(built_protos_dir / '**' / '*.py'),
+            recursive=True)
+
+        if not compiled_files:
             print('grpc_tools.protoc produced no Python files',
                   file=sys.stderr)
             sys.exit(1)
 
-        for f in compiled:
-            shutil.copy(f, proto_root_dir)
+        # Needed to support absolute imports in files. See
+        # https://github.com/protocolbuffers/protobuf/issues/1491
+        self.make_absolute_imports(compiled_files)
+
+        dir_util.copy_tree(built_protos_dir, str(proto_root_dir))
+
+    def make_absolute_imports(self, compiled_files):
+        for compiled in compiled_files:
+            with open(compiled, 'r+') as f:
+                content = f.read()
+                f.seek(0)
+                # Convert lines of the form:
+                # import xxx_pb2 as xxx__pb2 to
+                # from azure_functions_worker.protos import xxx_pb2 as..
+                p1 = re.sub(
+                    r'\nimport (.*?_pb2)',
+                    r'\nfrom azure_functions_worker.protos import \g<1>',
+                    content)
+                # Convert lines of the form:
+                # from identity import xxx_pb2 as.. to
+                # from azure_functions_worker.protos.identity import xxx_pb2..
+                p2 = re.sub(
+                    r'from ([a-z]*) (import.*_pb2)',
+                    r'from azure_functions_worker.protos.\g<1> \g<2>',
+                    p1)
+                f.write(p2)
+                f.truncate()
 
 
 class build(build.build, BuildGRPC):