Skip to content

Update proto and add custom build steps #601

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 8, 2020
Merged

Update proto and add custom build steps #601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e35d9e2
Update proto and add custom build steps
maiqbal11 Jan 7, 2020
2f65672
Remove redundant directory deletes
maiqbal11 Jan 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion azure_functions_worker/protos/_src/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Azure Functions Languge Worker Protobuf

This repository contains the protobuf definition file which defines the gRPC service which is used between the Azure WebJobs Script host and the Azure Functions language workers. This repo is shared across many repos in many languages (for each worker) by using git commands.
This repository contains the protobuf definition file which defines the gRPC service which is used between the [Azure Functions Host](https://github.com/Azure/azure-functions-host) and the Azure Functions language workers. This repo is shared across many repos in many languages (for each worker) by using git commands.

To use this repo in Azure Functions language workers, follow steps below to add this repo as a subtree (*Adding This Repo*). If this repo is already embedded in a language worker repo, follow the steps to update the consumed file (*Pulling Updates*).

Expand Down
32 changes: 29 additions & 3 deletions azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ option go_package ="github.com/Azure/azure-functions-go-worker/internal/rpc";
package AzureFunctionsRpcMessages;

import "google/protobuf/duration.proto";
import "azure_functions_worker/protos/ClaimsIdentityRpc.proto";
import "azure_functions_worker/protos/NullableTypes.proto";
import "identity/ClaimsIdentityRpc.proto";
import "shared/NullableTypes.proto";

// Interface exported by the server.
service FunctionRpc {
Expand Down Expand Up @@ -189,6 +189,8 @@ message WorkerStatusResponse {
message FunctionEnvironmentReloadRequest {
// Environment variables from the current process
map<string, string> environment_variables = 1;
// Current directory of function app
string function_app_directory = 2;
}

message FunctionEnvironmentReloadResponse {
Expand Down Expand Up @@ -255,6 +257,21 @@ message InvocationRequest {

// binding metadata from trigger
map<string, TypedData> trigger_metadata = 4;

// Populates activityId, tracestate and tags from host
RpcTraceContext trace_context = 5;
}

// Host sends ActivityId, traceStateString and Tags from host
message RpcTraceContext {
// This corresponds to Activity.Current?.Id
string trace_parent = 1;

// This corresponds to Activity.Current?.TraceStateString
string trace_state = 2;

// This corresponds to Activity.Current?.Tags
map<string, string> attributes = 3;
}

// Host requests worker to cancel invocation
Expand Down Expand Up @@ -368,6 +385,12 @@ message RpcLog {
None = 6;
}

// Category of the log. Defaults to User if not specified.
enum RpcLogCategory {
User = 0;
System = 1;
}

// Unique id for invocation (if exists)
string invocation_id = 1;

Expand All @@ -389,6 +412,9 @@ message RpcLog {

// json serialized property bag, or could use a type scheme like map<string, TypedData>
string properties = 7;

// Category of the log. Either user(default) or system.
RpcLogCategory log_category = 8;
}

// Encapsulates an Exception
Expand All @@ -399,7 +425,7 @@ message RpcException {
// Stack trace for the exception
string stack_trace = 1;

// Textual message describing hte exception
// Textual message describing the exception
string message = 2;
}

Expand Down
2 changes: 1 addition & 1 deletion azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ syntax = "proto3";

option java_package = "com.microsoft.azure.functions.rpc.messages";

import "azure_functions_worker/protos/NullableTypes.proto";
import "shared/NullableTypes.proto";

// Light-weight representation of a .NET System.Security.Claims.ClaimsIdentity object.
// This is the same serialization as found in EasyAuth, and needs to be kept in sync with
Expand Down
89 changes: 53 additions & 36 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import tempfile
import urllib.request
import zipfile
import re
from distutils import dir_util
from distutils.command import build

from setuptools import setup
Expand Down Expand Up @@ -73,56 +75,71 @@ def _gen_grpc(self):

proto_root_dir = root / 'azure_functions_worker' / 'protos'
proto_src_dir = proto_root_dir / '_src' / 'src' / 'proto'
staging_root_dir = root / 'build' / 'protos'
build_dir = root / 'build'
staging_root_dir = build_dir / 'protos'
staging_dir = (staging_root_dir
/ 'azure_functions_worker' / 'protos')
build_dir = staging_dir / 'azure_functions_worker' / 'protos'
built_protos_dir = root / 'build' / 'built_protos'
built_proto_files_dir = (built_protos_dir
/ 'azure_functions_worker' / 'protos')
built_protos_dir = build_dir / 'built_protos'

if os.path.exists(build_dir):
shutil.rmtree(build_dir)

if os.path.exists(built_protos_dir):
shutil.rmtree(built_protos_dir)
shutil.copytree(proto_src_dir, staging_dir)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't seems like staging_dir is used anywhere else. Maybe we can remove it?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's being used to do the copying as you pointed here. In the actual build command, we are using the same directory structure without explicitly referencing the staging_dir. Might be useful to walk through this offline as it's slightly tedious to explain here.


proto_files = glob.glob(str(proto_src_dir / '**' / '*.proto'),
recursive=True)

os.makedirs(build_dir)
for proto_file in proto_files:
shutil.copy(proto_file, build_dir)
os.makedirs(built_protos_dir)

protos = [os.path.basename(proto_file) for proto_file in proto_files]
protos = [
os.sep.join(('shared', 'NullableTypes.proto')),
os.sep.join(('identity', 'ClaimsIdentityRpc.proto')),
'FunctionRpc.proto'
]

full_protos = []
for proto in protos:
full_proto = os.sep.join(
('azure_functions_worker', 'protos',
'azure_functions_worker', 'protos', proto)
)
full_protos.append(full_proto)

os.makedirs(built_protos_dir)
subprocess.run([
sys.executable, '-m', 'grpc_tools.protoc',
'-I', os.sep.join(('azure_functions_worker', 'protos')),
'--python_out', str(built_protos_dir),
'--grpc_python_out', str(built_protos_dir),
*full_protos
], check=True, stdout=sys.stdout, stderr=sys.stderr,
cwd=staging_root_dir)

compiled = glob.glob(str(built_proto_files_dir / '*.py'))

if not compiled:
subprocess.run([
sys.executable, '-m', 'grpc_tools.protoc',
'-I', os.sep.join(('azure_functions_worker', 'protos')),
'--python_out', str(built_protos_dir),
'--grpc_python_out', str(built_protos_dir),
os.sep.join(('azure_functions_worker', 'protos', proto)),
], check=True, stdout=sys.stdout, stderr=sys.stderr,
cwd=staging_root_dir)

compiled_files = glob.glob(
str(built_protos_dir / '**' / '*.py'),
recursive=True)

if not compiled_files:
print('grpc_tools.protoc produced no Python files',
file=sys.stderr)
sys.exit(1)

for f in compiled:
shutil.copy(f, proto_root_dir)
# Needed to support absolute imports in files. See
# https://github.com/protocolbuffers/protobuf/issues/1491
self.make_absolute_imports(compiled_files)

dir_util.copy_tree(built_protos_dir, str(proto_root_dir))

def make_absolute_imports(self, compiled_files):
for compiled in compiled_files:
with open(compiled, 'r+') as f:
content = f.read()
f.seek(0)
# Convert lines of the form:
# import xxx_pb2 as xxx__pb2 to
# from azure_functions_worker.protos import xxx_pb2 as..
p1 = re.sub(
r'\nimport (.*?_pb2)',
r'\nfrom azure_functions_worker.protos import \g<1>',
content)
# Convert lines of the form:
# from identity import xxx_pb2 as.. to
# from azure_functions_worker.protos.identity import xxx_pb2..
p2 = re.sub(
r'from ([a-z]*) (import.*_pb2)',
r'from azure_functions_worker.protos.\g<1> \g<2>',
p1)
f.write(p2)
f.truncate()


class build(build.build, BuildGRPC):
Expand Down