diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4ec0935c2..c658e37e8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,7 +5,7 @@ trigger: - master variables: - DOTNET_VERSION: '2.2.300' + DOTNET_VERSION: '2.2.401' jobs: - job: Tests diff --git a/azure_functions_worker/protos/_src/README.md b/azure_functions_worker/protos/_src/README.md index a490b9603..0f0f2341f 100644 --- a/azure_functions_worker/protos/_src/README.md +++ b/azure_functions_worker/protos/_src/README.md @@ -26,19 +26,30 @@ From within the Azure Functions language worker repo: 1. Define remote branch for cleaner git commands - `git remote add proto-file https://github.com/azure/azure-functions-language-worker-protobuf.git` - `git fetch proto-file` -2. Merge updates - - `git merge -s subtree proto-file/ --squash --allow-unrelated-histories` - - You can also merge with an explicit path to subtree: `git merge -X subtree= --squash proto-file/ --allow-unrelated-histories` -3. Finalize with commit - - `git commit -m "Updated subtree from https://github.com/azure/azure-functions-language-worker-protobuf. Branch: . Commit: "` +2. Pull a specific release tag + - `git fetch proto-file refs/tags/` + - Example: `git fetch proto-file refs/tags/v1.1.0-protofile` +3. Merge updates + - Merge with an explicit path to subtree: `git merge -X subtree= --squash --allow-unrelated-histories --strategy-option theirs` + - Example: `git merge -X subtree=src/WebJobs.Script.Grpc/azure-functions-language-worker-protobuf --squash v1.1.0-protofile --allow-unrelated-histories --strategy-option theirs` +4. Finalize with commit + - `git commit -m "Updated subtree from https://github.com/azure/azure-functions-language-worker-protobuf. Tag: . Commit: "` - `git push` - + +## Releasing a Language Worker Protobuf version + +1. Draft a release in the GitHub UI + - Be sure to inculde details of the release +2. Create a release version, following semantic versioning guidelines ([semver.org](https://semver.org/)) +3. Tag the version with the pattern: `v..

-protofile` (example: `v1.1.0-protofile`) +3. Merge `dev` to `master` + ## Consuming FunctionRPC.proto *Note: Update versionNumber before running following commands* ## CSharp ``` -set NUGET_PATH=%UserProfile%\.nuget\packages +set NUGET_PATH="%UserProfile%\.nuget\packages" set GRPC_TOOLS_PATH=%NUGET_PATH%\grpc.tools\\tools\windows_x86 set PROTO_PATH=.\azure-functions-language-worker-protobuf\src\proto set PROTO=.\azure-functions-language-worker-protobuf\src\proto\FunctionRpc.proto diff --git a/azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto b/azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto index 44857b8b5..2d54cf945 100644 --- a/azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto +++ b/azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto @@ -10,6 +10,8 @@ option go_package ="github.com/Azure/azure-functions-go-worker/internal/rpc"; package AzureFunctionsRpcMessages; import "google/protobuf/duration.proto"; +import "identity/ClaimsIdentityRpc.proto"; +import "shared/NullableTypes.proto"; // Interface exported by the server. service FunctionRpc { @@ -201,6 +203,9 @@ message FunctionLoadRequest { // Metadata for the request RpcFunctionMetadata metadata = 2; + + // A flag indicating if managed dependency is enabled or not + bool managed_dependency_enabled = 3; } // Worker tells host result of reload @@ -211,6 +216,9 @@ message FunctionLoadResponse { // Result of load operation StatusResult result = 2; // TODO: return type expected? + + // Result of load operation + bool is_dependency_downloaded = 3; } // Information on how a Function should be loaded and its bindings @@ -229,6 +237,9 @@ message RpcFunctionMetadata { // Bindings info map bindings = 6; + + // Is set to true for proxy + bool is_proxy = 7; } // Host requests worker to invoke a Function @@ -278,11 +289,35 @@ message TypedData { bytes bytes = 3; bytes stream = 4; RpcHttp http = 5; - sint64 int = 6; + sint64 int = 6; double double = 7; + CollectionBytes collection_bytes = 8; + CollectionString collection_string = 9; + CollectionDouble collection_double = 10; + CollectionSInt64 collection_sint64 = 11; } } +// Used to encapsulate collection string +message CollectionString { + repeated string string = 1; +} + +// Used to encapsulate collection bytes +message CollectionBytes { + repeated bytes bytes = 1; +} + +// Used to encapsulate collection double +message CollectionDouble { + repeated double double = 1; +} + +// Used to encapsulate collection sint64 +message CollectionSInt64 { + repeated sint64 sint64 = 1; +} + // Used to describe a given binding on invocation message ParameterBinding { // Name for the binding @@ -368,6 +403,44 @@ message RpcException { string message = 2; } +// Http cookie type. Note that only name and value are used for Http requests +message RpcHttpCookie { + // Enum that lets servers require that a cookie shouoldn't be sent with cross-site requests + enum SameSite { + None = 0; + Lax = 1; + Strict = 2; + } + + // Cookie name + string name = 1; + + // Cookie value + string value = 2; + + // Specifies allowed hosts to receive the cookie + NullableString domain = 3; + + // Specifies URL path that must exist in the requested URL + NullableString path = 4; + + // Sets the cookie to expire at a specific date instead of when the client closes. + // It is generally recommended that you use "Max-Age" over "Expires". + NullableTimestamp expires = 5; + + // Sets the cookie to only be sent with an encrypted request + NullableBool secure = 6; + + // Sets the cookie to be inaccessible to JavaScript's Document.cookie API + NullableBool http_only = 7; + + // Allows servers to assert that a cookie ought not to be sent along with cross-site requests + SameSite same_site = 8; + + // Number of seconds until the cookie expires. A zero or negative number will expire the cookie immediately. + NullableDouble max_age = 9; +} + // TODO - solidify this or remove it message RpcHttp { string method = 1; @@ -379,4 +452,6 @@ message RpcHttp { map query = 15; bool enable_content_negotiation= 16; TypedData rawBody = 17; + repeated RpcClaimsIdentity identities = 18; + repeated RpcHttpCookie cookies = 19; } diff --git a/azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto b/azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto new file mode 100644 index 000000000..c3945bb8a --- /dev/null +++ b/azure_functions_worker/protos/_src/src/proto/identity/ClaimsIdentityRpc.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; +// protobuf vscode extension: https://marketplace.visualstudio.com/items?itemName=zxh404.vscode-proto3 + +option java_package = "com.microsoft.azure.functions.rpc.messages"; + +import "shared/NullableTypes.proto"; + +// Light-weight representation of a .NET System.Security.Claims.ClaimsIdentity object. +// This is the same serialization as found in EasyAuth, and needs to be kept in sync with +// its ClaimsIdentitySlim definition, as seen in the WebJobs extension: +// https://github.com/Azure/azure-webjobs-sdk-extensions/blob/dev/src/WebJobs.Extensions.Http/ClaimsIdentitySlim.cs +message RpcClaimsIdentity { + NullableString authentication_type = 1; + NullableString name_claim_type = 2; + NullableString role_claim_type = 3; + repeated RpcClaim claims = 4; +} + +// Light-weight representation of a .NET System.Security.Claims.Claim object. +// This is the same serialization as found in EasyAuth, and needs to be kept in sync with +// its ClaimSlim definition, as seen in the WebJobs extension: +// https://github.com/Azure/azure-webjobs-sdk-extensions/blob/dev/src/WebJobs.Extensions.Http/ClaimSlim.cs +message RpcClaim { + string value = 1; + string type = 2; +} diff --git a/azure_functions_worker/protos/_src/src/proto/shared/NullableTypes.proto b/azure_functions_worker/protos/_src/src/proto/shared/NullableTypes.proto new file mode 100644 index 000000000..4fb476502 --- /dev/null +++ b/azure_functions_worker/protos/_src/src/proto/shared/NullableTypes.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; +// protobuf vscode extension: https://marketplace.visualstudio.com/items?itemName=zxh404.vscode-proto3 + +option java_package = "com.microsoft.azure.functions.rpc.messages"; + +import "google/protobuf/timestamp.proto"; + +message NullableString { + oneof string { + string value = 1; + } +} + +message NullableDouble { + oneof double { + double value = 1; + } +} + +message NullableBool { + oneof bool { + bool value = 1; + } +} + +message NullableTimestamp { + oneof timestamp { + google.protobuf.Timestamp value = 1; + } +} diff --git a/setup.py b/setup.py index 42d40aa2b..dad5080e2 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,8 @@ import tempfile import urllib.request import zipfile +import re +from distutils import dir_util from distutils.command import build from setuptools import setup @@ -15,9 +17,9 @@ # TODO: change this to something more stable when available. -WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/1em48j4cd3odl4s9' +WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/sfelyng3x6p5sus0' '/artifacts' - '/Functions.Binaries.2.0.12521-prerelease.no-runtime.zip') + '/Functions.Binaries.2.0.12642.no-runtime.zip') # Extensions necessary for non-core bindings. AZURE_EXTENSIONS = [ @@ -73,36 +75,74 @@ def _gen_grpc(self): proto_root_dir = root / 'azure_functions_worker' / 'protos' proto_src_dir = proto_root_dir / '_src' / 'src' / 'proto' - staging_root_dir = root / 'build' / 'protos' + build_dir = root / 'build' + staging_root_dir = build_dir / 'protos' staging_dir = (staging_root_dir / 'azure_functions_worker' / 'protos') - build_dir = staging_dir / 'azure_functions_worker' / 'protos' + built_protos_dir = build_dir / 'built_protos' - if os.path.exists(build_dir): - shutil.rmtree(build_dir) + if os.path.exists(staging_root_dir): + shutil.rmtree(staging_root_dir) - shutil.copytree(proto_src_dir, build_dir) + if os.path.exists(built_protos_dir): + shutil.rmtree(built_protos_dir) - subprocess.run([ - sys.executable, '-m', 'grpc_tools.protoc', - '-I', os.sep.join(('azure_functions_worker', 'protos')), - '--python_out', str(staging_root_dir), - '--grpc_python_out', str(staging_root_dir), - os.sep.join(('azure_functions_worker', 'protos', - 'azure_functions_worker', 'protos', - 'FunctionRpc.proto')), - ], check=True, stdout=sys.stdout, stderr=sys.stderr, - cwd=staging_root_dir) + shutil.copytree(proto_src_dir, staging_dir) - compiled = glob.glob(str(staging_dir / '*.py')) + os.makedirs(built_protos_dir) - if not compiled: + protos = [ + os.sep.join(('shared', 'NullableTypes.proto')), + os.sep.join(('identity', 'ClaimsIdentityRpc.proto')), + 'FunctionRpc.proto' + ] + + for proto in protos: + subprocess.run([ + sys.executable, '-m', 'grpc_tools.protoc', + '-I', os.sep.join(('azure_functions_worker', 'protos')), + '--python_out', str(built_protos_dir), + '--grpc_python_out', str(built_protos_dir), + os.sep.join(('azure_functions_worker', 'protos', proto)), + ], check=True, stdout=sys.stdout, stderr=sys.stderr, + cwd=staging_root_dir) + + compiled_files = glob.glob( + str(built_protos_dir / '**' / '*.py'), + recursive=True) + + if not compiled_files: print('grpc_tools.protoc produced no Python files', file=sys.stderr) sys.exit(1) - for f in compiled: - shutil.copy(f, proto_root_dir) + # Needed to support absolute imports in files. See + # https://github.com/protocolbuffers/protobuf/issues/1491 + self.make_absolute_imports(compiled_files) + + dir_util.copy_tree(built_protos_dir, str(proto_root_dir)) + + def make_absolute_imports(self, compiled_files): + for compiled in compiled_files: + with open(compiled, 'r+') as f: + content = f.read() + f.seek(0) + # Convert lines of the form: + # import xxx_pb2 as xxx__pb2 to + # from azure_functions_worker.protos import xxx_pb2 as.. + p1 = re.sub( + r'\nimport (.*?_pb2)', + r'\nfrom azure_functions_worker.protos import \g<1>', + content) + # Convert lines of the form: + # from identity import xxx_pb2 as.. to + # from azure_functions_worker.protos.identity import xxx_pb2.. + p2 = re.sub( + r'from ([a-z]*) (import.*_pb2)', + r'from azure_functions_worker.protos.\g<1> \g<2>', + p1) + f.write(p2) + f.truncate() class build(build.build, BuildGRPC):