Skip to content

Commit 291a852

Browse files
gavin-aguiarvrdmr
andauthored
Added Retry Context for python worker (#909)
* Retry context initial commit * Added unit tests * Fixed spacing * Removed additional log * Fixed Lint check errors * Fixed line spacing errors * Refactored get_context * Reverting setup.py * Updated setup.py * Added doc links Co-authored-by: Varad Meru <[email protected]>
1 parent 51b47ed commit 291a852

File tree

11 files changed

+233
-23
lines changed

11 files changed

+233
-23
lines changed

azure_functions_worker/bindings/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Copyright (c) Microsoft Corporation. All rights reserved.
22
# Licensed under the MIT License.
33
from .tracecontext import TraceContext
4+
from .retrycontext import RetryContext
45
from .context import Context
56
from .meta import check_input_type_annotation
67
from .meta import check_output_type_annotation
@@ -16,6 +17,6 @@
1617
'is_trigger_binding',
1718
'check_input_type_annotation', 'check_output_type_annotation',
1819
'has_implicit_output',
19-
'from_incoming_proto', 'to_outgoing_proto', 'TraceContext',
20+
'from_incoming_proto', 'to_outgoing_proto', 'TraceContext', 'RetryContext',
2021
'to_outgoing_param_binding'
2122
)

azure_functions_worker/bindings/context.py

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,22 @@
11
# Copyright (c) Microsoft Corporation. All rights reserved.
22
# Licensed under the MIT License.
33
from . import TraceContext
4+
from . import RetryContext
45

56

67
class Context:
78

8-
def __init__(self, func_name: str, func_dir: str,
9-
invocation_id: str, trace_context: TraceContext) -> None:
9+
def __init__(self,
10+
func_name: str,
11+
func_dir: str,
12+
invocation_id: str,
13+
trace_context: TraceContext,
14+
retry_context: RetryContext) -> None:
1015
self.__func_name = func_name
1116
self.__func_dir = func_dir
1217
self.__invocation_id = invocation_id
1318
self.__trace_context = trace_context
19+
self.__retry_context = retry_context
1420

1521
@property
1622
def invocation_id(self) -> str:
@@ -27,3 +33,7 @@ def function_directory(self) -> str:
2733
@property
2834
def trace_context(self) -> TraceContext:
2935
return self.__trace_context
36+
37+
@property
38+
def retry_context(self) -> RetryContext:
39+
return self.__retry_context
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright (c) Microsoft Corporation. All rights reserved.
2+
# Licensed under the MIT License.
3+
4+
from . import rpcexception
5+
6+
7+
class RetryContext:
8+
"""Check https://docs.microsoft.com/en-us/azure/azure-functions/
9+
functions-bindings-error-pages?tabs=python#retry-policies-preview"""
10+
11+
def __init__(self,
12+
retry_count: int,
13+
max_retry_count: int,
14+
rpc_exception: rpcexception.RpcException) -> None:
15+
self.__retry_count = retry_count
16+
self.__max_retry_count = max_retry_count
17+
self.__rpc_exception = rpc_exception
18+
19+
@property
20+
def retry_count(self) -> int:
21+
"""Gets the current retry count from retry-context"""
22+
return self.__retry_count
23+
24+
@property
25+
def max_retry_count(self) -> int:
26+
"""Gets the max retry count from retry-context"""
27+
return self.__max_retry_count
28+
29+
@property
30+
def exception(self) -> rpcexception.RpcException:
31+
return self.__rpc_exception
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Copyright (c) Microsoft Corporation. All rights reserved.
2+
# Licensed under the MIT License.
3+
4+
5+
class RpcException:
6+
7+
def __init__(self,
8+
source: str,
9+
stack_trace: str,
10+
message: str) -> None:
11+
self.__source = source
12+
self.__stack_trace = stack_trace
13+
self.__message = message
14+
15+
@property
16+
def source(self) -> str:
17+
return self.__source
18+
19+
@property
20+
def stack_trace(self) -> str:
21+
return self.__stack_trace
22+
23+
@property
24+
def message(self) -> str:
25+
return self.__message

azure_functions_worker/dispatcher.py

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@
5050

5151

5252
class DispatcherMeta(type):
53-
5453
__current_dispatcher__ = None
5554

5655
@property
@@ -62,7 +61,6 @@ def current(mcls):
6261

6362

6463
class Dispatcher(metaclass=DispatcherMeta):
65-
6664
_GRPC_STOP_RESPONSE = object()
6765

6866
def __init__(self, loop: BaseEventLoop, host: str, port: int,
@@ -346,13 +344,9 @@ async def _handle__function_load_request(self, req):
346344

347345
async def _handle__invocation_request(self, req):
348346
invoc_request = req.invocation_request
349-
350347
invocation_id = invoc_request.invocation_id
351348
function_id = invoc_request.function_id
352-
trace_context = bindings.TraceContext(
353-
invoc_request.trace_context.trace_parent,
354-
invoc_request.trace_context.trace_state,
355-
invoc_request.trace_context.attributes)
349+
356350
# Set the current `invocation_id` to the current task so
357351
# that our logging handler can find it.
358352
current_task = _CURRENT_TASK(self._loop)
@@ -392,8 +386,7 @@ async def _handle__invocation_request(self, req):
392386
pytype=pb_type_info.pytype,
393387
shmem_mgr=self._shmem_mgr)
394388

395-
fi_context = bindings.Context(
396-
fi.name, fi.directory, invocation_id, trace_context)
389+
fi_context = self._get_context(invoc_request, fi.name, fi.directory)
397390
if fi.requires_context:
398391
args['context'] = fi_context
399392

@@ -470,7 +463,7 @@ async def _handle__function_environment_reload_request(self, req):
470463
# Import before clearing path cache so that the default
471464
# azure.functions modules is available in sys.modules for
472465
# customer use
473-
import azure.functions # NoQA
466+
import azure.functions # NoQA
474467

475468
# Append function project root to module finding sys.path
476469
if func_env_reload_request.function_app_directory:
@@ -556,6 +549,25 @@ async def _handle__close_shared_memory_resources_request(self, req):
556549
request_id=self.request_id,
557550
close_shared_memory_resources_response=response)
558551

552+
@staticmethod
553+
def _get_context(invoc_request: protos.InvocationRequest, name: str,
554+
directory: str) -> bindings.Context:
555+
""" For more information refer: https://aka.ms/azfunc-invocation-context
556+
"""
557+
trace_context = bindings.TraceContext(
558+
invoc_request.trace_context.trace_parent,
559+
invoc_request.trace_context.trace_state,
560+
invoc_request.trace_context.attributes)
561+
562+
retry_context = bindings.RetryContext(
563+
invoc_request.retry_context.retry_count,
564+
invoc_request.retry_context.max_retry_count,
565+
invoc_request.retry_context.exception)
566+
567+
return bindings.Context(
568+
name, directory, invoc_request.invocation_id,
569+
trace_context, retry_context)
570+
559571
@disable_feature_by(constants.PYTHON_ROLLBACK_CWD_PATH)
560572
def _change_cwd(self, new_cwd: str):
561573
if os.path.exists(new_cwd):
@@ -702,7 +714,6 @@ def emit(self, record: LogRecord) -> None:
702714

703715

704716
class ContextEnabledTask(asyncio.Task):
705-
706717
AZURE_INVOCATION_ID = '__azure_function_invocation_id__'
707718

708719
def __init__(self, coro, loop):

azure_functions_worker/protos/_src/src/proto/FunctionRpc.proto

Lines changed: 51 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,21 +75,25 @@ message StreamingMessage {
7575
// Ask the worker to close any open shared memory resources for a given invocation
7676
CloseSharedMemoryResourcesRequest close_shared_memory_resources_request = 27;
7777
CloseSharedMemoryResourcesResponse close_shared_memory_resources_response = 28;
78+
79+
// Worker indexing message types
80+
FunctionsMetadataRequest functions_metadata_request = 29;
81+
FunctionMetadataResponses function_metadata_responses = 30;
7882
}
7983
}
8084

8185
// Process.Start required info
8286
// connection details
8387
// protocol type
84-
// protocol version
88+
// protocol version
8589

8690
// Worker sends the host information identifying itself
8791
message StartStream {
8892
// id of the worker
8993
string worker_id = 2;
9094
}
9195

92-
// Host requests the worker to initialize itself
96+
// Host requests the worker to initialize itself
9397
message WorkerInitRequest {
9498
// version of the host sending init request
9599
string host_version = 1;
@@ -177,7 +181,7 @@ message WorkerActionResponse {
177181
Restart = 0;
178182
Reload = 1;
179183
}
180-
184+
181185
// action for this response
182186
Action action = 1;
183187

@@ -248,7 +252,7 @@ message RpcFunctionMetadata {
248252

249253
// base directory for the Function
250254
string directory = 1;
251-
255+
252256
// Script file specified
253257
string script_file = 2;
254258

@@ -260,6 +264,30 @@ message RpcFunctionMetadata {
260264

261265
// Is set to true for proxy
262266
bool is_proxy = 7;
267+
268+
// Function indexing status
269+
StatusResult status = 8;
270+
271+
// Function language
272+
string language = 9;
273+
274+
// Raw binding info
275+
repeated string raw_bindings = 10;
276+
}
277+
278+
// Host tells worker it is ready to receive metadata
279+
message FunctionsMetadataRequest {
280+
// base directory for function app
281+
string function_app_directory = 1;
282+
}
283+
284+
// Worker sends function metadata back to host
285+
message FunctionMetadataResponses {
286+
// list of function indexing responses
287+
repeated FunctionLoadRequest function_load_requests_results = 1;
288+
289+
// status of overall metadata request
290+
StatusResult result = 2;
263291
}
264292

265293
// Host requests worker to invoke a Function
@@ -278,6 +306,9 @@ message InvocationRequest {
278306

279307
// Populates activityId, tracestate and tags from host
280308
RpcTraceContext trace_context = 5;
309+
310+
// Current retry context
311+
RetryContext retry_context = 6;
281312
}
282313

283314
// Host sends ActivityId, traceStateString and Tags from host
@@ -292,6 +323,18 @@ message RpcTraceContext {
292323
map<string, string> attributes = 3;
293324
}
294325

326+
// Host sends retry context for a function invocation
327+
message RetryContext {
328+
// Current retry count
329+
int32 retry_count = 1;
330+
331+
// Max retry count
332+
int32 max_retry_count = 2;
333+
334+
// Exception that caused the retry
335+
RpcException exception = 3;
336+
}
337+
295338
// Host requests worker to cancel invocation
296339
message InvocationCancel {
297340
// Unique id for invocation
@@ -421,7 +464,7 @@ message BindingInfo {
421464
DataType data_type = 4;
422465
}
423466

424-
// Used to send logs back to the Host
467+
// Used to send logs back to the Host
425468
message RpcLog {
426469
// Matching ILogger semantics
427470
// https://github.com/aspnet/Logging/blob/9506ccc3f3491488fe88010ef8b9eb64594abf95/src/Microsoft.Extensions.Logging/Logger.cs
@@ -468,7 +511,7 @@ message RpcLog {
468511
RpcLogCategory log_category = 8;
469512
}
470513

471-
// Encapsulates an Exception
514+
// Encapsulates an Exception
472515
message RpcException {
473516
// Source of the exception
474517
string source = 3;
@@ -522,7 +565,7 @@ message RpcHttpCookie {
522565
// TODO - solidify this or remove it
523566
message RpcHttp {
524567
string method = 1;
525-
string url = 2;
568+
string url = 2;
526569
map<string,string> headers = 3;
527570
TypedData body = 4;
528571
map<string,string> params = 10;
@@ -535,4 +578,4 @@ message RpcHttp {
535578
map<string,NullableString> nullable_headers = 20;
536579
map<string,NullableString> nullable_params = 21;
537580
map<string,NullableString> nullable_query = 22;
538-
}
581+
}
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"scriptFile": "main.py",
3+
"bindings": [
4+
{
5+
"type": "httpTrigger",
6+
"direction": "in",
7+
"name": "req"
8+
},
9+
{
10+
"type": "http",
11+
"direction": "out",
12+
"name": "$return"
13+
}
14+
],
15+
"retry": {
16+
"strategy": "exponentialBackoff",
17+
"maxRetryCount": 3,
18+
"minimumInterval": "00:00:01",
19+
"maximumInterval": "00:00:05"
20+
}
21+
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright (c) Microsoft Corporation. All rights reserved.
2+
# Licensed under the MIT License.
3+
import azure.functions
4+
import logging
5+
6+
logger = logging.getLogger('my function')
7+
8+
9+
def main(req: azure.functions.HttpRequest, context: azure.functions.Context):
10+
logger.info(f'Current retry count: {context.retry_context.retry_count}')
11+
logger.info(f'Max retry count: {context.retry_context.max_retry_count}')
12+
13+
raise Exception("Testing retries")
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"scriptFile": "main.py",
3+
"bindings": [
4+
{
5+
"type": "httpTrigger",
6+
"direction": "in",
7+
"name": "req"
8+
},
9+
{
10+
"type": "http",
11+
"direction": "out",
12+
"name": "$return"
13+
}
14+
],
15+
"retry": {
16+
"strategy": "fixedDelay",
17+
"maxRetryCount": 3,
18+
"delayInterval": "00:00:01"
19+
}
20+
}

0 commit comments

Comments
 (0)