Skip to content

Python Worker Extension Interface (sdk) #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Mar 19, 2021
Merged
10 changes: 9 additions & 1 deletion azure/functions/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
from ._servicebus import ServiceBusMessage
from ._durable_functions import OrchestrationContext, EntityContext
from .meta import get_binding_registry
from .extension import (ExtensionMeta, FunctionExtensionException,
FuncExtensionBase, AppExtensionBase)

# Import binding implementations to register them
from . import blob # NoQA
Expand Down Expand Up @@ -54,7 +56,13 @@
'TimerRequest',

# Middlewares
'WsgiMiddleware'
'WsgiMiddleware',

# Extensions
'AppExtensionBase',
'FuncExtensionBase',
'ExtensionMeta',
'FunctionExtensionException'
)

__version__ = '1.6.0'
11 changes: 11 additions & 0 deletions azure/functions/extension/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
from .extension_meta import ExtensionMeta
from .function_extension_exception import FunctionExtensionException
from .app_extension_base import AppExtensionBase
from .func_extension_base import FuncExtensionBase

__all__ = [
'ExtensionMeta',
'FunctionExtensionException',
'AppExtensionBase',
'FuncExtensionBase'
]
122 changes: 122 additions & 0 deletions azure/functions/extension/app_extension_base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import typing
from logging import Logger
from .extension_meta import ExtensionMeta
from .extension_scope import ExtensionScope
from .._abc import Context


class AppExtensionBase(metaclass=ExtensionMeta):
"""An abstract class defines the global life-cycle hooks to be implemented
by customer's extension, will be applied to all functions.

An AppExtension should be treated as a static class. Must not contain
__init__ method since it is not instantiable.

Please place your initialization code in init() classmethod, consider
accepting extension settings in configure() classmethod from customers.
"""

_scope = ExtensionScope.APPLICATION

@classmethod
def init(cls):
"""The function will be executed when the extension is loaded.
Happens when Azure Functions customers import the extension module.
"""
pass

@classmethod
def configure(cls, *args, **kwargs):
"""This function is intended to be called by Azure Functions
customers. This is a contract between extension developers and
azure functions customers. If multiple .configure() are called,
the extension system cannot guarentee the calling order.
"""
pass

# DO NOT decorate this with @abc.abstractstatismethod
# since implementation by subclass is not mandatory
@classmethod
def post_function_load_app_level(cls,
function_name: str,
function_directory: str,
*args, **kwargs) -> None:
"""This must be implemented as a @classmethod. It will be called right
a customer's function is loaded. In this stage, the customer's logger
is not fully initialized from the Python worker. Please use print()
to emit message if necessary.

Parameters
----------
function_name: str
The name of customer's function (e.g. HttpTrigger)
function_directory: str
The path to customer's function directory
(e.g. /home/site/wwwroot/HttpTrigger)
"""
pass

# DO NOT decorate this with @abc.abstractstatismethod
# since implementation by subclass is not mandatory
@classmethod
def pre_invocation_app_level(cls,
logger: Logger,
context: Context,
func_args: typing.Dict[str, object] = {},
*args,
**kwargs) -> None:
"""This must be implemented as a @staticmethod. It will be called right
before a customer's function is being executed.

Parameters
----------
logger: logging.Logger
A logger provided by Python worker. Extension developer should
use this logger to emit telemetry to Azure Functions customers.
context: azure.functions.Context
This will include the function_name, function_directory and an
invocation_id of this specific invocation.
func_args: typing.Dict[str, object]
Arguments that are passed into the Azure Functions. The name of
each parameter is defined in function.json. Extension developers
may also want to do isinstance() check if you want to apply
operations to specific trigger types or input binding types.
"""
pass

# DO NOT decorate this with @abc.abstractstatismethod
# since implementation by subclass is not mandatory
@classmethod
def post_invocation_app_level(cls,
logger: Logger,
context: Context,
func_args: typing.Dict[str, object] = {},
func_ret: typing.Optional[object] = None,
*args,
**kwargs) -> None:
"""This must be implemented as a @staticmethod. It will be called right
before a customer's function is being executed.

Parameters
----------
logger: logging.Logger
A logger provided by Python worker. Extension developer should
use this logger to emit telemetry to Azure Functions customers.
context: azure.functions.Context
This will include the function_name, function_directory and an
invocation_id of this specific invocation.
func_args: typing.Dict[str, object]
Arguments that are passed into the Azure Functions. The name of
each parameter is defined in function.json. Extension developers
may also want to do isinstance() check if you want to apply
operations to specific trigger types or input binding types.
func_ret: typing.Optional[object]
Return value from Azure Functions. This is usually the value
defined in function.json $return section. Extension developers
may also want to do isinstance() check if you want to apply
operations to specific types or input binding types."
"""
pass
16 changes: 16 additions & 0 deletions azure/functions/extension/app_extension_hooks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

from typing import NamedTuple, List
from .extension_hook_meta import ExtensionHookMeta


class AppExtensionHooks(NamedTuple):
"""The definition of which type of global hooks are supported in SDK.
ExtensionMeta will lookup the AppExtension life-cycle type from here.
"""
# The default value ([] empty list) is not being set here intentionally
# since it is impacted by a Python bug https://bugs.python.org/issue33077.
post_function_load_app_level: List[ExtensionHookMeta]
pre_invocation_app_level: List[ExtensionHookMeta]
post_invocation_app_level: List[ExtensionHookMeta]
16 changes: 16 additions & 0 deletions azure/functions/extension/extension_hook_meta.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

from typing import Callable, NamedTuple


class ExtensionHookMeta(NamedTuple):
"""The metadata of a single life-cycle hook.
The ext_name has the class name of an extension class.
The ext_impl has the callable function that is used by the worker.
"""
ext_name: str
ext_impl: Callable

# When adding more fields, make sure they have default values (e.g.
# ext_new_field: Optional[str] = None
202 changes: 202 additions & 0 deletions azure/functions/extension/extension_meta.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

from typing import Optional, Union, Dict, List
import abc
import json
from .app_extension_hooks import AppExtensionHooks
from .func_extension_hooks import FuncExtensionHooks
from .extension_hook_meta import ExtensionHookMeta
from .extension_scope import ExtensionScope
from .function_extension_exception import FunctionExtensionException


class ExtensionMeta(abc.ABCMeta):
"""The metaclass handles extension registration.

AppExtension is regsistered in __init__, it is applied to all triggers.
FuncExtension is registered in __call__, as users need to instantiate it
inside hook script.

After registration, the extension class will be flatten into the following
structure to speed up worker lookup:
_func_exts[<trigger_name>].<hook_name>.(ext_name, ext_impl)
(e.g. _func_exts['HttpTrigger'].pre_invocation.ext_impl)

_app_exts.<hook_name>.(ext_name, ext_impl)
(e.g. _app_exts.pre_invocation_app_level.ext_impl)

The extension tree information is stored in _info for diagnostic
purpose. The dictionary is serializible to json:
_info['FuncExtension']['<Trigger>'] = list(<Extension>)
_info['AppExtension'] = list(<Extension>)
"""
_func_exts: Dict[str, FuncExtensionHooks] = {}
_app_exts: Optional[AppExtensionHooks] = None
_info: Dict[str, Union[Dict[str, List[str]], List[str]]] = {}

def __init__(cls, *args, **kwargs):
"""Executes on 'import extension', once the AppExtension class is
loaded, call the setup() method and add the life-cycle hooks into
_app_exts.
"""
super(ExtensionMeta, cls).__init__(*args, **kwargs)
scope = ExtensionMeta._get_extension_scope(cls)

# Only register application extension here
if scope is ExtensionScope.APPLICATION:
ExtensionMeta._register_application_extension(cls)

def __call__(cls, *args, **kwargs):
"""Executes on 'inst = extension(__file__)', once the FuncExtension
class is instantiate, overwrite the __init__() method and add the
instance into life-cycle hooks.
"""
scope = ExtensionMeta._get_extension_scope(cls)

# Only register function extension here
if scope is ExtensionScope.FUNCTION:
instance = super(ExtensionMeta, cls).__call__(*args, **kwargs)
ExtensionMeta._register_function_extension(instance)
return instance
elif scope is ExtensionScope.APPLICATION:
raise FunctionExtensionException(
f'Python worker extension with scope:{scope} should not be'
'instantiable. Please access via class method directly.'
)
else:
raise FunctionExtensionException(
f'Python worker extension:{cls.__name__} is not properly '
'implemented from AppExtensionBase or FuncExtensionBase.'
)

@classmethod
def get_function_hooks(cls, name: str) -> Optional[FuncExtensionHooks]:
"""Return all function extension hooks indexed by trigger name.

Returns
-------
Optional[FuncExtensionHooks]:
Example to look up a certain life-cycle name:
get_function_hooks('HttpTrigger').pre_invocation.ext_name
"""
return cls._func_exts.get(name.lower())

@classmethod
def get_application_hooks(cls) -> Optional[AppExtensionHooks]:
"""Return all application hooks

Returns
-------
Optional[AppExtensionHooks]:
Example to look up a certain life-cycle name:
get_application_hooks().pre_invocation_app_level.ext_name
"""
return cls._app_exts

@classmethod
def get_registered_extensions_json(cls) -> str:
"""Return a json string of the registered

Returns
-------
str:
The json string will be constructed in a structure of
{
"FuncExtension": {
"<TriggerA>": [
"ExtensionName"
]
},
"AppExtension": [
"ExtensionName"
]
}
"""
return json.dumps(cls._info)

@classmethod
def _get_extension_scope(cls, extension) -> ExtensionScope:
"""Return the scope of an extension"""
return getattr(extension, '_scope', # type: ignore
ExtensionScope.UNKNOWN)

@classmethod
def _set_hooks_for_function(cls, trigger_name: str, ext):
ext_hooks = cls._func_exts.setdefault(
trigger_name.lower(),
cls._create_default_function_hook()
)

# Flatten extension class to cls._func_exts
for hook_name in ext_hooks._fields:
hook_impl = getattr(ext, hook_name, None)
if hook_impl is not None:
hook_meta = ExtensionHookMeta(
ext_name=ext.__class__.__name__,
ext_impl=hook_impl,
)
getattr(ext_hooks, hook_name).append(hook_meta)

@classmethod
def _set_hooks_for_application(cls, ext):
if cls._app_exts is None:
cls._app_exts = cls._create_default_app_hook()

# Check for definition in AppExtensionHooks NamedTuple
for hook_name in cls._app_exts._fields:
hook_impl = getattr(ext, hook_name, None)
if hook_impl is not None:
getattr(cls._app_exts, hook_name).append(ExtensionHookMeta(
ext_name=ext.__name__,
ext_impl=hook_impl
))

@classmethod
def _register_function_extension(cls, extension):
"""Flatten the function extension structure into function hooks"""
# Should skip registering FuncExtensionBase, cannot use isinstance(),
# referring to func_extension_hooks introduces a dependency cycle
if extension.__class__.__name__ == 'FuncExtensionBase':
return

trigger_name = extension._trigger_name
cls._set_hooks_for_function(trigger_name, extension)

# Record function extension information
hooks_info = cls._info.setdefault( # type: ignore
'FuncExtension', {}).setdefault(trigger_name, [])
hooks_info.append(extension.__class__.__name__)

@classmethod
def _register_application_extension(cls, extension):
"""Flatten the application extension structure into function hooks"""
# Should skip registering AppExtensionBase, cannot use isinstance(),
# referring to app_extension_hooks introduces a dependency cycle
if extension.__name__ == 'AppExtensionBase':
return

if getattr(extension, 'init', None):
extension.init()

cls._set_hooks_for_application(extension)

# Record application extension information
hooks_info = cls._info.setdefault('AppExtension', [])
hooks_info.append(extension.__name__) # type: ignore

@classmethod
def _create_default_function_hook(cls) -> FuncExtensionHooks:
return FuncExtensionHooks(
post_function_load=[],
pre_invocation=[],
post_invocation=[]
)

@classmethod
def _create_default_app_hook(cls) -> AppExtensionHooks:
return AppExtensionHooks(
post_function_load_app_level=[],
pre_invocation_app_level=[],
post_invocation_app_level=[]
)
Loading