Python Worker Extension Interface (sdk) (#81)

* Added interface for extension registration

Fix nit

Ensure extension can be loaded successfully

Add unittests

Modify documentations

Change trace context attribute names

Fix wrongly declared abstract method

Accept

Revert changes in Context

Fix unittests

* Remove FuncExtensionInitError

* Add back metaclass

* Snapshot

* Add extension class

* Clean up repo

* Fix Python 36 mypy syntax

* Remove logger from function load hook

* Update unit test cases

* Rename hook as discussed offline

* Fix comment

* Change exception name

* Demonstrate trigger name in function extension is in script root

* Fix nits

* Refactor func extension and app extension generation

* Refactor NewAppExtension code

Author: Hazhzeng

azure/functions/__init__.py

@@ -13,6 +13,8 @@
 from ._servicebus import ServiceBusMessage
 from ._durable_functions import OrchestrationContext, EntityContext
 from .meta import get_binding_registry
+from .extension import (ExtensionMeta, FunctionExtensionException,
+                        FuncExtensionBase, AppExtensionBase)
 
 # Import binding implementations to register them
 from . import blob  # NoQA
@@ -54,7 +56,13 @@
     'TimerRequest',
 
     # Middlewares
-    'WsgiMiddleware'
+    'WsgiMiddleware',
+
+    # Extensions
+    'AppExtensionBase',
+    'FuncExtensionBase',
+    'ExtensionMeta',
+    'FunctionExtensionException'
 )
 
 __version__ = '1.6.0'

azure/functions/extension/__init__.py

@@ -0,0 +1,11 @@
+from .extension_meta import ExtensionMeta
+from .function_extension_exception import FunctionExtensionException
+from .app_extension_base import AppExtensionBase
+from .func_extension_base import FuncExtensionBase
+
+__all__ = [
+    'ExtensionMeta',
+    'FunctionExtensionException',
+    'AppExtensionBase',
+    'FuncExtensionBase'
+]

azure/functions/extension/app_extension_base.py

@@ -0,0 +1,122 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import typing
+from logging import Logger
+from .extension_meta import ExtensionMeta
+from .extension_scope import ExtensionScope
+from .._abc import Context
+
+
+class AppExtensionBase(metaclass=ExtensionMeta):
+    """An abstract class defines the global life-cycle hooks to be implemented
+    by customer's extension, will be applied to all functions.
+
+    An AppExtension should be treated as a static class. Must not contain
+    __init__ method since it is not instantiable.
+
+    Please place your initialization code in init() classmethod, consider
+    accepting extension settings in configure() classmethod from customers.
+    """
+
+    _scope = ExtensionScope.APPLICATION
+
+    @classmethod
+    def init(cls):
+        """The function will be executed when the extension is loaded.
+        Happens when Azure Functions customers import the extension module.
+        """
+        pass
+
+    @classmethod
+    def configure(cls, *args, **kwargs):
+        """This function is intended to be called by Azure Functions
+        customers. This is a contract between extension developers and
+        azure functions customers. If multiple .configure() are called,
+        the extension system cannot guarentee the calling order.
+        """
+        pass
+
+    # DO NOT decorate this with @abc.abstractstatismethod
+    # since implementation by subclass is not mandatory
+    @classmethod
+    def post_function_load_app_level(cls,
+                                     function_name: str,
+                                     function_directory: str,
+                                     *args, **kwargs) -> None:
+        """This must be implemented as a @classmethod. It will be called right
+        a customer's function is loaded. In this stage, the customer's logger
+        is not fully initialized from the Python worker. Please use print()
+        to emit message if necessary.
+
+        Parameters
+        ----------
+        function_name: str
+            The name of customer's function (e.g. HttpTrigger)
+        function_directory: str
+            The path to customer's function directory
+            (e.g. /home/site/wwwroot/HttpTrigger)
+        """
+        pass
+
+    # DO NOT decorate this with @abc.abstractstatismethod
+    # since implementation by subclass is not mandatory
+    @classmethod
+    def pre_invocation_app_level(cls,
+                                 logger: Logger,
+                                 context: Context,
+                                 func_args: typing.Dict[str, object] = {},
+                                 *args,
+                                 **kwargs) -> None:
+        """This must be implemented as a @staticmethod. It will be called right
+        before a customer's function is being executed.
+
+        Parameters
+        ----------
+        logger: logging.Logger
+            A logger provided by Python worker. Extension developer should
+            use this logger to emit telemetry to Azure Functions customers.
+        context: azure.functions.Context
+            This will include the function_name, function_directory and an
+            invocation_id of this specific invocation.
+        func_args: typing.Dict[str, object]
+            Arguments that are passed into the Azure Functions. The name of
+            each parameter is defined in function.json. Extension developers
+            may also want to do isinstance() check if you want to apply
+            operations to specific trigger types or input binding types.
+        """
+        pass
+
+    # DO NOT decorate this with @abc.abstractstatismethod
+    # since implementation by subclass is not mandatory
+    @classmethod
+    def post_invocation_app_level(cls,
+                                  logger: Logger,
+                                  context: Context,
+                                  func_args: typing.Dict[str, object] = {},
+                                  func_ret: typing.Optional[object] = None,
+                                  *args,
+                                  **kwargs) -> None:
+        """This must be implemented as a @staticmethod. It will be called right
+        before a customer's function is being executed.
+
+        Parameters
+        ----------
+        logger: logging.Logger
+            A logger provided by Python worker. Extension developer should
+            use this logger to emit telemetry to Azure Functions customers.
+        context: azure.functions.Context
+            This will include the function_name, function_directory and an
+            invocation_id of this specific invocation.
+        func_args: typing.Dict[str, object]
+            Arguments that are passed into the Azure Functions. The name of
+            each parameter is defined in function.json. Extension developers
+            may also want to do isinstance() check if you want to apply
+            operations to specific trigger types or input binding types.
+        func_ret: typing.Optional[object]
+            Return value from Azure Functions. This is usually the value
+            defined in function.json $return section. Extension developers
+            may also want to do isinstance() check if you want to apply
+            operations to specific types or input binding types."
+        """
+        pass

azure/functions/extension/app_extension_hooks.py

@@ -0,0 +1,16 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from typing import NamedTuple, List
+from .extension_hook_meta import ExtensionHookMeta
+
+
+class AppExtensionHooks(NamedTuple):
+    """The definition of which type of global hooks are supported in SDK.
+    ExtensionMeta will lookup the AppExtension life-cycle type from here.
+    """
+    # The default value ([] empty list) is not being set here intentionally
+    # since it is impacted by a Python bug https://bugs.python.org/issue33077.
+    post_function_load_app_level: List[ExtensionHookMeta]
+    pre_invocation_app_level: List[ExtensionHookMeta]
+    post_invocation_app_level: List[ExtensionHookMeta]

azure/functions/extension/extension_hook_meta.py

@@ -0,0 +1,16 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from typing import Callable, NamedTuple
+
+
+class ExtensionHookMeta(NamedTuple):
+    """The metadata of a single life-cycle hook.
+    The ext_name has the class name of an extension class.
+    The ext_impl has the callable function that is used by the worker.
+    """
+    ext_name: str
+    ext_impl: Callable
+
+    # When adding more fields, make sure they have default values (e.g.
+    # ext_new_field: Optional[str] = None

azure/functions/extension/extension_meta.py

@@ -0,0 +1,202 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from typing import Optional, Union, Dict, List
+import abc
+import json
+from .app_extension_hooks import AppExtensionHooks
+from .func_extension_hooks import FuncExtensionHooks
+from .extension_hook_meta import ExtensionHookMeta
+from .extension_scope import ExtensionScope
+from .function_extension_exception import FunctionExtensionException
+
+
+class ExtensionMeta(abc.ABCMeta):
+    """The metaclass handles extension registration.
+
+    AppExtension is regsistered in __init__, it is applied to all triggers.
+    FuncExtension is registered in __call__, as users need to instantiate it
+    inside hook script.
+
+    After registration, the extension class will be flatten into the following
+    structure to speed up worker lookup:
+        _func_exts[<trigger_name>].<hook_name>.(ext_name, ext_impl)
+        (e.g. _func_exts['HttpTrigger'].pre_invocation.ext_impl)
+
+        _app_exts.<hook_name>.(ext_name, ext_impl)
+        (e.g. _app_exts.pre_invocation_app_level.ext_impl)
+
+    The extension tree information is stored in _info for diagnostic
+    purpose. The dictionary is serializible to json:
+        _info['FuncExtension']['<Trigger>'] = list(<Extension>)
+        _info['AppExtension'] = list(<Extension>)
+    """
+    _func_exts: Dict[str, FuncExtensionHooks] = {}
+    _app_exts: Optional[AppExtensionHooks] = None
+    _info: Dict[str, Union[Dict[str, List[str]], List[str]]] = {}
+
+    def __init__(cls, *args, **kwargs):
+        """Executes on 'import extension', once the AppExtension class is
+        loaded, call the setup() method and add the life-cycle hooks into
+        _app_exts.
+        """
+        super(ExtensionMeta, cls).__init__(*args, **kwargs)
+        scope = ExtensionMeta._get_extension_scope(cls)
+
+        # Only register application extension here
+        if scope is ExtensionScope.APPLICATION:
+            ExtensionMeta._register_application_extension(cls)
+
+    def __call__(cls, *args, **kwargs):
+        """Executes on 'inst = extension(__file__)', once the FuncExtension
+        class is instantiate, overwrite the __init__() method and add the
+        instance into life-cycle hooks.
+        """
+        scope = ExtensionMeta._get_extension_scope(cls)
+
+        # Only register function extension here
+        if scope is ExtensionScope.FUNCTION:
+            instance = super(ExtensionMeta, cls).__call__(*args, **kwargs)
+            ExtensionMeta._register_function_extension(instance)
+            return instance
+        elif scope is ExtensionScope.APPLICATION:
+            raise FunctionExtensionException(
+                f'Python worker extension with scope:{scope} should not be'
+                'instantiable. Please access via class method directly.'
+            )
+        else:
+            raise FunctionExtensionException(
+                f'Python worker extension:{cls.__name__} is not properly '
+                'implemented from AppExtensionBase or FuncExtensionBase.'
+            )
+
+    @classmethod
+    def get_function_hooks(cls, name: str) -> Optional[FuncExtensionHooks]:
+        """Return all function extension hooks indexed by trigger name.
+
+        Returns
+        -------
+        Optional[FuncExtensionHooks]:
+            Example to look up a certain life-cycle name:
+            get_function_hooks('HttpTrigger').pre_invocation.ext_name
+        """
+        return cls._func_exts.get(name.lower())
+
+    @classmethod
+    def get_application_hooks(cls) -> Optional[AppExtensionHooks]:
+        """Return all application hooks
+
+        Returns
+        -------
+        Optional[AppExtensionHooks]:
+            Example to look up a certain life-cycle name:
+            get_application_hooks().pre_invocation_app_level.ext_name
+        """
+        return cls._app_exts
+
+    @classmethod
+    def get_registered_extensions_json(cls) -> str:
+        """Return a json string of the registered
+
+        Returns
+        -------
+        str:
+            The json string will be constructed in a structure of
+            {
+                "FuncExtension": {
+                    "<TriggerA>": [
+                        "ExtensionName"
+                    ]
+                },
+                "AppExtension": [
+                    "ExtensionName"
+                ]
+            }
+        """
+        return json.dumps(cls._info)
+
+    @classmethod
+    def _get_extension_scope(cls, extension) -> ExtensionScope:
+        """Return the scope of an extension"""
+        return getattr(extension, '_scope',  # type: ignore
+                       ExtensionScope.UNKNOWN)
+
+    @classmethod
+    def _set_hooks_for_function(cls, trigger_name: str, ext):
+        ext_hooks = cls._func_exts.setdefault(
+            trigger_name.lower(),
+            cls._create_default_function_hook()
+        )
+
+        # Flatten extension class to cls._func_exts
+        for hook_name in ext_hooks._fields:
+            hook_impl = getattr(ext, hook_name, None)
+            if hook_impl is not None:
+                hook_meta = ExtensionHookMeta(
+                    ext_name=ext.__class__.__name__,
+                    ext_impl=hook_impl,
+                )
+                getattr(ext_hooks, hook_name).append(hook_meta)
+
+    @classmethod
+    def _set_hooks_for_application(cls, ext):
+        if cls._app_exts is None:
+            cls._app_exts = cls._create_default_app_hook()
+
+        # Check for definition in AppExtensionHooks NamedTuple
+        for hook_name in cls._app_exts._fields:
+            hook_impl = getattr(ext, hook_name, None)
+            if hook_impl is not None:
+                getattr(cls._app_exts, hook_name).append(ExtensionHookMeta(
+                    ext_name=ext.__name__,
+                    ext_impl=hook_impl
+                ))
+
+    @classmethod
+    def _register_function_extension(cls, extension):
+        """Flatten the function extension structure into function hooks"""
+        # Should skip registering FuncExtensionBase, cannot use isinstance(),
+        # referring to func_extension_hooks introduces a dependency cycle
+        if extension.__class__.__name__ == 'FuncExtensionBase':
+            return
+
+        trigger_name = extension._trigger_name
+        cls._set_hooks_for_function(trigger_name, extension)
+
+        # Record function extension information
+        hooks_info = cls._info.setdefault(  # type: ignore
+            'FuncExtension', {}).setdefault(trigger_name, [])
+        hooks_info.append(extension.__class__.__name__)
+
+    @classmethod
+    def _register_application_extension(cls, extension):
+        """Flatten the application extension structure into function hooks"""
+        # Should skip registering AppExtensionBase, cannot use isinstance(),
+        # referring to app_extension_hooks introduces a dependency cycle
+        if extension.__name__ == 'AppExtensionBase':
+            return
+
+        if getattr(extension, 'init', None):
+            extension.init()
+
+        cls._set_hooks_for_application(extension)
+
+        # Record application extension information
+        hooks_info = cls._info.setdefault('AppExtension', [])
+        hooks_info.append(extension.__name__)   # type: ignore
+
+    @classmethod
+    def _create_default_function_hook(cls) -> FuncExtensionHooks:
+        return FuncExtensionHooks(
+            post_function_load=[],
+            pre_invocation=[],
+            post_invocation=[]
+        )
+
+    @classmethod
+    def _create_default_app_hook(cls) -> AppExtensionHooks:
+        return AppExtensionHooks(
+            post_function_load_app_level=[],
+            pre_invocation_app_level=[],
+            post_invocation_app_level=[]
+        )