Implemented capability to isolate worker dependencies (#794)

Author: Hazhzeng

azure_functions_worker/constants.py

@@ -13,14 +13,20 @@
 # Debug Flags
 PYAZURE_WEBHOST_DEBUG = "PYAZURE_WEBHOST_DEBUG"
 
+# Platform Environment Variables
+AZURE_WEBJOBS_SCRIPT_ROOT = "AzureWebJobsScriptRoot"
+
 # Python Specific Feature Flags and App Settings
 PYTHON_ROLLBACK_CWD_PATH = "PYTHON_ROLLBACK_CWD_PATH"
 PYTHON_THREADPOOL_THREAD_COUNT = "PYTHON_THREADPOOL_THREAD_COUNT"
+PYTHON_ISOLATE_WORKER_DEPENDENCIES = "PYTHON_ISOLATE_WORKER_DEPENDENCIES"
 
 # Setting Defaults
 PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT = 1
 PYTHON_THREADPOOL_THREAD_COUNT_MIN = 1
 PYTHON_THREADPOOL_THREAD_COUNT_MAX = 32
+PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT = False
+PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 = True
 
 # External Site URLs
 MODULE_NOT_FOUND_TS_URL = "https://aka.ms/functions-modulenotfound"

azure_functions_worker/dispatcher.py

@@ -7,8 +7,6 @@
 
 import asyncio
 import concurrent.futures
-import importlib
-import inspect
 import logging
 import os
 import queue
@@ -33,6 +31,7 @@
 from .logging import error_logger, is_system_log_category, logger
 from .utils.common import get_app_setting
 from .utils.tracing import marshall_exception_trace
+from .utils.dependency import DependencyManager
 from .utils.wrappers import disable_feature_by
 
 _TRUE = "true"
@@ -264,6 +263,9 @@ async def _handle__worker_init_request(self, req):
             constants.RPC_HTTP_TRIGGER_METADATA_REMOVED: _TRUE,
         }
 
+        # Can detech worker packages
+        DependencyManager.use_customer_dependencies()
+
         return protos.StreamingMessage(
             request_id=self.request_id,
             worker_init_response=protos.WorkerInitResponse(
@@ -450,26 +452,10 @@ async def _handle__function_environment_reload_request(self, req):
                 self._create_sync_call_tp(self._get_sync_tp_max_workers())
             )
 
-            # Reload package namespaces for customer's libraries
-            packages_to_reload = ['azure', 'google']
-            for p in packages_to_reload:
-                try:
-                    logger.info(f'Reloading {p} module')
-                    importlib.reload(sys.modules[p])
-                except Exception as ex:
-                    logger.info('Unable to reload {}: \n{}'.format(p, ex))
-                logger.info(f'Reloaded {p} module')
-
-            # Reload azure.functions to give user package precedence
-            logger.info('Reloading azure.functions module at %s',
-                        inspect.getfile(sys.modules['azure.functions']))
-            try:
-                importlib.reload(sys.modules['azure.functions'])
-                logger.info('Reloaded azure.functions module now at %s',
-                            inspect.getfile(sys.modules['azure.functions']))
-            except Exception as ex:
-                logger.info('Unable to reload azure.functions. '
-                            'Using default. Exception:\n{}'.format(ex))
+            # Reload azure google namespaces
+            DependencyManager.reload_azure_google_namespace(
+                func_env_reload_request.function_app_directory
+            )
 
             # Change function app directory
             if getattr(func_env_reload_request,

azure_functions_worker/main.py

@@ -5,11 +5,6 @@
 
 import argparse
 
-from . import dispatcher
-from . import logging
-from ._thirdparty import aio_compat
-from .logging import error_logger, logger
-
 
 def parse_args():
     parser = argparse.ArgumentParser(
@@ -36,6 +31,14 @@ def parse_args():
 
 
 def main():
+    from .utils.dependency import DependencyManager
+    DependencyManager.initialize()
+    DependencyManager.use_worker_dependencies()
+
+    from . import logging
+    from ._thirdparty import aio_compat
+    from .logging import error_logger, logger
+
     args = parse_args()
     logging.setup(log_level=args.log_level, log_destination=args.log_to)
 
@@ -52,6 +55,8 @@ def main():
 
 
 async def start_async(host, port, worker_id, request_id):
+    from . import dispatcher
+
     disp = await dispatcher.Dispatcher.connect(host=host, port=port,
                                                worker_id=worker_id,
                                                request_id=request_id,

azure_functions_worker/testutils.py

@@ -29,7 +29,6 @@
 import unittest
 import uuid
 
-import asynctest as asynctest
 import grpc
 import requests
 
@@ -123,7 +122,7 @@ def wrapper(*args, **kwargs):
         return wrapper
 
 
-class AsyncTestCase(asynctest.TestCase, metaclass=AsyncTestCaseMeta):
+class AsyncTestCase(unittest.TestCase, metaclass=AsyncTestCaseMeta):
     pass
 
 

azure_functions_worker/utils/common.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 from typing import Optional, Callable
 import os
+import sys
 
 
 def is_true_like(setting: str) -> bool:
@@ -11,13 +12,32 @@ def is_true_like(setting: str) -> bool:
     return setting.lower().strip() in ['1', 'true', 't', 'yes', 'y']
 
 
+def is_false_like(setting: str) -> bool:
+    if setting is None:
+        return False
+
+    return setting.lower().strip() in ['0', 'false', 'f', 'no', 'n']
+
+
 def is_envvar_true(env_key: str) -> bool:
     if os.getenv(env_key) is None:
         return False
 
     return is_true_like(os.environ[env_key])
 
 
+def is_envvar_false(env_key: str) -> bool:
+    if os.getenv(env_key) is None:
+        return False
+
+    return is_false_like(os.environ[env_key])
+
+
+def is_python_version(version: str) -> bool:
+    current_version = f'{sys.version_info.major}.{sys.version_info.minor}'
+    return current_version == version
+
+
 def get_app_setting(
     setting: str,
     default_value: Optional[str] = None,