From 6bb4e9e37b341e5a21313013cdd006f2c4b4b0c5 Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Mon, 14 Dec 2020 01:57:02 -0800 Subject: [PATCH 1/7] Implemented capability to isolate worker dependencies --- .github/workflows/ut_ci_workflow.yml | 4 +- azure_functions_worker/constants.py | 5 + azure_functions_worker/dispatcher.py | 30 +- azure_functions_worker/main.py | 15 +- azure_functions_worker/testutils.py | 3 +- azure_functions_worker/utils/common.py | 14 + azure_functions_worker/utils/dependency.py | 313 ++++++++++++ azure_functions_worker/utils/wrappers.py | 22 +- setup.py | 3 +- .../customer_deps_path/azure/__init__.py | 4 + .../azure/functions/__init__.py | 8 + .../common_module/__init__.py | 8 + .../resources/customer_deps_path/readme.md | 6 + .../HttpTrigger/__init__.py | 9 + .../HttpTrigger/function.json | 20 + .../common_module/__init__.py | 9 + .../func_specific_module/__init__.py | 8 + .../resources/customer_func_path/host.json | 15 + .../customer_func_path/requirements.txt | 1 + .../worker_deps_path/azure/__init__.py | 4 + .../azure/functions/__init__.py | 8 + .../common_module/__init__.py | 5 + .../resources/worker_deps_path/readme.md | 6 + tests/unittests/test_rpc_messages.py | 6 + tests/unittests/test_utilities.py | 64 +++ tests/unittests/test_utilities_dependency.py | 460 ++++++++++++++++++ 26 files changed, 1011 insertions(+), 39 deletions(-) create mode 100644 azure_functions_worker/utils/dependency.py create mode 100644 tests/unittests/resources/customer_deps_path/azure/__init__.py create mode 100644 tests/unittests/resources/customer_deps_path/azure/functions/__init__.py create mode 100644 tests/unittests/resources/customer_deps_path/common_module/__init__.py create mode 100644 tests/unittests/resources/customer_deps_path/readme.md create mode 100644 tests/unittests/resources/customer_func_path/HttpTrigger/__init__.py create mode 100644 tests/unittests/resources/customer_func_path/HttpTrigger/function.json create mode 100644 tests/unittests/resources/customer_func_path/common_module/__init__.py create mode 100644 tests/unittests/resources/customer_func_path/func_specific_module/__init__.py create mode 100644 tests/unittests/resources/customer_func_path/host.json create mode 100644 tests/unittests/resources/customer_func_path/requirements.txt create mode 100644 tests/unittests/resources/worker_deps_path/azure/__init__.py create mode 100644 tests/unittests/resources/worker_deps_path/azure/functions/__init__.py create mode 100644 tests/unittests/resources/worker_deps_path/common_module/__init__.py create mode 100644 tests/unittests/resources/worker_deps_path/readme.md create mode 100644 tests/unittests/test_utilities_dependency.py diff --git a/.github/workflows/ut_ci_workflow.yml b/.github/workflows/ut_ci_workflow.yml index 753721b50..412c1c3bc 100644 --- a/.github/workflows/ut_ci_workflow.yml +++ b/.github/workflows/ut_ci_workflow.yml @@ -40,7 +40,9 @@ jobs: python setup.py webhost - name: Test with pytest run: | - pytest --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests + CI_RUN=1 pytest --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests --ignore=tests/unittests/test_utilities.py + # Need to test test_utilities.py separately since it involve a lot of environment variable, import cache, module cache change. + CI_RUN=1 pytest --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests/test_utilities.py - name: Codecov uses: codecov/codecov-action@v1.0.13 with: diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index f9c1e0299..2bbcf92fc 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -13,14 +13,19 @@ # Debug Flags PYAZURE_WEBHOST_DEBUG = "PYAZURE_WEBHOST_DEBUG" +# Platform Environment Variables +AZURE_WEBJOBS_SCRIPT_ROOT = "AzureWebJobsScriptRoot" + # Python Specific Feature Flags and App Settings PYTHON_ROLLBACK_CWD_PATH = "PYTHON_ROLLBACK_CWD_PATH" PYTHON_THREADPOOL_THREAD_COUNT = "PYTHON_THREADPOOL_THREAD_COUNT" +PYTHON_ISOLATE_WORKER_DEPENDENCIES = "PYTHON_ISOLATE_WORKER_DEPENDENCIES" # Setting Defaults PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT = 1 PYTHON_THREADPOOL_THREAD_COUNT_MIN = 1 PYTHON_THREADPOOL_THREAD_COUNT_MAX = 32 +PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT = False # External Site URLs MODULE_NOT_FOUND_TS_URL = "https://aka.ms/functions-modulenotfound" diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 268361509..15f11b523 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -7,8 +7,6 @@ import asyncio import concurrent.futures -import importlib -import inspect import logging import os import queue @@ -33,6 +31,7 @@ from .logging import error_logger, is_system_log_category, logger from .utils.common import get_app_setting from .utils.tracing import marshall_exception_trace +from .utils.dependency import DependencyManager from .utils.wrappers import disable_feature_by _TRUE = "true" @@ -264,6 +263,9 @@ async def _handle__worker_init_request(self, req): constants.RPC_HTTP_TRIGGER_METADATA_REMOVED: _TRUE, } + # Can detech worker packages + DependencyManager.use_customer_dependencies() + return protos.StreamingMessage( request_id=self.request_id, worker_init_response=protos.WorkerInitResponse( @@ -450,26 +452,10 @@ async def _handle__function_environment_reload_request(self, req): self._create_sync_call_tp(self._get_sync_tp_max_workers()) ) - # Reload package namespaces for customer's libraries - packages_to_reload = ['azure', 'google'] - for p in packages_to_reload: - try: - logger.info(f'Reloading {p} module') - importlib.reload(sys.modules[p]) - except Exception as ex: - logger.info('Unable to reload {}: \n{}'.format(p, ex)) - logger.info(f'Reloaded {p} module') - - # Reload azure.functions to give user package precedence - logger.info('Reloading azure.functions module at %s', - inspect.getfile(sys.modules['azure.functions'])) - try: - importlib.reload(sys.modules['azure.functions']) - logger.info('Reloaded azure.functions module now at %s', - inspect.getfile(sys.modules['azure.functions'])) - except Exception as ex: - logger.info('Unable to reload azure.functions. ' - 'Using default. Exception:\n{}'.format(ex)) + # Reload azure google namespaces + DependencyManager.reload_azure_google_namespace( + func_env_reload_request.function_app_directory + ) # Change function app directory if getattr(func_env_reload_request, diff --git a/azure_functions_worker/main.py b/azure_functions_worker/main.py index c7db801e5..78f8b7f58 100644 --- a/azure_functions_worker/main.py +++ b/azure_functions_worker/main.py @@ -5,11 +5,6 @@ import argparse -from . import dispatcher -from . import logging -from ._thirdparty import aio_compat -from .logging import error_logger, logger - def parse_args(): parser = argparse.ArgumentParser( @@ -36,6 +31,14 @@ def parse_args(): def main(): + from .utils.dependency import DependencyManager + DependencyManager.initialize() + DependencyManager.use_worker_dependencies() + + from . import logging + from ._thirdparty import aio_compat + from .logging import error_logger, logger + args = parse_args() logging.setup(log_level=args.log_level, log_destination=args.log_to) @@ -52,6 +55,8 @@ def main(): async def start_async(host, port, worker_id, request_id): + from . import dispatcher + disp = await dispatcher.Dispatcher.connect(host=host, port=port, worker_id=worker_id, request_id=request_id, diff --git a/azure_functions_worker/testutils.py b/azure_functions_worker/testutils.py index 64444cf58..ca065b251 100644 --- a/azure_functions_worker/testutils.py +++ b/azure_functions_worker/testutils.py @@ -29,7 +29,6 @@ import unittest import uuid -import asynctest as asynctest import grpc import requests @@ -123,7 +122,7 @@ def wrapper(*args, **kwargs): return wrapper -class AsyncTestCase(asynctest.TestCase, metaclass=AsyncTestCaseMeta): +class AsyncTestCase(unittest.TestCase, metaclass=AsyncTestCaseMeta): pass diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index 3b2c6f1fb..d60ca34ba 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -11,6 +11,13 @@ def is_true_like(setting: str) -> bool: return setting.lower().strip() in ['1', 'true', 't', 'yes', 'y'] +def is_false_like(setting: str) -> bool: + if setting is None: + return False + + return setting.lower().strip() in ['0', 'false', 'f', 'no', 'n'] + + def is_envvar_true(env_key: str) -> bool: if os.getenv(env_key) is None: return False @@ -18,6 +25,13 @@ def is_envvar_true(env_key: str) -> bool: return is_true_like(os.environ[env_key]) +def is_envvar_false(env_key: str) -> bool: + if os.getenv(env_key) is None: + return False + + return is_false_like(os.environ[env_key]) + + def get_app_setting( setting: str, default_value: Optional[str] = None, diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py new file mode 100644 index 000000000..013ae26bd --- /dev/null +++ b/azure_functions_worker/utils/dependency.py @@ -0,0 +1,313 @@ +from azure_functions_worker.utils.common import is_true_like +from typing import List, Optional +import importlib +import inspect +import os +import re +import sys + +from ..logging import logger +from ..constants import ( + AZURE_WEBJOBS_SCRIPT_ROOT, + PYTHON_ISOLATE_WORKER_DEPENDENCIES, + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT +) +from ..utils.wrappers import enable_feature_by + + +# Enable this feature in Python 39 by default +PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 = ( + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT + or (sys.version_info.major == 3 and sys.version_info.minor == 9) +) + + +class DependencyManager: + """The dependency manager can be used to managed the current python package + environment. Azure Functions has three different set of sys.path ordering, + + Linux Consumption sys.path: [ + "/tmp/functions\\standby\\wwwroot", # Placeholder folder + "/home/site/wwwroot/.python_packages/lib/site-packages", # CX's deps + "/azure-functions-host/workers/python/3.6/LINUX/X64", # Worker's deps + "/home/site/wwwroot" # CX's Working Directory + ] + + Linux Dedicated/Premium sys.path: [ + "/home/site/wwwroot", # CX's Working Directory + "/home/site/wwwroot/.python_packages/lib/site-packages", # CX's deps + "/azure-functions-host/workers/python/3.6/LINUX/X64", # Worker's deps + ] + + Core Tools sys.path: [ + "%appdata%\\azure-functions-core-tools\\bin\\workers\\" + "python\\3.6\\WINDOWS\\X64", # Worker's deps + "C:\\Users\\user\\Project\\.venv38\\lib\\site-packages", # CX's deps + "C:\\Users\\user\\Project", # CX's Working Directory + ] + + When we first start up the Python worker, we should only loaded from + worker's deps and create module namespace (e.g. google.protobuf variable). + + Once the worker receives worker init request, we clear out the sys.path, + worker sys.modules cache and sys.path_import_cache so the libraries + will only get loaded from CX's deps path. + """ + + cx_deps_path: str = '' + cx_working_dir: str = '' + worker_deps_path: str = '' + + @classmethod + def initialize(cls): + cls.cx_deps_path = cls._get_cx_deps_path() + cls.cx_working_dir = cls._get_cx_working_dir() + cls.worker_deps_path = cls._get_worker_deps_path() + + @classmethod + @enable_feature_by( + flag=PYTHON_ISOLATE_WORKER_DEPENDENCIES, + flag_default=PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 + ) + def use_worker_dependencies(cls): + """Switch the sys.path and ensure the worker imports are loaded from + Worker's dependenciess. + + This will not affect already imported namespaces, but will clear out + the module cache and ensure the upcoming modules are loaded from + worker's dependency path. + """ + + # The following log line will not show up in core tools but should + # work in kusto since core tools only collects gRPC logs. This function + # is executed even before the gRPC logging channel is ready. + cls._remove_from_sys_path(cls.cx_deps_path) + cls._remove_from_sys_path(cls.cx_working_dir) + cls._add_to_sys_path(cls.worker_deps_path, True) + logger.info(f'Start using worker dependencies {cls.worker_deps_path}') + + @classmethod + @enable_feature_by( + flag=PYTHON_ISOLATE_WORKER_DEPENDENCIES, + flag_default=PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 + ) + def use_customer_dependencies(cls): + """Switch the sys.path and ensure the customer's code import are loaded + from CX's deppendencies. + + This will not affect already imported namespaces, but will clear out + the module cache and ensure the upcoming modules are loaded from + customer's dependency path. + + As for Linux Consumption, this will only remove worker_deps_path, + but the customer's path will be loaded in function_environment_reload. + + The search order of a module name in customer frame is: + 1. cx_deps_path + 2. cx_working_dir + """ + cls._remove_from_sys_path(cls.worker_deps_path) + cls._add_to_sys_path(cls.cx_deps_path, True) + cls._add_to_sys_path(cls.cx_working_dir, False) + logger.info(f'Start using customer dependencies {cls.cx_deps_path}') + + @classmethod + def reload_azure_google_namespace(cls, cx_working_dir: str): + """Reload azure and google namespace, this including any modules in + this namespace, such as azure-functions, grpcio, grpcio-tools etc. + + Depends on the PYTHON_ISOLATE_WORKER_DEPENDENCIES, the actual behavior + differs. + + Parameters + ---------- + cx_working_dir: str + The path which contains customer's project file (e.g. wwwroot). + """ + use_new_env = os.getenv(PYTHON_ISOLATE_WORKER_DEPENDENCIES) + if use_new_env is None: + use_new = PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 + else: + use_new = is_true_like(use_new_env) + + if use_new: + cls.reload_azure_google_namespace_from_customer_deps( + cx_working_dir + ) + else: + cls.reload_azure_google_namespace_from_worker_deps() + + @classmethod + def reload_azure_google_namespace_from_worker_deps(cls): + """This is the old implementation of reloading azure and google + namespace in Python worker directory. It is not actually re-importing + the module but only reloads the module scripts from the worker path. + + It is not doing what it is intended, but due to it is already released + on Linux Consumption production, we don't want to introduce regression + on existing customers. + + Only intended to be used in Linux Consumption scenario. + """ + # Reload package namespaces for customer's libraries + packages_to_reload = ['azure', 'google'] + for p in packages_to_reload: + try: + logger.info(f'Reloading {p} module') + importlib.reload(sys.modules[p]) + except Exception as ex: + logger.info('Unable to reload {}: \n{}'.format(p, ex)) + logger.info(f'Reloaded {p} module') + + # Reload azure.functions to give user package precedence + logger.info('Reloading azure.functions module at %s', + inspect.getfile(sys.modules['azure.functions'])) + try: + importlib.reload(sys.modules['azure.functions']) + logger.info('Reloaded azure.functions module now at %s', + inspect.getfile(sys.modules['azure.functions'])) + except Exception as ex: + logger.info('Unable to reload azure.functions. ' + 'Using default. Exception:\n{}'.format(ex)) + + @classmethod + def reload_azure_google_namespace_from_customer_deps(cls, + cx_working_dir: str): + """This is a new implementation of reloading azure and google + namespace from customer's .python_packages folder. Only intended to be + used in Linux Consumption scenario. + + Parameters + ---------- + cx_working_dir: str + The path which contains customer's project file (e.g. wwwroot). + """ + # Specialized working directory needs to be added + working_directory: str = os.path.abspath(cx_working_dir) + + # Switch to customer deps and clear out all module cache in worker deps + cls._remove_from_sys_path(cls.worker_deps_path) + cls._add_to_sys_path(cls.cx_deps_path, True) + cls._add_to_sys_path(working_directory, False) + logger.info('Reloaded azure google namespace from customer deps') + + @classmethod + def _add_to_sys_path(cls, path: str, add_to_first: bool): + """This will ensure no duplicated path are added into sys.path and + clear importer cache. No action if path already exists in sys.path. + + Parameters + ---------- + path: str + The path needs to be added into sys.path. + If the path is an empty string, no action will be taken. + add_to_first: bool + Should the path added to the first entry (highest priority) + """ + if path and path not in sys.path: + if add_to_first: + sys.path.insert(0, path) + else: + sys.path.append(path) + + if path in sys.path_importer_cache: + sys.path_importer_cache.pop(path) + cls._remove_module_cache(path) + + @classmethod + def _remove_from_sys_path(cls, path: str): + """This will remove path from sys.path and clear importer cache. + No action if the path does not exist in sys.path. + + Parameters + ---------- + path: str + The path attempts to be removed from sys.path. + If the path is an empty string, no action will be taken. + """ + if path and path in sys.path: + # Remove all occurances + sys.path = list(filter(lambda p: p != path, sys.path)) + if path in sys.path_importer_cache: + sys.path_importer_cache.pop(path) + cls._remove_module_cache(path) + + @staticmethod + def _get_cx_deps_path() -> str: + """Get the directory storing the customer's third-party libraries. + + Returns + ------- + str + Core Tools: path to customer's site pacakges + Linux Dedicated/Premium: path to customer's site pacakges + Linux Consumption: empty string + """ + prefix: Optional[str] = os.getenv(AZURE_WEBJOBS_SCRIPT_ROOT) + cx_paths: List[str] = [ + p for p in sys.path + if prefix and p.startswith(prefix) and ('site-packages' in p) + ] + # Return first or default of customer path + return (cx_paths or [''])[0] + + @staticmethod + def _get_cx_working_dir() -> str: + """Get the customer's working directory. + + Returns + ------- + str + Core Tools: AzureWebJobsScriptRoot env variable + Linux Dedicated/Premium: AzureWebJobsScriptRoot env variable + Linux Consumption: empty string + """ + return os.getenv(AZURE_WEBJOBS_SCRIPT_ROOT, '') + + @staticmethod + def _get_worker_deps_path() -> str: + """Get the worker dependency sys.path. This will always available + even in all skus. + + Returns + ------- + str + The worker packages path + """ + # 1. Try to parse the absolute path python/3.8/LINUX/X64 in sys.path + r = re.compile(r'.*python(\/|\\)\d+\.\d+(\/|\\)(WINDOWS|LINUX|OSX).*') + worker_deps_paths: List[str] = [p for p in sys.path if r.match(p)] + if worker_deps_paths: + return worker_deps_paths[0] + + # 2. If it fails to find one, try to find one from the parent path + # This is used for handling the CI/localdev environment + return os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..') + ) + + @staticmethod + def _remove_module_cache(path: str): + """Remove module cache if the module is imported from specific path. + This will not impact builtin modules + + Parameters + ---------- + path: str + The module cache to be removed if it is imported from this path. + """ + all_modules = set(sys.modules.keys()) - set(sys.builtin_module_names) + for module_name in all_modules: + module = sys.modules[module_name] + # Module path can be actual file path or a pure namespace path + # For actual files: use __file__ attribute to retrieve module path + # For namespace: use __path__[0] to retrieve module path + module_path = '' + if getattr(module, '__file__', None): + module_path = os.path.dirname(module.__file__) + elif getattr(module, '__path__', None) and getattr( + module.__path__, '_path', None): + module_path = module.__path__._path[0] + + if module_path.startswith(path): + sys.modules.pop(module_name) diff --git a/azure_functions_worker/utils/wrappers.py b/azure_functions_worker/utils/wrappers.py index 9c3f92c6b..a1d3af675 100644 --- a/azure_functions_worker/utils/wrappers.py +++ b/azure_functions_worker/utils/wrappers.py @@ -1,26 +1,34 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from .common import is_envvar_true +from .common import is_envvar_true, is_envvar_false from .tracing import extend_exception_message -from typing import Callable, Optional +from typing import Callable, Any -def enable_feature_by(flag: str, default: Optional[int] = None) -> Callable: +def enable_feature_by(flag: str, + default: Any = None, + flag_default: bool = False) -> Callable: def decorate(func): def call(*args, **kwargs): if is_envvar_true(flag): return func(*args, **kwargs) + if flag_default and not is_envvar_false(flag): + return func(*args, **kwargs) return default return call return decorate -def disable_feature_by(flag: str, default: None = None) -> Callable: +def disable_feature_by(flag: str, + default: Any = None, + flag_default: bool = False) -> Callable: def decorate(func): def call(*args, **kwargs): - if not is_envvar_true(flag): - return func(*args, **kwargs) - return default + if is_envvar_true(flag): + return default + if flag_default and not is_envvar_false(flag): + return default + return func(*args, **kwargs) return call return decorate diff --git a/setup.py b/setup.py index 1097734bc..97098251b 100644 --- a/setup.py +++ b/setup.py @@ -308,8 +308,7 @@ def run(self): 'pytest-xdist', 'pytest-randomly', 'pytest-instafail', - 'pytest-rerunfailures', - 'asynctest' + 'pytest-rerunfailures' ] }, include_package_data=True, diff --git a/tests/unittests/resources/customer_deps_path/azure/__init__.py b/tests/unittests/resources/customer_deps_path/azure/__init__.py new file mode 100644 index 000000000..649cbaa5f --- /dev/null +++ b/tests/unittests/resources/customer_deps_path/azure/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/tests/unittests/resources/customer_deps_path/azure/functions/__init__.py b/tests/unittests/resources/customer_deps_path/azure/functions/__init__.py new file mode 100644 index 000000000..31ec82012 --- /dev/null +++ b/tests/unittests/resources/customer_deps_path/azure/functions/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +__version__: str == 'customer' + +import os +# ./tests/unittests/resources/customer_deps_path/common_module +package_location: str = os.path.dirname(__file__) diff --git a/tests/unittests/resources/customer_deps_path/common_module/__init__.py b/tests/unittests/resources/customer_deps_path/common_module/__init__.py new file mode 100644 index 000000000..31ec82012 --- /dev/null +++ b/tests/unittests/resources/customer_deps_path/common_module/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +__version__: str == 'customer' + +import os +# ./tests/unittests/resources/customer_deps_path/common_module +package_location: str = os.path.dirname(__file__) diff --git a/tests/unittests/resources/customer_deps_path/readme.md b/tests/unittests/resources/customer_deps_path/readme.md new file mode 100644 index 000000000..385a10b2d --- /dev/null +++ b/tests/unittests/resources/customer_deps_path/readme.md @@ -0,0 +1,6 @@ +This is a folder for containing a common_module in customer dependencies. + +It is used for testing import behavior with worker_deps_path. + +Adding this folder to sys.path and importing common_module, printing out the +common_module.__version__ will show which module is loaded. \ No newline at end of file diff --git a/tests/unittests/resources/customer_func_path/HttpTrigger/__init__.py b/tests/unittests/resources/customer_func_path/HttpTrigger/__init__.py new file mode 100644 index 000000000..34451943f --- /dev/null +++ b/tests/unittests/resources/customer_func_path/HttpTrigger/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import azure.functions as func # NoQA + + +def main(): + return os.path.abspath(os.path.dirname(func.__file__)) diff --git a/tests/unittests/resources/customer_func_path/HttpTrigger/function.json b/tests/unittests/resources/customer_func_path/HttpTrigger/function.json new file mode 100644 index 000000000..4667f0aca --- /dev/null +++ b/tests/unittests/resources/customer_func_path/HttpTrigger/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "anonymous", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/tests/unittests/resources/customer_func_path/common_module/__init__.py b/tests/unittests/resources/customer_func_path/common_module/__init__.py new file mode 100644 index 000000000..c001ad451 --- /dev/null +++ b/tests/unittests/resources/customer_func_path/common_module/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +__version__: str == 'function_app' + +import os +# This module should be shadowed from customer_deps_path/common_module +# ./tests/unittests/resources/customer_func_path/common_module +package_location: str = os.path.dirname(__file__) diff --git a/tests/unittests/resources/customer_func_path/func_specific_module/__init__.py b/tests/unittests/resources/customer_func_path/func_specific_module/__init__.py new file mode 100644 index 000000000..7e8399fae --- /dev/null +++ b/tests/unittests/resources/customer_func_path/func_specific_module/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +__version__: str == 'function_app' + +import os +# ./tests/unittests/resources/customer_func_path/func_specific_module +package_location: str = os.path.dirname(__file__) diff --git a/tests/unittests/resources/customer_func_path/host.json b/tests/unittests/resources/customer_func_path/host.json new file mode 100644 index 000000000..05291ed43 --- /dev/null +++ b/tests/unittests/resources/customer_func_path/host.json @@ -0,0 +1,15 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[1.*, 2.0.0)" + } +} \ No newline at end of file diff --git a/tests/unittests/resources/customer_func_path/requirements.txt b/tests/unittests/resources/customer_func_path/requirements.txt new file mode 100644 index 000000000..f86a15a6a --- /dev/null +++ b/tests/unittests/resources/customer_func_path/requirements.txt @@ -0,0 +1 @@ +azure-functions \ No newline at end of file diff --git a/tests/unittests/resources/worker_deps_path/azure/__init__.py b/tests/unittests/resources/worker_deps_path/azure/__init__.py new file mode 100644 index 000000000..649cbaa5f --- /dev/null +++ b/tests/unittests/resources/worker_deps_path/azure/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/tests/unittests/resources/worker_deps_path/azure/functions/__init__.py b/tests/unittests/resources/worker_deps_path/azure/functions/__init__.py new file mode 100644 index 000000000..abde0b6a1 --- /dev/null +++ b/tests/unittests/resources/worker_deps_path/azure/functions/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +__version__: str == 'worker' + +import os +# ./tests/unittests/resources/worker_deps_path/common_module +package_location: str = os.path.dirname(__file__) diff --git a/tests/unittests/resources/worker_deps_path/common_module/__init__.py b/tests/unittests/resources/worker_deps_path/common_module/__init__.py new file mode 100644 index 000000000..9cb472c3f --- /dev/null +++ b/tests/unittests/resources/worker_deps_path/common_module/__init__.py @@ -0,0 +1,5 @@ +__version__: str == 'worker' + +import os +# ./tests/unittests/resources/worker_deps_path/common_module +package_location: str = os.path.dirname(__file__) diff --git a/tests/unittests/resources/worker_deps_path/readme.md b/tests/unittests/resources/worker_deps_path/readme.md new file mode 100644 index 000000000..c91e2bdc4 --- /dev/null +++ b/tests/unittests/resources/worker_deps_path/readme.md @@ -0,0 +1,6 @@ +This is a folder for containing a common_module in worker dependencies. + +It is used for testing import behavior with customer_deps_path. + +Adding this folder to sys.path and importing common_module, printing out the +common_module.__version__ will show which module is loaded. \ No newline at end of file diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py index 1c4730669..63d82980c 100644 --- a/tests/unittests/test_rpc_messages.py +++ b/tests/unittests/test_rpc_messages.py @@ -127,6 +127,12 @@ def test_failed_azure_namespace_import(self): @unittest.skipIf(sys.platform == 'win32', 'Linux .sh script only works on Linux') + @unittest.skipIf( + sys.version_info.major == 3 and sys.version_info.minor == 9, + 'In Python 3.9, isolate worker dependencies is turned on by default.' + ' Reloading all customer dependencies on specialization is a must.' + ' This partially reloading namespace feature is no longer needed.' + ) def test_successful_azure_namespace_import(self): self._verify_azure_namespace_import( 'true', diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index a8d0058ef..504d510bf 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -19,12 +19,24 @@ def mock_feature_enabled(self, output: typing.List[str]) -> str: output.append(result) return result + @wrappers.enable_feature_by(TEST_FEATURE_FLAG, flag_default=True) + def mock_enabled_default_true(self, output: typing.List[str]) -> str: + result = 'mock_enabled_default_true' + output.append(result) + return result + @wrappers.disable_feature_by(TEST_FEATURE_FLAG) def mock_feature_disabled(self, output: typing.List[str]) -> str: result = 'mock_feature_disabled' output.append(result) return result + @wrappers.disable_feature_by(TEST_FEATURE_FLAG, flag_default=True) + def mock_disabled_default_true(self, output: typing.List[str]) -> str: + result = 'mock_disabled_default_true' + output.append(result) + return result + @wrappers.enable_feature_by(TEST_FEATURE_FLAG, FEATURE_DEFAULT) def mock_feature_default(self, output: typing.List[str]) -> str: result = 'mock_feature_default' @@ -73,6 +85,18 @@ def test_is_true_like_rejected(self): self.assertFalse(common.is_true_like('')) self.assertFalse(common.is_true_like('secret')) + def test_is_false_like_accepted(self): + self.assertTrue(common.is_false_like('0')) + self.assertTrue(common.is_false_like('false')) + self.assertTrue(common.is_false_like('F')) + self.assertTrue(common.is_false_like('NO')) + self.assertTrue(common.is_false_like('n')) + + def test_is_false_like_rejected(self): + self.assertFalse(common.is_false_like(None)) + self.assertFalse(common.is_false_like('')) + self.assertFalse(common.is_false_like('secret')) + def test_is_envvar_true(self): os.environ[TEST_FEATURE_FLAG] = 'true' self.assertTrue(common.is_envvar_true(TEST_FEATURE_FLAG)) @@ -81,6 +105,14 @@ def test_is_envvar_not_true_on_unset(self): self._unset_feature_flag() self.assertFalse(common.is_envvar_true(TEST_FEATURE_FLAG)) + def test_is_envvar_false(self): + os.environ[TEST_FEATURE_FLAG] = 'false' + self.assertTrue(common.is_envvar_false(TEST_FEATURE_FLAG)) + + def test_is_envvar_not_false_on_unset(self): + self._unset_feature_flag() + self.assertFalse(common.is_envvar_true(TEST_FEATURE_FLAG)) + def test_disable_feature_with_no_feature_flag(self): mock_feature = MockFeature() output = [] @@ -88,6 +120,13 @@ def test_disable_feature_with_no_feature_flag(self): self.assertIsNone(result) self.assertListEqual(output, []) + def test_disable_feature_with_default_value(self): + mock_feature = MockFeature() + output = [] + result = mock_feature.mock_disabled_default_true(output) + self.assertIsNone(result) + self.assertListEqual(output, []) + def test_enable_feature_with_feature_flag(self): feature_flag = TEST_FEATURE_FLAG os.environ[feature_flag] = '1' @@ -97,6 +136,13 @@ def test_enable_feature_with_feature_flag(self): self.assertEqual(result, 'mock_feature_enabled') self.assertListEqual(output, ['mock_feature_enabled']) + def test_enable_feature_with_default_value(self): + mock_feature = MockFeature() + output = [] + result = mock_feature.mock_enabled_default_true(output) + self.assertEqual(result, 'mock_enabled_default_true') + self.assertListEqual(output, ['mock_enabled_default_true']) + def test_enable_feature_with_no_rollback_flag(self): mock_feature = MockFeature() output = [] @@ -104,6 +150,15 @@ def test_enable_feature_with_no_rollback_flag(self): self.assertEqual(result, 'mock_feature_disabled') self.assertListEqual(output, ['mock_feature_disabled']) + def test_ignore_disable_default_value_when_set_explicitly(self): + feature_flag = TEST_FEATURE_FLAG + os.environ[feature_flag] = '0' + mock_feature = MockFeature() + output = [] + result = mock_feature.mock_disabled_default_true(output) + self.assertEqual(result, 'mock_disabled_default_true') + self.assertListEqual(output, ['mock_disabled_default_true']) + def test_disable_feature_with_rollback_flag(self): rollback_flag = TEST_FEATURE_FLAG os.environ[rollback_flag] = '1' @@ -122,6 +177,15 @@ def test_enable_feature_with_rollback_flag_is_false(self): self.assertEqual(result, 'mock_feature_disabled') self.assertListEqual(output, ['mock_feature_disabled']) + def test_ignore_enable_default_value_when_set_explicitly(self): + feature_flag = TEST_FEATURE_FLAG + os.environ[feature_flag] = '0' + mock_feature = MockFeature() + output = [] + result = mock_feature.mock_enabled_default_true(output) + self.assertIsNone(result) + self.assertListEqual(output, []) + def test_fail_to_enable_feature_return_default_value(self): mock_feature = MockFeature() output = [] diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py new file mode 100644 index 000000000..2dd7e2940 --- /dev/null +++ b/tests/unittests/test_utilities_dependency.py @@ -0,0 +1,460 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import os +import sys +import unittest +from unittest.mock import patch + +from azure_functions_worker import testutils +from azure_functions_worker.utils.dependency import DependencyManager + + +class TestDependencyManager(unittest.TestCase): + + def setUp(self): + self._patch_environ = patch.dict('os.environ', {}) + self._patch_sys_path = patch('sys.path', []) + self._patch_importer_cache = patch.dict('sys.path_importer_cache', {}) + self._patch_modules = patch.dict('sys.modules', {}) + self._customer_func_path = os.path.abspath( + os.path.join( + testutils.UNIT_TESTS_ROOT, 'resources', 'customer_func_path' + ) + ) + self._worker_deps_path = os.path.abspath( + os.path.join( + testutils.UNIT_TESTS_ROOT, 'resources', 'worker_deps_path' + ) + ) + self._customer_deps_path = os.path.abspath( + os.path.join( + testutils.UNIT_TESTS_ROOT, 'resources', 'customer_deps_path' + ) + ) + + self._patch_environ.start() + self._patch_sys_path.start() + self._patch_importer_cache.start() + self._patch_modules.start() + + def tearDown(self): + self._patch_environ.stop() + self._patch_sys_path.stop() + self._patch_importer_cache.stop() + self._patch_modules.stop() + DependencyManager.cx_deps_path = '' + DependencyManager.cx_working_dir = '' + DependencyManager.worker_deps_path = '' + + def test_should_not_have_any_paths_initially(self): + self.assertEqual(DependencyManager.cx_deps_path, '') + self.assertEqual(DependencyManager.cx_working_dir, '') + self.assertEqual(DependencyManager.worker_deps_path, '') + + def test_initialize_in_linux_consumption(self): + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + sys.path.extend([ + '/tmp/functions\\standby\\wwwroot', + '/home/site/wwwroot/.python_packages/lib/site-packages', + '/azure-functions-host/workers/python/3.6/LINUX/X64', + '/home/site/wwwroot' + ]) + DependencyManager.initialize() + self.assertEqual( + DependencyManager.cx_deps_path, + '/home/site/wwwroot/.python_packages/lib/site-packages' + ) + self.assertEqual( + DependencyManager.cx_working_dir, + '/home/site/wwwroot', + ) + self.assertEqual( + DependencyManager.worker_deps_path, + '/azure-functions-host/workers/python/3.6/LINUX/X64' + ) + + def test_initialize_in_linux_dedicated(self): + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + sys.path.extend([ + '/home/site/wwwroot', + '/home/site/wwwroot/.python_packages/lib/site-packages', + '/azure-functions-host/workers/python/3.7/LINUX/X64' + ]) + DependencyManager.initialize() + self.assertEqual( + DependencyManager.cx_deps_path, + '/home/site/wwwroot/.python_packages/lib/site-packages' + ) + self.assertEqual( + DependencyManager.cx_working_dir, + '/home/site/wwwroot', + ) + self.assertEqual( + DependencyManager.worker_deps_path, + '/azure-functions-host/workers/python/3.7/LINUX/X64' + ) + + def test_initialize_in_windows_core_tools(self): + os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp' + sys.path.extend([ + 'C:\\Users\\hazeng\\AppData\\Roaming\\npm\\' + 'node_modules\\azure-functions-core-tools\\bin\\' + 'workers\\python\\3.6\\WINDOWS\\X64', + 'C:\\FunctionApp\\.venv38\\lib\\site-packages', + 'C:\\FunctionApp' + ]) + DependencyManager.initialize() + self.assertEqual( + DependencyManager.cx_deps_path, + 'C:\\FunctionApp\\.venv38\\lib\\site-packages' + ) + self.assertEqual( + DependencyManager.cx_working_dir, + 'C:\\FunctionApp', + ) + self.assertEqual( + DependencyManager.worker_deps_path, + 'C:\\Users\\hazeng\\AppData\\Roaming\\npm\\node_modules\\' + 'azure-functions-core-tools\\bin\\workers\\python\\3.6\\WINDOWS' + '\\X64' + ) + + def test_get_cx_deps_path_in_no_script_root(self): + result = DependencyManager._get_cx_deps_path() + self.assertEqual(result, '') + + def test_get_cx_deps_path_in_script_root_no_sys_path(self): + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + result = DependencyManager._get_cx_deps_path() + self.assertEqual(result, '') + + def test_get_cx_deps_path_in_script_root_with_sys_path_linux_py36(self): + # Test for Python 3.6 Azure Environment + sys.path.append('/home/site/wwwroot/.python_packages/sites/lib/' + 'python3.6/site-packages/') + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + result = DependencyManager._get_cx_deps_path() + self.assertEqual(result, '/home/site/wwwroot/.python_packages/sites/' + 'lib/python3.6/site-packages/') + + def test_get_cx_deps_path_in_script_root_with_sys_path_linux(self): + # Test for Python 3.7+ Azure Environment + sys.path.append('/home/site/wwwroot/.python_packages/sites/lib/' + 'site-packages/') + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + result = DependencyManager._get_cx_deps_path() + self.assertEqual(result, '/home/site/wwwroot/.python_packages/sites/' + 'lib/site-packages/') + + def test_get_cx_deps_path_in_script_root_with_sys_path_windows(self): + # Test for Windows Core Tools Environment + sys.path.append('C:\\FunctionApp\\sites\\lib\\site-packages') + os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp' + result = DependencyManager._get_cx_deps_path() + self.assertEqual(result, + 'C:\\FunctionApp\\sites\\lib\\site-packages') + + def test_get_cx_working_dir_no_script_root(self): + result = DependencyManager._get_cx_working_dir() + self.assertEqual(result, '') + + def test_get_cx_working_dir_with_script_root_linux(self): + # Test for Azure Environment + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + result = DependencyManager._get_cx_working_dir() + self.assertEqual(result, '/home/site/wwwroot') + + def test_get_cx_working_dir_with_script_root_windows(self): + # Test for Windows Core Tools Environment + os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp' + result = DependencyManager._get_cx_working_dir() + self.assertEqual(result, 'C:\\FunctionApp') + + def test_get_worker_deps_path_with_no_worker_sys_path(self): + result = DependencyManager._get_worker_deps_path() + worker_parent = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..') + ) + self.assertEqual(result.lower(), worker_parent.lower()) + + def test_get_worker_deps_path_from_windows_core_tools(self): + # Test for Windows Core Tools Environment + sys.path.append('C:\\Users\\hazeng\\AppData\\Roaming\\npm\\' + 'node_modules\\azure-functions-core-tools\\bin\\' + 'workers\\python\\3.6\\WINDOWS\\X64') + result = DependencyManager._get_worker_deps_path() + self.assertEqual(result, + 'C:\\Users\\hazeng\\AppData\\Roaming\\npm\\' + 'node_modules\\azure-functions-core-tools\\bin\\' + 'workers\\python\\3.6\\WINDOWS\\X64') + + def test_get_worker_deps_path_from_linux_azure_environment(self): + # Test for Azure Environment + sys.path.append('/azure-functions-host/workers/python/3.7/LINUX/X64') + result = DependencyManager._get_worker_deps_path() + self.assertEqual(result, + '/azure-functions-host/workers/python/3.7/LINUX/X64') + + def test_get_worker_deps_path_without_worker_path(self): + # Test when worker path is not provided + sys.path.append('/home/site/wwwroot') + result = DependencyManager._get_worker_deps_path() + worker_parent = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..') + ) + self.assertEqual(result.lower(), worker_parent.lower()) + + def test_add_to_sys_path_add_to_first(self): + DependencyManager._add_to_sys_path(self._customer_deps_path, True) + self.assertEqual(sys.path[0], self._customer_deps_path) + + def test_add_to_sys_path_add_to_last(self): + DependencyManager._add_to_sys_path(self._customer_deps_path, False) + self.assertEqual(sys.path[-1], self._customer_deps_path) + + def test_add_to_sys_path_no_duplication(self): + DependencyManager._add_to_sys_path(self._customer_deps_path, True) + DependencyManager._add_to_sys_path(self._customer_deps_path, True) + path_count = len(list(filter( + lambda x: x == self._customer_deps_path, sys.path + ))) + self.assertEqual(path_count, 1) + + def test_add_to_sys_path_import_module(self): + DependencyManager._add_to_sys_path(self._customer_deps_path, True) + import common_module # NoQA + self.assertEqual( + common_module.package_location, + os.path.join(self._customer_deps_path, 'common_module') + ) + + def test_add_to_sys_path_importer_cache(self): + DependencyManager._add_to_sys_path(self._customer_deps_path, True) + import common_module # NoQA + self.assertIn(self._customer_deps_path, sys.path_importer_cache) + + def test_add_to_sys_path_importer_cache_reloaded(self): + # First import the common module from worker_deps_path + DependencyManager._add_to_sys_path(self._worker_deps_path, True) + import common_module # NoQA + self.assertIn(self._worker_deps_path, sys.path_importer_cache) + self.assertEqual( + common_module.package_location, + os.path.join(self._worker_deps_path, 'common_module') + ) + + # Mock that the customer's script are running in a different module + # (e.g. HttpTrigger/__init__.py) + del sys.modules['common_module'] + del common_module + + # Import the common module from customer_deps_path + # Customer should only see their own module + DependencyManager._add_to_sys_path(self._customer_deps_path, True) + import common_module # NoQA + self.assertIn(self._customer_deps_path, sys.path_importer_cache) + self.assertEqual( + common_module.package_location, + os.path.join(self._customer_deps_path, 'common_module') + ) + + def test_remove_from_sys_path(self): + sys.path.append(self._customer_deps_path) + DependencyManager._remove_from_sys_path(self._customer_deps_path) + self.assertNotIn(self._customer_deps_path, sys.path) + + def test_remove_from_sys_path_should_remove_all_duplications(self): + sys.path.insert(0, self._customer_deps_path) + sys.path.append(self._customer_deps_path) + DependencyManager._remove_from_sys_path(self._customer_deps_path) + self.assertNotIn(self._customer_deps_path, sys.path) + + def test_remove_from_sys_path_should_remove_path_importer_cache(self): + # Import a common_module from customer deps will create a path finter + # cache in sys.path_importer_cache + sys.path.insert(0, self._customer_deps_path) + import common_module # NoQA + self.assertIn(self._customer_deps_path, sys.path_importer_cache) + + # Remove sys.path_importer_cache + DependencyManager._remove_from_sys_path(self._customer_deps_path) + self.assertNotIn(self._customer_deps_path, sys.path_importer_cache) + + def test_remove_from_sys_path_should_remove_related_module(self): + # Import a common_module from customer deps will create a module import + # cache in sys.module + sys.path.insert(0, self._customer_deps_path) + import common_module # NoQA + self.assertIn('common_module', sys.modules) + + # Remove sys.path_importer_cache + DependencyManager._remove_from_sys_path(self._customer_deps_path) + self.assertNotIn('common_module', sys.modules) + + def test_use_worker_dependencies(self): + # Setup app settings + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' + + # Setup paths + DependencyManager.worker_deps_path = self._worker_deps_path + DependencyManager.cx_deps_path = self._customer_deps_path + DependencyManager.cx_working_dir = self._customer_func_path + + # Ensure the common_module is imported from _worker_deps_path + DependencyManager.use_worker_dependencies() + import common_module # NoQA + self.assertEqual( + common_module.package_location, + os.path.join(self._worker_deps_path, 'common_module') + ) + + def test_use_worker_dependencies_disable(self): + # Setup app settings + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'false' + + # Setup paths + DependencyManager.worker_deps_path = self._worker_deps_path + DependencyManager.cx_deps_path = self._customer_deps_path + DependencyManager.cx_working_dir = self._customer_func_path + + # The common_module cannot be imported since feature is disabled + DependencyManager.use_worker_dependencies() + with self.assertRaises(ImportError): + import common_module # NoQA + + @unittest.skipUnless( + sys.version_info.major == 3 and sys.version_info.minor in (6, 7, 8), + 'Test only available for Python 3.6, 3.7, or 3.8' + ) + def test_use_worker_dependencies_default_python_36_37_38(self): + # Feature should be disabled in Python 3.6, 3.7, and 3.8 + # Setup paths + DependencyManager.worker_deps_path = self._worker_deps_path + DependencyManager.cx_deps_path = self._customer_deps_path + DependencyManager.cx_working_dir = self._customer_func_path + + # The common_module cannot be imported since feature is disabled + DependencyManager.use_worker_dependencies() + with self.assertRaises(ImportError): + import common_module # NoQA + + @unittest.skipUnless( + sys.version_info.major == 3 and sys.version_info.minor == 9, + 'Test only available for Python 3.9' + ) + def test_use_worker_dependencies_default_python_39(self): + # Feature should be enabled in Python 3.9 by default + # Setup paths + DependencyManager.worker_deps_path = self._worker_deps_path + DependencyManager.cx_deps_path = self._customer_deps_path + DependencyManager.cx_working_dir = self._customer_func_path + + # Ensure the common_module is imported from _worker_deps_path + DependencyManager.use_worker_dependencies() + import common_module # NoQA + self.assertEqual( + common_module.package_location, + os.path.join(self._worker_deps_path, 'common_module') + ) + + def test_use_customer_dependencies(self): + # Setup app settings + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' + + # Setup paths + DependencyManager.worker_deps_path = self._worker_deps_path + DependencyManager.cx_deps_path = self._customer_deps_path + DependencyManager.cx_working_dir = self._customer_func_path + + # Ensure the common_module is imported from _customer_deps_path + DependencyManager.use_customer_dependencies() + import common_module # NoQA + self.assertEqual( + common_module.package_location, + os.path.join(self._customer_deps_path, 'common_module') + ) + + def test_use_customer_dependencies_disable(self): + # Setup app settings + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'false' + + # Setup paths + DependencyManager.worker_deps_path = self._worker_deps_path + DependencyManager.cx_deps_path = self._customer_deps_path + DependencyManager.cx_working_dir = self._customer_func_path + + # Ensure the common_module is imported from _customer_deps_path + DependencyManager.use_customer_dependencies() + with self.assertRaises(ImportError): + import common_module # NoQA + + @unittest.skipUnless( + sys.version_info.major == 3 and sys.version_info.minor in (6, 7, 8), + 'Test only available for Python 3.6, 3.7, or 3.8' + ) + def test_use_customer_dependencies_default_python_36_37_38(self): + # Feature should be disabled in Python 3.6, 3.7, and 3.8 + # Setup paths + DependencyManager.worker_deps_path = self._worker_deps_path + DependencyManager.cx_deps_path = self._customer_deps_path + DependencyManager.cx_working_dir = self._customer_func_path + + # Ensure the common_module is imported from _customer_deps_path + DependencyManager.use_customer_dependencies() + with self.assertRaises(ImportError): + import common_module # NoQA + + @unittest.skipUnless( + sys.version_info.major == 3 and sys.version_info.minor == 9, + 'Test only available for Python 3.9' + ) + def test_use_customer_dependencies_default_python_39(self): + # Feature should be enabled in Python 3.9 by default + # Setup paths + DependencyManager.worker_deps_path = self._worker_deps_path + DependencyManager.cx_deps_path = self._customer_deps_path + DependencyManager.cx_working_dir = self._customer_func_path + + # Ensure the common_module is imported from _customer_deps_path + DependencyManager.use_customer_dependencies() + import common_module # NoQA + self.assertEqual( + common_module.package_location, + os.path.join(self._customer_deps_path, 'common_module') + ) + + def test_use_customer_dependencies_import_from_working_directory(self): + # Setup app settings + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' + + # Setup paths + DependencyManager.worker_deps_path = self._worker_deps_path + DependencyManager.cx_deps_path = self._customer_deps_path + DependencyManager.cx_working_dir = self._customer_func_path + + # Ensure the func_specific_module is imported from _customer_func_path + DependencyManager.use_customer_dependencies() + import func_specific_module # NoQA + self.assertEqual( + func_specific_module.package_location, + os.path.join(self._customer_func_path, 'func_specific_module') + ) + + def test_remove_module_cache(self): + # First import the common_module and create a sys.modules cache + sys.path.append(self._customer_deps_path) + import common_module # NoQA + self.assertIn('common_module', sys.modules) + + # Ensure the module cache will be remove + DependencyManager._remove_module_cache(self._customer_deps_path) + self.assertNotIn('common_module', sys.modules) + + def test_remove_module_cache_with_namespace_remain(self): + # Create common_module namespace + sys.path.append(self._customer_deps_path) + import common_module # NoQA + + # Ensure namespace remains after module cache is removed + DependencyManager._remove_module_cache(self._customer_deps_path) + self.assertIsNotNone(common_module) From 525d46e14f1adc0e0425359dfbe8010e757434ea Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Fri, 25 Dec 2020 20:07:03 -0800 Subject: [PATCH 2/7] Restore ut workflow --- .github/workflows/ut_ci_workflow.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ut_ci_workflow.yml b/.github/workflows/ut_ci_workflow.yml index 412c1c3bc..753721b50 100644 --- a/.github/workflows/ut_ci_workflow.yml +++ b/.github/workflows/ut_ci_workflow.yml @@ -40,9 +40,7 @@ jobs: python setup.py webhost - name: Test with pytest run: | - CI_RUN=1 pytest --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests --ignore=tests/unittests/test_utilities.py - # Need to test test_utilities.py separately since it involve a lot of environment variable, import cache, module cache change. - CI_RUN=1 pytest --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests/test_utilities.py + pytest --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests - name: Codecov uses: codecov/codecov-action@v1.0.13 with: From 49156fb08a05d5264dbd659735b00a6b31bc93a8 Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Fri, 8 Jan 2021 17:37:45 -0800 Subject: [PATCH 3/7] Refactor clear path importer code --- azure_functions_worker/utils/dependency.py | 46 +++++++++---- tests/unittests/test_utilities_dependency.py | 71 ++++++++++++++++++++ 2 files changed, 103 insertions(+), 14 deletions(-) diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index 013ae26bd..7d9f7e07e 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -23,8 +23,15 @@ class DependencyManager: - """The dependency manager can be used to managed the current python package - environment. Azure Functions has three different set of sys.path ordering, + """The dependency manager controls the Python packages source, preventing + worker packages interfer customer's code. + + It has two mode, in worker mode, the Python packages are loaded from worker + path, (e.g. workers/python///). In customer mode, + the packages are loaded from customer's .python_packages/ folder or from + their virtual environment. + + Azure Functions has three different set of sys.path ordering, Linux Consumption sys.path: [ "/tmp/functions\\standby\\wwwroot", # Placeholder folder @@ -131,9 +138,7 @@ def reload_azure_google_namespace(cls, cx_working_dir: str): use_new = is_true_like(use_new_env) if use_new: - cls.reload_azure_google_namespace_from_customer_deps( - cx_working_dir - ) + cls.reload_all_namespaces_from_customer_deps(cx_working_dir) else: cls.reload_azure_google_namespace_from_worker_deps() @@ -171,8 +176,7 @@ def reload_azure_google_namespace_from_worker_deps(cls): 'Using default. Exception:\n{}'.format(ex)) @classmethod - def reload_azure_google_namespace_from_customer_deps(cls, - cx_working_dir: str): + def reload_all_namespaces_from_customer_deps(cls, cx_working_dir: str): """This is a new implementation of reloading azure and google namespace from customer's .python_packages folder. Only intended to be used in Linux Consumption scenario. @@ -209,10 +213,7 @@ def _add_to_sys_path(cls, path: str, add_to_first: bool): sys.path.insert(0, path) else: sys.path.append(path) - - if path in sys.path_importer_cache: - sys.path_importer_cache.pop(path) - cls._remove_module_cache(path) + cls._clear_path_importer_cache_and_modules(path) @classmethod def _remove_from_sys_path(cls, path: str): @@ -222,14 +223,31 @@ def _remove_from_sys_path(cls, path: str): Parameters ---------- path: str - The path attempts to be removed from sys.path. + The path to be removed from sys.path. If the path is an empty string, no action will be taken. """ if path and path in sys.path: # Remove all occurances sys.path = list(filter(lambda p: p != path, sys.path)) - if path in sys.path_importer_cache: - sys.path_importer_cache.pop(path) + cls._clear_path_importer_cache_and_modules(path) + + @classmethod + def _clear_path_importer_cache_and_modules(cls, path: str): + """Removes path from sys.path_importer_cache and clear related + sys.modules cache. No action if the path is empty or no entries + in sys.path_importer_cache or sys.modules. + + Parameters + ---------- + path: str + The path to be removed from sys.path_importer_cache. All related + modules will be cleared out from sys.modules cache. + If the path is an empty string, no action will be taken. + """ + if path in sys.path_importer_cache: + sys.path_importer_cache.pop(path) + + if path: cls._remove_module_cache(path) @staticmethod diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py index 2dd7e2940..e3a83eca0 100644 --- a/tests/unittests/test_utilities_dependency.py +++ b/tests/unittests/test_utilities_dependency.py @@ -291,6 +291,77 @@ def test_remove_from_sys_path_should_remove_related_module(self): DependencyManager._remove_from_sys_path(self._customer_deps_path) self.assertNotIn('common_module', sys.modules) + def test_clear_path_importer_cache_and_modules(self): + # Ensure sys.path_importer_cache and sys.modules cache is cleared + sys.path.insert(0, self._customer_deps_path) + import common_module # NoQA + self.assertIn('common_module', sys.modules) + + # Clear out cache + DependencyManager._clear_path_importer_cache_and_modules( + self._customer_deps_path + ) + + # Ensure cache is cleared + self.assertNotIn('common_module', sys.modules) + + def test_clear_path_importer_cache_and_modules_reimport(self): + # First import common_module from _customer_deps_path + sys.path.insert(0, self._customer_deps_path) + import common_module # NoQA + self.assertIn('common_module', sys.modules) + self.assertEqual( + common_module.package_location, + os.path.join(self._customer_deps_path, 'common_module') + ) + + # Clean up cache + DependencyManager._clear_path_importer_cache_and_modules( + self._customer_deps_path + ) + self.assertNotIn('common_module', sys.modules) + + # Clean up namespace + del common_module + + # Try import common_module from _worker_deps_path + sys.path.insert(0, self._worker_deps_path) + + # Ensure new import is from _worker_deps_path + import common_module # NoQA + self.assertIn('common_module', sys.modules) + self.assertEqual( + common_module.package_location, + os.path.join(self._worker_deps_path, 'common_module') + ) + + def test_clear_path_importer_cache_and_modules_retain_namespace(self): + # First import common_module from _customer_deps_path as customer_mod + sys.path.insert(0, self._customer_deps_path) + import common_module as customer_mod # NoQA + self.assertIn('common_module', sys.modules) + self.assertEqual( + customer_mod.package_location, + os.path.join(self._customer_deps_path, 'common_module') + ) + + # Clean up cache + DependencyManager._clear_path_importer_cache_and_modules( + self._customer_deps_path + ) + self.assertNotIn('common_module', sys.modules) + + # Try import common_module from _worker_deps_path as worker_mod + sys.path.insert(0, self._worker_deps_path) + + # Ensure new import is from _worker_deps_path + import common_module as worker_mod # NoQA + self.assertIn('common_module', sys.modules) + self.assertEqual( + worker_mod.package_location, + os.path.join(self._worker_deps_path, 'common_module') + ) + def test_use_worker_dependencies(self): # Setup app settings os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' From 6118351ceebb8664966af20e482c39a84b3c547b Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Tue, 12 Jan 2021 15:06:17 -0800 Subject: [PATCH 4/7] Explicitly introduce python 3.9 default flag --- azure_functions_worker/constants.py | 1 + azure_functions_worker/utils/common.py | 6 +++++ azure_functions_worker/utils/dependency.py | 29 ++++++++++++++-------- tests/unittests/test_utilities.py | 14 +++++++++++ 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index 2bbcf92fc..2fcb83828 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -26,6 +26,7 @@ PYTHON_THREADPOOL_THREAD_COUNT_MIN = 1 PYTHON_THREADPOOL_THREAD_COUNT_MAX = 32 PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT = False +PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 = True # External Site URLs MODULE_NOT_FOUND_TS_URL = "https://aka.ms/functions-modulenotfound" diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index d60ca34ba..d203fb315 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. from typing import Optional, Callable import os +import sys def is_true_like(setting: str) -> bool: @@ -32,6 +33,11 @@ def is_envvar_false(env_key: str) -> bool: return is_false_like(os.environ[env_key]) +def is_python_version(version: str) -> bool: + current_version = f'{sys.version_info.major}.{sys.version_info.minor}' + return current_version == version + + def get_app_setting( setting: str, default_value: Optional[str] = None, diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index 7d9f7e07e..9eaef665e 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -10,18 +10,13 @@ from ..constants import ( AZURE_WEBJOBS_SCRIPT_ROOT, PYTHON_ISOLATE_WORKER_DEPENDENCIES, - PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT, + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 ) +from ..utils.common import is_python_version from ..utils.wrappers import enable_feature_by -# Enable this feature in Python 39 by default -PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 = ( - PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT - or (sys.version_info.major == 3 and sys.version_info.minor == 9) -) - - class DependencyManager: """The dependency manager controls the Python packages source, preventing worker packages interfer customer's code. @@ -74,7 +69,11 @@ def initialize(cls): @classmethod @enable_feature_by( flag=PYTHON_ISOLATE_WORKER_DEPENDENCIES, - flag_default=PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 + flag_default=( + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 if + is_python_version('3.9') else + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT + ) ) def use_worker_dependencies(cls): """Switch the sys.path and ensure the worker imports are loaded from @@ -96,7 +95,11 @@ def use_worker_dependencies(cls): @classmethod @enable_feature_by( flag=PYTHON_ISOLATE_WORKER_DEPENDENCIES, - flag_default=PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 + flag_default=( + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 if + is_python_version('3.9') else + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT + ) ) def use_customer_dependencies(cls): """Switch the sys.path and ensure the customer's code import are loaded @@ -133,7 +136,11 @@ def reload_azure_google_namespace(cls, cx_working_dir: str): """ use_new_env = os.getenv(PYTHON_ISOLATE_WORKER_DEPENDENCIES) if use_new_env is None: - use_new = PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 + use_new = ( + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_39 if + is_python_version('3.9') else + PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT + ) else: use_new = is_true_like(use_new_env) diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 504d510bf..935659816 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -298,6 +298,20 @@ def parse_int_no_raise(value: str): # Because 'invalid' is not an interger, falls back to default value self.assertEqual(app_setting, '42') + def test_is_python_version(self): + # Should pass at least 1 test + is_python_version_36 = common.is_python_version('3.6') + is_python_version_37 = common.is_python_version('3.7') + is_python_version_38 = common.is_python_version('3.8') + is_python_version_39 = common.is_python_version('3.9') + + self.assertTrue(any([ + is_python_version_36, + is_python_version_37, + is_python_version_38, + is_python_version_39 + ])) + def _unset_feature_flag(self): try: os.environ.pop(TEST_FEATURE_FLAG) From a498c85959e9c66ac543c046b31f607580bb554c Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Tue, 12 Jan 2021 15:22:49 -0800 Subject: [PATCH 5/7] Fix nit --- azure_functions_worker/utils/dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index 9eaef665e..5ac194f58 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -251,7 +251,7 @@ def _clear_path_importer_cache_and_modules(cls, path: str): modules will be cleared out from sys.modules cache. If the path is an empty string, no action will be taken. """ - if path in sys.path_importer_cache: + if path and path in sys.path_importer_cache: sys.path_importer_cache.pop(path) if path: From d6666dc534eb66004b91503dbcc8a4a4c453459a Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Tue, 12 Jan 2021 16:41:28 -0800 Subject: [PATCH 6/7] Fix cache clear logic --- azure_functions_worker/utils/dependency.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index 5ac194f58..84dfd529a 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -220,6 +220,9 @@ def _add_to_sys_path(cls, path: str, add_to_first: bool): sys.path.insert(0, path) else: sys.path.append(path) + + # Only clear path importer and sys.modules cache if path is not + # defined in sys.path cls._clear_path_importer_cache_and_modules(path) @classmethod @@ -234,9 +237,12 @@ def _remove_from_sys_path(cls, path: str): If the path is an empty string, no action will be taken. """ if path and path in sys.path: - # Remove all occurances + # Remove all occurances in sys.path sys.path = list(filter(lambda p: p != path, sys.path)) - cls._clear_path_importer_cache_and_modules(path) + + # In case if any part of worker initialization do sys.path.pop() + # Always do a cache clear in path importer and sys.modules + cls._clear_path_importer_cache_and_modules(path) @classmethod def _clear_path_importer_cache_and_modules(cls, path: str): From e001a7ef1f12db327eca863100b7a05926783c17 Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Tue, 12 Jan 2021 21:42:38 -0800 Subject: [PATCH 7/7] Fix log entry --- azure_functions_worker/utils/dependency.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index 84dfd529a..d7ffd2332 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -200,7 +200,8 @@ def reload_all_namespaces_from_customer_deps(cls, cx_working_dir: str): cls._remove_from_sys_path(cls.worker_deps_path) cls._add_to_sys_path(cls.cx_deps_path, True) cls._add_to_sys_path(working_directory, False) - logger.info('Reloaded azure google namespace from customer deps') + logger.info('Reloaded azure google namespaces from ' + 'customer dependencies') @classmethod def _add_to_sys_path(cls, path: str, add_to_first: bool):