From 8151231feaae7bfe53475dd1bec08afd44204957 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 27 May 2022 16:14:19 -0500 Subject: [PATCH 1/6] Fix ModuleNotFoundError for older pinned functions versions --- azure_functions_worker/functions.py | 5 ++- azure_functions_worker/loader.py | 2 +- .../utils/library_importer.py | 33 +++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 azure_functions_worker/utils/library_importer.py diff --git a/azure_functions_worker/functions.py b/azure_functions_worker/functions.py index 085fb5de6..1f88ce381 100644 --- a/azure_functions_worker/functions.py +++ b/azure_functions_worker/functions.py @@ -5,7 +5,7 @@ import pathlib import typing -from azure.functions import DataType, Function +from azure_functions_worker.utils.library_importer import Function from . import bindings as bindings_utils from . import protos @@ -216,8 +216,7 @@ def validate_function_params(params: dict, bound_params: dict, param_bind_type, param_py_type) if not checks_out: - if binding.data_type is not DataType( - protos.BindingInfo.undefined): + if binding.data_type is not protos.BindingInfo.undefined: raise FunctionLoadError( func_name, f'{param.name!r} binding type "{binding.type}" ' diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index e5120a2b9..4fb50f6d2 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -12,7 +12,7 @@ from os import PathLike, fspath from typing import List, Optional, Dict -from azure.functions import Function, FunctionApp +from .utils.library_importer import Function, FunctionApp from . import protos, functions from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \ diff --git a/azure_functions_worker/utils/library_importer.py b/azure_functions_worker/utils/library_importer.py new file mode 100644 index 000000000..b462399f3 --- /dev/null +++ b/azure_functions_worker/utils/library_importer.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import logging + +""" +File to import library dependencies. +This file should be used to import any dependency from the library since +directly importing azure.functions may cause ModuleNotFound error is the user +has pinned their azure.functions to an older release version. + +""" + + +def get_azure_function(): + try: + from azure.functions import Function as func + return func + except ImportError: + logging.error("azure.functions is pinned to an older version. " + "Please pin azure.functions to version 1.10.1 or higher") + + +def get_azure_function_app(): + try: + from azure.functions import FunctionApp as funcApp + return funcApp + except ImportError: + logging.error("azure.functions is pinned to an older version. " + "Please pin azure.functions to version 1.10.1 or higher") + + +Function = get_azure_function() +FunctionApp = get_azure_function_app() From d452dc4fb8d353fee5f3afa329e4e7befa108b2c Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 27 May 2022 16:28:12 -0500 Subject: [PATCH 2/6] Organising imports --- azure_functions_worker/functions.py | 1 - azure_functions_worker/loader.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/azure_functions_worker/functions.py b/azure_functions_worker/functions.py index 1f88ce381..41331ec69 100644 --- a/azure_functions_worker/functions.py +++ b/azure_functions_worker/functions.py @@ -6,7 +6,6 @@ import typing from azure_functions_worker.utils.library_importer import Function - from . import bindings as bindings_utils from . import protos from ._thirdparty import typing_inspect diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index 4fb50f6d2..37dea059d 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -12,11 +12,10 @@ from os import PathLike, fspath from typing import List, Optional, Dict -from .utils.library_importer import Function, FunctionApp - from . import protos, functions from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \ PYTHON_LANGUAGE_RUNTIME +from .utils.library_importer import Function, FunctionApp from .utils.wrappers import attach_message_to_exception _AZURE_NAMESPACE = '__app__' From 0784018281dbb51ea0816bdb7dd811fe5faa030b Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 31 May 2022 10:53:07 -0500 Subject: [PATCH 3/6] Fixed unit tests --- tests/unittests/test_broken_functions.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/unittests/test_broken_functions.py b/tests/unittests/test_broken_functions.py index 0be21c84d..7d2aa2ded 100644 --- a/tests/unittests/test_broken_functions.py +++ b/tests/unittests/test_broken_functions.py @@ -201,10 +201,9 @@ async def test_load_broken__invalid_http_trigger_anno(self): self.assertEqual( r.response.result.exception.message, - 'FunctionLoadError: cannot load the invalid_http_trigger_anno ' - 'function: \'req\' binding type "httpTrigger" and dataType "0"' - ' in function.json do not match the corresponding function' - ' parameter\'s Python type annotation "int"') + 'FunctionLoadError: cannot load the invalid_http_trigger_anno' + ' function: type of req binding in function.json "httpTrigger" ' + 'does not match its Python annotation "int"') async def test_load_broken__invalid_out_anno(self): async with testutils.start_mockhost( @@ -218,9 +217,8 @@ async def test_load_broken__invalid_out_anno(self): self.assertEqual( r.response.result.exception.message, 'FunctionLoadError: cannot load the invalid_out_anno function: ' - '\'ret\' binding type "http" and dataType "0" in function.json' - ' do not match the corresponding function parameter\'s Python' - ' type annotation "HttpRequest"') + r'type of ret binding in function.json "http" ' + r'does not match its Python annotation "HttpRequest"') async def test_load_broken__invalid_in_anno(self): async with testutils.start_mockhost( @@ -233,10 +231,9 @@ async def test_load_broken__invalid_in_anno(self): self.assertEqual( r.response.result.exception.message, - 'FunctionLoadError: cannot load the invalid_in_anno function:' - ' \'req\' binding type "httpTrigger" and dataType "0" in ' - 'function.json do not match the corresponding function ' - 'parameter\'s Python type annotation "HttpResponse"') + 'FunctionLoadError: cannot load the invalid_in_anno function: ' + r'type of req binding in function.json "httpTrigger" ' + r'does not match its Python annotation "HttpResponse"') async def test_load_broken__invalid_in_anno_non_type(self): async with testutils.start_mockhost( From 3c4f56da920698cd29f7f630de16d3f03c1f3bec Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Wed, 1 Jun 2022 15:56:54 -0500 Subject: [PATCH 4/6] Added unit test --- .../invalid_datatype/function.json | 11 +++++++++++ .../broken_functions/invalid_datatype/main.py | 7 +++++++ tests/unittests/test_broken_functions.py | 16 ++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 tests/unittests/broken_functions/invalid_datatype/function.json create mode 100644 tests/unittests/broken_functions/invalid_datatype/main.py diff --git a/tests/unittests/broken_functions/invalid_datatype/function.json b/tests/unittests/broken_functions/invalid_datatype/function.json new file mode 100644 index 000000000..247beea27 --- /dev/null +++ b/tests/unittests/broken_functions/invalid_datatype/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "type": "httpTrigger", + "direction": "in", + "dataType" : "string", + "name": "req" + } + ] +} diff --git a/tests/unittests/broken_functions/invalid_datatype/main.py b/tests/unittests/broken_functions/invalid_datatype/main.py new file mode 100644 index 000000000..0fbe6b520 --- /dev/null +++ b/tests/unittests/broken_functions/invalid_datatype/main.py @@ -0,0 +1,7 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import azure.functions as azf + + +def main(req: azf.HttpResponse): + return 'This function should fail!!' diff --git a/tests/unittests/test_broken_functions.py b/tests/unittests/test_broken_functions.py index 7d2aa2ded..bf92fb615 100644 --- a/tests/unittests/test_broken_functions.py +++ b/tests/unittests/test_broken_functions.py @@ -235,6 +235,22 @@ async def test_load_broken__invalid_in_anno(self): r'type of req binding in function.json "httpTrigger" ' r'does not match its Python annotation "HttpResponse"') + async def test_load_broken__invalid_datatype(self): + async with testutils.start_mockhost( + script_root=self.broken_funcs_dir) as host: + func_id, r = await host.load_function('invalid_datatype') + + self.assertEqual(r.response.function_id, func_id) + self.assertEqual(r.response.result.status, + protos.StatusResult.Failure) + + self.assertRegex( + r.response.result.exception.message, + r'.*cannot load the invalid_datatype function: ' + r'.*binding type "httpTrigger" and dataType "1" in ' + r'function.json do not match the corresponding function ' + r'parameter.* Python type annotation "HttpResponse"') + async def test_load_broken__invalid_in_anno_non_type(self): async with testutils.start_mockhost( script_root=self.broken_funcs_dir) as host: From ee43db3fac8cc6f91cbc7213cfbc9f8b394f7534 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Wed, 8 Jun 2022 17:31:31 -0500 Subject: [PATCH 5/6] Added unit tests --- tests/unittests/test_utilities.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 91db929b9..e28a26c19 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -6,7 +6,7 @@ from unittest.mock import patch import typing -from azure_functions_worker.utils import common, wrappers +from azure_functions_worker.utils import common, wrappers, library_importer TEST_APP_SETTING_NAME = "TEST_APP_SETTING_NAME" @@ -367,3 +367,14 @@ def _unset_feature_flag(self): os.environ.pop(TEST_FEATURE_FLAG) except KeyError: pass + + def test_library_importer_invalid_import(self): + sys.path.clear() + sys.modules.pop('azure.functions') + sys.path.insert(0, self._dummy_sdk_sys_path) + common.get_sdk_from_sys_path() + functionapp = library_importer.get_azure_function_app() + self.assertIsNone(functionapp) + + function = library_importer.get_azure_function_app() + self.assertIsNone(function) From 3a627df4e5f4752ac38378edea2d5b791232a3fb Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Tue, 14 Jun 2022 14:03:40 -0500 Subject: [PATCH 6/6] Remove library imports --- azure_functions_worker/functions.py | 3 +- azure_functions_worker/loader.py | 10 +++--- .../utils/library_importer.py | 33 ------------------- tests/unittests/test_utilities.py | 16 ++------- 4 files changed, 8 insertions(+), 54 deletions(-) delete mode 100644 azure_functions_worker/utils/library_importer.py diff --git a/azure_functions_worker/functions.py b/azure_functions_worker/functions.py index 41331ec69..04f7652c0 100644 --- a/azure_functions_worker/functions.py +++ b/azure_functions_worker/functions.py @@ -5,7 +5,6 @@ import pathlib import typing -from azure_functions_worker.utils.library_importer import Function from . import bindings as bindings_utils from . import protos from ._thirdparty import typing_inspect @@ -355,7 +354,7 @@ def add_function(self, function_id: str, output_types, return_type) def add_indexed_function(self, function_id: str, - function: Function): + function): func = function.get_user_function() func_name = function.get_function_name() return_binding_name: typing.Optional[str] = None diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index 37dea059d..36787e651 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -10,12 +10,11 @@ import sys import uuid from os import PathLike, fspath -from typing import List, Optional, Dict +from typing import Optional, Dict from . import protos, functions from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \ PYTHON_LANGUAGE_RUNTIME -from .utils.library_importer import Function, FunctionApp from .utils.wrappers import attach_message_to_exception _AZURE_NAMESPACE = '__app__' @@ -46,7 +45,7 @@ def uninstall() -> None: pass -def build_binding_protos(indexed_function: List[Function]) -> Dict: +def build_binding_protos(indexed_function) -> Dict: binding_protos = {} for binding in indexed_function.get_bindings(): binding_protos[binding.name] = protos.BindingInfo( @@ -58,7 +57,7 @@ def build_binding_protos(indexed_function: List[Function]) -> Dict: def process_indexed_function(functions_registry: functions.Registry, - indexed_functions: List[Function]): + indexed_functions): fx_metadata_results = [] for indexed_function in indexed_functions: function_id = str(uuid.uuid4()) @@ -140,10 +139,11 @@ def load_function(name: str, directory: str, script_file: str, expt_type=ImportError, message=f'Troubleshooting Guide: {MODULE_NOT_FOUND_TS_URL}' ) -def index_function_app(function_path: str) -> List[Function]: +def index_function_app(function_path: str): module_name = pathlib.Path(function_path).stem imported_module = importlib.import_module(module_name) + from azure.functions import FunctionApp app: Optional[FunctionApp] = None for i in imported_module.__dir__(): if isinstance(getattr(imported_module, i, None), FunctionApp): diff --git a/azure_functions_worker/utils/library_importer.py b/azure_functions_worker/utils/library_importer.py deleted file mode 100644 index b462399f3..000000000 --- a/azure_functions_worker/utils/library_importer.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -import logging - -""" -File to import library dependencies. -This file should be used to import any dependency from the library since -directly importing azure.functions may cause ModuleNotFound error is the user -has pinned their azure.functions to an older release version. - -""" - - -def get_azure_function(): - try: - from azure.functions import Function as func - return func - except ImportError: - logging.error("azure.functions is pinned to an older version. " - "Please pin azure.functions to version 1.10.1 or higher") - - -def get_azure_function_app(): - try: - from azure.functions import FunctionApp as funcApp - return funcApp - except ImportError: - logging.error("azure.functions is pinned to an older version. " - "Please pin azure.functions to version 1.10.1 or higher") - - -Function = get_azure_function() -FunctionApp = get_azure_function_app() diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index e28a26c19..7112a27c5 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -2,12 +2,11 @@ # Licensed under the MIT License. import os import sys +import typing import unittest from unittest.mock import patch -import typing - -from azure_functions_worker.utils import common, wrappers, library_importer +from azure_functions_worker.utils import common, wrappers TEST_APP_SETTING_NAME = "TEST_APP_SETTING_NAME" TEST_FEATURE_FLAG = "APP_SETTING_FEATURE_FLAG" @@ -367,14 +366,3 @@ def _unset_feature_flag(self): os.environ.pop(TEST_FEATURE_FLAG) except KeyError: pass - - def test_library_importer_invalid_import(self): - sys.path.clear() - sys.modules.pop('azure.functions') - sys.path.insert(0, self._dummy_sdk_sys_path) - common.get_sdk_from_sys_path() - functionapp = library_importer.get_azure_function_app() - self.assertIsNone(functionapp) - - function = library_importer.get_azure_function_app() - self.assertIsNone(function)