From fc2e738ac53c4afd11987e948f3b1d454d2d284a Mon Sep 17 00:00:00 2001 From: Varad Date: Thu, 10 Feb 2022 05:28:40 -0800 Subject: [PATCH 01/10] Removing loop in Queue and fixing Python version test --- azure_functions_worker/testutils.py | 2 +- tests/unittests/test_utilities.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/azure_functions_worker/testutils.py b/azure_functions_worker/testutils.py index e92108c5a..4638821fc 100644 --- a/azure_functions_worker/testutils.py +++ b/azure_functions_worker/testutils.py @@ -466,7 +466,7 @@ def __init__(self, loop, scripts_dir): self._connected_fut = loop.create_future() self._in_queue = queue.Queue() - self._out_aqueue = asyncio.Queue(loop=self._loop) + self._out_aqueue = asyncio.Queue() self._threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=1) self._server = grpc.server(self._threadpool) self._servicer = _MockWebHostServicer(self) diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index b658198cd..91db929b9 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -318,12 +318,14 @@ def test_is_python_version(self): is_python_version_37 = common.is_python_version('3.7') is_python_version_38 = common.is_python_version('3.8') is_python_version_39 = common.is_python_version('3.9') + is_python_version_310 = common.is_python_version('3.10') self.assertTrue(any([ is_python_version_36, is_python_version_37, is_python_version_38, - is_python_version_39 + is_python_version_39, + is_python_version_310 ])) def test_get_sdk_from_sys_path(self): From ab186336f78a00a719aaa51121b5b7a419e77fee Mon Sep 17 00:00:00 2001 From: Varad Date: Thu, 10 Feb 2022 07:19:14 -0800 Subject: [PATCH 02/10] Updating the workflow files for 3.10 CI --- .github/workflows/ci_e2e_workflow.yml | 13 ++++++++++++- .github/workflows/ut_ci_workflow.yml | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml index f0aad3634..eb80798fb 100644 --- a/.github/workflows/ci_e2e_workflow.yml +++ b/.github/workflows/ci_e2e_workflow.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ 3.6, 3.7, 3.8, 3.9 ] + python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ] steps: - name: Checkout code. @@ -113,6 +113,17 @@ jobs: AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }} run: | pytest --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend + - name: Running 3.10 Tests + if: matrix.python-version == 3.10 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} + AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString310 }} + AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString310 }} + AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString310 }} + AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }} + AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }} + run: | + pytest --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend - name: Codecov uses: codecov/codecov-action@v1.0.13 with: diff --git a/.github/workflows/ut_ci_workflow.yml b/.github/workflows/ut_ci_workflow.yml index ea2dd0b4f..a8c332a23 100644 --- a/.github/workflows/ut_ci_workflow.yml +++ b/.github/workflows/ut_ci_workflow.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: [ 3.6, 3.7, 3.8, 3.9 ] + python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 7e3b86201a930189f969d98bbd382bcde6d882fb Mon Sep 17 00:00:00 2001 From: Varad Date: Thu, 10 Feb 2022 07:22:08 -0800 Subject: [PATCH 03/10] Updating Prod V4 info --- python/prodV4/worker.config.json | 2 +- python/prodV4/worker.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/prodV4/worker.config.json b/python/prodV4/worker.config.json index d19d43b99..69e1adab1 100644 --- a/python/prodV4/worker.config.json +++ b/python/prodV4/worker.config.json @@ -3,7 +3,7 @@ "language":"python", "defaultRuntimeVersion":"3.9", "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"], - "supportedRuntimeVersions":["3.7", "3.8", "3.9"], + "supportedRuntimeVersions":["3.7", "3.8", "3.9", "3.10"], "supportedArchitectures":["X64", "X86"], "extensions":[".py"], "defaultExecutablePath":"python", diff --git a/python/prodV4/worker.py b/python/prodV4/worker.py index d9f4a697e..0aef47295 100644 --- a/python/prodV4/worker.py +++ b/python/prodV4/worker.py @@ -38,14 +38,14 @@ def determine_user_pkg_paths(): home = Path.home() pkgs_path = os.path.join(home, PKGS_PATH) - user_pkg_paths = [] + usr_packages_path = [] - if minor_version in (7, 8, 9): - user_pkg_paths.append(os.path.join(pkgs_path, PKGS)) + if minor_version in (7, 8, 9, 10): + usr_packages_path.append(os.path.join(pkgs_path, PKGS)) else: raise RuntimeError(f'Unsupported Python version: 3.{minor_version}') - return user_pkg_paths + return usr_packages_path if __name__ == '__main__': From 1c755b78a72d0ca7259e493099a1d2336edb9a39 Mon Sep 17 00:00:00 2001 From: Varad Date: Thu, 10 Feb 2022 07:27:27 -0800 Subject: [PATCH 04/10] Enabling UTs to run for any Push --- .github/workflows/ut_ci_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ut_ci_workflow.yml b/.github/workflows/ut_ci_workflow.yml index a8c332a23..dabee794d 100644 --- a/.github/workflows/ut_ci_workflow.yml +++ b/.github/workflows/ut_ci_workflow.yml @@ -4,12 +4,12 @@ name: CI Unit tests on: + workflow_dispatch: schedule: # Monday to Thursday 1 AM PDT build # * is a special character in YAML so you have to quote this string - cron: "0 8 * * 1,2,3,4" push: - branches: [ dev, master, main, release/* ] pull_request: branches: [ dev, master, main, release/* ] From 00d286b697e48e409ea10d720c2b4e67387ad043 Mon Sep 17 00:00:00 2001 From: Varad Date: Thu, 10 Feb 2022 07:59:32 -0800 Subject: [PATCH 05/10] Correcting the test skip for DI --- tests/unittests/test_rpc_messages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py index 63d82980c..2875a020e 100644 --- a/tests/unittests/test_rpc_messages.py +++ b/tests/unittests/test_rpc_messages.py @@ -128,8 +128,8 @@ def test_failed_azure_namespace_import(self): @unittest.skipIf(sys.platform == 'win32', 'Linux .sh script only works on Linux') @unittest.skipIf( - sys.version_info.major == 3 and sys.version_info.minor == 9, - 'In Python 3.9, isolate worker dependencies is turned on by default.' + sys.version_info.major == 3 and sys.version_info.minor == 10, + 'In Python 3.10, isolate worker dependencies is turned on by default.' ' Reloading all customer dependencies on specialization is a must.' ' This partially reloading namespace feature is no longer needed.' ) From ff607b4cda7d18af7142570e3923db9b69ff0a33 Mon Sep 17 00:00:00 2001 From: Varad Date: Thu, 10 Feb 2022 08:22:30 -0800 Subject: [PATCH 06/10] Updating readme.md for Python 3.10 --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7cdbb0e72..9d569d2f5 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,17 @@ |master|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=master)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=master)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=master)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=master) |dev|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=dev)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=dev)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=dev)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=dev) -Python support for Azure Functions is based on Python 3.6, Python 3.7, Python 3.8, and Python 3.9 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime. +Python support for Azure Functions is based on Python 3.6, Python 3.7, Python 3.8, Python 3.9, and Python 3.10 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime. Here is the current status of Python in Azure Functions: What are the supported Python versions? -|Azure Functions Runtime|Python 3.6|Python 3.7|Python 3.8|Python 3.9| -|---|---|---|---|---| -|Azure Functions 2.0|✔|✔|-|-| -|Azure Functions 3.0|✔|✔|✔|✔| -|Azure Functions 4.0|-|✔|✔|✔| +|Azure Functions Runtime|Python 3.6|Python 3.7|Python 3.8|Python 3.9|Python 3.10| +|---|---|---|---|---|---| +|Azure Functions 2.0|✔|✔|-|-|-| +|Azure Functions 3.0|✔|✔|✔|✔|-| +|Azure Functions 4.0|-|✔|✔|✔|preview| For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page. From 53420b36fe378ba6d523ea42708532ab5c4a29c9 Mon Sep 17 00:00:00 2001 From: Varad Date: Thu, 10 Feb 2022 19:56:56 -0800 Subject: [PATCH 07/10] Skipping LC E2E tests for 3.10 --- tests/endtoend/test_linux_consumption.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/endtoend/test_linux_consumption.py b/tests/endtoend/test_linux_consumption.py index df55ad0ac..dfa6693f4 100644 --- a/tests/endtoend/test_linux_consumption.py +++ b/tests/endtoend/test_linux_consumption.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from unittest import TestCase +from unittest import TestCase, skipIf import os import sys @@ -11,6 +11,9 @@ ) +@skipIf(sys.version_info.minor == 10, + "Skip the tests for Python 3.10 currently as the mesh images for " + "Python 3.10 aren't available currently.") class TestLinuxConsumption(TestCase): """Test worker behaviors on specific scenarios. From 83b56c753252280a595f55fe45767338928abb60 Mon Sep 17 00:00:00 2001 From: Varad Date: Fri, 11 Feb 2022 16:33:35 -0800 Subject: [PATCH 08/10] Bumping up EH version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f35c5142c..5a4612dc8 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ EXTRA_REQUIRES = { "dev": [ - "azure-eventhub~=5.1.0", + "azure-eventhub~=5.7.0", # Used for EventHub E2E tests "python-dateutil~=2.8.1", "pycryptodome~=3.10.1", "flake8~=3.7.9", From 2e7f175a4bf60351ee83afc2656a7a9058ad0d93 Mon Sep 17 00:00:00 2001 From: Varad Date: Fri, 11 Feb 2022 21:50:06 -0800 Subject: [PATCH 09/10] nit changes to use is_python_version and bumping up versions --- setup.py | 4 ++-- tests/endtoend/test_linux_consumption.py | 3 ++- tests/unittests/test_rpc_messages.py | 3 ++- tests/unittests/test_utilities_dependency.py | 15 ++++++--------- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index 5a4612dc8..315e936a8 100644 --- a/setup.py +++ b/setup.py @@ -115,9 +115,9 @@ EXTRA_REQUIRES = { "dev": [ "azure-eventhub~=5.7.0", # Used for EventHub E2E tests - "python-dateutil~=2.8.1", + "python-dateutil~=2.8.2", "pycryptodome~=3.10.1", - "flake8~=3.7.9", + "flake8~=4.0.1", "mypy", "pytest", "requests==2.*", diff --git a/tests/endtoend/test_linux_consumption.py b/tests/endtoend/test_linux_consumption.py index dfa6693f4..6a89cead4 100644 --- a/tests/endtoend/test_linux_consumption.py +++ b/tests/endtoend/test_linux_consumption.py @@ -9,9 +9,10 @@ from azure_functions_worker.testutils_lc import ( LinuxConsumptionWebHostController ) +from azure_functions_worker.utils.common import is_python_version -@skipIf(sys.version_info.minor == 10, +@skipIf(is_python_version('3.10'), "Skip the tests for Python 3.10 currently as the mesh images for " "Python 3.10 aren't available currently.") class TestLinuxConsumption(TestCase): diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py index 2875a020e..9026efe8c 100644 --- a/tests/unittests/test_rpc_messages.py +++ b/tests/unittests/test_rpc_messages.py @@ -9,6 +9,7 @@ from azure_functions_worker import protos from azure_functions_worker import testutils +from azure_functions_worker.utils.common import is_python_version class TestGRPC(testutils.AsyncTestCase): @@ -128,7 +129,7 @@ def test_failed_azure_namespace_import(self): @unittest.skipIf(sys.platform == 'win32', 'Linux .sh script only works on Linux') @unittest.skipIf( - sys.version_info.major == 3 and sys.version_info.minor == 10, + is_python_version('3.10'), 'In Python 3.10, isolate worker dependencies is turned on by default.' ' Reloading all customer dependencies on specialization is a must.' ' This partially reloading namespace feature is no longer needed.' diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py index 2f1ba56a1..59526997a 100644 --- a/tests/unittests/test_utilities_dependency.py +++ b/tests/unittests/test_utilities_dependency.py @@ -7,6 +7,7 @@ from unittest.mock import patch from azure_functions_worker import testutils +from azure_functions_worker.utils.common import is_python_version from azure_functions_worker.utils.dependency import DependencyManager @@ -571,7 +572,7 @@ def test_use_worker_dependencies_default_python_36_37_38_39(self): import common_module # NoQA @unittest.skipUnless( - sys.version_info.major == 3 and sys.version_info.minor == 10, + is_python_version('3.10'), 'Test only available for python 3.10' ) def test_use_worker_dependencies_default_python_310(self): @@ -627,10 +628,8 @@ def test_prioritize_customer_dependencies_disable(self): with self.assertRaises(ImportError): import common_module # NoQA - @unittest.skipIf( - sys.version_info.major == 3 and sys.version_info.minor == 10, - 'Test not available for python 3.10' - ) + @unittest.skipIf(is_python_version('3.10'), + 'Test not available for python 3.10') def test_prioritize_customer_dependencies_default_python_36_37_38_39(self): # Feature should be disabled in Python 3.6, 3.7, 3.8 and 3.9 # Setup paths @@ -643,10 +642,8 @@ def test_prioritize_customer_dependencies_default_python_36_37_38_39(self): with self.assertRaises(ImportError): import common_module # NoQA - @unittest.skipUnless( - sys.version_info.major == 3 and sys.version_info.minor == 10, - 'Test only available for python 3.10' - ) + @unittest.skipUnless(is_python_version('3.10'), + 'Test only available for python 3.10') def test_prioritize_customer_dependencies_default_python_310(self): # Feature should be enabled in Python 3.10 by default # Setup paths From d61f691bd911a8ca69a8b37a7d687c7ccb86357b Mon Sep 17 00:00:00 2001 From: Varad Date: Fri, 11 Feb 2022 22:10:49 -0800 Subject: [PATCH 10/10] Resolving flake8 issues --- .../metadata_output_batch/__init__.py | 2 +- .../eventhub_functions/metadata_output/__init__.py | 2 +- .../customer_func_path/common_module/__init__.py | 6 ++++-- .../customer_func_path/func_specific_module/__init__.py | 6 ++++-- tests/unittests/test_dispatcher.py | 8 ++++---- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py b/tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py index 0ceb782e6..88a50f3f3 100644 --- a/tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py +++ b/tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py @@ -33,4 +33,4 @@ def main(req: func.HttpRequest): with client: client.send_batch(event_data_batch) - return f'OK' + return 'OK' diff --git a/tests/endtoend/eventhub_functions/metadata_output/__init__.py b/tests/endtoend/eventhub_functions/metadata_output/__init__.py index 303d38d9e..5e7d32ebf 100644 --- a/tests/endtoend/eventhub_functions/metadata_output/__init__.py +++ b/tests/endtoend/eventhub_functions/metadata_output/__init__.py @@ -32,4 +32,4 @@ async def main(req: func.HttpRequest): finally: await client.close() - return f'OK' + return 'OK' diff --git a/tests/unittests/resources/customer_func_path/common_module/__init__.py b/tests/unittests/resources/customer_func_path/common_module/__init__.py index c001ad451..8a7910ea4 100644 --- a/tests/unittests/resources/customer_func_path/common_module/__init__.py +++ b/tests/unittests/resources/customer_func_path/common_module/__init__.py @@ -1,9 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import os -__version__: str == 'function_app' -import os +FUNCTION_APP = "function_app" +__version__: str == FUNCTION_APP + # This module should be shadowed from customer_deps_path/common_module # ./tests/unittests/resources/customer_func_path/common_module package_location: str = os.path.dirname(__file__) diff --git a/tests/unittests/resources/customer_func_path/func_specific_module/__init__.py b/tests/unittests/resources/customer_func_path/func_specific_module/__init__.py index 7e8399fae..a14ee353f 100644 --- a/tests/unittests/resources/customer_func_path/func_specific_module/__init__.py +++ b/tests/unittests/resources/customer_func_path/func_specific_module/__init__.py @@ -1,8 +1,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import os -__version__: str == 'function_app' -import os +FUNCTION_APP = "function_app" +__version__: str == FUNCTION_APP + # ./tests/unittests/resources/customer_func_path/func_specific_module package_location: str = os.path.dirname(__file__) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 2b6c3bfa0..40ac6f4f7 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -63,14 +63,14 @@ async def test_dispatcher_initialize_worker_logging(self): async with self._ctrl as host: r = await host.init_worker('3.0.12345') self.assertEqual( - len([l for l in r.logs if l.message.startswith( + len([log for log in r.logs if log.message.startswith( 'Received WorkerInitRequest' )]), 1 ) self.assertEqual( - len([l for l in r.logs if l.message.startswith( + len([log for log in r.logs if log.message.startswith( 'To enable debug level logging' )]), 1 @@ -86,14 +86,14 @@ async def test_dispatcher_environment_reload_logging(self): # Reload environment variable on specialization r = await host.reload_environment(environment={}) self.assertEqual( - len([l for l in r.logs if l.message.startswith( + len([log for log in r.logs if log.message.startswith( 'Received FunctionEnvironmentReloadRequest' )]), 1 ) self.assertEqual( - len([l for l in r.logs if l.message.startswith( + len([log for log in r.logs if log.message.startswith( 'To enable debug level logging' )]), 1