Enabling Python 3.10 CI and other changes (#961)

vrdmr · web-flow · commit 18aafe9f9309 · 2022-02-12T07:22:35.000-08:00
* Removing loop in Queue and fixing Python version test
* Updating the workflow files for 3.10 CI
* Updating Prod V4 info
* Enabling UTs to run for any Push
* Correcting the test skip for DI
* Updating readme.md for Python 3.10
* Skipping LC E2E tests for 3.10
* Bumping up EH version
* nit changes to use is_python_version and bumping up versions
* Resolving flake8 issues
diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ 3.6, 3.7, 3.8, 3.9 ]
+        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]
 
     steps:
       - name: Checkout code.
@@ -113,6 +113,17 @@ jobs:
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
         run: |
           pytest  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+      - name: Running 3.10 Tests
+        if: matrix.python-version == 3.10
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString310 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString310 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString310 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
+        run: |
+          pytest  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Codecov
         uses: codecov/codecov-action@v1.0.13
         with:
diff --git a/.github/workflows/ut_ci_workflow.yml b/.github/workflows/ut_ci_workflow.yml
@@ -4,12 +4,12 @@
 name: CI Unit tests
 
 on:
+  workflow_dispatch:
   schedule:
     # Monday to Thursday 1 AM PDT build
     # * is a special character in YAML so you have to quote this string
     - cron: "0 8 * * 1,2,3,4"
   push:
-    branches: [ dev, master, main, release/* ]
   pull_request:
     branches: [ dev, master, main, release/* ]
 
@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: [ 3.6, 3.7, 3.8, 3.9 ]
+        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
diff --git a/README.md b/README.md
@@ -5,17 +5,17 @@
 |master|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=master)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=master)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=master)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=master)
 |dev|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=dev)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=dev)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=dev)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=dev)
 
-Python support for Azure Functions is based on Python 3.6, Python 3.7, Python 3.8, and Python 3.9 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime.
+Python support for Azure Functions is based on Python 3.6, Python 3.7, Python 3.8, Python 3.9, and Python 3.10 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime.
 
 Here is the current status of Python in Azure Functions:
 
 What are the supported Python versions?
 
-|Azure Functions Runtime|Python 3.6|Python 3.7|Python 3.8|Python 3.9|
-|---|---|---|---|---|
-|Azure Functions 2.0|✔|✔|-|-|
-|Azure Functions 3.0|✔|✔|✔|✔|
-|Azure Functions 4.0|-|✔|✔|✔|
+|Azure Functions Runtime|Python 3.6|Python 3.7|Python 3.8|Python 3.9|Python 3.10|
+|---|---|---|---|---|---|
+|Azure Functions 2.0|✔|✔|-|-|-|
+|Azure Functions 3.0|✔|✔|✔|✔|-|
+|Azure Functions 4.0|-|✔|✔|✔|preview|
 
 For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page.
 
diff --git a/azure_functions_worker/testutils.py b/azure_functions_worker/testutils.py
@@ -466,7 +466,7 @@ def __init__(self, loop, scripts_dir):
 
         self._connected_fut = loop.create_future()
         self._in_queue = queue.Queue()
-        self._out_aqueue = asyncio.Queue(loop=self._loop)
+        self._out_aqueue = asyncio.Queue()
         self._threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
         self._server = grpc.server(self._threadpool)
         self._servicer = _MockWebHostServicer(self)
diff --git a/python/prodV4/worker.config.json b/python/prodV4/worker.config.json
@@ -3,7 +3,7 @@
         "language":"python",
         "defaultRuntimeVersion":"3.9",
         "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"],
-        "supportedRuntimeVersions":["3.7", "3.8", "3.9"],
+        "supportedRuntimeVersions":["3.7", "3.8", "3.9", "3.10"],
         "supportedArchitectures":["X64", "X86"],
         "extensions":[".py"],
         "defaultExecutablePath":"python",
diff --git a/python/prodV4/worker.py b/python/prodV4/worker.py
@@ -38,14 +38,14 @@ def determine_user_pkg_paths():
 
     home = Path.home()
     pkgs_path = os.path.join(home, PKGS_PATH)
-    user_pkg_paths = []
+    usr_packages_path = []
 
-    if minor_version in (7, 8, 9):
-        user_pkg_paths.append(os.path.join(pkgs_path, PKGS))
+    if minor_version in (7, 8, 9, 10):
+        usr_packages_path.append(os.path.join(pkgs_path, PKGS))
     else:
         raise RuntimeError(f'Unsupported Python version: 3.{minor_version}')
 
-    return user_pkg_paths
+    return usr_packages_path
 
 
 if __name__ == '__main__':
diff --git a/setup.py b/setup.py
@@ -114,10 +114,10 @@
 
 EXTRA_REQUIRES = {
     "dev": [
-        "azure-eventhub~=5.1.0",
-        "python-dateutil~=2.8.1",
+        "azure-eventhub~=5.7.0",  # Used for EventHub E2E tests
+        "python-dateutil~=2.8.2",
         "pycryptodome~=3.10.1",
-        "flake8~=3.7.9",
+        "flake8~=4.0.1",
         "mypy",
         "pytest",
         "requests==2.*",
diff --git a/tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py b/tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py
@@ -33,4 +33,4 @@ def main(req: func.HttpRequest):
     with client:
         client.send_batch(event_data_batch)
 
-    return f'OK'
+    return 'OK'
diff --git a/tests/endtoend/eventhub_functions/metadata_output/__init__.py b/tests/endtoend/eventhub_functions/metadata_output/__init__.py
@@ -32,4 +32,4 @@ async def main(req: func.HttpRequest):
     finally:
         await client.close()
 
-    return f'OK'
+    return 'OK'
diff --git a/tests/endtoend/test_linux_consumption.py b/tests/endtoend/test_linux_consumption.py
@@ -1,6 +1,6 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-from unittest import TestCase
+from unittest import TestCase, skipIf
 
 import os
 import sys
@@ -9,8 +9,12 @@
 from azure_functions_worker.testutils_lc import (
     LinuxConsumptionWebHostController
 )
+from azure_functions_worker.utils.common import is_python_version
 
 
+@skipIf(is_python_version('3.10'),
+        "Skip the tests for Python 3.10 currently as the mesh images for "
+        "Python 3.10 aren't available currently.")
 class TestLinuxConsumption(TestCase):
     """Test worker behaviors on specific scenarios.
 
diff --git a/tests/unittests/resources/customer_func_path/common_module/__init__.py b/tests/unittests/resources/customer_func_path/common_module/__init__.py
@@ -1,9 +1,11 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+import os
 
-__version__: str == 'function_app'
 
-import os
+FUNCTION_APP = "function_app"
+__version__: str == FUNCTION_APP
+
 # This module should be shadowed from customer_deps_path/common_module
 # ./tests/unittests/resources/customer_func_path/common_module
 package_location: str = os.path.dirname(__file__)
diff --git a/tests/unittests/resources/customer_func_path/func_specific_module/__init__.py b/tests/unittests/resources/customer_func_path/func_specific_module/__init__.py
@@ -1,8 +1,10 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+import os
 
-__version__: str == 'function_app'
 
-import os
+FUNCTION_APP = "function_app"
+__version__: str == FUNCTION_APP
+
 # ./tests/unittests/resources/customer_func_path/func_specific_module
 package_location: str = os.path.dirname(__file__)
diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py
@@ -63,14 +63,14 @@ async def test_dispatcher_initialize_worker_logging(self):
         async with self._ctrl as host:
             r = await host.init_worker('3.0.12345')
             self.assertEqual(
-                len([l for l in r.logs if l.message.startswith(
+                len([log for log in r.logs if log.message.startswith(
                     'Received WorkerInitRequest'
                 )]),
                 1
             )
 
             self.assertEqual(
-                len([l for l in r.logs if l.message.startswith(
+                len([log for log in r.logs if log.message.startswith(
                     'To enable debug level logging'
                 )]),
                 1
@@ -86,14 +86,14 @@ async def test_dispatcher_environment_reload_logging(self):
             # Reload environment variable on specialization
             r = await host.reload_environment(environment={})
             self.assertEqual(
-                len([l for l in r.logs if l.message.startswith(
+                len([log for log in r.logs if log.message.startswith(
                     'Received FunctionEnvironmentReloadRequest'
                 )]),
                 1
             )
 
             self.assertEqual(
-                len([l for l in r.logs if l.message.startswith(
+                len([log for log in r.logs if log.message.startswith(
                     'To enable debug level logging'
                 )]),
                 1
diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py
@@ -9,6 +9,7 @@
 
 from azure_functions_worker import protos
 from azure_functions_worker import testutils
+from azure_functions_worker.utils.common import is_python_version
 
 
 class TestGRPC(testutils.AsyncTestCase):
@@ -128,8 +129,8 @@ def test_failed_azure_namespace_import(self):
     @unittest.skipIf(sys.platform == 'win32',
                      'Linux .sh script only works on Linux')
     @unittest.skipIf(
-        sys.version_info.major == 3 and sys.version_info.minor == 9,
-        'In Python 3.9, isolate worker dependencies is turned on by default.'
+        is_python_version('3.10'),
+        'In Python 3.10, isolate worker dependencies is turned on by default.'
         ' Reloading all customer dependencies on specialization is a must.'
         ' This partially reloading namespace feature is no longer needed.'
     )
diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py
@@ -318,12 +318,14 @@ def test_is_python_version(self):
         is_python_version_37 = common.is_python_version('3.7')
         is_python_version_38 = common.is_python_version('3.8')
         is_python_version_39 = common.is_python_version('3.9')
+        is_python_version_310 = common.is_python_version('3.10')
 
         self.assertTrue(any([
             is_python_version_36,
             is_python_version_37,
             is_python_version_38,
-            is_python_version_39
+            is_python_version_39,
+            is_python_version_310
         ]))
 
     def test_get_sdk_from_sys_path(self):
diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py
@@ -7,6 +7,7 @@
 from unittest.mock import patch
 
 from azure_functions_worker import testutils
+from azure_functions_worker.utils.common import is_python_version
 from azure_functions_worker.utils.dependency import DependencyManager
 
 
@@ -571,7 +572,7 @@ def test_use_worker_dependencies_default_python_36_37_38_39(self):
             import common_module  # NoQA
 
     @unittest.skipUnless(
-        sys.version_info.major == 3 and sys.version_info.minor == 10,
+        is_python_version('3.10'),
         'Test only available for python 3.10'
     )
     def test_use_worker_dependencies_default_python_310(self):
@@ -627,10 +628,8 @@ def test_prioritize_customer_dependencies_disable(self):
         with self.assertRaises(ImportError):
             import common_module  # NoQA
 
-    @unittest.skipIf(
-        sys.version_info.major == 3 and sys.version_info.minor == 10,
-        'Test not available for python 3.10'
-    )
+    @unittest.skipIf(is_python_version('3.10'),
+                     'Test not available for python 3.10')
     def test_prioritize_customer_dependencies_default_python_36_37_38_39(self):
         # Feature should be disabled in Python 3.6, 3.7, 3.8 and 3.9
         # Setup paths
@@ -643,10 +642,8 @@ def test_prioritize_customer_dependencies_default_python_36_37_38_39(self):
         with self.assertRaises(ImportError):
             import common_module  # NoQA
 
-    @unittest.skipUnless(
-        sys.version_info.major == 3 and sys.version_info.minor == 10,
-        'Test only available for python 3.10'
-    )
+    @unittest.skipUnless(is_python_version('3.10'),
+                         'Test only available for python 3.10')
     def test_prioritize_customer_dependencies_default_python_310(self):
         # Feature should be enabled in Python 3.10 by default
         # Setup paths
