Setting up dev branch to be the v4 branch

.github/workflows/ci_e2e_workflow.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ 3.6, 3.7, 3.8, 3.9 ]
+        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]
 
     steps:
       - name: Checkout code.

.github/workflows/ut_ci_workflow.yml

@@ -4,6 +4,7 @@
 name: CI Unit tests
 
 on:
+  workflow_dispatch:
   schedule:
     # Monday to Thursday 1 AM PDT build
     # * is a special character in YAML so you have to quote this string
@@ -20,7 +21,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: [ 3.6, 3.7, 3.8, 3.9 ]
+        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

README.md

@@ -5,7 +5,7 @@
 |master|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=master)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=master)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=master)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=master)
 |dev|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=dev)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=dev)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=dev)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=dev)
 
-Python support for Azure Functions is based on Python 3.6, Python 3.7, and Python 3.8, serverless hosting on Linux and the Functions 2.0 and 3.0 runtime.
+Python support for Azure Functions is based on Python 3.6, Python 3.7, Python 3.8, and Python 3.9 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime.
 
 Here is the current status of Python in Azure Functions:
 
@@ -14,9 +14,12 @@ What are the supported Python versions?
 |Azure Functions Runtime|Python 3.6|Python 3.7|Python 3.8|Python 3.9|
 |---|---|---|---|---|
 |Azure Functions 2.0|✔|✔|-|-|
-|Azure Functions 3.0|✔|✔|✔|(preview)|
+|Azure Functions 3.0|✔|✔|✔|✔|
+|Azure Functions 4.0|-|✔|✔|✔|
 
-What's available?
+For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page.
+
+### What's available?
 
 - Build, test, debug and publish using Azure Functions Core Tools (CLI) or Visual Studio Code
 - Deploy Python Function project onto consumption, dedicated, or elastic premium plan.
@@ -28,7 +31,7 @@ What's coming?
 
 - [Durable Functions For Python](https://github.com/Azure/azure-functions-durable-python)
 
-# Get Started
+### Get Started
 
 - [Create your first Python function](https://docs.microsoft.com/en-us/azure/azure-functions/functions-create-first-function-python)
 - [Developer guide](https://docs.microsoft.com/en-us/azure/azure-functions/functions-reference-python)

azure-pipelines.yml

@@ -69,6 +69,15 @@ jobs:
       Python39V3:
         pythonVersion: '3.9'
         workerPath: 'python/prodV3/worker.py'
+      Python37V4:
+        pythonVersion: '3.7'
+        workerPath: 'python/prodV4/worker.py'
+      Python38V4:
+        pythonVersion: '3.8'
+        workerPath: 'python/prodV4/worker.py'
+      Python39V4:
+        pythonVersion: '3.9'
+        workerPath: 'python/prodV4/worker.py'
   steps:
   - template: pack/templates/win_env_gen.yml
     parameters:
@@ -94,6 +103,15 @@ jobs:
       Python39V3:
         pythonVersion: '3.9'
         workerPath: 'python/prodV3/worker.py'
+      Python37V4:
+        pythonVersion: '3.7'
+        workerPath: 'python/prodV4/worker.py'
+      Python38V4:
+        pythonVersion: '3.8'
+        workerPath: 'python/prodV4/worker.py'
+      Python39V4:
+        pythonVersion: '3.9'
+        workerPath: 'python/prodV4/worker.py'
   steps:
   - template: pack/templates/win_env_gen.yml
     parameters:
@@ -125,6 +143,15 @@ jobs:
       Python39V3:
         pythonVersion: '3.9'
         workerPath: 'python/prodV3/worker.py'
+      Python37V4:
+        pythonVersion: '3.7'
+        workerPath: 'python/prodV4/worker.py'
+      Python38V4:
+        pythonVersion: '3.8'
+        workerPath: 'python/prodV4/worker.py'
+      Python39V4:
+        pythonVersion: '3.9'
+        workerPath: 'python/prodV4/worker.py'
   steps:
   - template: pack/templates/nix_env_gen.yml
     parameters:
@@ -155,6 +182,15 @@ jobs:
       Python39V3:
         pythonVersion: '3.9'
         workerPath: 'python/prodV3/worker.py'
+      Python37V4:
+        pythonVersion: '3.7'
+        workerPath: 'python/prodV4/worker.py'
+      Python38V4:
+        pythonVersion: '3.8'
+        workerPath: 'python/prodV4/worker.py'
+      Python39V4:
+        pythonVersion: '3.9'
+        workerPath: 'python/prodV4/worker.py'
   steps:
   - template: pack/templates/nix_env_gen.yml
     parameters:
@@ -182,13 +218,18 @@ jobs:
         echo "Generating V3 Release Package for $BUILD_SOURCEBRANCHNAME"
         NUSPEC="pack\Microsoft.Azure.Functions.V3.PythonWorker.nuspec"
         WKVERSION="$BUILD_SOURCEBRANCHNAME"
+      elif [[ $BUILD_SOURCEBRANCHNAME = 4\.* ]]
+      then
+        echo "Generating V4 Release Package for $BUILD_SOURCEBRANCHNAME"
+        NUSPEC="pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec"
+        WKVERSION="$BUILD_SOURCEBRANCHNAME"
       elif [[ $BUILD_SOURCEBRANCHNAME = dev ]]
       then
-        echo "Generating V3 Integration Test Package for $BUILD_SOURCEBRANCHNAME"
-        LATEST_TAG=$(curl https://api.github.com/repos/Azure/azure-functions-python-worker/tags -s | jq '.[0].name' | sed 's/\"//g' |  cut -d'.' -f-2)
+        echo "Generating V4 Integration Test Package for $BUILD_SOURCEBRANCHNAME"
+        LATEST_TAG=$(cat azure_functions_worker/__init__.py | grep __version__ | cut -d'=' -f2 | tr -d \' | tr -d ' ')
         NUSPEC="pack\Microsoft.Azure.Functions.V3.PythonWorker.nuspec"
         # Only required for Integration Test. Version number contains date (e.g. 3.1.2.20211028-dev)
-        WKVERSION="3.$LATEST_TAG-$(patchBuildNumberForDev)"
+        WKVERSION="$LATEST_TAG.$(patchBuildNumberForDev)"
       else
         echo "No Matching Release Tag For $BUILD_SOURCEBRANCH"
       fi

azure_functions_worker/__init__.py

@@ -1,4 +1,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-__version__ = '1.2.6'
+__version__ = '4.0.0'

azure_functions_worker/testutils.py

@@ -444,15 +444,14 @@ class _WorkerResponseMessages(typing.NamedTuple):
 class _MockWebHost:
 
     def __init__(self, loop, scripts_dir):
-        self._loop = loop
         self._scripts_dir = scripts_dir
 
         self._available_functions = {}
         self._read_available_functions()
 
         self._connected_fut = loop.create_future()
-        self._in_queue = queue.Queue()
-        self._out_aqueue = asyncio.Queue(loop=self._loop)
+        self._in_queue: queue.Queue = queue.Queue()
+        self._out_aqueue: asyncio.Queue = asyncio.Queue()
         self._threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
         self._server = grpc.server(self._threadpool)
         self._servicer = _MockWebHostServicer(self)
@@ -894,7 +893,8 @@ def start_webhost(*, script_dir=None, stdout=None):
                 time.sleep(2)
                 break
             else:
-                print(f'Failed to ping {health_check_endpoint}', flush=True)
+                print(f'Failed to ping {health_check_endpoint} Please check the'
+                      f' log file for details: {stdout.name}', flush=True)
         except requests.exceptions.ConnectionError:
             pass
         time.sleep(2)

python/prodV4/worker.config.json

@@ -0,0 +1,12 @@
+{
+    "description":{
+        "language":"python",
+        "defaultRuntimeVersion":"3.9",
+        "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"],
+        "supportedRuntimeVersions":["3.7", "3.8", "3.9"],
+        "supportedArchitectures":["X64", "X86"],
+        "extensions":[".py"],
+        "defaultExecutablePath":"python",
+        "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py"
+    }
+}

python/prodV4/worker.py

@@ -0,0 +1,75 @@
+import os
+import sys
+
+from pathlib import Path
+
+# User packages
+PKGS_PATH = "site/wwwroot/.python_packages"
+VENV_PKGS_PATH = "site/wwwroot/worker_venv"
+PKGS = "lib/site-packages"
+
+# Azure environment variables
+AZURE_WEBSITE_INSTANCE_ID = "WEBSITE_INSTANCE_ID"
+AZURE_CONTAINER_NAME = "CONTAINER_NAME"
+AZURE_WEBJOBS_SCRIPT_ROOT = "AzureWebJobsScriptRoot"
+
+
+def is_azure_environment():
+    """Check if the function app is running on the cloud"""
+    return (AZURE_CONTAINER_NAME in os.environ
+            or AZURE_WEBSITE_INSTANCE_ID in os.environ)
+
+
+def add_script_root_to_sys_path():
+    """Append function project root to module finding sys.path"""
+    functions_script_root = os.getenv(AZURE_WEBJOBS_SCRIPT_ROOT)
+    if functions_script_root is not None:
+        sys.path.append(functions_script_root)
+
+
+def determine_user_pkg_paths():
+    """This finds the user packages when function apps are running on the cloud
+
+    For Python 3.7, 3.8, 3.9, we only accept:
+        /home/site/wwwroot/.python_packages/lib/site-packages
+    """
+    minor_version = sys.version_info[1]
+
+    home = Path.home()
+    pkgs_path = os.path.join(home, PKGS_PATH)
+    venv_pkgs_path = os.path.join(home, VENV_PKGS_PATH)
+
+    user_pkg_paths = []
+    if minor_version in (7, 8, 9):
+        user_pkg_paths.append(os.path.join(pkgs_path, PKGS))
+    else:
+        raise RuntimeError(f'Unsupported Python version: 3.{minor_version}')
+
+    return user_pkg_paths
+
+
+if __name__ == '__main__':
+    # worker.py lives in the same directory as azure_functions_worker
+    func_worker_dir = str(Path(__file__).absolute().parent)
+    env = os.environ
+
+    if is_azure_environment():
+        user_pkg_paths = determine_user_pkg_paths()
+
+        joined_pkg_paths = os.pathsep.join(user_pkg_paths)
+
+        # On cloud, we prioritize third-party user packages
+        # over worker packages in PYTHONPATH
+        env['PYTHONPATH'] = f'{joined_pkg_paths}:{func_worker_dir}'
+        os.execve(sys.executable,
+                  [sys.executable, '-m', 'azure_functions_worker']
+                  + sys.argv[1:],
+                  env)
+    else:
+        # On local development, we prioritize worker packages over
+        # third-party user packages (in .venv)
+        sys.path.insert(1, func_worker_dir)
+        add_script_root_to_sys_path()
+        from azure_functions_worker import main
+
+        main.main()

setup.py

@@ -34,27 +34,27 @@
   <ItemGroup>
     <PackageReference
         Include="Microsoft.NET.Sdk.Functions"
-        Version="3.0.3"
+        Version="3.0.13"
     />
     <PackageReference
         Include="Microsoft.Azure.WebJobs.Extensions.CosmosDB"
-        Version="3.0.5"
+        Version="3.0.9"
     />
     <PackageReference
         Include="Microsoft.Azure.WebJobs.Extensions.EventHubs"
-        Version="3.0.6"
+        Version="4.2.0"
     />
     <PackageReference
         Include="Microsoft.Azure.WebJobs.Extensions.EventGrid"
         Version="2.1.0"
     />
     <PackageReference
         Include="Microsoft.Azure.WebJobs.Extensions.Storage"
-        Version="3.0.10"
+        Version="4.0.4"
     />
     <PackageReference
-        Include="Microsoft.Azure.WebJobs.ServiceBus"
-        Version="3.0.0-beta8"
+        Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus"
+        Version="4.2.1"
     />
   </ItemGroup>
 </Project>
@@ -388,8 +388,9 @@ def run(self):
               'azure_functions_worker.utils',
               'azure_functions_worker._thirdparty'],
     install_requires=[
-        'grpcio~=1.33.2',
-        'grpcio-tools~=1.33.2',
+        'grpcio~=1.41.1',
+        'grpcio-tools~=1.41.1',
+        'protobuf~=3.19.1'
     ],
     extras_require={
         'dev': [

tests/unittests/test_utilities.py

@@ -314,16 +314,16 @@ def parse_int_no_raise(value: str):
 
     def test_is_python_version(self):
         # Should pass at least 1 test
-        is_python_version_36 = common.is_python_version('3.6')
         is_python_version_37 = common.is_python_version('3.7')
         is_python_version_38 = common.is_python_version('3.8')
         is_python_version_39 = common.is_python_version('3.9')
+        is_python_version_310 = common.is_python_version('3.10')
 
         self.assertTrue(any([
-            is_python_version_36,
             is_python_version_37,
             is_python_version_38,
-            is_python_version_39
+            is_python_version_39,
+            is_python_version_310
         ]))
 
     def test_get_sdk_from_sys_path(self):