From 8dec501e92621bfc57d7c14f897732c069668857 Mon Sep 17 00:00:00 2001 From: Varad Date: Thu, 11 Nov 2021 00:53:19 -0800 Subject: [PATCH 1/2] Setting up dev branch to be the v4 branch --- .github/workflows/ci_e2e_workflow.yml | 2 +- .github/workflows/ut_ci_workflow.yml | 3 +- README.md | 11 ++-- azure-pipelines.yml | 47 +++++++++++++++-- azure_functions_worker/__init__.py | 2 +- python/prodV4/worker.config.json | 12 +++++ python/prodV4/worker.py | 75 +++++++++++++++++++++++++++ setup.py | 17 +++--- 8 files changed, 151 insertions(+), 18 deletions(-) create mode 100644 python/prodV4/worker.config.json create mode 100644 python/prodV4/worker.py diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml index 006f8b466..4adac16ec 100644 --- a/.github/workflows/ci_e2e_workflow.yml +++ b/.github/workflows/ci_e2e_workflow.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ 3.6, 3.7, 3.8, 3.9 ] + python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ] steps: - name: Checkout code. diff --git a/.github/workflows/ut_ci_workflow.yml b/.github/workflows/ut_ci_workflow.yml index 6fc13a455..dffeb8b82 100644 --- a/.github/workflows/ut_ci_workflow.yml +++ b/.github/workflows/ut_ci_workflow.yml @@ -4,6 +4,7 @@ name: CI Unit tests on: + workflow_dispatch: schedule: # Monday to Thursday 1 AM PDT build # * is a special character in YAML so you have to quote this string @@ -20,7 +21,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: [ 3.6, 3.7, 3.8, 3.9 ] + python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/README.md b/README.md index ff526c582..d3ab71192 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ |master|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=master)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=master)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=master)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=master) |dev|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=dev)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=dev)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=dev)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=dev) -Python support for Azure Functions is based on Python 3.6, Python 3.7, and Python 3.8, serverless hosting on Linux and the Functions 2.0 and 3.0 runtime. +Python support for Azure Functions is based on Python 3.6, Python 3.7, Python 3.8, and Python 3.9 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime. Here is the current status of Python in Azure Functions: @@ -14,9 +14,12 @@ What are the supported Python versions? |Azure Functions Runtime|Python 3.6|Python 3.7|Python 3.8|Python 3.9| |---|---|---|---|---| |Azure Functions 2.0|✔|✔|-|-| -|Azure Functions 3.0|✔|✔|✔|(preview)| +|Azure Functions 3.0|✔|✔|✔|✔| +|Azure Functions 4.0|-|✔|✔|✔| -What's available? +For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page. + +### What's available? - Build, test, debug and publish using Azure Functions Core Tools (CLI) or Visual Studio Code - Deploy Python Function project onto consumption, dedicated, or elastic premium plan. @@ -28,7 +31,7 @@ What's coming? - [Durable Functions For Python](https://github.com/Azure/azure-functions-durable-python) -# Get Started +### Get Started - [Create your first Python function](https://docs.microsoft.com/en-us/azure/azure-functions/functions-create-first-function-python) - [Developer guide](https://docs.microsoft.com/en-us/azure/azure-functions/functions-reference-python) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b56ca090c..651a7e0c1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -69,6 +69,15 @@ jobs: Python39V3: pythonVersion: '3.9' workerPath: 'python/prodV3/worker.py' + Python37V4: + pythonVersion: '3.7' + workerPath: 'python/prodV4/worker.py' + Python38V4: + pythonVersion: '3.8' + workerPath: 'python/prodV4/worker.py' + Python39V4: + pythonVersion: '3.9' + workerPath: 'python/prodV4/worker.py' steps: - template: pack/templates/win_env_gen.yml parameters: @@ -94,6 +103,15 @@ jobs: Python39V3: pythonVersion: '3.9' workerPath: 'python/prodV3/worker.py' + Python37V4: + pythonVersion: '3.7' + workerPath: 'python/prodV4/worker.py' + Python38V4: + pythonVersion: '3.8' + workerPath: 'python/prodV4/worker.py' + Python39V4: + pythonVersion: '3.9' + workerPath: 'python/prodV4/worker.py' steps: - template: pack/templates/win_env_gen.yml parameters: @@ -125,6 +143,15 @@ jobs: Python39V3: pythonVersion: '3.9' workerPath: 'python/prodV3/worker.py' + Python37V4: + pythonVersion: '3.7' + workerPath: 'python/prodV4/worker.py' + Python38V4: + pythonVersion: '3.8' + workerPath: 'python/prodV4/worker.py' + Python39V4: + pythonVersion: '3.9' + workerPath: 'python/prodV4/worker.py' steps: - template: pack/templates/nix_env_gen.yml parameters: @@ -155,6 +182,15 @@ jobs: Python39V3: pythonVersion: '3.9' workerPath: 'python/prodV3/worker.py' + Python37V4: + pythonVersion: '3.7' + workerPath: 'python/prodV4/worker.py' + Python38V4: + pythonVersion: '3.8' + workerPath: 'python/prodV4/worker.py' + Python39V4: + pythonVersion: '3.9' + workerPath: 'python/prodV4/worker.py' steps: - template: pack/templates/nix_env_gen.yml parameters: @@ -182,13 +218,18 @@ jobs: echo "Generating V3 Release Package for $BUILD_SOURCEBRANCHNAME" NUSPEC="pack\Microsoft.Azure.Functions.V3.PythonWorker.nuspec" WKVERSION="$BUILD_SOURCEBRANCHNAME" + elif [[ $BUILD_SOURCEBRANCHNAME = 4\.* ]] + then + echo "Generating V4 Release Package for $BUILD_SOURCEBRANCHNAME" + NUSPEC="pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec" + WKVERSION="$BUILD_SOURCEBRANCHNAME" elif [[ $BUILD_SOURCEBRANCHNAME = dev ]] then - echo "Generating V3 Integration Test Package for $BUILD_SOURCEBRANCHNAME" - LATEST_TAG=$(curl https://api.github.com/repos/Azure/azure-functions-python-worker/tags -s | jq '.[0].name' | sed 's/\"//g' | cut -d'.' -f-2) + echo "Generating V4 Integration Test Package for $BUILD_SOURCEBRANCHNAME" + LATEST_TAG=$(cat azure_functions_worker/__init__.py | grep __version__ | cut -d'=' -f2 | tr -d \' | tr -d ' ') NUSPEC="pack\Microsoft.Azure.Functions.V3.PythonWorker.nuspec" # Only required for Integration Test. Version number contains date (e.g. 3.1.2.20211028-dev) - WKVERSION="3.$LATEST_TAG-$(patchBuildNumberForDev)" + WKVERSION="$LATEST_TAG.$(patchBuildNumberForDev)" else echo "No Matching Release Tag For $BUILD_SOURCEBRANCH" fi diff --git a/azure_functions_worker/__init__.py b/azure_functions_worker/__init__.py index 2e6e647f0..f85fa6a01 100644 --- a/azure_functions_worker/__init__.py +++ b/azure_functions_worker/__init__.py @@ -1,4 +1,4 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -__version__ = '1.2.6' +__version__ = '4.0.0' diff --git a/python/prodV4/worker.config.json b/python/prodV4/worker.config.json new file mode 100644 index 000000000..5bca51fab --- /dev/null +++ b/python/prodV4/worker.config.json @@ -0,0 +1,12 @@ +{ + "description":{ + "language":"python", + "defaultRuntimeVersion":"3.9", + "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"], + "supportedRuntimeVersions":["3.7", "3.8", "3.9"], + "supportedArchitectures":["X64", "X86"], + "extensions":[".py"], + "defaultExecutablePath":"python", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py" + } +} diff --git a/python/prodV4/worker.py b/python/prodV4/worker.py new file mode 100644 index 000000000..ea49d1e1d --- /dev/null +++ b/python/prodV4/worker.py @@ -0,0 +1,75 @@ +import os +import sys + +from pathlib import Path + +# User packages +PKGS_PATH = "site/wwwroot/.python_packages" +VENV_PKGS_PATH = "site/wwwroot/worker_venv" +PKGS = "lib/site-packages" + +# Azure environment variables +AZURE_WEBSITE_INSTANCE_ID = "WEBSITE_INSTANCE_ID" +AZURE_CONTAINER_NAME = "CONTAINER_NAME" +AZURE_WEBJOBS_SCRIPT_ROOT = "AzureWebJobsScriptRoot" + + +def is_azure_environment(): + """Check if the function app is running on the cloud""" + return (AZURE_CONTAINER_NAME in os.environ + or AZURE_WEBSITE_INSTANCE_ID in os.environ) + + +def add_script_root_to_sys_path(): + """Append function project root to module finding sys.path""" + functions_script_root = os.getenv(AZURE_WEBJOBS_SCRIPT_ROOT) + if functions_script_root is not None: + sys.path.append(functions_script_root) + + +def determine_user_pkg_paths(): + """This finds the user packages when function apps are running on the cloud + + For Python 3.7, 3.8, 3.9, we only accept: + /home/site/wwwroot/.python_packages/lib/site-packages + """ + minor_version = sys.version_info[1] + + home = Path.home() + pkgs_path = os.path.join(home, PKGS_PATH) + venv_pkgs_path = os.path.join(home, VENV_PKGS_PATH) + + user_pkg_paths = [] + if minor_version in (7, 8, 9): + user_pkg_paths.append(os.path.join(pkgs_path, PKGS)) + else: + raise RuntimeError(f'Unsupported Python version: 3.{minor_version}') + + return user_pkg_paths + + +if __name__ == '__main__': + # worker.py lives in the same directory as azure_functions_worker + func_worker_dir = str(Path(__file__).absolute().parent) + env = os.environ + + if is_azure_environment(): + user_pkg_paths = determine_user_pkg_paths() + + joined_pkg_paths = os.pathsep.join(user_pkg_paths) + + # On cloud, we prioritize third-party user packages + # over worker packages in PYTHONPATH + env['PYTHONPATH'] = f'{joined_pkg_paths}:{func_worker_dir}' + os.execve(sys.executable, + [sys.executable, '-m', 'azure_functions_worker'] + + sys.argv[1:], + env) + else: + # On local development, we prioritize worker packages over + # third-party user packages (in .venv) + sys.path.insert(1, func_worker_dir) + add_script_root_to_sys_path() + from azure_functions_worker import main + + main.main() diff --git a/setup.py b/setup.py index f173ae5d0..9507f2b61 100644 --- a/setup.py +++ b/setup.py @@ -34,15 +34,15 @@ @@ -388,8 +388,9 @@ def run(self): 'azure_functions_worker.utils', 'azure_functions_worker._thirdparty'], install_requires=[ - 'grpcio~=1.33.2', - 'grpcio-tools~=1.33.2', + 'grpcio~=1.41.1', + 'grpcio-tools~=1.41.1', + 'protobuf~=3.19.1' ], extras_require={ 'dev': [ From 25866f74c74bfe8bbcb6d2f2b429fdd495307581 Mon Sep 17 00:00:00 2001 From: Varad Date: Mon, 15 Nov 2021 15:02:38 -0800 Subject: [PATCH 2/2] Removing the queue as its deprecated for 3.10 --- azure_functions_worker/testutils.py | 8 ++++---- tests/unittests/test_utilities.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/azure_functions_worker/testutils.py b/azure_functions_worker/testutils.py index 91656cf81..0f90c7c60 100644 --- a/azure_functions_worker/testutils.py +++ b/azure_functions_worker/testutils.py @@ -444,15 +444,14 @@ class _WorkerResponseMessages(typing.NamedTuple): class _MockWebHost: def __init__(self, loop, scripts_dir): - self._loop = loop self._scripts_dir = scripts_dir self._available_functions = {} self._read_available_functions() self._connected_fut = loop.create_future() - self._in_queue = queue.Queue() - self._out_aqueue = asyncio.Queue(loop=self._loop) + self._in_queue: queue.Queue = queue.Queue() + self._out_aqueue: asyncio.Queue = asyncio.Queue() self._threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=1) self._server = grpc.server(self._threadpool) self._servicer = _MockWebHostServicer(self) @@ -894,7 +893,8 @@ def start_webhost(*, script_dir=None, stdout=None): time.sleep(2) break else: - print(f'Failed to ping {health_check_endpoint}', flush=True) + print(f'Failed to ping {health_check_endpoint} Please check the' + f' log file for details: {stdout.name}', flush=True) except requests.exceptions.ConnectionError: pass time.sleep(2) diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index b658198cd..110bb8f6e 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -314,16 +314,16 @@ def parse_int_no_raise(value: str): def test_is_python_version(self): # Should pass at least 1 test - is_python_version_36 = common.is_python_version('3.6') is_python_version_37 = common.is_python_version('3.7') is_python_version_38 = common.is_python_version('3.8') is_python_version_39 = common.is_python_version('3.9') + is_python_version_310 = common.is_python_version('3.10') self.assertTrue(any([ - is_python_version_36, is_python_version_37, is_python_version_38, - is_python_version_39 + is_python_version_39, + is_python_version_310 ])) def test_get_sdk_from_sys_path(self):