Setting up dev as V4 branch for PyWorker

.flake8

@@ -8,6 +8,8 @@ ignore = W503,E402,E731
 exclude = .git, __pycache__, build, dist, .eggs, .github, .local, docs/,
           Samples, azure_functions_worker/protos/,
           azure_functions_worker/_thirdparty/typing_inspect.py,
-          tests/unittests/test_typing_inspect.py, .venv*, .env*, .vscode, venv*
+          tests/unittests/test_typing_inspect.py,
+          tests/unittests/broken_functions/syntax_error/main.py,
+          .env*, .vscode, venv*, *.venv*,
 
-max-line-length = 80
+max-line-length = 80

.github/workflows/ci_e2e_workflow.yml

@@ -38,6 +38,10 @@ jobs:
         uses: actions/setup-dotnet@v1
         with:
           dotnet-version: '5.0.x'
+      - name: Set up Dotnet 6.x
+        uses: actions/setup-dotnet@v1
+        with:
+          dotnet-version: '6.x'
       - name: Install dependencies and the worker
         run: |
           retry() {

.github/workflows/ut_ci_workflow.yml

@@ -35,6 +35,10 @@ jobs:
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: '5.0.x'
+    - name: Set up Dotnet 6.x
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: '6.x'
     - name: Install dependencies and the worker
       run: |
         retry() {

.gitignore

@@ -125,4 +125,9 @@ prof/
 
 # E2E Integration Test Core Tools
 .ci/e2e_integration_test/Azure.Functions.Cli/
-.ci/e2e_integration_test/Azure.Functions.Cli*
+.ci/e2e_integration_test/Azure.Functions.Cli*
+
+# Lingering test files
+tests/**/host.json
+tests/**/bin
+tests/**/extensions.csproj

README.md

@@ -5,7 +5,7 @@
 |master|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=master)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=master)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=master)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=master)
 |dev|[![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=dev)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=dev)|[![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker)|![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=dev)|![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=dev)
 
-Python support for Azure Functions is based on Python 3.6, Python 3.7, and Python 3.8, serverless hosting on Linux and the Functions 2.0 and 3.0 runtime.
+Python support for Azure Functions is based on Python 3.6, Python 3.7, Python 3.8, and Python 3.9 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime.
 
 Here is the current status of Python in Azure Functions:
 
@@ -14,9 +14,12 @@ What are the supported Python versions?
 |Azure Functions Runtime|Python 3.6|Python 3.7|Python 3.8|Python 3.9|
 |---|---|---|---|---|
 |Azure Functions 2.0|✔|✔|-|-|
-|Azure Functions 3.0|✔|✔|✔|(preview)|
+|Azure Functions 3.0|✔|✔|✔|✔|
+|Azure Functions 4.0|-|✔|✔|✔|
 
-What's available?
+For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page.
+
+### What's available?
 
 - Build, test, debug and publish using Azure Functions Core Tools (CLI) or Visual Studio Code
 - Deploy Python Function project onto consumption, dedicated, or elastic premium plan.
@@ -28,7 +31,7 @@ What's coming?
 
 - [Durable Functions For Python](https://github.com/Azure/azure-functions-durable-python)
 
-# Get Started
+###Get Started
 
 - [Create your first Python function](https://docs.microsoft.com/en-us/azure/azure-functions/functions-create-first-function-python)
 - [Developer guide](https://docs.microsoft.com/en-us/azure/azure-functions/functions-reference-python)

azure-pipelines.yml

@@ -1,14 +1,17 @@
 name: $(Date:yyyyMMdd).$(Rev:r)
 
 trigger:
-- release/2.*
 - release/3.*
+- release/4.*
 - dev
 
 variables:
     DOTNET_VERSION: '3.1.405'
     DOTNET_VERSION_5: '5.0.x'
+    DOTNET_VERSION_6: '6.x'
     patchBuildNumberForDev: $(Build.BuildNumber)
+    PROD_V3_WORKER_PY : 'python/prodV3/worker.py'
+    PROD_V4_WORKER_PY : 'python/prodV4/worker.py'
 
 jobs:
 - job: Tests
@@ -18,14 +21,14 @@ jobs:
       - ImageOverride -equals MMSUbuntu20.04TLS
   strategy:
     matrix:
-      Python36:
-        pythonVersion: '3.6'
       Python37:
         pythonVersion: '3.7'
       Python38:
         pythonVersion: '3.8'
       Python39:
         pythonVersion: '3.9'
+      Python310:
+        pythonVersion: '3.10'
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -41,6 +44,11 @@ jobs:
     inputs:
       packageType: 'sdk'
       version: $(DOTNET_VERSION_5)
+  - task: UseDotNet@2
+    displayName: 'Install DotNet 6.x'
+    inputs:
+      packageType: 'sdk'
+      version: $(DOTNET_VERSION_6)
   - task: ShellScript@2
     inputs:
       disableAutoCwd: true
@@ -55,24 +63,27 @@ jobs:
       - ImageOverride -equals MMS2019TLS 
   strategy:
     matrix:
-      Python36V2:
-        pythonVersion: '3.6'
-        workerPath: 'python/prodV2/worker.py'
-      Python37V2:
-        pythonVersion: '3.7'
-        workerPath: 'python/prodV2/worker.py'
       Python36V3:
         pythonVersion: '3.6'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python37V3:
         pythonVersion: '3.7'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python38V3:
         pythonVersion: '3.8'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python39V3:
         pythonVersion: '3.9'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
+      Python37V4:
+        pythonVersion: '3.7'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python38V4:
+        pythonVersion: '3.8'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python39V4:
+        pythonVersion: '3.9'
+        workerPath: $(PROD_V4_WORKER_PY)
   steps:
   - template: pack/templates/win_env_gen.yml
     parameters:
@@ -88,18 +99,24 @@ jobs:
       - ImageOverride -equals MMS2019TLS 
   strategy:
     matrix:
-      Python37V2:
-        pythonVersion: '3.7'
-        workerPath: 'python/prodV2/worker.py'
       Python37V3:
         pythonVersion: '3.7'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python38V3:
         pythonVersion: '3.8'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python39V3:
         pythonVersion: '3.9'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
+      Python37V4:
+        pythonVersion: '3.7'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python38V4:
+        pythonVersion: '3.8'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python39V4:
+        pythonVersion: '3.9'
+        workerPath: $(PROD_V4_WORKER_PY)
   steps:
   - template: pack/templates/win_env_gen.yml
     parameters:
@@ -115,24 +132,27 @@ jobs:
       - ImageOverride -equals MMSUbuntu20.04TLS
   strategy:
     matrix:
-      Python36V2:
-        pythonVersion: '3.6'
-        workerPath: 'python/prodV2/worker.py'
-      Python37V2:
-        pythonVersion: '3.7'
-        workerPath: 'python/prodV2/worker.py'
       Python36V3:
         pythonVersion: '3.6'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python37V3:
         pythonVersion: '3.7'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python38V3:
         pythonVersion: '3.8'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python39V3:
         pythonVersion: '3.9'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
+      Python37V4:
+        pythonVersion: '3.7'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python38V4:
+        pythonVersion: '3.8'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python39V4:
+        pythonVersion: '3.9'
+        workerPath: $(PROD_V4_WORKER_PY)
   steps:
   - template: pack/templates/nix_env_gen.yml
     parameters:
@@ -145,24 +165,27 @@ jobs:
     vmImage: 'macOS-10.15'
   strategy:
     matrix:
-      Python36V2:
-        pythonVersion: '3.6'
-        workerPath: 'python/prodV2/worker.py'
-      Python37V2:
-        pythonVersion: '3.7'
-        workerPath: 'python/prodV2/worker.py'
       Python36V3:
         pythonVersion: '3.6'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python37V3:
         pythonVersion: '3.7'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python38V3:
         pythonVersion: '3.8'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
       Python39V3:
         pythonVersion: '3.9'
-        workerPath: 'python/prodV3/worker.py'
+        workerPath: $(PROD_V3_WORKER_PY)
+      Python37V4:
+        pythonVersion: '3.7'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python38V4:
+        pythonVersion: '3.8'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python39V4:
+        pythonVersion: '3.9'
+        workerPath: $(PROD_V4_WORKER_PY)
   steps:
   - template: pack/templates/nix_env_gen.yml
     parameters:
@@ -174,32 +197,31 @@ jobs:
   dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64']
   condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev'))
   pool:
-      #vmImage: 'vs2017-win2016'
-      name: '1ES-Hosted-AzFunc' #MMS2019TLS for Windows2019 or MMSUbuntu20.04TLS for ubuntu
+      # vmImage: 'vs2017-win2016'
+      name: '1ES-Hosted-AzFunc' # MMS2019TLS for Windows2019 or MMSUbuntu20.04TLS for ubuntu
       demands:
       - ImageOverride -equals MMS2019TLS
   steps:
   - bash: |
       echo "Releasing from $BUILD_SOURCEBRANCHNAME"
       apt install jq
 
-      if [[ $BUILD_SOURCEBRANCHNAME = 2\.* ]]
-      then
-        echo "Generating V2 Release Package for $BUILD_SOURCEBRANCHNAME"
-        NUSPEC="pack\Microsoft.Azure.Functions.V2.PythonWorker.nuspec"
-        WKVERSION="$BUILD_SOURCEBRANCHNAME"
-      elif [[ $BUILD_SOURCEBRANCHNAME = 3\.* ]]
+      if [[ $BUILD_SOURCEBRANCHNAME = 3\.* ]]
       then
         echo "Generating V3 Release Package for $BUILD_SOURCEBRANCHNAME"
         NUSPEC="pack\Microsoft.Azure.Functions.V3.PythonWorker.nuspec"
         WKVERSION="$BUILD_SOURCEBRANCHNAME"
+      elif [[ $BUILD_SOURCEBRANCHNAME = 4\.* ]]
+      then
+        echo "Generating V4 Release Package for $BUILD_SOURCEBRANCHNAME"
+        NUSPEC="pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec"
+        WKVERSION="$BUILD_SOURCEBRANCHNAME"
       elif [[ $BUILD_SOURCEBRANCHNAME = dev ]]
       then
-        echo "Generating V3 Integration Test Package for $BUILD_SOURCEBRANCHNAME"
-        LATEST_TAG=$(curl https://api.github.com/repos/Azure/azure-functions-python-worker/tags -s | jq '.[0].name' | sed 's/\"//g' |  cut -d'.' -f-2)
-        NUSPEC="pack\Microsoft.Azure.Functions.V3.PythonWorker.nuspec"
-        # Only required for Integration Test. Version number contains date (e.g. 3.1.2.20211028-dev)
-        WKVERSION="3.$LATEST_TAG-$(patchBuildNumberForDev)"
+        echo "Generating V4 Integration Test Package for $BUILD_SOURCEBRANCHNAME"
+        VERSION=$(cat azure_functions_worker/version.py | tail -1 | cut -d' ' -f3 | sed  "s/'//g")
+        NUSPEC="pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec"
+        WKVERSION="$VERSION-$(patchBuildNumberForDev)"
       else
         echo "No Matching Release Tag For $BUILD_SOURCEBRANCH"
       fi
@@ -236,4 +258,4 @@ jobs:
       nuGetFeedType: 'internal'
       publishVstsFeed: 'e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df'
       allowPackageConflicts: true
-    displayName: '[Integration Test] Push NuGet package to the AzureFunctionsPreRelease feed'
+    displayName: '[Integration Test] Push NuGet package to the AzureFunctionsPreRelease feed'

azure_functions_worker/__init__.py

@@ -1,4 +1,2 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-
-__version__ = '1.2.6'

azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor.py

@@ -82,3 +82,16 @@ def _set_mem_map_initialized(self, mem_map: mmap.mmap):
         mem_map.write(consts.HeaderFlags.Initialized)
         # Seek back the memory map to the original position
         mem_map.seek(original_pos)
+
+
+class DummyFileAccessor(FileAccessor):
+    def open_mem_map(self, mem_map_name: str, mem_map_size: int,
+                     access: int = mmap.ACCESS_READ) -> Optional[mmap.mmap]:
+        pass
+
+    def create_mem_map(self, mem_map_name: str,
+                       mem_map_size: int) -> Optional[mmap.mmap]:
+        pass
+
+    def delete_mem_map(self, mem_map_name: str, mem_map: mmap.mmap) -> bool:
+        pass

azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py

@@ -2,8 +2,13 @@
 # Licensed under the MIT License.
 
 import os
+import sys
+
+from .file_accessor import DummyFileAccessor
 from .file_accessor_unix import FileAccessorUnix
 from .file_accessor_windows import FileAccessorWindows
+from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED
+from ...utils.common import is_envvar_true
 
 
 class FileAccessorFactory:
@@ -13,6 +18,9 @@ class FileAccessorFactory:
     """
     @staticmethod
     def create_file_accessor():
-        if os.name == 'nt':
+        if sys.platform == "darwin" and not is_envvar_true(
+                FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED):
+            return DummyFileAccessor()
+        elif os.name == 'nt':
             return FileAccessorWindows()
         return FileAccessorUnix()

azure_functions_worker/dispatcher.py

@@ -18,7 +18,6 @@
 
 import grpc
 
-from . import __version__
 from . import bindings
 from . import constants
 from . import functions
@@ -39,6 +38,8 @@
 from .utils.dependency import DependencyManager
 from .utils.tracing import marshall_exception_trace
 from .utils.wrappers import disable_feature_by
+from .version import VERSION
+
 
 _TRUE = "true"
 
@@ -262,7 +263,7 @@ async def _dispatch_grpc_request(self, request):
     async def _handle__worker_init_request(self, req):
         logger.info('Received WorkerInitRequest, '
                     'python version %s, worker version %s, request ID %s',
-                    sys.version, __version__, self.request_id)
+                    sys.version, VERSION, self.request_id)
         enable_debug_logging_recommendation()
 
         worker_init_request = req.worker_init_request