From 74c77b115081e3dae0cef344bcd7f25017ea55fd Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Thu, 10 Oct 2024 10:29:36 -0500 Subject: [PATCH 1/6] pin to extension versions when testing --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cdba09cc7..657ddc62a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,12 +72,12 @@ dev = [ "invoke" ] test-http-v2 = [ - "azurefunctions-extensions-http-fastapi", + "azurefunctions-extensions-http-fastapi==1.0.0b1", "ujson", "orjson" ] test-deferred-bindings = [ - "azurefunctions-extensions-bindings-blob" + "azurefunctions-extensions-bindings-blob==1.0.0b2" ] [build-system] From 54bc3e0245aecd362ef58d82c6e40344fa845c1e Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Thu, 10 Oct 2024 13:46:23 -0500 Subject: [PATCH 2/6] add eg sourced blob functions --- .../blob_functions_stein/function_app.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py b/tests/endtoend/blob_functions/blob_functions_stein/function_app.py index 2621a3360..ea872bde2 100644 --- a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py +++ b/tests/endtoend/blob_functions/blob_functions_stein/function_app.py @@ -391,3 +391,55 @@ def put_get_multiple_blobs_as_bytes_return_http_response( mimetype="application/json", status_code=200 ) + + +@app.function_name(name="blob_trigger_default_source_enum") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source=func.BlobSource.LOGS_AND_CONTAINER_SCAN) +def blob_trigger_default_source_enum(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_eventgrid_source_enum") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source=func.BlobSource.EVENT_GRID) +def blob_trigger_eventgrid_source_enum(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_default_source_str") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source="LogsAndContainerScan") +def blob_trigger_default_source_str(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="blob_trigger_eventgrid_source_str") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage", + source="EventGrid") +def blob_trigger_eventgrid_source_str(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) From 7ba94222782bca087c8fda349ccdbab47a195c8c Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 15 Oct 2024 10:07:24 -0500 Subject: [PATCH 3/6] skipping dependency isolation test when using sdk artifact --- eng/templates/official/jobs/ci-e2e-tests.yml | 20 ++++++++++++++++++- .../test_dependency_isolation_functions.py | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index b69d90906..bd1e6254f 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -115,6 +115,24 @@ jobs: eng/scripts/test-setup.sh displayName: 'Install test python extension, dependencies and the worker' condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) + - powershell: | + $pipelineVarSet = "$(USETESTPYTHONSDK)" + Write-Host "pipelineVarSet: $pipelineVarSet" + $branch = "$(Build.SourceBranch)" + Write-Host "Branch: $branch" + if($branch.StartsWith("refs/heads/sdk/") -or $pipelineVarSet -eq "true") + { + Write-Host "##vso[task.setvariable variable=skipDependencyIsolationTests;]true" + } + else + { + Write-Host "##vso[task.setvariable variable=skipDependencyIsolationTests;]false" + } + displayName: 'Set SkipDependencyIsolationTests variable' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + - powershell: | + Write-Host "skipDependencyIsolationTests: $(skipDependencyIsolationTests)" + displayName: 'Display SkipDependencyIsolationTests variable' - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests env: @@ -125,5 +143,5 @@ jobs: AzureWebJobsSqlConnectionString: $(SQL_CONNECTION) AzureWebJobsEventGridTopicUri: $(EVENTGRID_URI) AzureWebJobsEventGridConnectionKey: $(EVENTGRID_CONNECTION) - USETESTPYTHONSDK: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + skipDependencyIsolationTests: $(skipDependencyIsolationTests) displayName: "Running $(PYTHON_VERSION) Python E2E Tests" diff --git a/tests/endtoend/test_dependency_isolation_functions.py b/tests/endtoend/test_dependency_isolation_functions.py index ec1cc251f..42efe551e 100644 --- a/tests/endtoend/test_dependency_isolation_functions.py +++ b/tests/endtoend/test_dependency_isolation_functions.py @@ -121,7 +121,7 @@ def test_paths_resolution(self): ).lower() ) - @skipIf(is_envvar_true('USETESTPYTHONSDK'), + @skipIf(is_envvar_true('skipDependencyIsolationTests'), 'Running tests using an editable azure-functions package.') def test_loading_libraries_from_customers_package(self): """Since the Python now loaded the customer's dependencies, the From 3608af15484c21b7dc443bfe7a070a36ceb5ecfe Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 15 Oct 2024 10:33:05 -0500 Subject: [PATCH 4/6] reference before assignment --- eng/templates/official/jobs/ci-e2e-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index bd1e6254f..481efd3db 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -133,6 +133,7 @@ jobs: - powershell: | Write-Host "skipDependencyIsolationTests: $(skipDependencyIsolationTests)" displayName: 'Display SkipDependencyIsolationTests variable' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests env: From 9c01d26c9a5488a7fe8e9b35e30a7382eca36b06 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 21 Nov 2024 11:44:40 -0600 Subject: [PATCH 5/6] run vulnerability scan --- eng/templates/jobs/build.yml | 6 +++++- pack/templates/macos_64_env_gen.yml | 4 ++++ pack/templates/nix_env_gen.yml | 4 ++++ pack/templates/win_env_gen.yml | 4 ++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/eng/templates/jobs/build.yml b/eng/templates/jobs/build.yml index 3b0500df2..dd422f4fa 100644 --- a/eng/templates/jobs/build.yml +++ b/eng/templates/jobs/build.yml @@ -21,4 +21,8 @@ jobs: python -m pip install . displayName: 'Build python worker' # Skip the build stage for SDK and Extensions release branches. This stage will fail because pyproject.toml contains the updated (and unreleased) library version - condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) \ No newline at end of file + condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) + - bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' \ No newline at end of file diff --git a/pack/templates/macos_64_env_gen.yml b/pack/templates/macos_64_env_gen.yml index 8fb49f73b..87fb2d205 100644 --- a/pack/templates/macos_64_env_gen.yml +++ b/pack/templates/macos_64_env_gen.yml @@ -12,6 +12,10 @@ steps: inputs: disableAutoCwd: true scriptPath: 'pack/scripts/mac_arm64_deps.sh' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' - task: CopyFiles@2 inputs: contents: | diff --git a/pack/templates/nix_env_gen.yml b/pack/templates/nix_env_gen.yml index 7c2b68701..9ef378d86 100644 --- a/pack/templates/nix_env_gen.yml +++ b/pack/templates/nix_env_gen.yml @@ -12,6 +12,10 @@ steps: inputs: disableAutoCwd: true scriptPath: 'pack/scripts/nix_deps.sh' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' - task: CopyFiles@2 inputs: contents: | diff --git a/pack/templates/win_env_gen.yml b/pack/templates/win_env_gen.yml index 0ae4f70ec..e07d84102 100644 --- a/pack/templates/win_env_gen.yml +++ b/pack/templates/win_env_gen.yml @@ -12,6 +12,10 @@ steps: - task: PowerShell@2 inputs: filePath: 'pack\scripts\win_deps.ps1' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' - task: CopyFiles@2 inputs: contents: | From e42618dfef0a183cf23a16f859cbeaf3b63668d9 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 3 Dec 2024 16:44:19 -0600 Subject: [PATCH 6/6] feedback --- README.md | 33 +++++++++---------- eng/templates/official/jobs/ci-e2e-tests.yml | 12 +++---- pack/templates/macos_64_env_gen.yml | 1 + pack/templates/nix_env_gen.yml | 1 + pack/templates/win_env_gen.yml | 1 + requirements.txt | 3 +- .../test_dependency_isolation_functions.py | 2 +- 7 files changed, 27 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index a889aef3d..08baeb6d1 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,34 @@ # Functions Header Image - Lightning Logo Azure Functions Python Worker -| Branch | Status | CodeCov | Unittests | E2E tests | -|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| -| main | [![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=main)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=main) | [![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/main/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker) | ![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=main) | ![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=main) | -| dev | [![Build Status](https://azfunc.visualstudio.com/Azure%20Functions/_apis/build/status/Azure.azure-functions-python-worker?branchName=dev)](https://azfunc.visualstudio.com/Azure%20Functions/_build/latest?definitionId=57&branchName=dev) | [![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker) | ![CI Unit tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20Unit%20tests/badge.svg?branch=dev) | ![CI E2E tests](https://github.com/Azure/azure-functions-python-worker/workflows/CI%20E2E%20tests/badge.svg?branch=dev) | +| Branch | Build Status | CodeCov | Test Status | +|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| dev | [![Build Status](https://img.shields.io/azure-devops/build/azfunc/public/658/dev)](https://azfunc.visualstudio.com/public/_build/latest?definitionId=658&branchName=dev) | [![codecov](https://codecov.io/gh/Azure/azure-functions-python-worker/branch/dev/graph/badge.svg)](https://codecov.io/gh/Azure/azure-functions-python-worker) | [![Test Status](https://img.shields.io/azure-devops/build/azfunc/public/658/dev)](https://azfunc.visualstudio.com/public/_build/latest?definitionId=658&branchName=dev) | -Python support for Azure Functions is based on Python 3.6, 3.7, 3.8, 3.9, and 3.10 serverless hosting on Linux and the Functions 2.0, 3.0 and 4.0 runtime. +Python support for Azure Functions is based on Python 3.8, 3.9, 3.10, 3.11, and 3.12 serverless hosting on Linux and the Functions 4.0 runtime. Here is the current status of Python in Azure Functions: What are the supported Python versions? -| Azure Functions Runtime | Python 3.6 | Python 3.7 | Python 3.8 | Python 3.9 | Python 3.10 | Python 3.11 | -|----------------------------------|------------|------------|------------|------------|-------------|-------------| -| Azure Functions 2.0 (deprecated) | ✔ | ✔ | - | - | - | - | -| Azure Functions 3.0 (deprecated) | ✔ | ✔ | ✔ | ✔ | - | - | -| Azure Functions 4.0 | - | - | ✔ | ✔ | ✔ | ✔ | +| Azure Functions Runtime | Python 3.8 | Python 3.9 | Python 3.10 | Python 3.11 | Python 3.12 | +|----------------------------------|------------|------------|-------------|-------------|-------------| +| Azure Functions 3.0 (deprecated) | ✔ | ✔ | - | - | - | +| Azure Functions 4.0 | ✔ | ✔ | ✔ | ✔ | ✔ | For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page. ### What's available? -- Build, test, debug and publish using Azure Functions Core Tools (CLI) or Visual Studio Code -- Deploy Python Function project onto consumption, dedicated, or elastic premium plan. -- Deploy Python Function project in a custom docker image onto dedicated, or elastic premium plan. -- Triggers / Bindings : HTTP, Blob, Queue, Timer, Cosmos DB, Event Grid, Event Hubs and Service Bus +- Build, test, debug, and publish using Azure Functions Core Tools (CLI) or Visual Studio Code +- Deploy Python Function project onto consumption, dedicated, elastic premium, or flex consumption plan. +- Deploy Python Function project in a custom docker image onto dedicated or elastic premium plan. +- Triggers / Bindings : Blob, Cosmos DB, Event Grid, Event Hub, HTTP, Kafka, MySQL, Queue, ServiceBus, SQL, Timer, and Warmup - Triggers / Bindings : Custom binding support -What's coming? +### What's new? -- [Durable Functions For Python](https://github.com/Azure/azure-functions-durable-python) +- [SDK Type Bindings for Blob](https://techcommunity.microsoft.com/t5/azure-compute-blog/azure-functions-sdk-type-bindings-for-azure-blob-storage-with/ba-p/4146744) +- [HTTP Streaming](https://techcommunity.microsoft.com/t5/azure-compute-blog/azure-functions-support-for-http-streams-in-python-is-now-in/ba-p/4146697) ### Get Started @@ -72,4 +71,4 @@ provided by the bot. You will only need to do this once across all repos using o This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. \ No newline at end of file diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index 481efd3db..b3ff4c57d 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -122,17 +122,17 @@ jobs: Write-Host "Branch: $branch" if($branch.StartsWith("refs/heads/sdk/") -or $pipelineVarSet -eq "true") { - Write-Host "##vso[task.setvariable variable=skipDependencyIsolationTests;]true" + Write-Host "##vso[task.setvariable variable=skipTest;]true" } else { - Write-Host "##vso[task.setvariable variable=skipDependencyIsolationTests;]false" + Write-Host "##vso[task.setvariable variable=skipTest;]false" } - displayName: 'Set SkipDependencyIsolationTests variable' + displayName: 'Set skipTest variable' condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) - powershell: | - Write-Host "skipDependencyIsolationTests: $(skipDependencyIsolationTests)" - displayName: 'Display SkipDependencyIsolationTests variable' + Write-Host "skipTest: $(skipTest)" + displayName: 'Display skipTest variable' condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests @@ -144,5 +144,5 @@ jobs: AzureWebJobsSqlConnectionString: $(SQL_CONNECTION) AzureWebJobsEventGridTopicUri: $(EVENTGRID_URI) AzureWebJobsEventGridConnectionKey: $(EVENTGRID_CONNECTION) - skipDependencyIsolationTests: $(skipDependencyIsolationTests) + skipTest: $(skipTest) displayName: "Running $(PYTHON_VERSION) Python E2E Tests" diff --git a/pack/templates/macos_64_env_gen.yml b/pack/templates/macos_64_env_gen.yml index 87fb2d205..9bf2027ab 100644 --- a/pack/templates/macos_64_env_gen.yml +++ b/pack/templates/macos_64_env_gen.yml @@ -39,4 +39,5 @@ steps: !distutils-precedence.pth !pkg_resources/** !*.dist-info/** + !werkzeug/debug/shared/debugger.js targetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/pack/templates/nix_env_gen.yml b/pack/templates/nix_env_gen.yml index 9ef378d86..b89d48133 100644 --- a/pack/templates/nix_env_gen.yml +++ b/pack/templates/nix_env_gen.yml @@ -39,4 +39,5 @@ steps: !distutils-precedence.pth !pkg_resources/** !*.dist-info/** + !werkzeug/debug/shared/debugger.js targetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/pack/templates/win_env_gen.yml b/pack/templates/win_env_gen.yml index e07d84102..8e9b0321c 100644 --- a/pack/templates/win_env_gen.yml +++ b/pack/templates/win_env_gen.yml @@ -39,4 +39,5 @@ steps: !distutils-precedence.pth !pkg_resources\** !*.dist-info\** + !werkzeug\debug\shared\debugger.js targetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/requirements.txt b/requirements.txt index 6b5f87b7a..3fdb69c81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -# Please list runtime dependencies in setup.py 'install_requires' and -# 'extras_require'. +# Required dependencies listed in pyproject.toml . diff --git a/tests/endtoend/test_dependency_isolation_functions.py b/tests/endtoend/test_dependency_isolation_functions.py index 42efe551e..7ada0ae69 100644 --- a/tests/endtoend/test_dependency_isolation_functions.py +++ b/tests/endtoend/test_dependency_isolation_functions.py @@ -121,7 +121,7 @@ def test_paths_resolution(self): ).lower() ) - @skipIf(is_envvar_true('skipDependencyIsolationTests'), + @skipIf(is_envvar_true('skipTest'), 'Running tests using an editable azure-functions package.') def test_loading_libraries_from_customers_package(self): """Since the Python now loaded the customer's dependencies, the