From 5e2810014500dbff7b9e434c20af928a836e2c7d Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 2 May 2025 11:55:40 -0500 Subject: [PATCH 01/10] initial changes for linux-arm64 build --- eng/ci/worker-release.yml | 4 -- .../official/jobs/build-artifacts.yml | 40 ++++++++++++++++++- eng/templates/official/jobs/ci-e2e-tests.yml | 7 +--- ...osoft.Azure.Functions.PythonWorker.targets | 1 + ...oft.Azure.Functions.V4.PythonWorker.nuspec | 5 +++ .../test_deferred_bindings.py | 9 ----- .../test_deferred_bindings_blob_functions.py | 4 ++ 7 files changed, 50 insertions(+), 20 deletions(-) diff --git a/eng/ci/worker-release.yml b/eng/ci/worker-release.yml index 987569c61..d189be866 100644 --- a/eng/ci/worker-release.yml +++ b/eng/ci/worker-release.yml @@ -22,10 +22,6 @@ extends: name: 1es-pool-azfunc image: 1es-windows-2022 os: windows - sdl: - codeSignValidation: - enabled: true - break: true stages: - stage: Release diff --git a/eng/templates/official/jobs/build-artifacts.yml b/eng/templates/official/jobs/build-artifacts.yml index bc0c5de1a..fe4cf0007 100644 --- a/eng/templates/official/jobs/build-artifacts.yml +++ b/eng/templates/official/jobs/build-artifacts.yml @@ -188,16 +188,52 @@ jobs: outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) - artifactName: "$(pythonVersion)_OSX_ARM4" + artifactName: "$(pythonVersion)_OSX_ARM64" steps: - template: ../../../../pack/templates/macos_64_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' workerPath: '$(workerPath)' artifactName: '$(pythonVersion)_OSX_ARM64' +- job: Build_LINUX_ARM64 + pool: + name: Azure Pipelines + image: macOS-latest + os: macOS + strategy: + matrix: + Python39V4: + pythonVersion: '3.9' + workerPath: 'python/prodV4/worker.py' + Python310V4: + pythonVersion: '3.10' + workerPath: 'python/prodV4/worker.py' + Python311V4: + pythonVersion: '3.11' + workerPath: 'python/prodV4/worker.py' + Python312V4: + pythonVersion: '3.12' + workerPath: 'python/prodV4/worker.py' + Python313V4: + pythonVersion: '3.13' + workerPath: 'python/proxyV4/worker.py' + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: "$(pythonVersion)_LINUX_ARM64" + steps: + - template: ../../../../pack/templates/nix_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + workerPath: '$(workerPath)' + artifactName: '$(pythonVersion)_LINUX_ARM64' + + - job: PackageWorkers - dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64'] + dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64', 'Build_LINUX_ARM64'] templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index 1a45e06ce..9699b28b2 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -134,7 +134,7 @@ jobs: Write-Host "Python version: $PY_VER" # Extract minor version as integers $PY_MINO = "${PY_VER#*.}" - Write-Host "Branch: PY_MINOR" + Write-Host "Minor version: $PY_MINO" if($branch.StartsWith("refs/heads/sdk/") -or $pipelineVarSet -eq "true" -or $PY_MINOR -ge 13 ) { @@ -144,11 +144,8 @@ jobs: { Write-Host "##vso[task.setvariable variable=skipTest;]false" } - displayName: 'Set skipTest variable' - - powershell: | Write-Host "skipTest: $(skipTest)" - displayName: 'Display skipTest variable' - condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + displayName: 'Set skipTest variable' - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests env: diff --git a/pack/Microsoft.Azure.Functions.PythonWorker.targets b/pack/Microsoft.Azure.Functions.PythonWorker.targets index 887dc8d52..882ff9fa1 100644 --- a/pack/Microsoft.Azure.Functions.PythonWorker.targets +++ b/pack/Microsoft.Azure.Functions.PythonWorker.targets @@ -10,6 +10,7 @@ <_PythonSupportedRuntime Include="linux-x64" WorkerPath="LINUX/X64" /> <_PythonSupportedRuntime Include="osx-x64" WorkerPath="OSX/X64" /> <_PythonSupportedRuntime Include="osx-arm64" WorkerPath="OSX/Arm64" /> + <_PythonSupportedRuntime Include="linux-arm64" WorkerPath="LINUX/Arm64" /> diff --git a/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec b/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec index 98b0c832a..b76b3ecb6 100644 --- a/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec +++ b/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec @@ -23,26 +23,31 @@ + + + + + diff --git a/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py index b8ced2834..84f348283 100644 --- a/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py +++ b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py @@ -43,9 +43,6 @@ def __init__(self, version: str, source: str, @unittest.skipIf(sys.version_info.minor <= 8, "The base extension" "is only supported for 3.9+.") -@unittest.skipIf(sys.version_info.minor >= 13, "For python 3.13+," - "this logic is in the" - "library worker.") class TestDeferredBindingsEnabled(testutils.AsyncTestCase): @testutils.retryable_test(3, 5) @@ -77,9 +74,6 @@ async def test_deferred_bindings_enabled_log(self): @unittest.skipIf(sys.version_info.minor <= 8, "The base extension" "is only supported for 3.9+.") -@unittest.skipIf(sys.version_info.minor >= 13, "For python 3.13+," - "this logic is in the" - "library worker.") class TestDeferredBindingsDisabled(testutils.AsyncTestCase): @testutils.retryable_test(3, 5) @@ -111,9 +105,6 @@ async def test_deferred_bindings_disabled_log(self): @unittest.skipIf(sys.version_info.minor <= 8, "The base extension" "is only supported for 3.9+.") -@unittest.skipIf(sys.version_info.minor >= 13, "For python 3.13+," - "this logic is in the" - "library worker.") class TestDeferredBindingsEnabledDual(testutils.AsyncTestCase): @testutils.retryable_test(3, 5) diff --git a/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py index ed441a077..cedd910ad 100644 --- a/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py +++ b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py @@ -171,6 +171,8 @@ def test_type_undefined(self): self.assertEqual(r.status_code, 200) self.assertEqual(r.text, 'test-data') + @unittest.skipIf(sys.version_info.minor >= 13, "For python 3.13+," + "the cache is maintained in the ext and TBD.") def test_caching(self): ''' The cache returns the same type based on resource and function name. @@ -204,6 +206,8 @@ def test_caching(self): self.assertEqual(r2.text, client2) self.assertNotEqual(r.text, r2.text) + @unittest.skipIf(sys.version_info.minor >= 13, "For python 3.13+," + "the cache is maintained in the ext and TBD.") def test_caching_same_resource(self): ''' The cache returns the same type based on param name. From 9650a9683b16eae803f074310e790ffee5037ecc Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 2 May 2025 12:06:11 -0500 Subject: [PATCH 02/10] fix variable def --- eng/templates/official/jobs/ci-e2e-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index 9699b28b2..441bb8852 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -146,6 +146,9 @@ jobs: } Write-Host "skipTest: $(skipTest)" displayName: 'Set skipTest variable' + - powershell: | + Write-Host "skipTest: $(skipTest)" + displayName: 'Display skipTest variable' - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests env: From edd775185f0e65ff122c46139475588cef939b9b Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 2 May 2025 12:08:05 -0500 Subject: [PATCH 03/10] fix variable --- eng/templates/official/jobs/ci-e2e-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index 441bb8852..a0d1e152f 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -144,7 +144,6 @@ jobs: { Write-Host "##vso[task.setvariable variable=skipTest;]false" } - Write-Host "skipTest: $(skipTest)" displayName: 'Set skipTest variable' - powershell: | Write-Host "skipTest: $(skipTest)" From 96ced268e9820d6a6df657a1c8cbfd8be925104a Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 2 May 2025 13:23:36 -0500 Subject: [PATCH 04/10] testing fixes --- eng/templates/official/jobs/ci-e2e-tests.yml | 4 ++-- .../deferred_bindings_tests/test_deferred_bindings.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index a0d1e152f..08d6f30d9 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -133,8 +133,8 @@ jobs: $PY_VER = "$(PYTHON_VERSION)" Write-Host "Python version: $PY_VER" # Extract minor version as integers - $PY_MINO = "${PY_VER#*.}" - Write-Host "Minor version: $PY_MINO" + $PY_MINOR = "${PY_VER#*.}" + Write-Host "Minor version: $PY_MINOR" if($branch.StartsWith("refs/heads/sdk/") -or $pipelineVarSet -eq "true" -or $PY_MINOR -ge 13 ) { diff --git a/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py index 84f348283..b8ced2834 100644 --- a/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py +++ b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py @@ -43,6 +43,9 @@ def __init__(self, version: str, source: str, @unittest.skipIf(sys.version_info.minor <= 8, "The base extension" "is only supported for 3.9+.") +@unittest.skipIf(sys.version_info.minor >= 13, "For python 3.13+," + "this logic is in the" + "library worker.") class TestDeferredBindingsEnabled(testutils.AsyncTestCase): @testutils.retryable_test(3, 5) @@ -74,6 +77,9 @@ async def test_deferred_bindings_enabled_log(self): @unittest.skipIf(sys.version_info.minor <= 8, "The base extension" "is only supported for 3.9+.") +@unittest.skipIf(sys.version_info.minor >= 13, "For python 3.13+," + "this logic is in the" + "library worker.") class TestDeferredBindingsDisabled(testutils.AsyncTestCase): @testutils.retryable_test(3, 5) @@ -105,6 +111,9 @@ async def test_deferred_bindings_disabled_log(self): @unittest.skipIf(sys.version_info.minor <= 8, "The base extension" "is only supported for 3.9+.") +@unittest.skipIf(sys.version_info.minor >= 13, "For python 3.13+," + "this logic is in the" + "library worker.") class TestDeferredBindingsEnabledDual(testutils.AsyncTestCase): @testutils.retryable_test(3, 5) From 1854a9cd2a022686e40e520a5cc54aabf310ab04 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 2 May 2025 14:13:01 -0500 Subject: [PATCH 05/10] parse minor version --- eng/templates/official/jobs/ci-e2e-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index 08d6f30d9..f347786f5 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -133,7 +133,8 @@ jobs: $PY_VER = "$(PYTHON_VERSION)" Write-Host "Python version: $PY_VER" # Extract minor version as integers - $PY_MINOR = "${PY_VER#*.}" + $versionParts = $PY_VER.Split('.') + $PY_MINOR = [int]$versionParts[1] Write-Host "Minor version: $PY_MINOR" if($branch.StartsWith("refs/heads/sdk/") -or $pipelineVarSet -eq "true" -or $PY_MINOR -ge 13 ) From 6adf31c5b7b3635e2a912dad4b8bd2d3018a3516 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Mon, 5 May 2025 10:05:06 -0500 Subject: [PATCH 06/10] refactor pack into eng --- ...osoft.Azure.Functions.PythonWorker.targets | 0 ...oft.Azure.Functions.V4.PythonWorker.nuspec | 55 +++++++++++++++ {pack => eng/pack}/scripts/mac_arm64_deps.sh | 0 {pack => eng/pack}/scripts/nix_deps.sh | 0 {pack => eng/pack}/scripts/win_deps.ps1 | 0 .../pack}/templates/macos_64_env_gen.yml | 6 +- {pack => eng/pack}/templates/nix_env_gen.yml | 67 +++++++++---------- {pack => eng/pack}/templates/win_env_gen.yml | 0 {pack => eng/pack}/utils/__init__.py | 0 .../official/jobs/build-artifacts.yml | 20 +++--- ...oft.Azure.Functions.V4.PythonWorker.nuspec | 55 --------------- 11 files changed, 100 insertions(+), 103 deletions(-) rename {pack => eng/pack}/Microsoft.Azure.Functions.PythonWorker.targets (100%) create mode 100644 eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec rename {pack => eng/pack}/scripts/mac_arm64_deps.sh (100%) rename {pack => eng/pack}/scripts/nix_deps.sh (100%) rename {pack => eng/pack}/scripts/win_deps.ps1 (100%) rename {pack => eng/pack}/templates/macos_64_env_gen.yml (96%) rename {pack => eng/pack}/templates/nix_env_gen.yml (59%) rename {pack => eng/pack}/templates/win_env_gen.yml (100%) rename {pack => eng/pack}/utils/__init__.py (100%) delete mode 100644 pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec diff --git a/pack/Microsoft.Azure.Functions.PythonWorker.targets b/eng/pack/Microsoft.Azure.Functions.PythonWorker.targets similarity index 100% rename from pack/Microsoft.Azure.Functions.PythonWorker.targets rename to eng/pack/Microsoft.Azure.Functions.PythonWorker.targets diff --git a/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec b/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec new file mode 100644 index 000000000..8e5a2a9ed --- /dev/null +++ b/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec @@ -0,0 +1,55 @@ + + + + Microsoft.Azure.Functions.PythonWorker + 1.1.0 + Microsoft + Microsoft + false + Microsoft Azure Functions Python Worker + © .NET Foundation. All rights reserved. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pack/scripts/mac_arm64_deps.sh b/eng/pack/scripts/mac_arm64_deps.sh similarity index 100% rename from pack/scripts/mac_arm64_deps.sh rename to eng/pack/scripts/mac_arm64_deps.sh diff --git a/pack/scripts/nix_deps.sh b/eng/pack/scripts/nix_deps.sh similarity index 100% rename from pack/scripts/nix_deps.sh rename to eng/pack/scripts/nix_deps.sh diff --git a/pack/scripts/win_deps.ps1 b/eng/pack/scripts/win_deps.ps1 similarity index 100% rename from pack/scripts/win_deps.ps1 rename to eng/pack/scripts/win_deps.ps1 diff --git a/pack/templates/macos_64_env_gen.yml b/eng/pack/templates/macos_64_env_gen.yml similarity index 96% rename from pack/templates/macos_64_env_gen.yml rename to eng/pack/templates/macos_64_env_gen.yml index 75f33bc5f..5502b442c 100644 --- a/pack/templates/macos_64_env_gen.yml +++ b/eng/pack/templates/macos_64_env_gen.yml @@ -18,7 +18,7 @@ steps: - task: ShellScript@2 inputs: disableAutoCwd: true - scriptPath: 'pack/scripts/mac_arm64_deps.sh' + scriptPath: 'eng/pack/scripts/mac_arm64_deps.sh' args: '${{ parameters.pythonVersion }}' displayName: 'Install Dependencies' - bash: | @@ -41,7 +41,7 @@ steps: !grpcio_tools*/* !build/** !docs/** - !pack/** + !eng/** !python/** !tests/** !setuptools*/** @@ -63,7 +63,7 @@ steps: !grpcio_tools*/* !build/** !docs/** - !pack/** + !eng/** !python/** !tests/** !setuptools*/** diff --git a/pack/templates/nix_env_gen.yml b/eng/pack/templates/nix_env_gen.yml similarity index 59% rename from pack/templates/nix_env_gen.yml rename to eng/pack/templates/nix_env_gen.yml index db3820153..d153d0ed9 100644 --- a/pack/templates/nix_env_gen.yml +++ b/eng/pack/templates/nix_env_gen.yml @@ -7,6 +7,7 @@ steps: - task: UsePythonVersion@0 inputs: versionSpec: ${{ parameters.pythonVersion }} + architecture: ${{ parameters.architecture }} addToPath: true - bash: | major=$(echo $(pythonVersion) | cut -d. -f1) @@ -15,12 +16,10 @@ steps: echo "##vso[task.setvariable variable=pythonMinor]$minor" echo $pythonMinor displayName: 'Parse pythonVersion' -- task: ShellScript@2 +- task: PowerShell@2 inputs: - disableAutoCwd: true - scriptPath: 'pack/scripts/nix_deps.sh' - args: '${{ parameters.pythonVersion }}' - displayName: 'Install Dependencies' + filePath: 'eng\pack\scripts\win_deps.ps1' + arguments: '${{ parameters.pythonVersion }}' - bash: | pip install pip-audit pip-audit -r requirements.txt @@ -34,46 +33,46 @@ steps: flattenFolders: true - task: CopyFiles@2 inputs: - sourceFolder: '$(Build.SourcesDirectory)/deps' + sourceFolder: '$(Build.SourcesDirectory)\deps' contents: | ** - !grpc_tools/**/* - !grpcio_tools*/* - !build/** - !docs/** - !pack/** - !python/** - !tests/** - !setuptools*/** - !_distutils_hack/** + !grpc_tools\**\* + !grpcio_tools*\* + !build\** + !docs\** + !eng\** + !python\** + !tests\** + !setuptools*\** + !_distutils_hack\** !distutils-precedence.pth - !pkg_resources/** - !*.dist-info/** - !werkzeug/debug/shared/debugger.js - !proxy_worker/** + !pkg_resources\** + !*.dist-info\** + !werkzeug\debug\shared\debugger.js + !proxy_worker\** targetFolder: '$(Build.ArtifactStagingDirectory)' condition: in(variables['pythonMinor'], '7', '8', '9', '10', '11', '12') displayName: 'Copy azure_functions_worker files' - task: CopyFiles@2 inputs: - sourceFolder: '$(Build.SourcesDirectory)/deps' + sourceFolder: '$(Build.SourcesDirectory)\deps' contents: | ** - !grpc_tools/**/* - !grpcio_tools*/* - !build/** - !docs/** - !pack/** - !python/** - !tests/** - !setuptools*/** - !_distutils_hack/** + !grpc_tools\**\* + !grpcio_tools*\* + !build\** + !docs\** + !eng\** + !python\** + !tests\** + !setuptools*\** + !_distutils_hack\** !distutils-precedence.pth - !pkg_resources/** - !*.dist-info/** - !werkzeug/debug/shared/debugger.js - !dateutil/** - !azure_functions_worker/** + !pkg_resources\** + !*.dist-info\** + !werkzeug\debug\shared\debugger.js + !dateutil\** + !azure_functions_worker\** targetFolder: '$(Build.ArtifactStagingDirectory)' condition: in(variables['pythonMinor'], '13') displayName: 'Copy proxy_worker files' diff --git a/pack/templates/win_env_gen.yml b/eng/pack/templates/win_env_gen.yml similarity index 100% rename from pack/templates/win_env_gen.yml rename to eng/pack/templates/win_env_gen.yml diff --git a/pack/utils/__init__.py b/eng/pack/utils/__init__.py similarity index 100% rename from pack/utils/__init__.py rename to eng/pack/utils/__init__.py diff --git a/eng/templates/official/jobs/build-artifacts.yml b/eng/templates/official/jobs/build-artifacts.yml index fe4cf0007..1b01872ed 100644 --- a/eng/templates/official/jobs/build-artifacts.yml +++ b/eng/templates/official/jobs/build-artifacts.yml @@ -34,7 +34,7 @@ jobs: targetPath: $(Build.ArtifactStagingDirectory) artifactName: "$(pythonVersion)_WINDOWS_X64" steps: - - template: ../../../../pack/templates/win_env_gen.yml + - template: ../../../pack/templates/win_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' workerPath: '$(workerPath)' @@ -75,7 +75,7 @@ jobs: targetPath: $(Build.ArtifactStagingDirectory) artifactName: "$(pythonVersion)_WINDOWS_X86" steps: - - template: ../../../../pack/templates/win_env_gen.yml + - template: ../../../pack/templates/win_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' workerPath: '$(workerPath)' @@ -116,7 +116,7 @@ jobs: targetPath: $(Build.ArtifactStagingDirectory) artifactName: "$(pythonVersion)_LINUX_X64" steps: - - template: ../../../../pack/templates/nix_env_gen.yml + - template: ../../../pack/templates/nix_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' workerPath: '$(workerPath)' @@ -156,7 +156,7 @@ jobs: targetPath: $(Build.ArtifactStagingDirectory) artifactName: "$(pythonVersion)_OSX_X64" steps: - - template: ../../../../pack/templates/nix_env_gen.yml + - template: ../../../pack/templates/nix_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' workerPath: '$(workerPath)' @@ -190,7 +190,7 @@ jobs: targetPath: $(Build.ArtifactStagingDirectory) artifactName: "$(pythonVersion)_OSX_ARM64" steps: - - template: ../../../../pack/templates/macos_64_env_gen.yml + - template: ../../../pack/templates/macos_64_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' workerPath: '$(workerPath)' @@ -224,14 +224,12 @@ jobs: targetPath: $(Build.ArtifactStagingDirectory) artifactName: "$(pythonVersion)_LINUX_ARM64" steps: - - template: ../../../../pack/templates/nix_env_gen.yml + - template: ../../../pack/templates/nix_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' workerPath: '$(workerPath)' artifactName: '$(pythonVersion)_LINUX_ARM64' - - - job: PackageWorkers dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64', 'Build_LINUX_ARM64'] templateContext: @@ -248,19 +246,19 @@ jobs: if [[ $BUILD_SOURCEBRANCHNAME = 4\.* ]] then echo "Generating V4 Release Package for $BUILD_SOURCEBRANCHNAME" - NUSPEC="pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec" + NUSPEC="eng\pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec" WKVERSION="$BUILD_SOURCEBRANCHNAME" elif [[ $BUILD_SOURCEBRANCHNAME = dev ]] then echo "Generating V4 Integration Test Package for $BUILD_SOURCEBRANCHNAME" VERSION=$(cat azure_functions_worker/version.py | tail -1 | cut -d' ' -f3 | sed "s/'//g") - NUSPEC="pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec" + NUSPEC="eng\pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec" WKVERSION="$VERSION-$(Build.BuildNumber)" else # this is only to test nuget related workflow because we are setting nuspec here echo "Generating Integration Test Package for $BUILD_SOURCEBRANCHNAME for testing purpose" LATEST_TAG=$(curl https://api.github.com/repos/Azure/azure-functions-python-worker/tags -s | jq '.[0].name' | sed 's/\"//g' | cut -d'.' -f-2) - NUSPEC="pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec" + NUSPEC="eng\pack\Microsoft.Azure.Functions.V4.PythonWorker.nuspec" # Only required for Integration Test. Version number contains date (e.g. 3.1.2.20211028-dev) WKVERSION="3.$LATEST_TAG-$(BUILD_BUILDID)-TEST" echo "No Matching Release Tag For $BUILD_SOURCEBRANCH" diff --git a/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec b/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec deleted file mode 100644 index b76b3ecb6..000000000 --- a/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec +++ /dev/null @@ -1,55 +0,0 @@ - - - - Microsoft.Azure.Functions.PythonWorker - 1.1.0 - Microsoft - Microsoft - false - Microsoft Azure Functions Python Worker - © .NET Foundation. All rights reserved. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 4b319dcaf0f3897ad174d3bc5aafdbec4a79f45b Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Mon, 5 May 2025 16:01:16 -0500 Subject: [PATCH 07/10] does not work: initial code for win-arm64 build --- ...osoft.Azure.Functions.PythonWorker.targets | 3 +- ...oft.Azure.Functions.V4.PythonWorker.nuspec | 25 ++++--- eng/pack/templates/macos_64_env_gen.yml | 8 +- eng/pack/templates/nix_env_gen.yml | 75 ++++++++++--------- eng/pack/templates/win_env_gen.yml | 52 +++++++++++-- .../official/jobs/build-artifacts.yml | 33 +++++++- 6 files changed, 139 insertions(+), 57 deletions(-) diff --git a/eng/pack/Microsoft.Azure.Functions.PythonWorker.targets b/eng/pack/Microsoft.Azure.Functions.PythonWorker.targets index 882ff9fa1..56f50699f 100644 --- a/eng/pack/Microsoft.Azure.Functions.PythonWorker.targets +++ b/eng/pack/Microsoft.Azure.Functions.PythonWorker.targets @@ -7,10 +7,11 @@ <_PythonSupportedRuntime Include="win-x86" WorkerPath="WINDOWS/X86" /> <_PythonSupportedRuntime Include="win-x64" WorkerPath="WINDOWS/X64" /> + <_PythonSupportedRuntime Include="win-arm64" WorkerPath="WINDOWS/Arm64" /> <_PythonSupportedRuntime Include="linux-x64" WorkerPath="LINUX/X64" /> + <_PythonSupportedRuntime Include="linux-arm64" WorkerPath="LINUX/Arm64" /> <_PythonSupportedRuntime Include="osx-x64" WorkerPath="OSX/X64" /> <_PythonSupportedRuntime Include="osx-arm64" WorkerPath="OSX/Arm64" /> - <_PythonSupportedRuntime Include="linux-arm64" WorkerPath="LINUX/Arm64" /> diff --git a/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec b/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec index 8e5a2a9ed..707932c8d 100644 --- a/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec +++ b/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec @@ -22,32 +22,37 @@ - - + + + - - + + + - - + + + - - + + + - - + + + diff --git a/eng/pack/templates/macos_64_env_gen.yml b/eng/pack/templates/macos_64_env_gen.yml index 5502b442c..4fb543335 100644 --- a/eng/pack/templates/macos_64_env_gen.yml +++ b/eng/pack/templates/macos_64_env_gen.yml @@ -9,12 +9,14 @@ steps: versionSpec: ${{ parameters.pythonVersion }} addToPath: true - bash: | - major=$(echo $(pythonVersion) | cut -d. -f1) - minor=$(echo $(pythonVersion) | cut -d. -f2) + major=$(echo "$pythonVersion" | cut -d. -f1) + minor=$(echo "$pythonVersion" | cut -d. -f2) echo "##vso[task.setvariable variable=pythonMajor]$major" echo "##vso[task.setvariable variable=pythonMinor]$minor" - echo $pythonMinor + echo "$minor" displayName: 'Parse pythonVersion' + env: + pythonVersion: ${{ parameters.pythonVersion }} - task: ShellScript@2 inputs: disableAutoCwd: true diff --git a/eng/pack/templates/nix_env_gen.yml b/eng/pack/templates/nix_env_gen.yml index d153d0ed9..a56634edd 100644 --- a/eng/pack/templates/nix_env_gen.yml +++ b/eng/pack/templates/nix_env_gen.yml @@ -7,19 +7,22 @@ steps: - task: UsePythonVersion@0 inputs: versionSpec: ${{ parameters.pythonVersion }} - architecture: ${{ parameters.architecture }} addToPath: true - bash: | - major=$(echo $(pythonVersion) | cut -d. -f1) - minor=$(echo $(pythonVersion) | cut -d. -f2) + major=$(echo "$pythonVersion" | cut -d. -f1) + minor=$(echo "$pythonVersion" | cut -d. -f2) echo "##vso[task.setvariable variable=pythonMajor]$major" echo "##vso[task.setvariable variable=pythonMinor]$minor" - echo $pythonMinor + echo "$minor" displayName: 'Parse pythonVersion' -- task: PowerShell@2 + env: + pythonVersion: ${{ parameters.pythonVersion }} +- task: ShellScript@2 inputs: - filePath: 'eng\pack\scripts\win_deps.ps1' - arguments: '${{ parameters.pythonVersion }}' + disableAutoCwd: true + scriptPath: 'eng/pack/scripts/nix_deps.sh' + args: '${{ parameters.pythonVersion }}' + displayName: 'Install Dependencies' - bash: | pip install pip-audit pip-audit -r requirements.txt @@ -33,46 +36,46 @@ steps: flattenFolders: true - task: CopyFiles@2 inputs: - sourceFolder: '$(Build.SourcesDirectory)\deps' + sourceFolder: '$(Build.SourcesDirectory)/deps' contents: | ** - !grpc_tools\**\* - !grpcio_tools*\* - !build\** - !docs\** - !eng\** - !python\** - !tests\** - !setuptools*\** - !_distutils_hack\** + !grpc_tools/**/* + !grpcio_tools*/* + !build/** + !docs/** + !eng/** + !python/** + !tests/** + !setuptools*/** + !_distutils_hack/** !distutils-precedence.pth - !pkg_resources\** - !*.dist-info\** - !werkzeug\debug\shared\debugger.js - !proxy_worker\** + !pkg_resources/** + !*.dist-info/** + !werkzeug/debug/shared/debugger.js + !proxy_worker/** targetFolder: '$(Build.ArtifactStagingDirectory)' condition: in(variables['pythonMinor'], '7', '8', '9', '10', '11', '12') displayName: 'Copy azure_functions_worker files' - task: CopyFiles@2 inputs: - sourceFolder: '$(Build.SourcesDirectory)\deps' + sourceFolder: '$(Build.SourcesDirectory)/deps' contents: | ** - !grpc_tools\**\* - !grpcio_tools*\* - !build\** - !docs\** - !eng\** - !python\** - !tests\** - !setuptools*\** - !_distutils_hack\** + !grpc_tools/**/* + !grpcio_tools*/* + !build/** + !docs/** + !eng/** + !python/** + !tests/** + !setuptools*/** + !_distutils_hack/** !distutils-precedence.pth - !pkg_resources\** - !*.dist-info\** - !werkzeug\debug\shared\debugger.js - !dateutil\** - !azure_functions_worker\** + !pkg_resources/** + !*.dist-info/** + !werkzeug/debug/shared/debugger.js + !dateutil/** + !azure_functions_worker/** targetFolder: '$(Build.ArtifactStagingDirectory)' condition: in(variables['pythonMinor'], '13') displayName: 'Copy proxy_worker files' diff --git a/eng/pack/templates/win_env_gen.yml b/eng/pack/templates/win_env_gen.yml index b85bf9f89..1ca4dba2d 100644 --- a/eng/pack/templates/win_env_gen.yml +++ b/eng/pack/templates/win_env_gen.yml @@ -9,16 +9,56 @@ steps: versionSpec: ${{ parameters.pythonVersion }} architecture: ${{ parameters.architecture }} addToPath: true + condition: ne('${{ parameters.downloadArm64 }}', 'true') +- task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $versionMap = @{ + "3.11" = "3.11.10" + "3.12" = "3.12.3" + "3.13" = "3.13.0" + } + + $shortVersion = "${{ parameters.pythonVersion }}" + if ($versionMap.ContainsKey($shortVersion)) { + $fullVersion = $versionMap[$shortVersion] + Write-Host "Mapped $shortVersion to full version $fullVersion" + Write-Host "##vso[task.setvariable variable=fullPythonVersion]$fullVersion" + } + else { + Write-Host "ERROR: Unsupported Python version '$shortVersion'" + exit 1 + } + displayName: 'Map pythonVersion to full version' + condition: eq('${{ parameters.downloadArm64 }}', 'true') +- powershell: | + $fullVersion = "$(fullPythonVersion)" + $url = "https://www.python.org/ftp/python/$fullVersion/python-$fullVersion-arm64.exe" + Write-Host "Downloading Python ARM64 from $url" + Invoke-WebRequest -Uri $url -OutFile python-arm64.exe + + Start-Process -Wait -FilePath .\python-arm64.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=0 Include_test=0' + + # Explicitly reference ARM64 Python + $majorMinor = $fullVersion.Substring(0, $fullVersion.LastIndexOf('.')).Replace('.', '') + $pythonPath = "C:\Program Files\Python$majorMinor\python.exe" + & "$pythonPath" --version + & "$pythonPath" -c "import platform; print(platform.machine())" + displayName: 'Download and install Python ARM64' + condition: eq('${{ parameters.downloadArm64 }}', 'true') - bash: | - major=$(echo $(pythonVersion) | cut -d. -f1) - minor=$(echo $(pythonVersion) | cut -d. -f2) + major=$(echo "$pythonVersion" | cut -d. -f1) + minor=$(echo "$pythonVersion" | cut -d. -f2) echo "##vso[task.setvariable variable=pythonMajor]$major" echo "##vso[task.setvariable variable=pythonMinor]$minor" - echo $pythonMinor + echo "$minor" displayName: 'Parse pythonVersion' + env: + pythonVersion: ${{ parameters.pythonVersion }} - task: PowerShell@2 inputs: - filePath: 'pack\scripts\win_deps.ps1' + filePath: 'eng\pack\scripts\win_deps.ps1' arguments: '${{ parameters.pythonVersion }}' - bash: | pip install pip-audit @@ -40,7 +80,7 @@ steps: !grpcio_tools*\* !build\** !docs\** - !pack\** + !eng\** !python\** !tests\** !setuptools*\** @@ -62,7 +102,7 @@ steps: !grpcio_tools*\* !build\** !docs\** - !pack\** + !eng\** !python\** !tests\** !setuptools*\** diff --git a/eng/templates/official/jobs/build-artifacts.yml b/eng/templates/official/jobs/build-artifacts.yml index 1b01872ed..052d67f5b 100644 --- a/eng/templates/official/jobs/build-artifacts.yml +++ b/eng/templates/official/jobs/build-artifacts.yml @@ -229,9 +229,40 @@ jobs: pythonVersion: '$(pythonVersion)' workerPath: '$(workerPath)' artifactName: '$(pythonVersion)_LINUX_ARM64' +- job: Build_WINDOWS_ARM64 + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + strategy: + matrix: + # ARM64 python is only supported for 3.11 and above + Python311V4: + pythonVersion: '3.11' + workerPath: 'python/prodV4/worker.py' + Python312V4: + pythonVersion: '3.12' + workerPath: 'python/prodV4/worker.py' + Python313V4: + pythonVersion: '3.13' + workerPath: 'python/proxyV4/worker.py' + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: "$(pythonVersion)_WINDOWS_ARM64" + steps: + - template: ../../../pack/templates/win_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + workerPath: '$(workerPath)' + architecture: 'x64' # only x64 and x86 are supported for windows agents + downloadArm64: true + artifactName: '$(pythonVersion)_WINDOWS_ARM64' - job: PackageWorkers - dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64', 'Build_LINUX_ARM64'] + dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64', 'Build_LINUX_ARM64', 'Build_WINDOWS_ARM64'] templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: From 33d329638bada793676c3a7cd732460d11015e41 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 6 May 2025 11:14:31 -0500 Subject: [PATCH 08/10] refactor workerPath into top-level variables --- eng/pack/templates/macos_64_env_gen.yml | 19 ++++++-- eng/pack/templates/nix_env_gen.yml | 19 ++++++-- eng/pack/templates/win_env_gen.yml | 19 ++++++-- .../official/jobs/build-artifacts.yml | 48 ------------------- eng/templates/utils/official-variables.yml | 6 ++- 5 files changed, 53 insertions(+), 58 deletions(-) diff --git a/eng/pack/templates/macos_64_env_gen.yml b/eng/pack/templates/macos_64_env_gen.yml index 4fb543335..331ee4a2c 100644 --- a/eng/pack/templates/macos_64_env_gen.yml +++ b/eng/pack/templates/macos_64_env_gen.yml @@ -1,7 +1,6 @@ parameters: pythonVersion: '' artifactName: '' - workerPath: '' steps: - task: UsePythonVersion@0 @@ -28,10 +27,24 @@ steps: pip-audit -r requirements.txt displayName: 'Run vulnerability scan' condition: ne(variables['pythonVersion'], '3.7') +- powershell: | + Write-Host "Minor version: $(pythonMinor)" + + if( $(pythonMinor) -ge 13 ) + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)" + } + else + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)" + } + displayName: 'Set workerPath variable' +- powershell: | + Write-Host "workerPath: $(workerPath)" + displayName: 'Display workerPath variable' - task: CopyFiles@2 inputs: - contents: | - ${{ parameters.workerPath }} + contents: '$(workerPath)' targetFolder: '$(Build.ArtifactStagingDirectory)' flattenFolders: true - task: CopyFiles@2 diff --git a/eng/pack/templates/nix_env_gen.yml b/eng/pack/templates/nix_env_gen.yml index a56634edd..ea35263c5 100644 --- a/eng/pack/templates/nix_env_gen.yml +++ b/eng/pack/templates/nix_env_gen.yml @@ -1,7 +1,6 @@ parameters: pythonVersion: '' artifactName: '' - workerPath: '' steps: - task: UsePythonVersion@0 @@ -28,10 +27,24 @@ steps: pip-audit -r requirements.txt displayName: 'Run vulnerability scan' condition: ne(variables['pythonVersion'], '3.7') +- powershell: | + Write-Host "Minor version: $(pythonMinor)" + + if( $(pythonMinor) -ge 13 ) + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)" + } + else + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)" + } + displayName: 'Set workerPath variable' +- powershell: | + Write-Host "workerPath: $(workerPath)" + displayName: 'Display workerPath variable' - task: CopyFiles@2 inputs: - contents: | - ${{ parameters.workerPath }} + contents: '$(workerPath)' targetFolder: '$(Build.ArtifactStagingDirectory)' flattenFolders: true - task: CopyFiles@2 diff --git a/eng/pack/templates/win_env_gen.yml b/eng/pack/templates/win_env_gen.yml index 1ca4dba2d..0094321ab 100644 --- a/eng/pack/templates/win_env_gen.yml +++ b/eng/pack/templates/win_env_gen.yml @@ -1,7 +1,6 @@ parameters: pythonVersion: '' artifactName: '' - workerPath: '' steps: - task: UsePythonVersion@0 @@ -65,10 +64,24 @@ steps: pip-audit -r requirements.txt displayName: 'Run vulnerability scan' condition: ne(variables['pythonVersion'], '3.7') +- powershell: | + Write-Host "Minor version: $(pythonMinor)" + + if( $(pythonMinor) -ge 13 ) + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)" + } + else + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)" + } + displayName: 'Set workerPath variable' +- powershell: | + Write-Host "workerPath: $(workerPath)" + displayName: 'Display workerPath variable' - task: CopyFiles@2 inputs: - contents: | - ${{ parameters.workerPath }} + contents: '$(workerPath)' targetFolder: '$(Build.ArtifactStagingDirectory)' flattenFolders: true - task: CopyFiles@2 diff --git a/eng/templates/official/jobs/build-artifacts.yml b/eng/templates/official/jobs/build-artifacts.yml index 052d67f5b..f33d1005d 100644 --- a/eng/templates/official/jobs/build-artifacts.yml +++ b/eng/templates/official/jobs/build-artifacts.yml @@ -8,25 +8,18 @@ jobs: matrix: Python37V4: pythonVersion: '3.7' - workerPath: 'python/prodV4/worker.py' Python38V4: pythonVersion: '3.8' - workerPath: 'python/prodV4/worker.py' Python39V4: pythonVersion: '3.9' - workerPath: 'python/prodV4/worker.py' Python310V4: pythonVersion: '3.10' - workerPath: 'python/prodV4/worker.py' Python311V4: pythonVersion: '3.11' - workerPath: 'python/prodV4/worker.py' Python312V4: pythonVersion: '3.12' - workerPath: 'python/prodV4/worker.py' Python313V4: pythonVersion: '3.13' - workerPath: 'python/proxyV4/worker.py' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: @@ -37,7 +30,6 @@ jobs: - template: ../../../pack/templates/win_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - workerPath: '$(workerPath)' architecture: 'x64' artifactName: '$(pythonVersion)_WINDOWS_X64' - job: Build_WINDOWS_X86 @@ -49,25 +41,18 @@ jobs: matrix: Python37V4: pythonVersion: '3.7' - workerPath: 'python/prodV4/worker.py' Python38V4: pythonVersion: '3.8' - workerPath: 'python/prodV4/worker.py' Python39V4: pythonVersion: '3.9' - workerPath: 'python/prodV4/worker.py' Python310V4: pythonVersion: '3.10' - workerPath: 'python/prodV4/worker.py' Python311V4: pythonVersion: '3.11' - workerPath: 'python/prodV4/worker.py' Python312V4: pythonVersion: '3.12' - workerPath: 'python/prodV4/worker.py' Python313V4: pythonVersion: '3.13' - workerPath: 'python/proxyV4/worker.py' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: @@ -78,7 +63,6 @@ jobs: - template: ../../../pack/templates/win_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - workerPath: '$(workerPath)' architecture: 'x86' artifactName: '$(pythonVersion)_WINDOWS_x86' - job: Build_LINUX_X64 @@ -90,25 +74,18 @@ jobs: matrix: Python37V4: pythonVersion: '3.7' - workerPath: 'python/prodV4/worker.py' Python38V4: pythonVersion: '3.8' - workerPath: 'python/prodV4/worker.py' Python39V4: pythonVersion: '3.9' - workerPath: 'python/prodV4/worker.py' Python310V4: pythonVersion: '3.10' - workerPath: 'python/prodV4/worker.py' Python311V4: pythonVersion: '3.11' - workerPath: 'python/prodV4/worker.py' Python312V4: pythonVersion: '3.12' - workerPath: 'python/prodV4/worker.py' Python313V4: pythonVersion: '3.13' - workerPath: 'python/proxyV4/worker.py' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: @@ -119,7 +96,6 @@ jobs: - template: ../../../pack/templates/nix_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - workerPath: '$(workerPath)' artifactName: '$(pythonVersion)_LINUX_X64' - job: Build_OSX_X64 pool: @@ -130,25 +106,18 @@ jobs: matrix: Python37V4: pythonVersion: '3.7' - workerPath: 'python/prodV4/worker.py' Python38V4: pythonVersion: '3.8' - workerPath: 'python/prodV4/worker.py' Python39V4: pythonVersion: '3.9' - workerPath: 'python/prodV4/worker.py' Python310V4: pythonVersion: '3.10' - workerPath: 'python/prodV4/worker.py' Python311V4: pythonVersion: '3.11' - workerPath: 'python/prodV4/worker.py' Python312V4: pythonVersion: '3.12' - workerPath: 'python/prodV4/worker.py' Python313V4: pythonVersion: '3.13' - workerPath: 'python/proxyV4/worker.py' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: @@ -159,7 +128,6 @@ jobs: - template: ../../../pack/templates/nix_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - workerPath: '$(workerPath)' artifactName: '$(pythonVersion)_OSX_X64' - job: Build_OSX_ARM64 pool: @@ -170,19 +138,14 @@ jobs: matrix: Python39V4: pythonVersion: '3.9' - workerPath: 'python/prodV4/worker.py' Python310V4: pythonVersion: '3.10' - workerPath: 'python/prodV4/worker.py' Python311V4: pythonVersion: '3.11' - workerPath: 'python/prodV4/worker.py' Python312V4: pythonVersion: '3.12' - workerPath: 'python/prodV4/worker.py' Python313V4: pythonVersion: '3.13' - workerPath: 'python/proxyV4/worker.py' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: @@ -193,7 +156,6 @@ jobs: - template: ../../../pack/templates/macos_64_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - workerPath: '$(workerPath)' artifactName: '$(pythonVersion)_OSX_ARM64' - job: Build_LINUX_ARM64 pool: @@ -204,19 +166,14 @@ jobs: matrix: Python39V4: pythonVersion: '3.9' - workerPath: 'python/prodV4/worker.py' Python310V4: pythonVersion: '3.10' - workerPath: 'python/prodV4/worker.py' Python311V4: pythonVersion: '3.11' - workerPath: 'python/prodV4/worker.py' Python312V4: pythonVersion: '3.12' - workerPath: 'python/prodV4/worker.py' Python313V4: pythonVersion: '3.13' - workerPath: 'python/proxyV4/worker.py' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: @@ -227,7 +184,6 @@ jobs: - template: ../../../pack/templates/nix_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - workerPath: '$(workerPath)' artifactName: '$(pythonVersion)_LINUX_ARM64' - job: Build_WINDOWS_ARM64 pool: @@ -239,13 +195,10 @@ jobs: # ARM64 python is only supported for 3.11 and above Python311V4: pythonVersion: '3.11' - workerPath: 'python/prodV4/worker.py' Python312V4: pythonVersion: '3.12' - workerPath: 'python/prodV4/worker.py' Python313V4: pythonVersion: '3.13' - workerPath: 'python/proxyV4/worker.py' templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: @@ -256,7 +209,6 @@ jobs: - template: ../../../pack/templates/win_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' - workerPath: '$(workerPath)' architecture: 'x64' # only x64 and x86 are supported for windows agents downloadArm64: true artifactName: '$(pythonVersion)_WINDOWS_ARM64' diff --git a/eng/templates/utils/official-variables.yml b/eng/templates/utils/official-variables.yml index 1654d7e6c..8bb7d74a4 100644 --- a/eng/templates/utils/official-variables.yml +++ b/eng/templates/utils/official-variables.yml @@ -1,4 +1,8 @@ variables: - template: /ci/variables/build.yml@eng - template: /ci/variables/cfs.yml@eng - - group: python-integration-resources \ No newline at end of file + - group: python-integration-resources + - name: prodV4Path + value: 'python/prodV4/worker.py' + - name: proxyV4Path + value: 'python/proxyV4/worker.py' \ No newline at end of file From 5af8e415dc91a5fa52294266c63b1f27e2042b50 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 6 May 2025 14:01:40 -0500 Subject: [PATCH 09/10] fix env_gen CopyFiles condition --- eng/pack/templates/macos_64_env_gen.yml | 51 +++++++++++++------------ eng/pack/templates/nix_env_gen.yml | 51 +++++++++++++------------ eng/pack/templates/win_env_gen.yml | 51 +++++++++++++------------ 3 files changed, 81 insertions(+), 72 deletions(-) diff --git a/eng/pack/templates/macos_64_env_gen.yml b/eng/pack/templates/macos_64_env_gen.yml index 331ee4a2c..0ab5d9054 100644 --- a/eng/pack/templates/macos_64_env_gen.yml +++ b/eng/pack/templates/macos_64_env_gen.yml @@ -7,15 +7,33 @@ steps: inputs: versionSpec: ${{ parameters.pythonVersion }} addToPath: true -- bash: | - major=$(echo "$pythonVersion" | cut -d. -f1) - minor=$(echo "$pythonVersion" | cut -d. -f2) - echo "##vso[task.setvariable variable=pythonMajor]$major" - echo "##vso[task.setvariable variable=pythonMinor]$minor" - echo "$minor" - displayName: 'Parse pythonVersion' +- powershell: | + # Parse the Python minor version + $PY_VER = "$(pythonVersion)" + Write-Host "Python version: $PY_VER" + # Extract minor version as integers + $versionParts = $PY_VER.Split('.') + $PY_MINOR = [int]$versionParts[1] + Write-Host "Minor version: $PY_MINOR" + + # Set build-related variables based on Python minor version + if( $PY_MINOR -ge 13 ) + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)" + Write-Host "##vso[task.setvariable variable=proxyWorker;]true" + } + else + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)" + Write-Host "##vso[task.setvariable variable=proxyWorker;]false" + } + displayName: 'Set necessary variables' env: pythonVersion: ${{ parameters.pythonVersion }} +- powershell: | + Write-Host "workerPath: $(workerPath)" + Write-Host "proxyWorker: $(proxyWorker)" + displayName: 'Display variables' - task: ShellScript@2 inputs: disableAutoCwd: true @@ -27,21 +45,6 @@ steps: pip-audit -r requirements.txt displayName: 'Run vulnerability scan' condition: ne(variables['pythonVersion'], '3.7') -- powershell: | - Write-Host "Minor version: $(pythonMinor)" - - if( $(pythonMinor) -ge 13 ) - { - Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)" - } - else - { - Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)" - } - displayName: 'Set workerPath variable' -- powershell: | - Write-Host "workerPath: $(workerPath)" - displayName: 'Display workerPath variable' - task: CopyFiles@2 inputs: contents: '$(workerPath)' @@ -67,7 +70,7 @@ steps: !werkzeug/debug/shared/debugger.js !proxy_worker/** targetFolder: '$(Build.ArtifactStagingDirectory)' - condition: in(variables['pythonMinor'], '7', '8', '9', '10', '11', '12') + condition: eq(variables['proxyWorker'], false) displayName: 'Copy azure_functions_worker files' - task: CopyFiles@2 inputs: @@ -90,5 +93,5 @@ steps: !azure_functions_worker/** !dateutil/** targetFolder: '$(Build.ArtifactStagingDirectory)' - condition: in(variables['pythonMinor'], '13') + condition: eq(variables['proxyWorker'], true) displayName: 'Copy proxy_worker files' diff --git a/eng/pack/templates/nix_env_gen.yml b/eng/pack/templates/nix_env_gen.yml index ea35263c5..982949ce4 100644 --- a/eng/pack/templates/nix_env_gen.yml +++ b/eng/pack/templates/nix_env_gen.yml @@ -7,15 +7,33 @@ steps: inputs: versionSpec: ${{ parameters.pythonVersion }} addToPath: true -- bash: | - major=$(echo "$pythonVersion" | cut -d. -f1) - minor=$(echo "$pythonVersion" | cut -d. -f2) - echo "##vso[task.setvariable variable=pythonMajor]$major" - echo "##vso[task.setvariable variable=pythonMinor]$minor" - echo "$minor" - displayName: 'Parse pythonVersion' +- powershell: | + # Parse the Python minor version + $PY_VER = "$(pythonVersion)" + Write-Host "Python version: $PY_VER" + # Extract minor version as integers + $versionParts = $PY_VER.Split('.') + $PY_MINOR = [int]$versionParts[1] + Write-Host "Minor version: $PY_MINOR" + + # Set build-related variables based on Python minor version + if( $PY_MINOR -ge 13 ) + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)" + Write-Host "##vso[task.setvariable variable=proxyWorker;]true" + } + else + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)" + Write-Host "##vso[task.setvariable variable=proxyWorker;]false" + } + displayName: 'Set necessary variables' env: pythonVersion: ${{ parameters.pythonVersion }} +- powershell: | + Write-Host "workerPath: $(workerPath)" + Write-Host "proxyWorker: $(proxyWorker)" + displayName: 'Display variables' - task: ShellScript@2 inputs: disableAutoCwd: true @@ -27,21 +45,6 @@ steps: pip-audit -r requirements.txt displayName: 'Run vulnerability scan' condition: ne(variables['pythonVersion'], '3.7') -- powershell: | - Write-Host "Minor version: $(pythonMinor)" - - if( $(pythonMinor) -ge 13 ) - { - Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)" - } - else - { - Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)" - } - displayName: 'Set workerPath variable' -- powershell: | - Write-Host "workerPath: $(workerPath)" - displayName: 'Display workerPath variable' - task: CopyFiles@2 inputs: contents: '$(workerPath)' @@ -67,7 +70,7 @@ steps: !werkzeug/debug/shared/debugger.js !proxy_worker/** targetFolder: '$(Build.ArtifactStagingDirectory)' - condition: in(variables['pythonMinor'], '7', '8', '9', '10', '11', '12') + condition: eq(variables['proxyWorker'], false) displayName: 'Copy azure_functions_worker files' - task: CopyFiles@2 inputs: @@ -90,5 +93,5 @@ steps: !dateutil/** !azure_functions_worker/** targetFolder: '$(Build.ArtifactStagingDirectory)' - condition: in(variables['pythonMinor'], '13') + condition: eq(variables['proxyWorker'], true) displayName: 'Copy proxy_worker files' diff --git a/eng/pack/templates/win_env_gen.yml b/eng/pack/templates/win_env_gen.yml index 0094321ab..e549d41d0 100644 --- a/eng/pack/templates/win_env_gen.yml +++ b/eng/pack/templates/win_env_gen.yml @@ -46,39 +46,42 @@ steps: & "$pythonPath" -c "import platform; print(platform.machine())" displayName: 'Download and install Python ARM64' condition: eq('${{ parameters.downloadArm64 }}', 'true') -- bash: | - major=$(echo "$pythonVersion" | cut -d. -f1) - minor=$(echo "$pythonVersion" | cut -d. -f2) - echo "##vso[task.setvariable variable=pythonMajor]$major" - echo "##vso[task.setvariable variable=pythonMinor]$minor" - echo "$minor" - displayName: 'Parse pythonVersion' - env: - pythonVersion: ${{ parameters.pythonVersion }} -- task: PowerShell@2 - inputs: - filePath: 'eng\pack\scripts\win_deps.ps1' - arguments: '${{ parameters.pythonVersion }}' -- bash: | - pip install pip-audit - pip-audit -r requirements.txt - displayName: 'Run vulnerability scan' - condition: ne(variables['pythonVersion'], '3.7') - powershell: | - Write-Host "Minor version: $(pythonMinor)" + # Parse the Python minor version + $PY_VER = "$(pythonVersion)" + Write-Host "Python version: $PY_VER" + # Extract minor version as integers + $versionParts = $PY_VER.Split('.') + $PY_MINOR = [int]$versionParts[1] + Write-Host "Minor version: $PY_MINOR" - if( $(pythonMinor) -ge 13 ) + # Set build-related variables based on Python minor version + if( $PY_MINOR -ge 13 ) { Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)" + Write-Host "##vso[task.setvariable variable=proxyWorker;]true" } else { Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)" + Write-Host "##vso[task.setvariable variable=proxyWorker;]false" } - displayName: 'Set workerPath variable' + displayName: 'Set necessary variables' + env: + pythonVersion: ${{ parameters.pythonVersion }} - powershell: | Write-Host "workerPath: $(workerPath)" - displayName: 'Display workerPath variable' + Write-Host "proxyWorker: $(proxyWorker)" + displayName: 'Display variables' +- task: PowerShell@2 + inputs: + filePath: 'eng\pack\scripts\win_deps.ps1' + arguments: '${{ parameters.pythonVersion }}' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' + condition: ne(variables['pythonVersion'], '3.7') - task: CopyFiles@2 inputs: contents: '$(workerPath)' @@ -104,7 +107,7 @@ steps: !werkzeug\debug\shared\debugger.js !proxy_worker\** targetFolder: '$(Build.ArtifactStagingDirectory)' - condition: in(variables['pythonMinor'], '7', '8', '9', '10', '11', '12') + condition: eq(variables['proxyWorker'], false) displayName: 'Copy azure_functions_worker files' - task: CopyFiles@2 inputs: @@ -127,5 +130,5 @@ steps: !dateutil\** !azure_functions_worker\** targetFolder: '$(Build.ArtifactStagingDirectory)' - condition: in(variables['pythonMinor'], '13') + condition: eq(variables['proxyWorker'], true) displayName: 'Copy proxy_worker files' From 20b0d8e69f24dde27d5d5d44da3865be01324492 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 6 May 2025 15:31:30 -0500 Subject: [PATCH 10/10] remove win-arm64 build --- ...osoft.Azure.Functions.PythonWorker.targets | 1 - ...oft.Azure.Functions.V4.PythonWorker.nuspec | 5 --- eng/pack/templates/win_env_gen.yml | 38 ------------------- .../official/jobs/build-artifacts.yml | 29 +------------- 4 files changed, 1 insertion(+), 72 deletions(-) diff --git a/eng/pack/Microsoft.Azure.Functions.PythonWorker.targets b/eng/pack/Microsoft.Azure.Functions.PythonWorker.targets index 56f50699f..e747df63d 100644 --- a/eng/pack/Microsoft.Azure.Functions.PythonWorker.targets +++ b/eng/pack/Microsoft.Azure.Functions.PythonWorker.targets @@ -7,7 +7,6 @@ <_PythonSupportedRuntime Include="win-x86" WorkerPath="WINDOWS/X86" /> <_PythonSupportedRuntime Include="win-x64" WorkerPath="WINDOWS/X64" /> - <_PythonSupportedRuntime Include="win-arm64" WorkerPath="WINDOWS/Arm64" /> <_PythonSupportedRuntime Include="linux-x64" WorkerPath="LINUX/X64" /> <_PythonSupportedRuntime Include="linux-arm64" WorkerPath="LINUX/Arm64" /> <_PythonSupportedRuntime Include="osx-x64" WorkerPath="OSX/X64" /> diff --git a/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec b/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec index 707932c8d..9fa4f554c 100644 --- a/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec +++ b/eng/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec @@ -24,27 +24,23 @@ - - - - @@ -52,7 +48,6 @@ - diff --git a/eng/pack/templates/win_env_gen.yml b/eng/pack/templates/win_env_gen.yml index e549d41d0..988c688e0 100644 --- a/eng/pack/templates/win_env_gen.yml +++ b/eng/pack/templates/win_env_gen.yml @@ -8,44 +8,6 @@ steps: versionSpec: ${{ parameters.pythonVersion }} architecture: ${{ parameters.architecture }} addToPath: true - condition: ne('${{ parameters.downloadArm64 }}', 'true') -- task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - $versionMap = @{ - "3.11" = "3.11.10" - "3.12" = "3.12.3" - "3.13" = "3.13.0" - } - - $shortVersion = "${{ parameters.pythonVersion }}" - if ($versionMap.ContainsKey($shortVersion)) { - $fullVersion = $versionMap[$shortVersion] - Write-Host "Mapped $shortVersion to full version $fullVersion" - Write-Host "##vso[task.setvariable variable=fullPythonVersion]$fullVersion" - } - else { - Write-Host "ERROR: Unsupported Python version '$shortVersion'" - exit 1 - } - displayName: 'Map pythonVersion to full version' - condition: eq('${{ parameters.downloadArm64 }}', 'true') -- powershell: | - $fullVersion = "$(fullPythonVersion)" - $url = "https://www.python.org/ftp/python/$fullVersion/python-$fullVersion-arm64.exe" - Write-Host "Downloading Python ARM64 from $url" - Invoke-WebRequest -Uri $url -OutFile python-arm64.exe - - Start-Process -Wait -FilePath .\python-arm64.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=0 Include_test=0' - - # Explicitly reference ARM64 Python - $majorMinor = $fullVersion.Substring(0, $fullVersion.LastIndexOf('.')).Replace('.', '') - $pythonPath = "C:\Program Files\Python$majorMinor\python.exe" - & "$pythonPath" --version - & "$pythonPath" -c "import platform; print(platform.machine())" - displayName: 'Download and install Python ARM64' - condition: eq('${{ parameters.downloadArm64 }}', 'true') - powershell: | # Parse the Python minor version $PY_VER = "$(pythonVersion)" diff --git a/eng/templates/official/jobs/build-artifacts.yml b/eng/templates/official/jobs/build-artifacts.yml index f33d1005d..aab12a746 100644 --- a/eng/templates/official/jobs/build-artifacts.yml +++ b/eng/templates/official/jobs/build-artifacts.yml @@ -185,36 +185,9 @@ jobs: parameters: pythonVersion: '$(pythonVersion)' artifactName: '$(pythonVersion)_LINUX_ARM64' -- job: Build_WINDOWS_ARM64 - pool: - name: 1es-pool-azfunc - image: 1es-windows-2022 - os: windows - strategy: - matrix: - # ARM64 python is only supported for 3.11 and above - Python311V4: - pythonVersion: '3.11' - Python312V4: - pythonVersion: '3.12' - Python313V4: - pythonVersion: '3.13' - templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - output: pipelineArtifact - targetPath: $(Build.ArtifactStagingDirectory) - artifactName: "$(pythonVersion)_WINDOWS_ARM64" - steps: - - template: ../../../pack/templates/win_env_gen.yml - parameters: - pythonVersion: '$(pythonVersion)' - architecture: 'x64' # only x64 and x86 are supported for windows agents - downloadArm64: true - artifactName: '$(pythonVersion)_WINDOWS_ARM64' - job: PackageWorkers - dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64', 'Build_LINUX_ARM64', 'Build_WINDOWS_ARM64'] + dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64', 'Build_LINUX_ARM64'] templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: