From d734a739781ebddbb6527d53495f4d290c337f6a Mon Sep 17 00:00:00 2001 From: Varad Date: Mon, 19 Jun 2023 18:54:33 -0500 Subject: [PATCH 1/5] Updating the protobuf dependency and build pipelines for ARM64 mac --- azure-pipelines.yml | 22 ++++++++++++- ...oft.Azure.Functions.V4.PythonWorker.nuspec | 3 ++ pack/scripts/mac_arm64_deps.sh | 9 ++++++ pack/templates/macos_64_env_gen.yml | 32 +++++++++++++++++++ python/prodV4/worker.config.json | 4 +-- setup.py | 14 ++++---- 6 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 pack/scripts/mac_arm64_deps.sh create mode 100644 pack/templates/macos_64_env_gen.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a56edcbda..df46b6960 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -122,9 +122,29 @@ jobs: pythonVersion: '$(pythonVersion)' workerPath: '$(workerPath)' artifactName: '$(pythonVersion)_OSX_X64' +- job: Build_OSX_ARM64 + pool: + vmImage: 'macOS-12' + strategy: + matrix: + Python39V4: + pythonVersion: '3.9' + workerPath: $(PROD_V4_WORKER_PY) + Python310V4: + pythonVersion: '3.10' + workerPath: $(PROD_V4_WORKER_PY) + Python311V4: + pythonVersion: '3.11' + workerPath: $(PROD_V4_WORKER_PY) + steps: + - template: pack/templates/macos_64_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + workerPath: '$(workerPath)' + artifactName: '$(pythonVersion)_OSX_ARM64' - job: PackageWorkers - dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64'] + dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64'] condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev')) pool: name: '1ES-Hosted-AzFunc' diff --git a/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec b/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec index 4ffc38b50..3426e72b8 100644 --- a/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec +++ b/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec @@ -22,14 +22,17 @@ + + + diff --git a/pack/scripts/mac_arm64_deps.sh b/pack/scripts/mac_arm64_deps.sh new file mode 100644 index 000000000..13a2315da --- /dev/null +++ b/pack/scripts/mac_arm64_deps.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python -m venv .env +source .env/bin/activate +python -m pip install --upgrade pip==23.0 + +python -m pip install . + +python -m pip install . --no-compile --target "$BUILD_SOURCESDIRECTORY/deps" \ No newline at end of file diff --git a/pack/templates/macos_64_env_gen.yml b/pack/templates/macos_64_env_gen.yml new file mode 100644 index 000000000..94c2260a1 --- /dev/null +++ b/pack/templates/macos_64_env_gen.yml @@ -0,0 +1,32 @@ +parameters: + pythonVersion: '' + artifactName: '' + workerPath: '' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: ${{ parameters.pythonVersion }} + addToPath: true +- task: ShellScript@2 + inputs: + disableAutoCwd: true + scriptPath: 'pack/scripts/mac_arm64_deps.sh' +- task: CopyFiles@2 + inputs: + contents: | + ${{ parameters.workerPath }} + targetFolder: '$(Build.ArtifactStagingDirectory)' + flattenFolders: true +- task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)/deps' + contents: | + ** + !grpc_tools/**/* + !grpcio_tools*/* + targetFolder: '$(Build.ArtifactStagingDirectory)' +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: ${{ parameters.artifactName }} diff --git a/python/prodV4/worker.config.json b/python/prodV4/worker.config.json index 552cecc29..f22d26f45 100644 --- a/python/prodV4/worker.config.json +++ b/python/prodV4/worker.config.json @@ -1,10 +1,10 @@ { "description":{ "language":"python", - "defaultRuntimeVersion":"3.9", + "defaultRuntimeVersion":"3.10", "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"], "supportedRuntimeVersions":["3.7", "3.8", "3.9", "3.10", "3.11"], - "supportedArchitectures":["X64", "X86"], + "supportedArchitectures":["X64", "X86", "Arm64"], "extensions":[".py"], "defaultExecutablePath":"python", "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py", diff --git a/setup.py b/setup.py index bda8b4d96..4a49556b5 100644 --- a/setup.py +++ b/setup.py @@ -110,14 +110,14 @@ "python-dateutil~=2.8.2" ] -if sys.version_info[:3] < (3, 11, 0): - INSTALL_REQUIRES.append("protobuf~=3.19.3") - INSTALL_REQUIRES.append("grpcio-tools~=1.43.0") - INSTALL_REQUIRES.append("grpcio~=1.43.0") +if sys.version_info[:3] < (3, 9, 0): + INSTALL_REQUIRES.extend( + ("protobuf~=3.19.3", "grpcio-tools~=1.43.0", "grpcio~=1.43.0") + ) else: - INSTALL_REQUIRES.append("protobuf~=4.22.0") - INSTALL_REQUIRES.append("grpcio-tools~=1.51.3") - INSTALL_REQUIRES.append("grpcio~=1.51.3") + INSTALL_REQUIRES.extend( + ("protobuf~=4.22.0", "grpcio-tools~=1.54.2", "grpcio~=1.54.2") + ) EXTRA_REQUIRES = { "dev": [ From 7e48466a0d2be4ec35f2e5f7670f534f604f8975 Mon Sep 17 00:00:00 2001 From: Varad Meru Date: Wed, 21 Jun 2023 18:52:33 -0500 Subject: [PATCH 2/5] Moving to use macOS-latest --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index df46b6960..e66419899 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -98,7 +98,7 @@ jobs: artifactName: '$(pythonVersion)_LINUX_X64' - job: Build_OSX_X64 pool: - vmImage: 'macOS-12' + vmImage: 'macOS-latest' strategy: matrix: Python37V4: @@ -124,7 +124,7 @@ jobs: artifactName: '$(pythonVersion)_OSX_X64' - job: Build_OSX_ARM64 pool: - vmImage: 'macOS-12' + vmImage: 'macOS-latest' strategy: matrix: Python39V4: From 950d0ab7f5b7e5a615768af1ef4e49b4163464a8 Mon Sep 17 00:00:00 2001 From: Varad Meru Date: Wed, 21 Jun 2023 18:54:10 -0500 Subject: [PATCH 3/5] Creating a mechanism to generate packages --- azure-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e66419899..ddb7db391 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,7 +7,8 @@ trigger: variables: patchBuildNumberForDev: $(Build.BuildNumber) - PROD_V4_WORKER_PY : 'python/prodV4/worker.py' + PROD_V4_WORKER_PY: 'python/prodV4/worker.py' + GeneratePackage: false jobs: - job: Build_WINDOWS_X64 @@ -145,7 +146,7 @@ jobs: - job: PackageWorkers dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64'] - condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev')) + condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), GeneratePackage) pool: name: '1ES-Hosted-AzFunc' demands: From 4db1666f920a8dead71018199e7f5f7d726bf2dc Mon Sep 17 00:00:00 2001 From: Varad Meru Date: Wed, 21 Jun 2023 21:57:13 -0500 Subject: [PATCH 4/5] Converting GeneratePackage to a Variable --- azure-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ddb7db391..79749b58c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,7 +8,6 @@ trigger: variables: patchBuildNumberForDev: $(Build.BuildNumber) PROD_V4_WORKER_PY: 'python/prodV4/worker.py' - GeneratePackage: false jobs: - job: Build_WINDOWS_X64 @@ -146,7 +145,7 @@ jobs: - job: PackageWorkers dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64'] - condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), GeneratePackage) + condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), variables['GeneratePackage']) pool: name: '1ES-Hosted-AzFunc' demands: From 0c1c114a8ec889de9afeaf17d560a7c0b972d6dd Mon Sep 17 00:00:00 2001 From: Varad Meru Date: Fri, 23 Jun 2023 10:23:38 -0500 Subject: [PATCH 5/5] Explicit Generate Package check --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 79749b58c..c0d87b28a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -145,7 +145,7 @@ jobs: - job: PackageWorkers dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64'] - condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), variables['GeneratePackage']) + condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), eq(variables['GeneratePackage'], True)) pool: name: '1ES-Hosted-AzFunc' demands: