diff --git a/eng/pack/scripts/nix_arm64_deps.sh b/eng/pack/scripts/nix_arm64_deps.sh new file mode 100644 index 000000000..770479d13 --- /dev/null +++ b/eng/pack/scripts/nix_arm64_deps.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +python -m venv .env +source .env/bin/activate +python -m pip install --upgrade pip + +version_minor=$(echo $1 | cut -d '.' -f 2) +mkdir -p $BUILD_SOURCESDIRECTORY/deps + +# Targeting: grpcio manylinux_2_17_aarch64.whl build + +# Starts a docker container using the linux/arm64 platform +# Inside the container, we perform the same steps as our typical builds +# However, since we're running them on the linux/arm64 platform, we ensure +# that we pull in the correct grpc, etc. builds +docker run --privileged --rm tonistiigi/binfmt --install all +docker run --name my-arm64-container --platform linux/arm64 \ + -v ./:/src \ + -w /src \ + python:3.$version_minor bash -c " + ls -la /src # debug: see what files exist + apt-get update && apt-get install -y git curl && \ + pip install --upgrade pip && \ + pip install . && \ + pip install . --target /src && \ + pip install invoke && \ + cd tests && \ + python -m invoke -c test_setup build-protos && \ + ls -la /src + " + +# This copies over the build files from the docker container to the local pipeline +docker cp my-arm64-container:/src/. $BUILD_SOURCESDIRECTORY/all/ +docker rm my-arm64-container + +# From the container, we have many unnecessary files. Here, we only +# copy over the relevant files to the 'deps/' directory. +copy_list=( + "azure" + "azure_functions_worker" + "azurefunctions" + "dateutil" + "google" + "grpc" + "markupsafe" + "proxy_worker" + "six.py" + "werkzeug" +) + +for dir in "${copy_list[@]}"; do + src="$BUILD_SOURCESDIRECTORY/all/$dir" + dest="$BUILD_SOURCESDIRECTORY/deps" + + if [ -e $src ]; then + echo "Copying $dir..." + cp -r $src $dest + else + echo "Directory $dir not found in deps — skipping" + fi + done + +cp .artifactignore "$BUILD_SOURCESDIRECTORY/deps" + +version_minor=$(echo $1 | cut -d '.' -f 2) +if [[ $version_minor -lt 13 ]]; then + cp -r azure_functions_worker/protos "$BUILD_SOURCESDIRECTORY/deps/azure_functions_worker" +else + cp -r proxy_worker/protos "$BUILD_SOURCESDIRECTORY/deps/proxy_worker" +fi + +echo "Listing contents of deps directory:" +ls -la $BUILD_SOURCESDIRECTORY/deps \ No newline at end of file diff --git a/eng/pack/templates/nix_arm64_env_gen.yml b/eng/pack/templates/nix_arm64_env_gen.yml new file mode 100644 index 000000000..5d3167bd8 --- /dev/null +++ b/eng/pack/templates/nix_arm64_env_gen.yml @@ -0,0 +1,103 @@ +parameters: + pythonVersion: '' + artifactName: '' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: ${{ parameters.pythonVersion }} + addToPath: true +- powershell: | + # Parse the Python minor version + $PY_VER = "$(pythonVersion)" + Write-Host "Python version: $PY_VER" + # Extract minor version as integers + $versionParts = $PY_VER.Split('.') + $PY_MINOR = [int]$versionParts[1] + Write-Host "Minor version: $PY_MINOR" + + # Set build-related variables based on Python minor version + if( $PY_MINOR -ge 13 ) + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(proxyV4Path)" + Write-Host "##vso[task.setvariable variable=proxyWorker;]true" + } + else + { + Write-Host "##vso[task.setvariable variable=workerPath;]$(prodV4Path)" + Write-Host "##vso[task.setvariable variable=proxyWorker;]false" + } + displayName: 'Set necessary variables' + env: + pythonVersion: ${{ parameters.pythonVersion }} +- powershell: | + Write-Host "workerPath: $(workerPath)" + Write-Host "proxyWorker: $(proxyWorker)" + displayName: 'Display variables' +- script: | + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + displayName: 'Docker login to avoid rate limits' + env: + DOCKER_USERNAME: $(DOCKER_USERNAME) + DOCKER_PASSWORD: $(DOCKER_PASSWORD) +- task: ShellScript@2 + inputs: + disableAutoCwd: true + scriptPath: 'eng/pack/scripts/nix_arm64_deps.sh' + args: '${{ parameters.pythonVersion }}' + displayName: 'Install Dependencies' +- bash: | + pip install pip-audit + pip-audit -r requirements.txt + displayName: 'Run vulnerability scan' + condition: ne(variables['pythonVersion'], '3.7') +- task: CopyFiles@2 + inputs: + contents: '$(workerPath)' + targetFolder: '$(Build.ArtifactStagingDirectory)' + flattenFolders: true +- task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)/deps' + contents: | + ** + !grpc_tools/**/* + !grpcio_tools*/* + !build/** + !docs/** + !eng/** + !python/** + !tests/** + !setuptools*/** + !_distutils_hack/** + !distutils-precedence.pth + !pkg_resources/** + !*.dist-info/** + !werkzeug/debug/shared/debugger.js + !proxy_worker/** + targetFolder: '$(Build.ArtifactStagingDirectory)' + condition: eq(variables['proxyWorker'], false) + displayName: 'Copy azure_functions_worker files' +- task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)/deps' + contents: | + ** + !grpc_tools/**/* + !grpcio_tools*/* + !build/** + !docs/** + !eng/** + !python/** + !tests/** + !setuptools*/** + !_distutils_hack/** + !distutils-precedence.pth + !pkg_resources/** + !*.dist-info/** + !werkzeug/debug/shared/debugger.js + !dateutil/** + !azure_functions_worker/** + targetFolder: '$(Build.ArtifactStagingDirectory)' + condition: eq(variables['proxyWorker'], true) + displayName: 'Copy proxy_worker files' diff --git a/eng/templates/official/jobs/build-artifacts.yml b/eng/templates/official/jobs/build-artifacts.yml index aab12a746..af6467c12 100644 --- a/eng/templates/official/jobs/build-artifacts.yml +++ b/eng/templates/official/jobs/build-artifacts.yml @@ -159,9 +159,9 @@ jobs: artifactName: '$(pythonVersion)_OSX_ARM64' - job: Build_LINUX_ARM64 pool: - name: Azure Pipelines - image: macOS-latest - os: macOS + name: 1es-pool-azfunc + image: 1es-ubuntu-22.04 + os: linux strategy: matrix: Python39V4: @@ -181,7 +181,7 @@ jobs: targetPath: $(Build.ArtifactStagingDirectory) artifactName: "$(pythonVersion)_LINUX_ARM64" steps: - - template: ../../../pack/templates/nix_env_gen.yml + - template: ../../../pack/templates/nix_arm64_env_gen.yml parameters: pythonVersion: '$(pythonVersion)' artifactName: '$(pythonVersion)_LINUX_ARM64'