Skip to content

Sync eng/common directory with azure-sdk-tools for PR 9993 #40021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Mar 11, 2025
Merged

Sync eng/common directory with azure-sdk-tools for PR 9993 #40021

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e9f207a
move the check to see if the file is a sole CI item. store a lookup f…
scbedd Mar 6, 2025
b265086
cache the results of GCi to speed up the diff action
scbedd Mar 6, 2025
c01b8ba
output so we can see how fast the processing is actually going
scbedd Mar 6, 2025
96e6c10
heavy utilization of grouping the impacted directories
scbedd Mar 7, 2025
bd70946
simplify the triggeringPath logic now that we don't need to resolve t…
scbedd Mar 7, 2025
c744622
remove the excluded paths from the targetedFiles entirely. they're ex…
scbedd Mar 7, 2025
ddd9cac
updates for clarity
scbedd Mar 7, 2025
87b46f9
only do additional work for indirect if we have decided that this pac…
scbedd Mar 7, 2025
035e717
caught some issues in various integration test runs
scbedd Mar 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 164 additions & 62 deletions eng/common/scripts/Package-Properties.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,12 +128,35 @@ class PackageProps {
if ($ciArtifactResult) {
$this.ArtifactDetails = [Hashtable]$ciArtifactResult.ArtifactConfig

$repoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..")
$ciYamlPath = (Resolve-Path -Path $ciArtifactResult.Location -Relative -RelativeBasePath $repoRoot).TrimStart(".").Replace("`\", "/")
$relRoot = [System.IO.Path]::GetDirectoryName($ciYamlPath).Replace("`\", "/")

if (-not $this.ArtifactDetails["triggeringPaths"]) {
$this.ArtifactDetails["triggeringPaths"] = @()
}
$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..")
$relativePath = (Resolve-Path -Path $ciArtifactResult.Location -Relative -RelativeBasePath $RepoRoot).TrimStart(".").Replace("`\", "/")
$this.ArtifactDetails["triggeringPaths"] += $relativePath
else {
$adjustedPaths = @()

# we need to convert relative references to absolute references within the repo
# this will make it extremely easy to compare triggering paths to files in the deleted+changed file list.
for ($i = 0; $i -lt $this.ArtifactDetails["triggeringPaths"].Count; $i++) {
$currentPath = $this.ArtifactDetails["triggeringPaths"][$i]
$newPath = Join-Path $repoRoot $currentPath
if (!$currentPath.StartsWith("/")) {
$newPath = Join-Path $repoRoot $relRoot $currentPath
}
# it is a possibility that users may have a triggerPath dependency on a file that no longer exists.
# before we resolve it to get rid of possible relative references, we should check if the file exists
# if it doesn't, we should just leave it as is. Otherwise we would _crash_ here when a user accidentally
# left a triggeringPath on a file that had been deleted
if (Test-Path $newPath) {
$adjustedPaths += (Resolve-Path -Path $newPath -Relative -RelativeBasePath $repoRoot).TrimStart(".").Replace("`\", "/")
}
}
$this.ArtifactDetails["triggeringPaths"] = $adjustedPaths
}
$this.ArtifactDetails["triggeringPaths"] += $ciYamlPath

$this.CIParameters["CIMatrixConfigs"] = @()

Expand Down Expand Up @@ -180,8 +203,86 @@ function Get-PkgProperties {
return $null
}


function Get-TriggerPaths([PSCustomObject]$AllPackageProperties) {
$existingTriggeringPaths = @()
$AllPackageProperties | ForEach-Object {
if ($_.ArtifactDetails) {
$pathsForArtifact = $_.ArtifactDetails["triggeringPaths"]
foreach ($triggerPath in $pathsForArtifact){
# we only care about triggering paths that are actual files, not directories
# go by by the assumption that if the triggerPath has an extension, it's a file :)
if ([System.IO.Path]::HasExtension($triggerPath)) {
$existingTriggeringPaths += $triggerPath
}
}
}
}

return ($existingTriggeringPaths | Select-Object -Unique)
}

function Update-TargetedFilesForTriggerPaths([string[]]$TargetedFiles, [string[]]$TriggeringPaths) {
# now we simply loop through the files a single time, keeping all the files that are a triggeringPath
# for the rest of the files, simply group by what directory they belong to
# the new TargetedFiles array will contain only the changed directories + the files that actually aligned to a triggeringPath
$processedFiles = @()
$Triggers = [System.Collections.ArrayList]$TriggeringPaths
$i = 0
foreach ($file in $TargetedFiles) {
$isExistingTriggerPath = $false

for ($i = 0; $i -lt $Triggers.Count; $i++) {
$triggerPath = $Triggers[$i]
if ($triggerPath -and $file -eq "$triggerPath") {
$isExistingTriggerPath = $true
break
}
}

if ($isExistingTriggerPath) {
# we know that we should have a valid $i that we can use to remove the triggerPath from the list
# so that it gets smaller as we find items
$Triggers.RemoveAt($i)
$processedFiles += $file
}
else {
# Get directory path by removing the filename
$directoryPath = Split-Path -Path $file -Parent
if ($directoryPath) {
$processedFiles += $directoryPath
} else {
# In case there's no parent directory (root file), keep the original
$processedFiles += $file
}
}
}

return ($processedFiles | Select-Object -Unique)
}

function Update-TargetedFilesForExclude([string[]]$TargetedFiles, [string[]]$ExcludePaths) {
$files = @()
foreach ($file in $TargetedFiles) {
$shouldExclude = $false
foreach ($exclude in $ExcludePaths) {
if (!$file.StartsWith($exclude,'CurrentCultureIgnoreCase')) {
$shouldExclude = $true
break
}
}
if (!$shouldExclude) {
$files += $file
}
}
return ,$files
}

function Get-PrPkgProperties([string]$InputDiffJson) {
$packagesWithChanges = @()
$additionalValidationPackages = @()
$lookup = @{}
$directoryIndex = @{}

$allPackageProperties = Get-AllPkgProperties
$diff = Get-Content $InputDiffJson | ConvertFrom-Json
Expand All @@ -194,10 +295,9 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
$targetedFiles += $diff.DeletedFiles
}

$excludePaths = $diff.ExcludePaths

$additionalValidationPackages = @()
$lookup = @{}
$existingTriggeringPaths = Get-TriggerPaths $allPackageProperties
$targetedFiles = Update-TargetedFilesForExclude $targetedFiles $diff.ExcludePaths
$targetedFiles = Update-TargetedFilesForTriggerPaths $targetedFiles $existingTriggeringPaths

# Sort so that we very quickly find any directly changed packages before hitting service level changes.
# This is important because due to the way we traverse the changed files, the instant we evaluate a pkg
Expand All @@ -206,9 +306,11 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
# To avoid this without wonky changes to the detection algorithm, we simply sort our files by their depth, so we will always
# detect direct package changes first!
$targetedFiles = $targetedFiles | Sort-Object { ($_.Split("/").Count) } -Descending
$pkgCounter = 1

# this is the primary loop that identifies the packages that have changes
foreach ($pkg in $allPackageProperties) {
Write-Host "Processing changed files against $($pkg.Name). $pkgCounter of $($allPackageProperties.Count)."
$pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)"
$lookupKey = ($pkg.DirectoryPath).Replace($RepoRoot, "").TrimStart('\/')
$lookup[$lookupKey] = $pkg
Expand All @@ -224,73 +326,71 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
}

foreach ($file in $targetedFiles) {
$pathComponents = $file -split "/"
$shouldExclude = $false
foreach ($exclude in $excludePaths) {
if ($file.StartsWith($exclude,'CurrentCultureIgnoreCase')) {
$shouldExclude = $true
break
}
}
if ($shouldExclude) {
continue
}
$filePath = (Join-Path $RepoRoot $file)

# handle direct changes to packages
$shouldInclude = $filePath -like (Join-Path "$pkgDirectory" "*")

# handle changes to files that are RELATED to each package
foreach($triggerPath in $triggeringPaths) {
$resolvedRelativePath = (Join-Path $RepoRoot $triggerPath)
if (!$triggerPath.StartsWith("/")){
$resolvedRelativePath = (Join-Path $RepoRoot "sdk" "$($pkg.ServiceDirectory)" $triggerPath)
}

# if we are including this package due to one of its additional trigger paths, we need
# to ensure we're counting it as included for validation, not as an actual package change
if ($resolvedRelativePath) {
$includedForValidation = $filePath -like (Join-Path "$resolvedRelativePath" "*")
$shouldInclude = $filePath -eq $pkgDirectory -or $filePath -like (Join-Path "$pkgDirectory" "*")

# we only need to do additional work for indirect packages if we haven't already decided
# to include this package due to this file
if (-not $shouldInclude) {
# handle changes to files that are RELATED to each package
foreach($triggerPath in $triggeringPaths) {
$resolvedRelativePath = (Join-Path $RepoRoot $triggerPath)
# triggerPaths can be direct files, so we need to check both startswith and direct equality
$includedForValidation = ($filePath -like (Join-Path "$resolvedRelativePath" "*") -or $filePath -eq $resolvedRelativePath)
$shouldInclude = $shouldInclude -or $includedForValidation
if ($includedForValidation) {
$pkg.IncludedForValidation = $true
}
break
}
}

# handle service-level changes to the ci.yml files
# we are using the ci.yml file being added automatically to each artifactdetails as the input
# for this task. This is because we can resolve a service directory from the ci.yml, and if
# there is a single ci.yml in that directory, we can assume that any file change in that directory
# will apply to all packages that exist in that directory.
$triggeringCIYmls = $triggeringPaths | Where-Object { $_ -like "*ci*.yml" }

foreach($yml in $triggeringCIYmls) {
# given that this path is coming from the populated triggering paths in the artifact,
# we can assume that the path to the ci.yml will successfully resolve.
$ciYml = Join-Path $RepoRoot $yml
# ensure we terminate the service directory with a /
$directory = [System.IO.Path]::GetDirectoryName($ciYml).Replace("`\", "/")
$soleCIYml = (Get-ChildItem -Path $directory -Filter "ci*.yml" -File).Count -eq 1

# we should only continue with this check if the file being changed is "in the service directory"
$serviceDirectoryChange = (Split-Path $filePath -Parent).Replace("`\", "/") -eq $directory
if (!$serviceDirectoryChange) {
break
}
# handle service-level changes to the ci.yml files
# we are using the ci.yml file being added automatically to each artifactdetails as the input
# for this task. This is because we can resolve a service directory from the ci.yml, and if
# there is a single ci.yml in that directory, we can assume that any file change in that directory
# will apply to all packages that exist in that directory.
$triggeringCIYmls = $triggeringPaths | Where-Object { $_ -like "*ci*.yml" }

foreach($yml in $triggeringCIYmls) {
# given that this path is coming from the populated triggering paths in the artifact,
# we can assume that the path to the ci.yml will successfully resolve.
$ciYml = Join-Path $RepoRoot $yml
# ensure we terminate the service directory with a /
$directory = [System.IO.Path]::GetDirectoryName($ciYml).Replace("`\", "/")

# we should only continue with this check if the file being changed is "in the service directory"
# files that are directly included in triggerPaths will kept in full form, but otherwise we pre-process the targetedFiles to the
# directory containing the change. Given that pre-process, we should check both direct equality (when not triggeringPath) and parent directory
# for the case where the full form of the file has been left behind (because it was a triggeringPath)
$serviceDirectoryChange = (Split-Path $filePath -Parent).Replace("`\", "/") -eq $directory -or $filePath.Replace("`\", "/") -eq $directory
if (!$serviceDirectoryChange) {
break
}

if ($soleCIYml -and $filePath.Replace("`\", "/").StartsWith($directory)) {
if (-not $shouldInclude) {
$pkg.IncludedForValidation = $true
$shouldInclude = $true
# this GCI is very expensive, so we want to cache the result
$soleCIYml = $true
if ($directoryIndex[$directory]) {
$soleCIYml = $directoryIndex[$directory]
}
else {
$soleCIYml = (Get-ChildItem -Path $directory -Filter "ci*.yml" -File).Count -eq 1
$directoryIndex[$directory] = $soleCIYml
}

if ($soleCIYml -and $filePath.Replace("`\", "/").StartsWith($directory)) {
if (-not $shouldInclude) {
$pkg.IncludedForValidation = $true
$shouldInclude = $true
}
break
}
else {
# if the ci.yml is not the only file in the directory, we cannot assume that any file changed within the directory that isn't the ci.yml
# should trigger this package
Write-Host "Skipping adding package for file `"$file`" because the ci yml `"$yml`" is not the only file in the service directory `"$directory`""
}
break
}
else {
# if the ci.yml is not the only file in the directory, we cannot assume that any file changed within the directory that isn't the ci.yml
# should trigger this package
Write-Host "Skipping adding package for file `"$file`" because the ci yml `"$yml`" is not the only file in the service directory `"$directory`""
}
}

Expand All @@ -305,6 +405,8 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
break
}
}

$pkgCounter++
}

# add all of the packages that were added purely for validation purposes
Expand Down
Loading