From ba6fbe9f2a4d23887fc1a03021cfc04ab3147c01 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 12 May 2022 10:03:40 -0700 Subject: [PATCH] Start rotating through all images (#17315) * Use day of week for fallback seed * Use new properties to select CI image * move container selection code into * use parameter * fix azDO variable refernce * move container name left in non-PR build name * Use get-command to detect package manager * update dependencies in build.psm1 * Add RHEL to WSMan support function * Update build.psm1 * Update tools/ci.psm1 Co-authored-by: James Truher [MSFT] Co-authored-by: James Truher [MSFT] --- .vsts-ci/linux.yml | 58 ++++--------------- build.psm1 | 7 +-- .../Modules/HelpersCommon/HelpersCommon.psm1 | 3 +- tools/ci.psm1 | 52 +++++++++++++++++ 4 files changed, 69 insertions(+), 51 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 7fd3d6b2a1..6a3f0280d0 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -1,3 +1,12 @@ +parameters: + - name: ContainerPattern + displayName: | + Pattern to match JobName of the container. + Update this to force a container. + `.` will match everything + type: string + default: . + name: PR-$(System.PullRequest.PullRequestNumber)-$(Date:yyyyMMdd)$(Rev:.rr) trigger: # Batch merge builds together while a merge build is running @@ -116,54 +125,11 @@ stages: clean: true - pwsh: | - $buildId = $env:BUILD_BUILDID - Write-Verbose "Fall back seed: $buildId" -Verbose - # For PRs set the seed to the PR number so that the image is always the same - $seed = $env:SYSTEM_PULLREQUEST_PULLREQUESTID - if(!$seed) { - # for non-PRs use the integer identifier of the build as the seed. - $seed = $buildId - } - Write-Verbose "Seed: $seed" -Verbose - - # Get the latest image matrix JSON for preview - $matrix = ./PowerShell-Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json - - # Filter out containers that won't run on AzDevOps Linux host as an agent - # Windows, nano server, alpine, and any ARM image - $linuxImages = $matrix.preview | - Where-Object {$_.osversion -notmatch 'windows|nano|alpine|arm'} | - Select-Object JobName, Taglist | - Sort-Object -property JobName - - # Force mariner for now - $marinerImage = $linuxImages | where-object {$_.JobName -like 'mariner*'} - - # Use the selected seed to pick a container - $index = Get-Random -Minimum 0 -Maximum $marinerImage.Count -SetSeed $seed - $selectedImage = $marinerImage[$index] - - # Filter to the first test-deps compatible tag - $tag = $selectedImage.Taglist -split ';' | Where-Object {$_ -match 'preview-\D+'} | Select-Object -First 1 - - # Calculate the container name - $containerName = "mcr.microsoft.com/powershell/test-deps:$tag" - - # Set variables for later jobs to use + # Initialize container test stage Import-Module ./PowerShell/tools/ci.psm1 - Set-BuildVariable -Name containerName -Value $containerName -IsOutput - Set-BuildVariable -Name containerBuildName -Value $selectedImage.JobName -IsOutput - - if($env:BUILD_REASON -eq 'PullRequest') { - Write-Host "##vso[build.updatebuildnumber]PR-$(System.PullRequest.PullRequestNumber)-$($selectedImage.JobName)-$((get-date).ToString("yyyyMMddhhmmss"))" - } else { - Write-Host "##vso[build.updatebuildnumber]${env:BUILD_SOURCEBRANCHNAME}-${env:BUILD_SOURCEVERSION}-$($selectedImage.JobName)-$((get-date).ToString("yyyyMMddhhmmss"))" - - # Cannot do this for a PR - Write-Host "##vso[build.addbuildtag]$($selectedImage.JobName)" - } + Invoke-InitializeContainerStage -ContainerPattern '${{ parameters.ContainerPattern }}' name: getContainerTask - displayName: Get Container + displayName: Initialize Container Stage continueOnError: true - template: templates/test/nix-container-test.yml diff --git a/build.psm1 b/build.psm1 index 0d1096d673..d8796b78e1 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1959,9 +1959,9 @@ function Install-Dotnet { } function Get-RedHatPackageManager { - if ($environment.IsCentOS) { + if ($environment.IsCentOS -or (Get-Command -Name yum -CommandType Application -ErrorAction SilentlyContinue)) { "yum install -y -q" - } elseif ($environment.IsFedora) { + } elseif ($environment.IsFedora -or (Get-Command -Name dnf -CommandType Application -ErrorAction SilentlyContinue)) { "dnf install -y -q" } else { throw "Error determining package manager for this distribution." @@ -2044,7 +2044,6 @@ function Start-PSBootstrap { $Deps += "curl", "wget" # .NET Core required runtime libraries - $Deps += "libunwind8" if ($environment.IsUbuntu16) { $Deps += "libicu55" } elseif ($environment.IsUbuntu18) { $Deps += "libicu60"} @@ -2070,7 +2069,7 @@ function Start-PSBootstrap { $Deps += "which", "curl", "wget" # .NET Core required runtime libraries - $Deps += "libicu", "libunwind" + $Deps += "libicu", "openssl-libs" # Packaging tools if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel', "gcc-c++" } diff --git a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 index dbd77186d2..e0b2c8c2e6 100644 --- a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 +++ b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 @@ -391,10 +391,11 @@ function Get-WsManSupport { $platformInfo = Get-PlatformInfo if ( ($platformInfo.Platform -match "alpine|raspbian") -or - ($platformInfo.Platform -eq "debian" -and $platformInfo.Version -ne '9') -or + ($platformInfo.Platform -eq "debian" -and $platformInfo.Version -ne '9') -or ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or ($platformInfo.Platform -eq 'mariner') -or + ($platformInfo.Platform -eq 'rhel') -or ($IsMacOS) ) { return $false diff --git a/tools/ci.psm1 b/tools/ci.psm1 index bc87db9db5..7991a03fba 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -811,3 +811,55 @@ function New-LinuxPackage Copy-Item $armPackage -Destination "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}" -Force } } + +function Invoke-InitializeContainerStage { + param( + [string] + $ContainerPattern = '.' + ) + + Write-Verbose "Invoking InitializeContainerStage with ContainerPattern: ${ContainerPattern}" -Verbose + + $fallbackSeed = (get-date).DayOfYear + Write-Verbose "Fall back seed: $fallbackSeed" -Verbose + + # For PRs set the seed to the PR number so that the image is always the same + $seed = $env:SYSTEM_PULLREQUEST_PULLREQUESTID + if(!$seed) { + # for non-PRs use the integer identifier of the build as the seed. + $seed = $fallbackSeed + } + + Write-Verbose "Seed: $seed" -Verbose + + # Get the latest image matrix JSON for preview + $matrix = ./PowerShell-Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json + + # Chose images that are validated or validating, Linux and can be used in CI. + $linuxImages = $matrix.preview | + Where-Object {$_.IsLinux -and $_.UseInCi -and $_.DistributionState -match 'Validat.*' -and $_.JobName -match $ContainerPattern} | + Select-Object JobName, Taglist | + Sort-Object -property JobName + + # Use the selected seed to pick a container + $index = Get-Random -Minimum 0 -Maximum $linuxImages.Count -SetSeed $seed + $selectedImage = $linuxImages[$index] + + # Filter to the first test-deps compatible tag + $tag = $selectedImage.Taglist -split ';' | Where-Object {$_ -match 'preview-\D+'} | Select-Object -First 1 + + # Calculate the container name + $containerName = "mcr.microsoft.com/powershell/test-deps:$tag" + + Set-BuildVariable -Name containerName -Value $containerName -IsOutput + Set-BuildVariable -Name containerBuildName -Value $selectedImage.JobName -IsOutput + + if($env:BUILD_REASON -eq 'PullRequest') { + Write-Host "##vso[build.updatebuildnumber]PR-${env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER}-$($selectedImage.JobName)-$((get-date).ToString("yyyyMMddhhmmss"))" + } else { + Write-Host "##vso[build.updatebuildnumber]${env:BUILD_SOURCEBRANCHNAME}-$($selectedImage.JobName)-${env:BUILD_SOURCEVERSION}-$((get-date).ToString("yyyyMMddhhmmss"))" + + # Cannot do this for a PR + Write-Host "##vso[build.addbuildtag]$($selectedImage.JobName)" + } +}