mirror of
https://github.com/PowerShell/PowerShell
synced 2026-06-08 12:12:50 +00:00
[release/v7.5]Add EV2 support for publishing PowerShell packages to PMC (#24856)
Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Justin Chung <chungjustin@microsoft.com> Co-authored-by: Justin Chung <t-juchung@microsoft.com> Co-authored-by: Justin Chung <124807742+jshigetomi@users.noreply.github.com> Co-authored-by: Aditya Patwardhan <adityap@microsoft.com> Co-authored-by: alerickson <25858831+alerickson@users.noreply.github.com> Co-authored-by: Patrick Meinecke <SeeminglyScience@users.noreply.github.com> Co-authored-by: Travis Plunk <travis.plunk@microsoft.com> Co-authored-by: PowerShell Team Bot <69177312+pwshBot@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jordan Borean <jborean93@gmail.com>
This commit is contained in:
Anam Navied
co-authored by
Justin Chung
Justin Chung
Justin Chung
Aditya Patwardhan
alerickson
Patrick Meinecke
Travis Plunk
PowerShell Team Bot
dependabot[bot]
Jordan Borean
Justin Chung
Justin Chung
Justin Chung
Aditya Patwardhan
alerickson
Patrick Meinecke
Travis Plunk
PowerShell Team Bot
dependabot[bot]
Jordan Borean
parent
f36dd32296
commit
06997b4005
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutSpecification.json",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"rolloutMetadata": {
|
||||
"serviceModelPath": "ServiceModel.json",
|
||||
"ScopeBindingsPath": "ScopeBindings.json",
|
||||
"name": "OneBranch-Demo-Container-Deployment",
|
||||
"rolloutType": "Major",
|
||||
"buildSource": {
|
||||
"parameters": {
|
||||
"versionFile": "buildver.txt"
|
||||
}
|
||||
},
|
||||
"Notification": {
|
||||
"Email": {
|
||||
"To": "default"
|
||||
}
|
||||
}
|
||||
},
|
||||
"orchestratedSteps": [
|
||||
{
|
||||
"name": "UploadLinuxContainer",
|
||||
"targetType": "ServiceResource",
|
||||
"targetName": "LinuxContainerUpload",
|
||||
"actions": ["Shell/Run"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"$schema": "https://ev2schema.azure.net/schemas/2020-01-01/scopeBindings.json",
|
||||
"contentVersion": "0.0.0.1",
|
||||
"scopeBindings": [
|
||||
{
|
||||
"scopeTagName": "Global",
|
||||
"bindings": [
|
||||
{
|
||||
"find": "__SUBSCRIPTION_ID__",
|
||||
"replaceWith": "$azureSubscriptionId()"
|
||||
},
|
||||
{
|
||||
"find": "__RESOURCE_GROUP__",
|
||||
"replaceWith": "$azureResourceGroup()"
|
||||
},
|
||||
{
|
||||
"find": "__BUILD_VERSION__",
|
||||
"replaceWith": "$buildVersion()"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"$schema": "https://ev2schema.azure.net/schemas/2020-01-01/serviceModel.json",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"serviceMetadata": {
|
||||
"serviceGroup": "OneBranch-PowerShellDocker",
|
||||
"environment": "Test"
|
||||
},
|
||||
"serviceResourceGroupDefinitions": [
|
||||
{
|
||||
"name": "OneBranch-PowerShellDocker-RGDef",
|
||||
"serviceResourceDefinitions": [
|
||||
{
|
||||
"name": "OneBranch-PowerShellDocker.Shell-SRDef",
|
||||
"composedOf": {
|
||||
"extension": {
|
||||
"shell": [
|
||||
{
|
||||
"type": "Run",
|
||||
"properties": {
|
||||
"imageName": "adm-mariner-20-l",
|
||||
"imageVersion": "v11"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"serviceResourceGroups": [
|
||||
{
|
||||
"azureResourceGroupName": "default",
|
||||
"location": "West US 3",
|
||||
"instanceOf": "OneBranch-PowerShellDocker-RGDef",
|
||||
"azureSubscriptionId": "default",
|
||||
"scopeTags": [
|
||||
{
|
||||
"name": "Global"
|
||||
}
|
||||
],
|
||||
"serviceResources": [
|
||||
{
|
||||
"Name": "LinuxContainerUpload",
|
||||
"InstanceOf": "OneBranch-PowerShellDocker.Shell-SRDef",
|
||||
"RolloutParametersPath": "UploadLinux.Rollout.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
<#
|
||||
This function gets info from pmc's derived list of all repositories and from mapping.json (which contains info on just the repositories powershell publishes packages to, their package formats, etc)
|
||||
to create a list of repositories PowerShell cares about along with repository Ids, repository full Urls and associated package that will be published to it.
|
||||
#>
|
||||
function Get-MappedRepositoryIds {
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[hashtable]
|
||||
$Mapping,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
$RepoList,
|
||||
|
||||
# LTS is not consider a package in this context.
|
||||
# LTS is just another package name.
|
||||
[Parameter(Mandatory)]
|
||||
[ValidateSet('stable', 'preview')]
|
||||
$Channel
|
||||
)
|
||||
|
||||
$mappedReposUsedByPwsh = @()
|
||||
foreach ($package in $Mapping.Packages)
|
||||
{
|
||||
Write-Verbose "package: $package"
|
||||
$packageChannel = $package.channel
|
||||
if (!$packageChannel) {
|
||||
$packageChannel = 'all'
|
||||
}
|
||||
|
||||
Write-Verbose "package channel: $packageChannel"
|
||||
if ($packageChannel -eq 'all' -or $packageChannel -eq $Channel)
|
||||
{
|
||||
$repoIds = [System.Collections.Generic.List[string]]::new()
|
||||
$packageFormat = $package.PackageFormat
|
||||
Write-Verbose "package format: $packageFormat" -Verbose
|
||||
$extension = [System.io.path]::GetExtension($packageFormat)
|
||||
$packageType = $extension -replace '^\.'
|
||||
|
||||
if ($package.distribution.count -gt 1) {
|
||||
throw "Package $($package | out-string) has more than one Distribution."
|
||||
}
|
||||
|
||||
foreach ($distribution in $package.distribution)
|
||||
{
|
||||
$urlGlob = $package.url
|
||||
switch ($packageType)
|
||||
{
|
||||
'deb' {
|
||||
$urlGlob = $urlGlob + '-apt'
|
||||
}
|
||||
'rpm' {
|
||||
$urlGlob = $urlGlob + '-yum'
|
||||
}
|
||||
default {
|
||||
throw "Unknown package type: $packageType"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose
|
||||
$repos = $RepoList | Where-Object { $_.name -eq $urlGlob }
|
||||
|
||||
if ($repos.id) {
|
||||
Write-Verbose "Found repo id: $($repos.id)" -Verbose
|
||||
$repoIds.AddRange(([string[]]$repos.id))
|
||||
}
|
||||
else {
|
||||
Write-Failure "Could not find repo for $urlGlob"
|
||||
}
|
||||
|
||||
if ($repoIds.Count -gt 0) {
|
||||
$mappedReposUsedByPwsh += ($package + @{ "RepoId" = $repoIds.ToArray() })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Write-Verbose -Verbose "mapped repos length: $($mappedReposUsedByPwsh.Length)"
|
||||
return $mappedReposUsedByPwsh
|
||||
}
|
||||
|
||||
<#
|
||||
This function creates package objects for the packages to be published,
|
||||
with the package name (ie package name format resolve with channel based PackageName and pwsh version), repoId, distribution and package path.
|
||||
#>
|
||||
function Get-PackageObjects() {
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[psobject[]]
|
||||
$RepoObjects,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
[string]
|
||||
$ReleaseVersion,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
[string[]]
|
||||
$PackageName
|
||||
)
|
||||
|
||||
$packages = @()
|
||||
|
||||
foreach ($pkg in $RepoObjects)
|
||||
{
|
||||
if ($pkg.RepoId.count -gt 1) {
|
||||
throw "Package $($pkg.name) has more than one repo id."
|
||||
}
|
||||
|
||||
if ($pkg.Distribution.count -gt 1) {
|
||||
throw "Package $($pkg.name) has more than one Distribution."
|
||||
}
|
||||
|
||||
$pkgRepo = $pkg.RepoId | Select-Object -First 1
|
||||
$pkgDistribution = $pkg.Distribution | Select-Object -First 1
|
||||
|
||||
foreach ($name in $PackageName) {
|
||||
$pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $ReleaseVersion)
|
||||
|
||||
if ($pkgName.EndsWith('.rpm')) {
|
||||
$pkgName = $pkgName.Replace($ReleaseVersion, $ReleaseVersion.Replace('-', '_'))
|
||||
}
|
||||
|
||||
$packagePath = "$pwshPackagesFolder/$pkgName"
|
||||
$packagePathExists = Test-Path -Path $packagePath
|
||||
if (!$packagePathExists)
|
||||
{
|
||||
throw "package path $packagePath does not exist"
|
||||
}
|
||||
|
||||
Write-Verbose "Creating package info object for package '$pkgName' for repo '$pkgRepo'"
|
||||
$packages += @{
|
||||
PackagePath = $packagePath
|
||||
PackageName = $pkgName
|
||||
RepoId = $pkgRepo
|
||||
Distribution = $pkgDistribution
|
||||
}
|
||||
|
||||
Write-Verbose -Verbose "package info obj: Name: $pkgName RepoId: $pkgRepo Distribution: $pkgDistribution PackagePath: $packagePath"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Verbose -Verbose "count of packages objects: $($packages.Length)"
|
||||
return $packages
|
||||
}
|
||||
|
||||
<#
|
||||
This function stages, uploads and publishes the powershell packages to their associated repositories in PMC.
|
||||
#>
|
||||
function Publish-PackageToPMC() {
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[pscustomobject[]]
|
||||
$PackageObject,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
[string]
|
||||
$ConfigPath,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
[bool]
|
||||
$SkipPublish
|
||||
)
|
||||
|
||||
# Don't fail outright when an error occurs, but instead pool them until
|
||||
# after attempting to publish every package. That way we can choose to
|
||||
# proceed for a partial failure.
|
||||
$errorMessage = [System.Collections.Generic.List[string]]::new()
|
||||
foreach ($finalPackage in $PackageObject)
|
||||
{
|
||||
Write-Verbose "---Staging package: $($finalPackage.PackageName)---" -Verbose
|
||||
$packagePath = $finalPackage.PackagePath
|
||||
$pkgRepo = $finalPackage.RepoId
|
||||
|
||||
$extension = [System.io.path]::GetExtension($packagePath)
|
||||
$packageType = $extension -replace '^\.'
|
||||
Write-Verbose "packageType: $packageType" -Verbose
|
||||
|
||||
$packageListJson = pmc --config $ConfigPath package $packageType list --file $packagePath
|
||||
$list = $packageListJson | ConvertFrom-Json
|
||||
|
||||
$packageId = @()
|
||||
if ($list.count -ne 0)
|
||||
{
|
||||
Write-Verbose "Package '$packagePath' already exists, skipping upload" -Verbose
|
||||
$packageId = $list.results.id | Select-Object -First 1
|
||||
}
|
||||
else {
|
||||
# PMC UPLOAD COMMAND
|
||||
Write-Verbose -Verbose "Uploading package, config: '$ConfigPath' package: '$packagePath'"
|
||||
$uploadResult = $null
|
||||
try {
|
||||
$uploadResult = pmc --config $ConfigPath package upload $packagePath --type $packageType
|
||||
}
|
||||
catch {
|
||||
$errorMessage.Add("Uploading package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.")
|
||||
continue
|
||||
}
|
||||
|
||||
$packageId = ($uploadResult | ConvertFrom-Json).id
|
||||
}
|
||||
|
||||
Write-Verbose "Got package ID: '$packageId'" -Verbose
|
||||
$distribution = $finalPackage.Distribution | select-object -First 1
|
||||
Write-Verbose "distribution: $distribution" -Verbose
|
||||
|
||||
if (!$SkipPublish)
|
||||
{
|
||||
Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose
|
||||
|
||||
if (($packageType -ne 'rpm') -and ($packageType -ne 'deb'))
|
||||
{
|
||||
throw "Unsupported package type: $packageType"
|
||||
return 1
|
||||
}
|
||||
else {
|
||||
# PMC UPDATE COMMAND
|
||||
$rawUpdateResponse = $null
|
||||
try {
|
||||
if ($packageType -eq 'rpm') {
|
||||
$rawUpdateResponse = pmc --config $ConfigPath repo package update $pkgRepo --add-packages $packageId
|
||||
} elseif ($packageType -eq 'deb') {
|
||||
$rawUpdateResponse = pmc --config $ConfigPath repo package update $pkgRepo $distribution --add-packages $packageId
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$errorMessage.Add("Invoking update for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.")
|
||||
continue
|
||||
}
|
||||
|
||||
$state = $rawUpdateResponse.state
|
||||
if ($state -ne 'Completed') {
|
||||
$errorMessage.Add("Publishing package $($finalPackage.PackageName) to $pkgRepo failed: $rawUpdateResponse")
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
# PMC PUBLISH COMMAND
|
||||
# The CLI outputs messages and JSON in the same stream, so we must sift through it for now
|
||||
# This is planned to be fixed with a switch in a later release
|
||||
Write-Verbose -Verbose ([pscustomobject]($package + @{
|
||||
PackageId = $packageId
|
||||
}))
|
||||
|
||||
# At this point, the changes are staged and will eventually be publish.
|
||||
# Running publish, causes them to go live "immediately"
|
||||
try {
|
||||
pmc --config $ConfigPath repo publish $pkgRepo
|
||||
}
|
||||
catch {
|
||||
$errorMessage.Add("Running final publish for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.")
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
Write-Verbose -Verbose "Skipping Uploading package --config-file '$ConfigPath' package add '$packagePath' --repoID '$pkgRepo'"
|
||||
}
|
||||
}
|
||||
|
||||
if ($errorMessage) {
|
||||
throw $errorMessage -join [Environment]::NewLine
|
||||
}
|
||||
}
|
||||
|
||||
if ($null -eq $env:MAPPING_FILE)
|
||||
{
|
||||
Write-Verbose -Verbose "MAPPING_FILE variable didn't get passed correctly"
|
||||
return 1
|
||||
}
|
||||
|
||||
if ($null -eq $env:PWSH_PACKAGES_TARGZIP)
|
||||
{
|
||||
Write-Verbose -Verbose "PWSH_PACKAGES_TARGZIP variable didn't get passed correctly"
|
||||
return 1
|
||||
}
|
||||
|
||||
if ($null -eq $env:PMC_METADATA)
|
||||
{
|
||||
Write-Verbose -Verbose "PMC_METADATA variable didn't get passed correctly"
|
||||
return 1
|
||||
}
|
||||
|
||||
try {
|
||||
Write-Verbose -Verbose "Downloading files"
|
||||
Invoke-WebRequest -Uri $env:MAPPING_FILE -OutFile mapping.json
|
||||
Invoke-WebRequest -Uri $env:PWSH_PACKAGES_TARGZIP -OutFile packages.tar.gz
|
||||
Invoke-WebRequest -Uri $env:PMC_METADATA -OutFile pmcMetadata.json
|
||||
|
||||
# create variables to those paths and test them
|
||||
$mappingFilePath = Join-Path "/package/unarchive/" -ChildPath "mapping.json"
|
||||
$mappingFilePathExists = Test-Path $mappingFilePath
|
||||
if (!$mappingFilePathExists)
|
||||
{
|
||||
Write-Verbose -Verbose "mapping.json expected at $mappingFilePath does not exist"
|
||||
return 1
|
||||
}
|
||||
|
||||
$packagesTarPath = Join-Path -Path "/package/unarchive/" -ChildPath "packages.tar.gz"
|
||||
$packagesTarPathExists = Test-Path $packagesTarPath
|
||||
if (!$packagesTarPathExists)
|
||||
{
|
||||
Write-Verbose -Verbose "packages.tar.gz expected at $packagesTarPath does not exist"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Extract files from 'packages.tar.gz'
|
||||
Write-Verbose -Verbose "---Extracting files from packages.tar.gz---"
|
||||
$pwshPackagesFolder = Join-Path -Path "/package/unarchive/" -ChildPath "packages"
|
||||
New-Item -Path $pwshPackagesFolder -ItemType Directory
|
||||
tar -xzvf $packagesTarPath -C $pwshPackagesFolder --force-local
|
||||
Get-ChildItem $pwshPackagesFolder -Recurse
|
||||
|
||||
$metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json"
|
||||
$metadataFilePathExists = Test-Path $metadataFilePath
|
||||
if (!$metadataFilePathExists)
|
||||
{
|
||||
Write-Verbose -Verbose "pmcMetadata.json expected at $metadataFilePath does not exist"
|
||||
return 1
|
||||
}
|
||||
|
||||
# files in the extracted Run dir
|
||||
$configPath = Join-Path '/package/unarchive/Run' -ChildPath 'settings.toml'
|
||||
$configPathExists = Test-Path -Path $configPath
|
||||
if (!$configPathExists)
|
||||
{
|
||||
Write-Verbose -Verbose "settings.toml expected at $configPath does not exist"
|
||||
return 1
|
||||
}
|
||||
|
||||
$pythonDlFolder = Join-Path '/package/unarchive/Run' -ChildPath 'python_dl'
|
||||
$pyPathExists = Test-Path -Path $pythonDlFolder
|
||||
if (!$pyPathExists)
|
||||
{
|
||||
Write-Verbose -Verbose "python_dl expected at $pythonDlFolder does not exist"
|
||||
return 1
|
||||
}
|
||||
|
||||
Write-Verbose -Verbose "Installing pmc-cli"
|
||||
pip install --upgrade pip
|
||||
pip --version --verbose
|
||||
pip install /package/unarchive/Run/python_dl/*.whl
|
||||
|
||||
# Get metadata
|
||||
$channel = ""
|
||||
$packageNames = @()
|
||||
$metadataContent = Get-Content -Path $metadataFilePath | ConvertFrom-Json
|
||||
$releaseVersion = $metadataContent.ReleaseTag.TrimStart('v')
|
||||
$skipPublish = $metadataContent.SkipPublish
|
||||
$lts = $metadataContent.LTS
|
||||
|
||||
if ($releaseVersion.Contains('-')) {
|
||||
$channel = 'preview'
|
||||
$packageNames = @('powershell-preview')
|
||||
}
|
||||
else {
|
||||
$channel = 'stable'
|
||||
$packageNames = @('powershell')
|
||||
}
|
||||
|
||||
if ($lts) {
|
||||
$packageNames += @('powershell-lts')
|
||||
}
|
||||
|
||||
Write-Verbose -Verbose "---Getting repository list---"
|
||||
$rawResponse = pmc --config $configPath repo list --limit 800
|
||||
$response = $rawResponse | ConvertFrom-Json
|
||||
$limit = $($response.limit)
|
||||
$count = $($response.count)
|
||||
Write-Verbose -Verbose "'pmc repo list' limit is: $limit and count is: $count"
|
||||
$repoList = $response.results
|
||||
|
||||
Write-Verbose -Verbose "---Getting package info---"
|
||||
|
||||
|
||||
Write-Verbose "Reading mapping file from '$mappingFilePath'" -Verbose
|
||||
$mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable
|
||||
$mappedReposUsedByPwsh = Get-MappedRepositoryIds -Mapping $mapping -RepoList $repoList -Channel $channel
|
||||
$packageObjects = Get-PackageObjects -RepoObjects $mappedReposUsedByPwsh -PackageName $packageNames -ReleaseVersion $releaseVersion
|
||||
Write-Verbose -Verbose "skip publish $skipPublish"
|
||||
Publish-PackageToPMC -PackageObject $packageObjects -ConfigPath $configPath -SkipPublish $skipPublish
|
||||
}
|
||||
catch {
|
||||
Write-Error -ErrorAction Stop $_.Exception.Message
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutParameters.json",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"shellExtensions": [
|
||||
{
|
||||
"name": "Run",
|
||||
"type": "Run",
|
||||
"properties": {
|
||||
"maxExecutionTime": "PT2H"
|
||||
},
|
||||
"package": {
|
||||
"reference": {
|
||||
"path": "Shell/Run.tar"
|
||||
}
|
||||
},
|
||||
"launch": {
|
||||
"command": [
|
||||
"/bin/bash",
|
||||
"-c",
|
||||
"pwsh ./Run/Run.ps1"
|
||||
],
|
||||
"environmentVariables": [
|
||||
{
|
||||
"name": "MAPPING_FILE",
|
||||
"reference":
|
||||
{
|
||||
"path": "Parameters\\mapping.json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "PWSH_PACKAGES_TARGZIP",
|
||||
"reference":
|
||||
{
|
||||
"path": "Parameters\\packages.tar.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "PMC_METADATA",
|
||||
"reference":
|
||||
{
|
||||
"path": "Parameters\\pmcMetadata.json"
|
||||
}
|
||||
}
|
||||
],
|
||||
"identity": {
|
||||
"type": "userAssigned",
|
||||
"userAssignedIdentities": [
|
||||
"default"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
1.0.1
|
||||
@@ -0,0 +1,101 @@
|
||||
trigger: none
|
||||
|
||||
parameters: # parameters are shown up in ADO UI in a build queue time
|
||||
- name: 'debug'
|
||||
displayName: 'Enable debug output'
|
||||
type: boolean
|
||||
default: false
|
||||
- name: skipPublish
|
||||
displayName: Skip PMC Publish
|
||||
type: boolean
|
||||
default: false
|
||||
- name: SKIP_SIGNING
|
||||
displayName: Skip Signing
|
||||
type: string
|
||||
default: 'NO'
|
||||
|
||||
variables:
|
||||
- name: CDP_DEFINITION_BUILD_COUNT
|
||||
value: $[counter('', 0)]
|
||||
- name: system.debug
|
||||
value: ${{ parameters.debug }}
|
||||
- name: ENABLE_PRS_DELAYSIGN
|
||||
value: 1
|
||||
- name: ROOT
|
||||
value: $(Build.SourcesDirectory)
|
||||
- name: REPOROOT
|
||||
value: $(Build.SourcesDirectory)
|
||||
- name: OUTPUTROOT
|
||||
value: $(REPOROOT)\out
|
||||
- name: NUGET_XMLDOC_MODE
|
||||
value: none
|
||||
- name: nugetMultiFeedWarnLevel
|
||||
value: none
|
||||
- name: NugetSecurityAnalysisWarningLevel
|
||||
value: none
|
||||
- name: skipNugetSecurityAnalysis
|
||||
value: true
|
||||
- name: ob_outputDirectory
|
||||
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
|
||||
- name: ob_sdl_tsa_configFile
|
||||
value: $(Build.SourcesDirectory)\.config\tsaoptions.json
|
||||
- name: WindowsContainerImage
|
||||
value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'
|
||||
- name: LinuxContainerImage
|
||||
value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
|
||||
- group: PoolNames
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: templates
|
||||
type: git
|
||||
name: OneBranch.Pipelines/GovernedTemplates
|
||||
ref: refs/heads/main
|
||||
|
||||
pipelines:
|
||||
- pipeline: CoOrdinatedBuildPipeline
|
||||
source: 'PowerShell-Coordinated Packages-Official'
|
||||
|
||||
- pipeline: PSPackagesOfficial
|
||||
source: 'PowerShell-Packages-Official'
|
||||
trigger:
|
||||
branches:
|
||||
include:
|
||||
- master
|
||||
- releases/*
|
||||
|
||||
extends:
|
||||
template: v2/OneBranch.Official.CrossPlat.yml@templates
|
||||
parameters:
|
||||
featureFlags:
|
||||
WindowsHostVersion:
|
||||
Version: 2022
|
||||
Network: Netlock
|
||||
linuxEsrpSigning: true
|
||||
cloudvault:
|
||||
enabled: false
|
||||
globalSdl:
|
||||
disableLegacyManifest: true
|
||||
# disabled Armory as we dont have any ARM templates to scan. It fails on some sample ARM templates.
|
||||
armory:
|
||||
enabled: false
|
||||
asyncSdl:
|
||||
enabled: true
|
||||
tsaOptionsFile: .config/tsaoptions.json
|
||||
tsa:
|
||||
enabled: true
|
||||
credscan:
|
||||
enabled: true
|
||||
scanFolder: $(Build.SourcesDirectory)
|
||||
suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json
|
||||
binskim:
|
||||
break: false # always break the build on binskim issues in addition to TSA upload
|
||||
policheck:
|
||||
break: true # always break the build on policheck issues. You can disable it by setting to 'false'
|
||||
tsaOptionsFile: .config\tsaoptions.json
|
||||
stages:
|
||||
- template: /.pipelines/templates/release-prep-for-ev2.yml@self
|
||||
parameters:
|
||||
skipPublish: ${{ parameters.skipPublish }}
|
||||
|
||||
- template: /.pipelines/templates/release-publish-pmc.yml@self
|
||||
@@ -1,3 +1,7 @@
|
||||
parameters:
|
||||
- name: restorePhase
|
||||
default: false
|
||||
|
||||
steps:
|
||||
- pwsh: |
|
||||
$variable = 'releaseTag'
|
||||
@@ -13,6 +17,8 @@ steps:
|
||||
Write-Host -Object "##$vstsCommandString"
|
||||
name: OutputReleaseTag
|
||||
displayName: Set Release Tag
|
||||
env:
|
||||
ob_restore_phase: ${{ parameters.restorePhase }}
|
||||
|
||||
- pwsh: |
|
||||
$azureVersion = '$(OutputReleaseTag.ReleaseTag)'.ToLowerInvariant() -replace '\.', '-'
|
||||
@@ -26,3 +32,5 @@ steps:
|
||||
Write-Host "##$vstsCommandString"
|
||||
name: OutputVersion
|
||||
displayName: Set container name
|
||||
env:
|
||||
ob_restore_phase: ${{ parameters.restorePhase }}
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
parameters:
|
||||
- name: skipPublish
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
stages:
|
||||
- stage: PrepForEV2
|
||||
displayName: 'Copy and prep all files needed for EV2 stage'
|
||||
jobs:
|
||||
- job: CopyEV2FilesToArtifact
|
||||
displayName: 'Copy EV2 Files to Artifact'
|
||||
pool:
|
||||
type: linux
|
||||
variables:
|
||||
- name: ob_outputDirectory
|
||||
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
|
||||
- name: repoRoot
|
||||
value: '$(Build.SourcesDirectory)/PowerShell'
|
||||
- name: ev2ServiceGroupRootFolder
|
||||
value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot'
|
||||
- name: ev2ParametersFolder
|
||||
value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters'
|
||||
- group: 'mscodehub-code-read-akv'
|
||||
- group: 'packages.microsoft.com'
|
||||
- name: ob_sdl_credscan_suppressionsFile
|
||||
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
|
||||
steps:
|
||||
- checkout: self ## the global setting on lfs didn't work
|
||||
lfs: false
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- template: release-SetReleaseTagandContainerName.yml
|
||||
parameters:
|
||||
restorePhase: true
|
||||
|
||||
- pwsh: |
|
||||
$packageVersion = '$(OutputReleaseTag.ReleaseTag)'.ToLowerInvariant() -replace '^v',''
|
||||
$vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion"
|
||||
Write-Host "sending " + $vstsCommandString
|
||||
Write-Host "##$vstsCommandString"
|
||||
displayName: Set Package version
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
$branch = 'mirror-target'
|
||||
$gitArgs = "clone",
|
||||
"--verbose",
|
||||
"--branch",
|
||||
"$branch",
|
||||
"https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools",
|
||||
'$(Pipeline.Workspace)/tools'
|
||||
$gitArgs | Write-Verbose -Verbose
|
||||
git $gitArgs
|
||||
displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose
|
||||
displayName: 'Capture Environment Variables'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
Get-ChildItem '$(Build.SourcesDirectory)'
|
||||
displayName: 'Capture BuildDirectory'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose
|
||||
displayName: 'Capture Workspace'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory
|
||||
displayName: 'Create Parameters folder under EV2Specs folder'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- task: PipAuthenticate@1
|
||||
inputs:
|
||||
artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages'
|
||||
displayName: 'Pip Authenticate'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
python3 -m pip install --upgrade pip
|
||||
pip --version --verbose
|
||||
|
||||
Write-Verbose -Verbose "Download pmc-cli to folder without installing it"
|
||||
$pythonDlFolderPath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath "python_dl"
|
||||
pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux_2_17_x86_64 --only-binary=:all: --verbose
|
||||
displayName: 'Download pmc-cli package'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- download: PSPackagesOfficial
|
||||
artifact: 'drop_linux_package_deb'
|
||||
displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- download: PSPackagesOfficial
|
||||
artifact: 'drop_linux_package_rpm'
|
||||
displayName: 'Download artifact containing .rh.x64_86.rpm file from PSPackagesOfficial triggering pipeline'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- download: PSPackagesOfficial
|
||||
artifact: 'drop_linux_package_mariner_x64'
|
||||
displayName: 'Download artifact containing .cm.x86_64.rpm file from PSPackagesOfficial triggering pipeline'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- download: PSPackagesOfficial
|
||||
artifact: 'drop_linux_package_mariner_arm64'
|
||||
displayName: 'Download artifact containing .cm.aarch64.rpm file from PSPackagesOfficial triggering pipeline'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages"
|
||||
$downloadedPipelineFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'PSPackagesOfficial'
|
||||
$srcFilesFolder = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'SourceFiles'
|
||||
New-Item -Path $srcFilesFolder -ItemType Directory
|
||||
$packagesFolder = Join-Path -Path $srcFilesFolder -ChildPath 'packages'
|
||||
New-Item -Path $packagesFolder -ItemType Directory
|
||||
|
||||
$packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm
|
||||
foreach ($file in $packageFiles)
|
||||
{
|
||||
Write-Verbose -Verbose "copying file: $($file.FullName)"
|
||||
Copy-Item -Path $($file.FullName) -Destination $packagesFolder -Verbose
|
||||
}
|
||||
|
||||
$packagesTarGzDestination = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'packages.tar.gz'
|
||||
tar -czvf $packagesTarGzDestination -C $packagesFolder .
|
||||
displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
$pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json'
|
||||
|
||||
$metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json
|
||||
$metadataHash = @{}
|
||||
$skipPublishValue = '${{ parameters.skipPublish }}'
|
||||
$metadataHash["ReleaseTag"] = '$(OutputReleaseTag.ReleaseTag)'
|
||||
$metadataHash["LTS"] = $metadata.LTSRelease.Latest
|
||||
$metadataHash["ForProduction"] = $true
|
||||
$metadataHash["SkipPublish"] = [System.Convert]::ToBoolean($skipPublishValue)
|
||||
|
||||
$metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile
|
||||
|
||||
$mappingFilePath = Join-Path -Path '$(repoRoot)/tools/packages.microsoft.com' -ChildPath 'mapping.json'
|
||||
$mappingFilePathExists = Test-Path $mappingFilePath
|
||||
$mappingFileEV2Path = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath "mapping.json"
|
||||
Write-Verbose -Verbose "Copy mapping.json file at: $mappingFilePath which exists: $mappingFilePathExists to: $mappingFileEV2Path"
|
||||
Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path
|
||||
displayName: 'Create pmcScriptMetadata.json and mapping.json file'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
$pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json'
|
||||
$content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json
|
||||
$content.RolloutMetadata.Notification.Email.To = '$(PmcEV2SupportEmail)'
|
||||
Remove-Item -Path $pathToJsonFile
|
||||
$content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile
|
||||
displayName: 'Replace values in RolloutSpecPath.json'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
$pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json'
|
||||
$content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json
|
||||
|
||||
$identityString = "/subscriptions/$(PmcSubscription)/resourcegroups/$(PmcResourceGroup)/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(PmcMIName)"
|
||||
$content.shellExtensions.launch.identity.userAssignedIdentities[0] = $identityString
|
||||
|
||||
Remove-Item -Path $pathToJsonFile
|
||||
$content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile
|
||||
displayName: 'Replace values in UploadLinux.Rollout.json file'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
$pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json'
|
||||
$content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json
|
||||
$content.ServiceResourceGroups[0].AzureResourceGroupName = '$(PmcResourceGroup)'
|
||||
$content.ServiceResourceGroups[0].AzureSubscriptionId = '$(PmcSubscription)'
|
||||
|
||||
Remove-Item -Path $pathToJsonFile
|
||||
$content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile
|
||||
displayName: 'Replace values in ServiceModel.json'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- pwsh: |
|
||||
$settingFilePath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath 'settings.toml'
|
||||
New-Item -Path $settingFilePath -ItemType File
|
||||
$pmcMIClientID = '$(PmcMIClientID)'
|
||||
$pmcEndpoint = '$(PmcEndpointUrl)'
|
||||
|
||||
Add-Content -Path $settingFilePath -Value "[default]"
|
||||
Add-Content -Path $settingFilePath -Value "base_url = `"$pmcEndpoint`""
|
||||
Add-Content -Path $settingFilePath -Value "auth_type = `"msi`""
|
||||
Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`""
|
||||
displayName: 'Create settings.toml file with MI clientId populated'
|
||||
env:
|
||||
ob_restore_phase: true
|
||||
|
||||
- task: onebranch.pipeline.signing@1
|
||||
inputs:
|
||||
command: 'sign'
|
||||
signing_profile: external_distribution
|
||||
files_to_sign: '*.ps1'
|
||||
search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run'
|
||||
displayName: Sign Run.ps1
|
||||
|
||||
- pwsh: |
|
||||
# folder to tar must have: Run.ps1, settings.toml, python_dl
|
||||
$srcPath = Join-Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'Shell'
|
||||
$pathToRunTarFile = Join-Path $srcPath -ChildPath "Run.tar"
|
||||
tar -cvf $pathToRunTarFile -C $srcPath ./Run
|
||||
displayName: 'Create archive for the shell extension'
|
||||
|
||||
- task: CopyFiles@2
|
||||
inputs:
|
||||
SourceFolder: '$(repoRoot)/.pipelines'
|
||||
Contents: 'EV2Specs/**'
|
||||
TargetFolder: $(ob_outputDirectory)
|
||||
@@ -1,90 +1,37 @@
|
||||
parameters:
|
||||
- name: skipPublish
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
- job: PMCPublish
|
||||
displayName: Publish to PMC
|
||||
condition: succeeded()
|
||||
pool:
|
||||
type: linux
|
||||
isCustom: true
|
||||
name: PowerShell1ES
|
||||
demands:
|
||||
- ImageOverride -equals PSMMSUbuntu20.04-Secure
|
||||
stages:
|
||||
- stage: 'Prod_Release'
|
||||
displayName: 'Deploy packages to PMC with EV2'
|
||||
dependsOn:
|
||||
- PrepForEV2
|
||||
variables:
|
||||
- name: runCodesignValidationInjection
|
||||
value: false
|
||||
- name: NugetSecurityAnalysisWarningLevel
|
||||
value: none
|
||||
- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
|
||||
value: 1
|
||||
- group: 'mscodehub-code-read-akv'
|
||||
- group: 'packages.microsoft.com'
|
||||
- name: ob_outputDirectory
|
||||
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
|
||||
- name: ob_sdl_codeSignValidation_enabled
|
||||
value: false
|
||||
- name: ob_sdl_binskim_enabled
|
||||
value: false
|
||||
- name: ob_sdl_tsa_configFile
|
||||
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
|
||||
- name: ob_sdl_credscan_suppressionsFile
|
||||
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
|
||||
- name: ob_release_environment
|
||||
value: "Production"
|
||||
- name: repoRoot
|
||||
value: $(Build.SourcesDirectory)
|
||||
jobs:
|
||||
- job: Prod_ReleaseJob
|
||||
displayName: Publish to PMC
|
||||
pool:
|
||||
type: release
|
||||
|
||||
steps:
|
||||
- checkout: self ## the global setting on lfs didn't work
|
||||
lfs: false
|
||||
steps:
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
targetPath: '$(Pipeline.Workspace)'
|
||||
artifact: drop_PrepForEV2_CopyEv2FilesToArtifact
|
||||
displayName: 'Download drop_PrepForEV2_CopyEv2FilesToArtifact artifact that has all files needed'
|
||||
|
||||
- template: release-SetReleaseTagAndContainerName.yml
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
targetPath: '$(Pipeline.Workspace)'
|
||||
displayName: 'Download to get EV2 Files'
|
||||
|
||||
- pwsh: |
|
||||
$packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v',''
|
||||
$vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion"
|
||||
Write-Host "sending " + $vstsCommandString
|
||||
Write-Host "##$vstsCommandString"
|
||||
displayName: Set Package version
|
||||
|
||||
- pwsh: |
|
||||
$branch = 'mirror-target'
|
||||
$gitArgs = "clone",
|
||||
"--verbose",
|
||||
"--branch",
|
||||
"$branch",
|
||||
"https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools",
|
||||
'$(Pipeline.Workspace)/tools'
|
||||
$gitArgs | Write-Verbose -Verbose
|
||||
git $gitArgs
|
||||
displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub
|
||||
|
||||
- task: PipAuthenticate@1
|
||||
inputs:
|
||||
artifactFeeds: 'pmc'
|
||||
pythonDownloadServiceConnections: pmcDownload
|
||||
|
||||
- pwsh: |
|
||||
pip install pmc-cli==1.12.0
|
||||
|
||||
$newPath = (resolve-path '~/.local/bin').providerpath
|
||||
$vstsCommandString = "vso[task.setvariable variable=PATH]${env:PATH}:$newPath"
|
||||
Write-Host "sending " + $vstsCommandString
|
||||
Write-Host "##$vstsCommandString"
|
||||
displayName: Install pmc cli
|
||||
|
||||
- pwsh: |
|
||||
$metadata = Get-Content -Path "$(Build.SourcesDirectory)/tools/metadata.json" -Raw | ConvertFrom-Json
|
||||
$params = @{
|
||||
ReleaseTag = "$(ReleaseTag)"
|
||||
AadClientId = "$(PmcCliClientID)"
|
||||
BlobFolderName = "$(ReleaseTag)"
|
||||
LTS = $metadata.LTSRelease.Latest
|
||||
ForProduction = $true
|
||||
SkipPublish = $${{ parameters.skipPublish }}
|
||||
MappingFilePath = '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com/mapping.json'
|
||||
}
|
||||
|
||||
$params | Out-String -width 9999 -Stream | write-Verbose -Verbose
|
||||
|
||||
& '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' @params
|
||||
displayName: Run release script
|
||||
- task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1
|
||||
displayName: 'Ev2: Push to PMC'
|
||||
inputs:
|
||||
UseServerMonitorTask: true
|
||||
EndpointProviderType: ApprovalService
|
||||
ApprovalServiceEnvironment: Production
|
||||
ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot'
|
||||
RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json'
|
||||
|
||||
Reference in New Issue
Block a user