diff --git a/.vscode/extensions.json b/.vscode/extensions.json index b0c23e31c2..310b1db2ec 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -6,6 +6,7 @@ "ms-vscode.csharp", "ms-vscode.PowerShell", "twxs.cmake", - "DavidAnson.vscode-markdownlint" + "DavidAnson.vscode-markdownlint", + "ms-azure-devops.azure-pipelines" ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 792e547d8a..bc8c2b8e61 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -22,16 +22,16 @@ // Sets the codeformatting options to follow the given indent style in a way that is compatible with PowerShell syntax. For more information about the brace styles please refer to https://github.com/PoshCode/PowerShellPracticeAndStyle/issues/81. "powershell.codeFormatting.preset": "OTBS", - + // Adds a space between a keyword and its associated scriptblock expression. "powershell.codeFormatting.whitespaceBeforeOpenBrace": true, - + // Adds a space between a keyword (if, elseif, while, switch, etc) and its associated conditional expression. "powershell.codeFormatting.whitespaceBeforeOpenParen": true, - + // Adds spaces before and after an operator ('=', '+', '-', etc.). "powershell.codeFormatting.whitespaceAroundOperator": true, - + // Adds a space after a separator (',' and ';'). "powershell.codeFormatting.whitespaceAfterSeparator": true } diff --git a/tools/releaseBuild/azureDevOps/diagram.png b/tools/releaseBuild/azureDevOps/diagram.png new file mode 100644 index 0000000000..0c53297198 Binary files /dev/null and b/tools/releaseBuild/azureDevOps/diagram.png differ diff --git a/tools/releaseBuild/azureDevOps/diagram.puml b/tools/releaseBuild/azureDevOps/diagram.puml new file mode 100644 index 0000000000..f2593038f0 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/diagram.puml @@ -0,0 +1,61 @@ +@startuml + +' Define the build tasks as business processes +archimate #Business "Build DEB" as BuildDEB <> +archimate #Business "Build RPM" as BuildRPM <> +archimate #Business "Build Alpine" as BuildAlpine <> +archimate #Business "Build Linux-FxDependent" as BuildLinuxFx <> +archimate #Business "Upload DEB" as UploadDEB <> +archimate #Business "Upload RPM" as UploadRPM <> +archimate #Business "Upload Alpine" as UploadAlpine <> +archimate #Business "Upload Linux-FxDependent" as UploadLinuxFx <> + +archimate #Business "Build macOS" as BuildMac <> +archimate #Business "macOS Package Signing and Upload" as SignMac <> + +archimate #Business "Build Windows - x64" as BuildWinX64 <> +archimate #Business "Build Windows - x86" as BuildWinX86 <> +archimate #Business "Build Windows - arm32" as BuildWinArm32 <> +archimate #Business "Build Windows - arm64" as BuildWinArm64 <> +archimate #Business "Build Windows - FxDependent" as BuildWinFx <> +archimate #Business "Windows Package Signing and Upload" as SignWin <> + +archimate #Business "ComponentRegistration" as BuildCG <> + +archimate #Business "Compliance" as Compliance <> + +' Define finishing the build as a goal filled +archimate #Goal "Finish" as Finish <> + +' map the various Upload task dependencies +BuildDEB -down-> UploadDEB +BuildRPM -down-> UploadRPM +BuildLinuxFx -down-> UploadLinuxFx +BuildAlpine -down-> UploadAlpine + +' map all of the SignMac task dependencies +BuildMac -down-> SignMac + +' map all of the SignWin task dependencies +BuildWinX64 -down-> SignWin +BuildWinX86 -down-> SignWin +BuildWinArm32 -down-> SignWin +BuildWinArm64 -down-> SignWin +BuildWinFx -down-> SignWin + +' map all of the Compliance task dependencies +BuildWinX86 -down-> Compliance +BuildWinX64 -down-> Compliance +BuildWinFx -down-> Compliance + +' map all leafs to finish +Compliance -down-> Finish +UploadAlpine -down-> Finish +UploadDEB -down-> Finish +UploadRPM -down-> Finish +UploadLinuxFx -down-> Finish +SignWin -down-> Finish +SignMac -down-> Finish +BuildCG -down-> Finish + +@enduml diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 9eef2a8ce1..0a3a627f78 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -85,3 +85,9 @@ jobs: - template: templates/mac-package-signing.yml +- template: templates/compliance.yml + parameters: + parentJobs: + - build_windows_x64 + - build_windows_x86 + - build_windows_fxdependent diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml new file mode 100644 index 0000000000..cc31dddcc9 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -0,0 +1,142 @@ +parameters: + parentJobs: [] + +jobs: +- job: compliance + displayName: Compliance + dependsOn: + ${{ parameters.parentJobs }} + pool: + name: Package ES CodeHub Lab E + + steps: + - powershell: | + tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTagVar) -Variable "ReleaseTagVar" + displayName: 'Set ReleaseTag Variable' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifacts' + inputs: + downloadType: specific + itemPattern: | + **/*.zip + + - powershell: | + dir "$(System.ArtifactsDirectory)\*" -Recurse + displayName: 'Capture artifacts directory' + continueOnError: true + + - powershell: | + $version = '$(ReleaseTagVar)'.Substring(1) + $vstsCommandString = "vso[task.setvariable variable=Version]$version" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + $azureVersion = '$(ReleaseTagVar)'.ToLowerInvariant() -replace '\.', '-' + $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + displayName: 'Set Version Variable' + + - template: expand-compliance.yml + parameters: + architecture: x86 + + - template: expand-compliance.yml + parameters: + architecture: x64 + + - template: expand-compliance.yml + parameters: + architecture: fxdependent + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 + displayName: 'Run MpCmdRun.exe' + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 + displayName: 'Run CredScan' + inputs: + suppressionsFile: tools/credScan/suppress.json + debugMode: false + continueOnError: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 + displayName: 'Run PoliCheck' + inputs: + targetType: F + optionsFC: 0 + optionsXS: 0 + optionsPE: '1|2|3|4' + optionsHMENABLE: 0 + optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb' + optionsFTPATH: '$(Build.SourcesDirectory)\tools\terms\FileTypeSet.xml' + continueOnError: true + + # add RoslynAnalyzers + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-autoapplicability.AutoApplicability@1 + displayName: 'Run AutoApplicability' + inputs: + ExternalRelease: true + IsSoftware: true + DataSensitivity: lbi + continueOnError: true + + # add codeMetrics + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-vulnerabilityassessment.VulnerabilityAssessment@0 + displayName: 'Run Vulnerability Assessment' + continueOnError: true + + # FXCop is not applicable + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 + displayName: 'Publish Security Analysis Logs to Build Artifacts' + continueOnError: true + + # PreFASt is not applicable + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 + displayName: 'Run PoliCheck' + inputs: + targetType: F + optionsFC: 0 + optionsXS: 0 + optionsPE: '1|2|3|4' + optionsHMENABLE: 0 + optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb' + optionsFTPATH: '$(Build.SourcesDirectory)\tools\terms\FileTypeSet.xml' + continueOnError: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-apiscan.APIScan@1 + displayName: 'Run APIScan' + inputs: + softwareFolder: '$(CompliancePath)' + softwareName: PowerShell + softwareVersionNum: '$(ReleaseTagVar)' + isLargeApp: false + preserveTempFiles: true + continueOnError: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1 + displayName: 'TSA upload to Codebase: PowerShellCore_201807 Stamp: Azure' + inputs: + tsaStamp: $(TsaStamp) + codeBaseName: $(CodeBaseName) + uploadFortifySCA: false + uploadFxCop: false + uploadModernCop: false + uploadPREfast: false + uploadRoslyn: false + uploadTSLint: false + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1 + displayName: 'Create Security Analysis Report' + inputs: + TsvFile: false + APIScan: true + BinSkim: true + CredScan: true + PoliCheck: true + PoliCheckBreakOn: Severity2Above diff --git a/tools/releaseBuild/azureDevOps/templates/expand-compliance.yml b/tools/releaseBuild/azureDevOps/templates/expand-compliance.yml new file mode 100644 index 0000000000..b925ffbca9 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/expand-compliance.yml @@ -0,0 +1,11 @@ +parameters: + architecture: x86 + +steps: + - powershell: | + Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-${{ parameters.architecture }}.zip" -Destination "$(Build.StagingDirectory)\symbols\${{ parameters.architecture }}" + displayName: Expand symbols zip - ${{ parameters.architecture }} + + - powershell: | + tools/releaseBuild/createComplianceFolder.ps1 -ArtifactFolder "$(Build.StagingDirectory)\symbols\${{ parameters.architecture }}" -VSTSVariableName 'CompliancePath' + displayName: Expand Compliance file - ${{ parameters.architecture }} diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index ba2cd4bdbb..6f74033119 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -5,7 +5,7 @@ jobs: - job: ${{ parameters.jobName }} displayName: Build macOS condition: succeeded() - pool: Hosted macOS Preview + pool: Hosted Mac Internal steps: #- task: @ # inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/upload.yml b/tools/releaseBuild/azureDevOps/templates/upload.yml new file mode 100644 index 0000000000..092cfbf292 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/upload.yml @@ -0,0 +1,40 @@ +parameters: + architecture: x86 + msi: yes + +steps: +- powershell: | + $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-${{ parameters.architecture }}.msi" + if(Test-Path -Path $packagePath) + { + Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" + } + displayName: '[create script] upload signed msi - ${{ parameters.architecture }}' + condition: and(succeeded(), eq(variables['msi'], 'yes')) + +- task: AzureFileCopy@1 + displayName: 'upload signed msi to Azure - ${{ parameters.architecture }}' + inputs: + SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-${{ parameters.architecture }}.msi' + azureSubscription: '$(AzureFileCopySubscription)' + Destination: AzureBlob + storage: '$(StorageAccount)' + ContainerName: '$(AzureVersion)' + condition: and(succeeded(), eq(variables['msi'], 'yes')) + +- powershell: | + $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-${{ parameters.architecture }}.zip" + if(Test-Path -Path $packagePath) + { + Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" + } + displayName: '[create script] upload signed zip - ${{ parameters.architecture }}' + +- task: AzureFileCopy@1 + displayName: 'upload signed zip to Azure - ${{ parameters.architecture }}' + inputs: + SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-${{ parameters.architecture }}.zip' + azureSubscription: '$(AzureFileCopySubscription)' + Destination: AzureBlob + storage: '$(StorageAccount)' + ContainerName: '$(AzureVersion)' diff --git a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml index cdbc62471d..244f554dad 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml @@ -56,6 +56,16 @@ jobs: # Diagnostics is not critical it passes every time it runs continueOnError: true + - powershell: | + Import-Module $(Build.SourcesDirectory)/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $(Build.SourcesDirectory)/src/Modules + + if(-not (Test-Path "$(Build.SourcesDirectory)/src/Modules/nuget.config")) + { + throw "nuget.config is not created" + } + displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' + - powershell: | ./tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-x64-component-registration displayName: 'Build Windows Universal - Component Registration' diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index 07fa67cd40..e3809a4e35 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -75,132 +75,28 @@ jobs: outPathRoot: '$(Build.StagingDirectory)\signedPackages' condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - powershell: | - $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[Create script] upload signed msi - x64' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) + - template: upload.yml + parameters: + architecture: x86 - - task: AzureFileCopy@1 - displayName: 'upload signed msi to Azure - x64' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) + - template: upload.yml + parameters: + architecture: x64 - - powershell: | - $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed msi - x86' + - template: upload.yml + parameters: + architecture: arm32 + msi: no - - task: AzureFileCopy@1 - displayName: 'upload signed msi to Azure - x86' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) + - template: upload.yml + parameters: + architecture: arm64 + msi: no - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[Create script] upload signed zip - x64' - - - task: AzureFileCopy@1 - displayName: 'upload signed zip to Azure - x64' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - x86' - - - task: AzureFileCopy@1 - displayName: 'upload signed zip to Azure - x86' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - arm' - - - task: AzureFileCopy@1 - displayName: 'upload signed zip to Azure - arm' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - arm64' - - - task: AzureFileCopy@1 - displayName: 'upload signed zip to Azure - arm64' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - fxdependent' - - - task: AzureFileCopy@1 - displayName: 'upload signed zip to Azure - fxdependent' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) + - template: upload.yml + parameters: + architecture: fxdependent + msi: no - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 displayName: 'Run MpCmdRun.exe' diff --git a/tools/releaseBuild/macOS/vsts.yml b/tools/releaseBuild/macOS/vsts.yml index 57cdb50738..b00fc9ae0b 100644 --- a/tools/releaseBuild/macOS/vsts.yml +++ b/tools/releaseBuild/macOS/vsts.yml @@ -2,7 +2,7 @@ resources: - repo: self clean: true queue: - name: Hosted macOS Preview + name: Hosted Mac Internal name: macOS_$(Build.SourceBranchName)_$(Build.BuildId) steps: #- task: @