[release/v7.3.0-rc.1] Move ApiScan to compliance build (#18278)

* Add build and APIScan to compliance build

* dynamically set version

set releaseTagVar default

* publish logs

* set to standard verbosity

only update status every 5 minutes

* add guardian export

Publish Guardian reports

publish whole artifact folder

update publish

switch to guardian publish task

* Download symbols

* Create configuration that doesn't make it ready to run

* temp files are huge, don't preserve

* remove APIScan from release build

* remove duplicate install tool task

Co-authored-by: Travis Plunk <github@ez13.net>

PowerShell.Common.props

@@ -137,8 +137,6 @@
 
     <TargetFramework>net7.0</TargetFramework>
     <LangVersion>10.0</LangVersion>
-    <PublishReadyToRun Condition=" '$(Configuration)' != 'Debug' ">true</PublishReadyToRun>
-    <PublishReadyToRunEmitSymbols>true</PublishReadyToRunEmitSymbols>
 
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
@@ -171,9 +169,17 @@
 
   <!-- Define all OS, release configuration properties -->
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
+    <PublishReadyToRun>true</PublishReadyToRun>
+    <PublishReadyToRunEmitSymbols>true</PublishReadyToRunEmitSymbols>
     <Optimize>true</Optimize>
   </PropertyGroup>
 
+  <!-- Define all OS, release configuration properties -->
+  <PropertyGroup Condition=" '$(Configuration)' == 'StaticAnalysis' ">
+    <Optimize>true</Optimize>
+    <DebugType>full</DebugType>
+  </PropertyGroup>
+
   <!-- Define windows, release configuration properties -->
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' And '$(IsWindows)' == 'true' ">
     <Optimize>true</Optimize>
@@ -193,6 +199,8 @@
   <!-- Define all OS, CodeCoverage configuration properties -->
   <PropertyGroup Condition=" '$(Configuration)' == 'CodeCoverage' ">
     <!-- This is required to be portable to Coverlet tool !-->
+    <PublishReadyToRun Condition=" '$(Configuration)' != 'Debug' ">true</PublishReadyToRun>
+    <PublishReadyToRunEmitSymbols>true</PublishReadyToRunEmitSymbols>
     <DebugType>portable</DebugType>
   </PropertyGroup>
 

build.psm1

@@ -319,7 +319,7 @@ function Start-PSBuild {
                      "win7-x86")]
         [string]$Runtime,
 
-        [ValidateSet('Debug', 'Release', 'CodeCoverage', '')] # We might need "Checked" as well
+        [ValidateSet('Debug', 'Release', 'CodeCoverage', 'StaticAnalysis', '')] # We might need "Checked" as well
         [string]$Configuration,
 
         [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")]
@@ -814,7 +814,7 @@ function Compress-TestContent {
 function New-PSOptions {
     [CmdletBinding()]
     param(
-        [ValidateSet("Debug", "Release", "CodeCoverage", '')]
+        [ValidateSet('Debug', 'Release', 'CodeCoverage', 'StaticAnalysis', '')]
         [string]$Configuration,
 
         [ValidateSet("net7.0")]
@@ -2234,7 +2234,7 @@ function Start-DevPowerShell {
         [string[]]$ArgumentList = @(),
         [switch]$LoadProfile,
         [Parameter(ParameterSetName='ConfigurationParamSet')]
-        [ValidateSet("Debug", "Release", "CodeCoverage", '')] # should match New-PSOptions -Configuration values
+        [ValidateSet('Debug', 'Release', 'CodeCoverage', 'StaticAnalysis', '')] # should match New-PSOptions -Configuration values
         [string]$Configuration,
         [Parameter(ParameterSetName='BinDirParamSet')]
         [string]$BinDir,

tools/releaseBuild/azureDevOps/compliance.yml

@@ -39,6 +39,13 @@ stages:
       - template: templates/compliance/compliance.yml
         parameters:
           parentJobs: []
+  - stage: APIScan
+    displayName: 'ApiScan'
+    dependsOn: []
+    jobs:
+      - template: templates/compliance/apiscan.yml
+        parameters:
+          parentJobs: []
   - stage: notice
     displayName: Generate Notice File
     dependsOn: []

tools/releaseBuild/azureDevOps/templates/compliance.yml

@@ -9,9 +9,6 @@ jobs:
   - name: NugetSecurityAnalysisWarningLevel
     value: none
 
-  # Defines the variables APIScanClient, APIScanTenant and APIScanSecret
-  - group: PS-PS-APIScan
-
   displayName: Compliance
   dependsOn:
     ${{ parameters.parentJobs }}
@@ -20,9 +17,6 @@ jobs:
     demands:
     - ImageOverride -equals PSMMS2019-Secure
 
-  # APIScan can take a long time
-  timeoutInMinutes: 180
-
   steps:
   - checkout: self
     clean: true
@@ -93,18 +87,6 @@ jobs:
 
   # PreFASt is not applicable
 
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-apiscan.APIScan@2
-    displayName: 'Run APIScan'
-    inputs:
-      softwareFolder: '$(CompliancePath)'
-      softwareName: PowerShell
-      softwareVersionNum: '$(ReleaseTagVar)'
-      isLargeApp: false
-      preserveTempFiles: true
-    env:
-      AzureServicesAuthConnectionString: RunAs=App;AppId=$(APIScanClient);TenantId=$(APIScanTenant);AppKey=$(APIScanSecret)
-    continueOnError: true
-
   - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
     displayName: 'Publish Security Analysis Logs to Build Artifacts'
     continueOnError: true
@@ -129,7 +111,7 @@ jobs:
     displayName: 'Create Security Analysis Report'
     inputs:
       TsvFile: false
-      APIScan: true
+      APIScan: false
       BinSkim: true
       CredScan: true
       PoliCheck: true

tools/releaseBuild/azureDevOps/templates/compliance/apiscan.yml

@@ -0,0 +1,112 @@
+jobs:
+  - job: APIScan
+    variables:
+      - name: runCodesignValidationInjection
+        value : false
+      - name: NugetSecurityAnalysisWarningLevel
+        value: none
+      - name: ReleaseTagVar
+        value: fromBranch
+      # Defines the variables APIScanClient, APIScanTenant and APIScanSecret
+      - group: PS-PS-APIScan
+      # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission.
+      # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct.
+      - group: symbols
+
+    pool:
+      name: PowerShell1ES
+      demands:
+        - ImageOverride -equals PSMMS2019-Secure
+
+    # APIScan can take a long time
+    timeoutInMinutes: 180
+
+    steps:
+    - template: ../SetVersionVariables.yml
+      parameters:
+        ReleaseTagVar: $(ReleaseTagVar)
+        CreateJson: yes
+        UseJson: no
+
+    - pwsh: |
+        Import-Module .\build.psm1 -force
+        Start-PSBootstrap
+      workingDirectory: '$(Build.SourcesDirectory)'
+      retryCountOnTaskFailure: 2
+      displayName: 'Bootstrap'
+
+    - pwsh: |
+        Import-Module .\build.psm1 -force
+        Find-DotNet
+        dotnet tool install dotnet-symbol --tool-path $(Agent.ToolsDirectory)\tools\dotnet-symbol
+        $symbolToolPath = Get-ChildItem -Path $(Agent.ToolsDirectory)\tools\dotnet-symbol\dotnet-symbol.exe | Select-Object -First 1 -ExpandProperty FullName
+        Write-Host "##vso[task.setvariable variable=symbolToolPath]$symbolToolPath"
+      displayName: Install dotnet-symbol
+      retryCountOnTaskFailure: 2
+
+    - pwsh: |
+        Import-Module .\build.psm1 -force
+        Find-DotNet
+        Start-PSBuild -Configuration StaticAnalysis -PSModuleRestore -Clean
+
+        $OutputFolder = Split-Path (Get-PSOutput)
+        Write-Host "##vso[task.setvariable variable=BinDir]$OutputFolder"
+      workingDirectory: '$(Build.SourcesDirectory)'
+      displayName: 'Build PowerShell Source'
+
+    - pwsh: |
+        Get-ChildItem -Path env:
+      displayName: Capture Environment
+      condition: succeededOrFailed()
+
+    # Explicitly download symbols for the drop since the SDL image doesn't have http://SymWeb access and APIScan cannot handle https yet.
+    - pwsh: |
+        Import-Module .\build.psm1 -force
+        Find-DotNet
+        $pat = '$(SymbolServerPAT)'
+        if ($pat -like '*PAT*' -or $pat -eq '')
+        {
+          throw 'No PAT defined'
+        }
+        $url = 'https://microsoft.artifacts.visualstudio.com/defaultcollection/_apis/symbol/symsrv'
+        $(symbolToolPath) --authenticated-server-path $(SymbolServerPAT) $url --symbols -d "$env:BinDir\*" --recurse-subdirectories
+      displayName: 'Download Symbols for binaries'
+      retryCountOnTaskFailure: 2
+      workingDirectory: '$(Build.SourcesDirectory)'
+
+    - task: securedevelopmentteam.vss-secure-development-tools.build-task-apiscan.APIScan@2
+      displayName: 'Run APIScan'
+      inputs:
+        softwareFolder: '$(BinDir)'
+        softwareName: PowerShell
+        softwareVersionNum: '$(ReleaseTagVar)'
+        isLargeApp: false
+        preserveTempFiles: false
+        verbosityLevel: standard
+        # write a status update every 5 minutes.  Default is 1 minute
+        statusUpdateInterval: '00:05:00'
+      env:
+        AzureServicesAuthConnectionString: RunAs=App;AppId=$(APIScanClient);TenantId=$(APIScanTenant);AppKey=$(APIScanSecret)
+
+    - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@2
+      continueOnError: true
+      displayName: 'Guardian Export'
+      inputs:
+        GdnExportVstsConsole: true
+        GdnExportSarifFile: true
+        GdnExportHtmlFile: true
+        GdnExportAllTools: false
+        GdnExportGdnToolApiScan: true
+        #this didn't do anything GdnExportCustomLogsFolder: '$(Build.ArtifactStagingDirectory)/Guardian'
+
+    - pwsh: |
+        Get-ChildItem -Path env:
+      displayName: Capture Environment
+      condition: succeededOrFailed()
+
+    - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
+      displayName: 'Publish Guardian Artifacts'
+      inputs:
+        AllTools: false
+        APIScan: true
+        ArtifactName: APIScan