Struct out PGO scripts and jobs borrowed from MUX

Author: miniksa

.vscode/settings.json

@@ -41,5 +41,9 @@
         "**/obj/**": true,
         "**/packages/**": true,
         "**/generated files/**": true
-    }
+    },
+    "vscode-nmake-tools.workspaceBuildDirectories": [
+        "d:\\src\\terminal"
+    ],
+    "C_Cpp.default.configurationProvider": "microsoft.vscode-nmake-tools"
 }

build/Helix/RunTestsInHelix.proj

@@ -14,5 +14,6 @@
 
   <!-- These .proj files are generated by the build machine prior to running tests via GenerateTestProjFile.ps1. -->
   <Import Project="$(ProjFilesPath)\RunTestsInHelix-TerminalAppLocalTests.proj" Condition=" '$(TestSuite)'=='DevTestSuite' " />
+  <Import Project="$(ProjFilesPath)\RunTestsInHelix-SettingsModelLocalTests.proj" Condition=" '$(TestSuite)'=='DevTestSuite' " />
   <Import Project="$(ProjFilesPath)\RunTestsInHelix-HostTestsUIA.proj" Condition=" '$(TestSuite)'=='DevTestSuite' " />
 </Project>

build/Helix/runtests.cmd

@@ -46,7 +46,7 @@ move te.wtl te_original.wtl
 
 copy /y te_original.wtl %HELIX_WORKITEM_UPLOAD_ROOT%
 copy /y WexLogFileOutput\*.jpg %HELIX_WORKITEM_UPLOAD_ROOT%
-for /f "tokens=* delims=" %%a in ('dir /b *.pgc') do ren "%%a" "%testnameprefix%.%%~na.pgc"
+:: for /f "tokens=* delims=" %%a in ('dir /b *.pgc') do ren "%%a" "%testnameprefix%.%%~na.pgc"
 copy /y *.pgc %HELIX_WORKITEM_UPLOAD_ROOT%
 
 set FailedTestQuery=

build/pipelines/pgo.yml

@@ -0,0 +1,27 @@
+trigger: none
+pr: none
+
+variables:
+  - name: runCodesignValidationInjectionBG
+    value: false
+
+#     0.0.yyMM.dd##
+#     0.0.1904.0900
+name: 0.0.$(Date:yyMM).$(Date:dd)$(Rev:rr)
+
+stages:
+  - stage: Build_x64
+    displayName: Build x64
+    dependsOn: []
+    condition: succeeded()
+    jobs:
+      - template: ./templates/build-console-pgo.yml
+        parameters:
+          platform: x64
+  - stage: Build_x86
+    displayName: Build x86
+    dependsOn: []
+    jobs:
+      - template: ./templates/build-console-pgo.yml
+        parameters:
+          platform: x86

build/pipelines/templates/build-console-pgo.yml

@@ -0,0 +1,52 @@
+parameters:
+  configuration: 'Release'
+  platform: ''
+  additionalBuildArguments: ''
+  minimumExpectedTestsExecutedCount: 10  # Sanity check for minimum expected tests to be reported
+  rerunPassesRequiredToAvoidFailure: 5
+
+jobs:
+- job: Build${{ parameters.platform }}${{ parameters.configuration }}
+  displayName: Build ${{ parameters.platform }} ${{ parameters.configuration }}
+  variables:
+    BuildConfiguration: ${{ parameters.configuration }}
+    BuildPlatform: ${{ parameters.platform }}
+    PGOBuildMode: 'Instrument'
+  pool: "windevbuildagents"
+  # The public pool is also an option!
+  # pool: { vmImage: windows-2019 }
+
+  steps:
+  - template: build-console-steps.yml
+    parameters:
+      additionalBuildArguments: ${{ parameters.additionalBuildArguments }}
+
+- template: helix-runtests-job.yml
+  parameters:
+    name: 'RunTestsInHelix'
+    dependsOn: Build${{ parameters.platform }}${{ parameters.configuration }}
+    condition: succeeded()
+    testSuite: 'PgoInstrumentationSuite'
+    rerunPassesRequiredToAvoidFailure: ${{ parameters.rerunPassesRequiredToAvoidFailure }}
+    
+- template: helix-processtestresults-job.yml
+  parameters:
+    name: 'ProcessTestResults'
+    dependsOn:
+    - RunTestsInHelix
+    condition: succeededOrFailed()
+    rerunPassesRequiredToAvoidFailure: ${{ parameters.rerunPassesRequiredToAvoidFailure }}
+    minimumExpectedTestsExecutedCount: ${{ parameters.minimumExpectedTestsExecutedCount }}
+
+- template: pgo-merge-pgd-job.yml
+  parameters:
+    name: 'MergePGD'
+    dependsOn:
+    - ProcessTestResults
+    pgoArtifact: 'PGO'
+
+- template: pgo-build-and-publish-nuget-job.yml
+  parameters:
+    dependsOn:
+    - MergePGD
+    pgoArtifact: 'PGO'

build/pipelines/templates/build-console-steps.yml

@@ -32,6 +32,29 @@ steps:
     restoreSolution: build/packages.config
     restoreDirectory: '$(Build.SourcesDirectory)\packages'
 
+# The environment variable VCToolsInstallDir isn't defined on lab machines, so we need to retrieve it ourselves.
+- script: |
+    "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -Latest -requires Microsoft.Component.MSBuild -property InstallationPath > %TEMP%\vsinstalldir.txt
+    set /p _VSINSTALLDIR15=<%TEMP%\vsinstalldir.txt
+    del %TEMP%\vsinstalldir.txt
+    call "%_VSINSTALLDIR15%\Common7\Tools\VsDevCmd.bat"
+    echo VCToolsInstallDir = %VCToolsInstallDir%
+    echo ##vso[task.setvariable variable=VCToolsInstallDir]%VCToolsInstallDir%
+  displayName: 'Retrieve VC tools directory'
+
+- task: CmdLine@1
+  displayName: 'Display build machine environment variables'
+  inputs:
+    filename: 'set'
+
+- task: powershell@2
+  displayName: 'Restore PGO database'
+  condition: eq(variables['PGOBuildMode'], 'Optimize')
+  inputs:
+    targetType: filePath
+    workingDirectory: $(Build.SourcesDirectory)\tools\PGODatabase
+    filePath: $(Build.SourcesDirectory)\tools\PGODatabase\restore-pgodb.ps1
+
 - task: VSBuild@1
   displayName: 'Build solution **\OpenConsole.sln'
   inputs:
@@ -51,60 +74,6 @@ steps:
       $Package = Get-ChildItem -Recurse -Filter "CascadiaPackage_*.msix"
       .\build\scripts\Test-WindowsTerminalPackage.ps1 -Verbose -Path $Package.FullName
 
-- task: powershell@2
-  displayName: 'Source Index PDBs'
-  inputs:
-    targetType: filePath
-    filePath: build\scripts\Index-Pdbs.ps1
-    arguments: -SearchDir '$(Build.SourcesDirectory)' -SourceRoot '$(Build.SourcesDirectory)' -recursive -Verbose -CommitId $(Build.SourceVersion)
-    errorActionPreference: silentlyContinue
-
-- task: PowerShell@2
-  displayName: 'Rationalize build platform'
-  inputs:
-    targetType: inline
-    script: |
-      $Arch = "$(BuildPlatform)"
-      If ($Arch -Eq "x86") { $Arch = "Win32" }
-      Write-Host "##vso[task.setvariable variable=RationalizedBuildPlatform]${Arch}"
-
-- task: PowerShell@2
-  displayName: 'Run Unit Tests'
-  inputs:
-    targetType: filePath
-    filePath: build\scripts\Run-Tests.ps1
-    arguments: -MatchPattern '*unit.test*.dll' -Platform '$(RationalizedBuildPlatform)' -Configuration '$(BuildConfiguration)' -LogPath '${{ parameters.testLogPath }}'
-  condition: and(succeeded(), or(eq(variables['BuildPlatform'], 'x64'), eq(variables['BuildPlatform'], 'x86')))
-
-- task: PowerShell@2
-  displayName: 'Run Feature Tests (x64 only)'
-  inputs:
-    targetType: filePath
-    filePath: build\scripts\Run-Tests.ps1
-    arguments: -MatchPattern '*feature.test*.dll' -Platform '$(RationalizedBuildPlatform)' -Configuration '$(BuildConfiguration)' -LogPath '${{ parameters.testLogPath }}'
-  condition: and(succeeded(), eq(variables['BuildPlatform'], 'x64'))
-
-- task: PowerShell@2
-  displayName: 'Convert Test Logs from WTL to xUnit format'
-  inputs:
-    targetType: filePath
-    filePath: build\Helix\ConvertWttLogToXUnit.ps1
-    arguments: -WttInputPath '${{ parameters.testLogPath }}' -WttSingleRerunInputPath 'unused.wtl' -WttMultipleRerunInputPath 'unused2.wtl' -XUnitOutputPath 'onBuildMachineResults.xml' -TestNamePrefix '$(BuildConfiguration).$(BuildPlatform)'
-  condition: or(eq(variables['BuildPlatform'], 'x64'), eq(variables['BuildPlatform'], 'x86'))
-
-- task: PublishTestResults@2
-  displayName: 'Upload converted test logs'
-  inputs:
-    testResultsFormat: 'xUnit' # Options: JUnit, NUnit, VSTest, xUnit, cTest
-    testResultsFiles: '**/onBuildMachineResults.xml'
-    #searchFolder: '$(System.DefaultWorkingDirectory)' # Optional
-    #mergeTestResults: false # Optional
-    #failTaskOnFailedTests: false # Optional
-    testRunTitle: 'On Build Machine Tests' # Optional
-    buildPlatform: $(BuildPlatform) # Optional
-    buildConfiguration: $(BuildConfiguration) # Optional
-    #publishRunAttachments: true # Optional
-
 - task: CopyFiles@2
   displayName: 'Copy result logs to Artifacts'
   inputs:
@@ -127,7 +96,7 @@ steps:
     TargetFolder: '$(Build.ArtifactStagingDirectory)/appx'
     OverWrite: true
     flattenFolders: true
-  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+  condition: succeeded()
 
 - task: CopyFiles@2
   displayName: 'Copy outputs needed for test runs to Artifacts'
@@ -141,7 +110,17 @@ steps:
     TargetFolder: '$(Build.ArtifactStagingDirectory)/$(BuildConfiguration)/$(BuildPlatform)/test'
     OverWrite: true
     flattenFolders: true
-  condition: and(and(succeeded(), eq(variables['BuildPlatform'], 'x64')), ne(variables['Build.Reason'], 'PullRequest'))
+  condition: succeeded()
+
+- task: CopyFiles@2
+  displayName: 'Copy PGO databases needed for PGO instrumentation run'
+  inputs:
+    Contents: |
+     $(Build.SourcesDirectory)/bin/$(BuildPlatform)/$(BuildConfiguration)/*.pgd
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/$(BuildConfiguration)/$(BuildPlatform)/PGO'
+    OverWrite: true
+    flattenFolders: true
+  condition: and(succeeded(), eq(variables['PGOBuildMode'], 'Instrument'))
 
 - task: PublishBuildArtifacts@1
   displayName: 'Publish All Build Artifacts'

build/pipelines/templates/helix-runtests-job.yml

@@ -99,15 +99,15 @@ jobs:
 
   - template: helix-createprojfile-steps.yml
     parameters:
-      condition: and(succeeded(),ne('${{ parameters.testSuite }}','NugetTestSuite'))
+      condition: and(succeeded(),eq('${{ parameters.testSuite }}','DevTestSuite'))
       testFilePath: '$(artifactsDir)\${{ parameters.artifactName }}\$(buildConfiguration)\$(buildPlatform)\Test\TerminalApp.LocalTests.dll'
       outputProjFileName: 'RunTestsInHelix-TerminalAppLocalTests.proj'
       testSuite: '${{ parameters.testSuite }}'
       taefQuery: ${{ parameters.taefQuery }}
 
   - template: helix-createprojfile-steps.yml
     parameters:
-      condition: and(succeeded(),ne('${{ parameters.testSuite }}','NugetTestSuite'))
+      condition: and(succeeded(),eq('${{ parameters.testSuite }}','DevTestSuite'))
       testFilePath: '$(artifactsDir)\${{ parameters.artifactName }}\$(buildConfiguration)\$(buildPlatform)\Test\SettingsModel.LocalTests.dll'
       outputProjFileName: 'RunTestsInHelix-SettingsModelLocalTests.proj'
       testSuite: '${{ parameters.testSuite }}'
@@ -116,12 +116,20 @@ jobs:
 
   - template: helix-createprojfile-steps.yml
     parameters:
-      condition: and(succeeded(),ne('${{ parameters.testSuite }}','NugetTestSuite'))
+      condition: and(succeeded(),eq('${{ parameters.testSuite }}','DevTestSuite'))
       testFilePath: '$(artifactsDir)\${{ parameters.artifactName }}\$(buildConfiguration)\$(buildPlatform)\Test\Conhost.UIA.Tests.dll'
       outputProjFileName: 'RunTestsInHelix-HostTestsUIA.proj'
       testSuite: '${{ parameters.testSuite }}'
       taefQuery: ${{ parameters.taefQuery }}
 
+  - template: helix-createprojfile-steps.yml
+    parameters:
+      condition: and(succeeded(),eq('${{ parameters.testSuite }}','PgoInstrumentationSuite'))
+      testFilePath: '$(artifactsDir)\${{ parameters.artifactName }}\$(buildConfiguration)\$(buildPlatform)\Test\TerminalApp.PGO.Tests.dll'
+      outputProjFileName: 'RunTestsInHelix-TerminalAppPGOTests.proj'
+      testSuite: '${{ parameters.testSuite }}'
+      taefQuery: ${{ parameters.taefQuery }}
+
   - task: PublishBuildArtifacts@1
     displayName: 'Publish generated .proj files'
     inputs:

build/pipelines/templates/pgo-build-and-publish-nuget-job.yml

@@ -0,0 +1,55 @@
+# From our friends at MUX: https://github.com/microsoft/microsoft-ui-xaml/blob/main/build/AzurePipelinesTemplates/MUX-BuildAndPublishPGONuGet-Job.yml
+
+parameters:
+  dependsOn: ''
+  pgoArtifact: PGO
+
+jobs:
+- job: BuildAndPublishPGONuGet
+  dependsOn: ${{ parameters.dependsOn }}
+  pool:
+    vmImage: 'windows-2019'
+  variables:
+    artifactsPath: $(Build.SourcesDirectory)\Artifacts
+    pgoToolsPath: $(Build.SourcesDirectory)\tools\PGODatabase
+    nuspecPath: $(pgoToolsPath)\NuSpecs
+    nuspecFilename: PGO.nuspec
+
+  steps:
+  - task: DownloadBuildArtifacts@0
+    inputs:
+      artifactName: ${{ parameters.pgoArtifact }}
+      downloadPath: $(artifactsPath)
+
+  - task: CopyFiles@2
+    displayName: 'Copy pgd files to NuGet build directory'
+    inputs:
+      sourceFolder: $(artifactsPath)\${{ parameters.pgoArtifact }}
+      contents: '**\*.pgd'
+      targetFolder: $(nuspecPath)\tools
+
+  - task: powershell@2
+    displayName: 'Generate NuSpec file'
+    inputs:
+      targetType: filePath
+      filePath: $(pgoToolsPath)\generate-nuspec.ps1
+      workingDirectory: $(pgoToolsPath)
+      arguments: $(nuspecPath)\$(nuspecFilename).template $(nuspecPath)\$(nuspecFilename)
+
+  - task: CmdLine@1
+    displayName: 'Build NuGet package'
+    inputs:
+      filename: '$(Build.SourcesDirectory)\tools\NugetWrapper.cmd'
+      arguments: pack "$(nuspecPath)\$(nuspecFilename)" -BasePath "$(nuspecPath)" -OutputDirectory "$(Build.ArtifactStagingDirectory)" -properties PROGRAMFILES="%ProgramFiles(x86)%"
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      pathToPublish: $(Build.ArtifactStagingDirectory)
+      artifactName: ${{ parameters.pgoArtifact }}
+
+  - task: 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2
+    displayName: 'NuGet push'
+    inputs:
+      command: push
+      publishVstsFeed: Terminal/TerminalDependencies
+      packagesToPush: $(Build.ArtifactStagingDirectory)/*.nupkg

build/pipelines/templates/pgo-merge-pgd-job.yml

@@ -0,0 +1,54 @@
+parameters:
+  dependsOn: ''
+  pgoArtifact: PGO
+
+jobs:
+- job: MergePGD
+  dependsOn: ${{ parameters.dependsOn }}
+  pool:
+    vmImage: 'windows-2019'
+  variables:
+    artifactsPath: $(Build.SourcesDirectory)\Artifacts
+    pgoArtifactsPath: $(artifactsPath)\${{ parameters.pgoArtifact }}
+  strategy:
+    matrix: ${{ parameters.matrix }}
+
+  steps:
+  # The environment variable VCToolsInstallDir isn't defined on lab machines, so we need to retrieve it ourselves.
+  - script: |
+      "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -Latest -requires Microsoft.Component.MSBuild -property InstallationPath > %TEMP%\vsinstalldir.txt
+      set /p _VSINSTALLDIR15=<%TEMP%\vsinstalldir.txt
+      del %TEMP%\vsinstalldir.txt
+      call "%_VSINSTALLDIR15%\Common7\Tools\VsDevCmd.bat"
+      echo VCToolsInstallDir = %VCToolsInstallDir%
+      echo ##vso[task.setvariable variable=VCToolsInstallDir]%VCToolsInstallDir%
+    displayName: 'Retrieve VC tools directory'
+
+  - task: DownloadBuildArtifacts@0
+    inputs:
+      artifactName: ${{ parameters.pgoArtifact }}
+      downloadPath: $(artifactsPath)
+
+  - script: |
+      cd $(buildPlatform)
+      "%VCToolsInstallDir%\bin\hostx64\x64\pgomgr.exe" /merge WindowsTerminal*.pgc WindowsTerminal.pgd
+    displayName: 'Merge Terminal pgc files into pgd'
+    workingDirectory: $(pgoArtifactsPath)
+
+  - script: |
+      cd $(buildPlatform)
+      "%VCToolsInstallDir%\bin\hostx64\x64\pgomgr.exe" /merge OpenConsole*.pgc OpenConsole.pgd
+    displayName: 'Merge OpenConsole pgc files into pgd'
+    workingDirectory: $(pgoArtifactsPath)
+
+  - task: CopyFiles@2
+    displayName: 'Copy merged pgd to artifact staging'
+    inputs:
+      sourceFolder: $(pgoArtifactsPath)
+      contents: '**\$(buildPlatform)\*.pgd'
+      targetFolder: $(Build.ArtifactStagingDirectory)
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      pathToPublish: $(Build.ArtifactStagingDirectory)
+      artifactName: ${{ parameters.pgoArtifact }}