diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 index 88fe906c9157..046c47a29f95 100644 --- a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 +++ b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 @@ -15,7 +15,7 @@ $artifactsDir = Join-Path $RepoRoot 'artifacts' $changedModulesDict = @{} $changedSubModulesDict = @{} -if ($env:TEST_CHANGED_MODULES_ONLY -eq "True") { +if ($env:TEST_CHANGED_MODULES_ONLY -eq 'true') { Write-Host "Run test on generated folder changed modules" # Only generated folder change should trigger the test for ($i = 0; $i -lt $ChangedFiles.Count; $i++) { diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 index 7f014d55a343..362209123575 100644 --- a/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 +++ b/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 @@ -15,7 +15,10 @@ $notificationContent = $notificationTemplate -replace "{{ pipelineName }}", $pip -replace "{{ pipelineUrl }}", $pipelineUrl ` -replace "{{ runUrl }}", $runUrl +$notificationReceivers = if ($env:NotificationReceiversOverride -and $env:NotificationReceiversOverride -ne 'none') { $env:NotificationReceiversOverride } else { $env:FailedJobNotificationReceivers } +Write-Host "Notification receivers: $notificationReceivers" + Send-Teams ` - -to $env:FailedJobNotificationReceivers ` + -to $notificationReceivers ` -title "Batch Generation Job Failed" ` -content $notificationContent diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 index 1c3543bb7ae1..7057bdee8c47 100644 --- a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 +++ b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 @@ -3,6 +3,15 @@ function Get-BatchGenerationModuleMap { [string]$srcPath ) $skippedModules = $env:SKIPPED_MODULES -split ',' | ForEach-Object { $_.Trim() } + $selectedTargetModules = @{} + if ($env:SELECTED_TARGET_MODULES -ne "none") { + $env:SELECTED_TARGET_MODULES -split ',' | ForEach-Object { + $key = $_.Trim() + if ($key -ne '') { + $selectedTargetModules[$key] = $true + } + } + } $result = @{} $modules = Get-ChildItem -Path $srcPath -Directory @@ -11,6 +20,12 @@ function Get-BatchGenerationModuleMap { Write-Warning "Skipping module: $($module.Name) as it is in the skipped modules list." continue } + + if ($selectedTargetModules.Count -gt 0 -and -not $selectedTargetModules.ContainsKey($module.Name)) { + Write-Warning "Skipping module: $($module.Name) as it is not in the selected target modules list." + continue + } + $subModules = Get-ChildItem -Path $module.FullName -Directory | Where-Object { $_.Name -like '*.autorest' } diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml index b20786a9da3f..eb66032b75db 100644 --- a/.azure-pipelines/batch-generation.yml +++ b/.azure-pipelines/batch-generation.yml @@ -1,8 +1,21 @@ parameters: - name: TestChangedModulesOnly - displayName: 'Only run tests on modules that are changed by regeneration' + type: boolean + default: true +# Selected modules to generate, build, analyze and test +- name: SelectedTargetModules type: string - default: 'True' + default: 'None' +- name: NotificationReceiversOverride + type: string + default: 'None' +# branch from when creating the generation branch +- name: GenerationBaseBranch + type: string + default: 'main' +- name: PrepareModulesOnGenerationBranch + type: boolean + default: false variables: IntermediateStepTimeoutInMinutes: 30 @@ -39,6 +52,22 @@ stages: pool: ${{ variables.WindowsAgentPoolName }} steps: + - task: PowerShell@2 + name: showVariables + displayName: 'Show Variables' + inputs: + targetType: inline + pwsh: true + script: | + Write-Host "Pipeline variable SkippedModules: $($env:SkippedModules)" + Write-Host "Pipeline variable FailedJobNotificationReceivers: $($env:FailedJobNotificationReceivers)" + + Write-Host "Parameter TestChangedModulesOnly: ${{ parameters.TestChangedModulesOnly }}" + Write-Host "Parameter SelectedTargetModules: ${{ parameters.SelectedTargetModules }}" + Write-Host "Parameter NotificationReceiversOverride: ${{ parameters.NotificationReceiversOverride }}" + Write-Host "Parameter GenerationBaseBranch: ${{ parameters.GenerationBaseBranch }}" + Write-Host "Parameter PrepareModulesOnGenerationBranch: ${{ parameters.PrepareModulesOnGenerationBranch }}" + # TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agents # - checkout: none - checkout: self @@ -59,7 +88,7 @@ stages: $token = "$(GitHubToken)" $createBranchPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'create-branch.ps1' - & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch "$(GenerationBaseBranch)" -NewBranch $generationBranch -Token $token + & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch "${{ parameters.GenerationBaseBranch }}" -NewBranch $generationBranch -Token $token Write-Host "##vso[task.setvariable variable=GenerationBranch;isOutput=true]$generationBranch" - task: PowerShell@2 @@ -82,10 +111,18 @@ stages: targetType: inline pwsh: true script: | + if ($env:PREPARE_MODULES_ON_GENERATION_BRANCH -eq 'true') { + Write-Host "Checkout to GenerationBranch: $(checkout.GenerationBranch)" + git fetch origin "$(checkout.GenerationBranch)" + git checkout "$(checkout.GenerationBranch)" + } + $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1' & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelGenerateJobs }}" env: SKIPPED_MODULES: $(SkippedModules) + SELECTED_TARGET_MODULES: ${{ parameters.SelectedTargetModules }} + PREPARE_MODULES_ON_GENERATION_BRANCH: ${{ parameters.PrepareModulesOnGenerationBranch }} - task: PublishPipelineArtifact@1 displayName: 'Upload generated targets' @@ -515,3 +552,4 @@ stages: env: TEAMS_URL: $(TEAMS_URL) FailedJobNotificationReceivers: $(FailedJobNotificationReceivers) + NotificationReceiversOverride: ${{ parameters.NotificationReceiversOverride }}