From a88623ecc6d8b8dae717382581bec8bda827b958 Mon Sep 17 00:00:00 2001 From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com> Date: Tue, 1 Jul 2025 00:35:14 +1000 Subject: [PATCH 1/7] Add batch generation parameter --- .azure-pipelines/batch-generation.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml index b20786a9da3f..718eae383802 100644 --- a/.azure-pipelines/batch-generation.yml +++ b/.azure-pipelines/batch-generation.yml @@ -1,8 +1,13 @@ parameters: - name: TestChangedModulesOnly - displayName: 'Only run tests on modules that are changed by regeneration' type: string default: 'True' +- name: SelectedTargetModules + type: string + default: '' +- name: NotificationReceiversOverride + type: string + default: '' variables: IntermediateStepTimeoutInMinutes: 30 @@ -39,6 +44,25 @@ stages: pool: ${{ variables.WindowsAgentPoolName }} steps: + - task: PowerShell@2 + name: showVariables + displayName: 'Show Variables' + inputs: + targetType: inline + pwsh: true + script: | + Write-Host "Pipeline variable SKIPPED_MODULES: $($env:SKIPPED_MODULES)" + Write-Host "Pipeline variable GENERATION_BASE_BRANCH: $($env:GENERATION_BASE_BRANCH)" + Write-Host "Pipeline variable FailedJobNotificationReceivers: $($env:FailedJobNotificationReceivers)" + + Write-Host "Parameter TestChangedModulesOnly: ${{ parameters.TestChangedModulesOnly }}" + Write-Host "Parameter SelectedTargetModules: ${{ parameters.SelectedTargetModules }}" + Write-Host "Parameter NotificationReceiversOverride: ${{ parameters.NotificationReceiversOverride }}" + + Write-Host "FINAL_NOTIFICATION_RECEIVERS: $($env:FINAL_NOTIFICATION_RECEIVERS)" + env: + FINAL_NOTIFICATION_RECEIVERS: ${{ coalesce(parameters.NotificationReceiversOverride, variables.FailedJobNotificationReceivers) }} + # TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agents # - checkout: none - checkout: self From e7b8f31627ceaa4488ec20afabdebac3d41ada6e Mon Sep 17 00:00:00 2001 From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com> Date: Tue, 1 Jul 2025 01:00:08 +1000 Subject: [PATCH 2/7] default null --- .azure-pipelines/batch-generation.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml index 718eae383802..907cf808d31f 100644 --- a/.azure-pipelines/batch-generation.yml +++ b/.azure-pipelines/batch-generation.yml @@ -4,10 +4,10 @@ parameters: default: 'True' - name: SelectedTargetModules type: string - default: '' + default: 'None' - name: NotificationReceiversOverride type: string - default: '' + default: 'None' variables: IntermediateStepTimeoutInMinutes: 30 @@ -59,10 +59,6 @@ stages: Write-Host "Parameter SelectedTargetModules: ${{ parameters.SelectedTargetModules }}" Write-Host "Parameter NotificationReceiversOverride: ${{ parameters.NotificationReceiversOverride }}" - Write-Host "FINAL_NOTIFICATION_RECEIVERS: $($env:FINAL_NOTIFICATION_RECEIVERS)" - env: - FINAL_NOTIFICATION_RECEIVERS: ${{ coalesce(parameters.NotificationReceiversOverride, variables.FailedJobNotificationReceivers) }} - # TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agents # - checkout: none - checkout: self From 0b862139450c87f8c906c96b4da3da0a49163501 Mon Sep 17 00:00:00 2001 From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com> Date: Tue, 1 Jul 2025 16:42:49 +1000 Subject: [PATCH 3/7] select value --- .../BatchGeneration/notify-failed-job.ps1 | 5 ++++- .../PipelineSteps/BatchGeneration/util.psm1 | 15 +++++++++++++++ .azure-pipelines/batch-generation.yml | 6 ++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 index 7f014d55a343..362209123575 100644 --- a/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 +++ b/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 @@ -15,7 +15,10 @@ $notificationContent = $notificationTemplate -replace "{{ pipelineName }}", $pip -replace "{{ pipelineUrl }}", $pipelineUrl ` -replace "{{ runUrl }}", $runUrl +$notificationReceivers = if ($env:NotificationReceiversOverride -and $env:NotificationReceiversOverride -ne 'none') { $env:NotificationReceiversOverride } else { $env:FailedJobNotificationReceivers } +Write-Host "Notification receivers: $notificationReceivers" + Send-Teams ` - -to $env:FailedJobNotificationReceivers ` + -to $notificationReceivers ` -title "Batch Generation Job Failed" ` -content $notificationContent diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 index 1c3543bb7ae1..7057bdee8c47 100644 --- a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 +++ b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 @@ -3,6 +3,15 @@ function Get-BatchGenerationModuleMap { [string]$srcPath ) $skippedModules = $env:SKIPPED_MODULES -split ',' | ForEach-Object { $_.Trim() } + $selectedTargetModules = @{} + if ($env:SELECTED_TARGET_MODULES -ne "none") { + $env:SELECTED_TARGET_MODULES -split ',' | ForEach-Object { + $key = $_.Trim() + if ($key -ne '') { + $selectedTargetModules[$key] = $true + } + } + } $result = @{} $modules = Get-ChildItem -Path $srcPath -Directory @@ -11,6 +20,12 @@ function Get-BatchGenerationModuleMap { Write-Warning "Skipping module: $($module.Name) as it is in the skipped modules list." continue } + + if ($selectedTargetModules.Count -gt 0 -and -not $selectedTargetModules.ContainsKey($module.Name)) { + Write-Warning "Skipping module: $($module.Name) as it is not in the selected target modules list." + continue + } + $subModules = Get-ChildItem -Path $module.FullName -Directory | Where-Object { $_.Name -like '*.autorest' } diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml index 907cf808d31f..0b8a80ff83fa 100644 --- a/.azure-pipelines/batch-generation.yml +++ b/.azure-pipelines/batch-generation.yml @@ -51,8 +51,8 @@ stages: targetType: inline pwsh: true script: | - Write-Host "Pipeline variable SKIPPED_MODULES: $($env:SKIPPED_MODULES)" - Write-Host "Pipeline variable GENERATION_BASE_BRANCH: $($env:GENERATION_BASE_BRANCH)" + Write-Host "Pipeline variable SkippedModules: $($env:SkippedModules)" + Write-Host "Pipeline variable GenerationBaseBranch: $($env:GenerationBaseBranch)" Write-Host "Pipeline variable FailedJobNotificationReceivers: $($env:FailedJobNotificationReceivers)" Write-Host "Parameter TestChangedModulesOnly: ${{ parameters.TestChangedModulesOnly }}" @@ -106,6 +106,7 @@ stages: & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelGenerateJobs }}" env: SKIPPED_MODULES: $(SkippedModules) + SELECTED_TARGET_MODULES: ${{ parameters.SelectedTargetModules }} - task: PublishPipelineArtifact@1 displayName: 'Upload generated targets' @@ -535,3 +536,4 @@ stages: env: TEAMS_URL: $(TEAMS_URL) FailedJobNotificationReceivers: $(FailedJobNotificationReceivers) + NotificationReceiversOverride: ${{ parameters.NotificationReceiversOverride }} From 986f96516af09803b4b6535b74ccbd6c0e7ddbc8 Mon Sep 17 00:00:00 2001 From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com> Date: Tue, 1 Jul 2025 23:53:25 +1000 Subject: [PATCH 4/7] move GenerationBaseBranch from variable to parameter --- .azure-pipelines/batch-generation.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml index 0b8a80ff83fa..ed8da59a8a99 100644 --- a/.azure-pipelines/batch-generation.yml +++ b/.azure-pipelines/batch-generation.yml @@ -8,6 +8,9 @@ parameters: - name: NotificationReceiversOverride type: string default: 'None' +- name: GenerationBaseBranch + type: string + default: 'main' variables: IntermediateStepTimeoutInMinutes: 30 @@ -52,12 +55,12 @@ stages: pwsh: true script: | Write-Host "Pipeline variable SkippedModules: $($env:SkippedModules)" - Write-Host "Pipeline variable GenerationBaseBranch: $($env:GenerationBaseBranch)" Write-Host "Pipeline variable FailedJobNotificationReceivers: $($env:FailedJobNotificationReceivers)" Write-Host "Parameter TestChangedModulesOnly: ${{ parameters.TestChangedModulesOnly }}" Write-Host "Parameter SelectedTargetModules: ${{ parameters.SelectedTargetModules }}" Write-Host "Parameter NotificationReceiversOverride: ${{ parameters.NotificationReceiversOverride }}" + Write-Host "Parameter GenerationBaseBranch: ${{ parameters.GenerationBaseBranch }}" # TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agents # - checkout: none @@ -79,7 +82,7 @@ stages: $token = "$(GitHubToken)" $createBranchPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'create-branch.ps1' - & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch "$(GenerationBaseBranch)" -NewBranch $generationBranch -Token $token + & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch "${{ parameters.GenerationBaseBranch }}" -NewBranch $generationBranch -Token $token Write-Host "##vso[task.setvariable variable=GenerationBranch;isOutput=true]$generationBranch" - task: PowerShell@2 From f726b098efee3e7637e317bec096e3fc3490d0c7 Mon Sep 17 00:00:00 2001 From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com> Date: Wed, 2 Jul 2025 03:28:25 +1000 Subject: [PATCH 5/7] filter modules on generation branch incase main branch changed --- .azure-pipelines/batch-generation.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml index ed8da59a8a99..30a43fb2fb7f 100644 --- a/.azure-pipelines/batch-generation.yml +++ b/.azure-pipelines/batch-generation.yml @@ -105,6 +105,10 @@ stages: targetType: inline pwsh: true script: | + Write-Host "Checkout to GenerationBranch: $(checkout.GenerationBranch)" + git fetch origin "$(checkout.GenerationBranch)" + git checkout "$(checkout.GenerationBranch)" + $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1' & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelGenerateJobs }}" env: From b8be3f7f6dc1c95ca2956b706836868ca7d25387 Mon Sep 17 00:00:00 2001 From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com> Date: Fri, 4 Jul 2025 00:29:04 +1000 Subject: [PATCH 6/7] prepare modules on generation branch --- .../PipelineSteps/BatchGeneration/filter.ps1 | 2 +- .azure-pipelines/batch-generation.yml | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 index 88fe906c9157..046c47a29f95 100644 --- a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 +++ b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 @@ -15,7 +15,7 @@ $artifactsDir = Join-Path $RepoRoot 'artifacts' $changedModulesDict = @{} $changedSubModulesDict = @{} -if ($env:TEST_CHANGED_MODULES_ONLY -eq "True") { +if ($env:TEST_CHANGED_MODULES_ONLY -eq 'true') { Write-Host "Run test on generated folder changed modules" # Only generated folder change should trigger the test for ($i = 0; $i -lt $ChangedFiles.Count; $i++) { diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml index 30a43fb2fb7f..625db9a62e92 100644 --- a/.azure-pipelines/batch-generation.yml +++ b/.azure-pipelines/batch-generation.yml @@ -1,16 +1,21 @@ parameters: - name: TestChangedModulesOnly - type: string - default: 'True' + type: boolean + default: true +# Selected modules to generate, build, analyze and test - name: SelectedTargetModules type: string default: 'None' - name: NotificationReceiversOverride type: string default: 'None' +# branch from when creating the generation branch - name: GenerationBaseBranch type: string default: 'main' +- name: PrepareModulesOnGenerationBranch + type: boolean + default: false variables: IntermediateStepTimeoutInMinutes: 30 @@ -61,6 +66,7 @@ stages: Write-Host "Parameter SelectedTargetModules: ${{ parameters.SelectedTargetModules }}" Write-Host "Parameter NotificationReceiversOverride: ${{ parameters.NotificationReceiversOverride }}" Write-Host "Parameter GenerationBaseBranch: ${{ parameters.GenerationBaseBranch }}" + Write-Host "Parameter PrepareModulesOnGenerationBranch: ${{ parameters.PrepareModulesOnGenerationBranch }}" # TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agents # - checkout: none @@ -105,9 +111,11 @@ stages: targetType: inline pwsh: true script: | - Write-Host "Checkout to GenerationBranch: $(checkout.GenerationBranch)" - git fetch origin "$(checkout.GenerationBranch)" - git checkout "$(checkout.GenerationBranch)" + if (${{ parameters.PrepareModulesOnGenerationBranch }} -eq 'true') { + Write-Host "Checkout to GenerationBranch: $(checkout.GenerationBranch)" + git fetch origin "$(checkout.GenerationBranch)" + git checkout "$(checkout.GenerationBranch)" + } $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1' & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelGenerateJobs }}" From 1c50013e0c702d7a4139072cf6aa3365c5496df8 Mon Sep 17 00:00:00 2001 From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com> Date: Fri, 4 Jul 2025 01:25:01 +1000 Subject: [PATCH 7/7] fix parameter reference --- .azure-pipelines/batch-generation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml index 625db9a62e92..eb66032b75db 100644 --- a/.azure-pipelines/batch-generation.yml +++ b/.azure-pipelines/batch-generation.yml @@ -111,7 +111,7 @@ stages: targetType: inline pwsh: true script: | - if (${{ parameters.PrepareModulesOnGenerationBranch }} -eq 'true') { + if ($env:PREPARE_MODULES_ON_GENERATION_BRANCH -eq 'true') { Write-Host "Checkout to GenerationBranch: $(checkout.GenerationBranch)" git fetch origin "$(checkout.GenerationBranch)" git checkout "$(checkout.GenerationBranch)" @@ -122,6 +122,7 @@ stages: env: SKIPPED_MODULES: $(SkippedModules) SELECTED_TARGET_MODULES: ${{ parameters.SelectedTargetModules }} + PREPARE_MODULES_ON_GENERATION_BRANCH: ${{ parameters.PrepareModulesOnGenerationBranch }} - task: PublishPipelineArtifact@1 displayName: 'Upload generated targets'