From 4747c9c4e7af3c98718ad25c380c24a6c9c47da5 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 25 Nov 2024 15:10:24 -0800 Subject: [PATCH] Update the docIndex default for manual runs (#31903) This PR only updates the manual run conditions which were completely backwards. When running manually they wouldn't update main unless a variable was set and would always kick off the daily run which is only supposed to be kicked off during the scheduled run. The daily branch updates kick off a docs run against the daily branch which is created/updated when scheduled runs publish daily libraries. On the docs side, when the run against the daily branch completes, it nukes the branch and renames it to "daily/-[ci-succeeded|ci-failed"]. Unless we're specifically testing something related to kicking off the daily runs, it should only ever be run from the scheduled doc index runs. Multiple daily runs can cause issues on the docs with the way the success/failure branch is created. UpdateMain and ForceDailyUpdate are both template parameters. UpdateMain defaults to true and ForceDailyUpdate defaults to false. I also moved the verify required metadata json members _before_ running anything because if any of these are borked, any one of the steps running afterwards could fail. With these changes: - Scheduled runs will still update the main and daily branches and kick off the daily run. - Manual runs will run update main unless UpdateMain is set to true (which is the default) - Manual runs will not update the daily branch and kick off the daily run unless ForceDailyUpdate is set to true but the default is false. The Daily runs should only really be being kicked off during the scheduled run which is at the end of the day UTC. Testing the changes: - [Manual docIndex run with defaults](https://dev.azure.com/azure-sdk/internal/_build/results?buildId=4363170&view=results). This will process/update main without touching the daily branch. - [Manual docIndex run with the UpdateMain unchecked (false) and ForceDailyUpdate checked (true)](https://dev.azure.com/azure-sdk/internal/_build/results?buildId=4363175&view=results). This will skip processing main and only process the daily branch and kick off the daily docs run. --- eng/pipelines/docindex.yml | 260 ++++++++++++++++++++----------------- 1 file changed, 140 insertions(+), 120 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 94beaf7ff9e9..aba74f4c0ffc 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -1,4 +1,18 @@ trigger: none + +parameters: +- name: UpdateMain + displayName: | + Update main branch. + type: boolean + default: true + +- name: ForceDailyUpdate + displayName: | + Force the daily branch update (includes starting daily branch run). + type: boolean + default: false + variables: - template: /eng/pipelines/templates/variables/globals.yml jobs: @@ -56,127 +70,133 @@ jobs: - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Move deprecated packages to legacy moniker - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Update Docs Onboarding for main branch - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Generate Service Level Readme for main branch - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsToc.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml - displayName: Generate ToC for main branch - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - displayName: Verify Required Docs Json Members - - # Push changes to docs repo - template: /eng/common/pipelines/templates/steps/set-default-branch.yml parameters: WorkingDirectory: $(DocRepoLocation) - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: $(DefaultBranch) - BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" - TargetRepoName: $(DocRepoName) - TargetRepoOwner: $(DocRepoOwner) - WorkingDirectory: $(DocRepoLocation) - - # Prepare daily docs CI - - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml - parameters: - DailyBranchVariableName: DailyDocsBranchName - - pwsh: | - $ErrorActionPreference = "Continue" - git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null - $LASTEXITCODE = 0 # This ignores any error from git checkout - git status - displayName: Checkout daily branch if it exists - workingDirectory: $(DocRepoLocation) - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Update Docs Onboarding for Daily docs - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Generate Service Level Readme for Daily docs - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsToc.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml - displayName: Generate ToC for Daily docs - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - displayName: Verify Required Docs Json Members - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: $(DailyDocsBranchName) - BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" - TargetRepoName: $(DocRepoName) - TargetRepoOwner: $(DocRepoOwner) - WorkingDirectory: $(DocRepoLocation) - ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts - PushArgs: -f - - - task: AzureCLI@2 - displayName: Queue Docs CI build - inputs: - azureSubscription: msdocs-apidrop-connection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv - $buildParamJson = '{"params":"{ \"target_repo\": { \"url\": \"https://github.com/MicrosoftDocs/azure-docs-sdk-node\", \"branch\": \"$(DailyDocsBranchName)\", \"folder\": \"./\" }, \"source_repos\": [] }"}' - eng/common/scripts/Queue-Pipeline.ps1 ` - -Organization "apidrop" ` - -Project "Content%20CI" ` - -DefinitionId 3452 ` - -BuildParametersJson $buildParamJson ` - -BearerToken $accessToken + # Updating main is the default reason to kick off manual builds of the docIndex run. + # Manual builds of docIndex are typically done to update Main when certain updates are + # made. These updates include new libraries and CSV updates in azure-sdk which, depending + # on the column(s) updated can cause anything from package deprecation to service level + # readme or ToC updates. + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), parameters.UpdateMain) }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + displayName: Verify Required Docs Json Members + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Move deprecated packages to legacy moniker + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding for main branch + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Generate Service Level Readme for main branch + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsToc.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml + displayName: Generate ToC for main branch + + # Push changes to docs repo + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DefaultBranch) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + + # The scenario for running a Manual build is normally only for the main updates. The daily build + # should really only get kicked off for scheduled runs. + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), parameters.ForceDailyUpdate) }}: + # Prepare daily docs CI + - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml + parameters: + DailyBranchVariableName: DailyDocsBranchName + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout daily branch if it exists + workingDirectory: $(DocRepoLocation) + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + displayName: Verify Required Docs Json Members + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding for Daily docs + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Generate Service Level Readme for Daily docs + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsToc.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml + displayName: Generate ToC for Daily docs + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DailyDocsBranchName) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + PushArgs: -f + + - task: AzureCLI@2 + displayName: Queue Docs CI build + inputs: + azureSubscription: msdocs-apidrop-connection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv + $buildParamJson = '{"params":"{ \"target_repo\": { \"url\": \"https://github.com/MicrosoftDocs/azure-docs-sdk-node\", \"branch\": \"$(DailyDocsBranchName)\", \"folder\": \"./\" }, \"source_repos\": [] }"}' + eng/common/scripts/Queue-Pipeline.ps1 ` + -Organization "apidrop" ` + -Project "Content%20CI" ` + -DefinitionId 3452 ` + -BuildParametersJson $buildParamJson ` + -BearerToken $accessToken