Support matrix trimming in pipeline

eng/common/templates/jobs/build-images.yml

@@ -4,12 +4,11 @@ parameters:
   matrix: {}
   dockerClientOS: null
   buildJobTimeout: 60
+  commonInitStepsForMatrixAndBuild: []
   customInitSteps: []
   noCache: false
   internalProjectName: null
   publicProjectName: null
-  internalVersionsRepoRef: null
-  publicVersionsRepoRef: null
 
 jobs:
 - job: ${{ parameters.name }}
@@ -24,59 +23,11 @@ jobs:
   timeoutInMinutes: ${{ parameters.buildJobTimeout }}
   variables:
     imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions)
-    versionsRepoPath: versions
     sbomDirectory: $(Build.ArtifactStagingDirectory)/sbom
     imageInfoHostDir: $(Build.ArtifactStagingDirectory)/imageInfo
     imageInfoContainerDir: $(artifactsPath)/imageInfo
-    ${{ if eq(parameters.noCache, false) }}:
-      versionsBasePath: $(versionsRepoPath)/
-      pipelineDisabledCache: false
-    ${{ if eq(parameters.noCache, true) }}:
-      versionsBasePath: ""
-      pipelineDisabledCache: true
   steps:
-  - checkout: self
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.publicProjectName), eq(parameters.noCache, false)) }}:
-    - checkout: ${{ parameters.publicVersionsRepoRef }}
-      path: s/$(versionsRepoPath)
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), eq(parameters.noCache, false)) }}:
-    - checkout: ${{ parameters.internalVersionsRepoRef }}
-      path: s/$(versionsRepoPath)
-  - ${{ if eq(parameters.noCache, false) }}:
-    - powershell: |
-        $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/")
-        if ($pathSeparatorIndex -ge 0) {
-          $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1)
-        }
-        else {
-          $buildRepoName = "$(Build.Repository.Name)"
-        }
-
-        $engCommonPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engCommonRelativePath)"
-        $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng"
-        $manifest = "$buildRepoName/$(manifest)"
-        $testResultsDirectory = "$buildRepoName/$testResultsDirectory"
-
-        if ("$(testScriptPath)") {
-          $testScriptPath = "$buildRepoName/$(testScriptPath)"
-        }
-
-        echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName"
-        echo "##vso[task.setvariable variable=manifest]$manifest"
-        echo "##vso[task.setvariable variable=engCommonPath]$engCommonPath"
-        echo "##vso[task.setvariable variable=engPath]$engPath"
-        echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath"
-        echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory"
-      displayName: Override Common Paths
-  - powershell: |
-      if ("${{ parameters.noCache }}" -eq "false") {
-        $baseContainerRepoPath = "/repo/$(buildRepoName)"
-      }
-      else {
-        $baseContainerRepoPath = "/repo"
-      }
-      echo "##vso[task.setvariable variable=baseContainerRepoPath]$baseContainerRepoPath"
-    displayName: Set Base Container Repo Path
+  - ${{ parameters.commonInitStepsForMatrixAndBuild }}
   - template: /eng/common/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self
     parameters:
       cleanupDocker: true
@@ -96,13 +47,9 @@ jobs:
       # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since
       # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
       # the environment variable for us.
-      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json"
+      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
       if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
-        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr-staging.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push"
-      }
-
-      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.publicProjectName }}" -and ${env:PUBLIC-MIRROR_SERVER} -ne "") {
-        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'"
+        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push"
       }
 
       # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
@@ -128,7 +75,6 @@ jobs:
         --os-type $(osType)
         --architecture $(architecture)
         --retry
-        --source-repo $(publicGitRepoUri)
         --digests-out-var 'builtImages'
         --acr-subscription '$(acr-staging.subscription)'
         --acr-resource-group '$(acr-staging.resourceGroup)'

eng/common/templates/jobs/generate-matrix.yml

@@ -5,28 +5,43 @@ parameters:
   customBuildLegGroupArgs: ""
   isTestStage: false
   internalProjectName: null
+  noCache: false
+  commonInitStepsForMatrixAndBuild: []
 
 jobs:
 - job: ${{ parameters.name }}
   pool: ${{ parameters.pool }}
   steps:
+  - ${{ parameters.commonInitStepsForMatrixAndBuild }}
   - template: /eng/common/templates/steps/retain-build.yml@self
   - template: /eng/common/templates/steps/init-docker-linux.yml@self
   - template: /eng/common/templates/steps/validate-branch.yml@self
     parameters:
       internalProjectName: ${{ parameters.internalProjectName }}
+  - template: /eng/common/templates/steps/set-image-info-path-var.yml
+    parameters:
+      publicSourceBranch: $(publicSourceBranch)
   - ${{ if eq(parameters.isTestStage, true) }}:
     - template: /eng/common/templates/steps/download-build-artifact.yml@self
       parameters:
         targetPath: $(Build.ArtifactStagingDirectory)
         artifactName: image-info
-    - script: echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]--image-info $(artifactsPath)/image-info.json"
-      displayName: Set GenerateBuildMatrix Variables
-  - ${{ if eq(parameters.isTestStage, false) }}:
-    - script: echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]"
-      displayName: Set GenerateBuildMatrix Variables
+  - powershell: |
+      $additionalGenerateBuildMatrixOptions = "$(additionalGenerateBuildMatrixOptions)"
+
+      if ("${{ parameters.isTestStage}}" -eq "true") {
+        $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json"
+      }
+      elseif ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") {
+        # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
+        $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images"
+      }
+
+      echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]$additionalGenerateBuildMatrixOptions"
+    displayName: Set GenerateBuildMatrix Variables
   - script: >
-      $(runImageBuilderCmd) generateBuildMatrix
+      echo "##vso[task.setvariable variable=generateBuildMatrixCommand]
+      generateBuildMatrix
       --manifest $(manifest)
       --type ${{ parameters.matrixType }}
       --os-type '*'
@@ -35,6 +50,18 @@ jobs:
       ${{ parameters.customBuildLegGroupArgs }}
       $(imageBuilder.pathArgs)
       $(manifestVariables)
-      $(additionalGenerateBuildMatrixOptions)
-    displayName: Generate ${{ parameters.matrixType }} Matrix
-    name: matrix
+      $(commonMatrixAndBuildOptions)
+      $(additionalGenerateBuildMatrixOptions)"
+    displayName: Set GenerateBuildMatrix Command
+  - ${{ if eq(parameters.isTestStage, false) }}:
+    - template: /eng/common/templates/steps/run-imagebuilder.yml@self
+      parameters:
+        name: matrix
+        displayName: Generate ${{ parameters.matrixType }} Matrix
+        serviceConnection: $(build.serviceConnectionName)
+        internalProjectName: internal
+        args: $(generateBuildMatrixCommand)
+  - ${{ else }}:
+    - script: $(generateBuildMatrixCommand)
+      displayName: Generate ${{ parameters.matrixType }} Matrix
+      name: matrix

eng/common/templates/jobs/post-build.yml

@@ -52,22 +52,32 @@ jobs:
           }
     displayName: Prune Publish Artifacts
   - powershell: |
+      $imageInfoFiles = Get-ChildItem "$(imageInfosHostDir)"
+      if ($imageInfoFiles.Count -eq 0) {
+        echo "No image info files found."
+        echo "##vso[task.setvariable variable=noImageInfos;isOutput=true]true"
+        exit 0
+      }
+
       New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force
       $(runImageBuilderCmd) mergeImageInfo `
         --manifest $(manifest) `
         $(imageInfosContainerDir) `
         $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json `
         $(manifestVariables)
+    name: MergeImageInfoFiles
     displayName: Merge Image Info Files
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
+      condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true'))
       path: $(sbomOutputDir)
       artifactName: sboms
       displayName: Publish SBOM Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
+      condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true'))
       path: $(imageInfosHostDir)$(imageInfosOutputSubDir)
       artifactName: image-info
       displayName: Publish Image Info File Artifact

eng/common/templates/stages/build-test-publish-repo.yml

@@ -79,80 +79,111 @@ stages:
       pool: ${{ parameters.linuxAmd64Pool }}
       customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }}
       internalProjectName: ${{ parameters.internalProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+      noCache: ${{ parameters.noCache }}
+      commonInitStepsForMatrixAndBuild:
+      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        parameters:
+          noCache: ${{ parameters.noCache }}
+          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
+          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Linux_amd64
       pool: ${{ parameters.linuxAmd64Pool }}
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxAmd64']
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }}
+      commonInitStepsForMatrixAndBuild:
+      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        parameters:
+          noCache: ${{ parameters.noCache }}
+          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
+          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Linux_arm64
       pool: ${{ parameters.linuxArm64Pool }}
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm64']
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
+      commonInitStepsForMatrixAndBuild:
+      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        parameters:
+          noCache: ${{ parameters.noCache }}
+          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
+          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Linux_arm32
       pool: ${{ parameters.linuxArm32Pool }}
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm32']
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
+      commonInitStepsForMatrixAndBuild:
+      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        parameters:
+          noCache: ${{ parameters.noCache }}
+          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
+          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Windows1809_amd64
       pool: ${{ parameters.windows1809Pool }}
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.Windows1809Amd64']
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
+      commonInitStepsForMatrixAndBuild:
+      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        parameters:
+          noCache: ${{ parameters.noCache }}
+          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
+          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Windows2022_amd64
       pool: ${{ parameters.windows2022Pool }}
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2022Amd64']
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
+      commonInitStepsForMatrixAndBuild:
+      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        parameters:
+          noCache: ${{ parameters.noCache }}
+          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
+          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Windows2025_amd64
       pool: ${{ parameters.windows2025Pool }}
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2025Amd64']
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
+      commonInitStepsForMatrixAndBuild:
+      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        parameters:
+          noCache: ${{ parameters.noCache }}
+          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
+          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -166,12 +197,16 @@ stages:
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2016Amd64']
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
+      commonInitStepsForMatrixAndBuild:
+      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        parameters:
+          noCache: ${{ parameters.noCache }}
+          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
+          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
 
 ################################################################################
 # Post-Build
@@ -194,6 +229,7 @@ stages:
     dependsOn: Post_Build
     condition: "
       and(
+        ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'),
         ne(variables['testScriptPath'], ''),
         and(
           contains(variables['stages'], 'test'),
@@ -280,6 +316,7 @@ stages:
   condition: "
     and(
       not(canceled()),
+      ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'),
       and(
         contains(variables['stages'], 'publish'),
         or(