[main] Update common Docker engineering infrastructure with latest

eng/docker-tools/templates/jobs/build-images.yml

@@ -4,9 +4,14 @@ parameters:
   matrix: {}
   dockerClientOS: null
   buildJobTimeout: 60
-  commonInitStepsForMatrixAndBuild: []
+  # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source).
+  # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull.
   customInitSteps: []
+  # Custom steps that run after ImageBuilder is set up but before the build starts.
+  # Use for build-specific initialization (e.g., setting variables, additional setup).
+  customBuildInitSteps: []
   publishConfig: null
+  versionsRepoRef: ""
   noCache: false
   internalProjectName: null
   publicProjectName: null
@@ -29,12 +34,14 @@ jobs:
     imageInfoHostDir: $(Build.ArtifactStagingDirectory)/imageInfo
     imageInfoContainerDir: $(artifactsPath)/imageInfo
   steps:
-  - ${{ parameters.commonInitStepsForMatrixAndBuild }}
-  - template: /eng/docker-tools/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self
+  - template: /eng/docker-tools/templates/steps/init-common.yml@self
     parameters:
+      dockerClientOS: ${{ parameters.dockerClientOS }}
       publishConfig: ${{ parameters.publishConfig }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
       cleanupDocker: true
-  - ${{ parameters.customInitSteps }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+  - ${{ parameters.customBuildInitSteps }}
   - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self
     parameters:
       publicSourceBranch: $(publicSourceBranch)
@@ -115,6 +122,7 @@ jobs:
     - template: /eng/docker-tools/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self
       parameters:
         condition: ne(variables.testScriptPath, '')
+        skipCommonInit: true
   - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}:
     - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self
       parameters:

eng/docker-tools/templates/jobs/copy-base-images-staging.yml

@@ -8,15 +8,24 @@ parameters:
 - name: publishConfig
   type: object
   default: null
+# Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source).
+# Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull.
 - name: customInitSteps
   type: stepList
   default: []
+# Custom steps that run after ImageBuilder is set up but before copy-base-images runs.
+- name: customCopyBaseImagesInitSteps
+  type: stepList
+  default: []
 - name: additionalOptions
   type: string
   default: ''
 - name: continueOnError
   type: string
   default: false
+- name: versionsRepoRef
+  type: string
+  default: ""
 
 jobs:
 - template: /eng/docker-tools/templates/jobs/copy-base-images.yml@self
@@ -25,5 +34,7 @@ jobs:
     pool: ${{ parameters.pool }}
     publishConfig: ${{ parameters.publishConfig }}
     customInitSteps: ${{ parameters.customInitSteps }}
+    customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }}
     additionalOptions: ${{ parameters.additionalOptions }}
     acr: ${{ parameters.publishConfig.InternalMirrorRegistry }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}

eng/docker-tools/templates/jobs/copy-base-images.yml

@@ -11,9 +11,15 @@ parameters:
 - name: acr
   type: object
   default: null
+# Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source).
+# Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull.
 - name: customInitSteps
   type: stepList
   default: []
+# Custom steps that run after ImageBuilder is set up but before copy-base-images runs.
+- name: customCopyBaseImagesInitSteps
+  type: stepList
+  default: []
 - name: additionalOptions
   type: string
   default: ''
@@ -23,15 +29,21 @@ parameters:
 - name: forceDryRun
   type: boolean
   default: false
+- name: versionsRepoRef
+  type: string
+  default: ""
 
 jobs:
 - job: ${{ parameters.name }}
   pool: ${{ parameters.pool }}
   steps:
-  - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self
+  - template: /eng/docker-tools/templates/steps/init-common.yml@self
     parameters:
+      dockerClientOS: linux
       publishConfig: ${{ parameters.publishConfig }}
-  - ${{ parameters.customInitSteps }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+  - ${{ parameters.customCopyBaseImagesInitSteps }}
   - template: /eng/docker-tools/templates/steps/copy-base-images.yml@self
     parameters:
       acr: ${{ parameters.acr }}

eng/docker-tools/templates/jobs/generate-matrix.yml

@@ -7,18 +7,26 @@ parameters:
   internalProjectName: null
   noCache: false
   publishConfig: null
+  # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source).
+  # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull.
   customInitSteps: []
-  commonInitStepsForMatrixAndBuild: []
+  # Custom steps that run after ImageBuilder is set up but before matrix generation runs.
+  customGenerateMatrixInitSteps: []
+  versionsRepoRef: ""
   sourceBuildPipelineRunId: ""
 
 jobs:
 - job: ${{ parameters.name }}
   pool: ${{ parameters.pool }}
   steps:
-  - ${{ parameters.commonInitStepsForMatrixAndBuild }}
+  - template: /eng/docker-tools/templates/steps/init-common.yml@self
+    parameters:
+      dockerClientOS: linux
+      publishConfig: ${{ parameters.publishConfig }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+  - ${{ parameters.customGenerateMatrixInitSteps }}
   - template: /eng/docker-tools/templates/steps/retain-build.yml@self
-  - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self
-  - ${{ parameters.customInitSteps }}
   - template: /eng/docker-tools/templates/steps/validate-branch.yml@self
     parameters:
       publishConfig: ${{ parameters.publishConfig }}

eng/docker-tools/templates/jobs/post-build.yml

@@ -2,6 +2,7 @@ parameters:
   pool: {}
   internalProjectName: null
   publicProjectName: null
+  customInitSteps: []
 
 jobs:
 - job: Build
@@ -13,7 +14,10 @@ jobs:
     imageInfosOutputSubDir: "/output"
     sbomOutputDir: "$(Build.ArtifactStagingDirectory)/sbom"
   steps:
-  - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self
+  - template: /eng/docker-tools/templates/steps/init-common.yml@self
+    parameters:
+      dockerClientOS: linux
+      customInitSteps: ${{ parameters.customInitSteps }}
   - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self
     parameters:
       targetPath: $(Build.ArtifactStagingDirectory)

eng/docker-tools/templates/jobs/publish.yml

@@ -3,6 +3,7 @@ parameters:
   internalProjectName: null
   publishConfig: null
   customInitSteps: []
+  customPostInitSteps: []
   customPublishVariables: []
   sourceBuildPipelineDefinitionId: ""
   sourceBuildPipelineRunId: ""
@@ -45,25 +46,21 @@ jobs:
   - ${{ parameters.customPublishVariables }}
 
   steps:
-  - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
+  - template: /eng/docker-tools/templates/steps/init-common.yml@self
     parameters:
+      dockerClientOS: linux
       publishConfig: ${{ parameters.publishConfig }}
       versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
 
   - template: /eng/docker-tools/templates/steps/retain-build.yml@self
 
-  - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self
-    parameters:
-      publishConfig: ${{ parameters.publishConfig }}
-
   - pwsh: |
       $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI
       echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName"
-      $versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}"
-      echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot"
     displayName: Set Publish Variables
 
-  - ${{ parameters.customInitSteps }}
+  - ${{ parameters.customPostInitSteps }}
 
   - template: /eng/docker-tools/templates/steps/validate-branch.yml@self
     parameters:

eng/docker-tools/templates/stages/build-and-test.yml

@@ -5,6 +5,11 @@ parameters:
   testMatrixCustomBuildLegGroupArgs: ""
   customCopyBaseImagesInitSteps: []
   customGenerateMatrixInitSteps: []
+  # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source).
+  # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull.
+  customInitSteps: []
+  # Custom steps that run after ImageBuilder is set up but before the build starts.
+  # Use for build-specific initialization (e.g., setting variables, additional setup).
   customBuildInitSteps: []
   customTestInitSteps: []
   sourceBuildPipelineRunId: ""
@@ -16,7 +21,6 @@ parameters:
   linuxAmdTestJobTimeout: 60
   linuxArmTestJobTimeout: 60
   windowsAmdTestJobTimeout: 60
-
   noCache: false
   publishConfig: null
 
@@ -78,7 +82,9 @@ stages:
       publishConfig: ${{ parameters.publishConfig }}
       pool: ${{ parameters.linuxAmd64Pool }}
       additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)"
-      customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
   - template: /eng/docker-tools/templates/jobs/generate-matrix.yml@self
     parameters:
@@ -89,12 +95,9 @@ stages:
       internalProjectName: ${{ parameters.internalProjectName }}
       noCache: ${{ parameters.noCache }}
       publishConfig: ${{ parameters.publishConfig }}
-      customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
-      commonInitStepsForMatrixAndBuild:
-      - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
-        parameters:
-          publishConfig: ${{ parameters.publishConfig }}
-          versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
   - template: /eng/docker-tools/templates/jobs/build-images.yml@self
     parameters:
@@ -103,12 +106,9 @@ stages:
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxAmd64']
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }}
-      commonInitStepsForMatrixAndBuild:
-      - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
-        parameters:
-          publishConfig: ${{ parameters.publishConfig }}
-          versionsRepoRef: ${{ parameters.versionsRepoRef }}
-      customInitSteps: ${{ parameters.customBuildInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      customBuildInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -121,12 +121,9 @@ stages:
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm64']
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
-      commonInitStepsForMatrixAndBuild:
-      - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
-        parameters:
-          publishConfig: ${{ parameters.publishConfig }}
-          versionsRepoRef: ${{ parameters.versionsRepoRef }}
-      customInitSteps: ${{ parameters.customBuildInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      customBuildInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -139,12 +136,9 @@ stages:
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm32']
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
-      commonInitStepsForMatrixAndBuild:
-      - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
-        parameters:
-          publishConfig: ${{ parameters.publishConfig }}
-          versionsRepoRef: ${{ parameters.versionsRepoRef }}
-      customInitSteps: ${{ parameters.customBuildInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      customBuildInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -157,12 +151,9 @@ stages:
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.Windows1809Amd64']
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
-      commonInitStepsForMatrixAndBuild:
-      - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
-        parameters:
-          publishConfig: ${{ parameters.publishConfig }}
-          versionsRepoRef: ${{ parameters.versionsRepoRef }}
-      customInitSteps: ${{ parameters.customBuildInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      customBuildInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -175,12 +166,9 @@ stages:
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2022Amd64']
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
-      commonInitStepsForMatrixAndBuild:
-      - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
-        parameters:
-          publishConfig: ${{ parameters.publishConfig }}
-          versionsRepoRef: ${{ parameters.versionsRepoRef }}
-      customInitSteps: ${{ parameters.customBuildInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      customBuildInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -193,17 +181,13 @@ stages:
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2025Amd64']
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
-      commonInitStepsForMatrixAndBuild:
-      - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
-        parameters:
-          publishConfig: ${{ parameters.publishConfig }}
-          versionsRepoRef: ${{ parameters.versionsRepoRef }}
-      customInitSteps: ${{ parameters.customBuildInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      customBuildInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      versionsRepoRef: ${{ parameters.versionsRepoRef }}
       storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }}
   - template: /eng/docker-tools/templates/jobs/build-images.yml@self
     parameters:
@@ -212,12 +196,9 @@ stages:
       matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2016Amd64']
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
-      commonInitStepsForMatrixAndBuild:
-      - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
-        parameters:
-          publishConfig: ${{ parameters.publishConfig }}
-          versionsRepoRef: ${{ parameters.versionsRepoRef }}
-      customInitSteps: ${{ parameters.customBuildInitSteps }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      customInitSteps: ${{ parameters.customInitSteps }}
+      customBuildInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -236,6 +217,7 @@ stages:
       pool: ${{ parameters.linuxAmd64Pool }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
+      customInitSteps: ${{ parameters.customInitSteps }}
 
 ################################################################################
 # Test Images
@@ -264,13 +246,10 @@ stages:
         isTestStage: true
         internalProjectName: ${{ parameters.internalProjectName }}
         publicProjectName: ${{ parameters.publicProjectName }}
-        customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
+        customInitSteps: ${{ parameters.customInitSteps }}
+        customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
         sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
-        commonInitStepsForMatrixAndBuild:
-        - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self
-          parameters:
-            versionsRepoRef: ${{ parameters.versionsRepoRef }}
-            publishConfig: ${{ parameters.publishConfig }}
+        versionsRepoRef: ${{ parameters.versionsRepoRef }}
     - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self
       parameters:
         name: Linux_amd64