Enable consuming public .NET SDK builds from the VMR using DarcLib

eng/pipelines/jobs/update-tools.yml

@@ -8,20 +8,14 @@ jobs:
     pool:
       vmImage: $(defaultLinuxAmd64PoolImage)
     steps:
-    - powershell: |
-        Import-Module -force $(engPath)/DependencyManagement.psm1
-
-        $args=("9.0" + `
-            " --tool ${{ tool }}" + `
-            " --version-source-name ${{ tool }}" + `
-            " --source-branch nightly" + `
-            " --org dnceng" + `
-            " --project $(System.TeamProject)" + `
-            " --repo $(Build.Repository.Name)" + `
-            " --target-branch $(Get-Branch)")
-
-        echo "##vso[task.setvariable variable=customArgsArray]$($args | ConvertTo-Json -Compress -AsArray)"
-      displayName: Set up args
     - template: /eng/pipelines/steps/update-dependencies.yml
       parameters:
-        customArgsArray: "$(customArgsArray)"
+        args: >
+          specific 9.0
+          --tool ${{ tool }}
+          --version-source-name ${{ tool }}
+          --source-branch nightly
+          --target-branch nightly
+          --org dnceng
+          --project $(System.TeamProject)
+          --repo $(Build.Repository.Name)

eng/pipelines/steps/update-dependencies-specific.yml

@@ -0,0 +1,39 @@
+parameters:
+  # The customArgsArray parameter is used to specify the configuration for multiple Dockerfile versions.
+  # This allows for a single branch to be generated for different internal .NET build versions.
+  customArgsArray: ""
+
+  useInternalBuild: false
+
+steps:
+- script: docker build -t update-dependencies -f $(engPath)/update-dependencies/Dockerfile --pull .
+  displayName: Build Update Dependencies Tool
+- script: docker run --name update-dependencies -d -t --entrypoint /bin/sh -v /var/run/docker.sock:/var/run/docker.sock update-dependencies
+  displayName: Create Update Dependencies container
+- powershell: |
+    if ("${{ parameters.useInternalBuild }}" -eq "true") {
+      $pat="$(dn-bot-devdiv-dnceng-rw-code-pat)"
+    } else {
+      $pat="$(BotAccount-dotnet-docker-bot-PAT)"
+    }
+
+    $credArgs = "--user $(dotnetDockerBot.userName) --email $(dotnetDockerBot.email) --password $pat"
+
+    # Execute update-deps for each of the items in the customArgsArray, representing different Dockerfile versions
+    # Ensure that the value is treated as an array by wrapping it in an array literal. This deals with the quirk of
+    # how PowerShell treats a single item as a string instead of an array.
+    $customArgsArray = @('${{ parameters.customArgsArray }}' | ConvertFrom-Json)
+    foreach ($customArgs in $customArgsArray) {
+      # If this is the last iteration, include the credentials to cause a PR to be generated.
+      # For internal builds this will cause branch to be pushed to AzDO, but no PRs will be generated.
+      if ($customArgs -eq $customArgsArray[-1]) {
+        $customArgs += " $credArgs"
+      }
+
+      $command = "docker exec update-dependencies update-dependencies $customArgs"
+      Invoke-Expression $command
+    }
+  displayName: Run Update Dependencies
+- script: docker rm -f update-dependencies
+  displayName: Remove Update Dependencies container
+  condition: always()

eng/pipelines/steps/update-dependencies.yml

@@ -1,39 +1,34 @@
 parameters:
-  # The customArgsArray parameter is used to specify the configuration for multiple Dockerfile versions.
-  # This allows for a single branch to be generated for different internal .NET build versions.
-  customArgsArray: ""
-
-  useInternalBuild: false
+  serviceConnection: ""
+  args: ""
 
 steps:
 - script: docker build -t update-dependencies -f $(engPath)/update-dependencies/Dockerfile --pull .
-  displayName: Build Update Dependencies Tool
+  displayName: Build Update Dependencies image
 - script: docker run --name update-dependencies -d -t --entrypoint /bin/sh -v /var/run/docker.sock:/var/run/docker.sock update-dependencies
-  displayName: Create Update Dependencies container
-- powershell: |
-    if ("${{ parameters.useInternalBuild }}" -eq "true") {
-      $pat="$(dn-bot-devdiv-dnceng-rw-code-pat)"
-    } else {
-      $pat="$(BotAccount-dotnet-docker-bot-PAT)"
-    }
+  displayName: Start Update Dependencies container
 
-    $credArgs = "--user $(dotnetDockerBot.userName) --email $(dotnetDockerBot.email) --password $pat"
+- ${{ if ne(parameters.serviceConnection, '') }}:
+  - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self
+    parameters:
+      displayName: Az login
+      serviceConnection: ${{ parameters.serviceConnection }}
+      command: |
+        $azLogin = "az login --service-principal --tenant $env:tenantId --username $env:servicePrincipalId --federated-token $env:idToken";
+        $loginCommand = "docker exec update-dependencies $azLogin";
+        Write-Host "Executing $loginCommand";
+        Invoke-Expression $loginCommand;
 
-    # Execute update-deps for each of the items in the customArgsArray, representing different Dockerfile versions
-    # Ensure that the value is treated as an array by wrapping it in an array literal. This deals with the quirk of
-    # how PowerShell treats a single item as a string instead of an array.
-    $customArgsArray = @('${{ parameters.customArgsArray }}' | ConvertFrom-Json)
-    foreach ($customArgs in $customArgsArray) {
-      # If this is the last iteration, include the credentials to cause a PR to be generated.
-      # For internal builds this will cause branch to be pushed to AzDO, but no PRs will be generated.
-      if ($customArgs -eq $customArgsArray[-1]) {
-        $customArgs += " $credArgs"
-      }
+- powershell: |
+    $args = "${{ parameters.args }}"
+    $pat="$(BotAccount-dotnet-docker-bot-PAT)"
+    $credArgs = "--user $(dotnetDockerBot.userName) --email $(dotnetDockerBot.email) --password $pat"
+    $args += " $credArgs"
 
-      $command = "docker exec update-dependencies update-dependencies $customArgs"
-      Invoke-Expression $command
-    }
+    $command = "docker exec update-dependencies update-dependencies $args"
+    Invoke-Expression $command
   displayName: Run Update Dependencies
+
 - script: docker rm -f update-dependencies
   displayName: Remove Update Dependencies container
   condition: always()

eng/pipelines/steps/update-dotnet-dependencies-specific.yml

@@ -0,0 +1,89 @@
+parameters:
+  useInternalBuild: false
+
+  # Comma-delimited list of SDK versions to target (overrides the use of channel var to determine latest version)
+  sdkVersions: ""
+
+  buildId: ""
+
+steps:
+- powershell: |
+    if ("${{ parameters.sdkVersions }}" -ne "") {
+      $args = @{
+        SdkVersions = "${{ parameters.sdkVersions }}" -split ","
+      }
+    }
+    elseif ("${{ parameters.useInternalBuild }}" -eq "false") {
+      $args = @{
+        Channel = "$(channel)"
+      }
+    }
+    else {
+      $args = @{
+        BuildId = "${{ parameters.buildId }}"
+      }
+    }
+
+    if ("${{ parameters.useInternalBuild }}" -eq "true") {
+      $args["UseInternalBuild"] = $true
+      $args["InternalAccessToken"] = '$(System.AccessToken)'
+      $args["AzdoVersionsRepoInfoAccessToken"] = "$(dn-bot-devdiv-dnceng-rw-code-pat)"
+    }
+
+    $(engPath)/Get-DropVersions.ps1 @args
+  displayName: Get Versions
+- powershell: |
+    Import-Module -force $(engPath)/DependencyManagement.psm1
+
+    $versionInfos = '$(versionInfos)' | ConvertFrom-Json
+
+    $index=0
+    foreach ($versionInfo in $versionInfos) {
+      $args = @{
+        ProductVersion = $versionInfo.DockerfileVersion
+        RuntimeVersion = $versionInfo.RuntimeVersion
+        AspnetVersion = $versionInfo.AspnetVersion
+        SdkVersion = $versionInfo.SdkVersion
+        ComputeShas = $true
+        AzdoVariableName = "updateDepsArgs-$index"
+        UseStableBranding = $versionInfo.StableBranding
+      }
+
+      if ("${{ parameters.useInternalBuild }}" -eq "true") {
+        $args["InternalAccessToken"] = '$(System.AccessToken)'
+        $args["InternalBaseUrl"] = '"$(internalBaseUrl)"'
+      } else {
+        $args["ReleaseState"] = $(Get-ProductReleaseState)
+      }
+
+      Write-Host "Executing Set-DotnetVersions.ps1 for $($versionInfo.DockerfileVersion)"
+      $(engPath)/Set-DotnetVersions.ps1 @args
+      $index++
+    }
+  displayName: Get update-dependencies args
+- powershell: |
+    Import-Module -force $(engPath)/DependencyManagement.psm1
+
+    $targetBranch = $(Get-Branch)
+    if ("${{ parameters.useInternalBuild }}" -eq "true") {
+      $targetBranch = "staging-${{ parameters.buildId }}-pipeline-$(Build.BuildId)"
+    }
+
+    $customArgsArray = @()
+    $index=0
+
+    # Grab the variables that were set by the multiple calls to Set-DotnetVersions.ps1 and
+    # add them to an array. This allows us to pass args for multiple Dockerfile versions.
+    while ([Environment]::GetEnvironmentVariable("UPDATEDEPSARGS-$index") -ne $null) {
+      $updateDepsArgs = [Environment]::GetEnvironmentVariable("UPDATEDEPSARGS-$index")
+      $updateDepsArgs = "$updateDepsArgs --org dnceng --project $(System.TeamProject) --repo $(Build.Repository.Name) --target-branch $targetBranch"
+      $customArgsArray += $updateDepsArgs
+      $index++
+    }
+
+    echo "##vso[task.setvariable variable=customArgsArray]$($customArgsArray | ConvertTo-Json -Compress -AsArray)"
+  displayName: Set Custom Args
+- template: update-dependencies-specific.yml
+  parameters:
+    customArgsArray: "$(customArgsArray)"
+    useInternalBuild: ${{ parameters.useInternalBuild }}

eng/pipelines/steps/update-dotnet-dependencies.yml

@@ -1,89 +1,12 @@
 parameters:
-  useInternalBuild: false
-
-  # Comma-delimited list of SDK versions to target (overrides the use of channel var to determine latest version)
-  sdkVersions: ""
-
-  buildId: ""
+  serviceConnection: ""
+  channel: ""
+  repo: "https://github.com/dotnet/dotnet"
+  versionSourceName: ""
 
 steps:
-- powershell: |
-    if ("${{ parameters.sdkVersions }}" -ne "") {
-      $args = @{
-        SdkVersions = "${{ parameters.sdkVersions }}" -split ","
-      }
-    }
-    elseif ("${{ parameters.useInternalBuild }}" -eq "false") {
-      $args = @{
-        Channel = "$(channel)"
-      }
-    }
-    else {
-      $args = @{
-        BuildId = "${{ parameters.buildId }}"
-      }
-    }
-
-    if ("${{ parameters.useInternalBuild }}" -eq "true") {
-      $args["UseInternalBuild"] = $true
-      $args["InternalAccessToken"] = '$(System.AccessToken)'
-      $args["AzdoVersionsRepoInfoAccessToken"] = "$(dn-bot-devdiv-dnceng-rw-code-pat)"
-    }
-
-    $(engPath)/Get-DropVersions.ps1 @args
-  displayName: Get Versions
-- powershell: |
-    Import-Module -force $(engPath)/DependencyManagement.psm1
-
-    $versionInfos = '$(versionInfos)' | ConvertFrom-Json
-
-    $index=0
-    foreach ($versionInfo in $versionInfos) {
-      $args = @{
-        ProductVersion = $versionInfo.DockerfileVersion
-        RuntimeVersion = $versionInfo.RuntimeVersion
-        AspnetVersion = $versionInfo.AspnetVersion
-        SdkVersion = $versionInfo.SdkVersion
-        ComputeShas = $true
-        AzdoVariableName = "updateDepsArgs-$index"
-        UseStableBranding = $versionInfo.StableBranding
-      }
-
-      if ("${{ parameters.useInternalBuild }}" -eq "true") {
-        $args["InternalAccessToken"] = '$(System.AccessToken)'
-        $args["InternalBaseUrl"] = '"$(internalBaseUrl)"'
-      } else {
-        $args["ReleaseState"] = $(Get-ProductReleaseState)
-      }
-
-      Write-Host "Executing Set-DotnetVersions.ps1 for $($versionInfo.DockerfileVersion)"
-      $(engPath)/Set-DotnetVersions.ps1 @args
-      $index++
-    }
-  displayName: Get update-dependencies args
-- powershell: |
-    Import-Module -force $(engPath)/DependencyManagement.psm1
-
-    $targetBranch = $(Get-Branch)
-    if ("${{ parameters.useInternalBuild }}" -eq "true") {
-      $targetBranch = "staging-${{ parameters.buildId }}-pipeline-$(Build.BuildId)"
-    }
-
-    $customArgsArray = @()
-    $index=0
-
-    # Grab the variables that were set by the multiple calls to Set-DotnetVersions.ps1 and
-    # add them to an array. This allows us to pass args for multiple Dockerfile versions.
-    while ([Environment]::GetEnvironmentVariable("UPDATEDEPSARGS-$index") -ne $null) {
-      $updateDepsArgs = [Environment]::GetEnvironmentVariable("UPDATEDEPSARGS-$index")
-      $updateDepsArgs = "$updateDepsArgs --org dnceng --project $(System.TeamProject) --repo $(Build.Repository.Name) --target-branch $targetBranch"
-      $customArgsArray += $updateDepsArgs
-      $index++
-    }
-
-    echo "##vso[task.setvariable variable=customArgsArray]$($customArgsArray | ConvertTo-Json -Compress -AsArray)"
-  displayName: Set Custom Args
 - template: update-dependencies.yml
   parameters:
-    customArgsArray: "$(customArgsArray)"
-    useInternalBuild: ${{ parameters.useInternalBuild }}
+    args: "from-channel ${{ parameters.channel }} ${{ parameters.repo }} --version-source-name '${{ parameters.versionSourceName }}'"
+    useInternalBuild: false
+    serviceConnection: ${{ parameters.serviceConnection }}

eng/pipelines/update-dependencies-internal.yml

@@ -12,7 +12,7 @@ stages:
     pool:
       vmImage: $(defaultLinuxAmd64PoolImage)
     steps:
-    - template: steps/update-dotnet-dependencies.yml
+    - template: steps/update-dotnet-dependencies-specific.yml
       parameters:
         useInternalBuild: true
         buildId: $(buildId)

eng/pipelines/update-dependencies.yml

@@ -20,6 +20,11 @@ stages:
       vmImage: $(defaultLinuxAmd64PoolImage)
     steps:
     - template: steps/update-dotnet-dependencies.yml
+      parameters:
+        channel: "${{ variables['channel'] }}"
+        repo: "https://github.com/dotnet/dotnet"
+        versionSourceName: "dotnet/dotnet"
+        serviceConnection: "Darc: Maestro Production"
   - template: jobs/update-tools.yml
     parameters:
       tools: ["chisel", "rocks-toolbox", "syft", "mingit"]