diff --git a/diagnostics-codeql.yml b/diagnostics-codeql.yml deleted file mode 100644 index 5cacd47b62..0000000000 --- a/diagnostics-codeql.yml +++ /dev/null @@ -1,155 +0,0 @@ -parameters: -- name: runtimeFeed - displayName: Feed for runtime installation - type: string - default: default - values: - - default -- name: runtimeFeedToken - displayName: Base 64 SAS Token for runtime installation - type: string - default: default - values: - - default - -trigger: - none - -schedules: - - cron: 0 3 * * SUN - displayName: Weekly Sunday CodeQL run - branches: - include: - - main - always: true - -variables: - # CG is handled in the primary CI pipeline - - name: skipComponentGovernanceDetection - value: true - -extends: - template: /eng/pipelines/pipeline-resources.yml - parameters: - stages: - - stage: build - displayName: Build and Test Diagnostics - jobs: - - template: /eng/pipelines/build.yml - parameters: - name: Windows - osGroup: Windows_NT - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - Build_Release_x86: - _BuildConfig: Release - _BuildArch: x86 - Build_Release_arm: - _BuildConfig: Release - _BuildArch: arm - Build_Release_arm64: - _BuildConfig: Release - _BuildArch: arm64 - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_x64 - osGroup: Linux - nativeBuildContainer: linux_x64 - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_musl - osGroup: Linux - nativeBuildContainer: linux_musl_x64 - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - - - template: /eng/pipelines/build.yml - parameters: - name: MacOS - osGroup: MacOS - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - - - template: /eng/pipelines/build.yml - parameters: - name: MacOS_arm64 - osGroup: MacOS_cross - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_arm - osGroup: Linux - nativeBuildContainer: linux_arm - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_arm64 - osGroup: Linux - nativeBuildContainer: linux_arm64 - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_musl_arm - osGroup: Linux - nativeBuildContainer: linux_musl_arm - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_musl_arm64 - osGroup: Linux - nativeBuildContainer: linux_musl_arm64 - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 diff --git a/diagnostics.yml b/diagnostics.yml index 0de1318cca..064035b5e6 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -40,12 +40,6 @@ variables: - ${{ else }}: - name: isOfficialBuild value: false -- ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - name: validationConfiguration - value: Debug -- ${{ else }}: - - name: validationConfiguration - value: Release - template : /eng/pipelines/global-variables.yml parameters: runtimeFeed: ${{ parameters.runtimeFeed }} @@ -79,198 +73,31 @@ extends: # # ############################ - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: Windows - osGroup: Windows_NT - buildOnly: true - publishTestArtifacts: true - buildConfigs: - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - configuration: Debug - architecture: x64 - artifactUploadPath: bin/Windows_NT.x64.Debug - - configuration: Debug - architecture: x86 - artifactUploadPath: bin/Windows_NT.x86.Debug - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - configuration: Debug - architecture: arm64 - artifactUploadPath: bin/Windows_NT.arm64.Debug - - ${{ else }}: - - configuration: Release - architecture: x64 - artifactUploadPath: bin - - configuration: Release - architecture: x86 - artifactUploadPath: bin/Windows_NT.x86.Release - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - configuration: Debug - architecture: x64 - artifactUploadPath: bin/Windows_NT.x64.Debug - - configuration: Debug - architecture: x86 - artifactUploadPath: bin/Windows_NT.x86.Debug - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - configuration: Release - architecture: arm64 - artifactUploadPath: bin/Windows_NT.arm64.Release - - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - osGroup: Linux - container: linux_x64 - crossBuild: true - buildOnly: true - publishTestArtifacts: true - buildConfigs: - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - configuration: Debug - architecture: x64 - artifactUploadPath: bin/linux.x64.Debug - - ${{ else }}: - - configuration: Release - architecture: x64 - artifactUploadPath: bin/linux.x64.Release - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - configuration: Debug - architecture: x64 - artifactUploadPath: bin/linux.x64.Debug - - - template: /eng/pipelines/build.yml + - template: /eng/pipelines/platform-matrix.yml parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: Linux_musl - osGroup: Linux - osSuffix: -musl - container: linux_musl_x64 - crossBuild: true - buildOnly: true - publishTestArtifacts: true - buildConfigs: + jobTemplate: /eng/pipelines/build.yml + isOfficialBuild: ${{ variables.isOfficialBuild }} + configurations: - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - configuration: Debug - architecture: x64 - artifactUploadPath: bin/linux.x64.Debug - artifactTargetPath: bin/linux-musl.x64.Debug + - Debug - ${{ else }}: - - configuration: Release - architecture: x64 - artifactUploadPath: bin/linux.x64.Release - artifactTargetPath: bin/linux-musl.x64.Release - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - configuration: Debug - architecture: x64 - artifactUploadPath: bin/linux.x64.Debug - artifactTargetPath: bin/linux-musl.x64.Debug - - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - osGroup: Linux - container: linux_arm64 - crossBuild: true - buildOnly: true - publishTestArtifacts: ${{ eq(variables['System.TeamProject'], 'public') }} - buildConfigs: - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - configuration: Debug - architecture: arm64 - artifactUploadPath: bin/linux.arm64.Debug - - ${{ else }}: - - configuration: Release - architecture: arm64 - artifactUploadPath: bin/linux.arm64.Release - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - configuration: Debug - architecture: arm64 - artifactUploadPath: bin/linux.arm64.Debug - - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - osGroup: MacOS - buildOnly: true - publishTestArtifacts: true - buildConfigs: - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - configuration: Debug - architecture: x64 - artifactUploadPath: bin/osx.x64.Debug - - ${{ else }}: - - configuration: Release - architecture: x64 - artifactUploadPath: bin/osx.x64.Release - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - configuration: Debug - architecture: x64 - artifactUploadPath: bin/osx.x64.Debug - - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - osGroup: MacOS - crossBuild: true - buildOnly: true - publishTestArtifacts: ${{ eq(variables['System.TeamProject'], 'public') }} - buildConfigs: - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - configuration: Debug - architecture: arm64 - artifactUploadPath: bin/osx.arm64.Debug - - ${{ else }}: - - configuration: Release - architecture: arm64 - artifactUploadPath: bin/osx.arm64.Release - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - configuration: Debug - architecture: arm64 - artifactUploadPath: bin/osx.arm64.Debug - - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - osGroup: Linux - container: linux_arm - crossBuild: true - buildOnly: true - buildConfigs: - - configuration: ${{ variables.validationConfiguration }} - architecture: arm - artifactUploadPath: ${{ format('bin/linux.arm.{0}', variables.validationConfiguration) }} - - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: Linux_musl - osGroup: Linux - osSuffix: -musl - container: linux_musl_arm - crossBuild: true - buildOnly: true - buildConfigs: - - configuration: ${{ variables.validationConfiguration }} - architecture: arm - artifactUploadPath: ${{ format('bin/linux.arm.{0}', variables.validationConfiguration) }} - artifactTargetPath: ${{ format('bin/linux-musl.arm.{0}', variables.validationConfiguration) }} - - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: Linux_musl - osGroup: Linux - osSuffix: -musl - container: linux_musl_arm64 - crossBuild: true - buildOnly: true - buildConfigs: - - configuration: ${{ variables.validationConfiguration }} - architecture: arm64 - artifactUploadPath: ${{ format('bin/linux.arm64.{0}', variables.validationConfiguration) }} - artifactTargetPath: ${{ format('bin/linux-musl.arm64.{0}', variables.validationConfiguration) }} + - Release + - Debug + platforms: + - win-x64 + - win-x86 + - linux-x64 + - linux-arm64 + - linux-musl-x64 + - osx-x64 + - osx-arm64 + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - win-arm64 + - linux-arm + - linux-musl-arm + - linux-musl-arm64 + jobParameters: + pipelineJobTemplate: ${{ variables.jobTemplate }} ############################ # # @@ -279,266 +106,51 @@ extends: ############################ - ${{ if ne(parameters.buildOnly, true) }}: - - template: /eng/pipelines/build.yml + - template: /eng/pipelines/platform-matrix.yml parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: Windows_Test - osGroup: Windows_NT - dependsOn: Windows - testOnly: true - buildConfigs: - - configuration: ${{ variables.validationConfiguration }} - architecture: x64 + jobTemplate: /eng/pipelines/build.yml + isOfficialBuild: ${{ variables.isOfficialBuild }} + configurations: + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - Debug + - ${{ else }}: + - Release + - Debug + platforms: + - win-x64 - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - configuration: ${{ variables.validationConfiguration }} - architecture: x86 - - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: MacOS_Test - osGroup: MacOS - dependsOn: MacOS - testOnly: true - buildConfigs: - - configuration: ${{ variables.validationConfiguration }} - architecture: x64 - - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: Ubuntu_22_04 - osGroup: Linux - container: test_ubuntu_22_04 - dependsOn: Linux - testOnly: true - buildConfigs: - - configuration: ${{ variables.validationConfiguration }} - architecture: x64 - - - template: /eng/pipelines/build.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: Alpine3_23 - osGroup: Linux - osSuffix: -musl - container: test_linux_musl_x64 - dependsOn: Linux_musl - testOnly: true - disableComponentGovernance: true - buildConfigs: - - configuration: ${{ variables.validationConfiguration }} - architecture: x64 - + - win-x86 + - osx-x64 + - linux-x64 + - linux-musl-x64 + jobParameters: + pipelineJobTemplate: ${{ variables.jobTemplate }} + testOnly: true + + # TODO: Enable dedicated SOS Helix jobs for internal builds after selecting internal queues + # and aligning Debug artifact publishing. Internal builds currently run SOS in the normal test legs. - ${{ if eq(variables['System.TeamProject'], 'public') }}: - # TODO: Adopt the runtime pipeline's platform matrix pattern so SOS Helix coverage can be - # enabled for new platforms without duplicating the full job definition. - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: ${{ format('SOS_Helix_Windows_x64_{0}', variables.validationConfiguration) }} - osGroup: Windows_NT - dependsOn: ${{ format('Windows_x64_{0}', variables.validationConfiguration) }} - artifactPhase: ${{ format('Windows_x64_{0}', variables.validationConfiguration) }} - configuration: ${{ variables.validationConfiguration }} - architecture: x64 - targetOS: Windows_NT - targetRid: win-x64 - queue: $(helix_windows_x64) - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: ${{ format('SOS_Helix_Windows_x86_{0}', variables.validationConfiguration) }} - osGroup: Windows_NT - dependsOn: ${{ format('Windows_x86_{0}', variables.validationConfiguration) }} - artifactPhase: ${{ format('Windows_x86_{0}', variables.validationConfiguration) }} - configuration: ${{ variables.validationConfiguration }} - architecture: x86 - targetOS: Windows_NT - targetRid: win-x86 - queue: $(helix_windows_x64) - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: ${{ format('SOS_Helix_Linux_x64_{0}', variables.validationConfiguration) }} - osGroup: Linux - container: test_ubuntu_22_04 - dependsOn: ${{ format('Linux_x64_{0}', variables.validationConfiguration) }} - artifactPhase: ${{ format('Linux_x64_{0}', variables.validationConfiguration) }} - configuration: ${{ variables.validationConfiguration }} - architecture: x64 - targetOS: linux - targetRid: linux-x64 - queue: $(helix_linux_x64) - - - template: /eng/pipelines/sos-helix.yml + - template: /eng/pipelines/platform-matrix.yml parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: ${{ format('SOS_Helix_Linux_arm64_{0}', variables.validationConfiguration) }} - osGroup: Linux - container: test_ubuntu_22_04 - dependsOn: ${{ format('Linux_arm64_{0}', variables.validationConfiguration) }} - artifactPhase: ${{ format('Linux_arm64_{0}', variables.validationConfiguration) }} - configuration: ${{ variables.validationConfiguration }} - architecture: arm64 - targetOS: linux - targetRid: linux-arm64 - queue: $(helix_linux_arm64) - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: ${{ format('SOS_Helix_Alpine_x64_{0}', variables.validationConfiguration) }} - osGroup: Linux - container: test_linux_musl_x64 - dependsOn: ${{ format('Linux_musl_x64_{0}', variables.validationConfiguration) }} - artifactPhase: ${{ format('Linux_musl_x64_{0}', variables.validationConfiguration) }} - configuration: ${{ variables.validationConfiguration }} - architecture: x64 - targetOS: linux - targetRid: linux-musl-x64 - queue: $(helix_linux_musl_x64) - helixPreCommands: sudo apk add --no-cache py3-lldb - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: ${{ format('SOS_Helix_MacOS_arm64_{0}', variables.validationConfiguration) }} - osGroup: MacOS - dependsOn: ${{ format('MacOS_arm64_{0}', variables.validationConfiguration) }} - artifactPhase: ${{ format('MacOS_arm64_{0}', variables.validationConfiguration) }} - configuration: ${{ variables.validationConfiguration }} - architecture: arm64 - targetOS: osx - targetRid: osx-arm64 - queue: $(helix_osx_arm64) - - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: SOS_Helix_Windows_x64_Debug - osGroup: Windows_NT - dependsOn: Windows_x64_Debug - artifactPhase: Windows_x64_Debug - configuration: Debug - architecture: x64 - targetOS: Windows_NT - targetRid: win-x64 - queue: $(helix_windows_x64) - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: SOS_Helix_Windows_x86_Debug - osGroup: Windows_NT - dependsOn: Windows_x86_Debug - artifactPhase: Windows_x86_Debug - configuration: Debug - architecture: x86 - targetOS: Windows_NT - targetRid: win-x86 - queue: $(helix_windows_x64) - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: SOS_Helix_Linux_x64_Debug - osGroup: Linux - container: test_ubuntu_22_04 - dependsOn: Linux_x64_Debug - artifactPhase: Linux_x64_Debug - configuration: Debug - architecture: x64 - targetOS: linux - targetRid: linux-x64 - queue: $(helix_linux_x64) - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: SOS_Helix_Linux_arm64_Debug - osGroup: Linux - container: test_ubuntu_22_04 - dependsOn: Linux_arm64_Debug - artifactPhase: Linux_arm64_Debug - configuration: Debug - architecture: arm64 - targetOS: linux - targetRid: linux-arm64 - queue: $(helix_linux_arm64) - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: SOS_Helix_Alpine_x64_Debug - osGroup: Linux - container: test_linux_musl_x64 - dependsOn: Linux_musl_x64_Debug - artifactPhase: Linux_musl_x64_Debug - configuration: Debug - architecture: x64 - targetOS: linux - targetRid: linux-musl-x64 - queue: $(helix_linux_musl_x64) - helixPreCommands: sudo apk add --no-cache py3-lldb - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: SOS_Helix_MacOS_arm64_Debug - osGroup: MacOS - dependsOn: MacOS_arm64_Debug - artifactPhase: MacOS_arm64_Debug - configuration: Debug - architecture: arm64 - targetOS: osx - targetRid: osx-arm64 - queue: $(helix_osx_arm64) - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: SOS_Helix_MacOS_x64_Debug - osGroup: MacOS - dependsOn: MacOS_x64_Debug - artifactPhase: MacOS_x64_Debug - configuration: Debug - architecture: x64 - targetOS: osx - targetRid: osx-x64 - queue: $(helix_osx_x64) - - - template: /eng/pipelines/sos-helix.yml - parameters: - jobTemplate: ${{ variables.jobTemplate }} - name: SOS_Helix_MacOS_x64_Release - osGroup: MacOS - dependsOn: MacOS_x64_Release - artifactPhase: MacOS_x64_Release - configuration: Release - architecture: x64 - targetOS: osx - targetRid: osx-x64 - queue: $(helix_osx_x64) - - # - ${{ if ne(variables['System.TeamProject'], 'public') }}: - # - template: /eng/pipelines/build.yml - # parameters: - # jobTemplate: ${{ variables.jobTemplate }} - # name: Fedora_43 - # osGroup: Linux - # container: test_fedora - # dependsOn: Linux - # testOnly: true - # buildConfigs: - # - configuration: Release - # architecture: x64 - # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - # - configuration: Debug - # architecture: x64 + jobTemplate: /eng/pipelines/sos-helix.yml + isOfficialBuild: ${{ variables.isOfficialBuild }} + configurations: + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - Debug + - ${{ else }}: + - Release + - Debug + platforms: + - win-x64 + - win-x86 + - linux-x64 + - linux-arm64 + - linux-musl-x64 + - osx-arm64 + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - osx-x64 + jobParameters: + pipelineJobTemplate: ${{ variables.jobTemplate }} ############################ # # @@ -637,7 +249,7 @@ extends: repoAllowList: https://github.com/dotnet/diagnostics https://dev.azure.com/dnceng/internal/_git/dotnet-diagnostics productAllowList: diagnostics dotnet-diagnostics skipFileList: MergedManifest.xml - ${{ if eq(variables['isOfficialBuild'], 'true') }}: + ${{ if eq(variables.isOfficialBuild, true) }}: dependsOn: [ publish_using_darc ] ${{ else }}: dependsOn: [ build ] diff --git a/eng/pipelines/build.yml b/eng/pipelines/build.yml index 5ac8c9dfc1..02ec833d51 100644 --- a/eng/pipelines/build.yml +++ b/eng/pipelines/build.yml @@ -1,322 +1,245 @@ parameters: - # Job name -- name: name +# Public or official Arcade job wrapper. +- name: pipelineJobTemplate type: string - default: '' - -- name: jobTemplate - type: string - default: /eng/common/templates-official/job/job.yml@self - values: - - /eng/common/templates-official/job/job.yml@self - - /eng/common/templates/job/job.yml@self - -- name: osGroup - type: string - default: Windows_NT - values: - - Windows_NT - - Linux - - MacOS - - FreeBSD - -- name: osSuffix +# Build configuration generated by platform-matrix.yml. +- name: configuration type: string - default: '' - - # Additional variables -- name: variables - type: object - default: {} - - # Optional: Job timeout -- name: timeoutInMinutes - type: number - default: 180 - - # Optional: Build container to use -- name: templateContext +# Platform metadata generated by platform-matrix.yml. +- name: platform type: object - default: {} - -# Build configs. An object with the following properties: configuration, architecture. -# Optionally it can also contain artifactUploadPath, artifactTargetPath -- name: buildConfigs - type: object - default: {} - - # Optional: container resource name -- name: container - type: string - default: '' - - # Optional: build only job if true -- name: buildOnly - type: boolean - default: false - - # Optional: test only job if true +# Run tests from previously published build outputs instead of building. - name: testOnly type: boolean default: false - - # Optional: publish the full build outputs (managed + native + debuggees) as a - # 'TestArtifacts_' artifact so the matching test-only legs can run without rebuilding. -- name: publishTestArtifacts - type: boolean - default: false - -# Optional: architecture cross build if true -- name: crossBuild - type: boolean - default: false - -- name: dependsOn - type: string - default: '' - -- name: disableComponentGovernance +- name: isOfficialBuild type: boolean default: false jobs: -- ${{ each config in parameters.buildConfigs }}: - - template: ${{ parameters.jobTemplate }} - parameters: - name: ${{ coalesce(parameters.name, parameters.osGroup) }}_${{ config.architecture }}_${{ config.configuration }} +- template: ${{ parameters.pipelineJobTemplate }} + parameters: + ${{ if eq(parameters.testOnly, true) }}: + name: ${{ parameters.platform.name }}_Test_${{ parameters.platform.arch }}_${{ parameters.configuration }} + ${{ else }}: + name: ${{ parameters.platform.name }}_${{ parameters.platform.arch }}_${{ parameters.configuration }} + ${{ if eq(parameters.testOnly, true) }}: + displayName: Test ${{ replace(parameters.platform.name, '_', '-') }} ${{ parameters.platform.arch }} ${{ parameters.configuration }} + ${{ else }}: + displayName: Build ${{ replace(parameters.platform.name, '_', '-') }} ${{ parameters.platform.arch }} ${{ parameters.configuration }} + timeoutInMinutes: 180 + enableMicrobuild: true + disableComponentGovernance: ${{ and(eq(parameters.testOnly, true), startsWith(parameters.platform.rid, 'linux-musl-')) }} + + pool: + ${{ if eq(parameters.platform.os, 'windows') }}: + name: $(BuildPool) + demands: ImageOverride -equals $(WindowsImage) + os: windows + ${{ elseif eq(parameters.platform.os, 'linux') }}: + name: $(BuildPool) + demands: ImageOverride -equals $(LinuxImage) + os: linux + ${{ elseif eq(parameters.platform.os, 'osx') }}: + name: Azure Pipelines + vmImage: $(macOSImage) + os: macOS + ${{ else }}: + 'error, unsupported OS': ${{ parameters.platform.os }} + + ${{ if and(eq(parameters.testOnly, true), eq(parameters.platform.os, 'linux')) }}: + container: test_${{ parameters.platform.queueKey }} + ${{ elseif and(eq(parameters.testOnly, false), eq(parameters.platform.os, 'linux')) }}: + container: ${{ parameters.platform.queueKey }} + + ${{ if eq(parameters.testOnly, true) }}: + dependsOn: ${{ parameters.platform.name }}_${{ parameters.platform.arch }}_${{ parameters.configuration }} + + workspace: + clean: all + + variables: + - ${{ if eq(parameters.testOnly, true) }}: + - _PhaseName: ${{ parameters.platform.name }}_Test_${{ parameters.platform.arch }}_${{ parameters.configuration }} + - ${{ else }}: + - _PhaseName: ${{ parameters.platform.name }}_${{ parameters.platform.arch }}_${{ parameters.configuration }} + - _Pipeline_StreamDumpDir: $(Build.SourcesDirectory)/artifacts/tmp/${{ parameters.configuration }}/streams + - _TestArgs: '-test' + - _Cross: '' + + # Test-only legs invoke test.cmd/test.sh, which already encode + # '-test -skipmanaged -skipnative'. Build legs invoke build.cmd/build.sh. + - ${{ if eq(parameters.testOnly, true) }}: + - ${{ if eq(parameters.platform.os, 'windows') }}: + - _buildScript: $(Build.SourcesDirectory)\test.cmd + - ${{ else }}: + - _buildScript: $(Build.SourcesDirectory)/test.sh + - _TestArgs: '' + - ${{ else }}: + - ${{ if eq(parameters.platform.os, 'windows') }}: + - _buildScript: $(Build.SourcesDirectory)\build.cmd + - ${{ else }}: + - _buildScript: $(Build.SourcesDirectory)/build.sh + - _TestArgs: '' + + # For testing MSRC and service releases. The runtime versions and feed + # identify the servicing runtime to install. + - _InternalInstallArgs: '' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - _InternalInstallArgs: + -dotnetruntimeversion '$(DotnetRuntimeVersion)' + -dotnetruntimedownloadversion '$(DotnetRuntimeDownloadVersion)' + -runtimesourcefeed '$(RuntimeFeedUrl)' + -runtimesourcefeedkey '$(RuntimeFeedBase64SasToken)' + + # Linux builds and macOS ARM64 builds target a different environment + # than the host agent. + - ${{ if and(eq(parameters.testOnly, false), or(eq(parameters.platform.os, 'linux'), and(eq(parameters.platform.os, 'osx'), eq(parameters.platform.arch, 'arm64')))) }}: + - _Cross: -cross + + steps: + - ${{ if eq(parameters.testOnly, true) }}: + # Restore the complete output from the matching build leg so tests can + # skip both managed and native compilation. + - task: DownloadPipelineArtifact@2 + displayName: Download Build Artifacts + inputs: + artifact: TestArtifacts_${{ parameters.platform.name }}_${{ parameters.platform.arch }}_${{ parameters.configuration }} + targetPath: '$(Build.SourcesDirectory)/artifacts' + checkDownloadedFiles: true + + - script: $(_buildScript) + -ci + -binaryLog + -configuration ${{ parameters.configuration }} + -architecture ${{ parameters.platform.arch }} + $(_TestArgs) + $(_Cross) + $(_InternalInstallArgs) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) ${{ if eq(parameters.testOnly, true) }}: - displayName: Test ${{ replace(parameters.osGroup, '_NT', '') }}${{ parameters.osSuffix }} ${{ config.architecture }} ${{ config.configuration }} - ${{ elseif eq(parameters.buildOnly, true) }}: - displayName: Build ${{ replace(parameters.osGroup, '_NT', '') }}${{ parameters.osSuffix }} ${{ config.architecture }} ${{ config.configuration }} + displayName: Test + continueOnError: ${{ parameters.isOfficialBuild }} ${{ else }}: - displayName: Build / Test ${{ replace(parameters.osGroup, '_NT', '') }}${{ parameters.osSuffix }} ${{ config.architecture }} ${{ config.configuration }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - enableMicrobuild: true - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - templateContext: ${{ parameters.templateContext }} - - pool: - # Public Linux Build Pool - ${{ if eq(parameters.osGroup, 'Linux') }}: - name: $(BuildPool) - demands: ImageOverride -equals $(LinuxImage) - os: linux - - ${{ elseif eq(parameters.osGroup, 'Windows_NT') }}: - name: $(BuildPool) - demands: ImageOverride -equals $(WindowsImage) - os: windows - - ${{ elseif in(parameters.osGroup, 'MacOS') }}: - name: Azure Pipelines - vmImage: $(macOSImage) - os: macOS - - ${{ elseif and(eq(parameters.osGroup, 'FreeBSD'), ne(variables['System.TeamProject'], 'public')) }}: - name: dnceng-freebsd-internal - os: freebsd - - ${{ else }}: - 'error, Unable to identify pool for required build': error - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.strategy, '') }}: - 'error, we can no longer support the strategy feature in the new pipeline system. Please remove the strategy from the job template.': error - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }}_${{ config.architecture }}_${{ config.configuration }} + displayName: Build + condition: succeeded() - workspace: - clean: all - - variables: - - ${{ insert }}: ${{ parameters.variables }} - - - _PhaseName: ${{ coalesce(parameters.name, parameters.osGroup) }}_${{ config.architecture }}_${{ config.configuration }} - - _Pipeline_StreamDumpDir: $(Build.SourcesDirectory)/artifacts/tmp/${{ config.configuration }}/streams - - - _TestArgs: '-test' - - _Cross: '' - - - ${{ if and(eq(parameters.testOnly, 'true'), eq(parameters.buildOnly, 'true')) }}: - 'error, testOnly and buildOnly cannot be true at the same time': error - # Test-only legs invoke test.cmd/test.sh, which already encode - # '-test -skipmanaged -skipnative' and do not prepend '-restore -build'. All other legs - # (build-only and build+test) use build.cmd/build.sh, which prepend '-restore -build'. - - ${{ if eq(parameters.testOnly, 'true') }}: - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)\test.cmd - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)/test.sh - - _TestArgs: '' - - - ${{ if ne(parameters.testOnly, 'true') }}: - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)\build.cmd - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)/build.sh - - - ${{ if eq(parameters.buildOnly, 'true') }}: - - _TestArgs: '' - - # For testing msrc's and service releases. The RuntimeSourceVersion is either "default" or the service release version to test - - _InternalInstallArgs: '' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - _InternalInstallArgs: - -dotnetruntimeversion '$(DotnetRuntimeVersion)' - -dotnetruntimedownloadversion '$(DotnetRuntimeDownloadVersion)' - -runtimesourcefeed '$(RuntimeFeedUrl)' - -runtimesourcefeedkey '$(RuntimeFeedBase64SasToken)' - - # This is only required for cross builds. - - ${{ if eq(parameters.crossBuild, true) }}: - - _Cross: -cross - - steps: - - ${{ if eq(parameters.testOnly, true) }}: - - ${{ if notIn(parameters.osGroup, 'Windows_NT', 'MacOS', 'Linux') }}: - - 'error, testOnly is only supported on Windows_NT, MacOS, and Linux': error - # The matching build leg published its full outputs (managed assemblies, native - # binaries, and debuggees) as TestArtifacts_. Restore them into the repo - # artifacts directory so the test run can skip both the native and managed builds - # (-skipnative -skipmanaged) and only execute the tests. - - task: DownloadPipelineArtifact@2 - displayName: 'Download Build Artifacts' - inputs: - artifact: TestArtifacts_${{ parameters.dependsOn }}_${{ config.architecture }}_${{ config.configuration }} - targetPath: '$(Build.SourcesDirectory)/artifacts' - checkDownloadedFiles: true - - - script: $(_buildScript) - -ci - -binaryLog - -configuration ${{ config.configuration }} - -architecture ${{ config.architecture }} - $(_TestArgs) - $(_Cross) - $(_InternalInstallArgs) - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - ${{ if eq(parameters.testOnly, 'true') }}: - displayName: Test - continueOnError: ${{ eq(variables.isOfficialBuild, 'true') }} - ${{ elseif eq(parameters.buildOnly, 'true') }}: - displayName: Build - ${{ else }}: - displayName: Build / Test - condition: succeeded() - - - ${{ if ne(config.artifactUploadPath, '') }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/${{ config.artifactUploadPath }}' - Contents: '**' - TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/${{ coalesce(config.artifactTargetPath, config.artifactUploadPath) }} - - - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self - parameters: - displayName: Publish Build Artifacts - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' - artifactName: Build_$(_PhaseName) - - # Publish the full build outputs for the matching test-only legs to consume. This lets the - # test legs run with -skipnative -skipmanaged instead of rebuilding the product themselves. - # Only valid on build-only legs (publishTestArtifacts without buildOnly is a configuration error). - - ${{ if and(eq(parameters.publishTestArtifacts, true), eq(parameters.buildOnly, true)) }}: - - task: CopyFiles@2 - displayName: Gather test artifacts for publish - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' - Contents: | - bin/** - obj/** - test/** - TargetFolder: $(Build.ArtifactStagingDirectory)/testartifacts - - - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self - parameters: - displayName: Publish Test Artifacts - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)/testartifacts' - artifactName: TestArtifacts_$(_PhaseName) + - ${{ if eq(parameters.testOnly, false) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish + inputs: + ${{ if and(eq(parameters.platform.os, 'windows'), eq(parameters.platform.arch, 'x64'), eq(parameters.configuration, 'Release')) }}: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/bin' + ${{ else }}: + SourceFolder: "$(Build.SourcesDirectory)/artifacts/bin/${{ replace(parameters.platform.os, 'windows', 'Windows_NT') }}.${{ parameters.platform.arch }}.${{ parameters.configuration }}" + Contents: '**' + ${{ if ne(coalesce(parameters.platform.artifactOS, ''), '') }}: + TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/${{ parameters.platform.artifactOS }}.${{ parameters.platform.arch }}.${{ parameters.configuration }} + ${{ elseif and(eq(parameters.platform.os, 'windows'), eq(parameters.platform.arch, 'x64'), eq(parameters.configuration, 'Release')) }}: + TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin + ${{ else }}: + TargetFolder: "$(Build.ArtifactStagingDirectory)/artifacts/bin/${{ replace(parameters.platform.os, 'windows', 'Windows_NT') }}.${{ parameters.platform.arch }}.${{ parameters.configuration }}" - - ${{ if ne(parameters.buildOnly, 'true') }}: - # Publish test results to Azure Pipelines - - task: PublishTestResults@2 + - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self + parameters: + displayName: Publish Build Artifacts inputs: - testResultsFormat: xUnit - testResultsFiles: '**/*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults' - testRunTitle: 'Tests $(_PhaseName)' - failTaskOnFailedTests: true - publishRunAttachments: true - mergeTestResults: true - buildConfiguration: ${{ config.configuration }} - continueOnError: true - condition: always() + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: Build_$(_PhaseName) + # Publish the complete build outputs consumed by normal-test and SOS jobs. + - ${{ if eq(parameters.testOnly, false) }}: - task: CopyFiles@2 - displayName: Gather Logs + displayName: Gather test artifacts for publish inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts' - contents: '?(log|TestResults)/**' - targetFolder: '$(Build.StagingDirectory)/BuildLogs' - continueOnError: true - condition: always() + SourceFolder: '$(Build.SourcesDirectory)/artifacts' + Contents: | + bin/** + obj/** + test/** + TargetFolder: $(Build.ArtifactStagingDirectory)/testartifacts - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self parameters: - displayName: Publish Logs + displayName: Publish Test Artifacts inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: Logs_$(_PhaseName)_$(System.JobAttempt) - sbomEnabled: false # we don't need SBOM for logs - continueOnError: true - condition: always() + targetPath: '$(Build.ArtifactStagingDirectory)/testartifacts' + artifactName: TestArtifacts_$(_PhaseName) - - pwsh: | - $source = "$(Build.SourcesDirectory)/artifacts" - $dest = "$(Build.ArtifactStagingDirectory)/artifacts_on_failure" - New-Item -ItemType Directory -Force -Path $dest - - # Move non-dump artifacts - $patterns = @("bin", "Debuggees", "log", "TestResults") - foreach ($p in $patterns) { - $src = Join-Path $source $p - if (Test-Path $src) { - $target = Join-Path $dest $p - Move-Item -Force $src $target - } - } - - # Tar dumps and streams directly from source to avoid ENOSPC. - # Peak disk = original files + compressed tar (smaller than a copy). - $dumpSrc = Join-Path $source "tmp/${{ config.configuration }}/dumps" - $streamSrc = Join-Path $source "tmp/${{ config.configuration }}/streams" + # Publish test results to Azure Pipelines. + - ${{ if eq(parameters.testOnly, true) }}: + - task: PublishTestResults@2 + inputs: + testResultsFormat: xUnit + testResultsFiles: '**/*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults' + testRunTitle: 'Tests $(_PhaseName)' + failTaskOnFailedTests: true + publishRunAttachments: true + mergeTestResults: true + buildConfiguration: ${{ parameters.configuration }} + continueOnError: ${{ parameters.isOfficialBuild }} + condition: always() - $tarInputs = @() - if (Test-Path $dumpSrc) { $tarInputs += "tmp/${{ config.configuration }}/dumps" } - if (Test-Path $streamSrc) { $tarInputs += "tmp/${{ config.configuration }}/streams" } + - task: CopyFiles@2 + displayName: Gather Logs + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '?(log|TestResults)/**' + targetFolder: '$(Build.StagingDirectory)/BuildLogs' + continueOnError: true + condition: always() + + - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self + parameters: + displayName: Publish Logs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: Logs_$(_PhaseName)_$(System.JobAttempt) + sbomEnabled: false + continueOnError: true + condition: always() - if ($tarInputs.Count -gt 0) { - New-Item -ItemType Directory -Force -Path $dest | Out-Null - $tarPath = Join-Path $dest "diagnostics-dumps.tar.gz" - tar -czf $tarPath -C $source @tarInputs - foreach ($t in $tarInputs) { Remove-Item -Recurse -Force (Join-Path $source $t) } + - pwsh: | + $source = "$(Build.SourcesDirectory)/artifacts" + $dest = "$(Build.ArtifactStagingDirectory)/artifacts_on_failure" + New-Item -ItemType Directory -Force -Path $dest + + # Move non-dump artifacts. + $patterns = @("bin", "Debuggees", "log", "TestResults") + foreach ($p in $patterns) { + $src = Join-Path $source $p + if (Test-Path $src) { + $target = Join-Path $dest $p + Move-Item -Force $src $target } - displayName: Gather diagnostic artifacts on failure + } + + # Tar dumps and streams directly from source to avoid ENOSPC. + # Peak disk use is the original files plus the smaller compressed archive. + $dumpSrc = Join-Path $source "tmp/${{ parameters.configuration }}/dumps" + $streamSrc = Join-Path $source "tmp/${{ parameters.configuration }}/streams" + + $tarInputs = @() + if (Test-Path $dumpSrc) { $tarInputs += "tmp/${{ parameters.configuration }}/dumps" } + if (Test-Path $streamSrc) { $tarInputs += "tmp/${{ parameters.configuration }}/streams" } + + if ($tarInputs.Count -gt 0) { + New-Item -ItemType Directory -Force -Path $dest | Out-Null + $tarPath = Join-Path $dest "diagnostics-dumps.tar.gz" + tar -czf $tarPath -C $source @tarInputs + foreach ($t in $tarInputs) { Remove-Item -Recurse -Force (Join-Path $source $t) } + } + displayName: Gather diagnostic artifacts on failure + continueOnError: true + condition: failed() + + - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self + parameters: + displayName: Publish diagnostic artifacts on failure + inputs: + targetPath: $(Build.ArtifactStagingDirectory)/artifacts_on_failure + artifactName: Artifacts_On_Failure_$(_PhaseName)_$(System.JobAttempt) + sbomEnabled: false continueOnError: true condition: failed() - - - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self - parameters: - displayName: Publish diagnostic artifacts on failure - inputs: - targetPath: $(Build.ArtifactStagingDirectory)/artifacts_on_failure - artifactName: Artifacts_On_Failure_$(_PhaseName)_$(System.JobAttempt) - sbomEnabled: false # we don't need SBOM for non-shipping diagnostics assets - continueOnError: true - condition: failed() diff --git a/eng/pipelines/helix-platforms.yml b/eng/pipelines/helix-platforms.yml index 9a29202d8a..f5aee83be0 100644 --- a/eng/pipelines/helix-platforms.yml +++ b/eng/pipelines/helix-platforms.yml @@ -1,13 +1,53 @@ +# Helix Platform Definitions +# This template defines versioned Helix queues and stable platform aliases. + variables: -- name: helix_windows_x64 - value: Windows.Amd64.Server2022.Open -- name: helix_linux_x64 - value: Ubuntu.2204.Amd64.Open -- name: helix_linux_arm64 - value: Ubuntu.2204.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-arm64v8 -- name: helix_linux_musl_x64 - value: '(Alpine.323.Amd64.Open)azurelinux.3.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.23-helix-amd64' -- name: helix_osx_x64 - value: OSX.15.Amd64.Open -- name: helix_osx_arm64 - value: OSX.15.Arm64.Open + + # macOS platforms + + - name: helix_macos_arm64_osx_15 + value: OSX.15.Arm64.Open + + - name: helix_macos_x64_osx_15 + value: OSX.15.Amd64.Open + + # Linux platforms + + - name: helix_linux_x64_ubuntu_2204 + value: Ubuntu.2204.Amd64.Open + + - name: helix_linux_arm64_ubuntu_2204 + value: Ubuntu.2204.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-arm64v8 + + - name: helix_linux_musl_x64_alpine_323 + value: '(Alpine.323.Amd64.Open)azurelinux.3.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.23-helix-amd64' + + # Windows platforms + + - name: helix_windows_x64_server_2022 + value: Windows.Amd64.Server2022.Open + + # Common aliases + # These must remain after the versioned definitions they reference. + + - name: helix_osx_arm64 + value: ${{ variables.helix_macos_arm64_osx_15 }} + + - name: helix_osx_x64 + value: ${{ variables.helix_macos_x64_osx_15 }} + + - name: helix_linux_x64 + value: ${{ variables.helix_linux_x64_ubuntu_2204 }} + + - name: helix_linux_arm64 + value: ${{ variables.helix_linux_arm64_ubuntu_2204 }} + + - name: helix_linux_musl_x64 + value: ${{ variables.helix_linux_musl_x64_alpine_323 }} + + - name: helix_win_x64 + value: ${{ variables.helix_windows_x64_server_2022 }} + + # Windows x86 tests run on the Windows x64 Helix queue. + - name: helix_win_x86 + value: ${{ variables.helix_win_x64 }} diff --git a/eng/pipelines/pipeline-resources.yml b/eng/pipelines/pipeline-resources.yml index 64f140ceb7..90793eecca 100644 --- a/eng/pipelines/pipeline-resources.yml +++ b/eng/pipelines/pipeline-resources.yml @@ -68,7 +68,8 @@ extends: ROOTFS_DIR: /crossrootfs/riscv64 test_linux_x64: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-9 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + options: '--env PYTHONPATH=/usr/bin/python3.10' test_linux_musl_x64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.23-amd64 @@ -83,7 +84,3 @@ extends: test_opensuse_16_0: image: mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-16.0-helix-amd64 - - test_ubuntu_22_04: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - options: '--env PYTHONPATH=/usr/bin/python3.10' diff --git a/eng/pipelines/platform-matrix.yml b/eng/pipelines/platform-matrix.yml new file mode 100644 index 0000000000..7f88cae23b --- /dev/null +++ b/eng/pipelines/platform-matrix.yml @@ -0,0 +1,162 @@ +parameters: +- name: jobTemplate + type: string +- name: configurations + type: object +- name: platforms + type: object +- name: jobParameters + type: object + default: {} +- name: isOfficialBuild + type: boolean + default: false + +jobs: +- ${{ each configuration in parameters.configurations }}: + - ${{ if containsValue(parameters.platforms, 'win-x64') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: win-x64 + name: Windows + os: windows + arch: x64 + queueKey: win_x64 + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'win-x86') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: win-x86 + name: Windows + os: windows + arch: x86 + queueKey: win_x86 + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'win-arm64') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: win-arm64 + name: Windows + os: windows + arch: arm64 + queueKey: win_arm64 + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'linux-x64') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: linux-x64 + name: Linux + os: linux + arch: x64 + queueKey: linux_x64 + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'linux-arm') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: linux-arm + name: Linux + os: linux + arch: arm + queueKey: linux_arm + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'linux-arm64') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: linux-arm64 + name: Linux + os: linux + arch: arm64 + queueKey: linux_arm64 + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'linux-musl-x64') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: linux-musl-x64 + name: Linux_musl + os: linux + arch: x64 + artifactOS: linux-musl + queueKey: linux_musl_x64 + helixPreCommands: sudo apk add --no-cache py3-lldb + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'linux-musl-arm') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: linux-musl-arm + name: Linux_musl + os: linux + arch: arm + artifactOS: linux-musl + queueKey: linux_musl_arm + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'linux-musl-arm64') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: linux-musl-arm64 + name: Linux_musl + os: linux + arch: arm64 + artifactOS: linux-musl + queueKey: linux_musl_arm64 + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'osx-x64') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: osx-x64 + name: MacOS + os: osx + arch: x64 + queueKey: osx_x64 + ${{ insert }}: ${{ parameters.jobParameters }} + + - ${{ if containsValue(parameters.platforms, 'osx-arm64') }}: + - template: ${{ parameters.jobTemplate }} + parameters: + configuration: ${{ configuration }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + platform: + rid: osx-arm64 + name: MacOS + os: osx + arch: arm64 + queueKey: osx_arm64 + ${{ insert }}: ${{ parameters.jobParameters }} diff --git a/eng/pipelines/sos-helix.yml b/eng/pipelines/sos-helix.yml index 76a555dafe..aa14e93c22 100644 --- a/eng/pipelines/sos-helix.yml +++ b/eng/pipelines/sos-helix.yml @@ -1,65 +1,45 @@ parameters: -- name: jobTemplate - type: string - default: /eng/common/templates-official/job/job.yml@self -- name: name - type: string -- name: osGroup - type: string -- name: container - type: string - default: '' -- name: dependsOn - type: string - default: '' -- name: artifactPhase +- name: pipelineJobTemplate type: string - name: configuration type: string -- name: architecture - type: string -- name: targetOS - type: string -- name: targetRid - type: string -- name: queue - type: string -- name: helixPreCommands - type: string - default: '' +- name: platform + type: object +- name: isOfficialBuild + type: boolean + default: false + jobs: -- template: ${{ parameters.jobTemplate }} +- template: ${{ parameters.pipelineJobTemplate }} parameters: - name: ${{ parameters.name }} - displayName: SOS Helix ${{ replace(replace(replace(replace(parameters.targetRid, 'linux-musl-', 'Alpine '), 'linux-', 'Linux '), 'win-', 'Windows '), 'osx-', 'macOS ') }} ${{ parameters.configuration }} + name: SOS_Helix_${{ parameters.platform.name }}_${{ parameters.platform.arch }}_${{ parameters.configuration }} + displayName: SOS Helix ${{ parameters.platform.rid }} ${{ parameters.configuration }} timeoutInMinutes: 420 enableMicrobuild: true pool: - ${{ if eq(parameters.osGroup, 'Linux') }}: - name: $(BuildPool) - demands: ImageOverride -equals $(LinuxImage) - os: linux - ${{ elseif eq(parameters.osGroup, 'Windows_NT') }}: + ${{ if eq(parameters.platform.os, 'windows') }}: name: $(BuildPool) demands: ImageOverride -equals $(WindowsImage) os: windows - ${{ elseif eq(parameters.osGroup, 'MacOS') }}: + ${{ elseif eq(parameters.platform.os, 'linux') }}: + name: $(BuildPool) + demands: ImageOverride -equals $(LinuxImage) + os: linux + ${{ elseif eq(parameters.platform.os, 'osx') }}: name: Azure Pipelines vmImage: $(macOSImage) os: macOS + ${{ else }}: + 'error, unsupported OS': ${{ parameters.platform.os }} - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} + dependsOn: ${{ parameters.platform.name }}_${{ parameters.platform.arch }}_${{ parameters.configuration }} steps: - task: DownloadPipelineArtifact@2 displayName: Download SOS test payload inputs inputs: - artifact: TestArtifacts_${{ parameters.artifactPhase }} + artifact: TestArtifacts_${{ parameters.platform.name }}_${{ parameters.platform.arch }}_${{ parameters.configuration }} targetPath: '$(Build.SourcesDirectory)/artifacts' checkDownloadedFiles: true @@ -68,44 +48,44 @@ jobs: $env:BUILD_SOURCESDIRECTORY\src\tests\SOS.Tests\SOS.Tests.csproj /restore /p:TreatWarningsAsErrors=false /t:SendToHelix /p:Configuration=${{ parameters.configuration }} - /p:TargetArch=${{ parameters.architecture }} - /p:TargetOS=${{ parameters.targetOS }} - /p:TargetRid=${{ parameters.targetRid }} + /p:TargetArch=${{ parameters.platform.arch }} + /p:TargetOS=${{ replace(parameters.platform.os, 'windows', 'Windows_NT') }} + /p:TargetRid=${{ parameters.platform.rid }} /p:OfficialBuildId=$env:BUILD_BUILDNUMBER /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\${{ parameters.configuration }}\SendToHelix.binlog" displayName: Run SOS.Tests on Helix env: HelixSource: pr/dotnet/diagnostics/sos-tests HelixType: tests/sos/harness/ - HelixConfiguration: ${{ parameters.targetRid }}_${{ parameters.configuration }} - HelixTargetQueues: ${{ parameters.queue }} - HelixPreCommands: ${{ parameters.helixPreCommands }} + HelixConfiguration: ${{ parameters.platform.rid }}_${{ parameters.configuration }} + HelixTargetQueues: $(helix_${{ parameters.platform.queueKey }}) + HelixPreCommands: ${{ coalesce(parameters.platform.helixPreCommands, '') }} Creator: dotnet-bot SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: eq(variables['Agent.Os'], 'Windows_NT') - continueOnError: ${{ eq(variables.isOfficialBuild, 'true') }} + continueOnError: ${{ parameters.isOfficialBuild }} - script: > $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/src/tests/SOS.Tests/SOS.Tests.csproj /restore /p:TreatWarningsAsErrors=false /t:SendToHelix /p:Configuration=${{ parameters.configuration }} - /p:TargetArch=${{ parameters.architecture }} - /p:TargetOS=${{ parameters.targetOS }} - /p:TargetRid=${{ parameters.targetRid }} + /p:TargetArch=${{ parameters.platform.arch }} + /p:TargetOS=${{ replace(parameters.platform.os, 'windows', 'Windows_NT') }} + /p:TargetRid=${{ parameters.platform.rid }} /p:OfficialBuildId=$BUILD_BUILDNUMBER /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/${{ parameters.configuration }}/SendToHelix.binlog displayName: Run SOS.Tests on Helix env: HelixSource: pr/dotnet/diagnostics/sos-tests HelixType: tests/sos/harness/ - HelixConfiguration: ${{ parameters.targetRid }}_${{ parameters.configuration }} - HelixTargetQueues: ${{ parameters.queue }} - HelixPreCommands: ${{ parameters.helixPreCommands }} + HelixConfiguration: ${{ parameters.platform.rid }}_${{ parameters.configuration }} + HelixTargetQueues: $(helix_${{ parameters.platform.queueKey }}) + HelixPreCommands: ${{ coalesce(parameters.platform.helixPreCommands, '') }} Creator: dotnet-bot SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: ne(variables['Agent.Os'], 'Windows_NT') - continueOnError: ${{ eq(variables.isOfficialBuild, 'true') }} + continueOnError: ${{ parameters.isOfficialBuild }} - task: CopyFiles@2 displayName: Gather Helix logs @@ -121,7 +101,7 @@ jobs: displayName: Publish Helix logs inputs: targetPath: '$(Build.StagingDirectory)/HelixLogs' - artifactName: Logs_${{ parameters.name }}_$(System.JobAttempt) + artifactName: Logs_SOS_Helix_${{ parameters.platform.name }}_${{ parameters.platform.arch }}_${{ parameters.configuration }}_$(System.JobAttempt) sbomEnabled: false continueOnError: true condition: always()