From f2f0603d83772859cc73a80506d90fab25361ddd Mon Sep 17 00:00:00 2001 From: Theo van Kraay Date: Fri, 13 Feb 2026 19:42:52 +0000 Subject: [PATCH 1/3] Improve Cosmos DB Emulator startup reliability with retry logic and extended timeout --- .github/workflows/dotnet-build-and-test.yml | 123 +++++++++++++++++- .../CosmosChatHistoryProvider.cs | 1 + 2 files changed, 121 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dotnet-build-and-test.yml b/.github/workflows/dotnet-build-and-test.yml index 95842e703a2..b932982cfd7 100644 --- a/.github/workflows/dotnet-build-and-test.yml +++ b/.github/workflows/dotnet-build-and-test.yml @@ -89,10 +89,127 @@ jobs: if: ${{ runner.os == 'Windows' && (needs.paths-filter.outputs.cosmosDbChanges == 'true' || (github.event_name != 'pull_request' && matrix.integration-tests)) }} shell: pwsh run: | - Write-Host "Launching Azure Cosmos DB Emulator" + $ErrorActionPreference = "Stop" + $emulatorPath = "$env:ProgramFiles\Azure Cosmos DB Emulator\Microsoft.Azure.Cosmos.Emulator.exe" + $emulatorKey = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" + $maxRetries = 3 + $timeout = 600 # 10 minutes instead of default 4 minutes + $retryCount = 0 + $started = $false + + Write-Host "======================================" -ForegroundColor Cyan + Write-Host "Starting Azure Cosmos DB Emulator" -ForegroundColor Cyan + Write-Host "Timeout: $timeout seconds, Max retries: $maxRetries" -ForegroundColor Cyan + Write-Host "======================================" -ForegroundColor Cyan + Import-Module "$env:ProgramFiles\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator" - Start-CosmosDbEmulator -NoUI -Key "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" - echo "COSMOS_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV + + while (-not $started -and $retryCount -lt $maxRetries) { + $retryCount++ + + if ($retryCount -gt 1) { + Write-Host "`n--- Retry attempt $retryCount of $maxRetries ---" -ForegroundColor Yellow + + # Stop and cleanup before retry + Write-Host "Stopping existing emulator instances..." -ForegroundColor Yellow + Stop-CosmosDbEmulator -ErrorAction SilentlyContinue + Start-Sleep -Seconds 5 + + # Force kill if still running + Get-Process -Name "Microsoft.Azure.Cosmos.Emulator" -ErrorAction SilentlyContinue | Stop-Process -Force + Start-Sleep -Seconds 3 + + # Cleanup old data + $dataPath = Join-Path $env:LOCALAPPDATA "CosmosDbEmulator" + if (Test-Path $dataPath) { + Write-Host "Cleaning up emulator data..." -ForegroundColor Yellow + Remove-Item -Recurse -Force $dataPath -ErrorAction SilentlyContinue + Start-Sleep -Seconds 2 + } + } + + try { + Write-Host "`nStarting emulator (attempt $retryCount)..." -ForegroundColor Cyan + Start-CosmosDbEmulator -NoUI -Key $emulatorKey -AllowNetworkAccess -NoFirewall + + # Wait for emulator to be ready with extended timeout + $startTime = [DateTimeOffset]::Now + $stopTime = $startTime.AddSeconds($timeout) + $lastStatus = -1 + + Write-Host "Waiting for emulator to be ready..." -ForegroundColor Yellow + + do { + Start-Sleep -Seconds 5 + $elapsed = [int](([DateTimeOffset]::Now - $startTime).TotalSeconds) + + # Check status + $statusProcess = Start-Process $emulatorPath -ArgumentList "/getstatus" -PassThru -Wait -NoNewWindow -ErrorAction SilentlyContinue + $statusCode = if ($statusProcess) { $statusProcess.ExitCode } else { -1 } + + # Log status changes + if ($statusCode -ne $lastStatus) { + $statusMsg = switch ($statusCode) { + 1 { "Emulator is starting..." } + 2 { "Emulator is running!"; $started = $true } + 3 { "Emulator is stopped" } + default { "Unknown status (code: $statusCode)" } + } + Write-Host "[${elapsed}s] $statusMsg" -ForegroundColor $(if ($statusCode -eq 2) { "Green" } else { "Yellow" }) + $lastStatus = $statusCode + } + + if ($started) { break } + + # Check if process crashed + $processRunning = Get-Process -Name "Microsoft.Azure.Cosmos.Emulator" -ErrorAction SilentlyContinue + if (-not $processRunning -and $statusCode -ne 2) { + Write-Warning "Emulator process is not running" + break + } + + } while ([DateTimeOffset]::Now -lt $stopTime) + + if (-not $started) { + Write-Error "Emulator failed to start within $timeout seconds" + } + } + catch { + Write-Warning "Error on attempt ${retryCount}: $_" + if ($retryCount -ge $maxRetries) { throw } + } + } + + if ($started) { + Write-Host "`n======================================" -ForegroundColor Green + Write-Host "Cosmos DB Emulator started successfully!" -ForegroundColor Green + Write-Host "======================================`n" -ForegroundColor Green + + # Verify connectivity + try { + $response = Invoke-WebRequest -Uri "https://localhost:8081/" -UseBasicParsing -SkipCertificateCheck -TimeoutSec 10 -ErrorAction Stop + Write-Host "Emulator is responding (Status: $($response.StatusCode))" -ForegroundColor Green + } + catch { + Write-Warning "Emulator may not be fully ready: $_" + } + + echo "COSMOS_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV + } + else { + Write-Host "`n======================================" -ForegroundColor Red + Write-Host "Failed to start emulator after $maxRetries attempts" -ForegroundColor Red + Write-Host "======================================" -ForegroundColor Red + + # Show diagnostic info + $logPath = Join-Path $env:LOCALAPPDATA "CosmosDbEmulator\cosmosdb.log" + if (Test-Path $logPath) { + Write-Host "`nLast 30 lines of emulator log:" -ForegroundColor Yellow + Get-Content $logPath -Tail 30 + } + + throw "Failed to start Cosmos DB Emulator" + } - name: Setup dotnet uses: actions/setup-dotnet@v5.1.0 diff --git a/dotnet/src/Microsoft.Agents.AI.CosmosNoSql/CosmosChatHistoryProvider.cs b/dotnet/src/Microsoft.Agents.AI.CosmosNoSql/CosmosChatHistoryProvider.cs index 265f3a36752..1ca29c63b57 100644 --- a/dotnet/src/Microsoft.Agents.AI.CosmosNoSql/CosmosChatHistoryProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.CosmosNoSql/CosmosChatHistoryProvider.cs @@ -1,4 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. +// Trivial change to trigger CosmosDB emulator testing in CI/CD using System; using System.Collections.Generic; From 18cd0acee3f8974288ba2ceaf44a19ad8f9ca269 Mon Sep 17 00:00:00 2001 From: Theo van Kraay Date: Fri, 13 Feb 2026 19:55:18 +0000 Subject: [PATCH 2/3] Fix formatting issue --- .../CosmosChatHistoryProvider.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.CosmosNoSql/CosmosChatHistoryProvider.cs b/dotnet/src/Microsoft.Agents.AI.CosmosNoSql/CosmosChatHistoryProvider.cs index 1ca29c63b57..ebf9d731cb3 100644 --- a/dotnet/src/Microsoft.Agents.AI.CosmosNoSql/CosmosChatHistoryProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.CosmosNoSql/CosmosChatHistoryProvider.cs @@ -1,5 +1,4 @@ // Copyright (c) Microsoft. All rights reserved. -// Trivial change to trigger CosmosDB emulator testing in CI/CD using System; using System.Collections.Generic; @@ -16,7 +15,7 @@ namespace Microsoft.Agents.AI; /// -/// Provides a Cosmos DB implementation of the abstract class. +/// Provides a Cosmos DB implementation of the abstract class for persistent chat history storage. /// [RequiresUnreferencedCode("The CosmosChatHistoryProvider uses JSON serialization which is incompatible with trimming.")] [RequiresDynamicCode("The CosmosChatHistoryProvider uses JSON serialization which is incompatible with NativeAOT.")] From 36b67f56cd87d7271873a9bd878202735031b7f2 Mon Sep 17 00:00:00 2001 From: Theo van Kraay Date: Thu, 11 Jun 2026 20:58:41 +0100 Subject: [PATCH 3/3] Merge upstream/main and integrate enhanced Cosmos DB Emulator startup - Updated workflow structure to match new dotnet-build/dotnet-test split - Integrated enhanced emulator startup with 600s timeout and 3 retries - Updated environment variable to COSMOSDB_EMULATOR_AVAILABLE - Preserved all reliability improvements from original PR --- .github/workflows/dotnet-build-and-test.yml | 568 ++++++++++++++++---- 1 file changed, 464 insertions(+), 104 deletions(-) diff --git a/.github/workflows/dotnet-build-and-test.yml b/.github/workflows/dotnet-build-and-test.yml index b932982cfd7..6a1d3371cdf 100644 --- a/.github/workflows/dotnet-build-and-test.yml +++ b/.github/workflows/dotnet-build-and-test.yml @@ -37,9 +37,12 @@ jobs: outputs: dotnetChanges: ${{ steps.filter.outputs.dotnet }} cosmosDbChanges: ${{ steps.filter.outputs.cosmosdb }} + foundryHostingChanges: ${{ steps.filter.outputs.foundryHosting }} + functionsChanged: ${{ steps.filter.outputs.functions }} + coreChanged: ${{ steps.filter.outputs.core }} steps: - - uses: actions/checkout@v6 - - uses: dorny/paths-filter@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3 id: filter with: filters: | @@ -47,6 +50,40 @@ jobs: - 'dotnet/**' cosmosdb: - 'dotnet/src/Microsoft.Agents.AI.CosmosNoSql/**' + # The Foundry hosted-agent IT is costly (builds a container, pushes to ACR, + # provisions live agents). Only run it when the project under test, its + # dependency chain, the test container, the test fixture, or their tooling + # changed. Keep this list in sync with $hashedDirs in scripts/it-build-image.ps1. + foundryHosting: + - 'dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/**' + - 'dotnet/src/Microsoft.Agents.AI.Foundry/**' + - 'dotnet/src/Microsoft.Agents.AI/**' + - 'dotnet/src/Microsoft.Agents.AI.Abstractions/**' + - 'dotnet/src/Microsoft.Agents.AI.Workflows/**' + - 'dotnet/tests/Foundry.Hosting.IntegrationTests/**' + - 'dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/**' + - 'dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-AzureSearchRag/**' + - 'dotnet/Directory.Packages.props' + - 'dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1' + - '.github/workflows/dotnet-build-and-test.yml' + functions: + - 'dotnet/src/Microsoft.Agents.AI.DurableTask/**' + - 'dotnet/src/Microsoft.Agents.AI.Hosting.AzureFunctions/**' + - 'dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/**' + - 'dotnet/tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/**' + - '.github/actions/azure-functions-integration-setup/**' + - '.github/workflows/dotnet-build-and-test.yml' + core: + - 'dotnet/src/Microsoft.Agents.AI/**' + - 'dotnet/src/Microsoft.Agents.AI.Abstractions/**' + - 'dotnet/src/Microsoft.Agents.AI.OpenAI/**' + - 'dotnet/src/Microsoft.Agents.AI.Workflows/**' + - 'dotnet/src/Microsoft.Agents.AI.Workflows.Generators/**' + - 'dotnet/eng/scripts/New-FilteredSolution.ps1' + - 'dotnet/tests/Directory.Build.props' + - 'dotnet/Directory.Packages.props' + - 'dotnet/global.json' + - '.github/workflows/dotnet-build-and-test.yml' # run only if 'dotnet' files were changed - name: dotnet tests if: steps.filter.outputs.dotnet == 'true' @@ -59,22 +96,95 @@ jobs: if: steps.filter.outputs.dotnet != 'true' run: echo "NOT dotnet file" - dotnet-build-and-test: + # Build the full solution (including samples) on all TFMs. No tests. + dotnet-build: needs: paths-filter if: needs.paths-filter.outputs.dotnetChanges == 'true' strategy: fail-fast: false matrix: include: - - { targetFramework: "net10.0", os: "ubuntu-latest", configuration: Release, integration-tests: true, environment: "integration" } + - { targetFramework: "net10.0", os: "ubuntu-latest", configuration: Release } - { targetFramework: "net9.0", os: "windows-latest", configuration: Debug } - { targetFramework: "net8.0", os: "ubuntu-latest", configuration: Release } + - { targetFramework: "net472", os: "windows-latest", configuration: Release } + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + sparse-checkout: | + . + .github + dotnet + python + declarative-agents + + - name: Free runner disk space + uses: ./.github/actions/free-runner-disk-space + + - name: Setup dotnet + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 + with: + global-json-file: ${{ github.workspace }}/dotnet/global.json + - name: Build dotnet solutions + shell: bash + run: | + export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') + for solution in $SOLUTIONS; do + dotnet build $solution -c ${{ matrix.configuration }} --warnaserror + done + - name: Package install check + shell: bash + # All frameworks are only built for the release configuration, so we only run this step for the release configuration + # and dotnet new doesn't support net472 + if: matrix.configuration == 'Release' && matrix.targetFramework != 'net472' + run: | + TEMP_DIR=$(mktemp -d) + + export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') + for solution in $SOLUTIONS; do + dotnet pack $solution /property:TargetFrameworks=${{ matrix.targetFramework }} -c ${{ matrix.configuration }} --no-build --no-restore --output "$TEMP_DIR/artifacts" + done + + pushd "$TEMP_DIR" + + # Create a new console app to test the package installation + dotnet new console -f ${{ matrix.targetFramework }} --name packcheck --output consoleapp + + # Create minimal nuget.config and use only dotnet nuget commands + echo '' > consoleapp/nuget.config + + # Add sources with local first using dotnet nuget commands + dotnet nuget add source ../artifacts --name local --configfile consoleapp/nuget.config + dotnet nuget add source https://api.nuget.org/v3/index.json --name nuget.org --configfile consoleapp/nuget.config + + # Change to project directory to ensure local nuget.config is used + pushd consoleapp + dotnet add packcheck.csproj package Microsoft.Agents.AI --prerelease + dotnet build -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} packcheck.csproj + + # Clean up + popd + popd + rm -rf "$TEMP_DIR" + + # Build src+tests only (no samples) for a single TFM and run tests. + dotnet-test: + needs: paths-filter + if: needs.paths-filter.outputs.dotnetChanges == 'true' + strategy: + fail-fast: false + matrix: + include: + - { targetFramework: "net10.0", os: "ubuntu-latest", configuration: Release, integration-tests: true, environment: "integration" } - { targetFramework: "net472", os: "windows-latest", configuration: Release, integration-tests: true, environment: "integration" } runs-on: ${{ matrix.os }} environment: ${{ matrix.environment }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false sparse-checkout: | @@ -82,7 +192,10 @@ jobs: .github dotnet python - workflow-samples + declarative-agents + + - name: Free runner disk space + uses: ./.github/actions/free-runner-disk-space # Start Cosmos DB Emulator for all integration tests and only for unit tests when CosmosDB changes happened) - name: Start Azure Cosmos DB Emulator @@ -194,7 +307,7 @@ jobs: Write-Warning "Emulator may not be fully ready: $_" } - echo "COSMOS_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV + echo "COSMOSDB_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV } else { Write-Host "`n======================================" -ForegroundColor Red @@ -212,70 +325,64 @@ jobs: } - name: Setup dotnet - uses: actions/setup-dotnet@v5.1.0 + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 with: global-json-file: ${{ github.workspace }}/dotnet/global.json - - name: Build dotnet solutions - shell: bash + + - name: Generate test solution (no samples) + shell: pwsh run: | - export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') - for solution in $SOLUTIONS; do - dotnet build $solution -c ${{ matrix.configuration }} --warnaserror - done - - name: Package install check + ./dotnet/eng/scripts/New-FilteredSolution.ps1 ` + -Solution dotnet/agent-framework-dotnet.slnx ` + -TargetFramework ${{ matrix.targetFramework }} ` + -Configuration ${{ matrix.configuration }} ` + -ExcludeSamples ` + -OutputPath dotnet/filtered.slnx ` + -Verbose + + - name: Build src and tests shell: bash - # All frameworks are only built for the release configuration, so we only run this step for the release configuration - # and dotnet new doesn't support net472 - if: matrix.configuration == 'Release' && matrix.targetFramework != 'net472' - run: | - TEMP_DIR=$(mktemp -d) + run: dotnet build dotnet/filtered.slnx -c ${{ matrix.configuration }} -f ${{ matrix.targetFramework }} --warnaserror - export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') - for solution in $SOLUTIONS; do - dotnet pack $solution /property:TargetFrameworks=${{ matrix.targetFramework }} -c ${{ matrix.configuration }} --no-build --no-restore --output "$TEMP_DIR/artifacts" - done - - pushd "$TEMP_DIR" - - # Create a new console app to test the package installation - dotnet new console -f ${{ matrix.targetFramework }} --name packcheck --output consoleapp - - # Create minimal nuget.config and use only dotnet nuget commands - echo '' > consoleapp/nuget.config - - # Add sources with local first using dotnet nuget commands - dotnet nuget add source ../artifacts --name local --configfile consoleapp/nuget.config - dotnet nuget add source https://api.nuget.org/v3/index.json --name nuget.org --configfile consoleapp/nuget.config - - # Change to project directory to ensure local nuget.config is used - pushd consoleapp - dotnet add packcheck.csproj package Microsoft.Agents.AI --prerelease - dotnet build -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} packcheck.csproj - - # Clean up - popd - popd - rm -rf "$TEMP_DIR" + - name: Generate test-type filtered solutions + shell: pwsh + run: | + $commonArgs = @{ + Solution = "dotnet/filtered.slnx" + TargetFramework = "${{ matrix.targetFramework }}" + Configuration = "${{ matrix.configuration }}" + Verbose = $true + } + ./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs ` + -TestProjectNameIncludeFilter "*UnitTests*" ` + -OutputPath dotnet/filtered-unit.slnx + ./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs ` + -TestProjectNameIncludeFilter "*IntegrationTests*" ` + -TestProjectNameExcludeFilter "*DurableTask.IntegrationTests*","*AzureFunctions.IntegrationTests*" ` + -OutputPath dotnet/filtered-integration.slnx - name: Run Unit Tests - shell: bash + shell: pwsh + working-directory: dotnet run: | - export UT_PROJECTS=$(find ./dotnet -type f -name "*.UnitTests.csproj" | tr '\n' ' ') - for project in $UT_PROJECTS; do - # Query the project's target frameworks using MSBuild with the current configuration - target_frameworks=$(dotnet msbuild $project -getProperty:TargetFrameworks -p:Configuration=${{ matrix.configuration }} -nologo 2>/dev/null | tr -d '\r') - - # Check if the project supports the target framework - if [[ "$target_frameworks" == *"${{ matrix.targetFramework }}"* ]]; then - if [[ "${{ matrix.targetFramework }}" == "${{ env.COVERAGE_FRAMEWORK }}" ]]; then - dotnet test -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx --collect:"XPlat Code Coverage" --results-directory:"TestResults/Coverage/" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.ExcludeByAttribute=GeneratedCodeAttribute,CompilerGeneratedAttribute,ExcludeFromCodeCoverageAttribute - else - dotnet test -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx - fi - else - echo "Skipping $project - does not support target framework ${{ matrix.targetFramework }} (supports: $target_frameworks)" - fi - done + $coverageSettings = Join-Path $PWD "tests/coverage.runsettings" + $coverageArgs = @() + if ("${{ matrix.targetFramework }}" -eq "${{ env.COVERAGE_FRAMEWORK }}") { + $coverageArgs = @( + "--coverage", + "--coverage-output-format", "cobertura", + "--coverage-settings", $coverageSettings, + "--results-directory", "../TestResults/Coverage/" + ) + } + + dotnet test --solution ./filtered-unit.slnx ` + -f ${{ matrix.targetFramework }} ` + -c ${{ matrix.configuration }} ` + --no-build -v Normal ` + --report-xunit-trx ` + --ignore-exit-code 8 ` + @coverageArgs env: # Cosmos DB Emulator connection settings COSMOSDB_ENDPOINT: https://localhost:8081 @@ -287,68 +394,62 @@ jobs: - name: Azure CLI Login if: github.event_name != 'pull_request' && matrix.integration-tests - uses: azure/login@v2 + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2 with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - # This setup action is required for both Durable Task and Azure Functions integration tests. - # We only run it on Ubuntu since the Durable Task and Azure Functions features are not available - # on .NET Framework (net472) which is what we use the Windows runner for. - - name: Set up Durable Task and Azure Functions Integration Test Emulators - if: github.event_name != 'pull_request' && matrix.integration-tests && matrix.os == 'ubuntu-latest' - uses: ./.github/actions/azure-functions-integration-setup - id: azure-functions-setup - - name: Run Integration Tests - shell: bash + shell: pwsh + working-directory: dotnet if: github.event_name != 'pull_request' && matrix.integration-tests run: | - export INTEGRATION_TEST_PROJECTS=$(find ./dotnet -type f -name "*IntegrationTests.csproj" | tr '\n' ' ') - for project in $INTEGRATION_TEST_PROJECTS; do - # Query the project's target frameworks using MSBuild with the current configuration - target_frameworks=$(dotnet msbuild $project -getProperty:TargetFrameworks -p:Configuration=${{ matrix.configuration }} -nologo 2>/dev/null | tr -d '\r') - - # Check if the project supports the target framework - if [[ "$target_frameworks" == *"${{ matrix.targetFramework }}"* ]]; then - dotnet test -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx --filter "Category!=IntegrationDisabled" - else - echo "Skipping $project - does not support target framework ${{ matrix.targetFramework }} (supports: $target_frameworks)" - fi - done + dotnet test --solution ./filtered-integration.slnx ` + -f ${{ matrix.targetFramework }} ` + -c ${{ matrix.configuration }} ` + --no-build -v Normal ` + --report-xunit-trx ` + --report-junit ` + --results-directory ../IntegrationTestResults/ ` + --ignore-exit-code 8 ` + --filter-not-trait "Category=IntegrationDisabled" ` + --filter-not-trait "Category=FoundryHostedAgents" ` + --parallel-algorithm aggressive ` + --max-threads 2.0x env: # Cosmos DB Emulator connection settings COSMOSDB_ENDPOINT: https://localhost:8081 COSMOSDB_KEY: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw== # OpenAI Models - OpenAI__ApiKey: ${{ secrets.OPENAI__APIKEY }} - OpenAI__ChatModelId: ${{ vars.OPENAI__CHATMODELID }} - OpenAI__ChatReasoningModelId: ${{ vars.OPENAI__CHATREASONINGMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_MODEL_NAME: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_REASONING_MODEL_NAME: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # Azure OpenAI Models - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} - AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} # Azure AI Foundry - AzureAI__Endpoint: ${{ secrets.AZUREAI__ENDPOINT }} - AzureAI__DeploymentName: ${{ vars.AZUREAI__DEPLOYMENTNAME }} - AzureAI__BingConnectionId: ${{ vars.AZUREAI__BINGCONECTIONID }} - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} - FOUNDRY_MEDIA_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MEDIA_DEPLOYMENT_NAME }} - FOUNDRY_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL_DEPLOYMENT_NAME }} - FOUNDRY_CONNECTION_GROUNDING_TOOL: ${{ vars.FOUNDRY_CONNECTION_GROUNDING_TOOL }} + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }} + # Anthropic Models + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_CHAT_MODEL_NAME: ${{ vars.ANTHROPIC_CHAT_MODEL_NAME }} + ANTHROPIC_REASONING_MODEL_NAME: ${{ vars.ANTHROPIC_REASONING_MODEL_NAME }} # Generate test reports and check coverage - name: Generate test reports if: matrix.targetFramework == env.COVERAGE_FRAMEWORK - uses: danielpalme/ReportGenerator-GitHub-Action@5.5.1 + uses: danielpalme/ReportGenerator-GitHub-Action@2a82782178b2816d9d6960a7345fdd164791b323 # 5.5.3 with: - reports: "./TestResults/Coverage/**/coverage.cobertura.xml" + reports: "./TestResults/Coverage/**/*.cobertura.xml" targetdir: "./TestResults/Reports" reporttypes: "HtmlInline;JsonSummary" - name: Upload coverage report artifact if: matrix.targetFramework == env.COVERAGE_FRAMEWORK - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: CoverageReport-${{ matrix.os }}-${{ matrix.targetFramework }}-${{ matrix.configuration }} # Artifact name path: ./TestResults/Reports # Directory containing files to upload @@ -356,13 +457,211 @@ jobs: - name: Check coverage if: matrix.targetFramework == env.COVERAGE_FRAMEWORK shell: pwsh - run: .github/workflows/dotnet-check-coverage.ps1 -JsonReportPath "TestResults/Reports/Summary.json" -CoverageThreshold $env:COVERAGE_THRESHOLD + run: ./dotnet/eng/scripts/dotnet-check-coverage.ps1 -JsonReportPath "TestResults/Reports/Summary.json" -CoverageThreshold $env:COVERAGE_THRESHOLD + + - name: Upload integration test results + if: always() && github.event_name != 'pull_request' && matrix.integration-tests + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: dotnet-test-results-${{ matrix.targetFramework }}-${{ matrix.os }} + path: IntegrationTestResults/**/*.junit + if-no-files-found: ignore + + # The Foundry hosted-agent IT is costly (it builds a container, pushes to ACR, and provisions + # live agents on a separate Foundry project). Running it in its own job keeps the overall + # workflow time roughly flat: it executes in parallel to dotnet-build and dotnet-test and is + # gated on paths-filter.outputs.foundryHostingChanges so unrelated edits skip the work. + dotnet-foundry-hosted-it: + needs: paths-filter + if: github.event_name != 'pull_request' && needs.paths-filter.outputs.foundryHostingChanges == 'true' + runs-on: ubuntu-latest + environment: integration + env: + configuration: Release + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + sparse-checkout: | + . + .github + dotnet + python + + - name: Free runner disk space + uses: ./.github/actions/free-runner-disk-space + + - name: Setup dotnet + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 + with: + global-json-file: ${{ github.workspace }}/dotnet/global.json + + # Build the test csproj directly instead of a filtered slnx + -f override. + # The test project pins TargetFrameworks=net10.0 and its ProjectReference closure + # gives MSBuild a single-rooted graph, so each multi-targeted dependency is invoked + # exactly once for net10.0. This avoids the MSB3026/MSB3491/MSB4018/MSB3883 file-lock + # collisions caused by parallel inner-builds racing on shared bin/obj output paths + # under the previous slnx + global TFM override approach. + - name: Build Foundry hosted IT (and its deps) + shell: bash + run: dotnet build dotnet/tests/Foundry.Hosting.IntegrationTests/Foundry.Hosting.IntegrationTests.csproj -c "$configuration" --warnaserror + + - name: Azure CLI Login + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + # We rebuild and push the test container image on every IT run so framework code changes + # are picked up; the image tag is content-hashed across the test container source AND its + # framework project references, so identical content is a no-op push. + # + # The script always passes --no-dependencies to dotnet publish so publish never re-touches + # the framework lib DLLs the prior "Build Foundry hosted IT (and its deps)" step produced. + # This structurally eliminates the MSB3026 collision that VBCSCompiler from the prebuild + # would otherwise cause by holding file handles to those DLLs. Do not remove the prebuild + # step: the subsequent `dotnet test --no-build` step and the publish's ProjectReference + # resolution both depend on the prebuilt outputs being present. + - name: Build and push Foundry Hosted Agents test container + id: build-foundry-hosted-image + shell: pwsh + working-directory: ${{ github.workspace }} + run: | + $registry = "${{ vars.IT_HOSTED_AGENT_REGISTRY }}" + if ([string]::IsNullOrWhiteSpace($registry)) { + throw "IT_HOSTED_AGENT_REGISTRY not set in the integration environment." + } + & "${{ github.workspace }}/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1" -Registry $registry | Tee-Object -FilePath $env:GITHUB_ENV -Append + + - name: Run Foundry Hosted Agents Integration Tests + shell: pwsh + working-directory: dotnet + run: | + dotnet test --project tests/Foundry.Hosting.IntegrationTests/Foundry.Hosting.IntegrationTests.csproj ` + -c $env:configuration ` + --no-build -v Normal ` + --report-xunit-trx ` + --ignore-exit-code 8 ` + --filter-trait "Category=FoundryHostedAgents" + env: + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.IT_HOSTED_AGENT_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.IT_HOSTED_AGENT_MODEL_DEPLOYMENT_NAME }} + # Azure AI Search (for the azure-search-rag scenario). Reuses the integration + # environment secrets shared with python-sample-validation.yml. The index is + # provisioned out of band; see dotnet/tests/Foundry.Hosting.IntegrationTests/README.md + # for the required schema and seed content. + AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }} + AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }} + # IT_HOSTED_AGENT_IMAGE was exported into $GITHUB_ENV by the previous step. + + # DurableTask and AzureFunctions integration tests (ubuntu/net10.0 only). + # Split from main dotnet-test job for path-based filtering and parallelism. + dotnet-test-functions: + needs: [paths-filter] + if: > + github.event_name != 'pull_request' && + (needs.paths-filter.outputs.functionsChanged == 'true' || + needs.paths-filter.outputs.coreChanged == 'true' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + environment: integration + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + sparse-checkout: | + . + .github + dotnet + python + declarative-agents + + - name: Free runner disk space + uses: ./.github/actions/free-runner-disk-space + + - name: Setup dotnet + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 + with: + global-json-file: ${{ github.workspace }}/dotnet/global.json + + - name: Build functions integration test projects + shell: bash + working-directory: dotnet + run: | + dotnet build ./tests/Microsoft.Agents.AI.DurableTask.IntegrationTests -c Release -f net10.0 --warnaserror + dotnet build ./tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests -c Release -f net10.0 --warnaserror + + - name: Azure CLI Login + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Set up Durable Task and Azure Functions Integration Test Emulators + uses: ./.github/actions/azure-functions-integration-setup + id: azure-functions-setup + + - name: Run Functions Integration Tests + shell: pwsh + working-directory: dotnet + run: | + # Run DurableTask integration tests + dotnet test ` + --project ./tests/Microsoft.Agents.AI.DurableTask.IntegrationTests ` + -f net10.0 ` + -c Release ` + --no-build -v Normal ` + --report-xunit-trx ` + --report-junit ` + --results-directory ../IntegrationTestResults/ ` + --ignore-exit-code 8 ` + --filter-not-trait "Category=IntegrationDisabled" ` + --parallel-algorithm aggressive ` + --max-threads 2.0x + + # Run AzureFunctions integration tests + dotnet test ` + --project ./tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests ` + -f net10.0 ` + -c Release ` + --no-build -v Normal ` + --report-xunit-trx ` + --report-junit ` + --results-directory ../IntegrationTestResults/ ` + --ignore-exit-code 8 ` + --filter-not-trait "Category=IntegrationDisabled" ` + --parallel-algorithm aggressive ` + --max-threads 2.0x + env: + # OpenAI Models + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_MODEL_NAME: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_REASONING_MODEL_NAME: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + # Azure OpenAI Models + AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + # Azure AI Foundry + AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }} + AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }} + + - name: Upload functions test results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: dotnet-test-results-functions-net10.0-ubuntu-latest + path: IntegrationTestResults/**/*.junit + if-no-files-found: ignore # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed dotnet-build-and-test-check: if: always() runs-on: ubuntu-latest - needs: [dotnet-build-and-test] + needs: [dotnet-build, dotnet-test, dotnet-foundry-hosted-it, dotnet-test-functions] steps: - name: Get Date shell: bash @@ -390,13 +689,74 @@ jobs: - name: Fail workflow if tests failed id: check_tests_failed if: contains(join(needs.*.result, ','), 'failure') - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 with: script: core.setFailed('Integration Tests Failed!') - name: Fail workflow if tests cancelled id: check_tests_cancelled if: contains(join(needs.*.result, ','), 'cancelled') - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 with: script: core.setFailed('Integration Tests Cancelled!') + + # Integration test trend report (aggregates JUnit XML results from dotnet test jobs) + dotnet-integration-test-report: + name: Integration Test Report + if: > + always() && + github.event_name != 'pull_request' && + (contains(join(needs.*.result, ','), 'success') || + contains(join(needs.*.result, ','), 'failure')) + needs: [dotnet-test, dotnet-test-functions] + runs-on: ubuntu-latest + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + sparse-checkout: | + .github/actions/python-setup + python + - name: Set up python and install the project + uses: ./.github/actions/python-setup + with: + python-version: "3.13" + os: ${{ runner.os }} + - name: Download all test results from current run + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: dotnet-test-results-* + path: dotnet-test-results/ + - name: Restore report history cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: python/dotnet-integration-report-history.json + key: dotnet-integration-report-history-${{ github.run_id }} + restore-keys: | + dotnet-integration-report-history- + - name: Generate trend report + run: > + uv run python scripts/integration_test_report/aggregate.py + ../dotnet-test-results/ + dotnet-integration-report-history.json + dotnet-integration-test-report.md + - name: Post to Job Summary + if: always() + run: cat dotnet-integration-test-report.md >> $GITHUB_STEP_SUMMARY + - name: Save report history cache + if: always() + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: python/dotnet-integration-report-history.json + key: dotnet-integration-report-history-${{ github.run_id }} + - name: Upload trend report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: dotnet-integration-test-report + path: | + python/dotnet-integration-test-report.md + python/dotnet-integration-report-history.json