From 20ed7b463d9abf6dc25136422b6a53c53d0e12e3 Mon Sep 17 00:00:00 2001 From: Alon Gubkin Date: Mon, 5 Oct 2026 16:14:53 +0300 Subject: [PATCH 1/2] ci: serve Terraform providers from a read-only mirror in the Rust job The Terraform generator tests run `terraform init` in a fresh directory for each stack, downloading every provider again (~920 test-seconds per run). A shared plugin cache breaks under parallel inits; a filesystem mirror is only read. Providers not in the mirror still install from the registry. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-fast.yml | 7 ++++ scripts/terraform-provider-mirror.sh | 60 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100755 scripts/terraform-provider-mirror.sh diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 7f87d511bd..90fe857735 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -195,6 +195,13 @@ jobs: terraform_version: 1.15.3 terraform_wrapper: false + # The generator tests run `terraform init` in a fresh directory per stack. Serve the + # providers from a read-only local mirror instead of downloading them each time. + - name: Terraform provider mirror + run: | + scripts/terraform-provider-mirror.sh "$RUNNER_TEMP/terraform-mirror" "$RUNNER_TEMP/terraform.rc" + echo "TF_CLI_CONFIG_FILE=$RUNNER_TEMP/terraform.rc" >> "$GITHUB_ENV" + - uses: azure/setup-helm@v5 - name: Install kubeconform diff --git a/scripts/terraform-provider-mirror.sh b/scripts/terraform-provider-mirror.sh new file mode 100755 index 0000000000..b8da071427 --- /dev/null +++ b/scripts/terraform-provider-mirror.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Builds a read-only Terraform provider mirror for the Terraform generator tests. +# +# Every `terraform_validate` in crates/alien-terraform/src/test_utils.rs runs `terraform init` +# in a fresh directory, which downloads each provider again. A shared TF_PLUGIN_CACHE_DIR is not +# safe under parallel inits (they write the cache and fail checksum checks), but a filesystem +# mirror is only ever read. +# +# Usage: scripts/terraform-provider-mirror.sh +# Then export TF_CLI_CONFIG_FILE=. +# +# Keep the providers below in sync with `required_providers` in +# crates/alien-terraform/src/generator.rs. A provider missing here still installs from the +# registry, because the config only routes the mirrored providers to the mirror. +set -euo pipefail + +mirror="${1:?usage: $0 }" +config="${2:?usage: $0 }" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +cat > "$work/main.tf" <<'TF' +terraform { + required_providers { + aws = { source = "hashicorp/aws", version = ">= 5.0" } + awscc = { source = "hashicorp/awscc", version = ">= 1.0" } + tls = { source = "hashicorp/tls", version = ">= 4.0" } + google = { source = "hashicorp/google", version = ">= 5.0" } + google-beta = { source = "hashicorp/google-beta", version = ">= 6.0" } + azurerm = { source = "hashicorp/azurerm", version = ">= 4.75, < 5.0" } + azapi = { source = "Azure/azapi", version = ">= 2.6, < 3.0" } + time = { source = "hashicorp/time", version = ">= 0.9" } + kubernetes = { source = "hashicorp/kubernetes", version = ">= 2.30" } + helm = { source = "hashicorp/helm", version = ">= 3.0" } + random = { source = "hashicorp/random", version = ">= 3.6" } + } +} +TF + +terraform -chdir="$work" init -backend=false -input=false >/dev/null +mkdir -p "$mirror" +cp -R "$work/.terraform/providers/." "$mirror/" + +# Route exactly the mirrored providers (host/namespace/type) to the mirror. +providers="$(cd "$mirror" && find . -mindepth 3 -maxdepth 3 -type d | sed 's|^\./||' | sort)" +list="$(printf '%s\n' "$providers" | awk 'NF { printf "%s\"%s\"", (n++ ? ", " : ""), $0 }')" + +cat > "$config" < Date: Mon, 5 Oct 2026 16:22:25 +0300 Subject: [PATCH 2/2] test: keep the Terraform provider mirror in sync with the generator The generator's provider constraints now come from one PROVIDER_REQUIREMENTS table, and a unit test fails when scripts/terraform-provider-mirror.sh declares different providers or constraints. Co-Authored-By: Claude Opus 5.5 --- crates/alien-terraform/src/generator.rs | 88 ++++++++++++++++--------- scripts/terraform-provider-mirror.sh | 6 +- 2 files changed, 59 insertions(+), 35 deletions(-) diff --git a/crates/alien-terraform/src/generator.rs b/crates/alien-terraform/src/generator.rs index 1fb5b4362d..e56e137f34 100644 --- a/crates/alien-terraform/src/generator.rs +++ b/crates/alien-terraform/src/generator.rs @@ -1408,63 +1408,42 @@ fn versions_body( let mut provider_attrs: Vec = Vec::new(); if matches!(target.cloud_platform(), alien_core::Platform::Aws) { - provider_attrs.push(attr("aws", provider_decl_attr("hashicorp/aws", ">= 5.0"))); + provider_attrs.push(attr("aws", provider_requirement("aws"))); if include_awscc_provider { - provider_attrs.push(attr( - "awscc", - provider_decl_attr("hashicorp/awscc", ">= 1.0"), - )); + provider_attrs.push(attr("awscc", provider_requirement("awscc"))); } if matches!(target, TerraformTarget::Eks) { - provider_attrs.push(attr("tls", provider_decl_attr("hashicorp/tls", ">= 4.0"))); + provider_attrs.push(attr("tls", provider_requirement("tls"))); } } if matches!(target.cloud_platform(), alien_core::Platform::Gcp) { - provider_attrs.push(attr( - "google", - provider_decl_attr("hashicorp/google", ">= 5.0"), - )); + provider_attrs.push(attr("google", provider_requirement("google"))); if include_google_beta_provider { // The floor is where `google_vertex_ai_reasoning_engine` exists. - provider_attrs.push(attr( - "google-beta", - provider_decl_attr("hashicorp/google-beta", ">= 6.0"), - )); + provider_attrs.push(attr("google-beta", provider_requirement("google-beta"))); } } if matches!(target.cloud_platform(), alien_core::Platform::Azure) { // Upper bound deliberate: an open-ended constraint promises every future // major works, and azurerm 5 renamed arguments we emit. Raise it once the // emitters are ported, rather than letting a release decide for us. - provider_attrs.push(attr( - "azurerm", - provider_decl_attr("hashicorp/azurerm", ">= 4.75, < 5.0"), - )); + provider_attrs.push(attr("azurerm", provider_requirement("azurerm"))); if include_azapi_provider { // Bounded for the same reason as azurerm, and more sharply: the sandbox group is a // preview type, and a major bump is free to change what `body` accepts. - provider_attrs.push(attr( - "azapi", - provider_decl_attr("Azure/azapi", ">= 2.6, < 3.0"), - )); + provider_attrs.push(attr("azapi", provider_requirement("azapi"))); } } if include_time_provider { - provider_attrs.push(attr("time", provider_decl_attr("hashicorp/time", ">= 0.9"))); + provider_attrs.push(attr("time", provider_requirement("time"))); } if include_kubernetes_provider { - provider_attrs.push(attr( - "kubernetes", - provider_decl_attr("hashicorp/kubernetes", ">= 2.30"), - )); + provider_attrs.push(attr("kubernetes", provider_requirement("kubernetes"))); } if include_helm_provider { - provider_attrs.push(attr("helm", provider_decl_attr("hashicorp/helm", ">= 3.0"))); + provider_attrs.push(attr("helm", provider_requirement("helm"))); } - provider_attrs.push(attr( - "random", - provider_decl_attr("hashicorp/random", ">= 3.6"), - )); + provider_attrs.push(attr("random", provider_requirement("random"))); if let Some(registration) = registration { provider_attrs.push(attr( ®istration.provider_name, @@ -1489,6 +1468,32 @@ fn versions_body( Body::from(vec![Structure::Block(terraform_block)]) } +/// Every provider the generated stacks can declare: local name, source and version constraint. +/// `scripts/terraform-provider-mirror.sh` mirrors exactly these for CI, and +/// `provider_mirror_matches_generator_requirements` keeps the two in sync. +pub(crate) const PROVIDER_REQUIREMENTS: &[(&str, &str, &str)] = &[ + ("aws", "hashicorp/aws", ">= 5.0"), + ("awscc", "hashicorp/awscc", ">= 1.0"), + ("tls", "hashicorp/tls", ">= 4.0"), + ("google", "hashicorp/google", ">= 5.0"), + ("google-beta", "hashicorp/google-beta", ">= 6.0"), + ("azurerm", "hashicorp/azurerm", ">= 4.75, < 5.0"), + ("azapi", "Azure/azapi", ">= 2.6, < 3.0"), + ("time", "hashicorp/time", ">= 0.9"), + ("kubernetes", "hashicorp/kubernetes", ">= 2.30"), + ("helm", "hashicorp/helm", ">= 3.0"), + ("random", "hashicorp/random", ">= 3.6"), +]; + +/// The `required_providers` entry for a provider in [`PROVIDER_REQUIREMENTS`]. +fn provider_requirement(name: &str) -> Expression { + let (_, source, version) = PROVIDER_REQUIREMENTS + .iter() + .find(|(provider, _, _)| *provider == name) + .unwrap_or_else(|| panic!("provider '{name}' is not in PROVIDER_REQUIREMENTS")); + provider_decl_attr(source, version) +} + fn provider_decl_attr(source: &str, version: &str) -> Expression { expr::object([ ("source", Expression::String(source.to_string())), @@ -4322,6 +4327,25 @@ fn readme_kubernetes_destroy_order() -> &'static str { #[cfg(test)] mod tests { + /// CI serves these providers from a mirror that falls back to the registry only for providers + /// it doesn't list, so a constraint changed here without the script would break CI's inits. + #[test] + fn provider_mirror_matches_generator_requirements() { + let script = include_str!("../../../scripts/terraform-provider-mirror.sh"); + let mirrored: Vec = script + .lines() + .filter(|line| line.contains("source = ") && line.contains("version = ")) + .map(|line| line.split_whitespace().collect::>().join(" ")) + .collect(); + let expected: Vec = PROVIDER_REQUIREMENTS + .iter() + .map(|(name, source, version)| { + format!("{name} = {{ source = \"{source}\", version = \"{version}\" }}") + }) + .collect(); + assert_eq!(mirrored, expected); + } + use super::*; use alien_core::{Queue, RemoteStackManagement, ResourceLifecycle, ResourceRef}; diff --git a/scripts/terraform-provider-mirror.sh b/scripts/terraform-provider-mirror.sh index b8da071427..22ca6c4b0f 100755 --- a/scripts/terraform-provider-mirror.sh +++ b/scripts/terraform-provider-mirror.sh @@ -9,9 +9,9 @@ # Usage: scripts/terraform-provider-mirror.sh # Then export TF_CLI_CONFIG_FILE=. # -# Keep the providers below in sync with `required_providers` in -# crates/alien-terraform/src/generator.rs. A provider missing here still installs from the -# registry, because the config only routes the mirrored providers to the mirror. +# The providers below must match PROVIDER_REQUIREMENTS in crates/alien-terraform/src/generator.rs, +# line for line and in order; the `provider_mirror_matches_generator_requirements` test fails +# otherwise. A provider that isn't mirrored still installs from the registry. set -euo pipefail mirror="${1:?usage: $0 }"