AztecProtocol · AztecBot · Jun 9, 2026 · Jun 9, 2026
@@ -9,19 +9,7 @@ state_dir=$4
 
 cd $(dirname $0)
 
-bid_per_cpu_hour=0.0433
-
-instance_types_amd64=(
-  m6a
-  m7a
-  m7i
-)
-
-instance_types_arm64=(
-  r7g
-)
-
-# Declare an associative array to map CPU counts to instance type suffixes.
+# Map a cpu count to its instance size suffix.
 declare -A cpu_map
 cpu_map=(
   [2]="large"
@@ -37,32 +25,69 @@ cpu_map=(
   [0192]="metal"
 )
 
+# Spot pool preference by cpu count (lower number = tried first). 128 is the
+# known-good build size, then 192, then 64 as a last resort before on-demand.
+declare -A cpu_priority=([128]=1 [192]=2 [64]=3)
+
+# Capacity-diverse spot pools per cpu count. Restricted to families with
+# >=4 GB/vCPU (m/r) so the build's measured ~1.9 GB/vCPU peak keeps ~2x
+# headroom; c-series (2 GB/vCPU) is opt-in via CI_SPOT_INCLUDE_C. All entries
+# are confirmed offered as spot across every us-east-2 AZ.
+function spot_pools_for {
+  local cpu=$1
+  case "$arch" in
+    x86_64|amd64)
+      case "$cpu" in
+        128) echo "m6a.32xlarge m7a.32xlarge m6i.32xlarge r6a.32xlarge r7a.32xlarge" ;;
+        192) echo "m6a.48xlarge m7a.48xlarge m7i.48xlarge r6a.48xlarge r7a.48xlarge" ;;
+        64)  echo "m6a.16xlarge m7a.16xlarge m7i.16xlarge m6i.16xlarge m5.16xlarge r6a.16xlarge" ;;
+        *)   local s=${cpu_map[$cpu]:?Unknown cpu count: $cpu}; echo "m6a.$s m7a.$s m6i.$s r6a.$s" ;;
+      esac ;;
+    arm64)
+      case "$cpu" in
+        64) echo "r7g.16xlarge r8g.16xlarge m7g.16xlarge m8g.16xlarge" ;;
+        *)  local s=${cpu_map[$cpu]:?Unknown cpu count: $cpu}; echo "r7g.$s r8g.$s m7g.$s m8g.$s" ;;
+      esac ;;
+  esac
+}
+
+# Optional c-series pools (2 GB/vCPU). Gated behind CI_SPOT_INCLUDE_C until
+# build-phase parallelism is throttled to fit the lower memory headroom.
+function spot_pools_c_for {
+  local cpu=$1
+  case "$arch" in
+    x86_64|amd64)
+      case "$cpu" in
+        128) echo "c6a.32xlarge c6i.32xlarge" ;;
+        192) echo "c6a.48xlarge c7a.48xlarge" ;;
+        64)  echo "c6a.16xlarge c6i.16xlarge" ;;
+      esac ;;
+  esac
+}
+
 if [ "$arch" == "x86_64" ] || [ "$arch" == "amd64" ]; then
-  instances=("${instance_types_amd64[@]}")
   ami=${AMI:-$(cat ./aws/ami_id_amd64)}
 elif [ "$arch" == "arm64" ]; then
-  instances=("${instance_types_arm64[@]}")
   ami=${AMI:-$(cat ./aws/ami_id_arm64)}
 else
   echo "Unsupported arch: $arch" >&2
   exit 1
 fi
 
+# Build the fleet override set as space-separated "instanceType:priority" tokens.
+# AWS_INSTANCE pins a single pool (e.g. for debugging); otherwise diversify
+# across every requested size.
+overrides=""
 if [ -n "${AWS_INSTANCE:-}" ]; then
-  instances=("$AWS_INSTANCE")
+  overrides="$AWS_INSTANCE:1"
+else
+  IFS=',' read -ra cpu_list <<< "$cpus"
+  for cpu in "${cpu_list[@]}"; do
+    prio=${cpu_priority[$cpu]:-9}
+    types="$(spot_pools_for "$cpu")"
+    [ "${CI_SPOT_INCLUDE_C:-0}" -eq 1 ] && types+=" $(spot_pools_c_for "$cpu")"
+    for t in $types; do overrides+=" $t:$prio"; done
+  done
 fi
 
-set +e
-IFS=',' read -ra cpu_list <<< "$cpus"
-for cpu in "${cpu_list[@]}"; do
-  price=$(jq -n "$bid_per_cpu_hour*$cpu*100000 | round / 100000")
-  suffix=${cpu_map[$cpu]}
-  for inst in "${instances[@]}"; do
-    instance_type="$inst.$suffix"
-    aws_request_instance_type $name $instance_type $price $ami $state_dir
-    code=$?
-    [[ "$code" -eq 0 || "$code" -eq 143 || "$code" -eq 130 ]] && exit $code
-    echo "Instance request exited with code: $code"
-  done
-done
-exit 1
+FLEET_OVERRIDES="${overrides# }" aws_request_instance_type "$name" "$ami" "$state_dir"
@@ -2,22 +2,36 @@
 source $(git rev-parse --show-toplevel)/ci3/source
 
 name=$1
-instance_type=$2
-price=$3
-ami=$4
-state_dir=$5
+ami=$2
+state_dir=$3
 
-spec_path=$state_dir/spec.json
-sir_path=$state_dir/sir
+# Space-separated "instanceType:priority" tokens describing the spot pools to
+# diversify across (set by aws_request_instance).
+overrides_spec=${FLEET_OVERRIDES:?FLEET_OVERRIDES must be set}
+
+spec_path=$state_dir/lt_data.json
 iid_path=$state_dir/iid
 ip_path=$state_dir/ip
+lt_path=$state_dir/launch_template_id
+
+export AWS_DEFAULT_REGION=us-east-2
 
+# Delete the ephemeral launch template. Safe once instances are launched (they
+# no longer reference it) and idempotent, so we run it on every exit.
+function delete_lt {
+  [ -f "$lt_path" ] || return 0
+  aws ec2 delete-launch-template --launch-template-id "$(cat "$lt_path")" >/dev/null 2>&1 || true
+  rm -f "$lt_path"
+}
+
+# On interrupt, tear down whatever we acquired. On normal exit we only drop the
+# launch template (the instance must survive for the caller).
 function cleanup {
-  aws_terminate_instance $state_dir
+  delete_lt
+  aws_terminate_instance "$state_dir"
 }
 trap 'cleanup' SIGINT SIGTERM
-
-export AWS_DEFAULT_REGION=us-east-2
+trap 'delete_lt' EXIT
 
 # KeyName is optional: omit when KEY_NAME is empty (SSM-only mode).
 key_name_json=""
@@ -27,7 +41,6 @@ fi
 
 # Always attach the instance profile so containers can use instance-profile credentials.
 CI3_INSTANCE_PROFILE_NAME=${CI3_INSTANCE_PROFILE_NAME:-ci3-build-instance-profile}
-iam_profile_json="\"IamInstanceProfile\": { \"Name\": \"$CI3_INSTANCE_PROFILE_NAME\" },"
 
 # In SSM mode use the CI3 security group; in SSH mode use the original SG (allows port 22).
 if [ -z "${KEY_NAME:-}" ]; then
@@ -36,13 +49,35 @@ else
   sg_id="sg-0ccd4e5df0dcca0c9"
 fi
 
-launch_spec=$(cat <<EOF
+# create-fleet spreads across AZs via subnet overrides, which must live in the
+# same VPC as the security group. Pick one subnet per AZ.
+vpc_id=$(aws ec2 describe-security-groups --group-ids "$sg_id" \
+  --query 'SecurityGroups[0].VpcId' --output text)
+mapfile -t subnets < <(aws ec2 describe-subnets \
+  --filters "Name=vpc-id,Values=$vpc_id" \
+  --query 'Subnets[].[AvailabilityZone,SubnetId]' --output text | sort -k1,1 -u | awk '{print $2}')
+if [ "${#subnets[@]}" -eq 0 ]; then
+  echo_stderr "No subnets found in VPC $vpc_id for security group $sg_id."
+  exit 1
+fi
+
+# Instance tags, applied at launch via the launch template (propagate to fleet instances).
+tags_json="{\"Key\":\"Name\",\"Value\":\"$name\"},{\"Key\":\"Group\",\"Value\":\"build-instance\"}"
+[ -n "${GITHUB_ACTOR:-}" ] && tags_json+=",{\"Key\":\"GithubActor\",\"Value\":\"${GITHUB_ACTOR//[\[\]]/}\"}"
+[ -n "${CI_MODE:-}" ] && tags_json+=",{\"Key\":\"CICommand\",\"Value\":\"$CI_MODE\"}"
+[ -n "${CI_DASHBOARD:-}" ] && tags_json+=",{\"Key\":\"Dashboard\",\"Value\":\"$CI_DASHBOARD\"}"
+if [ "${UNSAFE_AWS_KEEP_ALIVE:-0}" -eq 1 ]; then
+  echo_stderr "You have set UNSAFE_AWS_KEEP_ALIVE=1, so the instance will not be terminated after 1.5 hours by the reaper script. Make sure you shut the machine down when done."
+  tags_json+=",{\"Key\":\"Keep-Alive\",\"Value\":\"true\"}"
+fi
+
+# Common launch config shared by every pool. Sizes/AZs vary per fleet override.
+lt_data=$(cat <<EOF
 {
   "ImageId": "$ami",
   $key_name_json
-  $iam_profile_json
+  "IamInstanceProfile": { "Name": "$CI3_INSTANCE_PROFILE_NAME" },
   "SecurityGroupIds": ["$sg_id"],
-  "InstanceType": "$instance_type",
   "BlockDeviceMappings": [
     {
       "DeviceName": "/dev/sda1",
@@ -53,75 +88,101 @@ launch_spec=$(cat <<EOF
         "Iops": 4000
       }
     }
+  ],
+  "TagSpecifications": [
+    { "ResourceType": "instance", "Tags": [ $tags_json ] }
   ]
 }
 EOF
 )
+echo "$lt_data" > "$spec_path"
+
+lt_name="ci3-${name}-$$-$(date +%s)"
+lt_id=$(aws ec2 create-launch-template \
+  --launch-template-name "$lt_name" \
+  --launch-template-data "$lt_data" \
+  --query 'LaunchTemplate.LaunchTemplateId' --output text)
+echo "$lt_id" > "$lt_path"
+
+# Cross every pool with every AZ. Priority biases capacity-optimized toward the
+# preferred sizes (see aws_request_instance) while still optimizing for depth.
+overrides_json=""
+for tok in $overrides_spec; do
+  itype=${tok%%:*}
+  prio=${tok##*:}
+  for sn in "${subnets[@]}"; do
+    overrides_json+="{\"InstanceType\":\"$itype\",\"SubnetId\":\"$sn\",\"Priority\":$prio},"
+  done
+done
+ltc="[{\"LaunchTemplateSpecification\":{\"LaunchTemplateId\":\"$lt_id\",\"Version\":\"\$Latest\"},\"Overrides\":[${overrides_json%,}]}]"
 
-# Save the launch specification to a temporary file.
-echo "$launch_spec" > "$spec_path"
+info="(name: $name) (pools: ${overrides_spec// /,}) (ami: $ami)"
 
-info="(name: $name) (type: $instance_type) (ami: $ami) (bid: $price)"
+iid=""
+instance_type=""
+lifecycle=""
 
+# Try spot first, retrying the whole diversified fleet until the time budget
+# expires (preserves the historical "give spot ~1 min, then on-demand" pattern).
 if [ "${NO_SPOT:-0}" -ne 1 ]; then
-  echo "Requesting $instance_type spot instance $info..."
-  sir=$(aws ec2 request-spot-instances \
-    --spot-price "$price" \
-    --instance-count 1 \
-    --type "one-time" \
-    --launch-specification file://$spec_path \
-    --query "SpotInstanceRequests[*].[SpotInstanceRequestId]" \
-    --output text)
-  echo $sir > $sir_path
-
-  echo "Waiting for instance id for spot request: $sir..."
-  sleep 5
-  for i in {1..6}; do
-    iid=$(aws ec2 describe-spot-instance-requests \
-      --spot-instance-request-ids $sir \
-      --query "SpotInstanceRequests[*].[InstanceId]" \
-      --output text)
-    [ -z "$iid" -o "$iid" == "None" ] || break
-
-    if [ $i -eq 6 ]; then
-      echo "Timeout waiting for spot request."
-      # Cancel spot request. We may still get allocated an instance if it's *just* happened.
-      aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $sir > /dev/null
+  spot_timeout=${CI_SPOT_TIMEOUT:-60}
+  spot_poll=${CI_SPOT_POLL:-10}
+  SECONDS=0
+  while true; do
+    echo "Requesting spot fleet $info (capacity-optimized-prioritized, ${spot_timeout}s budget)..."
+    resp=$(aws ec2 create-fleet \
+      --type instant \
+      --target-capacity-specification "TotalTargetCapacity=1,DefaultTargetCapacityType=spot" \
+      --spot-options "AllocationStrategy=capacity-optimized-prioritized" \
+      --launch-template-configs "$ltc" \
+      --output json 2>"$state_dir/fleet_err") || true
+    iid=$(echo "$resp" | jq -r '.Instances[0].InstanceIds[0] // empty' 2>/dev/null || true)
+    if [ -n "$iid" ]; then
+      instance_type=$(echo "$resp" | jq -r '.Instances[0].InstanceType // empty')
+      lifecycle=spot
+      break
     fi
-
-    sleep 5
+    if (( SECONDS + spot_poll >= spot_timeout )); then
+      echo "Spot fleet unfulfilled after ${spot_timeout}s; falling back to on-demand."
+      break
+    fi
+    sleep "$spot_poll"
   done
-  echo $iid > $iid_path
 fi
 
-if [ -z "${iid:-}" -o "${iid:-}" == "None" ]; then
-  # Request on-demand instance (with tags at launch for IAM RequestTag conditions).
-  echo "Requesting $instance_type on-demand instance $info..."
-  iid=$(aws ec2 run-instances \
-    --cli-input-json file://$spec_path \
-    --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$name},{Key=Group,Value=build-instance}]" \
-    --query "Instances[*].[InstanceId]" \
-    --output text)
-  echo $iid > $iid_path
+# On-demand fallback (cheapest pool) only if spot couldn't be had.
+if [ -z "$iid" ]; then
+  echo "Requesting on-demand fleet $info..."
+  resp=$(aws ec2 create-fleet \
+    --type instant \
+    --target-capacity-specification "TotalTargetCapacity=1,DefaultTargetCapacityType=on-demand" \
+    --on-demand-options "AllocationStrategy=lowest-price" \
+    --launch-template-configs "$ltc" \
+    --output json 2>"$state_dir/fleet_err") || true
+  iid=$(echo "$resp" | jq -r '.Instances[0].InstanceIds[0] // empty' 2>/dev/null || true)
+  instance_type=$(echo "$resp" | jq -r '.Instances[0].InstanceType // empty')
+  lifecycle=ondemand
 fi
 
-echo "Instance id: $iid"
-
-tags="Key=Name,Value=$name Key=Group,Value=build-instance"
-[ -n "${GITHUB_ACTOR:-}" ] && tags+=" Key=GithubActor,Value=${GITHUB_ACTOR//[\[\]]/}"
-[ -n "${CI_MODE:-}" ] && tags+=" Key=CICommand,Value=$CI_MODE"
-[ -n "${CI_DASHBOARD:-}" ] && tags+=" Key=Dashboard,Value=$CI_DASHBOARD"
-if [ "${UNSAFE_AWS_KEEP_ALIVE:-0}" -eq 1 ]; then
-  echo_stderr "You have set UNSAFE_AWS_KEEP_ALIVE=1, so the instance will not be terminated after 1.5 hours by the reaper script. Make sure you shut the machine down when done."
-  tags+=" Key=Keep-Alive,Value=true"
+if [ -z "$iid" ]; then
+  echo_stderr "Failed to acquire any instance (spot or on-demand)."
+  echo "$resp" | jq -r '.Errors[]? | "  \(.ErrorCode): \(.ErrorMessage)"' >&2 2>/dev/null || true
+  cat "$state_dir/fleet_err" >&2 2>/dev/null || true
+  exit 1
 fi
-aws ec2 create-tags --resources $iid --tags $tags
+
+echo "Instance id: $iid ($instance_type, $lifecycle)"
+echo "$iid" > "$iid_path"
+
+# Instances are launched; the launch template is no longer needed.
+delete_lt
 
 # Record the instance type so callers can pass it downstream (e.g. into Docker).
-echo $instance_type > $state_dir/instance_type
+echo "$instance_type" > "$state_dir/instance_type"
 # Record whether this is spot or on-demand.
-[ -f "$sir_path" ] && echo spot > $state_dir/spot || echo ondemand > $state_dir/spot
+echo "$lifecycle" > "$state_dir/spot"
 
+ip=""
 while [ -z "${ip:-}" ]; do
   sleep 1
   ip=$(aws ec2 describe-instances \
@@ -172,9 +233,3 @@ else
     sleep 1
   done
 fi
-
-# For spot instances, tags must be applied post-launch (launch-specification doesn't support TagSpecifications).
-# On-demand instances are tagged at launch via --tag-specifications.
-if [ -n "${sir:-}" ]; then
-  aws ec2 create-tags --resources $iid --tags "Key=Name,Value=$name" "Key=Group,Value=build-instance"
-fi
@@ -35,7 +35,7 @@ if [ "$arch" == "arm64" ]; then
   cores=64
   export AWS_SHUTDOWN_TIME=${AWS_SHUTDOWN_TIME_ARM:-${AWS_SHUTDOWN_TIME:-60}}
 else
-  cores=128,64
+  cores=192,128,64
   if [ "${CI_FULL:-0}" -eq 1 ]; then
     export AWS_SHUTDOWN_TIME=${AWS_SHUTDOWN_TIME:-75}
   else