diff --git a/ci3/aws_request_instance b/ci3/aws_request_instance index cd9cba8d0629..2382873370cd 100755 --- a/ci3/aws_request_instance +++ b/ci3/aws_request_instance @@ -9,19 +9,7 @@ state_dir=$4 cd $(dirname $0) -bid_per_cpu_hour=0.0433 - -instance_types_amd64=( - m6a - m7a - m7i -) - -instance_types_arm64=( - r7g -) - -# Declare an associative array to map CPU counts to instance type suffixes. +# Map a cpu count to its instance size suffix. declare -A cpu_map cpu_map=( [2]="large" @@ -37,32 +25,69 @@ cpu_map=( [0192]="metal" ) +# Spot pool preference by cpu count (lower number = tried first). 128 is the +# known-good build size, then 192, then 64 as a last resort before on-demand. +declare -A cpu_priority=([128]=1 [192]=2 [64]=3) + +# Capacity-diverse spot pools per cpu count. Restricted to families with +# >=4 GB/vCPU (m/r) so the build's measured ~1.9 GB/vCPU peak keeps ~2x +# headroom; c-series (2 GB/vCPU) is opt-in via CI_SPOT_INCLUDE_C. All entries +# are confirmed offered as spot across every us-east-2 AZ. +function spot_pools_for { + local cpu=$1 + case "$arch" in + x86_64|amd64) + case "$cpu" in + 128) echo "m6a.32xlarge m7a.32xlarge m6i.32xlarge r6a.32xlarge r7a.32xlarge" ;; + 192) echo "m6a.48xlarge m7a.48xlarge m7i.48xlarge r6a.48xlarge r7a.48xlarge" ;; + 64) echo "m6a.16xlarge m7a.16xlarge m7i.16xlarge m6i.16xlarge m5.16xlarge r6a.16xlarge" ;; + *) local s=${cpu_map[$cpu]:?Unknown cpu count: $cpu}; echo "m6a.$s m7a.$s m6i.$s r6a.$s" ;; + esac ;; + arm64) + case "$cpu" in + 64) echo "r7g.16xlarge r8g.16xlarge m7g.16xlarge m8g.16xlarge" ;; + *) local s=${cpu_map[$cpu]:?Unknown cpu count: $cpu}; echo "r7g.$s r8g.$s m7g.$s m8g.$s" ;; + esac ;; + esac +} + +# Optional c-series pools (2 GB/vCPU). Gated behind CI_SPOT_INCLUDE_C until +# build-phase parallelism is throttled to fit the lower memory headroom. +function spot_pools_c_for { + local cpu=$1 + case "$arch" in + x86_64|amd64) + case "$cpu" in + 128) echo "c6a.32xlarge c6i.32xlarge" ;; + 192) echo "c6a.48xlarge c7a.48xlarge" ;; + 64) echo "c6a.16xlarge c6i.16xlarge" ;; + esac ;; + esac +} + if [ "$arch" == "x86_64" ] || [ "$arch" == "amd64" ]; then - instances=("${instance_types_amd64[@]}") ami=${AMI:-$(cat ./aws/ami_id_amd64)} elif [ "$arch" == "arm64" ]; then - instances=("${instance_types_arm64[@]}") ami=${AMI:-$(cat ./aws/ami_id_arm64)} else echo "Unsupported arch: $arch" >&2 exit 1 fi +# Build the fleet override set as space-separated "instanceType:priority" tokens. +# AWS_INSTANCE pins a single pool (e.g. for debugging); otherwise diversify +# across every requested size. +overrides="" if [ -n "${AWS_INSTANCE:-}" ]; then - instances=("$AWS_INSTANCE") + overrides="$AWS_INSTANCE:1" +else + IFS=',' read -ra cpu_list <<< "$cpus" + for cpu in "${cpu_list[@]}"; do + prio=${cpu_priority[$cpu]:-9} + types="$(spot_pools_for "$cpu")" + [ "${CI_SPOT_INCLUDE_C:-0}" -eq 1 ] && types+=" $(spot_pools_c_for "$cpu")" + for t in $types; do overrides+=" $t:$prio"; done + done fi -set +e -IFS=',' read -ra cpu_list <<< "$cpus" -for cpu in "${cpu_list[@]}"; do - price=$(jq -n "$bid_per_cpu_hour*$cpu*100000 | round / 100000") - suffix=${cpu_map[$cpu]} - for inst in "${instances[@]}"; do - instance_type="$inst.$suffix" - aws_request_instance_type $name $instance_type $price $ami $state_dir - code=$? - [[ "$code" -eq 0 || "$code" -eq 143 || "$code" -eq 130 ]] && exit $code - echo "Instance request exited with code: $code" - done -done -exit 1 +FLEET_OVERRIDES="${overrides# }" aws_request_instance_type "$name" "$ami" "$state_dir" diff --git a/ci3/aws_request_instance_type b/ci3/aws_request_instance_type index 7d93f56068d6..acccf6faf898 100755 --- a/ci3/aws_request_instance_type +++ b/ci3/aws_request_instance_type @@ -2,22 +2,36 @@ source $(git rev-parse --show-toplevel)/ci3/source name=$1 -instance_type=$2 -price=$3 -ami=$4 -state_dir=$5 +ami=$2 +state_dir=$3 -spec_path=$state_dir/spec.json -sir_path=$state_dir/sir +# Space-separated "instanceType:priority" tokens describing the spot pools to +# diversify across (set by aws_request_instance). +overrides_spec=${FLEET_OVERRIDES:?FLEET_OVERRIDES must be set} + +spec_path=$state_dir/lt_data.json iid_path=$state_dir/iid ip_path=$state_dir/ip +lt_path=$state_dir/launch_template_id + +export AWS_DEFAULT_REGION=us-east-2 +# Delete the ephemeral launch template. Safe once instances are launched (they +# no longer reference it) and idempotent, so we run it on every exit. +function delete_lt { + [ -f "$lt_path" ] || return 0 + aws ec2 delete-launch-template --launch-template-id "$(cat "$lt_path")" >/dev/null 2>&1 || true + rm -f "$lt_path" +} + +# On interrupt, tear down whatever we acquired. On normal exit we only drop the +# launch template (the instance must survive for the caller). function cleanup { - aws_terminate_instance $state_dir + delete_lt + aws_terminate_instance "$state_dir" } trap 'cleanup' SIGINT SIGTERM - -export AWS_DEFAULT_REGION=us-east-2 +trap 'delete_lt' EXIT # KeyName is optional: omit when KEY_NAME is empty (SSM-only mode). key_name_json="" @@ -27,7 +41,6 @@ fi # Always attach the instance profile so containers can use instance-profile credentials. CI3_INSTANCE_PROFILE_NAME=${CI3_INSTANCE_PROFILE_NAME:-ci3-build-instance-profile} -iam_profile_json="\"IamInstanceProfile\": { \"Name\": \"$CI3_INSTANCE_PROFILE_NAME\" }," # In SSM mode use the CI3 security group; in SSH mode use the original SG (allows port 22). if [ -z "${KEY_NAME:-}" ]; then @@ -36,13 +49,35 @@ else sg_id="sg-0ccd4e5df0dcca0c9" fi -launch_spec=$(cat < "$spec_path" + +lt_name="ci3-${name}-$$-$(date +%s)" +lt_id=$(aws ec2 create-launch-template \ + --launch-template-name "$lt_name" \ + --launch-template-data "$lt_data" \ + --query 'LaunchTemplate.LaunchTemplateId' --output text) +echo "$lt_id" > "$lt_path" + +# Cross every pool with every AZ. Priority biases capacity-optimized toward the +# preferred sizes (see aws_request_instance) while still optimizing for depth. +overrides_json="" +for tok in $overrides_spec; do + itype=${tok%%:*} + prio=${tok##*:} + for sn in "${subnets[@]}"; do + overrides_json+="{\"InstanceType\":\"$itype\",\"SubnetId\":\"$sn\",\"Priority\":$prio}," + done +done +ltc="[{\"LaunchTemplateSpecification\":{\"LaunchTemplateId\":\"$lt_id\",\"Version\":\"\$Latest\"},\"Overrides\":[${overrides_json%,}]}]" -# Save the launch specification to a temporary file. -echo "$launch_spec" > "$spec_path" +info="(name: $name) (pools: ${overrides_spec// /,}) (ami: $ami)" -info="(name: $name) (type: $instance_type) (ami: $ami) (bid: $price)" +iid="" +instance_type="" +lifecycle="" +# Try spot first, retrying the whole diversified fleet until the time budget +# expires (preserves the historical "give spot ~1 min, then on-demand" pattern). if [ "${NO_SPOT:-0}" -ne 1 ]; then - echo "Requesting $instance_type spot instance $info..." - sir=$(aws ec2 request-spot-instances \ - --spot-price "$price" \ - --instance-count 1 \ - --type "one-time" \ - --launch-specification file://$spec_path \ - --query "SpotInstanceRequests[*].[SpotInstanceRequestId]" \ - --output text) - echo $sir > $sir_path - - echo "Waiting for instance id for spot request: $sir..." - sleep 5 - for i in {1..6}; do - iid=$(aws ec2 describe-spot-instance-requests \ - --spot-instance-request-ids $sir \ - --query "SpotInstanceRequests[*].[InstanceId]" \ - --output text) - [ -z "$iid" -o "$iid" == "None" ] || break - - if [ $i -eq 6 ]; then - echo "Timeout waiting for spot request." - # Cancel spot request. We may still get allocated an instance if it's *just* happened. - aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $sir > /dev/null + spot_timeout=${CI_SPOT_TIMEOUT:-60} + spot_poll=${CI_SPOT_POLL:-10} + SECONDS=0 + while true; do + echo "Requesting spot fleet $info (capacity-optimized-prioritized, ${spot_timeout}s budget)..." + resp=$(aws ec2 create-fleet \ + --type instant \ + --target-capacity-specification "TotalTargetCapacity=1,DefaultTargetCapacityType=spot" \ + --spot-options "AllocationStrategy=capacity-optimized-prioritized" \ + --launch-template-configs "$ltc" \ + --output json 2>"$state_dir/fleet_err") || true + iid=$(echo "$resp" | jq -r '.Instances[0].InstanceIds[0] // empty' 2>/dev/null || true) + if [ -n "$iid" ]; then + instance_type=$(echo "$resp" | jq -r '.Instances[0].InstanceType // empty') + lifecycle=spot + break fi - - sleep 5 + if (( SECONDS + spot_poll >= spot_timeout )); then + echo "Spot fleet unfulfilled after ${spot_timeout}s; falling back to on-demand." + break + fi + sleep "$spot_poll" done - echo $iid > $iid_path fi -if [ -z "${iid:-}" -o "${iid:-}" == "None" ]; then - # Request on-demand instance (with tags at launch for IAM RequestTag conditions). - echo "Requesting $instance_type on-demand instance $info..." - iid=$(aws ec2 run-instances \ - --cli-input-json file://$spec_path \ - --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$name},{Key=Group,Value=build-instance}]" \ - --query "Instances[*].[InstanceId]" \ - --output text) - echo $iid > $iid_path +# On-demand fallback (cheapest pool) only if spot couldn't be had. +if [ -z "$iid" ]; then + echo "Requesting on-demand fleet $info..." + resp=$(aws ec2 create-fleet \ + --type instant \ + --target-capacity-specification "TotalTargetCapacity=1,DefaultTargetCapacityType=on-demand" \ + --on-demand-options "AllocationStrategy=lowest-price" \ + --launch-template-configs "$ltc" \ + --output json 2>"$state_dir/fleet_err") || true + iid=$(echo "$resp" | jq -r '.Instances[0].InstanceIds[0] // empty' 2>/dev/null || true) + instance_type=$(echo "$resp" | jq -r '.Instances[0].InstanceType // empty') + lifecycle=ondemand fi -echo "Instance id: $iid" - -tags="Key=Name,Value=$name Key=Group,Value=build-instance" -[ -n "${GITHUB_ACTOR:-}" ] && tags+=" Key=GithubActor,Value=${GITHUB_ACTOR//[\[\]]/}" -[ -n "${CI_MODE:-}" ] && tags+=" Key=CICommand,Value=$CI_MODE" -[ -n "${CI_DASHBOARD:-}" ] && tags+=" Key=Dashboard,Value=$CI_DASHBOARD" -if [ "${UNSAFE_AWS_KEEP_ALIVE:-0}" -eq 1 ]; then - echo_stderr "You have set UNSAFE_AWS_KEEP_ALIVE=1, so the instance will not be terminated after 1.5 hours by the reaper script. Make sure you shut the machine down when done." - tags+=" Key=Keep-Alive,Value=true" +if [ -z "$iid" ]; then + echo_stderr "Failed to acquire any instance (spot or on-demand)." + echo "$resp" | jq -r '.Errors[]? | " \(.ErrorCode): \(.ErrorMessage)"' >&2 2>/dev/null || true + cat "$state_dir/fleet_err" >&2 2>/dev/null || true + exit 1 fi -aws ec2 create-tags --resources $iid --tags $tags + +echo "Instance id: $iid ($instance_type, $lifecycle)" +echo "$iid" > "$iid_path" + +# Instances are launched; the launch template is no longer needed. +delete_lt # Record the instance type so callers can pass it downstream (e.g. into Docker). -echo $instance_type > $state_dir/instance_type +echo "$instance_type" > "$state_dir/instance_type" # Record whether this is spot or on-demand. -[ -f "$sir_path" ] && echo spot > $state_dir/spot || echo ondemand > $state_dir/spot +echo "$lifecycle" > "$state_dir/spot" +ip="" while [ -z "${ip:-}" ]; do sleep 1 ip=$(aws ec2 describe-instances \ @@ -172,9 +233,3 @@ else sleep 1 done fi - -# For spot instances, tags must be applied post-launch (launch-specification doesn't support TagSpecifications). -# On-demand instances are tagged at launch via --tag-specifications. -if [ -n "${sir:-}" ]; then - aws ec2 create-tags --resources $iid --tags "Key=Name,Value=$name" "Key=Group,Value=build-instance" -fi diff --git a/ci3/bootstrap_ec2 b/ci3/bootstrap_ec2 index 77018e708d65..a15113a4c126 100755 --- a/ci3/bootstrap_ec2 +++ b/ci3/bootstrap_ec2 @@ -35,7 +35,7 @@ if [ "$arch" == "arm64" ]; then cores=64 export AWS_SHUTDOWN_TIME=${AWS_SHUTDOWN_TIME_ARM:-${AWS_SHUTDOWN_TIME:-60}} else - cores=128,64 + cores=192,128,64 if [ "${CI_FULL:-0}" -eq 1 ]; then export AWS_SHUTDOWN_TIME=${AWS_SHUTDOWN_TIME:-75} else