Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 191 additions & 0 deletions .github/actions/apt-install/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
name: Install APT packages
description: Install required and optional packages with bounded, recoverable APT operations on Ubuntu runners.

inputs:
required-packages:
description: Space-separated package names that must be installed.
required: false
default: ""
optional-packages:
description: Space-separated package names whose installation may fail without failing the caller.
required: false
default: ""
update-mode:
description: "none, system, or source-file. source-file refreshes only the supplied deb822 source."
required: false
default: none
source-file:
description: Deb822 .sources file to refresh when update-mode is source-file.
required: false
default: ""
update-attempts:
description: Number of bounded update attempts before installation proceeds from the existing indexes.
required: false
default: "1"
update-timeout-seconds:
description: Wall-clock bound for each apt-get update attempt.
required: false
default: "120"
install-timeout-seconds:
description: Wall-clock bound for each apt-get install invocation.
required: false
default: "240"
install-attempts:
description: Number of bounded install attempts before required packages fail.
required: false
default: "1"

runs:
using: composite
steps:
- name: Install APT packages
shell: bash
env:
REQUIRED_PACKAGES: ${{ inputs.required-packages }}
OPTIONAL_PACKAGES: ${{ inputs.optional-packages }}
UPDATE_MODE: ${{ inputs.update-mode }}
SOURCE_FILE: ${{ inputs.source-file }}
UPDATE_ATTEMPTS: ${{ inputs.update-attempts }}
UPDATE_TIMEOUT_SECONDS: ${{ inputs.update-timeout-seconds }}
INSTALL_ATTEMPTS: ${{ inputs.install-attempts }}
INSTALL_TIMEOUT_SECONDS: ${{ inputs.install-timeout-seconds }}
run: |
set -euo pipefail

if ! command -v apt-get >/dev/null 2>&1; then
echo "::error::apt-install requires an apt-based runner"
exit 1
fi

apt_options=(
-o Acquire::Retries=0
-o Acquire::http::Timeout=20
-o Acquire::https::Timeout=20
)
update_options=()
sourceparts_dir=""

cleanup_sourceparts() {
if [[ -n "$sourceparts_dir" ]]; then
rm -rf "$sourceparts_dir"
fi
}
trap cleanup_sourceparts EXIT

is_positive_integer() {
[[ "$1" =~ ^[1-9][0-9]*$ ]]
}

for value in "$UPDATE_ATTEMPTS" "$UPDATE_TIMEOUT_SECONDS" "$INSTALL_ATTEMPTS" "$INSTALL_TIMEOUT_SECONDS"; do
if ! is_positive_integer "$value"; then
echo "::error::APT attempt counts and timeouts must be positive integers"
exit 1
fi
done
case "$UPDATE_MODE" in
none|system)
;;
source-file)
if [[ -z "$SOURCE_FILE" || ! -f "$SOURCE_FILE" || "${SOURCE_FILE##*.}" != sources ]]; then
echo "::error::source-file update mode requires a readable .sources file"
exit 1
fi
;;
*)
echo "::error::unknown update mode: $UPDATE_MODE"
exit 1
;;
esac

configure_update() {
local mode="$1"
update_options=()
case "$mode" in
none|system)
;;
source-file)
if [[ -z "$sourceparts_dir" ]]; then
sourceparts_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/apt-install-sources.XXXXXX")"
install -m 0644 "$SOURCE_FILE" "$sourceparts_dir/${SOURCE_FILE##*/}"
fi
update_options=(
-o Dir::Etc::SourceList=/dev/null
-o "Dir::Etc::SourceParts=$sourceparts_dir"
-o APT::Get::List-Cleanup=0
)
;;
esac
}

recover_apt() {
sudo rm -rf /var/lib/apt/lists/partial
sudo DEBIAN_FRONTEND=noninteractive timeout --kill-after=15s 60s dpkg --configure -a || true
}

run_update() {
local mode="$1"
local attempts="$2"
local attempt
local -a update_command
update_command=(apt-get "${apt_options[@]}")
configure_update "$mode"
if [[ "$mode" == none ]]; then
return 0
fi
if [[ "$mode" == source-file ]]; then
update_command+=("${update_options[@]}")
fi
for ((attempt = 1; attempt <= attempts; ++attempt)); do
if sudo DEBIAN_FRONTEND=noninteractive timeout --kill-after=15s "${UPDATE_TIMEOUT_SECONDS}s" \
"${update_command[@]}" update; then
return 0
fi
recover_apt
echo "::warning::apt-get $mode update attempt ${attempt}/${attempts} failed; using existing package indexes"
done
return 1
}

install_packages_once() {
if (( $# == 0 )); then
return 0
fi
sudo DEBIAN_FRONTEND=noninteractive timeout --kill-after=15s "${INSTALL_TIMEOUT_SECONDS}s" \
apt-get "${apt_options[@]}" install -y "$@"
}

install_required_packages() {
local attempt
for ((attempt = 1; attempt <= INSTALL_ATTEMPTS; ++attempt)); do
if install_packages_once "$@"; then
return 0
fi
recover_apt
echo "::warning::apt required-package install attempt ${attempt}/${INSTALL_ATTEMPTS} failed"
done
if [[ "$UPDATE_MODE" == none ]]; then
echo "::warning::apt install failed on cached indexes; refreshing system indexes once"
run_update system 1 || true
if install_packages_once "$@"; then
return 0
fi
recover_apt
fi
echo "::error::apt could not install $*"
return 1
}

install_optional_packages() {
if install_packages_once "$@"; then
return 0
fi
recover_apt
echo "::warning::apt could not install $*"
return 1
}

read -r -a required_packages <<<"$REQUIRED_PACKAGES"
read -r -a optional_packages <<<"$OPTIONAL_PACKAGES"
run_update "$UPDATE_MODE" "$UPDATE_ATTEMPTS" || true
install_required_packages "${required_packages[@]}"
install_optional_packages "${optional_packages[@]}" || true
45 changes: 37 additions & 8 deletions .github/actions/setup-gcc-15/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ inputs:
runs:
using: composite
steps:
- name: Install GCC 15 and build tools
- name: Prepare GCC 15 and build tools
id: packages
if: runner.os == 'Linux'
shell: bash
env:
Expand Down Expand Up @@ -88,15 +89,43 @@ runs:
sudo tee /etc/apt/sources.list.d/ubuntu-toolchain-r-test.sources >/dev/null
fi

apt_options=(
-o Acquire::Retries=3
-o Acquire::http::Timeout=20
-o Acquire::https::Timeout=20
)
sudo apt-get "${apt_options[@]}" update
sudo apt-get "${apt_options[@]}" install -y "${packages[@]}"
if [[ "$needs_gcc_ppa" == true ]]; then
echo "update-mode=source-file" >> "$GITHUB_OUTPUT"
echo "source-file=/etc/apt/sources.list.d/ubuntu-toolchain-r-test.sources" >> "$GITHUB_OUTPUT"
echo "update-attempts=2" >> "$GITHUB_OUTPUT"
echo "install-attempts=2" >> "$GITHUB_OUTPUT"
else
echo "update-mode=none" >> "$GITHUB_OUTPUT"
echo "source-file=" >> "$GITHUB_OUTPUT"
echo "update-attempts=1" >> "$GITHUB_OUTPUT"
echo "install-attempts=1" >> "$GITHUB_OUTPUT"
fi
echo "required-packages=${packages[*]}" >> "$GITHUB_OUTPUT"
fi

- name: Install GCC 15 and build tools
if: runner.os == 'Linux' && steps.packages.outputs.required-packages != ''
uses: ./.github/actions/apt-install
with:
required-packages: ${{ steps.packages.outputs.required-packages }}
update-mode: ${{ steps.packages.outputs.update-mode }}
source-file: ${{ steps.packages.outputs.source-file }}
update-attempts: ${{ steps.packages.outputs.update-attempts }}
update-timeout-seconds: "60"
install-timeout-seconds: "240"
install-attempts: ${{ steps.packages.outputs.install-attempts }}

- name: Verify GCC 15 and build tools
if: runner.os == 'Linux'
shell: bash
run: |
set -euo pipefail

is_version_15() {
command -v "$1" >/dev/null 2>&1 &&
[[ "$("$1" -dumpversion | cut -d. -f1)" == 15 ]]
}

command -v gcc-15
command -v g++-15
command -v ninja
Expand Down
36 changes: 25 additions & 11 deletions .github/workflows/_packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,27 +38,41 @@ jobs:
persist-credentials: false

- name: Set up C++ compiler
id: packaging-tools
if: inputs.setup_variant == 'github'
run: |
# ninja arrives from PyPI with the venv, so the only package a
# packaging build cannot proceed without is a host compiler. ccache
# merely warms the matrix's later modes and the matrix step already
# runs without it, so a package manager that cannot serve it is a
# warning, not a failure.
if [[ "${{ runner.os }}" == "Linux" ]]; then
missing_packages=()
command -v ninja >/dev/null 2>&1 || missing_packages+=(ninja-build)
command -v ccache >/dev/null 2>&1 || missing_packages+=(ccache)
required_packages=()
if ! command -v gcc >/dev/null 2>&1 || ! command -v g++ >/dev/null 2>&1; then
missing_packages+=(g++)
fi
if (( ${#missing_packages[@]} )); then
sudo apt-get update
sudo apt-get install -y "${missing_packages[@]}"
required_packages+=(g++)
fi
optional_packages=()
command -v ccache >/dev/null 2>&1 || optional_packages+=(ccache)
echo "required-packages=${required_packages[*]}" >> "$GITHUB_OUTPUT"
echo "optional-packages=${optional_packages[*]}" >> "$GITHUB_OUTPUT"
else
command -v ninja >/dev/null 2>&1 || brew install ninja
command -v ccache >/dev/null 2>&1 || brew install ccache
if ! command -v ccache >/dev/null 2>&1; then
brew install ccache ||
echo "::warning::brew could not install ccache; continuing without it"
fi
if ! command -v gcc-15 >/dev/null 2>&1 || ! command -v g++-15 >/dev/null 2>&1; then
brew install gcc@15 || brew install gcc
fi
fi

- name: Install Linux C++ compiler tools
if: inputs.setup_variant == 'github' && runner.os == 'Linux' && (steps.packaging-tools.outputs.required-packages != '' || steps.packaging-tools.outputs.optional-packages != '')
uses: ./.github/actions/apt-install
with:
required-packages: ${{ steps.packaging-tools.outputs.required-packages }}
optional-packages: ${{ steps.packaging-tools.outputs.optional-packages }}
install-timeout-seconds: "240"

- name: Set up Python ${{ inputs.python_version }}
if: inputs.setup_variant == 'github'
uses: actions/setup-python@v6
Expand All @@ -72,7 +86,7 @@ jobs:
- name: Install build + runtime deps in venv
uses: ./.github/actions/setup-venv
with:
packages: "scikit-build-core>=0.10.0 nanobind>=2.0.0 cmake>=3.15 pytest>=6.0"
packages: "scikit-build-core>=0.10.0 nanobind>=2.0.0 cmake>=3.15 ninja>=1.11 pytest>=6.0"
Comment thread
doraemonmj marked this conversation as resolved.
Resolved

- name: Run packaging matrix
run: |
Expand Down
15 changes: 10 additions & 5 deletions .github/workflows/_pre-commit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ jobs:

echo "needs_build=$needs_build" >> "$GITHUB_OUTPUT"

- name: Install build and lint tools
- name: Find missing build and lint tools
id: apt-packages
if: inputs.setup_variant == 'github' && steps.lint-build.outputs.needs_build == 'true'
run: |
missing_packages=()
Expand All @@ -94,10 +95,14 @@ jobs:
if ! command -v clang-tidy >/dev/null 2>&1; then
missing_packages+=(clang-tidy)
fi
if (( ${#missing_packages[@]} )); then
sudo apt-get update
sudo apt-get install -y "${missing_packages[@]}"
fi
echo "required-packages=${missing_packages[*]}" >> "$GITHUB_OUTPUT"

- name: Install build and lint tools
if: inputs.setup_variant == 'github' && runner.os == 'Linux' && steps.lint-build.outputs.needs_build == 'true' && steps.apt-packages.outputs.required-packages != ''
uses: ./.github/actions/apt-install
with:
required-packages: ${{ steps.apt-packages.outputs.required-packages }}
install-timeout-seconds: "240"

- name: Verify provisioned toolchain
if: inputs.setup_variant == 'self-cpu' && steps.lint-build.outputs.needs_build == 'true'
Expand Down
17 changes: 9 additions & 8 deletions .github/workflows/_ut-no-hardware.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,15 @@ jobs:
python-version: ${{ inputs.python_version }}

- name: Install GoogleTest
if: inputs.setup_variant == 'github'
run: |
if [[ "${{ runner.os }}" == "Linux" ]]; then
sudo apt-get update
sudo apt-get install -y libgtest-dev
else
brew install googletest
fi
if: inputs.setup_variant == 'github' && runner.os == 'Linux'
uses: ./.github/actions/apt-install
with:
required-packages: libgtest-dev
install-timeout-seconds: "240"

- name: Install GoogleTest on macOS
if: inputs.setup_variant == 'github' && runner.os != 'Linux'
run: brew install googletest

- name: Cache pip packages
uses: ./.github/actions/cache-pip
Expand Down
Loading
Loading