Skip to content
155 changes: 118 additions & 37 deletions .github/workflows/build-kernel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,33 +28,39 @@ on:
description: "Version (e.g. 7.0.0-1006.8) - builds the exact Ubuntu-qcom-<version> tag. Empty = branch HEAD."
required: false
type: string
arch:
description: "Target Debian architecture"
devel_prs:
description: "Space-separated PR numbers against resolute-qcom-devel to merge before building (e.g. 42 43)"
required: false
default: "arm64"
type: choice
options:
- arm64
- amd64
flavor:
description: "Kernel flavour to build"
default: ""
type: string

workflow_call:
inputs:
suite:
description: "Branch to build from"
required: false
default: "qcom"
type: choice
options:
- qcom
- generic
- lowlatency
- all
runner:
description: "Runner to use for the build (lecore-production is the only one that uploads to S3)"
default: "resolute-qcom-devel"
type: string
kernel_version:
description: "Version tag to build. Empty = branch HEAD."
required: false
default: ""
type: string
devel_prs:
description: "Space-separated PR numbers against resolute-qcom-devel to merge before building"
required: false
default: ""
type: string
ref:
description: "Exact git ref to checkout. Overrides suite/kernel_version when set (used by premerge-pr.yml to build the PR merge ref)."
required: false
default: ""
type: string
skip_s3:
description: "Skip uploading built .deb packages to S3 (used by premerge-pr.yml for build-only checks)."
required: false
default: "lecore-production"
type: choice
options:
- lecore-production
- ubuntu-24.04-arm
- self-hosted
default: false
type: boolean

permissions:
contents: read # checkout + tag ls-remote only; output goes to S3
Expand All @@ -65,17 +71,14 @@ jobs:
# --------------------------------------------------------------------------
build:
name: "Build"
# Map friendly runner names to their actual runs-on labels.
# lecore-production → ["self-hosted","lecore-prd-u2404-arm64-xlrg-od-ephem"] (default; uploads to S3)
# ubuntu-24.04-arm → "ubuntu-24.04-arm" (GitHub-hosted)
# self-hosted → "self-hosted"
runs-on: ${{ fromJSON('{"ubuntu-24.04-arm":"ubuntu-24.04-arm","self-hosted":"self-hosted","lecore-production":["self-hosted","lecore-prd-u2404-arm64-xlrg-od-ephem"]}')[inputs.runner || 'lecore-production'] }}
# Runner is always lecore-production for this repo.
runs-on: ["self-hosted", "lecore-prd-u2404-arm64-xlrg-od-ephem"]
timeout-minutes: 360

env:
SUITE: ${{ inputs.suite || 'resolute-qcom' }}
ARCH: ${{ inputs.arch || 'arm64' }}
FLAVOR: ${{ inputs.flavor || 'qcom' }}
SUITE: ${{ inputs.suite || 'resolute-qcom-devel' }}
ARCH: arm64
FLAVOR: qcom

steps:
# -----------------------------------------------------------------------
Expand Down Expand Up @@ -168,9 +171,84 @@ jobs:
- name: Checkout kernel source
uses: actions/checkout@v6
with:
ref: ${{ inputs.kernel_version != '' && format('{0}-{1}', env.UPSTREAM_PREFIX, inputs.kernel_version) || inputs.suite || 'resolute-qcom' }}
ref: ${{ inputs.ref != '' && inputs.ref || (inputs.kernel_version != '' && format('{0}-{1}', env.UPSTREAM_PREFIX, inputs.kernel_version) || inputs.suite || 'resolute-qcom-devel') }}
path: kernel-src

# -----------------------------------------------------------------------
# 4b. Merge additional PRs against resolute-qcom-devel into the
# checked-out source tree (engineering builds only).
# Each PR number is validated as numeric before use. Conflicts abort
# the build immediately with a clear error.
# -----------------------------------------------------------------------
- name: Merge devel PRs into source tree
if: inputs.devel_prs != ''
env:
DEVEL_PRS: ${{ inputs.devel_prs }}
GH_TOKEN: ${{ github.token }}
run: |
cd kernel-src/
git config user.email "github-actions[bot]@users.noreply.github.com"
git config user.name "github-actions[bot]"
git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git"

echo "PRs to merge: ${DEVEL_PRS}"
for pr in ${DEVEL_PRS}; do
[[ "${pr}" =~ ^[0-9]+$ ]] || {
echo "ERROR: invalid PR number '${pr}' -- must be numeric" >&2
exit 1
}

# Validate: the PR must exist, be open, and target the suite branch.
# A closed or wrong-base PR number would otherwise be merged silently.
info="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${pr}" \
--jq '[.state, .base.ref] | @tsv')" || {
echo "ERROR: PR #${pr} not found in ${GITHUB_REPOSITORY}." >&2
exit 1
}
state="$(printf '%s' "${info}" | cut -f1)"
base="$(printf '%s' "${info}" | cut -f2)"
[ "${state}" = "open" ] || {
echo "ERROR: PR #${pr} is '${state}' -- only open PRs can be merged." >&2
exit 1
}
[ "${base}" = "${SUITE}" ] || {
echo "ERROR: PR #${pr} targets '${base}', not '${SUITE}'." >&2
exit 1
}

echo "::group::Merging ${SUITE} PR #${pr}"
# The build checkout is shallow (depth 1), so the merge base with the
# PR is not present. Fetch the PR head and deepen BOTH histories until
# a merge base exists - PRs fork from a recent tip, so this stays a
# small fetch and never pulls the full ~1.43M-commit history.
git fetch --no-tags --depth=1 origin "refs/pull/${pr}/head:refs/pr/${pr}"
step=100; total=0; max=6400
until git merge-base HEAD "refs/pr/${pr}" >/dev/null 2>&1; do
if [ "${total}" -ge "${max}" ]; then
echo "ERROR: no common ancestor with PR #${pr} within ${max} commits." >&2
echo "Rebase the PR onto ${SUITE} and retry." >&2
exit 1
fi
echo " deepening history by ${step} commits to find the merge base..."
git fetch --deepen="${step}" origin "refs/heads/${SUITE}" "refs/pull/${pr}/head"
total=$((total + step)); step=$((step * 2))
done

if ! git merge --no-ff --no-commit "refs/pr/${pr}"; then
echo "ERROR: merge conflict while merging PR #${pr}. Aborting." >&2
git merge --abort || true
exit 1
fi
if git diff --cached --quiet; then
echo "PR #${pr} already present in tree -- nothing to merge."
git merge --abort >/dev/null 2>&1 || true
else
git commit -m "Merged ${SUITE} PR #${pr}"
echo "PR #${pr} merged successfully."
fi
echo "::endgroup::"
done

# -----------------------------------------------------------------------
# 5. Checkout docker-pkg-build
# -----------------------------------------------------------------------
Expand Down Expand Up @@ -293,12 +371,10 @@ jobs:

# -----------------------------------------------------------------------
# 9. Upload to S3 (lecore-production runner only)
# Skipped on GitHub-hosted and generic self-hosted runners.
# Condition checks the runner input, not runner.name, because the
# runner's registered display name may differ from its label.
# Skipped when skip_s3 is set (premerge-pr.yml build-only checks).
# -----------------------------------------------------------------------
- name: Upload kernel .deb packages to S3
if: inputs.runner == 'lecore-production'
if: inputs.skip_s3 != true
uses: qualcomm-linux/upload-private-artifact-action@aws-v4
with:
s3_bucket: qli-prd-lecore-gh-artifacts
Expand All @@ -310,6 +386,8 @@ jobs:
# -----------------------------------------------------------------------
- name: Print summary
if: always()
env:
DEVEL_PRS: ${{ inputs.devel_prs }}
run: |
{
echo "## Kernel Build Summary"
Expand All @@ -321,6 +399,9 @@ jobs:
echo "| Flavour | \`${FLAVOR}\` |"
echo "| Container | \`ghcr.io/qualcomm-linux/pkg-builder:${BASE_SUITE}\` |"
echo "| Runner | \`${{ runner.name }}\` |"
if [ -n "${DEVEL_PRS}" ]; then
echo "| Devel PRs | \`${DEVEL_PRS}\` |"
fi
echo ""
echo "### Built packages"
echo '```'
Expand Down
5 changes: 1 addition & 4 deletions .github/workflows/fetch-source-pkg.yml
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,4 @@ jobs:
gh workflow run build-kernel.yml \
--repo "${{ github.repository }}" \
--field suite="${BRANCH}" \
--field kernel_version="${VERSION}" \
--field arch="arm64" \
--field flavor="qcom" \
--field runner="lecore-production"
--field kernel_version="${VERSION}"
110 changes: 94 additions & 16 deletions docs/PIPELINE.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ pkg-linux-qcom-canonical
│ ├── .github/workflows/
│ │ ├── fetch-source-pkg.yml ← manual incremental mirror sync
│ │ ├── bootstrap-history.yml ← one-time full-history seed
│ │ └── build-kernel.yml ← build .deb packages
│ │ └── build-kernel.yml ← build .deb packages (+ reusable workflow_call)
│ ├── scripts/
│ │ ├── sync-mirror.sh ← incremental mirror-repoint (CI)
│ │ └── seed-history.sh ← one-time full-history bootstrap (CI)
Expand All @@ -25,6 +25,7 @@ pkg-linux-qcom-canonical
│ └── ~1.4M commits; immutable tag per upload: Ubuntu-qcom-X.Y.Z-A.B (verbatim)
│
├── resolute-qcom-devel branch ← developer integration branch (see INTEGRATION.md)
│ └── .github/workflows/premerge-pr.yml ← pre-merge PR build check (lives here, not on main)
│
└── resolute-qcom-seed branch ← transient bootstrap staging (only during a seed)
```
Expand Down Expand Up @@ -190,7 +191,7 @@ triggers a build of the newest one.

Trigger manually: **Actions → Build: Canonical Kernel .deb Packages → Run
workflow**. It builds the `resolute-qcom-devel` integration branch by default; set
`suite` to `resolute-qcom` to build the mirror instead. The `kernel_version` input
the **Branch to build from** field to `resolute-qcom` to build from the mirror branch instead. The `kernel_version` input
controls what is checked out:

| `kernel_version` | Source checked out | Use |
Expand All @@ -216,7 +217,7 @@ Mirrors new upstream uploads into `resolute-qcom`, history-preserving, via
|-----|-------------|
| `check-version` | Two `ls-remote` calls (no clone): newest upstream `Ubuntu-qcom-*` tag vs our tags. Sets `should_sync`. |
| `sync` | Bare-clones our mirror; for each un-mirrored upload (ascending): fetches the upstream tag (delta only), advances `resolute-qcom`, and atomically pushes the branch + the verbatim tag (lease-pinned, fail-fast). Never merges/rebases. |
| `trigger-build` | Dispatches `build-kernel.yml` for the newest synced upload of the mirror (`suite=resolute-qcom`, `arch=arm64`, `flavor=qcom`, `runner=lecore-production`). |
| `trigger-build` | Dispatches `build-kernel.yml` for the newest synced upload of the mirror (`suite=resolute-qcom`, `kernel_version=<synced version>`). |

Idempotent: if the newest upload is already mirrored, the run exits without cloning
anything.
Expand All @@ -230,28 +231,105 @@ packages inside the base-suite-matched
- **Trigger**: dispatched by the sync, or manually.
- **Output**: **S3 only** (`lecore-production` runner). No artifacts, no releases.

| Input | Default | Description |
|-------|---------|-------------|
| `suite` | `resolute-qcom-devel` | Branch to build from; base suite (`resolute`) derived for container selection. |
| `kernel_version` | *(empty)* | Builds the exact `Ubuntu-qcom-<version>` tag (validated first). Empty = branch HEAD. |
| `arch` | `arm64` | Target architecture. |
| `flavor` | `qcom` | Kernel flavour. **`qcom` is the only flavour this tree builds** (`generic`/`lowlatency` do not exist for the qcom derivative). `all` builds `qcom` + `qcom-rt`. |
| `runner` | `ubuntu-24.04-arm` | `ubuntu-24.04-arm` (hosted, no S3), `self-hosted`, or `lecore-production` (the only runner that uploads to S3). |
| Input | Default | Trigger | Description |
|-------|---------|---------|-------------|
| `suite` | `resolute-qcom-devel` | dispatch + call | Branch to build from; base suite (`resolute`) derived for container selection. |
| `kernel_version` | *(empty)* | dispatch + call | Builds the exact `Ubuntu-qcom-<version>` tag (validated first). Empty = branch HEAD. |
| `devel_prs` | *(empty)* | dispatch + call | Space-separated PR numbers against `resolute-qcom-devel` to merge before building (e.g. `42 43`). Conflict aborts with a clear error. |
| `ref` | *(empty)* | call only | Exact git ref to checkout, overrides `suite`/`kernel_version` when set. Used by `premerge-pr.yml` to build the PR merge ref. |
| `skip_s3` | `false` | call only | Skip the S3 upload step. Used by `premerge-pr.yml` so premerge builds are build-only and never upload. |

**Build steps**:
1. Free disk space.
2. Checkout the tag/branch → `kernel-src/`.
3. Checkout `qualcomm-linux/docker-pkg-build@main`.
4. Derive `BASE_SUITE` (`resolute-qcom` → `resolute`).
5. Build the image: `docker_deb_build.py --rebuild -d <base_suite>`.
6. In-container: `apt-get build-dep linux`; `fakeroot make -f debian/rules clean`;
2. Checkout the tag/branch (or `ref` if set) → `kernel-src/`.
3. If `devel_prs` is set: merge each PR against `resolute-qcom-devel` into `kernel-src/` (conflict aborts).
4. Checkout `qualcomm-linux/docker-pkg-build@main`.
5. Derive `BASE_SUITE` (`resolute-qcom` → `resolute`).
6. Build the image: `docker_deb_build.py --rebuild -d <base_suite>`.
7. In-container: `apt-get build-dep linux`; `fakeroot make -f debian/rules clean`;
`fakeroot debian/rules binary-<flavor> do_skip_checks=true`
(see [Build container notes](#build-container-notes)).
7. Collect `.deb` files and upload to S3.
8. Collect `.deb` files and upload to S3 (skipped when `skip_s3=true`).

**Self-hosted runner requirements** (`lecore-production`): Ubuntu 24.04 arm64,
Docker (runner user in the `docker` group), ≥ 25 GB free disk.

### Pre-merge validation

The `premerge-pr.yml` check gives every PR into `resolute-qcom-devel` a build-only
kernel build as a status check. It builds the PR's **merge ref** (the PR as it would look merged into the
current branch HEAD) and uploads nothing (`skip_s3=true`).

- **Trigger**: `pull_request` on `resolute-qcom-devel` (opened, synchronize,
reopened), run via `build-kernel.yml`'s `workflow_call` interface.
- **Where it lives**: GitHub resolves `pull_request` workflows from the PR's
**base branch** - here `resolute-qcom-devel`, which is a kernel tree carrying
none of `main`'s workflows. (Base equals the default branch only in the common
case; here they differ, so a workflow on `main` alone would never fire.) So the
workflow lives **on the integration branch itself**, at
`resolute-qcom-devel:.github/workflows/premerge-pr.yml` - a thin caller that
invokes `build-kernel.yml@main`, so no build logic lives on the kernel branch.
It is added and maintained by its own PR into `resolute-qcom-devel` (it is
**not** carried on `main`); the reference copy is below.
- **Security**: it uses `pull_request`, **not** `pull_request_target`, so the run
has a read-only token and no secrets, and it builds the merge ref rather than a
raw attacker-controlled head. Contributor PRs come from **forks**, so the build
runs untrusted PR code (`debian/rules`) on the S3-credentialed
`lecore-production` runner. The repo **must** enable **"Require approval for all
fork pull requests"** (Settings > Actions > Fork pull request workflows) so a
maintainer reviews and approves each run before it executes - without it, this
is a pwn-request. Stronger still, and recommended before going public: run
pre-merge builds on a non-credentialed runner, or scope S3 to short-lived OIDC
so validation builds carry no write capability at all.
- **Concurrency**: one build per PR; a new push cancels the in-progress build.

Before the check is added: `main` must already carry the `workflow_call` interface
(added here), and the repo must have **"Require approval for all fork pull
requests"** enabled (see Security above) so fork PRs cannot run code on the runner
unreviewed. To make the check blocking, add **`Build check / Build`** as a required
status check on the `resolute-qcom-devel` ruleset.

<details>
<summary>Reference: <code>resolute-qcom-devel:.github/workflows/premerge-pr.yml</code></summary>

Copy this onto `resolute-qcom-devel` (via a PR into that branch) to add or restore
the check.

```yaml
# SPDX-License-Identifier: BSD-3-Clause
#
# premerge-pr.yml - pre-merge validation build for resolute-qcom-devel.
# Lives on resolute-qcom-devel (GitHub resolves pull_request workflows from the PR
# base branch). Thin caller of the reusable build-kernel.yml on main. Contributor
# PRs come from forks, so the repo must enable "Require approval for all fork pull
# requests". Rationale + security note: docs/PIPELINE.md#pre-merge-validation.

name: "Pre-merge PR build"

on:
pull_request:
branches:
- resolute-qcom-devel
types: [opened, synchronize, reopened]

permissions:
contents: read

concurrency:
group: premerge-pr-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
build:
name: "Build check"
uses: qualcomm-linux/pkg-linux-qcom-canonical/.github/workflows/build-kernel.yml@main
with:
suite: resolute-qcom-devel
ref: refs/pull/${{ github.event.pull_request.number }}/merge
skip_s3: true
```
</details>

## The CI scripts

| Script | Purpose | Used by |
Expand Down
Loading