Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions bench/fastled-examples/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,8 @@ fn measure_example(
framework_install_path: framework_root,
framework_version: "bench-fastled-examples-v1",
preprocessor_defines: &preprocessor_defines,
// The bench measures scan/cache throughput, not declaration handling.
declared_deps: &[],
};

let (cold, cold_ms) = timed(|| resolve_cached(&seeds, &search_paths, libraries, &inputs, &kv))?;
Expand Down
24 changes: 20 additions & 4 deletions crates/fbuild-build-arm/src/stm32/orchestrator/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! STM32 build orchestrator — wires together config, packages, compiler, linker.
//! STM32 build orchestrator — wires together config, packages, compiler, linker.
//!
//! Build phases:
//! 1. Parse platformio.ini
Expand Down Expand Up @@ -33,8 +33,8 @@ use fbuild_packages::{Framework, Toolchain};

use crate::compile_database::TargetArchitecture;
use crate::framework_libs::{
library_select_kv_store, resolve_framework_library_sources_active,
resolve_framework_library_sources_cached,
library_select_kv_store, resolve_framework_library_sources_active_declared,
resolve_framework_library_sources_cached, warn_if_lib_ldf_mode_unsupported,
};
use crate::generic_arm::{ArmCompiler, ArmLinker};
use crate::pipeline;
Expand Down Expand Up @@ -180,13 +180,28 @@ impl BuildOrchestrator for Stm32Orchestrator {
// is handled there — this string only needs to disambiguate stm32
// from teensy etc. so cross-platform key collisions are impossible.
let framework_info = fbuild_packages::Package::get_info(&framework);
// See the Teensy orchestrator: `lib_deps` is an explicit declaration,
// the only lever a project has for a framework library the shallow
// scan cannot infer (FastLED/fbuild#1214).
let declared_deps = ctx
.config
.get_lib_deps(&params.env_name)
.unwrap_or_default();
warn_if_lib_ldf_mode_unsupported(
ctx.config
.get_lib_ldf_mode(&params.env_name)
.ok()
.flatten()
.as_deref(),
);
let framework_library_sources = match library_select_kv_store() {
Some(store) => {
let key_inputs = fbuild_library_select::cache::CacheKeyInputs {
toolchain_triple: "stm32-arm-none-eabi",
framework_install_path: &framework_info.install_path,
framework_version: &framework_info.version,
preprocessor_defines: &defines,
declared_deps: &declared_deps,
};
resolve_framework_library_sources_cached(
&framework_libs,
Expand All @@ -196,11 +211,12 @@ impl BuildOrchestrator for Stm32Orchestrator {
store,
)
}
None => resolve_framework_library_sources_active(
None => resolve_framework_library_sources_active_declared(
&framework_libs,
&params.project_dir,
&ctx.src_dir,
&defines,
&declared_deps,
),
};
if !framework_library_sources.is_empty() {
Expand Down
23 changes: 20 additions & 3 deletions crates/fbuild-build-arm/src/teensy/orchestrator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ use crate::build_fingerprint::{
use crate::compile_database::TargetArchitecture;
use crate::compiler::Compiler as _;
use crate::framework_libs::{
library_select_kv_store, resolve_framework_library_sources_active,
resolve_framework_library_sources_cached,
library_select_kv_store, resolve_framework_library_sources_active_declared,
resolve_framework_library_sources_cached, warn_if_lib_ldf_mode_unsupported,
};
use crate::pipeline;
use crate::{BuildOrchestrator, BuildParams, BuildResult, SourceScanner};
Expand Down Expand Up @@ -195,13 +195,29 @@ impl BuildOrchestrator for TeensyOrchestrator {
// so bumping it invalidates the entire teensy slice without
// touching SCANNER_VERSION / LDF_MODE_VERSION.
let framework_info = fbuild_packages::Package::get_info(&framework);
// `lib_deps` is an explicit user declaration, not a deeper LDF: it
// lets a project name a framework library the shallow scan cannot
// infer. Previously only the ESP32 orchestrator consumed it, so on
// Teensy a project had no lever at all (FastLED/fbuild#1214).
let declared_deps = ctx
.config
.get_lib_deps(&params.env_name)
.unwrap_or_default();
warn_if_lib_ldf_mode_unsupported(
ctx.config
.get_lib_ldf_mode(&params.env_name)
.ok()
.flatten()
.as_deref(),
);
let framework_library_sources = match library_select_kv_store() {
Some(store) => {
let key_inputs = fbuild_library_select::cache::CacheKeyInputs {
toolchain_triple: "teensy-arm-none-eabi",
framework_install_path: &framework_info.install_path,
framework_version: &framework_info.version,
preprocessor_defines: &ldf_defines,
declared_deps: &declared_deps,
};
resolve_framework_library_sources_cached(
&framework_libs,
Expand All @@ -211,11 +227,12 @@ impl BuildOrchestrator for TeensyOrchestrator {
store,
)
}
None => resolve_framework_library_sources_active(
None => resolve_framework_library_sources_active_declared(
&framework_libs,
&params.project_dir,
&ctx.src_dir,
&ldf_defines,
&declared_deps,
),
};
if !framework_library_sources.is_empty() {
Expand Down
58 changes: 54 additions & 4 deletions crates/fbuild-build-engine/src/framework_libs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@ use std::path::{Path, PathBuf};
use std::sync::OnceLock;

use fbuild_library_select::cache::{CacheKeyInputs, FileKvStore, resolve_cached};
use fbuild_library_select::{
resolve as resolve_library_selection, resolve_active as resolve_active_library_selection,
};
use fbuild_library_select::resolve as resolve_library_selection;
use fbuild_packages::library::FrameworkLibrary;
use walkdir::{DirEntry, WalkDir};

Expand All @@ -40,12 +38,63 @@ pub fn resolve_framework_library_sources_active(
project_dir: &Path,
src_dir: &Path,
defines: &HashMap<String, String>,
) -> Vec<PathBuf> {
resolve_framework_library_sources_active_declared(libraries, project_dir, src_dir, defines, &[])
}

/// [`resolve_framework_library_sources_active`] honoring `lib_deps`.
///
/// `declared` are the `platformio.ini` `lib_deps` entries for the env being
/// built. A framework library named there is selected even though the header
/// scan never reaches it — the escape hatch for a dependency the finder
/// cannot infer, which previously had no lever at all on the Teensy/STM32
/// path (FastLED/fbuild#1214).
///
/// The scan itself is unchanged: seeds are still project translation units
/// only, so #1094's "an inactive local library header must not select a
/// framework library" invariant still holds.
pub fn resolve_framework_library_sources_active_declared(
libraries: &[FrameworkLibrary],
project_dir: &Path,
src_dir: &Path,
defines: &HashMap<String, String>,
declared: &[String],
) -> Vec<PathBuf> {
let roots = framework_include_scan_roots(project_dir, src_dir);
let filtered = filter_framework_libs_shadowed_by_project(libraries, &roots);
let seeds = collect_project_seeds(&roots);
let search_paths = project_search_paths(&roots);
resolve_active_library_selection(&seeds, &search_paths, &filtered, defines).source_files
fbuild_library_select::resolve_with_stats_active_declared(
&seeds,
&search_paths,
&filtered,
defines,
declared,
)
.0
.source_files
}

/// Warn when a project sets `lib_ldf_mode`, which fbuild does not implement.
///
/// The resolver is fixed at a `chain`-style scan seeded from project sources.
/// Accepting the key silently lets a project believe `deep` is in effect and
/// spend a debugging session wondering why it changed nothing
/// (FastLED/fbuild#1214). `chain` and `off` are close enough to the actual
/// behavior to pass without noise.
pub fn warn_if_lib_ldf_mode_unsupported(mode: Option<&str>) {
let Some(mode) = mode.map(str::trim).filter(|m| !m.is_empty()) else {
return;
};
if mode.eq_ignore_ascii_case("chain") || mode.eq_ignore_ascii_case("off") {
return;
}
tracing::warn!(
lib_ldf_mode = %mode,
"lib_ldf_mode is not implemented and has no effect; fbuild always \
resolves libraries with a chain-style scan seeded from project \
sources. Declare the dependency with `lib_deps` instead."
);
}

/// Drop framework libraries whose primary header (`<lib_name>.h`) is
Expand Down Expand Up @@ -921,6 +970,7 @@ mod tests {
framework_install_path: &framework_root,
framework_version: "0.0.0-test",
preprocessor_defines: &defines,
declared_deps: &[],
};

let kv = FileKvStore::open(tmp.path().join("kv")).unwrap();
Expand Down
12 changes: 12 additions & 0 deletions crates/fbuild-config/src/ini_parser/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,18 @@ impl PlatformIOConfig {
}
}

/// Get `lib_ldf_mode` for an environment, if set.
///
/// fbuild does not implement PlatformIO's LDF modes — the resolver is
/// fixed at a `chain`-style scan from project sources. This getter exists
/// so callers can *warn* that the setting is inert rather than silently
/// ignoring it, which is how a project ends up believing `deep` is in
/// effect (FastLED/fbuild#1214).
pub fn get_lib_ldf_mode(&self, env_name: &str) -> fbuild_core::Result<Option<String>> {
let config = self.get_env_config(env_name)?;
Ok(config.get("lib_ldf_mode").map(|m| m.trim().to_string()))
}

/// Get extra library search directories for an environment.
pub fn get_lib_extra_dirs(&self, env_name: &str) -> fbuild_core::Result<Vec<String>> {
if let Some(dirs) = self.overrides.get_lib_extra_dirs() {
Expand Down
1 change: 1 addition & 0 deletions crates/fbuild-library-select/benches/resolve_warm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ fn bench_resolve_warm(c: &mut Criterion) {
framework_install_path: &framework_root,
framework_version: "1.59.0",
preprocessor_defines: &defines,
declared_deps: &[],
};

// Prime the cache so the timed loop measures the hit path only.
Expand Down
74 changes: 72 additions & 2 deletions crates/fbuild-library-select/src/cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ use std::path::{Path, PathBuf};
use fbuild_packages::library::FrameworkLibrary;
use prost::Message;

use crate::{Selection, canon, resolve_active};
use crate::{Selection, canon};

/// Bump when the scanner's lexical grammar changes in a way that could change
/// which `#include` directives it emits for the same source.
Expand Down Expand Up @@ -189,6 +189,12 @@ pub struct CacheKeyInputs<'a> {
/// Defines supplied to the compiler. They select the active include graph
/// and therefore must be part of the cache identity.
pub preprocessor_defines: &'a HashMap<String, String>,
/// `lib_deps` entries from `platformio.ini`. These force-select framework
/// libraries the header scan never reaches, so editing them changes the
/// selection with no change to any scanned file — meaning they MUST be
/// part of the key or the edit is invisible behind a warm cache
/// (FastLED/fbuild#1214).
pub declared_deps: &'a [String],
}

/// Result of [`resolve_cached`]. `from_cache` distinguishes hit from miss so
Expand Down Expand Up @@ -232,6 +238,18 @@ pub fn cache_key(
h.update(inputs.framework_version.as_bytes());
h.update(b"\n");

// Sorted: `lib_deps` is a set of declarations, so reordering the ini
// lines must not invalidate the cache.
let mut declared: Vec<&str> = inputs.declared_deps.iter().map(String::as_str).collect();
declared.sort_unstable();
declared.dedup();
h.update(b"declared_deps:");
h.update(&(declared.len() as u64).to_le_bytes());
for dep in declared {
h.update(&(dep.len() as u64).to_le_bytes());
h.update(dep.as_bytes());
}

let mut defines: Vec<(&String, &String)> = inputs.preprocessor_defines.iter().collect();
defines.sort_unstable_by(|left, right| left.0.cmp(right.0));
h.update(b"defines:");
Expand Down Expand Up @@ -348,7 +366,14 @@ pub fn resolve_cached(
}
}

let selection = resolve_active(seeds, search_paths, libraries, inputs.preprocessor_defines);
let selection = crate::resolve_with_stats_active_declared(
seeds,
search_paths,
libraries,
inputs.preprocessor_defines,
inputs.declared_deps,
)
.0;
// serde's `PathBuf` Serialize impl errors when a path component is not
// valid UTF-8 (legal on Unix, possible on Windows via canonicalize edge
// cases). Treat that as a cache write miss — degraded performance is
Expand Down Expand Up @@ -407,6 +432,7 @@ mod tests {
framework_install_path: framework_root,
framework_version: "1.59.0",
preprocessor_defines: &EMPTY_DEFINES,
declared_deps: &[],
}
}

Expand Down Expand Up @@ -472,12 +498,14 @@ mod tests {
framework_install_path: tmp.path(),
framework_version: "1.59.0",
preprocessor_defines: &EMPTY_DEFINES,
declared_deps: &[],
};
let b = CacheKeyInputs {
toolchain_triple: "xtensa-esp32-elf",
framework_install_path: tmp.path(),
framework_version: "1.59.0",
preprocessor_defines: &EMPTY_DEFINES,
declared_deps: &[],
};
assert_ne!(
cache_key(&seeds, &search_paths, &libs, &a).as_bytes(),
Expand All @@ -494,8 +522,49 @@ mod tests {
framework_install_path: a.framework_install_path,
framework_version: "1.60.0",
preprocessor_defines: a.preprocessor_defines,
declared_deps: &[],
};
assert_ne!(
cache_key(&seeds, &search_paths, &libs, &a).as_bytes(),
cache_key(&seeds, &search_paths, &libs, &b).as_bytes()
);
}

/// A `lib_deps` edit changes the selection without touching any scanned
/// file, so if it weren't in the key a warm cache would silently serve the
/// old selection — exactly the "latent until a cold resolution" failure
/// mode described in FastLED/fbuild#1214.
#[test]
fn c04c_declared_deps_change_invalidates_key() {
let (tmp, seeds, search_paths, libs) = build_simple_project();
let none = fixture_inputs(tmp.path());
let declared = vec!["SPI".to_string()];
let with_spi = CacheKeyInputs {
declared_deps: &declared,
..fixture_inputs(tmp.path())
};
assert_ne!(
cache_key(&seeds, &search_paths, &libs, &none).as_bytes(),
cache_key(&seeds, &search_paths, &libs, &with_spi).as_bytes()
);
}

/// `lib_deps` is a set of declarations: reordering the ini lines must not
/// throw away a valid cache entry.
#[test]
fn c04d_declared_deps_order_does_not_affect_key() {
let (tmp, seeds, search_paths, libs) = build_simple_project();
let forward = vec!["SPI".to_string(), "Wire".to_string()];
let reversed = vec!["Wire".to_string(), "SPI".to_string()];
let a = CacheKeyInputs {
declared_deps: &forward,
..fixture_inputs(tmp.path())
};
let b = CacheKeyInputs {
declared_deps: &reversed,
..fixture_inputs(tmp.path())
};
assert_eq!(
cache_key(&seeds, &search_paths, &libs, &a).as_bytes(),
cache_key(&seeds, &search_paths, &libs, &b).as_bytes()
);
Expand All @@ -512,6 +581,7 @@ mod tests {
framework_install_path: a.framework_install_path,
framework_version: a.framework_version,
preprocessor_defines: &defines,
declared_deps: &[],
};
assert_ne!(
cache_key(&seeds, &search_paths, &libs, &a).as_bytes(),
Expand Down
Loading
Loading