From bd7f144c4286ba833ad1092efdfd1f9a73168a71 Mon Sep 17 00:00:00 2001 From: zackees Date: Tue, 14 Jul 2026 20:34:01 -0700 Subject: [PATCH] refactor: remove runtime embedded USB catalogues --- crates/fbuild-core/Cargo.toml | 7 +++---- crates/fbuild-core/src/usb/data.rs | 14 +++++++++++--- crates/fbuild-core/src/usb/mod.rs | 26 +++++++++----------------- crates/fbuild-core/src/usb/resolver.rs | 22 ++++++++++++++++++++-- docs/usb-vidpid-audit.md | 4 ++-- 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/crates/fbuild-core/Cargo.toml b/crates/fbuild-core/Cargo.toml index 0d3ceceb9..0579d25d3 100644 --- a/crates/fbuild-core/Cargo.toml +++ b/crates/fbuild-core/Cargo.toml @@ -21,11 +21,8 @@ sha2 = { workspace = true } # compile-time-embedded `usb-vendors.tar.zst` archive instead (see # `crate::usb::embedded`). usb-ids = { workspace = true } -# Embedded USB-vendor archive decompression + extraction at first use. -# Pulled in as workspace deps so other crates can share the same zstd / tar -# wire format without per-crate version drift. +# Runtime decompression for the downloaded FastLED/boards cache. zstd = { workspace = true } -tar = { workspace = true } prost = { workspace = true } # Process containment primitive (Job Objects on Windows; process groups + # PR_SET_PDEATHSIG on Linux; process groups on macOS). The single global @@ -55,3 +52,5 @@ libc = "0.2" [dev-dependencies] tempfile = { workspace = true } +# Test-only embedded USB-vendor fixture extraction. Production never links it. +tar = { workspace = true } diff --git a/crates/fbuild-core/src/usb/data.rs b/crates/fbuild-core/src/usb/data.rs index 5f14c227b..08d6ed002 100644 --- a/crates/fbuild-core/src/usb/data.rs +++ b/crates/fbuild-core/src/usb/data.rs @@ -59,7 +59,9 @@ use prost::Message; use serde_json::Value; use std::collections::HashMap; use std::path::Path; -use std::sync::{OnceLock, RwLock}; +#[cfg(test)] +use std::sync::OnceLock; +use std::sync::RwLock; use std::time::Duration; /// FastLED/boards published registry (canonical USB VID:PID source). @@ -86,14 +88,13 @@ static ONLINE_MAP: RwLock>> = RwLock::new(None); // path. Keep the historical blob available only to unit tests as a fixture. #[cfg(test)] const EMBEDDED_PROTO: &[u8] = include_bytes!("../../data/usb-vids.proto.zstd"); -#[cfg(not(test))] -const EMBEDDED_PROTO: &[u8] = &[]; /// Both projections of the embedded proto. The compact `usb-vids.proto.zstd` /// carries a `Vendor{vid, name, [Product{pid, name}]}` tree, so a single /// artifact yields BOTH a VID→vendor map AND a VID:PID→{vendor, product} /// map — no separate per-VID blob or hardcoded table needed. Parsed exactly /// once on first use. +#[cfg(test)] #[derive(Default)] struct EmbeddedOverlay { /// VID:PID → {vendor, product}. @@ -104,8 +105,10 @@ struct EmbeddedOverlay { vendors: HashMap, } +#[cfg(test)] static EMBEDDED: OnceLock = OnceLock::new(); +#[cfg(test)] fn embedded() -> &'static EmbeddedOverlay { EMBEDDED.get_or_init(|| decode_embedded_overlay(EMBEDDED_PROTO).unwrap_or_default()) } @@ -113,6 +116,7 @@ fn embedded() -> &'static EmbeddedOverlay { /// Inflate + parse the embedded proto into both projections. Errors bubble /// up to `unwrap_or_default()` (empty overlay) so a bad blob degrades to /// tier-1 vendor resolution rather than crashing. +#[cfg(test)] fn decode_embedded_overlay(raw: &[u8]) -> Result { let mut decoded = Vec::with_capacity(raw.len() * 4); zstd::stream::copy_decode(raw, &mut decoded).map_err(|e| format!("zstd: {e}"))?; @@ -143,16 +147,19 @@ fn decode_embedded_overlay(raw: &[u8]) -> Result { /// The embedded VID→vendor map (from the same proto). `None` if the VID is /// absent from the embedded overlay. +#[cfg(test)] pub(crate) fn embedded_vendor(vid: u16) -> Option<&'static str> { embedded().vendors.get(&vid).map(|s| s.as_str()) } /// Number of VID:PID rows in the embedded overlay (test/introspection aid). +#[cfg(test)] pub fn embedded_vidpid_count() -> usize { embedded().vidpid.len() } /// Number of VID→vendor rows in the embedded overlay (test/introspection aid). +#[cfg(test)] pub fn embedded_vendor_count() -> usize { embedded().vendors.len() } @@ -471,6 +478,7 @@ pub(crate) fn online_lookup(vid: u16, pid: u16) -> Option { } /// Compile-time embedded overlay only (FastLED/boards curated device map). +#[cfg(test)] pub(crate) fn embedded_lookup(vid: u16, pid: u16) -> Option { embedded().vidpid.get(&pack(vid, pid)).cloned() } diff --git a/crates/fbuild-core/src/usb/mod.rs b/crates/fbuild-core/src/usb/mod.rs index 877ad7812..bf8deb341 100644 --- a/crates/fbuild-core/src/usb/mod.rs +++ b/crates/fbuild-core/src/usb/mod.rs @@ -1,19 +1,11 @@ //! USB VID:PID → human-readable vendor/product name resolution. //! -//! Three resolution tiers, queried in order: +//! Two production resolution tiers, queried in order: //! -//! 1. **Online overlay** — an optional `{ "VVVV:PPPP": {vendor, product} }` -//! JSON map loaded at runtime (typically from a daemon-managed cache -//! file that mirrors the `online-data` branch of this repo). This is -//! the richest source — it has both vendor AND product names — and is -//! queried first. -//! 2. **Embedded vendor archive** — a 22 KB `tar.zst` blob compiled in -//! via `include_bytes!` (see [`embedded`]). Vendor names only — for -//! VIDs the overlay doesn't carry, we resolve the vendor offline and -//! synthesize `"Device 0xPPPP"` as the product placeholder. Per-PID -//! detail is intentionally not bundled — clients can hit the -//! SQLite-over-HTTP database on the `www` branch for that. -//! 3. **Fallback** — synthetic `"Unknown vendor 0xVVVV"` placeholder so +//! 1. **FastLED/boards cache** — the verified published USB identity +//! artifact loaded by the daemon. This is the only production device +//! catalogue. +//! 2. **Fallback** — synthetic `"Unknown vendor 0xVVVV"` placeholder so //! callers can always print something deterministic. //! //! See [`resolve`] (best-effort, never `None`), [`try_resolve`] (returns @@ -21,12 +13,11 @@ //! `"vendor product (VVVV:PPPP)"` for connect / scan / `device list` log //! lines). //! -//! The daemon calls [`install_online_cache`] at startup with the path to -//! the locally-cached `usb-vid.json`. The CLI / nightly workflow keeps -//! that file in sync with the manifest URL exposed by the `online-data` -//! branch — see [`MANIFEST_URL`] and [`USB_VID_JSON_URL`]. +//! Embedded vendor/device archives are available only in unit-test builds. +//! Production never silently falls back to compiled USB identity data. pub mod data; +#[cfg(test)] pub mod embedded; pub mod profiles; pub mod resolver; @@ -37,5 +28,6 @@ pub use data::{ try_install_online_cache_proto_zstd, MANIFEST_URL, ONLINE_CACHE_TTL_SECS, USB_VIDS_PROTO_ZSTD_URL, USB_VID_JSON_URL, }; +#[cfg(test)] pub use embedded::vendor_name as embedded_vendor_name; pub use resolver::{pretty, resolve, resolve_bundled, try_resolve, UsbInfo}; diff --git a/crates/fbuild-core/src/usb/resolver.rs b/crates/fbuild-core/src/usb/resolver.rs index 1aab4f2b1..afd2815eb 100644 --- a/crates/fbuild-core/src/usb/resolver.rs +++ b/crates/fbuild-core/src/usb/resolver.rs @@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize}; +#[cfg(test)] use super::embedded; /// Resolved USB device identity. @@ -33,12 +34,20 @@ pub fn resolve(vid: u16, pid: u16) -> UsbInfo { /// information; we only fall through to the embedded archive when the /// overlay misses the VID entirely. pub fn try_resolve(vid: u16, pid: u16) -> Option { - // 1. Online overlay wins entirely (freshest, curated at workflow time). if let Some(info) = super::data::online_lookup(vid, pid) { return Some(info); } - // 2. Embedded overlay: take the PRODUCT from the FastLED/boards curated + #[cfg(not(test))] + { + None + } + + #[cfg(test)] + { + // Test builds may exercise an embedded fixture. Release/runtime builds + // must use only the verified FastLED/boards cache above. + // Take the PRODUCT from the FastLED/boards curated // device map (e.g. "NXP LPC-Link2", "Teensy (Serial mode)"), but // resolve the VENDOR through the best available source rather than the // proto's per-VID:PID vendor column (which can be blank, or — for the @@ -63,6 +72,7 @@ pub fn try_resolve(vid: u16, pid: u16) -> Option { // the curated proto entirely. (None, _) => resolve_bundled(vid, pid), } + } } /// Vendor-name-only tier (no per-PID product). Two compile-time-embedded @@ -83,11 +93,19 @@ pub fn try_resolve(vid: u16, pid: u16) -> Option { /// `"Device 0xPPPP"` placeholder since per-PID resolution lives in the /// VID:PID overlay (tier-2). pub fn resolve_bundled(vid: u16, pid: u16) -> Option { + #[cfg(not(test))] + { + let _ = (vid, pid); + None + } + #[cfg(test)] + { let vendor = embedded::vendor_name(vid).or_else(|| super::data::embedded_vendor(vid))?; Some(UsbInfo { vendor: vendor.to_string(), product: format!("Device 0x{pid:04X}"), }) + } } /// `"vendor product (VVVV:PPPP)"` — the canonical display format used by diff --git a/docs/usb-vidpid-audit.md b/docs/usb-vidpid-audit.md index 7b1762304..4dc977500 100644 --- a/docs/usb-vidpid-audit.md +++ b/docs/usb-vidpid-audit.md @@ -14,8 +14,8 @@ catalogue blob (`EMBEDDED_PROTO` is empty outside tests). | `crates/fbuild-config/assets/boards/json/*.json` (1,645 checked-in board records; 516 contain `build.vid` and/or `build.pid`) | Canonical board metadata snapshot | These fields are copied from the FastLED/boards publication and are the only board-level identity input. The snapshot must be refreshed from FastLED/boards; fbuild must not add guessed values. | | `crates/fbuild-config/src/board/mcu_vid.rs` + `online-data-tools/seed_mcu_to_vid.json` | Legacy MCU-family heuristic | `mcu_vid.rs` embeds the seed at compile time and maps MCU names to a guessed VID when board metadata is absent. This is production identity knowledge, not a test fixture; it requires FastLED/boards#47 to publish an explicit semantic identity/role field before removal. | | `crates/fbuild-core/src/usb/data.rs` (`MANIFEST_URL`, protobuf/JSON cache boundary) | Canonical ingestion | Runtime cache populated from FastLED/boards; legacy JSON URL remains compatibility-only and must be removed after consumers migrate. | -| `crates/fbuild-core/data/usb-vendors.tar.zst` + `crates/fbuild-core/src/usb/embedded.rs` + `crates/fbuild-core/src/usb/resolver.rs` | Production vendor-only fallback | This archive is compile-time embedded in every release and `resolve_bundled()` falls back to it for vendor names. It is not a board-role catalogue, but it is still production USB identity data and must remain explicitly vendor-only; product/role resolution must come from FastLED/boards. | -| `crates/fbuild-serial/src/boards.rs` (`BOARD_FINGERPRINTS`, `ENVIRONMENT_TO_VCOM`, `family_for_vid_pid`) | Legacy runtime catalogue | Unsafe duplicate of board identity data. Existing rows are retained for compatibility in this PR and are the migration blocker: each row needs a matching FastLED/boards record plus a schema field expressing deploy family/reset semantics before removal. | +| `crates/fbuild-core/data/usb-vendors.tar.zst` + `crates/fbuild-core/src/usb/embedded.rs` | Test-only fixture | The module, archive extraction dependency, and fallback are compiled only under `cfg(test)`. Release/runtime resolution uses the verified FastLED/boards cache or an explicit unknown-device label. | +| `crates/fbuild-serial/src/boards.rs` (`BOARD_FINGERPRINTS`, `ENVIRONMENT_TO_VCOM`, `family_for_vid_pid`) | Migrated; test fixtures remain | Production hints, VCOM selection, and reset-family classification now derive from verified typed FastLED/boards profiles. Concrete tables and range matching are compiled only under `cfg(test)`. | | `crates/fbuild-serial/src/bootloader_watcher.rs` | Legacy bootloader VID/PID signatures | RP2040/SAMD/Teensy bootloader detection still uses concrete signatures; boards metadata needs a bootloader identity/role field before this can become data-driven. | | `crates/fbuild-daemon/src/handlers/operations/deploy_port.rs` | Legacy runtime VID fallback | Expected vendor IDs are deploy-port selection heuristics, not names; they still duplicate identity knowledge and require boards-derived upload metadata before removal. | | `crates/fbuild-deploy/src/lpc_debugger_reflash.rs` | Protocol/device compatibility constants | LPC-Link2 firmware recovery requires the exact probe identity. Provenance is NXP/FastLED LPC-Link2 documentation; move to boards metadata when the probe schema supports non-board recovery targets. |