diff --git a/online-data-tools/README.md b/online-data-tools/README.md index 45fb760b3..668d1ff04 100644 --- a/online-data-tools/README.md +++ b/online-data-tools/README.md @@ -103,6 +103,19 @@ and Seeed VIDs (`2341:0072`, `2886:0062`, `2886:8062`), so this supplement does not infer Silicon Labs board names from bridge-chip or debug-interface IDs. +Ambiq/Apollo3 currently has no wired PID supplement. The public USB-ID +sources already identify VID `0x2aec` as Ambiq Micro with PID `6011` +(`Converter`), while VID `0x1cbe` belongs to Luminary Micro/TI rather than +Ambiq. SparkFun's `Arduino_Apollo3` board package has no `vid.N`/`pid.N` +rows to ingest, and the Apollo3 boards present under +`crates/fbuild-config/assets/boards` are SparkFun Artemis boards uploaded via +serial loader rather than a documented Ambiq PID table. The `AM_APOLLO3` +MCU-to-VID seed therefore uses the weak CH340 bridge VID `0x1a86` only as a +board-search hint for those SparkFun boards; it does not add Ambiq product +PIDs without a first-party source. Third-party SDK or board-package rows may +be added later as supplemental data, but they should merge after first-party +and generic USB-ID sources so they fill gaps only. + ## Tests ```bash diff --git a/online-data-tools/seed_mcu_to_vid.json b/online-data-tools/seed_mcu_to_vid.json index 69ebfdb3d..13e804c1b 100644 --- a/online-data-tools/seed_mcu_to_vid.json +++ b/online-data-tools/seed_mcu_to_vid.json @@ -60,7 +60,7 @@ {"mcu_family": "LPC8", "vid": "1fc9", "score": 0.65, "reason": "NXP LPC8xx CMSIS-DAP / DAPLink"}, {"mcu_family": "LPC8", "vid": "0d28", "score": 0.55, "reason": "DAPLink CMSIS-DAP"}, - {"mcu_family": "AM_APOLLO3", "vid": "1cbe", "score": 0.70, "reason": "Sparkfun Apollo3 bootloader"}, + {"mcu_family": "AM_APOLLO3", "vid": "1a86", "score": 0.60, "reason": "SparkFun Artemis CH340 USB-serial bridge"}, {"mcu_family": "MGM240P", "vid": "10c4", "score": 0.65, "reason": "Silicon Labs CP210x"}, diff --git a/online-data-tools/test_orchestrators.py b/online-data-tools/test_orchestrators.py index 9115df6bc..236108e63 100644 --- a/online-data-tools/test_orchestrators.py +++ b/online-data-tools/test_orchestrators.py @@ -25,7 +25,6 @@ import sys import zipfile from pathlib import Path -from typing import Callable import pytest @@ -189,6 +188,73 @@ def test_update_www_second_run_keeps_existing_mcu_to_vid( assert row[0] == pytest.approx(0.99) +def test_seed_does_not_map_apollo3_to_luminary_vid() -> None: + seed = json.loads((HERE / "seed_mcu_to_vid.json").read_text(encoding="utf-8")) + apollo3 = [row for row in seed if row["mcu_family"] == "AM_APOLLO3"] + + assert apollo3 == [{ + "mcu_family": "AM_APOLLO3", + "vid": "1a86", + "score": 0.60, + "reason": "SparkFun Artemis CH340 USB-serial bridge", + }] + + +def test_update_www_replaces_deprecated_apollo3_vid( + workspace: Path, online_worktree: Path, www_worktree: Path, +) -> None: + # Simulate the stale online-data row that was copied before the seed was + # corrected. The migration must remove only that row and preserve curator + # additions for the same MCU family. + stale = [ + {"mcu_family": "AM_APOLLO3", "vid": "1cbe", + "score": 0.70, "reason": "Sparkfun Apollo3 bootloader"}, + {"mcu_family": "AM_APOLLO3", "vid": "2aec", + "score": 0.40, "reason": "custom Ambiq row"}, + {"mcu_family": "ESP32S3", "vid": "303a", + "score": 0.99, "reason": "custom curated"}, + ] + (online_worktree / "data" / "mcu_to_vid.json").write_text( + json.dumps(stale), encoding="utf-8" + ) + cfg = update_www.Config( + workspace = workspace, + online_worktree = online_worktree, + www_worktree = www_worktree, + today = "2026-06-20", + website_url = "https://example.invalid/fbuild/", + sqljs_zip_url = "https://unused.invalid/sqljs.zip", + ) + + summary = update_www.run(cfg, fetch_sqljs=lambda _url: _fake_sqljs_zip()) + + assert summary["mcu_to_vid_bootstrapped"] is False + assert summary["mcu_to_vid_corrections"] == 1 + corrected = json.loads( + (online_worktree / "data" / "mcu_to_vid.json").read_text(encoding="utf-8") + ) + by_family_vid = { + (row["mcu_family"], row["vid"]): row + for row in corrected + } + assert ("AM_APOLLO3", "1cbe") not in by_family_vid + assert by_family_vid[("AM_APOLLO3", "1a86")]["score"] == pytest.approx(0.60) + assert by_family_vid[("AM_APOLLO3", "2aec")]["reason"] == "custom Ambiq row" + + with sqlite3.connect(www_worktree / "2026-06-20.db") as conn: + rows = conn.execute( + "SELECT vid, score, reason FROM mcu_to_vid WHERE mcu_family=?", + ("AM_APOLLO3",), + ).fetchall() + assert (int("1cbe", 16), 0.70, "Sparkfun Apollo3 bootloader") not in rows + assert any( + vid == int("1a86", 16) + and score == pytest.approx(0.60) + and reason == "SparkFun Artemis CH340 USB-serial bridge" + for vid, score, reason in rows + ) + + def test_update_www_rotation_drops_old_dbs( workspace: Path, online_worktree: Path, www_worktree: Path, ) -> None: diff --git a/online-data-tools/update_www.py b/online-data-tools/update_www.py index 4118323f5..d9a279deb 100644 --- a/online-data-tools/update_www.py +++ b/online-data-tools/update_www.py @@ -32,6 +32,7 @@ import argparse import datetime as _dt import io +import json import shutil import sys import urllib.request @@ -53,6 +54,14 @@ # YAML doesn't need to enumerate them. STATIC_ASSETS = ("index.html", "app.js", "style.css") +# Narrow migrations for historical seed mistakes that may already have been +# copied to the curated online-data branch. Keep this list explicit: the +# online-data copy can carry curator edits, so seed changes must not become a +# blanket overwrite. +DEPRECATED_MCU_TO_VID_ROWS = { + ("AM_APOLLO3", "1cbe"), +} + @dataclass class Config: @@ -103,6 +112,84 @@ def bootstrap_mcu_to_vid(cfg: Config) -> bool: return True +def _vid_key(value: object) -> str: + """Normalize int, "1234", or "0x1234" VID values to 4-hex lowercase.""" + if isinstance(value, int): + return f"{value:04x}" + text = str(value).strip().lower() + if text.startswith("0x"): + text = text[2:] + return f"{int(text, 16):04x}" + + +def apply_mcu_to_vid_corrections(cfg: Config) -> int: + """Patch known-bad historical MCU VID rows in the online-data copy. + + `data/mcu_to_vid.json` is intentionally curator-owned after bootstrap, so + this does not resync the whole seed. It only removes explicitly deprecated + rows and adds the current seed row(s) for the affected family if absent. + Returns the number of deprecated rows removed. + """ + if not cfg.online_mcu_to_vid.is_file(): + return 0 + + online_rows = json.loads(cfg.online_mcu_to_vid.read_text(encoding="utf-8")) + seed_rows = json.loads(cfg.seed_mcu_to_vid.read_text(encoding="utf-8")) + if not isinstance(online_rows, list) or not isinstance(seed_rows, list): + return 0 + + seed_by_family: dict[str, list[dict]] = {} + for row in seed_rows: + if not isinstance(row, dict): + continue + family = row.get("mcu_family") + if isinstance(family, str): + seed_by_family.setdefault(family, []).append(row) + + kept_rows: list[dict] = [] + corrected_families: set[str] = set() + removed = 0 + for row in online_rows: + if not isinstance(row, dict): + kept_rows.append(row) + continue + family = row.get("mcu_family") + try: + vid = _vid_key(row.get("vid")) + except (TypeError, ValueError): + kept_rows.append(row) + continue + if (family, vid) in DEPRECATED_MCU_TO_VID_ROWS: + corrected_families.add(str(family)) + removed += 1 + continue + kept_rows.append(row) + + if removed == 0: + return 0 + + existing: set[tuple[object, str]] = set() + for row in kept_rows: + if not isinstance(row, dict) or not row.get("mcu_family") or not row.get("vid"): + continue + try: + existing.add((row.get("mcu_family"), _vid_key(row.get("vid")))) + except (TypeError, ValueError): + continue + for family in sorted(corrected_families): + for seed_row in seed_by_family.get(family, []): + key = (seed_row.get("mcu_family"), _vid_key(seed_row.get("vid"))) + if key not in existing: + kept_rows.append(dict(seed_row)) + existing.add(key) + + cfg.online_mcu_to_vid.write_text( + json.dumps(kept_rows, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + return removed + + def build_todays_db(cfg: Config) -> Path: data_dir = cfg.online_worktree / "data" build_sqlite.build_db( @@ -164,7 +251,6 @@ def rotate_dbs(cfg: Config) -> list[Path]: def write_www_manifest(cfg: Config) -> dict: manifest = build_www_manifest.build(cfg.www_worktree) - import json cfg.www_manifest.write_text( json.dumps(manifest, indent=2, sort_keys=False) + "\n", encoding="utf-8", @@ -173,7 +259,6 @@ def write_www_manifest(cfg: Config) -> dict: def annotate_online(cfg: Config, www_manifest: dict) -> dict: - import json online = json.loads(cfg.online_manifest.read_text(encoding="utf-8")) annotated = annotate_online_manifest.annotate( online_manifest = online, @@ -195,6 +280,7 @@ def run(cfg: Config, *, fetch_sqljs: Callable[[str], bytes] | None = None) -> di """Execute all steps in order. Returns a summary dict for logging.""" summary: dict = {"today": cfg.today, "website_url": cfg.website_url} summary["mcu_to_vid_bootstrapped"] = bootstrap_mcu_to_vid(cfg) + summary["mcu_to_vid_corrections"] = apply_mcu_to_vid_corrections(cfg) db = build_todays_db(cfg) summary["db_path"] = str(db) summary["db_bytes"] = db.stat().st_size