Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions online-data-tools/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,19 @@ and Seeed VIDs (`2341:0072`, `2886:0062`, `2886:8062`), so this supplement
does not infer Silicon Labs board names from bridge-chip or debug-interface
IDs.

Ambiq/Apollo3 currently has no wired PID supplement. The public USB-ID
sources already identify VID `0x2aec` as Ambiq Micro with PID `6011`
(`Converter`), while VID `0x1cbe` belongs to Luminary Micro/TI rather than
Ambiq. SparkFun's `Arduino_Apollo3` board package has no `vid.N`/`pid.N`
rows to ingest, and the Apollo3 boards present under
`crates/fbuild-config/assets/boards` are SparkFun Artemis boards uploaded via
serial loader rather than a documented Ambiq PID table. The `AM_APOLLO3`
MCU-to-VID seed therefore uses the weak CH340 bridge VID `0x1a86` only as a
board-search hint for those SparkFun boards; it does not add Ambiq product
PIDs without a first-party source. Third-party SDK or board-package rows may
be added later as supplemental data, but they should merge after first-party
and generic USB-ID sources so they fill gaps only.

## Tests

```bash
Expand Down
2 changes: 1 addition & 1 deletion online-data-tools/seed_mcu_to_vid.json
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
{"mcu_family": "LPC8", "vid": "1fc9", "score": 0.65, "reason": "NXP LPC8xx CMSIS-DAP / DAPLink"},
{"mcu_family": "LPC8", "vid": "0d28", "score": 0.55, "reason": "DAPLink CMSIS-DAP"},

{"mcu_family": "AM_APOLLO3", "vid": "1cbe", "score": 0.70, "reason": "Sparkfun Apollo3 bootloader"},
{"mcu_family": "AM_APOLLO3", "vid": "1a86", "score": 0.60, "reason": "SparkFun Artemis CH340 USB-serial bridge"},

{"mcu_family": "MGM240P", "vid": "10c4", "score": 0.65, "reason": "Silicon Labs CP210x"},

Expand Down
68 changes: 67 additions & 1 deletion online-data-tools/test_orchestrators.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import sys
import zipfile
from pathlib import Path
from typing import Callable

import pytest

Expand Down Expand Up @@ -189,6 +188,73 @@ def test_update_www_second_run_keeps_existing_mcu_to_vid(
assert row[0] == pytest.approx(0.99)


def test_seed_does_not_map_apollo3_to_luminary_vid() -> None:
seed = json.loads((HERE / "seed_mcu_to_vid.json").read_text(encoding="utf-8"))
apollo3 = [row for row in seed if row["mcu_family"] == "AM_APOLLO3"]

assert apollo3 == [{
"mcu_family": "AM_APOLLO3",
"vid": "1a86",
"score": 0.60,
"reason": "SparkFun Artemis CH340 USB-serial bridge",
}]


def test_update_www_replaces_deprecated_apollo3_vid(
workspace: Path, online_worktree: Path, www_worktree: Path,
) -> None:
# Simulate the stale online-data row that was copied before the seed was
# corrected. The migration must remove only that row and preserve curator
# additions for the same MCU family.
stale = [
{"mcu_family": "AM_APOLLO3", "vid": "1cbe",
"score": 0.70, "reason": "Sparkfun Apollo3 bootloader"},
{"mcu_family": "AM_APOLLO3", "vid": "2aec",
"score": 0.40, "reason": "custom Ambiq row"},
{"mcu_family": "ESP32S3", "vid": "303a",
"score": 0.99, "reason": "custom curated"},
]
(online_worktree / "data" / "mcu_to_vid.json").write_text(
json.dumps(stale), encoding="utf-8"
)
cfg = update_www.Config(
workspace = workspace,
online_worktree = online_worktree,
www_worktree = www_worktree,
today = "2026-06-20",
website_url = "https://example.invalid/fbuild/",
sqljs_zip_url = "https://unused.invalid/sqljs.zip",
)

summary = update_www.run(cfg, fetch_sqljs=lambda _url: _fake_sqljs_zip())

assert summary["mcu_to_vid_bootstrapped"] is False
assert summary["mcu_to_vid_corrections"] == 1
corrected = json.loads(
(online_worktree / "data" / "mcu_to_vid.json").read_text(encoding="utf-8")
)
by_family_vid = {
(row["mcu_family"], row["vid"]): row
for row in corrected
}
assert ("AM_APOLLO3", "1cbe") not in by_family_vid
assert by_family_vid[("AM_APOLLO3", "1a86")]["score"] == pytest.approx(0.60)
assert by_family_vid[("AM_APOLLO3", "2aec")]["reason"] == "custom Ambiq row"

with sqlite3.connect(www_worktree / "2026-06-20.db") as conn:
rows = conn.execute(
"SELECT vid, score, reason FROM mcu_to_vid WHERE mcu_family=?",
("AM_APOLLO3",),
).fetchall()
assert (int("1cbe", 16), 0.70, "Sparkfun Apollo3 bootloader") not in rows
assert any(
vid == int("1a86", 16)
and score == pytest.approx(0.60)
and reason == "SparkFun Artemis CH340 USB-serial bridge"
for vid, score, reason in rows
)


def test_update_www_rotation_drops_old_dbs(
workspace: Path, online_worktree: Path, www_worktree: Path,
) -> None:
Expand Down
90 changes: 88 additions & 2 deletions online-data-tools/update_www.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import argparse
import datetime as _dt
import io
import json
import shutil
import sys
import urllib.request
Expand All @@ -53,6 +54,14 @@
# YAML doesn't need to enumerate them.
STATIC_ASSETS = ("index.html", "app.js", "style.css")

# Narrow migrations for historical seed mistakes that may already have been
# copied to the curated online-data branch. Keep this list explicit: the
# online-data copy can carry curator edits, so seed changes must not become a
# blanket overwrite.
DEPRECATED_MCU_TO_VID_ROWS = {
("AM_APOLLO3", "1cbe"),
}


@dataclass
class Config:
Expand Down Expand Up @@ -103,6 +112,84 @@ def bootstrap_mcu_to_vid(cfg: Config) -> bool:
return True


def _vid_key(value: object) -> str:
"""Normalize int, "1234", or "0x1234" VID values to 4-hex lowercase."""
if isinstance(value, int):
return f"{value:04x}"
text = str(value).strip().lower()
if text.startswith("0x"):
text = text[2:]
return f"{int(text, 16):04x}"


def apply_mcu_to_vid_corrections(cfg: Config) -> int:
"""Patch known-bad historical MCU VID rows in the online-data copy.

`data/mcu_to_vid.json` is intentionally curator-owned after bootstrap, so
this does not resync the whole seed. It only removes explicitly deprecated
rows and adds the current seed row(s) for the affected family if absent.
Returns the number of deprecated rows removed.
"""
if not cfg.online_mcu_to_vid.is_file():
return 0

online_rows = json.loads(cfg.online_mcu_to_vid.read_text(encoding="utf-8"))
seed_rows = json.loads(cfg.seed_mcu_to_vid.read_text(encoding="utf-8"))
if not isinstance(online_rows, list) or not isinstance(seed_rows, list):
return 0

seed_by_family: dict[str, list[dict]] = {}
for row in seed_rows:
if not isinstance(row, dict):
continue
family = row.get("mcu_family")
if isinstance(family, str):
seed_by_family.setdefault(family, []).append(row)

kept_rows: list[dict] = []
corrected_families: set[str] = set()
removed = 0
for row in online_rows:
if not isinstance(row, dict):
kept_rows.append(row)
continue
family = row.get("mcu_family")
try:
vid = _vid_key(row.get("vid"))
except (TypeError, ValueError):
kept_rows.append(row)
continue
if (family, vid) in DEPRECATED_MCU_TO_VID_ROWS:
corrected_families.add(str(family))
removed += 1
continue
kept_rows.append(row)

if removed == 0:
return 0

existing: set[tuple[object, str]] = set()
for row in kept_rows:
if not isinstance(row, dict) or not row.get("mcu_family") or not row.get("vid"):
continue
try:
existing.add((row.get("mcu_family"), _vid_key(row.get("vid"))))
except (TypeError, ValueError):
continue
for family in sorted(corrected_families):
for seed_row in seed_by_family.get(family, []):
key = (seed_row.get("mcu_family"), _vid_key(seed_row.get("vid")))
if key not in existing:
kept_rows.append(dict(seed_row))
existing.add(key)

cfg.online_mcu_to_vid.write_text(
json.dumps(kept_rows, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
return removed


def build_todays_db(cfg: Config) -> Path:
data_dir = cfg.online_worktree / "data"
build_sqlite.build_db(
Expand Down Expand Up @@ -164,7 +251,6 @@ def rotate_dbs(cfg: Config) -> list[Path]:

def write_www_manifest(cfg: Config) -> dict:
manifest = build_www_manifest.build(cfg.www_worktree)
import json
cfg.www_manifest.write_text(
json.dumps(manifest, indent=2, sort_keys=False) + "\n",
encoding="utf-8",
Expand All @@ -173,7 +259,6 @@ def write_www_manifest(cfg: Config) -> dict:


def annotate_online(cfg: Config, www_manifest: dict) -> dict:
import json
online = json.loads(cfg.online_manifest.read_text(encoding="utf-8"))
annotated = annotate_online_manifest.annotate(
online_manifest = online,
Expand All @@ -195,6 +280,7 @@ def run(cfg: Config, *, fetch_sqljs: Callable[[str], bytes] | None = None) -> di
"""Execute all steps in order. Returns a summary dict for logging."""
summary: dict = {"today": cfg.today, "website_url": cfg.website_url}
summary["mcu_to_vid_bootstrapped"] = bootstrap_mcu_to_vid(cfg)
summary["mcu_to_vid_corrections"] = apply_mcu_to_vid_corrections(cfg)
db = build_todays_db(cfg)
summary["db_path"] = str(db)
summary["db_bytes"] = db.stat().st_size
Expand Down
Loading