From d071a33543a3c543c48cbf654a04a821ada51f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:37:40 -0300 Subject: [PATCH 1/8] feat(tooling): add event-monitor arrival-time dashboard Standalone Rust/axum collector + vanilla-JS dashboard that dials the /lean/v0/events SSE stream of several ethlambda nodes, timestamps each event on one collector clock, and visualizes arrival offset within the slot (rolling per-node beeswarm) and inter-node propagation delay per block/aggregate/head (a second delta beeswarm, delay behind the first node to see each id). Own Cargo workspace with no ethlambda-crate deps; speaks only the documented SSE/HTTP wire shape frozen in CONTRACT.md. Rolling window is adjustable live from the header, a fresh page load backfills recent history via GET /api/history so it is never blank, and ?demo=1 runs the frontend fully offline on synthetic data. --- tooling/event-monitor/.gitignore | 2 + tooling/event-monitor/CONTRACT.md | 243 ++ tooling/event-monitor/Cargo.lock | 2129 +++++++++++++++++ tooling/event-monitor/Cargo.toml | 32 + tooling/event-monitor/README.md | 33 + tooling/event-monitor/config.example.toml | 35 + tooling/event-monitor/src/collector.rs | 249 ++ tooling/event-monitor/src/config.rs | 153 ++ tooling/event-monitor/src/hub.rs | 231 ++ tooling/event-monitor/src/lib.rs | 10 + tooling/event-monitor/src/main.rs | 69 + tooling/event-monitor/src/model.rs | 401 ++++ tooling/event-monitor/src/server.rs | 125 + tooling/event-monitor/src/timing.rs | 194 ++ .../event-monitor/tests/sse_integration.rs | 169 ++ tooling/event-monitor/web/app.js | 206 ++ tooling/event-monitor/web/beeswarm.js | 267 +++ tooling/event-monitor/web/demo.js | 169 ++ tooling/event-monitor/web/index.html | 55 + tooling/event-monitor/web/propagation.js | 337 +++ tooling/event-monitor/web/style.css | 339 +++ 21 files changed, 5448 insertions(+) create mode 100644 tooling/event-monitor/.gitignore create mode 100644 tooling/event-monitor/CONTRACT.md create mode 100644 tooling/event-monitor/Cargo.lock create mode 100644 tooling/event-monitor/Cargo.toml create mode 100644 tooling/event-monitor/README.md create mode 100644 tooling/event-monitor/config.example.toml create mode 100644 tooling/event-monitor/src/collector.rs create mode 100644 tooling/event-monitor/src/config.rs create mode 100644 tooling/event-monitor/src/hub.rs create mode 100644 tooling/event-monitor/src/lib.rs create mode 100644 tooling/event-monitor/src/main.rs create mode 100644 tooling/event-monitor/src/model.rs create mode 100644 tooling/event-monitor/src/server.rs create mode 100644 tooling/event-monitor/src/timing.rs create mode 100644 tooling/event-monitor/tests/sse_integration.rs create mode 100644 tooling/event-monitor/web/app.js create mode 100644 tooling/event-monitor/web/beeswarm.js create mode 100644 tooling/event-monitor/web/demo.js create mode 100644 tooling/event-monitor/web/index.html create mode 100644 tooling/event-monitor/web/propagation.js create mode 100644 tooling/event-monitor/web/style.css diff --git a/tooling/event-monitor/.gitignore b/tooling/event-monitor/.gitignore new file mode 100644 index 00000000..fabfb875 --- /dev/null +++ b/tooling/event-monitor/.gitignore @@ -0,0 +1,2 @@ +/target +/config.toml diff --git a/tooling/event-monitor/CONTRACT.md b/tooling/event-monitor/CONTRACT.md new file mode 100644 index 00000000..c18430ad --- /dev/null +++ b/tooling/event-monitor/CONTRACT.md @@ -0,0 +1,243 @@ +# event-monitor — interface contract (authoritative) + +This file is the **single source of truth** for every interface shared between the +Rust backend (`src/`) and the vanilla-JS frontend (`web/`). Neither side may +change a field name, endpoint, or event name defined here without the other. + +Hard constraints: + +- Standalone Cargo project under `tooling/event-monitor/`. Its `Cargo.toml` + declares an empty `[workspace]` table so it is its own workspace root, fully + decoupled from the parent `ethlambda` workspace. +- **No dependency on any ethlambda crate.** The tool only knows the SSE *wire + shape* documented below (copied from `docs/rpc.md`); it never imports + `ethlambda-*`. + +--- + +## 1. What it does + +Dials the `GET /lean/v0/events` SSE endpoint of several ethlambda nodes, +timestamps each event on arrival (one collector clock), normalizes it, and +re-serves the browser a single merged stream plus a static dashboard. The +dashboard shows, for `block` / `attestation` / `aggregate` events: + +- a **rolling beeswarm** of arrival offset *within the slot* per node (top), and +- a **propagation-delta** view: for one block/aggregate, how long after the + first node each other node saw it (bottom). + +``` + ethlambda nodes event-monitor (Rust/axum) browser + node-2 :5052 ─/events─┐ per-node SSE client → stamp arrival EventSource + node-3 :5052 ─/events─┼─▶ → normalize → tokio::broadcast hub ─────▶ /stream + node-N ─/events─┘ axum: GET / GET /stream GET /api/meta beeswarm+propagation +``` + +--- + +## 2. Upstream input — ethlambda SSE (what the collector consumes) + +Endpoint per node: `GET {node.url}/lean/v0/events?topics=`. +Default subscribed topics (configurable): `block,attestation,aggregate`. + +The response is `text/event-stream`. Each frame has an `event:` line (the topic +name) and a `data:` line (flat JSON). The collector must read **both**: the topic +comes from the `event:` line, the payload from `data:`. Comment lines +(`: keep-alive`, `: error - dropped N messages`) must be ignored. + +Per-topic `data:` JSON shapes the collector must handle: + +| topic (`event:` line) | `data:` JSON | slot is at | +|---|---|---| +| `block` | `{ "slot": 128, "block": "0x…" }` | `slot` | +| `attestation` | `{ "validator_id": 7, "data": { "slot": 12, "head": {…}, "target": {…}, "source": {…} } }` | `data.slot` | +| `aggregate` | `{ "participants": [0,1,2], "data": { "slot": 12, "head": {…}, "target": {…}, "source": {…} } }` | `data.slot` | +| `head` | `{ "slot": 128, "block": "0x…", "state": "0x…" }` | `slot` | +| `justified_checkpoint` | `{ "slot": 128, "block": "0x…", "state": "0x…" }` | `slot` | +| `finalized_checkpoint` | `{ "slot": 128, "block": "0x…", "state": "0x…" }` | `slot` | +| `safe_target` | `{ "slot": 128, "block": "0x…" }` | `slot` | +| `chain_reorg` | `{ "slot":…, "depth":…, "old_head_block":"0x…", "old_head_state":"0x…", "new_head_block":"0x…", "new_head_state":"0x…" }` | `slot` | +| `block_gossip` | `{ "slot": 128, "block": "0x…" }` | `slot` | + +A `Checkpoint` (`head`/`target`/`source`) is `{ "root": "0x…", "slot": N }`. +The collector must be resilient: an unknown topic or a payload it can't parse is +logged and skipped, never fatal. + +### Timing bootstrap (how the collector learns slot geometry) + +On startup, fetch once from the first reachable node: + +- `GET {node.url}/lean/v0/genesis` → `{ "genesis_time": 1770407233, "validator_count": 16 }` +- `GET {node.url}/lean/v0/config/spec` → `{ "MILLISECONDS_PER_SLOT": 4000, "INTERVALS_PER_SLOT": 5, … }` + +`genesis_time` is in **seconds**. Offset formula (all ms): + +``` +slot_start_ms = genesis_time * 1000 + slot * MILLISECONDS_PER_SLOT +offset_ms = arrival_ms - slot_start_ms // may be negative under clock skew +``` + +`arrival_ms` = collector wall-clock at the moment the frame is received +(`SystemTime::now()` → epoch ms). Config may override `genesis_time` / +`ms_per_slot` for offline testing. + +--- + +## 3. NormalizedEvent (collector → browser payload) + +Serialized as JSON. Field names are frozen: + +```json +{ + "node": "node-2", + "topic": "block", + "slot": 128, + "arrival_ms": 1770407745123, + "offset_ms": 742, + "id": "0xabc123…", + "validator_id": null, + "participants": null +} +``` + +| field | type | meaning | +|---|---|---| +| `node` | string | configured node name | +| `topic` | string | one of the topic names in §2 | +| `slot` | u64 | slot the event refers to (from `slot` or `data.slot`) | +| `arrival_ms` | i64 | collector receive time, epoch ms | +| `offset_ms` | i64 | `arrival_ms - slot_start_ms`; can be negative | +| `id` | string \| null | grouping/propagation identity: block/head/safe_target/gossip → the `block` root; reorg → `new_head_block`; checkpoints → `block`; **aggregate → a session-stable content hash** of `(data, sorted participants)`, hex `0x…`; **attestation → null** | +| `validator_id` | u64 \| null | set only for `attestation` | +| `participants` | u32 \| null | set only for `aggregate`: participant **count** (never the full list — keep frames light) | + +The aggregate `id` need only be stable **within one collector session** (used to +group the same aggregate seen across nodes); a simple FNV-1a / hash of the +canonical JSON of `{data, sorted participants}` rendered as `0x…` hex is fine. + +--- + +## 4. Collector HTTP API (browser → collector) + +Served by axum on the configured `listen` address. + +### `GET /` +Serves `web/index.html` (and `web/` assets under their paths, e.g. `/app.js`, +`/style.css`). Static file serving rooted at the configured `static_dir` +(default `web`). + +### `GET /stream` (SSE, `text/event-stream`) +The merged live stream. Two SSE **event names**: + +- `event: chain` — `data:` is one NormalizedEvent (§3). +- `event: status` — `data:` is a node status object: + ```json + { "node": "node-2", "state": "connected", "events_per_sec": 4.2 } + ``` + `state` ∈ `"connected" | "reconnecting" | "down"`. Emitted on every state + change and at least every few seconds as a heartbeat with a refreshed rate. + +Keep-alive comments are sent to hold idle connections open. Best-effort: a slow +browser may miss events (same contract as upstream). + +### `GET /api/meta` (JSON) +One-shot bootstrap the frontend fetches on load: + +```json +{ + "genesis_time": 1770407233, + "ms_per_slot": 4000, + "intervals_per_slot": 5, + "window_slots": 30, + "topics": ["block", "attestation", "aggregate"], + "nodes": [ + { "name": "node-2", "url": "http://127.0.0.1:5052" }, + { "name": "node-3", "url": "http://127.0.0.1:5053" } + ] +} +``` + +### `GET /api/history` (JSON) +Startup backfill so a freshly-opened dashboard isn't blank. Returns the +collector's bounded in-memory ring of recent events (retained for +`history_slots` slots, hard-capped) plus the latest status per node: + +```json +{ + "events": [ /* NormalizedEvent (§3), oldest first */ ], + "status": [ { "node": "node-2", "state": "connected", "events_per_sec": 4.2 } ] +} +``` + +Each `events` element is byte-identical in shape to a `/stream` `chain` +event. **Startup ordering:** the frontend opens `/stream` first (buffering +live events), then fetches `/api/history`, seeds history, and flushes the +buffer, de-duping the overlap by `(node, topic, slot, id, validator_id, +arrival_ms)`. The broadcast never replays to new subscribers, so this +guarantees no gap and no double-count. + +--- + +## 5. Config file (TOML) + +Path via `--config ` (default `config.toml`). See `config.example.toml`. + +```toml +listen = "127.0.0.1:8080" # collector bind address +window_slots = 30 # initial rolling window; adjustable live in the UI +history_slots = 64 # slots of events buffered for GET /api/history backfill +static_dir = "web" # dir served at GET / +topics = ["block", "attestation", "aggregate"] # upstream topics to subscribe + +# optional offline overrides; normally auto-fetched from the first node +# genesis_time = 1770407233 +# ms_per_slot = 4000 + +[[nodes]] +name = "node-2" +url = "http://127.0.0.1:5052" + +[[nodes]] +name = "node-3" +url = "http://127.0.0.1:5053" +``` + +--- + +## 6. Frontend visualization spec + +Single dark/light-adaptive page, no framework, no build step. Fetches +`/api/meta`, backfills from `/api/history`, and streams live from +`EventSource("/stream")` (startup ordering per §4). + +**Window control** (header): a numeric input, seeded from `meta.window_slots`, +that live-adjusts the rolling `window_slots` for **both** panels (clamped +`1..500`). Client-side only; no collector restart. + +**Top — rolling beeswarm** (canvas): x-axis `0 … ms_per_slot`, gridlines at every +`ms_per_slot / intervals_per_slot`. One horizontal lane per node. Each incoming +`chain` event for topic `block`/`attestation`/`aggregate` is a dot at +`x = clamp(offset_ms, 0, ms_per_slot)`, small vertical jitter within its lane, +colored by topic: block `#4f8cff`, attestation `#37b24d`, aggregate `#f59f00`. +Keep only events from the last `window_slots` slots; older dots fade then drop. +Cap points per node (e.g. 2000) with oldest-first decimation so an attestation +flood can't wedge rendering. Legend + a note that older slots fade. + +**Bottom — propagation delta** (canvas beeswarm): a topic toggle +(`block` default / `aggregate` / `head`). Group events of that topic by `id`; +for every id in the last `window_slots`, plot one dot per node in that node's +lane at `x = clamp(arrival_ms − min(arrival_ms over nodes for that id), 0, +ms_per_slot)`, jittered, faded by slot age. **Fixed 0…`ms_per_slot` x-axis** +(not rescaled to the data), so a dot's position is comparable across ids and +slots; a delta beyond one slot saturates at the right edge. Colors are kept off +the topic hues so the panels don't read as the same scale: the first node to +see an id (`delta == 0`) is `--prop-first` (violet), normal lag is +`--prop-normal` (teal), over-one-slot is `--prop-over` (magenta). Legend maps +the three colors; empty topics show a "Waiting for … events" note. + +**Status bar**: one chip per node showing `state` (green/amber/red) and +`events_per_sec`, driven by `status` events. + +Design language: match the calm, technical look of the approved mockup (thin +gridlines, muted labels, the three topic colors above). Must be readable in both +light and dark; degrade gracefully if a node is `down` (empty lane, red chip). diff --git a/tooling/event-monitor/Cargo.lock b/tooling/event-monitor/Cargo.lock new file mode 100644 index 00000000..28d942d4 --- /dev/null +++ b/tooling/event-monitor/Cargo.lock @@ -0,0 +1,2129 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "aws-lc-rs" +version = "1.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "clap" +version = "4.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f2392eae7f16557a3d727ef3a12e57b2b2ca6f98566a5f4fb41ffe305df077" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-monitor" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "clap", + "eventsource-stream", + "futures-util", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", + "toml", + "tower-http 0.7.0", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "eventsource-stream" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" +dependencies = [ + "futures-core", + "nom", + "pin-project-lite", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22053b6a34f84abc97f9129e61334f40174659a1b9bd18c970b83db6a9a6348b" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tooling/event-monitor/Cargo.toml b/tooling/event-monitor/Cargo.toml new file mode 100644 index 00000000..5c7862c1 --- /dev/null +++ b/tooling/event-monitor/Cargo.toml @@ -0,0 +1,32 @@ +[workspace] + +[package] +name = "event-monitor" +version = "0.1.0" +edition = "2024" +publish = false + +[[bin]] +name = "event-monitor" +path = "src/main.rs" + +[lib] +name = "event_monitor" +path = "src/lib.rs" + +[dependencies] +anyhow = "1.0.104" +axum = "0.8.9" +clap = { version = "4.6.3", features = ["derive"] } +eventsource-stream = "0.2.3" +futures-util = "0.3.33" +reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "stream", "json"] } +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" +thiserror = "2.0.19" +tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "signal", "time", "net", "sync", "fs"] } +tokio-stream = { version = "0.1.18", features = ["sync"] } +toml = "1.1.3" +tower-http = { version = "0.7.0", features = ["fs"] } +tracing = "0.1.44" +tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } diff --git a/tooling/event-monitor/README.md b/tooling/event-monitor/README.md new file mode 100644 index 00000000..353d6c0c --- /dev/null +++ b/tooling/event-monitor/README.md @@ -0,0 +1,33 @@ +# event-monitor + +Live arrival-time monitor for lean-consensus (ethlambda) nodes. + +It dials the `GET /lean/v0/events` SSE stream of several nodes, timestamps each +event on arrival, and serves a browser dashboard that visualizes when +`block` / `attestation` / `aggregate` events arrive **relative to the slot** +(rolling beeswarm, per node) and how a given block/aggregate/head **propagates** +between nodes (a second beeswarm of per-node delay behind the first node to see +each id). The rolling window is adjustable live from the header, and a fresh +page load backfills recent history from the collector so it's never blank. + +Standalone: its own Cargo workspace, **no dependency on any ethlambda crate** — +it only speaks the documented SSE/HTTP wire shape. See [`CONTRACT.md`](./CONTRACT.md) +for the authoritative interface between the Rust backend and the JS frontend. + +## Run + +```bash +cp config.example.toml config.toml +$EDITOR config.toml # list your nodes' RPC URLs +cargo run --release -- --config config.toml +# open the `listen` address (default http://127.0.0.1:8080) in a browser +``` + +## Layout + +``` +src/ Rust collector + axum server (owns Cargo.toml) +web/ vanilla HTML/JS/CSS dashboard (no build step) +CONTRACT.md frozen interface: SSE input, NormalizedEvent, HTTP API, viz spec +config.example.toml +``` diff --git a/tooling/event-monitor/config.example.toml b/tooling/event-monitor/config.example.toml new file mode 100644 index 00000000..491be2db --- /dev/null +++ b/tooling/event-monitor/config.example.toml @@ -0,0 +1,35 @@ +# event-monitor configuration. Copy to config.toml and edit. +# +# Run: event-monitor --config config.toml +# Then open the `listen` address in a browser. + +listen = "127.0.0.1:8080" # collector bind address (the dashboard URL) +window_slots = 30 # initial rolling window; adjustable live in the UI +history_slots = 64 # slots of events buffered to backfill a fresh page load +static_dir = "web" # directory served at GET / (the frontend) +# Upstream SSE topics to subscribe. `head` powers the propagation panel's head +# toggle; drop it if you only care about block/aggregate propagation. +topics = ["block", "attestation", "aggregate", "head"] + +# Optional offline overrides. Normally the collector auto-fetches slot geometry +# from the first reachable node via /lean/v0/genesis and /lean/v0/config/spec. +# genesis_time = 1770407233 # seconds +# ms_per_slot = 4000 + +# One block per node. `url` is the node's RPC API base (no trailing path); +# the collector appends /lean/v0/events, /lean/v0/genesis, /lean/v0/config/spec. +[[nodes]] +name = "node-2" +url = "http://127.0.0.1:5052" + +[[nodes]] +name = "node-3" +url = "http://127.0.0.1:5053" + +[[nodes]] +name = "node-4" +url = "http://127.0.0.1:5054" + +[[nodes]] +name = "node-5" +url = "http://127.0.0.1:5055" diff --git a/tooling/event-monitor/src/collector.rs b/tooling/event-monitor/src/collector.rs new file mode 100644 index 00000000..612b3498 --- /dev/null +++ b/tooling/event-monitor/src/collector.rs @@ -0,0 +1,249 @@ +//! Per-node SSE collector: dials `GET {node.url}/lean/v0/events`, stamps +//! arrival time, normalizes, and republishes on the [`Hub`]. Reconnects with +//! capped exponential backoff and reports connection state changes plus a +//! periodic heartbeat (CONTRACT.md §2, §4). + +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use eventsource_stream::{Event as SseEvent, Eventsource}; +use futures_util::StreamExt; + +use crate::config::NodeConfig; +use crate::hub::Hub; +use crate::model::{self, NodeState, NodeStatus}; +use crate::timing::Timing; + +const INITIAL_BACKOFF: Duration = Duration::from_millis(250); +const MAX_BACKOFF: Duration = Duration::from_secs(10); +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Debug, thiserror::Error)] +enum CollectorError { + #[error("http error: {0}")] + Http(#[from] reqwest::Error), + #[error("event stream error: {0}")] + Stream(String), +} + +/// Exponential backoff capped at [`MAX_BACKOFF`]. Used both to pace +/// reconnect attempts and to decide whether the collector should report +/// `reconnecting` (still ramping up retries) or `down` (settled into +/// sustained failure at the cap). +struct Backoff { + delay: Duration, +} + +impl Backoff { + fn new() -> Self { + Self { + delay: INITIAL_BACKOFF, + } + } + + fn reset(&mut self) { + self.delay = INITIAL_BACKOFF; + } + + /// Returns the delay to wait before the next attempt, then doubles + /// (capped) for next time. + fn advance(&mut self) -> Duration { + let current = self.delay; + self.delay = (self.delay * 2).min(MAX_BACKOFF); + current + } +} + +/// Tracks a rolling events-per-second rate over the time since the last +/// reset, driven by the collector's heartbeat. +struct RateTracker { + count: u64, + window_start: Instant, +} + +impl RateTracker { + fn new() -> Self { + Self { + count: 0, + window_start: Instant::now(), + } + } + + fn tick(&mut self) { + self.count += 1; + } + + /// Events/sec since the last call, then resets the window. + fn rate_and_reset(&mut self) -> f64 { + let elapsed = self.window_start.elapsed().as_secs_f64().max(0.001); + let rate = self.count as f64 / elapsed; + self.count = 0; + self.window_start = Instant::now(); + rate + } +} + +fn now_ms() -> i64 { + let duration = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) +} + +/// Runs forever: connects, streams, and on any disconnect/error reconnects +/// with capped exponential backoff. Intended to be spawned as one long-lived +/// task per configured node. +pub async fn run_collector( + node: NodeConfig, + topics: Vec, + timing: Arc, + hub: Hub, + client: reqwest::Client, +) { + let mut backoff = Backoff::new(); + loop { + hub.publish_status(NodeStatus { + node: node.name.clone(), + state: NodeState::Reconnecting, + events_per_sec: 0.0, + }); + + match connect_and_stream(&node, &topics, &timing, &hub, &client).await { + Ok(()) => { + tracing::info!(node = %node.name, "SSE stream ended; reconnecting"); + backoff.reset(); + } + Err(err) => { + tracing::warn!(node = %node.name, %err, "SSE connection failed; will retry"); + } + } + + let delay = backoff.advance(); + let state = if delay >= MAX_BACKOFF { + NodeState::Down + } else { + NodeState::Reconnecting + }; + hub.publish_status(NodeStatus { + node: node.name.clone(), + state, + events_per_sec: 0.0, + }); + tokio::time::sleep(delay).await; + } +} + +/// Opens one SSE connection and streams frames until the connection ends or +/// errors. Returns `Ok(())` on a clean end-of-stream (server closed it), +/// `Err` on a transport/parse failure. +async fn connect_and_stream( + node: &NodeConfig, + topics: &[String], + timing: &Timing, + hub: &Hub, + client: &reqwest::Client, +) -> Result<(), CollectorError> { + let url = format!( + "{}/lean/v0/events?topics={}", + node.url.trim_end_matches('/'), + topics.join(",") + ); + + let response = client.get(&url).send().await?.error_for_status()?; + let mut stream = response.bytes_stream().eventsource(); + + hub.publish_status(NodeStatus { + node: node.name.clone(), + state: NodeState::Connected, + events_per_sec: 0.0, + }); + tracing::info!(node = %node.name, %url, "connected to SSE stream"); + + let mut rate = RateTracker::new(); + let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); + heartbeat.tick().await; // the first tick fires immediately; consume it + + loop { + tokio::select! { + frame = stream.next() => { + match frame { + Some(Ok(event)) => { + rate.tick(); + handle_frame(node, &event, timing, hub); + } + Some(Err(err)) => return Err(CollectorError::Stream(err.to_string())), + None => return Ok(()), + } + } + _ = heartbeat.tick() => { + hub.publish_status(NodeStatus { + node: node.name.clone(), + state: NodeState::Connected, + events_per_sec: rate.rate_and_reset(), + }); + } + } + } +} + +/// Normalizes one already-parsed SSE frame and publishes it on the hub. +/// Never panics: an unknown topic or payload we can't parse is logged and +/// dropped (CONTRACT.md §2). +fn handle_frame(node: &NodeConfig, event: &SseEvent, timing: &Timing, hub: &Hub) { + // Defensive: eventsource-stream already suppresses comment/keep-alive + // lines (they never build a non-empty data buffer), but guard anyway. + if event.data.is_empty() { + return; + } + match model::normalize(&node.name, &event.event, &event.data, now_ms(), timing) { + Ok(normalized) => hub.publish_chain(normalized), + Err(err) => { + tracing::debug!( + node = %node.name, + topic = %event.event, + %err, + "skipping unparsable SSE frame" + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backoff_doubles_until_capped() { + let mut backoff = Backoff::new(); + assert_eq!(backoff.advance(), Duration::from_millis(250)); + assert_eq!(backoff.advance(), Duration::from_millis(500)); + assert_eq!(backoff.advance(), Duration::from_millis(1_000)); + assert_eq!(backoff.advance(), Duration::from_millis(2_000)); + assert_eq!(backoff.advance(), Duration::from_millis(4_000)); + assert_eq!(backoff.advance(), Duration::from_millis(8_000)); + // 8s * 2 = 16s, capped to 10s. + assert_eq!(backoff.advance(), MAX_BACKOFF); + assert_eq!(backoff.advance(), MAX_BACKOFF); + } + + #[test] + fn backoff_reset_returns_to_initial_delay() { + let mut backoff = Backoff::new(); + backoff.advance(); + backoff.advance(); + backoff.reset(); + assert_eq!(backoff.advance(), INITIAL_BACKOFF); + } + + #[test] + fn rate_tracker_counts_ticks_since_last_reset() { + let mut rate = RateTracker::new(); + rate.tick(); + rate.tick(); + // Elapsed time is tiny but non-zero (clamped to a 1ms floor), so the + // computed rate is finite and positive rather than NaN/infinite. + let observed = rate.rate_and_reset(); + assert!(observed.is_finite()); + assert!(observed > 0.0); + } +} diff --git a/tooling/event-monitor/src/config.rs b/tooling/event-monitor/src/config.rs new file mode 100644 index 00000000..f87a6b1e --- /dev/null +++ b/tooling/event-monitor/src/config.rs @@ -0,0 +1,153 @@ +//! TOML configuration shape (CONTRACT.md §5). + +use std::net::SocketAddr; +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::timing::TimingOverrides; + +#[derive(Debug, Clone, Deserialize)] +pub struct Config { + /// Collector bind address (the dashboard URL). + pub listen: SocketAddr, + /// Rolling window of slots the frontend keeps (initial value; adjustable + /// live from the dashboard). + #[serde(default = "default_window_slots")] + pub window_slots: u32, + /// How many slots of recent events the collector buffers in memory to + /// backfill a freshly-opened dashboard via `GET /api/history`. + #[serde(default = "default_history_slots")] + pub history_slots: u32, + /// Directory served at `GET /`. + #[serde(default = "default_static_dir")] + pub static_dir: String, + /// Upstream SSE topics to subscribe to. + #[serde(default = "default_topics")] + pub topics: Vec, + /// Optional offline override for slot-0 wall-clock time (seconds). + pub genesis_time: Option, + /// Optional offline override for slot duration (milliseconds). + pub ms_per_slot: Option, + /// Nodes to dial for events. + pub nodes: Vec, +} + +/// Also `Serialize` so it can be embedded directly in the `/api/meta` +/// response's `nodes` array (CONTRACT.md §4), which mirrors this shape +/// exactly: `{ "name": ..., "url": ... }`. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct NodeConfig { + pub name: String, + pub url: String, +} + +fn default_window_slots() -> u32 { + 30 +} + +fn default_history_slots() -> u32 { + 64 +} + +fn default_static_dir() -> String { + "web".to_string() +} + +fn default_topics() -> Vec { + vec![ + "block".to_string(), + "attestation".to_string(), + "aggregate".to_string(), + ] +} + +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("failed to read config file {path}: {source}")] + Read { + path: String, + #[source] + source: std::io::Error, + }, + #[error("failed to parse config file {path}: {source}")] + Parse { + path: String, + #[source] + source: Box, + }, +} + +impl Config { + pub fn load(path: &Path) -> Result { + let raw = std::fs::read_to_string(path).map_err(|source| ConfigError::Read { + path: path.display().to_string(), + source, + })?; + toml::from_str(&raw).map_err(|source| ConfigError::Parse { + path: path.display().to_string(), + source: Box::new(source), + }) + } + + pub fn timing_overrides(&self) -> TimingOverrides { + TimingOverrides { + genesis_time: self.genesis_time, + ms_per_slot: self.ms_per_slot, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_apply_when_omitted() { + let toml_str = r#" + listen = "127.0.0.1:8080" + + [[nodes]] + name = "node-2" + url = "http://127.0.0.1:5052" + "#; + let cfg: Config = toml::from_str(toml_str).unwrap(); + assert_eq!(cfg.window_slots, 30); + assert_eq!(cfg.history_slots, 64); + assert_eq!(cfg.static_dir, "web"); + assert_eq!(cfg.topics, vec!["block", "attestation", "aggregate"]); + assert_eq!(cfg.nodes.len(), 1); + assert!(cfg.genesis_time.is_none()); + assert!(cfg.ms_per_slot.is_none()); + } + + #[test] + fn overrides_and_multiple_nodes_parse() { + let toml_str = r#" + listen = "127.0.0.1:8080" + window_slots = 10 + history_slots = 128 + static_dir = "public" + topics = ["block"] + genesis_time = 1770407233 + ms_per_slot = 4000 + + [[nodes]] + name = "node-2" + url = "http://127.0.0.1:5052" + + [[nodes]] + name = "node-3" + url = "http://127.0.0.1:5053" + "#; + let cfg: Config = toml::from_str(toml_str).unwrap(); + assert_eq!(cfg.window_slots, 10); + assert_eq!(cfg.history_slots, 128); + assert_eq!(cfg.static_dir, "public"); + assert_eq!(cfg.topics, vec!["block"]); + assert_eq!(cfg.genesis_time, Some(1770407233)); + assert_eq!(cfg.ms_per_slot, Some(4000)); + assert_eq!(cfg.nodes.len(), 2); + assert_eq!(cfg.nodes[1].name, "node-3"); + } +} diff --git a/tooling/event-monitor/src/hub.rs b/tooling/event-monitor/src/hub.rs new file mode 100644 index 00000000..4a95b1ae --- /dev/null +++ b/tooling/event-monitor/src/hub.rs @@ -0,0 +1,231 @@ +//! The merged event bus: collector tasks publish, `GET /stream` subscribes, +//! and a bounded in-memory history lets `GET /api/history` backfill a +//! freshly-opened dashboard so it isn't blank on load (CONTRACT.md §4). + +use std::collections::{BTreeMap, VecDeque}; +use std::sync::{Arc, Mutex}; + +use tokio::sync::broadcast; + +use crate::model::{NodeStatus, NormalizedEvent}; + +/// Capacity of the broadcast channel. A slow subscriber (browser) that falls +/// behind by more than this many messages will observe a `Lagged` error on +/// its receiver and skip ahead — best-effort, same contract as the upstream +/// ethlambda SSE endpoint (CONTRACT.md §4). +const HUB_CAPACITY: usize = 4096; + +/// Hard upper bound on retained history events, independent of the slot-based +/// window, so a high-rate stream (attestation flood) can't grow memory +/// without bound before the slot-age prune catches up. +const HISTORY_MAX_EVENTS: usize = 50_000; + +/// One message on the hub: either a normalized chain event (`event: chain`) +/// or a node status update (`event: status`) per CONTRACT.md §4. +#[derive(Debug, Clone)] +pub enum HubMessage { + Chain(NormalizedEvent), + Status(NodeStatus), +} + +/// Point-in-time backfill payload served by `GET /api/history`: the retained +/// recent chain events plus the latest status per node (CONTRACT.md §4). +#[derive(Debug, Clone, Default)] +pub struct HistorySnapshot { + pub events: Vec, + pub status: Vec, +} + +/// Bounded ring of recent events: retained by slot age up to `retain_slots` +/// (relative to the newest slot seen) and hard-capped at +/// [`HISTORY_MAX_EVENTS`], plus the latest status per node. +struct History { + events: VecDeque, + status: BTreeMap, + max_slot: u64, + retain_slots: u64, +} + +impl History { + fn new(retain_slots: u64) -> Self { + Self { + events: VecDeque::new(), + status: BTreeMap::new(), + max_slot: 0, + retain_slots, + } + } + + fn push_event(&mut self, event: NormalizedEvent) { + self.max_slot = self.max_slot.max(event.slot); + self.events.push_back(event); + self.prune(); + } + + fn record_status(&mut self, status: NodeStatus) { + self.status.insert(status.node.clone(), status); + } + + /// Drops events older than `retain_slots` relative to the newest slot + /// seen, then enforces the hard event cap from the front (oldest first). + /// Events arrive in roughly slot order across nodes/topics, so scanning + /// from the front is a good approximation of oldest-first. + fn prune(&mut self) { + while let Some(front) = self.events.front() { + if self.max_slot.saturating_sub(front.slot) >= self.retain_slots { + self.events.pop_front(); + } else { + break; + } + } + while self.events.len() > HISTORY_MAX_EVENTS { + self.events.pop_front(); + } + } + + fn snapshot(&self) -> HistorySnapshot { + HistorySnapshot { + events: self.events.iter().cloned().collect(), + status: self.status.values().cloned().collect(), + } + } +} + +/// Cheaply cloneable handle to the shared broadcast bus and history ring. +/// Every collector task and every `/stream` subscriber holds a clone. +#[derive(Clone)] +pub struct Hub { + tx: broadcast::Sender, + history: Arc>, +} + +impl Hub { + /// `history_slots` is how many slots of recent events are retained for + /// backfill; clamped to at least 1. + pub fn new(history_slots: u64) -> Self { + let (tx, _rx) = broadcast::channel(HUB_CAPACITY); + Self { + tx, + history: Arc::new(Mutex::new(History::new(history_slots.max(1)))), + } + } + + /// Publishes a normalized chain event. Records it into history *before* + /// broadcasting so a snapshot taken concurrently with a `/stream` + /// subscribe can never miss an event a live subscriber will also see (the + /// frontend de-dups the small overlap). Ignores the "no subscribers" + /// send error: normal when no browser is connected yet. + pub fn publish_chain(&self, event: NormalizedEvent) { + if let Ok(mut history) = self.history.lock() { + history.push_event(event.clone()); + } + let _ = self.tx.send(HubMessage::Chain(event)); + } + + /// Publishes a node status update. See [`Hub::publish_chain`] for why + /// send errors are ignored. + pub fn publish_status(&self, status: NodeStatus) { + if let Ok(mut history) = self.history.lock() { + history.record_status(status.clone()); + } + let _ = self.tx.send(HubMessage::Status(status)); + } + + pub fn subscribe(&self) -> broadcast::Receiver { + self.tx.subscribe() + } + + /// Snapshot of retained history for `GET /api/history`. Returns an empty + /// snapshot rather than propagating a (practically impossible) poisoned + /// lock — the critical sections never panic. + pub fn history_snapshot(&self) -> HistorySnapshot { + self.history + .lock() + .map(|history| history.snapshot()) + .unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::NodeState; + + fn chain_event(node: &str, slot: u64) -> NormalizedEvent { + NormalizedEvent { + node: node.to_string(), + topic: "block".to_string(), + slot, + arrival_ms: slot as i64 * 4_000, + offset_ms: 0, + id: Some(format!("0x{slot:064x}")), + validator_id: None, + participants: None, + } + } + + #[tokio::test] + async fn subscriber_receives_published_messages() { + let hub = Hub::new(64); + let mut rx = hub.subscribe(); + + hub.publish_status(NodeStatus { + node: "node-2".to_string(), + state: NodeState::Connected, + events_per_sec: 1.5, + }); + + let msg = rx.recv().await.unwrap(); + match msg { + HubMessage::Status(status) => assert_eq!(status.node, "node-2"), + HubMessage::Chain(_) => panic!("expected a Status message"), + } + } + + #[test] + fn publish_without_subscribers_does_not_panic() { + let hub = Hub::new(64); + hub.publish_status(NodeStatus { + node: "node-2".to_string(), + state: NodeState::Down, + events_per_sec: 0.0, + }); + } + + #[test] + fn history_snapshot_returns_published_events_and_latest_status() { + let hub = Hub::new(64); + hub.publish_chain(chain_event("node-0", 10)); + hub.publish_chain(chain_event("node-1", 11)); + hub.publish_status(NodeStatus { + node: "node-0".to_string(), + state: NodeState::Reconnecting, + events_per_sec: 0.0, + }); + hub.publish_status(NodeStatus { + node: "node-0".to_string(), + state: NodeState::Connected, + events_per_sec: 2.0, + }); + + let snap = hub.history_snapshot(); + assert_eq!(snap.events.len(), 2); + assert_eq!(snap.events[0].slot, 10); + // Only the latest status per node is retained. + assert_eq!(snap.status.len(), 1); + assert_eq!(snap.status[0].state, NodeState::Connected); + } + + #[test] + fn history_prunes_events_older_than_the_retain_window() { + let hub = Hub::new(5); // retain 5 slots + for slot in 0..10 { + hub.publish_chain(chain_event("node-0", slot)); + } + let snap = hub.history_snapshot(); + // max_slot = 9, retain 5 → keep slots with age < 5 (slots 5..=9). + assert_eq!(snap.events.len(), 5); + assert_eq!(snap.events.first().unwrap().slot, 5); + assert_eq!(snap.events.last().unwrap().slot, 9); + } +} diff --git a/tooling/event-monitor/src/lib.rs b/tooling/event-monitor/src/lib.rs new file mode 100644 index 00000000..3920c152 --- /dev/null +++ b/tooling/event-monitor/src/lib.rs @@ -0,0 +1,10 @@ +//! event-monitor library: SSE collector + normalizer + axum dashboard server +//! for lean-consensus (ethlambda) nodes. See `CONTRACT.md` for the frozen +//! wire interface shared with the `web/` frontend. + +pub mod collector; +pub mod config; +pub mod hub; +pub mod model; +pub mod server; +pub mod timing; diff --git a/tooling/event-monitor/src/main.rs b/tooling/event-monitor/src/main.rs new file mode 100644 index 00000000..4258925c --- /dev/null +++ b/tooling/event-monitor/src/main.rs @@ -0,0 +1,69 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use clap::Parser; +use tokio::net::TcpListener; + +use event_monitor::config::Config; +use event_monitor::hub::Hub; +use event_monitor::{collector, server, timing}; + +/// Live arrival-time monitor for lean-consensus (ethlambda) nodes. +#[derive(Parser, Debug)] +#[command(name = "event-monitor")] +struct Args { + /// Path to the TOML config file. + #[arg(long, default_value = "config.toml")] + config: PathBuf, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + let args = Args::parse(); + + let config_path = args.config.clone(); + let config = Config::load(&args.config).inspect_err(|err| { + tracing::error!(%err, config = %config_path.display(), "failed to load config"); + })?; + + let client = reqwest::Client::new(); + let timing = timing::bootstrap(&config.nodes, config.timing_overrides(), &client) + .await + .inspect_err(|err| tracing::error!(%err, "failed to bootstrap slot timing"))?; + tracing::info!( + genesis_time = timing.genesis_time, + ms_per_slot = timing.ms_per_slot, + intervals_per_slot = timing.intervals_per_slot, + "resolved slot geometry" + ); + let timing = Arc::new(timing); + + let hub = Hub::new(config.history_slots as u64); + for node in &config.nodes { + tokio::spawn(collector::run_collector( + node.clone(), + config.topics.clone(), + timing.clone(), + hub.clone(), + client.clone(), + )); + } + + let meta = server::Meta::new(&config, &timing); + let static_dir = Path::new(&config.static_dir).to_path_buf(); + let app = server::build_router(hub, meta, &static_dir); + + let listen_addr = config.listen; + let listener = TcpListener::bind(listen_addr).await?; + tracing::info!(%listen_addr, "event-monitor dashboard ready; open this address in a browser"); + axum::serve(listener, app).await?; + + Ok(()) +} diff --git a/tooling/event-monitor/src/model.rs b/tooling/event-monitor/src/model.rs new file mode 100644 index 00000000..25ab7087 --- /dev/null +++ b/tooling/event-monitor/src/model.rs @@ -0,0 +1,401 @@ +//! Wire shapes for upstream ethlambda SSE payloads (CONTRACT.md §2) and the +//! `NormalizedEvent` re-served to the browser (CONTRACT.md §3). + +use serde::{Deserialize, Serialize}; + +use crate::timing::Timing; + +/// A `Checkpoint` (`head`/`target`/`source`): `{ "root": "0x...", "slot": N }`. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Checkpoint { + pub root: String, + pub slot: u64, +} + +/// Shared attestation-vote payload embedded in both `attestation` and +/// `aggregate` topics. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AttestationData { + pub slot: u64, + pub head: Checkpoint, + pub target: Checkpoint, + pub source: Checkpoint, +} + +/// `block`, `safe_target`, `block_gossip`: `{ "slot": N, "block": "0x..." }`. +#[derive(Debug, Clone, Deserialize)] +struct SlotBlockPayload { + slot: u64, + block: String, +} + +/// `head`, `justified_checkpoint`, `finalized_checkpoint`: +/// `{ "slot": N, "block": "0x...", "state": "0x..." }`. +#[derive(Debug, Clone, Deserialize)] +struct CheckpointEventPayload { + slot: u64, + block: String, + #[allow(dead_code)] // part of the wire shape; not surfaced on NormalizedEvent + state: String, +} + +/// `attestation`: `{ "validator_id": N, "data": {...} }`. +#[derive(Debug, Clone, Deserialize)] +struct AttestationPayload { + validator_id: u64, + data: AttestationData, +} + +/// `aggregate`: `{ "participants": [...], "data": {...} }`. +#[derive(Debug, Clone, Deserialize)] +struct AggregatePayload { + participants: Vec, + data: AttestationData, +} + +/// `chain_reorg`: +/// `{ "slot":N, "depth":N, "old_head_block":"0x...", "old_head_state":"0x...", +/// "new_head_block":"0x...", "new_head_state":"0x..." }`. +#[derive(Debug, Clone, Deserialize)] +struct ReorgPayload { + slot: u64, + #[allow(dead_code)] + depth: u64, + #[allow(dead_code)] + old_head_block: String, + #[allow(dead_code)] + old_head_state: String, + new_head_block: String, + #[allow(dead_code)] + new_head_state: String, +} + +/// Collector -> browser payload (CONTRACT.md §3). Field names and shape are +/// frozen; do not rename without updating CONTRACT.md and `web/`. +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct NormalizedEvent { + pub node: String, + pub topic: String, + pub slot: u64, + pub arrival_ms: i64, + pub offset_ms: i64, + pub id: Option, + pub validator_id: Option, + pub participants: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum NormalizeError { + #[error("unknown topic: {0}")] + UnknownTopic(String), + #[error("failed to parse payload for topic {topic}: {source}")] + Json { + topic: String, + #[source] + source: serde_json::Error, + }, +} + +/// Canonical struct hashed to derive the aggregate `id`: `{data, participants}` +/// with `participants` sorted ascending, serialized deterministically via our +/// own field order (never via an arbitrary `serde_json::Value`). +#[derive(Serialize)] +struct AggregateIdInput<'a> { + data: &'a AttestationData, + participants: Vec, +} + +/// Session-stable FNV-1a hash of the canonical JSON of `{data, sorted +/// participants}`, rendered as `0x` + 16 lowercase hex digits. Only needs to +/// be stable within one collector process (CONTRACT.md §3). +fn aggregate_id(data: &AttestationData, participants: &[u64]) -> String { + let mut sorted = participants.to_vec(); + sorted.sort_unstable(); + let input = AggregateIdInput { + data, + participants: sorted, + }; + // Infallible: AggregateIdInput contains only plain data, no maps/floats. + let canonical = serde_json::to_string(&input).expect("aggregate id input is always valid JSON"); + format!("0x{:016x}", fnv1a_64(canonical.as_bytes())) +} + +fn fnv1a_64(data: &[u8]) -> u64 { + const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; + const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + let mut hash = FNV_OFFSET_BASIS; + for byte in data { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(FNV_PRIME); + } + hash +} + +/// Maps one upstream SSE frame (`event:` topic + `data:` JSON) into a +/// [`NormalizedEvent`]. Never panics: an unknown topic or unparsable payload +/// yields `Err`, which callers log and skip (CONTRACT.md §2). +pub fn normalize( + node: &str, + topic: &str, + data: &str, + arrival_ms: i64, + timing: &Timing, +) -> Result { + let to_json_err = |source: serde_json::Error| NormalizeError::Json { + topic: topic.to_string(), + source, + }; + + let (slot, id, validator_id, participants) = match topic { + "block" | "safe_target" | "block_gossip" => { + let payload: SlotBlockPayload = serde_json::from_str(data).map_err(to_json_err)?; + (payload.slot, Some(payload.block), None, None) + } + "head" | "justified_checkpoint" | "finalized_checkpoint" => { + let payload: CheckpointEventPayload = + serde_json::from_str(data).map_err(to_json_err)?; + (payload.slot, Some(payload.block), None, None) + } + "chain_reorg" => { + let payload: ReorgPayload = serde_json::from_str(data).map_err(to_json_err)?; + (payload.slot, Some(payload.new_head_block), None, None) + } + "attestation" => { + let payload: AttestationPayload = serde_json::from_str(data).map_err(to_json_err)?; + (payload.data.slot, None, Some(payload.validator_id), None) + } + "aggregate" => { + let payload: AggregatePayload = serde_json::from_str(data).map_err(to_json_err)?; + let id = aggregate_id(&payload.data, &payload.participants); + let count = u32::try_from(payload.participants.len()).unwrap_or(u32::MAX); + (payload.data.slot, Some(id), None, Some(count)) + } + other => return Err(NormalizeError::UnknownTopic(other.to_string())), + }; + + Ok(NormalizedEvent { + node: node.to_string(), + topic: topic.to_string(), + slot, + arrival_ms, + offset_ms: timing.offset_ms(slot, arrival_ms), + id, + validator_id, + participants, + }) +} + +/// Live status of one node's collector connection (CONTRACT.md §4). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum NodeState { + Connected, + Reconnecting, + Down, +} + +#[derive(Debug, Clone, Serialize)] +pub struct NodeStatus { + pub node: String, + pub state: NodeState, + pub events_per_sec: f64, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn timing() -> Timing { + Timing { + genesis_time: 0, + ms_per_slot: 4_000, + intervals_per_slot: 5, + } + } + + #[test] + fn block_topic_maps_id_to_block_root() { + let data = r#"{ "slot": 128, "block": "0xabc123" }"#; + let ev = normalize("node-2", "block", data, 1_000, &timing()).unwrap(); + assert_eq!(ev.topic, "block"); + assert_eq!(ev.slot, 128); + assert_eq!(ev.id, Some("0xabc123".to_string())); + assert_eq!(ev.validator_id, None); + assert_eq!(ev.participants, None); + assert_eq!(ev.offset_ms, 1_000 - 128 * 4_000); + } + + #[test] + fn safe_target_topic_maps_id_to_block_root() { + let data = r#"{ "slot": 127, "block": "0xdeadbeef" }"#; + let ev = normalize("node-2", "safe_target", data, 500, &timing()).unwrap(); + assert_eq!(ev.topic, "safe_target"); + assert_eq!(ev.slot, 127); + assert_eq!(ev.id, Some("0xdeadbeef".to_string())); + } + + #[test] + fn block_gossip_topic_maps_id_to_block_root() { + let data = r#"{ "slot": 128, "block": "0xabc123" }"#; + let ev = normalize("node-2", "block_gossip", data, 1_000, &timing()).unwrap(); + assert_eq!(ev.topic, "block_gossip"); + assert_eq!(ev.id, Some("0xabc123".to_string())); + } + + #[test] + fn head_topic_maps_id_to_block_root_ignoring_state() { + let data = r#"{ "slot": 128, "block": "0x1a2b", "state": "0x3c4d" }"#; + let ev = normalize("node-2", "head", data, 2_000, &timing()).unwrap(); + assert_eq!(ev.topic, "head"); + assert_eq!(ev.slot, 128); + assert_eq!(ev.id, Some("0x1a2b".to_string())); + } + + #[test] + fn justified_checkpoint_maps_id_to_block_root() { + let data = r#"{ "slot": 120, "block": "0xaaaa", "state": "0xbbbb" }"#; + let ev = normalize("node-2", "justified_checkpoint", data, 0, &timing()).unwrap(); + assert_eq!(ev.id, Some("0xaaaa".to_string())); + } + + #[test] + fn finalized_checkpoint_maps_id_to_block_root() { + let data = r#"{ "slot": 96, "block": "0xcccc", "state": "0xdddd" }"#; + let ev = normalize("node-2", "finalized_checkpoint", data, 0, &timing()).unwrap(); + assert_eq!(ev.id, Some("0xcccc".to_string())); + } + + #[test] + fn chain_reorg_maps_id_to_new_head_block() { + let data = r#"{ + "slot": 128, "depth": 2, + "old_head_block": "0xold1", "old_head_state": "0xold2", + "new_head_block": "0xnew1", "new_head_state": "0xnew2" + }"#; + let ev = normalize("node-2", "chain_reorg", data, 0, &timing()).unwrap(); + assert_eq!(ev.slot, 128); + assert_eq!(ev.id, Some("0xnew1".to_string())); + } + + #[test] + fn attestation_topic_has_null_id_and_validator_id_set() { + let data = r#"{ + "validator_id": 7, + "data": { + "slot": 12, + "head": {"root": "0xh", "slot": 12}, + "target": {"root": "0xt", "slot": 8}, + "source": {"root": "0xs", "slot": 4} + } + }"#; + let ev = normalize("node-2", "attestation", data, 0, &timing()).unwrap(); + assert_eq!(ev.topic, "attestation"); + assert_eq!(ev.slot, 12); + assert_eq!(ev.id, None); + assert_eq!(ev.validator_id, Some(7)); + assert_eq!(ev.participants, None); + } + + #[test] + fn aggregate_topic_sets_participant_count_and_hash_id() { + let data = r#"{ + "participants": [0, 1, 2], + "data": { + "slot": 12, + "head": {"root": "0xh", "slot": 12}, + "target": {"root": "0xt", "slot": 8}, + "source": {"root": "0xs", "slot": 4} + } + }"#; + let ev = normalize("node-2", "aggregate", data, 0, &timing()).unwrap(); + assert_eq!(ev.topic, "aggregate"); + assert_eq!(ev.slot, 12); + assert_eq!(ev.validator_id, None); + assert_eq!(ev.participants, Some(3)); + let id = ev.id.expect("aggregate must set an id"); + assert!(id.starts_with("0x")); + assert_eq!(id.len(), 2 + 16); + } + + #[test] + fn aggregate_id_is_stable_regardless_of_participant_order() { + let data_a = r#"{ + "participants": [0, 1, 2], + "data": { + "slot": 12, + "head": {"root": "0xh", "slot": 12}, + "target": {"root": "0xt", "slot": 8}, + "source": {"root": "0xs", "slot": 4} + } + }"#; + let data_b = r#"{ + "participants": [2, 0, 1], + "data": { + "slot": 12, + "head": {"root": "0xh", "slot": 12}, + "target": {"root": "0xt", "slot": 8}, + "source": {"root": "0xs", "slot": 4} + } + }"#; + let ev_a = normalize("node-2", "aggregate", data_a, 0, &timing()).unwrap(); + let ev_b = normalize("node-3", "aggregate", data_b, 999, &timing()).unwrap(); + assert_eq!(ev_a.id, ev_b.id); + } + + #[test] + fn aggregate_id_differs_for_different_participants() { + let base = |participants: &str| { + format!( + r#"{{ + "participants": {participants}, + "data": {{ + "slot": 12, + "head": {{"root": "0xh", "slot": 12}}, + "target": {{"root": "0xt", "slot": 8}}, + "source": {{"root": "0xs", "slot": 4}} + }} + }}"# + ) + }; + let ev_a = normalize("node-2", "aggregate", &base("[0,1,2]"), 0, &timing()).unwrap(); + let ev_b = normalize("node-2", "aggregate", &base("[0,1,3]"), 0, &timing()).unwrap(); + assert_ne!(ev_a.id, ev_b.id); + } + + #[test] + fn aggregate_id_differs_for_different_data() { + let data_a = r#"{ + "participants": [0, 1, 2], + "data": { + "slot": 12, + "head": {"root": "0xh", "slot": 12}, + "target": {"root": "0xt", "slot": 8}, + "source": {"root": "0xs", "slot": 4} + } + }"#; + let data_b = r#"{ + "participants": [0, 1, 2], + "data": { + "slot": 13, + "head": {"root": "0xh2", "slot": 13}, + "target": {"root": "0xt", "slot": 8}, + "source": {"root": "0xs", "slot": 4} + } + }"#; + let ev_a = normalize("node-2", "aggregate", data_a, 0, &timing()).unwrap(); + let ev_b = normalize("node-2", "aggregate", data_b, 0, &timing()).unwrap(); + assert_ne!(ev_a.id, ev_b.id); + } + + #[test] + fn unknown_topic_is_an_error_not_a_panic() { + let err = normalize("node-2", "mystery", "{}", 0, &timing()).unwrap_err(); + assert!(matches!(err, NormalizeError::UnknownTopic(_))); + } + + #[test] + fn malformed_payload_is_an_error_not_a_panic() { + let err = normalize("node-2", "block", "{not json", 0, &timing()).unwrap_err(); + assert!(matches!(err, NormalizeError::Json { .. })); + } +} diff --git a/tooling/event-monitor/src/server.rs b/tooling/event-monitor/src/server.rs new file mode 100644 index 00000000..7ec722df --- /dev/null +++ b/tooling/event-monitor/src/server.rs @@ -0,0 +1,125 @@ +//! axum HTTP server: static dashboard, the merged SSE `/stream`, and the +//! `/api/meta` bootstrap endpoint (CONTRACT.md §4). + +use std::convert::Infallible; +use std::path::Path; +use std::time::Duration; + +use axum::Router; +use axum::extract::State; +use axum::response::Json; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::routing::get; +use futures_util::{Stream, StreamExt}; +use serde::Serialize; +use tokio_stream::wrappers::BroadcastStream; +use tower_http::services::{ServeDir, ServeFile}; + +use crate::config::{Config, NodeConfig}; +use crate::hub::{Hub, HubMessage}; +use crate::model::{NodeStatus, NormalizedEvent}; +use crate::timing::Timing; + +/// Keep-alive comment interval on `/stream`, independent of any per-node +/// heartbeat published on the hub (CONTRACT.md §4). +const SSE_KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(15); + +/// One-shot bootstrap payload the frontend fetches on load (CONTRACT.md §4). +/// Small and cheap to clone per-request, so `AppState` holds it directly +/// rather than behind an `Arc` (serde's `Serialize` isn't derived for +/// `Arc` without the optional `rc` feature). +#[derive(Debug, Clone, Serialize)] +pub struct Meta { + pub genesis_time: u64, + pub ms_per_slot: u64, + pub intervals_per_slot: u64, + pub window_slots: u32, + pub topics: Vec, + pub nodes: Vec, +} + +impl Meta { + pub fn new(config: &Config, timing: &Timing) -> Self { + Self { + genesis_time: timing.genesis_time, + ms_per_slot: timing.ms_per_slot, + intervals_per_slot: timing.intervals_per_slot, + window_slots: config.window_slots, + topics: config.topics.clone(), + nodes: config.nodes.clone(), + } + } +} + +/// Backfill payload for `GET /api/history` (CONTRACT.md §4): recent chain +/// events (each identical in shape to a `/stream` `chain` event) plus the +/// latest status per node. The frontend seeds both panels from this before +/// going live, de-duping the small overlap with the live stream. +#[derive(Debug, Serialize)] +struct HistoryResponse { + events: Vec, + status: Vec, +} + +#[derive(Clone)] +struct AppState { + hub: Hub, + meta: Meta, +} + +/// Builds the full axum app: `/stream`, `/api/meta`, `/api/history`, and +/// static file serving (with an `index.html` fallback) rooted at `static_dir`. +pub fn build_router(hub: Hub, meta: Meta, static_dir: &Path) -> Router { + let state = AppState { hub, meta }; + + let index_html = static_dir.join("index.html"); + let serve_dir = ServeDir::new(static_dir).fallback(ServeFile::new(index_html)); + + Router::new() + .route("/stream", get(stream_handler)) + .route("/api/meta", get(meta_handler)) + .route("/api/history", get(history_handler)) + .with_state(state) + .fallback_service(serve_dir) +} + +async fn meta_handler(State(state): State) -> Json { + Json(state.meta.clone()) +} + +async fn history_handler(State(state): State) -> Json { + let snapshot = state.hub.history_snapshot(); + Json(HistoryResponse { + events: snapshot.events, + status: snapshot.status, + }) +} + +async fn stream_handler( + State(state): State, +) -> Sse>> { + let receiver = state.hub.subscribe(); + let stream = BroadcastStream::new(receiver).filter_map(|message| async move { + match message { + Ok(HubMessage::Chain(event)) => Some(Ok(sse_event("chain", &event))), + Ok(HubMessage::Status(status)) => Some(Ok(sse_event("status", &status))), + // Best-effort stream: a slow browser subscriber that falls + // behind simply skips the messages it missed, same contract as + // the upstream ethlambda SSE endpoint (CONTRACT.md §4). + Err(_lagged) => None, + } + }); + + Sse::new(stream).keep_alive(KeepAlive::default().interval(SSE_KEEP_ALIVE_INTERVAL)) +} + +/// Builds one named SSE event carrying `payload` as its JSON `data:` line. +/// Serialization of our own well-formed model types never fails; fall back +/// to an empty event of the same name in the unreachable error case rather +/// than panicking. +fn sse_event(name: &str, payload: &T) -> Event { + Event::default() + .event(name) + .json_data(payload) + .unwrap_or_else(|_| Event::default().event(name)) +} diff --git a/tooling/event-monitor/src/timing.rs b/tooling/event-monitor/src/timing.rs new file mode 100644 index 00000000..d11ac51f --- /dev/null +++ b/tooling/event-monitor/src/timing.rs @@ -0,0 +1,194 @@ +//! Slot-geometry bootstrap and the `offset_ms` calculation (CONTRACT.md §2). +//! +//! On startup the collector needs three numbers to translate an event's +//! `slot` into "how far into (or before) its slot did this arrive": the +//! chain's `genesis_time` (seconds), `MILLISECONDS_PER_SLOT`, and +//! `INTERVALS_PER_SLOT` (the last is only used for `/api/meta`, never for the +//! `offset_ms` math itself). `genesis_time` / `ms_per_slot` may be overridden +//! by config for offline testing; `intervals_per_slot` has no config +//! override and always comes from the network fetch (falling back to +//! [`DEFAULT_INTERVALS_PER_SLOT`] if no node is reachable). + +use std::time::Duration; + +use serde::Deserialize; + +use crate::config::NodeConfig; + +/// Fallback used only when no node answered `/lean/v0/config/spec` and the +/// config didn't need a network fetch at all (both `genesis_time` and +/// `ms_per_slot` overridden). Matches ethlambda's own default (5 intervals +/// per 4s slot). +pub const DEFAULT_INTERVALS_PER_SLOT: u64 = 5; + +const FETCH_TIMEOUT: Duration = Duration::from_secs(3); + +/// Resolved slot geometry used to compute `offset_ms` for incoming events. +#[derive(Debug, Clone, Copy)] +pub struct Timing { + pub genesis_time: u64, + pub ms_per_slot: u64, + pub intervals_per_slot: u64, +} + +impl Timing { + /// `offset_ms = arrival_ms - (genesis_time*1000 + slot*ms_per_slot)`. + /// + /// May be negative: an event can arrive before its nominal slot start + /// under clock skew between the collector and the node, or when the + /// event's own timestamp precedes the slot boundary. + pub fn offset_ms(&self, slot: u64, arrival_ms: i64) -> i64 { + let slot_start_ms = self.genesis_time as i64 * 1000 + slot as i64 * self.ms_per_slot as i64; + arrival_ms - slot_start_ms + } +} + +/// Config-supplied overrides for offline testing (CONTRACT.md §5). +#[derive(Debug, Clone, Copy, Default)] +pub struct TimingOverrides { + pub genesis_time: Option, + pub ms_per_slot: Option, +} + +#[derive(Debug, Deserialize)] +struct GenesisResponse { + genesis_time: u64, +} + +#[derive(Debug, Deserialize)] +struct SpecResponse { + #[serde(rename = "MILLISECONDS_PER_SLOT")] + milliseconds_per_slot: u64, + #[serde(rename = "INTERVALS_PER_SLOT")] + intervals_per_slot: u64, +} + +#[derive(Debug, thiserror::Error)] +pub enum BootstrapError { + #[error( + "no reachable node provided slot geometry and config did not override genesis_time/ms_per_slot" + )] + NoTimingSource, +} + +struct Fetched { + genesis_time: u64, + ms_per_slot: u64, + intervals_per_slot: u64, +} + +/// Fetches genesis + spec from the first node that answers both, trying +/// nodes in configured order. Returns `None` if none are reachable; this is +/// not necessarily fatal since `overrides` may fully cover `genesis_time` +/// and `ms_per_slot`. +async fn fetch_from_first_reachable( + nodes: &[NodeConfig], + client: &reqwest::Client, +) -> Option { + for node in nodes { + let genesis_url = format!("{}/lean/v0/genesis", node.url.trim_end_matches('/')); + let spec_url = format!("{}/lean/v0/config/spec", node.url.trim_end_matches('/')); + + let genesis = fetch_json::(client, &genesis_url).await; + let spec = fetch_json::(client, &spec_url).await; + + match (genesis, spec) { + (Ok(genesis), Ok(spec)) => { + return Some(Fetched { + genesis_time: genesis.genesis_time, + ms_per_slot: spec.milliseconds_per_slot, + intervals_per_slot: spec.intervals_per_slot, + }); + } + _ => { + tracing::debug!(node = %node.name, "timing bootstrap: node unreachable or malformed response, trying next"); + } + } + } + None +} + +async fn fetch_json Deserialize<'de>>( + client: &reqwest::Client, + url: &str, +) -> Result { + client + .get(url) + .timeout(FETCH_TIMEOUT) + .send() + .await? + .error_for_status()? + .json::() + .await +} + +/// Resolves [`Timing`], preferring config overrides and falling back to the +/// first reachable node for anything not overridden. +pub async fn bootstrap( + nodes: &[NodeConfig], + overrides: TimingOverrides, + client: &reqwest::Client, +) -> Result { + let fetched = fetch_from_first_reachable(nodes, client).await; + + let genesis_time = overrides + .genesis_time + .or_else(|| fetched.as_ref().map(|f| f.genesis_time)) + .ok_or(BootstrapError::NoTimingSource)?; + let ms_per_slot = overrides + .ms_per_slot + .or_else(|| fetched.as_ref().map(|f| f.ms_per_slot)) + .ok_or(BootstrapError::NoTimingSource)?; + let intervals_per_slot = fetched + .map(|f| f.intervals_per_slot) + .unwrap_or(DEFAULT_INTERVALS_PER_SLOT); + + Ok(Timing { + genesis_time, + ms_per_slot, + intervals_per_slot, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn timing() -> Timing { + Timing { + genesis_time: 1_770_407_233, + ms_per_slot: 4_000, + intervals_per_slot: 5, + } + } + + #[test] + fn offset_ms_matches_contract_example() { + // slot_start_ms = 1_770_407_233_000 + 128*4000 = 1_770_407_745_000 + let t = timing(); + let arrival_ms = 1_770_407_745_123_i64; + assert_eq!(t.offset_ms(128, arrival_ms), 123); + } + + #[test] + fn offset_ms_is_zero_exactly_at_slot_start() { + let t = timing(); + let slot_start = t.genesis_time as i64 * 1000 + 10 * t.ms_per_slot as i64; + assert_eq!(t.offset_ms(10, slot_start), 0); + } + + #[test] + fn offset_ms_can_be_negative_under_clock_skew() { + // Event arrives 50ms before its nominal slot boundary. + let t = timing(); + let slot_start = t.genesis_time as i64 * 1000 + 10 * t.ms_per_slot as i64; + assert_eq!(t.offset_ms(10, slot_start - 50), -50); + } + + #[test] + fn offset_ms_at_genesis_slot_zero() { + let t = timing(); + let genesis_ms = t.genesis_time as i64 * 1000; + assert_eq!(t.offset_ms(0, genesis_ms + 500), 500); + } +} diff --git a/tooling/event-monitor/tests/sse_integration.rs b/tooling/event-monitor/tests/sse_integration.rs new file mode 100644 index 00000000..2b44217b --- /dev/null +++ b/tooling/event-monitor/tests/sse_integration.rs @@ -0,0 +1,169 @@ +//! Integration test: a fake in-process SSE server stands in for an +//! ethlambda node. A real `collector::run_collector` task dials it and we +//! assert the expected `NormalizedEvent`s land on the [`Hub`]. + +use std::convert::Infallible; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::response::sse::{Event as SseEvent, Sse}; +use axum::routing::get; +use futures_util::stream::{self, Stream}; +use tokio::net::TcpListener; +use tokio::time::timeout; + +use event_monitor::collector::run_collector; +use event_monitor::config::NodeConfig; +use event_monitor::hub::{Hub, HubMessage}; +use event_monitor::model::NormalizedEvent; +use event_monitor::timing::Timing; + +const RECV_TIMEOUT: Duration = Duration::from_secs(5); + +/// Serves exactly the two frames the test asserts on: one `block` event and +/// one `attestation` event, using the exact JSON shapes from CONTRACT.md §2. +async fn fake_events_handler() -> Sse>> { + let frames = vec![ + Ok(SseEvent::default() + .event("block") + .data(r#"{"slot":128,"block":"0xabc123"}"#)), + Ok(SseEvent::default().event("attestation").data( + r#"{"validator_id":7,"data":{"slot":128,"head":{"root":"0xh","slot":128},"target":{"root":"0xt","slot":124},"source":{"root":"0xs","slot":120}}}"#, + )), + ]; + Sse::new(stream::iter(frames)) +} + +/// Pulls `HubMessage`s off `rx` until `want` [`NormalizedEvent`]s (ignoring +/// `Status` heartbeats) have been collected, or `RECV_TIMEOUT` elapses. +async fn collect_chain_events( + rx: &mut tokio::sync::broadcast::Receiver, + want: usize, +) -> Vec { + let mut collected = Vec::with_capacity(want); + timeout(RECV_TIMEOUT, async { + while collected.len() < want { + match rx.recv().await.expect("hub sender dropped unexpectedly") { + HubMessage::Chain(event) => collected.push(event), + HubMessage::Status(_) => continue, + } + } + }) + .await + .expect("timed out waiting for normalized events on the hub"); + collected +} + +#[tokio::test] +async fn collector_normalizes_frames_from_a_live_sse_server() { + let app = Router::new().route("/lean/v0/events", get(fake_events_handler)); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind fake server"); + let addr = listener + .local_addr() + .expect("fake server has no local addr"); + tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("fake SSE server crashed"); + }); + + let node = NodeConfig { + name: "fake-node".to_string(), + url: format!("http://{addr}"), + }; + let timing = Arc::new(Timing { + genesis_time: 0, + ms_per_slot: 4_000, + intervals_per_slot: 5, + }); + let hub = Hub::new(64); + let mut rx = hub.subscribe(); + let client = reqwest::Client::new(); + + let topics = vec![ + "block".to_string(), + "attestation".to_string(), + "aggregate".to_string(), + ]; + let collector_handle = tokio::spawn(run_collector(node, topics, timing, hub.clone(), client)); + + let events = collect_chain_events(&mut rx, 2).await; + collector_handle.abort(); + + assert_eq!(events.len(), 2); + + let block_event = &events[0]; + assert_eq!(block_event.node, "fake-node"); + assert_eq!(block_event.topic, "block"); + assert_eq!(block_event.slot, 128); + assert_eq!(block_event.id, Some("0xabc123".to_string())); + assert_eq!(block_event.validator_id, None); + assert_eq!(block_event.participants, None); + // genesis_time=0, ms_per_slot=4000 => slot_start_ms = 512_000; arrival + // is "now" (real wall clock), so offset_ms is a large positive number. + assert!(block_event.offset_ms > 0); + + let attestation_event = &events[1]; + assert_eq!(attestation_event.node, "fake-node"); + assert_eq!(attestation_event.topic, "attestation"); + assert_eq!(attestation_event.slot, 128); + assert_eq!(attestation_event.id, None); + assert_eq!(attestation_event.validator_id, Some(7)); + assert_eq!(attestation_event.participants, None); +} + +#[tokio::test] +async fn collector_publishes_connected_status_on_success() { + let app = Router::new().route("/lean/v0/events", get(fake_events_handler)); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind fake server"); + let addr = listener + .local_addr() + .expect("fake server has no local addr"); + tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("fake SSE server crashed"); + }); + + let node = NodeConfig { + name: "fake-node-2".to_string(), + url: format!("http://{addr}"), + }; + let timing = Arc::new(Timing { + genesis_time: 0, + ms_per_slot: 4_000, + intervals_per_slot: 5, + }); + let hub = Hub::new(64); + let mut rx = hub.subscribe(); + let client = reqwest::Client::new(); + + let collector_handle = tokio::spawn(run_collector( + node, + vec!["block".to_string()], + timing, + hub.clone(), + client, + )); + + let saw_connected = timeout(RECV_TIMEOUT, async { + loop { + if let HubMessage::Status(status) = rx.recv().await.expect("hub sender dropped") + && status.node == "fake-node-2" + && matches!(status.state, event_monitor::model::NodeState::Connected) + { + return true; + } + } + }) + .await + .unwrap_or(false); + + collector_handle.abort(); + assert!(saw_connected, "expected a Connected status update"); +} diff --git a/tooling/event-monitor/web/app.js b/tooling/event-monitor/web/app.js new file mode 100644 index 00000000..b6600c72 --- /dev/null +++ b/tooling/event-monitor/web/app.js @@ -0,0 +1,206 @@ +// app.js — bootstrap. Fetches /api/meta once, opens EventSource("/stream"), +// wires the "chain"/"status" events (CONTRACT.md §4) into the two renderers +// and the status bar. In `?demo=1` mode, meta and the event source come from +// demo.js's synthetic generator instead — everything downstream is identical. + +import { createBeeswarm } from "./beeswarm.js"; +import { createPropagation } from "./propagation.js"; + +const DEMO_MODE = new URLSearchParams(location.search).get("demo") === "1"; + +function renderStatusBar(nodes) { + const bar = document.getElementById("status-bar"); + bar.innerHTML = ""; + const chips = new Map(); + for (const node of nodes) { + const chip = document.createElement("div"); + chip.className = "status-chip status-unknown"; + + const dot = document.createElement("span"); + dot.className = "status-dot"; + + const name = document.createElement("span"); + name.className = "status-name"; + name.textContent = node.name; + + const rate = document.createElement("span"); + rate.className = "status-rate"; + rate.textContent = "—"; + + chip.append(dot, name, rate); + bar.appendChild(chip); + chips.set(node.name, chip); + } + return chips; +} + +function applyStatus(chips, status) { + const chip = chips.get(status.node); + if (!chip) return; + chip.classList.remove("status-connected", "status-reconnecting", "status-down", "status-unknown"); + chip.classList.add(`status-${status.state}`); + const rate = chip.querySelector(".status-rate"); + if (rate && typeof status.events_per_sec === "number") { + rate.textContent = `${status.events_per_sec.toFixed(1)}/s`; + } +} + +// Wraps a real EventSource("/stream") and surfaces connection loss via the +// #conn-banner element. EventSource itself retries the connection +// automatically on drop (readyState goes to CONNECTING, then back to OPEN); +// we just reflect that state so the user isn't left guessing. +function createLiveSource() { + const es = new EventSource("/stream"); + const banner = document.getElementById("conn-banner"); + es.addEventListener("open", () => { + banner.hidden = true; + }); + es.addEventListener("error", () => { + if (es.readyState !== EventSource.OPEN) banner.hidden = false; + }); + return es; +} + +async function fetchMeta() { + const res = await fetch("/api/meta"); + if (!res.ok) throw new Error(`GET /api/meta failed: ${res.status}`); + return res.json(); +} + +async function fetchHistory() { + const res = await fetch("/api/history"); + if (!res.ok) throw new Error(`GET /api/history failed: ${res.status}`); + return res.json(); +} + +// Composite key to de-dup the small overlap between the history snapshot and +// the live stream during startup (an event can appear in both). +function eventKey(e) { + return `${e.node}|${e.topic}|${e.slot}|${e.id ?? ""}|${e.validator_id ?? ""}|${e.arrival_ms}`; +} + +function wireWindowControl(meta, renderers) { + const input = document.getElementById("window-slots"); + if (!input) return; + input.value = String(meta.window_slots || 30); + const apply = () => { + let n = parseInt(input.value, 10); + if (!Number.isFinite(n)) return; + n = Math.min(500, Math.max(1, n)); + if (String(n) !== input.value) input.value = String(n); + for (const r of renderers) r.setWindowSlots(n); + }; + input.addEventListener("input", apply); + input.addEventListener("change", apply); +} + +async function boot() { + const modeIndicator = document.getElementById("mode-indicator"); + let meta; + let source; + + if (DEMO_MODE) { + const demo = await import("./demo.js"); + meta = demo.createDemoMeta(); + source = demo.createDemoSource(meta); + modeIndicator.textContent = "demo mode — synthetic data (?demo=1)"; + } else { + meta = await fetchMeta(); + source = createLiveSource(); + } + + const chips = renderStatusBar(meta.nodes); + + const beeswarm = createBeeswarm({ + canvas: document.getElementById("beeswarm-canvas"), + legendEl: document.getElementById("beeswarm-legend"), + noteEl: document.getElementById("beeswarm-note"), + meta, + }); + + const propagation = createPropagation({ + toggleEl: document.getElementById("propagation-toggle"), + canvas: document.getElementById("propagation-canvas"), + legendEl: document.getElementById("propagation-legend"), + noteEl: document.getElementById("propagation-note"), + meta, + }); + + wireWindowControl(meta, [beeswarm, propagation]); + + const ingest = (ev) => { + beeswarm.addEvent(ev); + propagation.addEvent(ev); + }; + + // Startup backfill (live mode only): open the stream first and buffer, so no + // live event is dropped in the gap while /api/history is fetched; then seed + // history, flush the buffer, and de-dup the overlap. Demo mode has no + // history endpoint and streams synthetic data straight through. + const backfilling = !DEMO_MODE; + const seen = new Set(); + const liveBuffer = []; + let loading = backfilling; + + const ingestDeduped = (ev) => { + const key = eventKey(ev); + if (seen.has(key)) return; + seen.add(key); + ingest(ev); + }; + + source.addEventListener("chain", (evt) => { + let ev; + try { + ev = JSON.parse(evt.data); + } catch { + return; // malformed frame; drop silently, never fatal to the page + } + if (loading) { + liveBuffer.push(ev); + return; + } + ingest(ev); + }); + + source.addEventListener("status", (evt) => { + let status; + try { + status = JSON.parse(evt.data); + } catch { + return; + } + applyStatus(chips, status); + }); + + if (backfilling) { + try { + const history = await fetchHistory(); + if (Array.isArray(history.status)) { + history.status.forEach((s) => applyStatus(chips, s)); + } + if (Array.isArray(history.events)) { + history.events.forEach(ingestDeduped); + } + } catch (err) { + console.warn("history backfill failed; starting with an empty view", err); + } + // Flush live events that arrived during the fetch, de-duping the overlap, + // then switch to direct ingest (the overlap window is over). + liveBuffer.forEach(ingestDeduped); + liveBuffer.length = 0; + loading = false; + seen.clear(); + } +} + +boot().catch((err) => { + console.error("event-monitor failed to start", err); + const main = document.querySelector("main"); + if (main) { + const banner = document.createElement("p"); + banner.className = "fatal-error"; + banner.textContent = `Failed to start: ${err.message}`; + main.prepend(banner); + } +}); diff --git a/tooling/event-monitor/web/beeswarm.js b/tooling/event-monitor/web/beeswarm.js new file mode 100644 index 00000000..605657dc --- /dev/null +++ b/tooling/event-monitor/web/beeswarm.js @@ -0,0 +1,267 @@ +// beeswarm.js — top panel: rolling per-node beeswarm of arrival offset within +// the slot. See CONTRACT.md §6. Consumes NormalizedEvent objects (§3) for +// topics block/attestation/aggregate; ignores everything else. + +const TOPICS = ["block", "attestation", "aggregate"]; +const MAX_POINTS_PER_NODE = 2000; + +const LANE_HEIGHT = 40; +const TOP_MARGIN = 14; +const BOTTOM_MARGIN = 30; +const LEFT_MARGIN = 92; +const RIGHT_MARGIN = 16; +const JITTER_RANGE = 11; // px, +/- around the lane center +const DOT_RADIUS = 2.6; + +// Fade older slots non-linearly: the newest slot is fully opaque and each +// older slot drops geometrically toward a faint floor (front-loaded, so +// recent events pop). opacity = FADE_FLOOR + (1 - FADE_FLOOR) * FADE_DECAY^age, +// e.g. age 0/1/2/3 → ~1.0 / 0.75 / 0.57 / 0.44. +const FADE_DECAY = 0.7; +const FADE_FLOOR = 0.15; + +// Deterministic pseudo-random in [-1, 1) from a string seed, so a dot's +// jitter stays put across animation frames instead of flickering. +function hashJitter(seed) { + let h = 2166136261; + for (let i = 0; i < seed.length; i++) { + h ^= seed.charCodeAt(i); + h = Math.imul(h, 16777619); + } + h >>>= 0; + return ((h % 2000) / 1000) - 1; +} + +function readTheme() { + const cs = getComputedStyle(document.documentElement); + const get = (name, fallback) => { + const v = cs.getPropertyValue(name); + return v && v.trim() ? v.trim() : fallback; + }; + return { + grid: get("--grid", "#d0d3d9"), + text: get("--muted", "#6b7280"), + laneAlt: get("--lane-alt", "rgba(0,0,0,0.04)"), + topics: { + block: get("--topic-block", "#4f8cff"), + attestation: get("--topic-attestation", "#37b24d"), + aggregate: get("--topic-aggregate", "#f59f00"), + }, + }; +} + +/** + * @param {{canvas: HTMLCanvasElement, legendEl?: Element, noteEl?: Element, meta: object}} opts + */ +export function createBeeswarm({ canvas, legendEl, noteEl, meta }) { + const ctx = canvas.getContext("2d"); + const nodeNames = meta.nodes.map((n) => n.name); + const laneIndex = new Map(nodeNames.map((name, i) => [name, i])); + const perNode = new Map(nodeNames.map((name) => [name, []])); + let windowSlots = meta.window_slots || 30; + const msPerSlot = meta.ms_per_slot || 4000; + const intervals = meta.intervals_per_slot || 5; + + let maxSlotSeen = 0; + let theme = readTheme(); + let dpr = window.devicePixelRatio || 1; + let cssWidth = 0; + let cssHeight = 0; + let stopped = false; + + function resize() { + const rect = canvas.parentElement.getBoundingClientRect(); + cssWidth = Math.max(320, Math.floor(rect.width)); + cssHeight = TOP_MARGIN + BOTTOM_MARGIN + LANE_HEIGHT * Math.max(1, nodeNames.length); + dpr = window.devicePixelRatio || 1; + canvas.style.width = `${cssWidth}px`; + canvas.style.height = `${cssHeight}px`; + canvas.width = Math.round(cssWidth * dpr); + canvas.height = Math.round(cssHeight * dpr); + } + + let ro = null; + if (typeof ResizeObserver !== "undefined") { + ro = new ResizeObserver(() => resize()); + ro.observe(canvas.parentElement); + } else { + window.addEventListener("resize", resize); + } + + let mq = null; + const onThemeChange = () => { + theme = readTheme(); + }; + if (window.matchMedia) { + mq = window.matchMedia("(prefers-color-scheme: dark)"); + if (mq.addEventListener) mq.addEventListener("change", onThemeChange); + else if (mq.addListener) mq.addListener(onThemeChange); + } + + function addEvent(ev) { + if (!TOPICS.includes(ev.topic)) return; + const arr = perNode.get(ev.node); + if (!arr) return; // event from a node not in meta.nodes; ignore defensively + if (ev.slot > maxSlotSeen) maxSlotSeen = ev.slot; + const offsetMs = Math.min(Math.max(ev.offset_ms, 0), msPerSlot); + const seed = `${ev.node}|${ev.topic}|${ev.slot}|${ev.id ?? ""}|${ev.validator_id ?? ""}|${ev.arrival_ms}`; + arr.push({ + topic: ev.topic, + offsetMs, + slot: ev.slot, + jitter: hashJitter(seed) * JITTER_RANGE, + }); + // Oldest-first decimation: cap points per node so a flood (e.g. + // attestations) can't grow memory/render cost unbounded. + if (arr.length > MAX_POINTS_PER_NODE) { + arr.splice(0, arr.length - MAX_POINTS_PER_NODE); + } + } + + function xForOffset(offsetMs) { + const plotWidth = cssWidth - LEFT_MARGIN - RIGHT_MARGIN; + return LEFT_MARGIN + (offsetMs / msPerSlot) * plotWidth; + } + + function laneY(nodeName) { + const idx = laneIndex.get(nodeName); + return TOP_MARGIN + idx * LANE_HEIGHT + LANE_HEIGHT / 2; + } + + function draw() { + if (cssWidth === 0) return; + ctx.save(); + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, cssWidth, cssHeight); + + const plotBottom = TOP_MARGIN + LANE_HEIGHT * nodeNames.length; + + // alternating lane backgrounds + nodeNames.forEach((_, idx) => { + if (idx % 2 === 1) { + ctx.fillStyle = theme.laneAlt; + ctx.fillRect( + LEFT_MARGIN, + TOP_MARGIN + idx * LANE_HEIGHT, + cssWidth - LEFT_MARGIN - RIGHT_MARGIN, + LANE_HEIGHT + ); + } + }); + + // gridlines every ms_per_slot / intervals_per_slot, plus axis labels + const step = msPerSlot / intervals; + ctx.strokeStyle = theme.grid; + ctx.lineWidth = 1; + ctx.font = "10px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace"; + ctx.fillStyle = theme.text; + ctx.textAlign = "center"; + ctx.textBaseline = "top"; + for (let ms = 0; ms <= msPerSlot + 0.001; ms += step) { + const x = xForOffset(ms); + ctx.beginPath(); + ctx.moveTo(x, TOP_MARGIN); + ctx.lineTo(x, plotBottom); + ctx.stroke(); + ctx.fillText(`${Math.round(ms)}`, x, plotBottom + 6); + } + ctx.textAlign = "left"; + ctx.fillText("ms into slot", LEFT_MARGIN, plotBottom + 18); + + // lane labels + ctx.textAlign = "right"; + ctx.textBaseline = "middle"; + nodeNames.forEach((name) => { + ctx.fillStyle = theme.text; + ctx.fillText(name, LEFT_MARGIN - 10, laneY(name)); + }); + + // dots, faded by slot age relative to the rolling window + nodeNames.forEach((name) => { + const arr = perNode.get(name); + const y = laneY(name); + for (let i = 0; i < arr.length; i++) { + const pt = arr[i]; + const age = maxSlotSeen - pt.slot; + if (age >= windowSlots) continue; + const x = xForOffset(pt.offsetMs); + ctx.globalAlpha = FADE_FLOOR + (1 - FADE_FLOOR) * Math.pow(FADE_DECAY, age); + ctx.fillStyle = theme.topics[pt.topic]; + ctx.beginPath(); + ctx.arc(x, y + pt.jitter, DOT_RADIUS, 0, Math.PI * 2); + ctx.fill(); + } + }); + ctx.globalAlpha = 1; + ctx.restore(); + } + + // Periodically drop points that have fully aged out of the window, so the + // per-node arrays don't hold onto stale data just because they're under + // the point cap. Each node's array is arrival-ordered, so scanning from + // the front is a good approximation of oldest-first. + function prune() { + for (const arr of perNode.values()) { + let i = 0; + while (i < arr.length && maxSlotSeen - arr[i].slot >= windowSlots) i++; + if (i > 0) arr.splice(0, i); + } + } + + function renderLegend() { + if (!legendEl) return; + legendEl.innerHTML = ""; + const items = [ + ["block", "Block"], + ["attestation", "Attestation"], + ["aggregate", "Aggregate"], + ]; + for (const [topic, label] of items) { + const chip = document.createElement("span"); + chip.className = `legend-item legend-${topic}`; + const swatch = document.createElement("span"); + swatch.className = "legend-swatch"; + chip.appendChild(swatch); + chip.appendChild(document.createTextNode(label)); + legendEl.appendChild(chip); + } + } + + let frameCount = 0; + function loop() { + if (stopped) return; + frameCount++; + if (frameCount % 60 === 0) prune(); + draw(); + requestAnimationFrame(loop); + } + + function updateNote() { + if (noteEl) { + noteEl.textContent = `Showing the last ${windowSlots} slots. Older slots fade out, then drop.`; + } + } + + resize(); + renderLegend(); + updateNote(); + requestAnimationFrame(loop); + + return { + addEvent, + resize, + setWindowSlots(n) { + windowSlots = Math.max(1, Math.floor(n)); + updateNote(); + }, + destroy() { + stopped = true; + if (ro) ro.disconnect(); + else window.removeEventListener("resize", resize); + if (mq) { + if (mq.removeEventListener) mq.removeEventListener("change", onThemeChange); + else if (mq.removeListener) mq.removeListener(onThemeChange); + } + }, + }; +} diff --git a/tooling/event-monitor/web/demo.js b/tooling/event-monitor/web/demo.js new file mode 100644 index 00000000..8f22cdb7 --- /dev/null +++ b/tooling/event-monitor/web/demo.js @@ -0,0 +1,169 @@ +// demo.js — self-contained synthetic data generator for the `?demo=1` mode. +// Only ever imported by app.js when that URL param is present (see boot() in +// app.js); never runs otherwise. It fabricates a plausible /api/meta (§4) and +// then drives the exact same "chain"/"status" EventTarget interface that a +// real EventSource would (see CONTRACT.md §4 GET /stream), so app.js and the +// two renderers (beeswarm.js, propagation.js) don't need to know the +// difference. + +const DEMO_NODES = ["node-2", "node-3", "node-4", "node-5"]; +const MS_PER_SLOT = 4000; +const INTERVALS_PER_SLOT = 5; +const WINDOW_SLOTS = 30; +const VALIDATOR_COUNT = 16; +const SLOW_NODE = "node-5"; // consistently later, so the panels have something to show + +export function createDemoMeta() { + return { + genesis_time: Math.floor(Date.now() / 1000) - (MS_PER_SLOT / 1000) * 1000, + ms_per_slot: MS_PER_SLOT, + intervals_per_slot: INTERVALS_PER_SLOT, + window_slots: WINDOW_SLOTS, + topics: ["block", "attestation", "aggregate"], + nodes: DEMO_NODES.map((name) => ({ name, url: `demo://${name}` })), + }; +} + +// Small non-cryptographic hash for fabricating plausible-looking 0x… ids. +function fnv1aHex(str) { + let h = 2166136261; + for (let i = 0; i < str.length; i++) { + h ^= str.charCodeAt(i); + h = Math.imul(h, 16777619); + } + return (h >>> 0).toString(16).padStart(8, "0"); +} + +function fakeRoot(seed) { + return `0x${fnv1aHex(seed).padEnd(64, "0")}`; +} + +/** + * Returns an object with the same `addEventListener("chain"|"status", cb)` + * shape as the real `EventSource` from CONTRACT.md §4, but driven by timers + * instead of a network connection. + * + * @param {object} meta as produced by createDemoMeta() + */ +export function createDemoSource(meta) { + const target = new EventTarget(); + const timers = []; + let slot = 1000; + const statusState = new Map(meta.nodes.map((n) => [n.name, "connected"])); + + function emitChain(ev) { + target.dispatchEvent(new MessageEvent("chain", { data: JSON.stringify(ev) })); + } + function emitStatus(st) { + target.dispatchEvent(new MessageEvent("status", { data: JSON.stringify(st) })); + } + + // The slow node arrives well after everyone else; the rest jitter a little. + function nodeDelayMs(name) { + return name === SLOW_NODE ? 350 + Math.random() * 250 : Math.random() * 60; + } + + function schedule(fn, delayMs) { + timers.push(setTimeout(fn, delayMs)); + } + + function runSlot(currentSlot) { + const blockRoot = fakeRoot(`block:${currentSlot}`); + const aggId = fakeRoot(`aggregate:${currentSlot}`); + + // blocks: ~0.6s into the slot + for (const node of meta.nodes) { + const offset = 600 + nodeDelayMs(node.name); + schedule( + () => + emitChain({ + node: node.name, + topic: "block", + slot: currentSlot, + arrival_ms: Date.now(), + offset_ms: Math.round(offset), + id: blockRoot, + validator_id: null, + participants: null, + }), + offset + ); + } + + // attestations: ~1.1s into the slot, one per validator per node + for (const node of meta.nodes) { + for (let validatorId = 0; validatorId < VALIDATOR_COUNT; validatorId++) { + const offset = 1100 + nodeDelayMs(node.name) + Math.random() * 200; + schedule( + () => + emitChain({ + node: node.name, + topic: "attestation", + slot: currentSlot, + arrival_ms: Date.now(), + offset_ms: Math.round(offset), + id: null, + validator_id: validatorId, + participants: null, + }), + offset + ); + } + } + + // aggregate: ~2.2s into the slot + for (const node of meta.nodes) { + const offset = 2200 + nodeDelayMs(node.name); + const participants = Math.max(1, VALIDATOR_COUNT - Math.round(Math.random() * 2)); + schedule( + () => + emitChain({ + node: node.name, + topic: "aggregate", + slot: currentSlot, + arrival_ms: Date.now(), + offset_ms: Math.round(offset), + id: aggId, + validator_id: null, + participants, + }), + offset + ); + } + } + + runSlot(slot); + timers.push( + setInterval(() => { + slot += 1; + runSlot(slot); + }, meta.ms_per_slot) + ); + + // occasional reconnecting blip on the slow node, plus a periodic rate heartbeat + timers.push( + setInterval(() => { + for (const node of meta.nodes) { + let state = statusState.get(node.name); + if (node.name === SLOW_NODE && Math.random() < 0.12) { + state = state === "connected" ? "reconnecting" : "connected"; + statusState.set(node.name, state); + } + emitStatus({ + node: node.name, + state, + events_per_sec: state === "down" ? 0 : 4 + Math.random() * 2, + }); + } + }, 2000) + ); + + target.close = () => { + for (const t of timers) { + clearTimeout(t); + clearInterval(t); + } + }; + + return target; +} diff --git a/tooling/event-monitor/web/index.html b/tooling/event-monitor/web/index.html new file mode 100644 index 00000000..b976457c --- /dev/null +++ b/tooling/event-monitor/web/index.html @@ -0,0 +1,55 @@ + + + + + + event-monitor + + + +
+
+

event-monitor

+ +
+ +
+ + + slots +
+
+
+ +
+
+
+

Arrival offset within slot

+
+
+
+ +
+

+
+ +
+
+

Propagation delta

+
+
+
+ +
+
+

+
+
+ +
+ event-monitor +
+ + + + diff --git a/tooling/event-monitor/web/propagation.js b/tooling/event-monitor/web/propagation.js new file mode 100644 index 00000000..911faf95 --- /dev/null +++ b/tooling/event-monitor/web/propagation.js @@ -0,0 +1,337 @@ +// propagation.js — bottom panel: propagation delta as a rolling per-node +// beeswarm. See CONTRACT.md §6. Groups NormalizedEvents (§3) by their `id` +// field per topic; each point is one node's arrival delay relative to the +// first node to see that id, on a fixed 0…ms_per_slot x-axis. A delta beyond +// one slot saturates at the right edge and turns magenta. +// +// Canvas scaffolding intentionally mirrors beeswarm.js (lanes, jitter, fade, +// rAF loop) rather than sharing a core, to keep each panel independently +// readable; keep the two in sync when changing the visual language. + +const TOPICS = ["block", "aggregate", "head"]; +const MAX_IDS_PER_TOPIC = 400; + +const LANE_HEIGHT = 40; +const TOP_MARGIN = 14; +const BOTTOM_MARGIN = 30; +const LEFT_MARGIN = 92; +const RIGHT_MARGIN = 16; +const JITTER_RANGE = 11; // px, +/- around the lane center +const DOT_RADIUS = 2.6; + +// Fade older slots non-linearly: the newest slot is fully opaque and each +// older slot drops geometrically toward a faint floor (front-loaded, so +// recent ids pop). Kept identical to beeswarm.js's fade for a consistent feel. +const FADE_DECAY = 0.7; +const FADE_FLOOR = 0.15; + +// Deterministic pseudo-random in [-1, 1) from a string seed, so a dot's +// jitter stays put across animation frames instead of flickering. +function hashJitter(seed) { + let h = 2166136261; + for (let i = 0; i < seed.length; i++) { + h ^= seed.charCodeAt(i); + h = Math.imul(h, 16777619); + } + h >>>= 0; + return ((h % 2000) / 1000) - 1; +} + +function readTheme() { + const cs = getComputedStyle(document.documentElement); + const get = (name, fallback) => { + const v = cs.getPropertyValue(name); + return v && v.trim() ? v.trim() : fallback; + }; + return { + grid: get("--grid", "#d0d3d9"), + text: get("--muted", "#6b7280"), + laneAlt: get("--lane-alt", "rgba(0,0,0,0.04)"), + first: get("--prop-first", "#7048e8"), + normal: get("--prop-normal", "#1098ad"), + over: get("--prop-over", "#e64980"), + }; +} + +/** + * @param {{toggleEl: Element, canvas: HTMLCanvasElement, legendEl?: Element, noteEl?: Element, meta: object}} opts + */ +export function createPropagation({ toggleEl, canvas, legendEl, noteEl, meta }) { + const ctx = canvas.getContext("2d"); + const nodeNames = meta.nodes.map((n) => n.name); + const laneIndex = new Map(nodeNames.map((name, i) => [name, i])); + + // topic -> Map }> + const groups = new Map(TOPICS.map((t) => [t, new Map()])); + // topic -> [id, ...] in first-seen order, for the per-topic id cap. + const insertionOrder = new Map(TOPICS.map((t) => [t, []])); + + let selectedTopic = "block"; + let windowSlots = meta.window_slots || 30; + const msPerSlot = meta.ms_per_slot || 4000; + const intervals = meta.intervals_per_slot || 5; + + let maxSlotSeen = 0; + let theme = readTheme(); + let dpr = window.devicePixelRatio || 1; + let cssWidth = 0; + let cssHeight = 0; + let stopped = false; + + function resize() { + const rect = canvas.parentElement.getBoundingClientRect(); + cssWidth = Math.max(320, Math.floor(rect.width)); + cssHeight = TOP_MARGIN + BOTTOM_MARGIN + LANE_HEIGHT * Math.max(1, nodeNames.length); + dpr = window.devicePixelRatio || 1; + canvas.style.width = `${cssWidth}px`; + canvas.style.height = `${cssHeight}px`; + canvas.width = Math.round(cssWidth * dpr); + canvas.height = Math.round(cssHeight * dpr); + } + + let ro = null; + if (typeof ResizeObserver !== "undefined") { + ro = new ResizeObserver(() => resize()); + ro.observe(canvas.parentElement); + } else { + window.addEventListener("resize", resize); + } + + let mq = null; + const onThemeChange = () => { + theme = readTheme(); + }; + if (window.matchMedia) { + mq = window.matchMedia("(prefers-color-scheme: dark)"); + if (mq.addEventListener) mq.addEventListener("change", onThemeChange); + else if (mq.addListener) mq.addListener(onThemeChange); + } + + function addEvent(ev) { + if (!TOPICS.includes(ev.topic)) return; + if (ev.id == null) return; // ungroupable (e.g. attestation) + + const group = groups.get(ev.topic); + const order = insertionOrder.get(ev.topic); + let entry = group.get(ev.id); + if (!entry) { + entry = { slot: ev.slot, arrivals: new Map() }; + group.set(ev.id, entry); + order.push(ev.id); + if (order.length > MAX_IDS_PER_TOPIC) { + const dropped = order.shift(); + group.delete(dropped); + } + } + // Keep the earliest arrival per node for this id (first sighting wins). + const prior = entry.arrivals.get(ev.node); + if (prior === undefined || ev.arrival_ms < prior) { + entry.arrivals.set(ev.node, ev.arrival_ms); + } + if (ev.slot > maxSlotSeen) maxSlotSeen = ev.slot; + } + + function xForDelta(deltaMs) { + const plotWidth = cssWidth - LEFT_MARGIN - RIGHT_MARGIN; + const clamped = Math.min(Math.max(deltaMs, 0), msPerSlot); + return LEFT_MARGIN + (clamped / msPerSlot) * plotWidth; + } + + function laneY(nodeName) { + const idx = laneIndex.get(nodeName); + return TOP_MARGIN + idx * LANE_HEIGHT + LANE_HEIGHT / 2; + } + + function drawFrame(plotBottom) { + // alternating lane backgrounds + nodeNames.forEach((_, idx) => { + if (idx % 2 === 1) { + ctx.fillStyle = theme.laneAlt; + ctx.fillRect( + LEFT_MARGIN, + TOP_MARGIN + idx * LANE_HEIGHT, + cssWidth - LEFT_MARGIN - RIGHT_MARGIN, + LANE_HEIGHT + ); + } + }); + + // gridlines every ms_per_slot / intervals_per_slot, plus axis labels + const step = msPerSlot / intervals; + ctx.strokeStyle = theme.grid; + ctx.lineWidth = 1; + ctx.font = "10px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace"; + ctx.fillStyle = theme.text; + ctx.textAlign = "center"; + ctx.textBaseline = "top"; + for (let ms = 0; ms <= msPerSlot + 0.001; ms += step) { + const x = xForDelta(ms); + ctx.beginPath(); + ctx.moveTo(x, TOP_MARGIN); + ctx.lineTo(x, plotBottom); + ctx.stroke(); + ctx.fillText(`${Math.round(ms)}`, x, plotBottom + 6); + } + ctx.textAlign = "left"; + ctx.fillText("delta ms (from first node)", LEFT_MARGIN, plotBottom + 18); + + // lane labels + ctx.textAlign = "right"; + ctx.textBaseline = "middle"; + nodeNames.forEach((name) => { + ctx.fillStyle = theme.text; + ctx.fillText(name, LEFT_MARGIN - 10, laneY(name)); + }); + } + + function draw() { + if (cssWidth === 0) return; + ctx.save(); + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, cssWidth, cssHeight); + + const plotBottom = TOP_MARGIN + LANE_HEIGHT * nodeNames.length; + drawFrame(plotBottom); + + const group = groups.get(selectedTopic); + let plotted = 0; + for (const [id, entry] of group) { + const age = maxSlotSeen - entry.slot; + if (age >= windowSlots) continue; + if (entry.arrivals.size === 0) continue; + const minArrival = Math.min(...entry.arrivals.values()); + const alpha = FADE_FLOOR + (1 - FADE_FLOOR) * Math.pow(FADE_DECAY, age); + + for (const [node, arrival] of entry.arrivals) { + const y = laneY(node); + if (y === undefined) continue; + const delta = arrival - minArrival; + const x = xForDelta(delta); + const color = delta === 0 ? theme.first : delta > msPerSlot ? theme.over : theme.normal; + const jitter = hashJitter(`${id}|${node}`) * JITTER_RANGE; + ctx.globalAlpha = alpha; + ctx.fillStyle = color; + ctx.beginPath(); + ctx.arc(x, y + jitter, DOT_RADIUS, 0, Math.PI * 2); + ctx.fill(); + plotted++; + } + } + ctx.globalAlpha = 1; + + if (plotted === 0) { + ctx.fillStyle = theme.text; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + ctx.font = "13px -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif"; + ctx.fillText( + `Waiting for ${selectedTopic} events…`, + LEFT_MARGIN + (cssWidth - LEFT_MARGIN - RIGHT_MARGIN) / 2, + TOP_MARGIN + (LANE_HEIGHT * nodeNames.length) / 2 + ); + } + ctx.restore(); + } + + // Drop ids that have fully aged out of the window from every topic, so the + // group maps don't retain stale ids just because they're under the cap. + function prune() { + for (const topic of TOPICS) { + const group = groups.get(topic); + const order = insertionOrder.get(topic); + let kept = []; + for (const id of order) { + const entry = group.get(id); + if (entry && maxSlotSeen - entry.slot >= windowSlots) { + group.delete(id); + } else if (entry) { + kept.push(id); + } + } + insertionOrder.set(topic, kept); + } + } + + function renderLegend() { + if (!legendEl) return; + legendEl.innerHTML = ""; + const items = [ + [theme.first, "first to see"], + [theme.normal, "lag"], + [theme.over, "> 1 slot behind"], + ]; + for (const [color, label] of items) { + const chip = document.createElement("span"); + chip.className = "legend-item"; + const swatch = document.createElement("span"); + swatch.className = "legend-swatch"; + swatch.style.background = color; + chip.appendChild(swatch); + chip.appendChild(document.createTextNode(label)); + legendEl.appendChild(chip); + } + } + + function updateNote() { + if (noteEl) { + noteEl.textContent = + `Each dot is one node's delay behind the first node to see an id, ` + + `over the last ${windowSlots} slots. Fixed 0–${msPerSlot}ms scale.`; + } + } + + function buildToggle() { + toggleEl.innerHTML = ""; + for (const topic of TOPICS) { + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "toggle-btn"; + btn.textContent = topic; + btn.setAttribute("aria-pressed", String(topic === selectedTopic)); + if (topic === selectedTopic) btn.classList.add("toggle-btn-active"); + btn.addEventListener("click", () => { + if (topic === selectedTopic) return; + selectedTopic = topic; + for (const sibling of toggleEl.children) { + const active = sibling === btn; + sibling.classList.toggle("toggle-btn-active", active); + sibling.setAttribute("aria-pressed", String(active)); + } + }); + toggleEl.appendChild(btn); + } + } + + let frameCount = 0; + function loop() { + if (stopped) return; + frameCount++; + if (frameCount % 60 === 0) prune(); + draw(); + requestAnimationFrame(loop); + } + + resize(); + buildToggle(); + renderLegend(); + updateNote(); + requestAnimationFrame(loop); + + return { + addEvent, + resize, + setWindowSlots(n) { + windowSlots = Math.max(1, Math.floor(n)); + updateNote(); + }, + destroy() { + stopped = true; + if (ro) ro.disconnect(); + else window.removeEventListener("resize", resize); + if (mq) { + if (mq.removeEventListener) mq.removeEventListener("change", onThemeChange); + else if (mq.removeListener) mq.removeListener(onThemeChange); + } + }, + }; +} diff --git a/tooling/event-monitor/web/style.css b/tooling/event-monitor/web/style.css new file mode 100644 index 00000000..1787042b --- /dev/null +++ b/tooling/event-monitor/web/style.css @@ -0,0 +1,339 @@ +/* event-monitor — calm, technical dashboard look. Light by default, dark via + prefers-color-scheme. Topic colors (block/attestation/aggregate) and status + colors (connected/reconnecting/down) are frozen by CONTRACT.md §6 and stay + constant across themes. */ + +:root { + --bg: #f6f7f9; + --panel-bg: #ffffff; + --panel-border: #e1e4e9; + --text: #1b1f24; + --muted: #6b7280; + --muted-2: #8b93a1; + --grid: #e4e7ec; + --lane-alt: rgba(80, 96, 120, 0.045); + --accent: #4f8cff; + --chip-bg: #f0f2f5; + + --topic-block: #4f8cff; + --topic-attestation: #37b24d; + --topic-aggregate: #f59f00; + + --status-connected: #2f9e44; + --status-reconnecting: #f59f00; + --status-down: #e03131; + --status-unknown: #9aa4b2; + + /* Propagation-bar state colors. Deliberately off the topic hues + (block/attestation/aggregate) so the two panels never read as the same + scale. Constant across themes, like the topic colors. */ + --prop-first: #7048e8; /* first node to see the id */ + --prop-normal: #1098ad; /* normal inter-node lag */ + --prop-over: #e64980; /* behind by more than a full slot */ +} + +@media (prefers-color-scheme: dark) { + :root { + --bg: #0d0f13; + --panel-bg: #14171d; + --panel-border: #262a33; + --text: #e7e9ec; + --muted: #9aa4b2; + --muted-2: #737d8c; + --grid: #262a33; + --lane-alt: rgba(160, 180, 220, 0.05); + --accent: #6ea1ff; + --chip-bg: #1b1f27; + } +} + +/* explicit theme override hooks, in case a host page ever stamps data-theme */ +:root[data-theme="dark"] { + --bg: #0d0f13; + --panel-bg: #14171d; + --panel-border: #262a33; + --text: #e7e9ec; + --muted: #9aa4b2; + --muted-2: #737d8c; + --grid: #262a33; + --lane-alt: rgba(160, 180, 220, 0.05); + --accent: #6ea1ff; + --chip-bg: #1b1f27; +} + +:root[data-theme="light"] { + --bg: #f6f7f9; + --panel-bg: #ffffff; + --panel-border: #e1e4e9; + --text: #1b1f24; + --muted: #6b7280; + --muted-2: #8b93a1; + --grid: #e4e7ec; + --lane-alt: rgba(80, 96, 120, 0.045); + --accent: #4f8cff; + --chip-bg: #f0f2f5; +} + +* { + box-sizing: border-box; +} + +html, body { + margin: 0; + padding: 0; +} + +body { + background: var(--bg); + color: var(--text); + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + font-size: 14px; + line-height: 1.45; + min-height: 100vh; + display: flex; + flex-direction: column; +} + +h1, h2 { + margin: 0; + font-weight: 600; +} + +h1 { + font-size: 15px; + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--text); +} + +h2 { + font-size: 12px; + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--muted); +} + +/* ---------- header ---------- */ + +.topbar { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 16px 24px; + padding: 18px 28px; + border-bottom: 1px solid var(--panel-border); + background: var(--panel-bg); +} + +.topbar-title { + display: flex; + align-items: baseline; + gap: 10px; +} + +.mode-indicator { + font-size: 11px; + color: var(--status-reconnecting); + letter-spacing: 0.03em; +} + +.conn-banner { + font-size: 12px; + color: var(--status-reconnecting); + border: 1px solid var(--status-reconnecting); + border-radius: 999px; + padding: 3px 10px; +} + +.window-control { + display: inline-flex; + align-items: center; + gap: 6px; + font-size: 12px; + color: var(--muted); +} + +.window-control input { + width: 58px; + padding: 3px 6px; + font-size: 12px; + color: var(--text); + background: var(--panel-bg); + border: 1px solid var(--panel-border); + border-radius: 6px; + font-variant-numeric: tabular-nums; +} + +.window-unit { + color: var(--muted-2); +} + +.status-bar { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-left: auto; +} + +.status-chip { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 4px 10px; + border-radius: 999px; + background: var(--chip-bg); + border: 1px solid var(--panel-border); + font-size: 12px; + color: var(--muted); +} + +.status-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--status-unknown); + flex: none; +} + +.status-name { + color: var(--text); + font-weight: 500; +} + +.status-rate { + color: var(--muted-2); + font-variant-numeric: tabular-nums; +} + +.status-connected .status-dot { background: var(--status-connected); } +.status-reconnecting .status-dot { background: var(--status-reconnecting); } +.status-down .status-dot { background: var(--status-down); } +.status-unknown .status-dot { background: var(--status-unknown); } + +/* ---------- layout ---------- */ + +main { + flex: 1; + display: flex; + flex-direction: column; + gap: 20px; + padding: 24px 28px 32px; + max-width: 1280px; + width: 100%; + margin: 0 auto; +} + +.panel { + background: var(--panel-bg); + border: 1px solid var(--panel-border); + border-radius: 10px; + padding: 18px 20px 20px; +} + +.panel-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + margin-bottom: 14px; + flex-wrap: wrap; +} + +.hint { + margin: 10px 2px 0; + font-size: 12px; + color: var(--muted-2); +} + +.fatal-error { + color: var(--status-down); + border: 1px solid var(--status-down); + border-radius: 8px; + padding: 10px 14px; + font-size: 13px; +} + +/* ---------- legend ---------- */ + +.legend { + display: flex; + gap: 14px; + flex-wrap: wrap; +} + +.legend-item { + display: inline-flex; + align-items: center; + gap: 6px; + font-size: 12px; + color: var(--muted); +} + +.legend-swatch { + width: 9px; + height: 9px; + border-radius: 50%; + display: inline-block; +} + +.legend-block .legend-swatch { background: var(--topic-block); } +.legend-attestation .legend-swatch { background: var(--topic-attestation); } +.legend-aggregate .legend-swatch { background: var(--topic-aggregate); } + +/* ---------- beeswarm ---------- */ + +.beeswarm-wrap { + width: 100%; + overflow-x: auto; +} + +#beeswarm-canvas { + display: block; + width: 100%; +} + +/* ---------- propagation ---------- */ + +.toggle-group { + display: inline-flex; + gap: 4px; + background: var(--chip-bg); + border-radius: 8px; + padding: 3px; +} + +.toggle-btn { + border: none; + background: transparent; + color: var(--muted); + font-size: 12px; + letter-spacing: 0.02em; + text-transform: uppercase; + padding: 5px 12px; + border-radius: 6px; + cursor: pointer; +} + +.toggle-btn:hover { + color: var(--text); +} + +.toggle-btn-active { + background: var(--panel-bg); + color: var(--text); + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.08); +} + +#propagation-canvas { + display: block; + width: 100%; +} + +/* ---------- footer ---------- */ + +.foot { + padding: 12px 28px 20px; + font-size: 11px; + color: var(--muted-2); + text-align: center; +} From 02e55db084804c6543d5ddc4db0ff4b7e43bed79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 23 Jul 2026 13:17:44 -0300 Subject: [PATCH 2/8] refactor(event-monitor): dedup and tidy per review Quality cleanups from a /simplify pass; no behavior change and the serialized wire shape is untouched: - serialize HistorySnapshot directly, dropping the field-identical HistoryResponse DTO and its hand copy - add NodeConfig::endpoint() as the single home for the node URL join, used by the collector and the timing bootstrap - fold the checkpoint-topic payload into SlotBlockPayload (serde ignores the extra state field) and trim ReorgPayload to the surfaced fields - extract node_status() for the four repeated status literals - fetch genesis + spec concurrently so a dead node costs one timeout - propagation panel: reject unknown nodes at ingest, drop a dead branch, and cache per-dot jitter + the id's min arrival at insert rather than recomputing both every animation frame --- tooling/event-monitor/src/collector.rs | 40 +++++++++++------------- tooling/event-monitor/src/config.rs | 8 +++++ tooling/event-monitor/src/hub.rs | 6 ++-- tooling/event-monitor/src/main.rs | 3 +- tooling/event-monitor/src/model.rs | 38 +++++++++------------- tooling/event-monitor/src/server.rs | 25 +++++---------- tooling/event-monitor/src/timing.rs | 14 ++++++--- tooling/event-monitor/web/propagation.js | 27 +++++++++------- 8 files changed, 76 insertions(+), 85 deletions(-) diff --git a/tooling/event-monitor/src/collector.rs b/tooling/event-monitor/src/collector.rs index 612b3498..53b50809 100644 --- a/tooling/event-monitor/src/collector.rs +++ b/tooling/event-monitor/src/collector.rs @@ -90,6 +90,14 @@ fn now_ms() -> i64 { i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) } +fn node_status(name: &str, state: NodeState, events_per_sec: f64) -> NodeStatus { + NodeStatus { + node: name.to_string(), + state, + events_per_sec, + } +} + /// Runs forever: connects, streams, and on any disconnect/error reconnects /// with capped exponential backoff. Intended to be spawned as one long-lived /// task per configured node. @@ -102,11 +110,7 @@ pub async fn run_collector( ) { let mut backoff = Backoff::new(); loop { - hub.publish_status(NodeStatus { - node: node.name.clone(), - state: NodeState::Reconnecting, - events_per_sec: 0.0, - }); + hub.publish_status(node_status(&node.name, NodeState::Reconnecting, 0.0)); match connect_and_stream(&node, &topics, &timing, &hub, &client).await { Ok(()) => { @@ -124,11 +128,7 @@ pub async fn run_collector( } else { NodeState::Reconnecting }; - hub.publish_status(NodeStatus { - node: node.name.clone(), - state, - events_per_sec: 0.0, - }); + hub.publish_status(node_status(&node.name, state, 0.0)); tokio::time::sleep(delay).await; } } @@ -144,19 +144,15 @@ async fn connect_and_stream( client: &reqwest::Client, ) -> Result<(), CollectorError> { let url = format!( - "{}/lean/v0/events?topics={}", - node.url.trim_end_matches('/'), + "{}?topics={}", + node.endpoint("/lean/v0/events"), topics.join(",") ); let response = client.get(&url).send().await?.error_for_status()?; let mut stream = response.bytes_stream().eventsource(); - hub.publish_status(NodeStatus { - node: node.name.clone(), - state: NodeState::Connected, - events_per_sec: 0.0, - }); + hub.publish_status(node_status(&node.name, NodeState::Connected, 0.0)); tracing::info!(node = %node.name, %url, "connected to SSE stream"); let mut rate = RateTracker::new(); @@ -176,11 +172,11 @@ async fn connect_and_stream( } } _ = heartbeat.tick() => { - hub.publish_status(NodeStatus { - node: node.name.clone(), - state: NodeState::Connected, - events_per_sec: rate.rate_and_reset(), - }); + hub.publish_status(node_status( + &node.name, + NodeState::Connected, + rate.rate_and_reset(), + )); } } } diff --git a/tooling/event-monitor/src/config.rs b/tooling/event-monitor/src/config.rs index f87a6b1e..606df950 100644 --- a/tooling/event-monitor/src/config.rs +++ b/tooling/event-monitor/src/config.rs @@ -42,6 +42,14 @@ pub struct NodeConfig { pub url: String, } +impl NodeConfig { + /// Joins `path` (leading slash included) onto this node's base URL, the + /// single place the trailing-slash-tolerant URL convention lives. + pub fn endpoint(&self, path: &str) -> String { + format!("{}{}", self.url.trim_end_matches('/'), path) + } +} + fn default_window_slots() -> u32 { 30 } diff --git a/tooling/event-monitor/src/hub.rs b/tooling/event-monitor/src/hub.rs index 4a95b1ae..c09fd2ca 100644 --- a/tooling/event-monitor/src/hub.rs +++ b/tooling/event-monitor/src/hub.rs @@ -5,6 +5,7 @@ use std::collections::{BTreeMap, VecDeque}; use std::sync::{Arc, Mutex}; +use serde::Serialize; use tokio::sync::broadcast; use crate::model::{NodeStatus, NormalizedEvent}; @@ -29,8 +30,9 @@ pub enum HubMessage { } /// Point-in-time backfill payload served by `GET /api/history`: the retained -/// recent chain events plus the latest status per node (CONTRACT.md §4). -#[derive(Debug, Clone, Default)] +/// recent chain events plus the latest status per node (CONTRACT.md §4). Its +/// field names match that endpoint's JSON exactly, so it is serialized directly. +#[derive(Debug, Clone, Default, Serialize)] pub struct HistorySnapshot { pub events: Vec, pub status: Vec, diff --git a/tooling/event-monitor/src/main.rs b/tooling/event-monitor/src/main.rs index 4258925c..b946ab34 100644 --- a/tooling/event-monitor/src/main.rs +++ b/tooling/event-monitor/src/main.rs @@ -28,9 +28,8 @@ async fn main() -> anyhow::Result<()> { let args = Args::parse(); - let config_path = args.config.clone(); let config = Config::load(&args.config).inspect_err(|err| { - tracing::error!(%err, config = %config_path.display(), "failed to load config"); + tracing::error!(%err, config = %args.config.display(), "failed to load config"); })?; let client = reqwest::Client::new(); diff --git a/tooling/event-monitor/src/model.rs b/tooling/event-monitor/src/model.rs index 25ab7087..374f5aaf 100644 --- a/tooling/event-monitor/src/model.rs +++ b/tooling/event-monitor/src/model.rs @@ -23,22 +23,16 @@ pub struct AttestationData { } /// `block`, `safe_target`, `block_gossip`: `{ "slot": N, "block": "0x..." }`. +/// +/// Also used for `head` / `justified_checkpoint` / `finalized_checkpoint`, whose +/// wire shape adds a `state: "0x..."` field; serde ignores it, since only `slot` +/// and `block` are surfaced on the [`NormalizedEvent`]. #[derive(Debug, Clone, Deserialize)] struct SlotBlockPayload { slot: u64, block: String, } -/// `head`, `justified_checkpoint`, `finalized_checkpoint`: -/// `{ "slot": N, "block": "0x...", "state": "0x..." }`. -#[derive(Debug, Clone, Deserialize)] -struct CheckpointEventPayload { - slot: u64, - block: String, - #[allow(dead_code)] // part of the wire shape; not surfaced on NormalizedEvent - state: String, -} - /// `attestation`: `{ "validator_id": N, "data": {...} }`. #[derive(Debug, Clone, Deserialize)] struct AttestationPayload { @@ -56,18 +50,14 @@ struct AggregatePayload { /// `chain_reorg`: /// `{ "slot":N, "depth":N, "old_head_block":"0x...", "old_head_state":"0x...", /// "new_head_block":"0x...", "new_head_state":"0x..." }`. +/// +/// Only `slot` and `new_head_block` are surfaced on the [`NormalizedEvent`]; the +/// remaining fields of the wire shape (documented above) are ignored on +/// deserialize. #[derive(Debug, Clone, Deserialize)] struct ReorgPayload { slot: u64, - #[allow(dead_code)] - depth: u64, - #[allow(dead_code)] - old_head_block: String, - #[allow(dead_code)] - old_head_state: String, new_head_block: String, - #[allow(dead_code)] - new_head_state: String, } /// Collector -> browser payload (CONTRACT.md §3). Field names and shape are @@ -147,15 +137,15 @@ pub fn normalize( }; let (slot, id, validator_id, participants) = match topic { - "block" | "safe_target" | "block_gossip" => { + "block" + | "safe_target" + | "block_gossip" + | "head" + | "justified_checkpoint" + | "finalized_checkpoint" => { let payload: SlotBlockPayload = serde_json::from_str(data).map_err(to_json_err)?; (payload.slot, Some(payload.block), None, None) } - "head" | "justified_checkpoint" | "finalized_checkpoint" => { - let payload: CheckpointEventPayload = - serde_json::from_str(data).map_err(to_json_err)?; - (payload.slot, Some(payload.block), None, None) - } "chain_reorg" => { let payload: ReorgPayload = serde_json::from_str(data).map_err(to_json_err)?; (payload.slot, Some(payload.new_head_block), None, None) diff --git a/tooling/event-monitor/src/server.rs b/tooling/event-monitor/src/server.rs index 7ec722df..f4b0a456 100644 --- a/tooling/event-monitor/src/server.rs +++ b/tooling/event-monitor/src/server.rs @@ -16,8 +16,7 @@ use tokio_stream::wrappers::BroadcastStream; use tower_http::services::{ServeDir, ServeFile}; use crate::config::{Config, NodeConfig}; -use crate::hub::{Hub, HubMessage}; -use crate::model::{NodeStatus, NormalizedEvent}; +use crate::hub::{HistorySnapshot, Hub, HubMessage}; use crate::timing::Timing; /// Keep-alive comment interval on `/stream`, independent of any per-node @@ -51,16 +50,6 @@ impl Meta { } } -/// Backfill payload for `GET /api/history` (CONTRACT.md §4): recent chain -/// events (each identical in shape to a `/stream` `chain` event) plus the -/// latest status per node. The frontend seeds both panels from this before -/// going live, de-duping the small overlap with the live stream. -#[derive(Debug, Serialize)] -struct HistoryResponse { - events: Vec, - status: Vec, -} - #[derive(Clone)] struct AppState { hub: Hub, @@ -87,12 +76,12 @@ async fn meta_handler(State(state): State) -> Json { Json(state.meta.clone()) } -async fn history_handler(State(state): State) -> Json { - let snapshot = state.hub.history_snapshot(); - Json(HistoryResponse { - events: snapshot.events, - status: snapshot.status, - }) +/// Backfill for `GET /api/history` (CONTRACT.md §4): recent chain events (each +/// identical in shape to a `/stream` `chain` event) plus the latest status per +/// node. The frontend seeds both panels from this before going live, de-duping +/// the small overlap with the live stream. +async fn history_handler(State(state): State) -> Json { + Json(state.hub.history_snapshot()) } async fn stream_handler( diff --git a/tooling/event-monitor/src/timing.rs b/tooling/event-monitor/src/timing.rs index d11ac51f..7893c914 100644 --- a/tooling/event-monitor/src/timing.rs +++ b/tooling/event-monitor/src/timing.rs @@ -86,11 +86,15 @@ async fn fetch_from_first_reachable( client: &reqwest::Client, ) -> Option { for node in nodes { - let genesis_url = format!("{}/lean/v0/genesis", node.url.trim_end_matches('/')); - let spec_url = format!("{}/lean/v0/config/spec", node.url.trim_end_matches('/')); - - let genesis = fetch_json::(client, &genesis_url).await; - let spec = fetch_json::(client, &spec_url).await; + let genesis_url = node.endpoint("/lean/v0/genesis"); + let spec_url = node.endpoint("/lean/v0/config/spec"); + + // The two fetches are independent; run them concurrently so a slow or + // dead node costs one FETCH_TIMEOUT, not two in series. + let (genesis, spec) = tokio::join!( + fetch_json::(client, &genesis_url), + fetch_json::(client, &spec_url), + ); match (genesis, spec) { (Ok(genesis), Ok(spec)) => { diff --git a/tooling/event-monitor/web/propagation.js b/tooling/event-monitor/web/propagation.js index 911faf95..ba3b9307 100644 --- a/tooling/event-monitor/web/propagation.js +++ b/tooling/event-monitor/web/propagation.js @@ -61,7 +61,7 @@ export function createPropagation({ toggleEl, canvas, legendEl, noteEl, meta }) const nodeNames = meta.nodes.map((n) => n.name); const laneIndex = new Map(nodeNames.map((name, i) => [name, i])); - // topic -> Map }> + // topic -> Map }> const groups = new Map(TOPICS.map((t) => [t, new Map()])); // topic -> [id, ...] in first-seen order, for the per-topic id cap. const insertionOrder = new Map(TOPICS.map((t) => [t, []])); @@ -110,12 +110,13 @@ export function createPropagation({ toggleEl, canvas, legendEl, noteEl, meta }) function addEvent(ev) { if (!TOPICS.includes(ev.topic)) return; if (ev.id == null) return; // ungroupable (e.g. attestation) + if (!laneIndex.has(ev.node)) return; // node not in meta.nodes; ignore defensively const group = groups.get(ev.topic); const order = insertionOrder.get(ev.topic); let entry = group.get(ev.id); if (!entry) { - entry = { slot: ev.slot, arrivals: new Map() }; + entry = { slot: ev.slot, arrivals: new Map(), minArrival: Infinity }; group.set(ev.id, entry); order.push(ev.id); if (order.length > MAX_IDS_PER_TOPIC) { @@ -124,10 +125,14 @@ export function createPropagation({ toggleEl, canvas, legendEl, noteEl, meta }) } } // Keep the earliest arrival per node for this id (first sighting wins). + // Jitter is hashed once here (stable per id+node) rather than every frame, + // and the id's min arrival is tracked incrementally for the delta baseline. const prior = entry.arrivals.get(ev.node); - if (prior === undefined || ev.arrival_ms < prior) { - entry.arrivals.set(ev.node, ev.arrival_ms); + if (prior === undefined || ev.arrival_ms < prior.arrival) { + const jitter = prior ? prior.jitter : hashJitter(`${ev.id}|${ev.node}`) * JITTER_RANGE; + entry.arrivals.set(ev.node, { arrival: ev.arrival_ms, jitter }); } + if (ev.arrival_ms < entry.minArrival) entry.minArrival = ev.arrival_ms; if (ev.slot > maxSlotSeen) maxSlotSeen = ev.slot; } @@ -195,24 +200,22 @@ export function createPropagation({ toggleEl, canvas, legendEl, noteEl, meta }) const group = groups.get(selectedTopic); let plotted = 0; - for (const [id, entry] of group) { + for (const [, entry] of group) { const age = maxSlotSeen - entry.slot; if (age >= windowSlots) continue; if (entry.arrivals.size === 0) continue; - const minArrival = Math.min(...entry.arrivals.values()); + const minArrival = entry.minArrival; const alpha = FADE_FLOOR + (1 - FADE_FLOOR) * Math.pow(FADE_DECAY, age); - for (const [node, arrival] of entry.arrivals) { - const y = laneY(node); - if (y === undefined) continue; - const delta = arrival - minArrival; + for (const [node, rec] of entry.arrivals) { + const delta = rec.arrival - minArrival; const x = xForDelta(delta); + const y = laneY(node) + rec.jitter; const color = delta === 0 ? theme.first : delta > msPerSlot ? theme.over : theme.normal; - const jitter = hashJitter(`${id}|${node}`) * JITTER_RANGE; ctx.globalAlpha = alpha; ctx.fillStyle = color; ctx.beginPath(); - ctx.arc(x, y + jitter, DOT_RADIUS, 0, Math.PI * 2); + ctx.arc(x, y, DOT_RADIUS, 0, Math.PI * 2); ctx.fill(); plotted++; } From 4eeafffa80331879fa193db244f2e4658dbba3e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:31:30 -0300 Subject: [PATCH 3/8] refactor(event-monitor): drop head, safe_target and chain_reorg events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The propagation panel offered a `head` toggle, but head arrival spread is a fork-choice output derived from the same attestation set as `block`, so it told us nothing the block lane didn't already show. Dropping the toggle also lets the collector stop subscribing to a topic no panel renders. `safe_target` and `chain_reorg` were normalized backend-side and then discarded by both panels' topic filters, so they only ever cost a match arm and a payload struct. `chain_reorg` in particular is a point-in-time event rather than a per-node arrival race, so it does not belong on a beeswarm. Keeps CONTRACT.md authoritative: the §2 topic table and the propagation toggle spec now match what the code accepts and renders. --- tooling/event-monitor/CONTRACT.md | 6 +-- tooling/event-monitor/README.md | 2 +- tooling/event-monitor/config.example.toml | 6 +-- tooling/event-monitor/src/model.rs | 47 +---------------------- tooling/event-monitor/web/propagation.js | 2 +- 5 files changed, 9 insertions(+), 54 deletions(-) diff --git a/tooling/event-monitor/CONTRACT.md b/tooling/event-monitor/CONTRACT.md index c18430ad..7d8ad8ee 100644 --- a/tooling/event-monitor/CONTRACT.md +++ b/tooling/event-monitor/CONTRACT.md @@ -55,8 +55,6 @@ Per-topic `data:` JSON shapes the collector must handle: | `head` | `{ "slot": 128, "block": "0x…", "state": "0x…" }` | `slot` | | `justified_checkpoint` | `{ "slot": 128, "block": "0x…", "state": "0x…" }` | `slot` | | `finalized_checkpoint` | `{ "slot": 128, "block": "0x…", "state": "0x…" }` | `slot` | -| `safe_target` | `{ "slot": 128, "block": "0x…" }` | `slot` | -| `chain_reorg` | `{ "slot":…, "depth":…, "old_head_block":"0x…", "old_head_state":"0x…", "new_head_block":"0x…", "new_head_state":"0x…" }` | `slot` | | `block_gossip` | `{ "slot": 128, "block": "0x…" }` | `slot` | A `Checkpoint` (`head`/`target`/`source`) is `{ "root": "0x…", "slot": N }`. @@ -107,7 +105,7 @@ Serialized as JSON. Field names are frozen: | `slot` | u64 | slot the event refers to (from `slot` or `data.slot`) | | `arrival_ms` | i64 | collector receive time, epoch ms | | `offset_ms` | i64 | `arrival_ms - slot_start_ms`; can be negative | -| `id` | string \| null | grouping/propagation identity: block/head/safe_target/gossip → the `block` root; reorg → `new_head_block`; checkpoints → `block`; **aggregate → a session-stable content hash** of `(data, sorted participants)`, hex `0x…`; **attestation → null** | +| `id` | string \| null | grouping/propagation identity: block/head/gossip → the `block` root; checkpoints → `block`; **aggregate → a session-stable content hash** of `(data, sorted participants)`, hex `0x…`; **attestation → null** | | `validator_id` | u64 \| null | set only for `attestation` | | `participants` | u32 \| null | set only for `aggregate`: participant **count** (never the full list — keep frames light) | @@ -224,7 +222,7 @@ Cap points per node (e.g. 2000) with oldest-first decimation so an attestation flood can't wedge rendering. Legend + a note that older slots fade. **Bottom — propagation delta** (canvas beeswarm): a topic toggle -(`block` default / `aggregate` / `head`). Group events of that topic by `id`; +(`block` default / `aggregate`). Group events of that topic by `id`; for every id in the last `window_slots`, plot one dot per node in that node's lane at `x = clamp(arrival_ms − min(arrival_ms over nodes for that id), 0, ms_per_slot)`, jittered, faded by slot age. **Fixed 0…`ms_per_slot` x-axis** diff --git a/tooling/event-monitor/README.md b/tooling/event-monitor/README.md index 353d6c0c..943fa330 100644 --- a/tooling/event-monitor/README.md +++ b/tooling/event-monitor/README.md @@ -5,7 +5,7 @@ Live arrival-time monitor for lean-consensus (ethlambda) nodes. It dials the `GET /lean/v0/events` SSE stream of several nodes, timestamps each event on arrival, and serves a browser dashboard that visualizes when `block` / `attestation` / `aggregate` events arrive **relative to the slot** -(rolling beeswarm, per node) and how a given block/aggregate/head **propagates** +(rolling beeswarm, per node) and how a given block/aggregate **propagates** between nodes (a second beeswarm of per-node delay behind the first node to see each id). The rolling window is adjustable live from the header, and a fresh page load backfills recent history from the collector so it's never blank. diff --git a/tooling/event-monitor/config.example.toml b/tooling/event-monitor/config.example.toml index 491be2db..8a210ef5 100644 --- a/tooling/event-monitor/config.example.toml +++ b/tooling/event-monitor/config.example.toml @@ -7,9 +7,9 @@ listen = "127.0.0.1:8080" # collector bind address (the dashboard URL) window_slots = 30 # initial rolling window; adjustable live in the UI history_slots = 64 # slots of events buffered to backfill a fresh page load static_dir = "web" # directory served at GET / (the frontend) -# Upstream SSE topics to subscribe. `head` powers the propagation panel's head -# toggle; drop it if you only care about block/aggregate propagation. -topics = ["block", "attestation", "aggregate", "head"] +# Upstream SSE topics to subscribe. Only these three are rendered by the +# frontend; any other topic is normalized and then dropped by both panels. +topics = ["block", "attestation", "aggregate"] # Optional offline overrides. Normally the collector auto-fetches slot geometry # from the first reachable node via /lean/v0/genesis and /lean/v0/config/spec. diff --git a/tooling/event-monitor/src/model.rs b/tooling/event-monitor/src/model.rs index 374f5aaf..a9d9233b 100644 --- a/tooling/event-monitor/src/model.rs +++ b/tooling/event-monitor/src/model.rs @@ -22,7 +22,7 @@ pub struct AttestationData { pub source: Checkpoint, } -/// `block`, `safe_target`, `block_gossip`: `{ "slot": N, "block": "0x..." }`. +/// `block`, `block_gossip`: `{ "slot": N, "block": "0x..." }`. /// /// Also used for `head` / `justified_checkpoint` / `finalized_checkpoint`, whose /// wire shape adds a `state: "0x..."` field; serde ignores it, since only `slot` @@ -47,19 +47,6 @@ struct AggregatePayload { data: AttestationData, } -/// `chain_reorg`: -/// `{ "slot":N, "depth":N, "old_head_block":"0x...", "old_head_state":"0x...", -/// "new_head_block":"0x...", "new_head_state":"0x..." }`. -/// -/// Only `slot` and `new_head_block` are surfaced on the [`NormalizedEvent`]; the -/// remaining fields of the wire shape (documented above) are ignored on -/// deserialize. -#[derive(Debug, Clone, Deserialize)] -struct ReorgPayload { - slot: u64, - new_head_block: String, -} - /// Collector -> browser payload (CONTRACT.md §3). Field names and shape are /// frozen; do not rename without updating CONTRACT.md and `web/`. #[derive(Debug, Clone, Serialize, PartialEq)] @@ -137,19 +124,10 @@ pub fn normalize( }; let (slot, id, validator_id, participants) = match topic { - "block" - | "safe_target" - | "block_gossip" - | "head" - | "justified_checkpoint" - | "finalized_checkpoint" => { + "block" | "block_gossip" | "head" | "justified_checkpoint" | "finalized_checkpoint" => { let payload: SlotBlockPayload = serde_json::from_str(data).map_err(to_json_err)?; (payload.slot, Some(payload.block), None, None) } - "chain_reorg" => { - let payload: ReorgPayload = serde_json::from_str(data).map_err(to_json_err)?; - (payload.slot, Some(payload.new_head_block), None, None) - } "attestation" => { let payload: AttestationPayload = serde_json::from_str(data).map_err(to_json_err)?; (payload.data.slot, None, Some(payload.validator_id), None) @@ -215,15 +193,6 @@ mod tests { assert_eq!(ev.offset_ms, 1_000 - 128 * 4_000); } - #[test] - fn safe_target_topic_maps_id_to_block_root() { - let data = r#"{ "slot": 127, "block": "0xdeadbeef" }"#; - let ev = normalize("node-2", "safe_target", data, 500, &timing()).unwrap(); - assert_eq!(ev.topic, "safe_target"); - assert_eq!(ev.slot, 127); - assert_eq!(ev.id, Some("0xdeadbeef".to_string())); - } - #[test] fn block_gossip_topic_maps_id_to_block_root() { let data = r#"{ "slot": 128, "block": "0xabc123" }"#; @@ -255,18 +224,6 @@ mod tests { assert_eq!(ev.id, Some("0xcccc".to_string())); } - #[test] - fn chain_reorg_maps_id_to_new_head_block() { - let data = r#"{ - "slot": 128, "depth": 2, - "old_head_block": "0xold1", "old_head_state": "0xold2", - "new_head_block": "0xnew1", "new_head_state": "0xnew2" - }"#; - let ev = normalize("node-2", "chain_reorg", data, 0, &timing()).unwrap(); - assert_eq!(ev.slot, 128); - assert_eq!(ev.id, Some("0xnew1".to_string())); - } - #[test] fn attestation_topic_has_null_id_and_validator_id_set() { let data = r#"{ diff --git a/tooling/event-monitor/web/propagation.js b/tooling/event-monitor/web/propagation.js index ba3b9307..caa89c1c 100644 --- a/tooling/event-monitor/web/propagation.js +++ b/tooling/event-monitor/web/propagation.js @@ -8,7 +8,7 @@ // rAF loop) rather than sharing a core, to keep each panel independently // readable; keep the two in sync when changing the visual language. -const TOPICS = ["block", "aggregate", "head"]; +const TOPICS = ["block", "aggregate"]; const MAX_IDS_PER_TOPIC = 400; const LANE_HEIGHT = 40; From b450d79593f09379c6b96675cac55de7528a1f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:43:16 -0300 Subject: [PATCH 4/8] fix(event-monitor): stop three defects that misreport or blank the view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconnect backoff reset was keyed on a clean end-of-stream. The common case is the opposite: a node restart surfaces as a stream error after hours of healthy streaming, so every restart ratcheted the delay one step permanently and after ~7 of them a perfectly healthy node reported `down` and took MAX_BACKOFF to reconnect. Key the reset on how long the session lasted instead, via an `Attempt` that is healthy once it survives one heartbeat. That also closes the inverse hazard the old rule left open: a peer which accepts the request and immediately closes the stream ended "cleanly", hit the reset, and was retried at INITIAL_BACKOFF forever instead of backing off. Both the collector's history ring and the dashboard's rolling window key retention off the highest slot seen, and that watermark only ever moves up. One event from a node on a different genesis therefore aged out every real event and blanked both panels until a restart. Drop events whose slot is more than MAX_FUTURE_SLOTS ahead of the slot the collector's own clock is in, warning once per node per connection so the misconfiguration is visible rather than silent. Only the future side is bounded: old slots are legitimate and common (finalized_checkpoint trails head, a syncing node replays history) and cannot move the watermark. Placing the check in `normalize` covers the history ring and both canvas panels from one spot, since no bogus event reaches the frontend. The frontend applies `status` immediately but buffers `chain` during startup backfill, so the /api/history snapshot could overwrite a fresher live status with an older one. Live status now wins. Test fixtures paired slot-128 and slot-12 payloads with sub-second arrival times, i.e. events far in the future, which the new bound rejects. They now place arrival inside a plausible slot via `arrival_for`, keeping every original assertion; `block_topic_maps_id_to_block_root` asserts CONTRACT §2's worked offset of 123ms rather than a negative offset, which `offset_ms_can_be_negative_under_clock_skew` already covers. CONTRACT.md gains the slot bound in §2 and the live-status-wins rule in §4. --- tooling/event-monitor/CONTRACT.md | 18 +++ tooling/event-monitor/src/collector.rs | 147 ++++++++++++++++++++++--- tooling/event-monitor/src/model.rs | 133 +++++++++++++++++++--- tooling/event-monitor/src/timing.rs | 44 ++++++++ tooling/event-monitor/web/app.js | 9 +- 5 files changed, 321 insertions(+), 30 deletions(-) diff --git a/tooling/event-monitor/CONTRACT.md b/tooling/event-monitor/CONTRACT.md index 7d8ad8ee..0731c681 100644 --- a/tooling/event-monitor/CONTRACT.md +++ b/tooling/event-monitor/CONTRACT.md @@ -79,6 +79,19 @@ offset_ms = arrival_ms - slot_start_ms // may be negative under clock (`SystemTime::now()` → epoch ms). Config may override `genesis_time` / `ms_per_slot` for offline testing. +### Slot plausibility bound + +Once slot geometry is known, the collector drops any event whose `slot` is more +than `MAX_FUTURE_SLOTS` (8) ahead of the slot its own clock is in, warning once +per node per connection. Both the collector's history ring (§4) and the +dashboard's rolling window (§6) key retention off the highest slot seen, and +that watermark only ever moves up, so a single event from a node on a different +genesis would age out every real event and blank the view until a restart. + +Only the future side is bounded. Old slots are legitimate and common +(`finalized_checkpoint` trails head, a syncing node replays history) and cannot +move the watermark, so they are always accepted. + --- ## 3. NormalizedEvent (collector → browser payload) @@ -174,6 +187,11 @@ buffer, de-duping the overlap by `(node, topic, slot, id, validator_id, arrival_ms)`. The broadcast never replays to new subscribers, so this guarantees no gap and no double-count. +Status is applied immediately rather than buffered, so **live status wins**: the +frontend ignores a snapshot `status` entry for any node whose status already +arrived on the open stream, since the snapshot is a point-in-time record and can +be the older of the two. + --- ## 5. Config file (TOML) diff --git a/tooling/event-monitor/src/collector.rs b/tooling/event-monitor/src/collector.rs index 53b50809..7df0a4ad 100644 --- a/tooling/event-monitor/src/collector.rs +++ b/tooling/event-monitor/src/collector.rs @@ -11,7 +11,7 @@ use futures_util::StreamExt; use crate::config::NodeConfig; use crate::hub::Hub; -use crate::model::{self, NodeState, NodeStatus}; +use crate::model::{self, NodeState, NodeStatus, NormalizeError}; use crate::timing::Timing; const INITIAL_BACKOFF: Duration = Duration::from_millis(250); @@ -26,6 +26,31 @@ enum CollectorError { Stream(String), } +/// Outcome of one connection attempt, as the reconnect loop needs it. +struct Attempt { + /// `true` once the connection stayed up for at least one + /// [`HEARTBEAT_INTERVAL`] while streaming. + /// + /// Keyed on how *long* the session lasted rather than on how it ended: a + /// node restart surfaces as an error after hours of healthy streaming and + /// must not inherit the failure ramp, while a peer that accepts the + /// request and instantly drops the stream ends cleanly yet must keep + /// ramping instead of being hammered at [`INITIAL_BACKOFF`]. + healthy: bool, + /// `None` on a clean end-of-stream, `Some` on a transport/parse failure. + error: Option, +} + +impl Attempt { + /// The connection was never established. + fn failed(error: CollectorError) -> Self { + Self { + healthy: false, + error: Some(error), + } + } +} + /// Exponential backoff capped at [`MAX_BACKOFF`]. Used both to pace /// reconnect attempts and to decide whether the collector should report /// `reconnecting` (still ramping up retries) or `down` (settled into @@ -45,6 +70,17 @@ impl Backoff { self.delay = INITIAL_BACKOFF; } + /// Folds one attempt's outcome into the ramp: a session that proved + /// healthy clears it, so the next reconnect starts from + /// [`INITIAL_BACKOFF`] instead of inheriting the delay earned by earlier + /// failures. Without this, every node restart ratchets the delay one step + /// permanently and a perfectly healthy node eventually reports `down`. + fn record(&mut self, attempt: &Attempt) { + if attempt.healthy { + self.reset(); + } + } + /// Returns the delay to wait before the next attempt, then doubles /// (capped) for next time. fn advance(&mut self) -> Duration { @@ -112,15 +148,14 @@ pub async fn run_collector( loop { hub.publish_status(node_status(&node.name, NodeState::Reconnecting, 0.0)); - match connect_and_stream(&node, &topics, &timing, &hub, &client).await { - Ok(()) => { - tracing::info!(node = %node.name, "SSE stream ended; reconnecting"); - backoff.reset(); - } - Err(err) => { + let attempt = connect_and_stream(&node, &topics, &timing, &hub, &client).await; + match &attempt.error { + None => tracing::info!(node = %node.name, "SSE stream ended; reconnecting"), + Some(err) => { tracing::warn!(node = %node.name, %err, "SSE connection failed; will retry"); } } + backoff.record(&attempt); let delay = backoff.advance(); let state = if delay >= MAX_BACKOFF { @@ -134,22 +169,30 @@ pub async fn run_collector( } /// Opens one SSE connection and streams frames until the connection ends or -/// errors. Returns `Ok(())` on a clean end-of-stream (server closed it), -/// `Err` on a transport/parse failure. +/// errors. The returned [`Attempt`] carries both how the session ended and +/// whether it lasted long enough to count as healthy. async fn connect_and_stream( node: &NodeConfig, topics: &[String], timing: &Timing, hub: &Hub, client: &reqwest::Client, -) -> Result<(), CollectorError> { +) -> Attempt { let url = format!( "{}?topics={}", node.endpoint("/lean/v0/events"), topics.join(",") ); - let response = client.get(&url).send().await?.error_for_status()?; + let response = match client + .get(&url) + .send() + .await + .and_then(|response| response.error_for_status()) + { + Ok(response) => response, + Err(err) => return Attempt::failed(err.into()), + }; let mut stream = response.bytes_stream().eventsource(); hub.publish_status(node_status(&node.name, NodeState::Connected, 0.0)); @@ -158,6 +201,8 @@ async fn connect_and_stream( let mut rate = RateTracker::new(); let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); heartbeat.tick().await; // the first tick fires immediately; consume it + let mut healthy = false; + let mut warned_implausible_slot = false; loop { tokio::select! { @@ -165,13 +210,19 @@ async fn connect_and_stream( match frame { Some(Ok(event)) => { rate.tick(); - handle_frame(node, &event, timing, hub); + handle_frame(node, &event, timing, hub, &mut warned_implausible_slot); + } + Some(Err(err)) => { + let error = Some(CollectorError::Stream(err.to_string())); + return Attempt { healthy, error }; } - Some(Err(err)) => return Err(CollectorError::Stream(err.to_string())), - None => return Ok(()), + None => return Attempt { healthy, error: None }, } } _ = heartbeat.tick() => { + // Surviving a whole heartbeat interval while streaming is what + // marks the session healthy for backoff purposes. + healthy = true; hub.publish_status(node_status( &node.name, NodeState::Connected, @@ -185,7 +236,17 @@ async fn connect_and_stream( /// Normalizes one already-parsed SSE frame and publishes it on the hub. /// Never panics: an unknown topic or payload we can't parse is logged and /// dropped (CONTRACT.md §2). -fn handle_frame(node: &NodeConfig, event: &SseEvent, timing: &Timing, hub: &Hub) { +/// +/// `warned_implausible_slot` latches the one-per-session warning for slots the +/// collector clock says cannot be real; a node on the wrong genesis produces +/// one such frame per event, and the point is to be noticed, not to flood. +fn handle_frame( + node: &NodeConfig, + event: &SseEvent, + timing: &Timing, + hub: &Hub, + warned_implausible_slot: &mut bool, +) { // Defensive: eventsource-stream already suppresses comment/keep-alive // lines (they never build a non-empty data buffer), but guard anyway. if event.data.is_empty() { @@ -193,6 +254,16 @@ fn handle_frame(node: &NodeConfig, event: &SseEvent, timing: &Timing, hub: &Hub) } match model::normalize(&node.name, &event.event, &event.data, now_ms(), timing) { Ok(normalized) => hub.publish_chain(normalized), + Err(err @ NormalizeError::ImplausibleSlot { .. }) => { + if !*warned_implausible_slot { + *warned_implausible_slot = true; + tracing::warn!( + node = %node.name, + %err, + "dropping events with implausible slots; is this node on a different genesis?" + ); + } + } Err(err) => { tracing::debug!( node = %node.name, @@ -231,6 +302,52 @@ mod tests { assert_eq!(backoff.advance(), INITIAL_BACKOFF); } + #[test] + fn healthy_session_clears_the_ramp_even_when_it_ends_with_an_error() { + // The common shape of a node restart: hours of healthy streaming, then + // the connection drops with an error. The next reconnect must start + // from INITIAL_BACKOFF, not inherit the ramp. + let mut backoff = Backoff::new(); + backoff.advance(); + backoff.advance(); + backoff.record(&Attempt { + healthy: true, + error: Some(CollectorError::Stream( + "connection reset by peer".to_string(), + )), + }); + assert_eq!(backoff.advance(), INITIAL_BACKOFF); + } + + #[test] + fn unhealthy_session_keeps_the_ramp_climbing_even_when_it_ends_cleanly() { + // A peer that accepts the request and immediately closes the stream + // ends cleanly, but must not be retried at INITIAL_BACKOFF forever. + let mut backoff = Backoff::new(); + assert_eq!(backoff.advance(), INITIAL_BACKOFF); + backoff.record(&Attempt { + healthy: false, + error: None, + }); + assert_eq!(backoff.advance(), INITIAL_BACKOFF * 2); + } + + #[test] + fn repeated_healthy_sessions_never_ratchet_toward_down() { + // Regression: with the reset keyed on clean end-of-stream instead of on + // session health, each restart ratcheted the delay one step and after + // ~7 restarts a healthy node was reported Down with 10s reconnects. + let mut backoff = Backoff::new(); + for _ in 0..20 { + let delay = backoff.advance(); + assert!(delay < MAX_BACKOFF, "delay ratcheted to the Down threshold"); + backoff.record(&Attempt { + healthy: true, + error: Some(CollectorError::Stream("node restarted".to_string())), + }); + } + } + #[test] fn rate_tracker_counts_ticks_since_last_reset() { let mut rate = RateTracker::new(); diff --git a/tooling/event-monitor/src/model.rs b/tooling/event-monitor/src/model.rs index a9d9233b..d8b86355 100644 --- a/tooling/event-monitor/src/model.rs +++ b/tooling/event-monitor/src/model.rs @@ -61,6 +61,20 @@ pub struct NormalizedEvent { pub participants: Option, } +/// How far ahead of the collector's own clock an event's slot may be before we +/// treat it as bogus. A node slightly ahead of us is normal (clock skew, an +/// event emitted just before its slot boundary), but a slot far in the future +/// means the node is on a different genesis or the payload is corrupt. Such an +/// event must not be accepted: both the collector's history ring and the +/// dashboard's rolling window key their retention off the highest slot seen, +/// which only ever moves up, so one bogus slot would age out every real event +/// and blank the view until a restart. +/// +/// Only the future side is bounded. Old slots are legitimate and common +/// (`finalized_checkpoint` trails head, a syncing node replays history) and +/// cannot move the watermark. +const MAX_FUTURE_SLOTS: u64 = 8; + #[derive(Debug, thiserror::Error)] pub enum NormalizeError { #[error("unknown topic: {0}")] @@ -71,6 +85,14 @@ pub enum NormalizeError { #[source] source: serde_json::Error, }, + #[error( + "topic {topic} reports slot {slot}, more than {MAX_FUTURE_SLOTS} slots ahead of the collector's own slot {collector_slot}" + )] + ImplausibleSlot { + topic: String, + slot: u64, + collector_slot: u64, + }, } /// Canonical struct hashed to derive the aggregate `id`: `{data, participants}` @@ -141,6 +163,15 @@ pub fn normalize( other => return Err(NormalizeError::UnknownTopic(other.to_string())), }; + let collector_slot = timing.slot_at(arrival_ms); + if slot > collector_slot.saturating_add(MAX_FUTURE_SLOTS) { + return Err(NormalizeError::ImplausibleSlot { + topic: topic.to_string(), + slot, + collector_slot, + }); + } + Ok(NormalizedEvent { node: node.to_string(), topic: topic.to_string(), @@ -181,22 +212,31 @@ mod tests { } } + /// Collector-clock arrival `offset_ms` into `slot`, so fixtures place the + /// arrival inside a plausible slot the way real events do (see + /// [`MAX_FUTURE_SLOTS`]). + fn arrival_for(slot: u64, offset_ms: i64) -> i64 { + slot as i64 * 4_000 + offset_ms + } + #[test] fn block_topic_maps_id_to_block_root() { let data = r#"{ "slot": 128, "block": "0xabc123" }"#; - let ev = normalize("node-2", "block", data, 1_000, &timing()).unwrap(); + let arrival = arrival_for(128, 123); + let ev = normalize("node-2", "block", data, arrival, &timing()).unwrap(); assert_eq!(ev.topic, "block"); assert_eq!(ev.slot, 128); assert_eq!(ev.id, Some("0xabc123".to_string())); assert_eq!(ev.validator_id, None); assert_eq!(ev.participants, None); - assert_eq!(ev.offset_ms, 1_000 - 128 * 4_000); + assert_eq!(ev.offset_ms, 123); } #[test] fn block_gossip_topic_maps_id_to_block_root() { let data = r#"{ "slot": 128, "block": "0xabc123" }"#; - let ev = normalize("node-2", "block_gossip", data, 1_000, &timing()).unwrap(); + let arrival = arrival_for(128, 1_000); + let ev = normalize("node-2", "block_gossip", data, arrival, &timing()).unwrap(); assert_eq!(ev.topic, "block_gossip"); assert_eq!(ev.id, Some("0xabc123".to_string())); } @@ -204,7 +244,8 @@ mod tests { #[test] fn head_topic_maps_id_to_block_root_ignoring_state() { let data = r#"{ "slot": 128, "block": "0x1a2b", "state": "0x3c4d" }"#; - let ev = normalize("node-2", "head", data, 2_000, &timing()).unwrap(); + let arrival = arrival_for(128, 2_000); + let ev = normalize("node-2", "head", data, arrival, &timing()).unwrap(); assert_eq!(ev.topic, "head"); assert_eq!(ev.slot, 128); assert_eq!(ev.id, Some("0x1a2b".to_string())); @@ -212,15 +253,18 @@ mod tests { #[test] fn justified_checkpoint_maps_id_to_block_root() { + // Checkpoints trail head, so the arrival sits well past their own slot. let data = r#"{ "slot": 120, "block": "0xaaaa", "state": "0xbbbb" }"#; - let ev = normalize("node-2", "justified_checkpoint", data, 0, &timing()).unwrap(); + let arrival = arrival_for(128, 0); + let ev = normalize("node-2", "justified_checkpoint", data, arrival, &timing()).unwrap(); assert_eq!(ev.id, Some("0xaaaa".to_string())); } #[test] fn finalized_checkpoint_maps_id_to_block_root() { let data = r#"{ "slot": 96, "block": "0xcccc", "state": "0xdddd" }"#; - let ev = normalize("node-2", "finalized_checkpoint", data, 0, &timing()).unwrap(); + let arrival = arrival_for(128, 0); + let ev = normalize("node-2", "finalized_checkpoint", data, arrival, &timing()).unwrap(); assert_eq!(ev.id, Some("0xcccc".to_string())); } @@ -235,7 +279,8 @@ mod tests { "source": {"root": "0xs", "slot": 4} } }"#; - let ev = normalize("node-2", "attestation", data, 0, &timing()).unwrap(); + let arrival = arrival_for(12, 800); + let ev = normalize("node-2", "attestation", data, arrival, &timing()).unwrap(); assert_eq!(ev.topic, "attestation"); assert_eq!(ev.slot, 12); assert_eq!(ev.id, None); @@ -254,7 +299,8 @@ mod tests { "source": {"root": "0xs", "slot": 4} } }"#; - let ev = normalize("node-2", "aggregate", data, 0, &timing()).unwrap(); + let arrival = arrival_for(12, 1_600); + let ev = normalize("node-2", "aggregate", data, arrival, &timing()).unwrap(); assert_eq!(ev.topic, "aggregate"); assert_eq!(ev.slot, 12); assert_eq!(ev.validator_id, None); @@ -284,8 +330,15 @@ mod tests { "source": {"root": "0xs", "slot": 4} } }"#; - let ev_a = normalize("node-2", "aggregate", data_a, 0, &timing()).unwrap(); - let ev_b = normalize("node-3", "aggregate", data_b, 999, &timing()).unwrap(); + let ev_a = normalize("node-2", "aggregate", data_a, arrival_for(12, 0), &timing()).unwrap(); + let ev_b = normalize( + "node-3", + "aggregate", + data_b, + arrival_for(12, 999), + &timing(), + ) + .unwrap(); assert_eq!(ev_a.id, ev_b.id); } @@ -304,8 +357,9 @@ mod tests { }}"# ) }; - let ev_a = normalize("node-2", "aggregate", &base("[0,1,2]"), 0, &timing()).unwrap(); - let ev_b = normalize("node-2", "aggregate", &base("[0,1,3]"), 0, &timing()).unwrap(); + let arrival = arrival_for(12, 1_600); + let ev_a = normalize("node-2", "aggregate", &base("[0,1,2]"), arrival, &timing()).unwrap(); + let ev_b = normalize("node-2", "aggregate", &base("[0,1,3]"), arrival, &timing()).unwrap(); assert_ne!(ev_a.id, ev_b.id); } @@ -329,11 +383,62 @@ mod tests { "source": {"root": "0xs", "slot": 4} } }"#; - let ev_a = normalize("node-2", "aggregate", data_a, 0, &timing()).unwrap(); - let ev_b = normalize("node-2", "aggregate", data_b, 0, &timing()).unwrap(); + let arrival = arrival_for(13, 1_600); + let ev_a = normalize("node-2", "aggregate", data_a, arrival, &timing()).unwrap(); + let ev_b = normalize("node-2", "aggregate", data_b, arrival, &timing()).unwrap(); assert_ne!(ev_a.id, ev_b.id); } + #[test] + fn slot_far_ahead_of_the_collector_clock_is_rejected() { + // A node on a different genesis reports slots wildly ahead of ours. + // Accepting one would ratchet the history/window watermark past every + // real event and blank the dashboard, so it must be dropped. + let data = r#"{ "slot": 900000, "block": "0xdead" }"#; + let err = normalize("node-2", "block", data, arrival_for(128, 0), &timing()).unwrap_err(); + assert!(matches!( + err, + NormalizeError::ImplausibleSlot { slot: 900000, .. } + )); + } + + #[test] + fn slot_slightly_ahead_of_the_collector_clock_is_accepted() { + // Modest clock skew, or an event emitted just before its slot boundary, + // is normal and must still get through. + let t = timing(); + let collector_slot = 128; + for ahead in 0..=MAX_FUTURE_SLOTS { + let data = format!( + r#"{{ "slot": {}, "block": "0xabc" }}"#, + collector_slot + ahead + ); + let arrival = arrival_for(collector_slot, 0); + let ev = normalize("node-2", "block", &data, arrival, &t) + .unwrap_or_else(|err| panic!("{ahead} slots ahead should be accepted: {err}")); + assert_eq!(ev.slot, collector_slot + ahead); + } + + let data = format!( + r#"{{ "slot": {}, "block": "0xabc" }}"#, + collector_slot + MAX_FUTURE_SLOTS + 1 + ); + let arrival = arrival_for(collector_slot, 0); + let err = normalize("node-2", "block", &data, arrival, &t).unwrap_err(); + assert!(matches!(err, NormalizeError::ImplausibleSlot { .. })); + } + + #[test] + fn old_slots_are_always_accepted() { + // Past slots are legitimate and common: finalized/justified checkpoints + // trail head, and a syncing node replays history. They also cannot move + // the watermark, so there is no reason to bound them. + let data = r#"{ "slot": 1, "block": "0xold", "state": "0xstate" }"#; + let arrival = arrival_for(500_000, 0); + let ev = normalize("node-2", "finalized_checkpoint", data, arrival, &timing()).unwrap(); + assert_eq!(ev.slot, 1); + } + #[test] fn unknown_topic_is_an_error_not_a_panic() { let err = normalize("node-2", "mystery", "{}", 0, &timing()).unwrap_err(); diff --git a/tooling/event-monitor/src/timing.rs b/tooling/event-monitor/src/timing.rs index 7893c914..9a57e828 100644 --- a/tooling/event-monitor/src/timing.rs +++ b/tooling/event-monitor/src/timing.rs @@ -41,6 +41,20 @@ impl Timing { let slot_start_ms = self.genesis_time as i64 * 1000 + slot as i64 * self.ms_per_slot as i64; arrival_ms - slot_start_ms } + + /// The slot the collector's own clock is in at `now_ms`, used to bound how + /// far ahead of us an event's slot may plausibly be. Saturates at slot 0 + /// for timestamps at or before genesis. + pub fn slot_at(&self, now_ms: i64) -> u64 { + let genesis_ms = self.genesis_time as i64 * 1000; + let elapsed_ms = now_ms.saturating_sub(genesis_ms); + if elapsed_ms <= 0 { + return 0; + } + // `ms_per_slot` comes off the wire, so treat a bogus 0 as 1ms rather + // than dividing by zero. + elapsed_ms as u64 / self.ms_per_slot.max(1) + } } /// Config-supplied overrides for offline testing (CONTRACT.md §5). @@ -195,4 +209,34 @@ mod tests { let genesis_ms = t.genesis_time as i64 * 1000; assert_eq!(t.offset_ms(0, genesis_ms + 500), 500); } + + #[test] + fn slot_at_counts_whole_slots_since_genesis() { + let t = timing(); + let genesis_ms = t.genesis_time as i64 * 1000; + assert_eq!(t.slot_at(genesis_ms), 0); + assert_eq!(t.slot_at(genesis_ms + 3_999), 0); + assert_eq!(t.slot_at(genesis_ms + 4_000), 1); + assert_eq!(t.slot_at(genesis_ms + 128 * 4_000 + 123), 128); + } + + #[test] + fn slot_at_saturates_at_zero_before_genesis() { + let t = timing(); + let genesis_ms = t.genesis_time as i64 * 1000; + assert_eq!(t.slot_at(genesis_ms - 1), 0); + assert_eq!(t.slot_at(0), 0); + } + + #[test] + fn slot_at_survives_a_bogus_zero_ms_per_slot() { + // `ms_per_slot` comes off the wire, so a malformed spec response must + // not divide by zero. + let t = Timing { + genesis_time: 0, + ms_per_slot: 0, + intervals_per_slot: 5, + }; + assert_eq!(t.slot_at(5_000), 5_000); + } } diff --git a/tooling/event-monitor/web/app.js b/tooling/event-monitor/web/app.js index b6600c72..1032ed84 100644 --- a/tooling/event-monitor/web/app.js +++ b/tooling/event-monitor/web/app.js @@ -141,6 +141,10 @@ async function boot() { const seen = new Set(); const liveBuffer = []; let loading = backfilling; + // Nodes whose status already arrived live while /api/history was in flight. + // Unlike chain events, status is applied immediately rather than buffered, so + // the snapshot (a point-in-time record, possibly older) must not clobber it. + const liveStatusNodes = new Set(); const ingestDeduped = (ev) => { const key = eventKey(ev); @@ -170,6 +174,7 @@ async function boot() { } catch { return; } + if (loading) liveStatusNodes.add(status.node); applyStatus(chips, status); }); @@ -177,7 +182,9 @@ async function boot() { try { const history = await fetchHistory(); if (Array.isArray(history.status)) { - history.status.forEach((s) => applyStatus(chips, s)); + history.status + .filter((s) => !liveStatusNodes.has(s.node)) + .forEach((s) => applyStatus(chips, s)); } if (Array.isArray(history.events)) { history.events.forEach(ingestDeduped); From 1cc932313dceac228332eb97825f539f79fc99d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:22:22 -0300 Subject: [PATCH 5/8] ci: cover the standalone tooling workspaces Each tool under tooling/ declares its own [workspace] table, which is what keeps it decoupled from the root workspace but also means every existing check misses it: `cargo fmt --all`, `cargo check --workspace` and `cargo clippy --workspace` all stop at the root workspace members, as do `make lint` and `make test`. tooling/event-monitor therefore landed with nothing verifying it, and would have stayed green only for as long as someone remembered to run cargo by hand in that directory. Add `tooling-lint` (fmt --check + clippy -D warnings) and `tooling-test`, both looping over `tooling/*/Cargo.toml` so a second tool is picked up without touching the Makefile. `set -e` inside the loop is load-bearing: without it a failing clippy would be masked by the exit status of the next command in the loop body. The `[ -e ]` guard keeps the recipes a clean no-op if the glob ever matches nothing. Wire them into `lint` and `test` rather than leaving them standalone, so the documented pre-commit workflow cannot pass locally while CI fails, and extend `fmt` to format tooling too, so `make fmt` does not leave work that `tooling-lint` then rejects. The CI Lint job calls the same two targets instead of restating the commands, and its rust-cache now lists tooling/event-monitor: separate workspaces have their own target dir and Cargo.lock, so without listing it the tooling build is recompiled from scratch every run and its lockfile does not feed the cache key. Tests run in Lint rather than in the Test job because clippy --all-targets has already compiled them there, and because they need none of the Test job's leanSpec fixtures. --- .github/workflows/ci.yml | 17 +++++++++++++++++ CLAUDE.md | 4 ++++ Makefile | 31 ++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 953b9910..b5c09b80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,14 @@ jobs: components: rustfmt, clippy - name: Setup cache + # Tools under tooling/ are separate Cargo workspaces with their own + # target dir and Cargo.lock, so they need listing explicitly or their + # builds are neither cached nor reflected in the cache key. uses: Swatinem/rust-cache@v2 + with: + workspaces: | + . + tooling/event-monitor - name: Check formatting run: cargo fmt --all -- --check @@ -41,6 +48,16 @@ jobs: - name: Clippy run: cargo clippy --workspace --all-targets -- -D warnings + # tooling/ is outside the root workspace, so the steps above never touch + # it. Its tests run here rather than in the `test` job because clippy has + # already compiled the test targets, and because they need none of that + # job's leanSpec fixtures. + - name: Lint tooling + run: make tooling-lint + + - name: Test tooling + run: make tooling-test + test: name: Test runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index 2a1c99a9..8a4f0468 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,6 +87,10 @@ make lint # Clippy with -D warnings make test # All tests + forkchoice spec tests ``` +Each of the three also covers the standalone tooling workspaces under `tooling/` +(see `tooling-lint` / `tooling-test`), which `--workspace` commands at the repo +root do not reach. CI runs the same two targets in its `Lint` job. + ### Common Operations ```bash .claude/skills/test-pr-devnet/scripts/test-branch.sh # Test branch in multi-client devnet diff --git a/Makefile b/Makefile index d28dc505..7f7e32ff 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,43 @@ -.PHONY: help fmt lint docker-build shadow-build shadow-docker-build run-devnet test docs docs-deps docs-serve +.PHONY: help fmt lint docker-build shadow-build shadow-docker-build run-devnet test tooling-lint tooling-test docs docs-deps docs-serve + +# Each tool under tooling/ is its own Cargo workspace (own [workspace] table and +# Cargo.lock), so `--workspace` commands at the repo root never reach them. They +# have to be driven one manifest at a time. +TOOLING_MANIFESTS = tooling/*/Cargo.toml help: ## 📚 Show help for each of the Makefile recipes @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' fmt: ## 🎨 Format all code using rustfmt cargo fmt --all + @set -e; for manifest in $(TOOLING_MANIFESTS); do \ + [ -e "$$manifest" ] || continue; \ + echo "cargo fmt --manifest-path $$manifest --all"; \ + cargo fmt --manifest-path "$$manifest" --all; \ + done -lint: ## 🔍 Run clippy on all workspace crates +lint: tooling-lint ## 🔍 Run clippy on all workspace crates cargo clippy --workspace --all-targets -- -D warnings -test: leanSpec/fixtures ## 🧪 Run all tests +test: leanSpec/fixtures tooling-test ## 🧪 Run all tests # Tests need to be run on release to avoid stack overflows during signature verification/aggregation cargo test --workspace --release +tooling-lint: ## 🔧 Format-check and clippy the standalone tooling workspaces + @set -e; for manifest in $(TOOLING_MANIFESTS); do \ + [ -e "$$manifest" ] || continue; \ + echo "==> lint $$manifest"; \ + cargo fmt --manifest-path "$$manifest" --all -- --check; \ + cargo clippy --manifest-path "$$manifest" --all-targets -- -D warnings; \ + done + +tooling-test: ## 🔧 Run the standalone tooling workspaces' test suites + @set -e; for manifest in $(TOOLING_MANIFESTS); do \ + [ -e "$$manifest" ] || continue; \ + echo "==> test $$manifest"; \ + cargo test --manifest-path "$$manifest"; \ + done + GIT_COMMIT=$(shell git rev-parse HEAD) GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD) DOCKER_TAG?=local From ed578b382e43973183d963c9a4db7dc5796b6008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:41:55 -0300 Subject: [PATCH 6/8] ci: run the tooling checks inline instead of via make Keeps the Makefile out of it: the tooling checks live entirely in the workflow, so `make fmt` / `lint` / `test` go back to meaning exactly what they meant before and nothing in the root build description knows about tooling/. The two steps now run cargo directly under `working-directory: tooling/event-monitor`. Failure propagation still holds without the explicit `set -e` the Makefile recipes needed, because the runner's default shell for a multi-line `run` is `bash -e`, so a failing `cargo fmt --check` aborts the step before clippy runs (verified: fmt violation exits 1, clippy violation exits 101). The rust-cache `workspaces` entry stays, since caching a separate workspace's target dir and folding its Cargo.lock into the cache key is a workflow concern either way. Trade-off this accepts: `make lint` and `make test` no longer cover tooling, so they can pass locally while CI fails on it. Running cargo inside tooling/event-monitor is the local equivalent. --- .github/workflows/ci.yml | 17 +++++++++++------ CLAUDE.md | 4 ---- Makefile | 31 +++---------------------------- 3 files changed, 14 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5c09b80..4a67a6e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,15 +48,20 @@ jobs: - name: Clippy run: cargo clippy --workspace --all-targets -- -D warnings - # tooling/ is outside the root workspace, so the steps above never touch - # it. Its tests run here rather than in the `test` job because clippy has - # already compiled the test targets, and because they need none of that - # job's leanSpec fixtures. + # tooling/event-monitor declares its own [workspace] table, so every step + # above stops at the root workspace members and never reaches it. Its + # tests run in this job rather than in `test` because clippy has already + # compiled the test targets, and because they need none of that job's + # leanSpec fixtures. - name: Lint tooling - run: make tooling-lint + working-directory: tooling/event-monitor + run: | + cargo fmt --all -- --check + cargo clippy --all-targets -- -D warnings - name: Test tooling - run: make tooling-test + working-directory: tooling/event-monitor + run: cargo test test: name: Test diff --git a/CLAUDE.md b/CLAUDE.md index 8a4f0468..2a1c99a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,10 +87,6 @@ make lint # Clippy with -D warnings make test # All tests + forkchoice spec tests ``` -Each of the three also covers the standalone tooling workspaces under `tooling/` -(see `tooling-lint` / `tooling-test`), which `--workspace` commands at the repo -root do not reach. CI runs the same two targets in its `Lint` job. - ### Common Operations ```bash .claude/skills/test-pr-devnet/scripts/test-branch.sh # Test branch in multi-client devnet diff --git a/Makefile b/Makefile index 7f7e32ff..d28dc505 100644 --- a/Makefile +++ b/Makefile @@ -1,43 +1,18 @@ -.PHONY: help fmt lint docker-build shadow-build shadow-docker-build run-devnet test tooling-lint tooling-test docs docs-deps docs-serve - -# Each tool under tooling/ is its own Cargo workspace (own [workspace] table and -# Cargo.lock), so `--workspace` commands at the repo root never reach them. They -# have to be driven one manifest at a time. -TOOLING_MANIFESTS = tooling/*/Cargo.toml +.PHONY: help fmt lint docker-build shadow-build shadow-docker-build run-devnet test docs docs-deps docs-serve help: ## 📚 Show help for each of the Makefile recipes @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' fmt: ## 🎨 Format all code using rustfmt cargo fmt --all - @set -e; for manifest in $(TOOLING_MANIFESTS); do \ - [ -e "$$manifest" ] || continue; \ - echo "cargo fmt --manifest-path $$manifest --all"; \ - cargo fmt --manifest-path "$$manifest" --all; \ - done -lint: tooling-lint ## 🔍 Run clippy on all workspace crates +lint: ## 🔍 Run clippy on all workspace crates cargo clippy --workspace --all-targets -- -D warnings -test: leanSpec/fixtures tooling-test ## 🧪 Run all tests +test: leanSpec/fixtures ## 🧪 Run all tests # Tests need to be run on release to avoid stack overflows during signature verification/aggregation cargo test --workspace --release -tooling-lint: ## 🔧 Format-check and clippy the standalone tooling workspaces - @set -e; for manifest in $(TOOLING_MANIFESTS); do \ - [ -e "$$manifest" ] || continue; \ - echo "==> lint $$manifest"; \ - cargo fmt --manifest-path "$$manifest" --all -- --check; \ - cargo clippy --manifest-path "$$manifest" --all-targets -- -D warnings; \ - done - -tooling-test: ## 🔧 Run the standalone tooling workspaces' test suites - @set -e; for manifest in $(TOOLING_MANIFESTS); do \ - [ -e "$$manifest" ] || continue; \ - echo "==> test $$manifest"; \ - cargo test --manifest-path "$$manifest"; \ - done - GIT_COMMIT=$(shell git rev-parse HEAD) GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD) DOCKER_TAG?=local From 7cf137d51fba7b31ad5e006c7b2866625bd035b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:04:57 -0300 Subject: [PATCH 7/8] fix(event-monitor): keep offsets honest across a genesis change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slot geometry was resolved once at startup and never revisited. A regenerated genesis is routine on these devnets and silently invalidates every `offset_ms` computed afterwards: nothing errors, every dot just piles up against one edge and the panel quietly lies. Re-resolve every REFRESH_INTERVAL and republish through a watch channel, so collectors pick it up per frame and `/api/meta` per request without a restart. On a change the retained history is dropped. Those events' offsets belong to the previous epoch and their slot numbers to a different chain, so serving them in the same backfill would mix two incomparable series. Dropping them also clears the slot watermark, which matters more than the events: the ring only ever moves that mark upward, so a chain restarted at low slots would otherwise have every one of its events pruned as "older than the retain window". Verified against a stub node: after the regen the ring holds the new chain's low slots with in-slot offsets, which does not happen without the reset. Already-loaded tabs keep the geometry and client-side watermark they fetched, so they need a reload; that is now stated in CONTRACT.md §2 and the README rather than left to be discovered. Widen the arrival axis to two slots at two scales. It clamped to [0, ms_per_slot], which made a block arriving a full slot late look exactly like one landing on the boundary — and spilling past the boundary is the failure mode worth seeing. The first slot keeps full resolution (90.9% of the width at 5 intervals) and the second is compressed into a tinted band half a first-slot interval wide, saturating beyond that. Solving `total = main + main * FRACTION / intervals` for the split keeps the band exactly that fraction of an interval regardless of canvas width; checked numerically for monotonicity and endpoints. Document that `offset_ms` includes the collector<->node round trip. One clock is deliberate and is what makes propagation deltas skew-free, since the events carry no node-side timestamp, but it means nodes reached over different links carry a systematic offset that reads as lag. Better stated than silently over-read. Share retained events as Arc. Each event was deep-copied once into the ring and once per broadcast receiver, and `/api/history` cloned up to HISTORY_MAX_EVENTS of them while holding the mutex every collector needs to publish. Serde's `rc` feature keeps the wire shape identical, as the contract requires. Reject `topics = []` and `nodes = []` at load. An empty topic list produces `?topics=`, which every node answers with 400, so the collector retried forever without ever saying why. Also drop tokio's `signal` feature, which was enabled but unused, and fix the DEFAULT_INTERVALS_PER_SLOT comment, which described a condition ("config didn't need a network fetch at all") that `bootstrap` never has, since it always fetches. --- tooling/event-monitor/CONTRACT.md | 57 +++++++++++-- tooling/event-monitor/Cargo.lock | 21 ----- tooling/event-monitor/Cargo.toml | 8 +- tooling/event-monitor/README.md | 14 +++ tooling/event-monitor/src/collector.rs | 15 ++-- tooling/event-monitor/src/config.rs | 85 ++++++++++++++++++- tooling/event-monitor/src/hub.rs | 66 ++++++++++++-- tooling/event-monitor/src/main.rs | 20 +++-- tooling/event-monitor/src/server.rs | 49 ++++++++--- tooling/event-monitor/src/timing.rs | 75 ++++++++++++++-- .../event-monitor/tests/sse_integration.rs | 10 ++- tooling/event-monitor/web/beeswarm.js | 64 +++++++++++++- tooling/event-monitor/web/style.css | 4 + 13 files changed, 414 insertions(+), 74 deletions(-) diff --git a/tooling/event-monitor/CONTRACT.md b/tooling/event-monitor/CONTRACT.md index 0731c681..564598b0 100644 --- a/tooling/event-monitor/CONTRACT.md +++ b/tooling/event-monitor/CONTRACT.md @@ -79,6 +79,24 @@ offset_ms = arrival_ms - slot_start_ms // may be negative under clock (`SystemTime::now()` → epoch ms). Config may override `genesis_time` / `ms_per_slot` for offline testing. +**What `offset_ms` actually measures.** It is stamped when the *collector* +receives the frame, so it includes the collector↔node round trip, not just the +node's own event time. One clock is deliberate: it makes propagation deltas +skew-free, which no per-node timestamp could (the events carry none). The +consequence is that nodes reached over different links (loopback vs a WAN +tunnel) carry a systematic per-node offset that reads as lag in both panels. +Cross-node comparisons are only as good as the comparability of those paths. + +**Refresh.** Geometry is re-resolved every `REFRESH_INTERVAL` (60s) and +republished when it changes, so a regenerated genesis does not silently +invalidate every subsequent `offset_ms`. On a change the collector drops its +retained history and slot watermark: those events were computed against the old +epoch and their slot numbers belong to a different chain. A failed refresh keeps +the current geometry. Already-loaded browser tabs keep the `ms_per_slot` / +`intervals_per_slot` they fetched from `/api/meta` plus their own client-side +watermark, so **reload the page after a genesis change**; server-computed +`offset_ms` values are correct immediately. + ### Slot plausibility bound Once slot geometry is known, the collector drops any event whose `slot` is more @@ -230,14 +248,39 @@ Single dark/light-adaptive page, no framework, no build step. Fetches that live-adjusts the rolling `window_slots` for **both** panels (clamped `1..500`). Client-side only; no collector restart. -**Top — rolling beeswarm** (canvas): x-axis `0 … ms_per_slot`, gridlines at every -`ms_per_slot / intervals_per_slot`. One horizontal lane per node. Each incoming +**Top — rolling beeswarm** (canvas): one horizontal lane per node. Each incoming `chain` event for topic `block`/`attestation`/`aggregate` is a dot at -`x = clamp(offset_ms, 0, ms_per_slot)`, small vertical jitter within its lane, -colored by topic: block `#4f8cff`, attestation `#37b24d`, aggregate `#f59f00`. -Keep only events from the last `window_slots` slots; older dots fade then drop. -Cap points per node (e.g. 2000) with oldest-first decimation so an attestation -flood can't wedge rendering. Legend + a note that older slots fade. +`x = xForOffset(clamp(offset_ms, 0, 2 * ms_per_slot))`, small vertical jitter +within its lane, colored by topic: block `#4f8cff`, attestation `#37b24d`, +aggregate `#f59f00`. Keep only events from the last `window_slots` slots; older +dots fade then drop. Cap points per node (e.g. 2000) with oldest-first +decimation so an attestation flood can't wedge rendering. Legend + a note that +older slots fade. + +The x-axis spans **two slots at two scales**. The first slot occupies the bulk +of the width at full resolution with gridlines every +`ms_per_slot / intervals_per_slot`; the second is compressed into a tinted +overflow band whose width is `OVERFLOW_INTERVAL_FRACTION` (0.5) of one +first-slot interval, i.e. the first slot keeps `1 / (1 + 0.5/intervals)` of the +plot width (90.9% at 5 intervals). Solve +`total = main + main * FRACTION / intervals` for the split: + +``` +offset_ms <= ms_per_slot : x = LEFT + (offset_ms / ms_per_slot) * main +offset_ms > ms_per_slot : x = LEFT + main + + (min(offset_ms - ms_per_slot, ms_per_slot) / ms_per_slot) * overflow +``` + +An arrival that spills past its slot boundary is the failure mode worth seeing, +so it gets a real position rather than piling up against a clamp, without +costing resolution where every healthy event lands. Beyond two slots it +saturates at the right edge. The slot boundary is drawn in the overflow accent +(`--prop-over`) over the gridlines, and only the far end of the band is labelled +(`+ms_per_slot`). + +The propagation panel keeps its single fixed `0 … ms_per_slot` scale: it already +distinguishes over-one-slot deltas by color (`--prop-over`), so it needs no +second region. **Bottom — propagation delta** (canvas beeswarm): a topic toggle (`block` default / `aggregate`). Group events of that topic by `id`; diff --git a/tooling/event-monitor/Cargo.lock b/tooling/event-monitor/Cargo.lock index 28d942d4..049670a3 100644 --- a/tooling/event-monitor/Cargo.lock +++ b/tooling/event-monitor/Cargo.lock @@ -320,16 +320,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "event-monitor" version = "0.1.0" @@ -1358,16 +1348,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" -[[package]] -name = "signal-hook-registry" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" -dependencies = [ - "errno", - "libc", -] - [[package]] name = "simd_cesu8" version = "1.2.0" @@ -1530,7 +1510,6 @@ dependencies = [ "libc", "mio", "pin-project-lite", - "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", diff --git a/tooling/event-monitor/Cargo.toml b/tooling/event-monitor/Cargo.toml index 5c7862c1..5854c932 100644 --- a/tooling/event-monitor/Cargo.toml +++ b/tooling/event-monitor/Cargo.toml @@ -21,10 +21,14 @@ clap = { version = "4.6.3", features = ["derive"] } eventsource-stream = "0.2.3" futures-util = "0.3.33" reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "stream", "json"] } -serde = { version = "1.0.229", features = ["derive"] } +# `rc` lets the history ring hold Arc and still serialize to +# the same JSON as a /stream event, so backfill shares one allocation per event +# instead of deep-copying it (Arc sharing is not preserved on the wire, which is +# exactly what we want here). +serde = { version = "1.0.229", features = ["derive", "rc"] } serde_json = "1.0.151" thiserror = "2.0.19" -tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "signal", "time", "net", "sync", "fs"] } +tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "time", "net", "sync", "fs"] } tokio-stream = { version = "0.1.18", features = ["sync"] } toml = "1.1.3" tower-http = { version = "0.7.0", features = ["fs"] } diff --git a/tooling/event-monitor/README.md b/tooling/event-monitor/README.md index 943fa330..93bf0807 100644 --- a/tooling/event-monitor/README.md +++ b/tooling/event-monitor/README.md @@ -14,6 +14,20 @@ Standalone: its own Cargo workspace, **no dependency on any ethlambda crate** it only speaks the documented SSE/HTTP wire shape. See [`CONTRACT.md`](./CONTRACT.md) for the authoritative interface between the Rust backend and the JS frontend. +## Reading the numbers + +Two caveats worth knowing before drawing conclusions from a panel: + +- Arrival is timestamped **on the collector**, so every offset includes the + collector↔node round trip. That single clock is what makes propagation deltas + skew-free, but it also means nodes reached over different links (loopback vs a + WAN tunnel) carry a systematic offset that looks like lag. Compare nodes whose + paths are comparable. +- The collector re-resolves slot geometry every 60s and drops its retained + history if `genesis_time` changes, so a regenerated genesis no longer corrupts + offsets silently. An already-open dashboard still needs a **page reload** after + such a change to pick up the new geometry and reset its own slot watermark. + ## Run ```bash diff --git a/tooling/event-monitor/src/collector.rs b/tooling/event-monitor/src/collector.rs index 7df0a4ad..bb6b9aec 100644 --- a/tooling/event-monitor/src/collector.rs +++ b/tooling/event-monitor/src/collector.rs @@ -3,11 +3,11 @@ //! capped exponential backoff and reports connection state changes plus a //! periodic heartbeat (CONTRACT.md §2, §4). -use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use eventsource_stream::{Event as SseEvent, Eventsource}; use futures_util::StreamExt; +use tokio::sync::watch; use crate::config::NodeConfig; use crate::hub::Hub; @@ -140,7 +140,7 @@ fn node_status(name: &str, state: NodeState, events_per_sec: f64) -> NodeStatus pub async fn run_collector( node: NodeConfig, topics: Vec, - timing: Arc, + timing: watch::Receiver, hub: Hub, client: reqwest::Client, ) { @@ -174,7 +174,7 @@ pub async fn run_collector( async fn connect_and_stream( node: &NodeConfig, topics: &[String], - timing: &Timing, + timing: &watch::Receiver, hub: &Hub, client: &reqwest::Client, ) -> Attempt { @@ -210,7 +210,10 @@ async fn connect_and_stream( match frame { Some(Ok(event)) => { rate.tick(); - handle_frame(node, &event, timing, hub, &mut warned_implausible_slot); + // Re-read per frame so a geometry refresh takes effect + // without tearing down the connection. + let geometry = *timing.borrow(); + handle_frame(node, &event, geometry, hub, &mut warned_implausible_slot); } Some(Err(err)) => { let error = Some(CollectorError::Stream(err.to_string())); @@ -243,7 +246,7 @@ async fn connect_and_stream( fn handle_frame( node: &NodeConfig, event: &SseEvent, - timing: &Timing, + timing: Timing, hub: &Hub, warned_implausible_slot: &mut bool, ) { @@ -252,7 +255,7 @@ fn handle_frame( if event.data.is_empty() { return; } - match model::normalize(&node.name, &event.event, &event.data, now_ms(), timing) { + match model::normalize(&node.name, &event.event, &event.data, now_ms(), &timing) { Ok(normalized) => hub.publish_chain(normalized), Err(err @ NormalizeError::ImplausibleSlot { .. }) => { if !*warned_implausible_slot { diff --git a/tooling/event-monitor/src/config.rs b/tooling/event-monitor/src/config.rs index 606df950..c6e07ea2 100644 --- a/tooling/event-monitor/src/config.rs +++ b/tooling/event-monitor/src/config.rs @@ -84,6 +84,8 @@ pub enum ConfigError { #[source] source: Box, }, + #[error("config file {path} is invalid: {reason}")] + Invalid { path: String, reason: String }, } impl Config { @@ -92,10 +94,34 @@ impl Config { path: path.display().to_string(), source, })?; - toml::from_str(&raw).map_err(|source| ConfigError::Parse { + let config: Config = toml::from_str(&raw).map_err(|source| ConfigError::Parse { path: path.display().to_string(), source: Box::new(source), - }) + })?; + config.validate(path)?; + Ok(config) + } + + /// Rejects configs that would only fail later as an opaque retry loop. + fn validate(&self, path: &Path) -> Result<(), ConfigError> { + let invalid = |reason: &str| ConfigError::Invalid { + path: path.display().to_string(), + reason: reason.to_string(), + }; + // Upstream requires a non-empty `topics`: an empty list produces + // `?topics=`, which every node answers with 400, so the collector would + // otherwise retry forever without ever saying why. + if self.topics.is_empty() { + return Err(invalid( + "`topics` is empty; list at least one topic to subscribe to", + )); + } + if self.nodes.is_empty() { + return Err(invalid( + "`nodes` is empty; add at least one [[nodes]] entry", + )); + } + Ok(()) } pub fn timing_overrides(&self) -> TimingOverrides { @@ -129,6 +155,61 @@ mod tests { assert!(cfg.ms_per_slot.is_none()); } + fn parse(toml_str: &str) -> Config { + toml::from_str(toml_str).expect("fixture should parse") + } + + #[test] + fn empty_topics_is_rejected_rather_than_retried_forever() { + let cfg = parse( + r#" + listen = "127.0.0.1:8080" + topics = [] + + [[nodes]] + name = "node-2" + url = "http://127.0.0.1:5052" + "#, + ); + let err = cfg.validate(Path::new("config.toml")).unwrap_err(); + assert!(matches!(err, ConfigError::Invalid { .. })); + } + + #[test] + fn empty_nodes_is_rejected() { + // `nodes` has no serde default, so omitting it fails deserialization + // before validation ever runs; an explicit empty list is the only way to + // express this, and it is what the check exists for. + let cfg = parse( + r#" + listen = "127.0.0.1:8080" + nodes = [] + "#, + ); + let err = cfg.validate(Path::new("config.toml")).unwrap_err(); + assert!(matches!(err, ConfigError::Invalid { .. })); + } + + #[test] + fn a_config_without_a_nodes_key_fails_to_parse() { + let err = toml::from_str::(r#"listen = "127.0.0.1:8080""#).unwrap_err(); + assert!(err.to_string().contains("nodes"), "unexpected error: {err}"); + } + + #[test] + fn a_populated_config_passes_validation() { + let cfg = parse( + r#" + listen = "127.0.0.1:8080" + + [[nodes]] + name = "node-2" + url = "http://127.0.0.1:5052" + "#, + ); + assert!(cfg.validate(Path::new("config.toml")).is_ok()); + } + #[test] fn overrides_and_multiple_nodes_parse() { let toml_str = r#" diff --git a/tooling/event-monitor/src/hub.rs b/tooling/event-monitor/src/hub.rs index c09fd2ca..79ef096d 100644 --- a/tooling/event-monitor/src/hub.rs +++ b/tooling/event-monitor/src/hub.rs @@ -23,18 +23,24 @@ const HISTORY_MAX_EVENTS: usize = 50_000; /// One message on the hub: either a normalized chain event (`event: chain`) /// or a node status update (`event: status`) per CONTRACT.md §4. +/// +/// Chain events are `Arc`-wrapped because every one is fanned out to each +/// `/stream` subscriber *and* retained in the history ring; sharing one +/// allocation keeps that from being a deep copy per destination. #[derive(Debug, Clone)] pub enum HubMessage { - Chain(NormalizedEvent), + Chain(Arc), Status(NodeStatus), } /// Point-in-time backfill payload served by `GET /api/history`: the retained /// recent chain events plus the latest status per node (CONTRACT.md §4). Its /// field names match that endpoint's JSON exactly, so it is serialized directly. +/// `Arc` is transparent to serde (the `rc` feature), so the wire shape is +/// identical to a `/stream` `chain` event as the contract requires. #[derive(Debug, Clone, Default, Serialize)] pub struct HistorySnapshot { - pub events: Vec, + pub events: Vec>, pub status: Vec, } @@ -42,7 +48,7 @@ pub struct HistorySnapshot { /// (relative to the newest slot seen) and hard-capped at /// [`HISTORY_MAX_EVENTS`], plus the latest status per node. struct History { - events: VecDeque, + events: VecDeque>, status: BTreeMap, max_slot: u64, retain_slots: u64, @@ -58,7 +64,7 @@ impl History { } } - fn push_event(&mut self, event: NormalizedEvent) { + fn push_event(&mut self, event: Arc) { self.max_slot = self.max_slot.max(event.slot); self.events.push_back(event); self.prune(); @@ -68,6 +74,13 @@ impl History { self.status.insert(status.node.clone(), status); } + /// Drops every retained event and the slot watermark, keeping per-node + /// status (a connection's state survives a geometry change). + fn reset(&mut self) { + self.events.clear(); + self.max_slot = 0; + } + /// Drops events older than `retain_slots` relative to the newest slot /// seen, then enforces the hard event cap from the front (oldest first). /// Events arrive in roughly slot order across nodes/topics, so scanning @@ -85,9 +98,12 @@ impl History { } } + /// Cheap because the events are `Arc`s: this copies pointers, not the + /// events themselves, which matters because the caller holds the mutex + /// across it while every collector is trying to publish. fn snapshot(&self) -> HistorySnapshot { HistorySnapshot { - events: self.events.iter().cloned().collect(), + events: self.events.iter().map(Arc::clone).collect(), status: self.status.values().cloned().collect(), } } @@ -118,8 +134,9 @@ impl Hub { /// frontend de-dups the small overlap). Ignores the "no subscribers" /// send error: normal when no browser is connected yet. pub fn publish_chain(&self, event: NormalizedEvent) { + let event = Arc::new(event); if let Ok(mut history) = self.history.lock() { - history.push_event(event.clone()); + history.push_event(Arc::clone(&event)); } let _ = self.tx.send(HubMessage::Chain(event)); } @@ -146,6 +163,16 @@ impl Hub { .map(|history| history.snapshot()) .unwrap_or_default() } + + /// Drops retained events and the slot watermark. Called when slot geometry + /// changes, since events carrying the old epoch's `offset_ms` and the old + /// chain's slot numbers are not comparable with what follows + /// ([`crate::timing::run_refresher`]). + pub fn reset_history(&self) { + if let Ok(mut history) = self.history.lock() { + history.reset(); + } + } } #[cfg(test)] @@ -218,6 +245,33 @@ mod tests { assert_eq!(snap.status[0].state, NodeState::Connected); } + #[test] + fn reset_history_drops_events_and_the_slot_watermark_but_keeps_status() { + let hub = Hub::new(5); + hub.publish_status(NodeStatus { + node: "node-0".to_string(), + state: NodeState::Connected, + events_per_sec: 3.0, + }); + for slot in 1_000..1_010 { + hub.publish_chain(chain_event("node-0", slot)); + } + hub.reset_history(); + + let snap = hub.history_snapshot(); + assert!(snap.events.is_empty()); + // Status survives: a connection's state is unaffected by geometry. + assert_eq!(snap.status.len(), 1); + + // The watermark reset is the point: a chain restarted at low slots must + // not be pruned as "older than the retain window" by the old high mark. + hub.publish_chain(chain_event("node-0", 1)); + hub.publish_chain(chain_event("node-0", 2)); + let snap = hub.history_snapshot(); + assert_eq!(snap.events.len(), 2); + assert_eq!(snap.events[0].slot, 1); + } + #[test] fn history_prunes_events_older_than_the_retain_window() { let hub = Hub::new(5); // retain 5 slots diff --git a/tooling/event-monitor/src/main.rs b/tooling/event-monitor/src/main.rs index b946ab34..117e62a0 100644 --- a/tooling/event-monitor/src/main.rs +++ b/tooling/event-monitor/src/main.rs @@ -1,8 +1,8 @@ use std::path::{Path, PathBuf}; -use std::sync::Arc; use clap::Parser; use tokio::net::TcpListener; +use tokio::sync::watch; use event_monitor::config::Config; use event_monitor::hub::Hub; @@ -42,22 +42,32 @@ async fn main() -> anyhow::Result<()> { intervals_per_slot = timing.intervals_per_slot, "resolved slot geometry" ); - let timing = Arc::new(timing); + // Geometry is shared by watch channel rather than a plain Arc so a + // regenerated genesis can be picked up without a restart: the refresher + // publishes, collectors re-read per frame, and `/api/meta` reads per request. + let (timing_tx, timing_rx) = watch::channel(timing); let hub = Hub::new(config.history_slots as u64); for node in &config.nodes { tokio::spawn(collector::run_collector( node.clone(), config.topics.clone(), - timing.clone(), + timing_rx.clone(), hub.clone(), client.clone(), )); } + tokio::spawn(timing::run_refresher( + config.nodes.clone(), + config.timing_overrides(), + client.clone(), + timing_tx, + hub.clone(), + )); - let meta = server::Meta::new(&config, &timing); + let meta = server::MetaConfig::new(&config); let static_dir = Path::new(&config.static_dir).to_path_buf(); - let app = server::build_router(hub, meta, &static_dir); + let app = server::build_router(hub, meta, timing_rx, &static_dir); let listen_addr = config.listen; let listener = TcpListener::bind(listen_addr).await?; diff --git a/tooling/event-monitor/src/server.rs b/tooling/event-monitor/src/server.rs index f4b0a456..0c93deba 100644 --- a/tooling/event-monitor/src/server.rs +++ b/tooling/event-monitor/src/server.rs @@ -12,6 +12,7 @@ use axum::response::sse::{Event, KeepAlive, Sse}; use axum::routing::get; use futures_util::{Stream, StreamExt}; use serde::Serialize; +use tokio::sync::watch; use tokio_stream::wrappers::BroadcastStream; use tower_http::services::{ServeDir, ServeFile}; @@ -24,9 +25,6 @@ use crate::timing::Timing; const SSE_KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(15); /// One-shot bootstrap payload the frontend fetches on load (CONTRACT.md §4). -/// Small and cheap to clone per-request, so `AppState` holds it directly -/// rather than behind an `Arc` (serde's `Serialize` isn't derived for -/// `Arc` without the optional `rc` feature). #[derive(Debug, Clone, Serialize)] pub struct Meta { pub genesis_time: u64, @@ -37,29 +35,54 @@ pub struct Meta { pub nodes: Vec, } -impl Meta { - pub fn new(config: &Config, timing: &Timing) -> Self { +/// The config-derived half of [`Meta`]. The timing half is read live per +/// request from the geometry watch channel, so a refresh +/// ([`crate::timing::run_refresher`]) reaches the next page load without a +/// restart. +#[derive(Debug, Clone)] +pub struct MetaConfig { + window_slots: u32, + topics: Vec, + nodes: Vec, +} + +impl MetaConfig { + pub fn new(config: &Config) -> Self { Self { - genesis_time: timing.genesis_time, - ms_per_slot: timing.ms_per_slot, - intervals_per_slot: timing.intervals_per_slot, window_slots: config.window_slots, topics: config.topics.clone(), nodes: config.nodes.clone(), } } + + fn with_timing(&self, timing: Timing) -> Meta { + Meta { + genesis_time: timing.genesis_time, + ms_per_slot: timing.ms_per_slot, + intervals_per_slot: timing.intervals_per_slot, + window_slots: self.window_slots, + topics: self.topics.clone(), + nodes: self.nodes.clone(), + } + } } #[derive(Clone)] struct AppState { hub: Hub, - meta: Meta, + meta: MetaConfig, + timing: watch::Receiver, } /// Builds the full axum app: `/stream`, `/api/meta`, `/api/history`, and /// static file serving (with an `index.html` fallback) rooted at `static_dir`. -pub fn build_router(hub: Hub, meta: Meta, static_dir: &Path) -> Router { - let state = AppState { hub, meta }; +pub fn build_router( + hub: Hub, + meta: MetaConfig, + timing: watch::Receiver, + static_dir: &Path, +) -> Router { + let state = AppState { hub, meta, timing }; let index_html = static_dir.join("index.html"); let serve_dir = ServeDir::new(static_dir).fallback(ServeFile::new(index_html)); @@ -73,7 +96,9 @@ pub fn build_router(hub: Hub, meta: Meta, static_dir: &Path) -> Router { } async fn meta_handler(State(state): State) -> Json { - Json(state.meta.clone()) + // Copy the geometry out immediately rather than holding the watch borrow. + let timing = *state.timing.borrow(); + Json(state.meta.with_timing(timing)) } /// Backfill for `GET /api/history` (CONTRACT.md §4): recent chain events (each diff --git a/tooling/event-monitor/src/timing.rs b/tooling/event-monitor/src/timing.rs index 9a57e828..d2bda6d3 100644 --- a/tooling/event-monitor/src/timing.rs +++ b/tooling/event-monitor/src/timing.rs @@ -12,19 +12,27 @@ use std::time::Duration; use serde::Deserialize; +use tokio::sync::watch; use crate::config::NodeConfig; +use crate::hub::Hub; -/// Fallback used only when no node answered `/lean/v0/config/spec` and the -/// config didn't need a network fetch at all (both `genesis_time` and -/// `ms_per_slot` overridden). Matches ethlambda's own default (5 intervals -/// per 4s slot). +/// Fallback used when no node answered `/lean/v0/config/spec`. Matches +/// ethlambda's own default (5 intervals per 4s slot). Unlike `genesis_time` and +/// `ms_per_slot` this has no config override, so it is the only source left when +/// the fetch comes back empty. pub const DEFAULT_INTERVALS_PER_SLOT: u64 = 5; const FETCH_TIMEOUT: Duration = Duration::from_secs(3); +/// How often slot geometry is re-resolved. A regenerated genesis is routine on +/// a devnet and silently invalidates every `offset_ms` computed against the old +/// epoch, so poll for it rather than relying on someone noticing that every dot +/// has piled up against one edge. +const REFRESH_INTERVAL: Duration = Duration::from_secs(60); + /// Resolved slot geometry used to compute `offset_ms` for incoming events. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Timing { pub genesis_time: u64, pub ms_per_slot: u64, @@ -168,6 +176,63 @@ pub async fn bootstrap( }) } +/// Re-resolves slot geometry every [`REFRESH_INTERVAL`] and republishes it on +/// `timing_tx` when it changes, so a regenerated genesis stops silently +/// corrupting every subsequent `offset_ms`. +/// +/// On a change the retained history is dropped: those events' offsets were +/// computed against the previous epoch and their slot numbers belong to a +/// different chain, so keeping them would mix two incomparable series in one +/// backfill. Dropping them also clears the slot watermark, which the history +/// ring only ever moves upward and which a restarted chain's low slots would +/// otherwise sit below. +/// +/// A failed refresh is logged and ignored: the current geometry is better than +/// none, and an unreachable node is expected during a rolling restart. +/// +/// Already-loaded browser tabs keep the `ms_per_slot` / `intervals_per_slot` +/// they fetched from `/api/meta`, and their own client-side slot watermark, so +/// they need a reload after a geometry change. Server-computed `offset_ms` +/// values are correct immediately. +pub async fn run_refresher( + nodes: Vec, + overrides: TimingOverrides, + client: reqwest::Client, + timing_tx: watch::Sender, + hub: Hub, +) { + let mut ticker = tokio::time::interval(REFRESH_INTERVAL); + ticker.tick().await; // the first tick fires immediately; bootstrap just ran + + loop { + ticker.tick().await; + + let fresh = match bootstrap(&nodes, overrides, &client).await { + Ok(fresh) => fresh, + Err(err) => { + tracing::debug!(%err, "slot geometry refresh failed; keeping current geometry"); + continue; + } + }; + + let current = *timing_tx.borrow(); + if fresh == current { + continue; + } + + tracing::warn!( + old_genesis_time = current.genesis_time, + new_genesis_time = fresh.genesis_time, + old_ms_per_slot = current.ms_per_slot, + new_ms_per_slot = fresh.ms_per_slot, + "slot geometry changed; dropping retained history and re-basing offsets" + ); + hub.reset_history(); + // A send error means every receiver is gone, i.e. we are shutting down. + let _ = timing_tx.send(fresh); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/tooling/event-monitor/tests/sse_integration.rs b/tooling/event-monitor/tests/sse_integration.rs index 2b44217b..f5637dc3 100644 --- a/tooling/event-monitor/tests/sse_integration.rs +++ b/tooling/event-monitor/tests/sse_integration.rs @@ -36,11 +36,13 @@ async fn fake_events_handler() -> Sse, want: usize, -) -> Vec { +) -> Vec> { let mut collected = Vec::with_capacity(want); timeout(RECV_TIMEOUT, async { while collected.len() < want { @@ -74,7 +76,7 @@ async fn collector_normalizes_frames_from_a_live_sse_server() { name: "fake-node".to_string(), url: format!("http://{addr}"), }; - let timing = Arc::new(Timing { + let (_timing_tx, timing) = tokio::sync::watch::channel(Timing { genesis_time: 0, ms_per_slot: 4_000, intervals_per_slot: 5, @@ -134,7 +136,7 @@ async fn collector_publishes_connected_status_on_success() { name: "fake-node-2".to_string(), url: format!("http://{addr}"), }; - let timing = Arc::new(Timing { + let (_timing_tx, timing) = tokio::sync::watch::channel(Timing { genesis_time: 0, ms_per_slot: 4_000, intervals_per_slot: 5, diff --git a/tooling/event-monitor/web/beeswarm.js b/tooling/event-monitor/web/beeswarm.js index 605657dc..199f2e89 100644 --- a/tooling/event-monitor/web/beeswarm.js +++ b/tooling/event-monitor/web/beeswarm.js @@ -5,6 +5,15 @@ const TOPICS = ["block", "attestation", "aggregate"]; const MAX_POINTS_PER_NODE = 2000; +// The x-axis covers two slots, but not at one scale. The first slot gets the +// bulk of the width at full resolution; the second is compressed into a narrow +// overflow band whose width is OVERFLOW_INTERVAL_FRACTION of one first-slot +// interval. Arrivals that spill past their slot boundary are the interesting +// failure mode, so they need real positions rather than piling up against a +// clamp, but they should not cost resolution in the region where every healthy +// event lands. Anything beyond two slots saturates at the right edge. +const OVERFLOW_INTERVAL_FRACTION = 0.5; + const LANE_HEIGHT = 40; const TOP_MARGIN = 14; const BOTTOM_MARGIN = 30; @@ -42,6 +51,8 @@ function readTheme() { grid: get("--grid", "#d0d3d9"), text: get("--muted", "#6b7280"), laneAlt: get("--lane-alt", "rgba(0,0,0,0.04)"), + overflowBand: get("--overflow-band", "rgba(230,73,128,0.07)"), + overflowEdge: get("--prop-over", "#e64980"), topics: { block: get("--topic-block", "#4f8cff"), attestation: get("--topic-attestation", "#37b24d"), @@ -103,7 +114,9 @@ export function createBeeswarm({ canvas, legendEl, noteEl, meta }) { const arr = perNode.get(ev.node); if (!arr) return; // event from a node not in meta.nodes; ignore defensively if (ev.slot > maxSlotSeen) maxSlotSeen = ev.slot; - const offsetMs = Math.min(Math.max(ev.offset_ms, 0), msPerSlot); + // Clamp to the full two-slot span, not to one slot: the overflow band is + // what makes a late arrival distinguishable from an on-boundary one. + const offsetMs = Math.min(Math.max(ev.offset_ms, 0), msPerSlot * 2); const seed = `${ev.node}|${ev.topic}|${ev.slot}|${ev.id ?? ""}|${ev.validator_id ?? ""}|${ev.arrival_ms}`; arr.push({ topic: ev.topic, @@ -118,9 +131,25 @@ export function createBeeswarm({ canvas, legendEl, noteEl, meta }) { } } + // Splits the available width between the full-resolution first slot and the + // compressed overflow band. Solving + // total = main + main * FRACTION / intervals + // keeps the band exactly FRACTION of one first-slot interval wide. + function axis() { + const total = cssWidth - LEFT_MARGIN - RIGHT_MARGIN; + const main = total / (1 + OVERFLOW_INTERVAL_FRACTION / intervals); + return { main, overflow: total - main }; + } + + // Piecewise-linear: linear within the slot, then compressed across the + // second slot. `offsetMs` is expected pre-clamped to [0, 2 * msPerSlot]. function xForOffset(offsetMs) { - const plotWidth = cssWidth - LEFT_MARGIN - RIGHT_MARGIN; - return LEFT_MARGIN + (offsetMs / msPerSlot) * plotWidth; + const { main, overflow } = axis(); + if (offsetMs <= msPerSlot) { + return LEFT_MARGIN + (Math.max(offsetMs, 0) / msPerSlot) * main; + } + const over = Math.min(offsetMs - msPerSlot, msPerSlot); + return LEFT_MARGIN + main + (over / msPerSlot) * overflow; } function laneY(nodeName) { @@ -149,6 +178,13 @@ export function createBeeswarm({ canvas, legendEl, noteEl, meta }) { } }); + // Tint the compressed overflow band so its different scale is visible + // before anyone reads the axis labels. + const boundaryX = xForOffset(msPerSlot); + const rightEdge = cssWidth - RIGHT_MARGIN; + ctx.fillStyle = theme.overflowBand; + ctx.fillRect(boundaryX, TOP_MARGIN, rightEdge - boundaryX, plotBottom - TOP_MARGIN); + // gridlines every ms_per_slot / intervals_per_slot, plus axis labels const step = msPerSlot / intervals; ctx.strokeStyle = theme.grid; @@ -165,8 +201,24 @@ export function createBeeswarm({ canvas, legendEl, noteEl, meta }) { ctx.stroke(); ctx.fillText(`${Math.round(ms)}`, x, plotBottom + 6); } + + // The slot boundary is drawn over the gridlines in the overflow accent, so + // "spilled into the next slot" reads at a glance. + ctx.strokeStyle = theme.overflowEdge; + ctx.beginPath(); + ctx.moveTo(boundaryX, TOP_MARGIN); + ctx.lineTo(boundaryX, plotBottom); + ctx.stroke(); + + // Only the far end of the band is labelled; there is no room for more. + ctx.fillStyle = theme.text; + ctx.textAlign = "right"; + ctx.fillText(`+${msPerSlot}`, rightEdge, plotBottom + 6); + ctx.textAlign = "left"; ctx.fillText("ms into slot", LEFT_MARGIN, plotBottom + 18); + ctx.textAlign = "right"; + ctx.fillText("next slot (compressed)", rightEdge, plotBottom + 18); // lane labels ctx.textAlign = "right"; @@ -238,7 +290,11 @@ export function createBeeswarm({ canvas, legendEl, noteEl, meta }) { function updateNote() { if (noteEl) { - noteEl.textContent = `Showing the last ${windowSlots} slots. Older slots fade out, then drop.`; + noteEl.textContent = + `Showing the last ${windowSlots} slots. Older slots fade out, then drop. ` + + `Past the slot boundary the axis compresses the next full slot into the ` + + `tinted band, so late arrivals stay distinguishable; beyond that they ` + + `saturate at the right edge.`; } } diff --git a/tooling/event-monitor/web/style.css b/tooling/event-monitor/web/style.css index 1787042b..e9edd37b 100644 --- a/tooling/event-monitor/web/style.css +++ b/tooling/event-monitor/web/style.css @@ -12,6 +12,7 @@ --muted-2: #8b93a1; --grid: #e4e7ec; --lane-alt: rgba(80, 96, 120, 0.045); + --overflow-band: rgba(230, 73, 128, 0.06); --accent: #4f8cff; --chip-bg: #f0f2f5; @@ -42,6 +43,7 @@ --muted-2: #737d8c; --grid: #262a33; --lane-alt: rgba(160, 180, 220, 0.05); + --overflow-band: rgba(230, 73, 128, 0.10); --accent: #6ea1ff; --chip-bg: #1b1f27; } @@ -57,6 +59,7 @@ --muted-2: #737d8c; --grid: #262a33; --lane-alt: rgba(160, 180, 220, 0.05); + --overflow-band: rgba(230, 73, 128, 0.10); --accent: #6ea1ff; --chip-bg: #1b1f27; } @@ -70,6 +73,7 @@ --muted-2: #8b93a1; --grid: #e4e7ec; --lane-alt: rgba(80, 96, 120, 0.045); + --overflow-band: rgba(230, 73, 128, 0.06); --accent: #4f8cff; --chip-bg: #f0f2f5; } From 95bdae3341c91e06259f726df7ac34ea70218f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:48:11 -0300 Subject: [PATCH 8/8] fix(event-monitor): make broken nodes and stale views say so MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every finding here is the same defect class: the tool reporting a healthy-looking state for something that is actually broken, which for a monitor is worse than reporting nothing. The shared HTTP client had no connect or read timeout, so a socket that completes the handshake and then never answers parked the collector in `send()` indefinitely. It reported `reconnecting` forever and `down` was unreachable for that entire failure class, making a blackholed node indistinguishable from one that is merely slow to come up; no retry was ever even logged. A half-open connection where the node stops sending was worse still: the heartbeat kept publishing `connected` at 0.0/s. Both timeouts belong on the client, since `RequestBuilder::timeout` is a total deadline that would tear down a healthy long-lived SSE stream on schedule. The read timeout sits at three of upstream's 15s keep-alives, trading slow convergence to `down` on a blackhole (a few minutes) for tolerance of a loaded node. A geometry change left every open tab permanently blank. The server resets its slot watermark; the frontend's only ever moves up, so a restarted chain's low slots computed an age past the window and were filtered out on arrival. That reads as "the chain died", the one conclusion this tool must not fabricate. The change is now broadcast on `/stream` as `event: geometry` and the tab raises a reload banner instead of emptying quietly. `?demo=1` also grows a clock-skewed node, since it was the one path that could not exercise the negative-offset rendering below. A negative `offset_ms` is documented as legitimate but was clamped to x=0, where an event 30ms before its boundary drew identically to one landing on it, inside the range this panel resolves. There is no room left of the lane labels to plot it, so it still clamps, but keeps its topic fill and gains a ring in the overflow accent: the same treatment the right edge already gives the mirror-image case. A missing `static_dir` started cleanly, logged that the dashboard was ready, and answered `GET /` with 404. It defaults to the relative `web`, so this fired on any invocation from outside `tooling/event-monitor/`; `validate` exists for exactly this class and now checks for `index.html`. The static fallback answered every unmatched path with `200 text/html`, so a typo'd asset surfaced as a JS parse error and a wrong API path lost a frozen contract's only wrong-path signal. This is one page, not a client-routed SPA: `/` is routed explicitly and the rest 404s. Bootstrap passed a malformed spec response straight through. Nothing crashed (`slot_at` floors the divisor, the frontend defaults it), but every dot piled against one edge unexplained, and a `genesis_time` off the wire could overflow the `offset_ms` multiply, a debug-build panic. One range check at the single point geometry enters the process covers both, and covers the config overrides too. A future `genesis_time` stays legal: a devnet is routinely configured before its genesis fires. Three smaller things. The reconnect loop announced `Reconnecting` on its way into every attempt, so a node that stayed down flipped its chip red/amber/red once per cycle; an unchanged state is no longer re-announced, while heartbeats still always publish their refreshed rate. `/api/history` shipped up to HISTORY_MAX_EVENTS that the panels discard against their own 2000-per-node cap, now capped per `(node, topic)` so an attestation flood cannot crowd blocks out of the propagation backfill either. The tooling CI steps pass `--locked`, so the committed Cargo.lock is actually enforced. CONTRACT.md gains `event: geometry` and the 404 rule in §4, the timeout and geometry-validation rationale in §2, and the clamped-negative dot plus the banner in §6. --- .github/workflows/ci.yml | 7 +- tooling/event-monitor/CONTRACT.md | 74 +++++- tooling/event-monitor/README.md | 9 +- tooling/event-monitor/src/collector.rs | 217 ++++++++++++++++-- tooling/event-monitor/src/config.rs | 50 +++- tooling/event-monitor/src/hub.rs | 110 ++++++++- tooling/event-monitor/src/main.rs | 4 +- tooling/event-monitor/src/server.rs | 18 +- tooling/event-monitor/src/timing.rs | 146 +++++++++++- tooling/event-monitor/tests/http_routes.rs | 135 +++++++++++ .../event-monitor/tests/sse_integration.rs | 6 +- tooling/event-monitor/web/app.js | 17 ++ tooling/event-monitor/web/beeswarm.js | 22 +- tooling/event-monitor/web/demo.js | 15 +- tooling/event-monitor/web/index.html | 4 + tooling/event-monitor/web/style.css | 35 +++ 16 files changed, 812 insertions(+), 57 deletions(-) create mode 100644 tooling/event-monitor/tests/http_routes.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a67a6e1..c472fd7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,15 +53,18 @@ jobs: # tests run in this job rather than in `test` because clippy has already # compiled the test targets, and because they need none of that job's # leanSpec fixtures. + # `--locked` so the committed Cargo.lock is actually enforced: without it + # cargo silently resolves and rewrites the lockfile in CI, and a stale or + # missing entry never fails the build. - name: Lint tooling working-directory: tooling/event-monitor run: | cargo fmt --all -- --check - cargo clippy --all-targets -- -D warnings + cargo clippy --locked --all-targets -- -D warnings - name: Test tooling working-directory: tooling/event-monitor - run: cargo test + run: cargo test --locked test: name: Test diff --git a/tooling/event-monitor/CONTRACT.md b/tooling/event-monitor/CONTRACT.md index 564598b0..6b6b478b 100644 --- a/tooling/event-monitor/CONTRACT.md +++ b/tooling/event-monitor/CONTRACT.md @@ -79,6 +79,22 @@ offset_ms = arrival_ms - slot_start_ms // may be negative under clock (`SystemTime::now()` → epoch ms). Config may override `genesis_time` / `ms_per_slot` for offline testing. +Geometry is range-checked wherever it enters the process — both the network fetch +and the config overrides — and the collector refuses to start on a `ms_per_slot` +or `intervals_per_slot` of zero, an absurd `ms_per_slot`, or a `genesis_time` +large enough to overflow the `offset_ms` multiply. None of those crash if let +through; they just pile every dot against one edge with no explanation. A +`genesis_time` in the *future* is legitimate and accepted: a devnet is routinely +configured before its genesis fires. + +**Timeouts.** The shared HTTP client sets `connect_timeout` and `read_timeout` +(the latter comfortably above upstream's 15s SSE keep-alive). They belong on the +client, not on a `RequestBuilder`: a per-request `timeout` is a *total* deadline +and would tear down a healthy long-lived SSE stream on a timer. Without them, a +socket that completes the handshake and then never answers parks the collector in +`send()` forever, so it reports `reconnecting` and can never reach `down` — and a +half-open connection leaves the heartbeat publishing `connected` at 0.0/s. + **What `offset_ms` actually measures.** It is stamped when the *collector* receives the frame, so it includes the collector↔node round trip, not just the node's own event time. One clock is deliberate: it makes propagation deltas @@ -92,10 +108,14 @@ republished when it changes, so a regenerated genesis does not silently invalidate every subsequent `offset_ms`. On a change the collector drops its retained history and slot watermark: those events were computed against the old epoch and their slot numbers belong to a different chain. A failed refresh keeps -the current geometry. Already-loaded browser tabs keep the `ms_per_slot` / -`intervals_per_slot` they fetched from `/api/meta` plus their own client-side -watermark, so **reload the page after a genesis change**; server-computed -`offset_ms` values are correct immediately. +the current geometry. Server-computed `offset_ms` values are correct immediately. + +Already-loaded browser tabs keep the `ms_per_slot` / `intervals_per_slot` they +fetched from `/api/meta` plus their own client-side watermark, which only ever +moves up — so a restarted chain's low slots age out on arrival and every panel +empties, which is indistinguishable from the chain having died. The change is +therefore broadcast on `/stream` as `event: geometry` (§4) and the tab shows a +reload banner instead of going quietly blank. ### Slot plausibility bound @@ -151,12 +171,19 @@ canonical JSON of `{data, sorted participants}` rendered as `0x…` hex is fine. Served by axum on the configured `listen` address. ### `GET /` -Serves `web/index.html` (and `web/` assets under their paths, e.g. `/app.js`, -`/style.css`). Static file serving rooted at the configured `static_dir` -(default `web`). +Serves `web/index.html`, with `web/` assets under their own paths (e.g. +`/app.js`, `/style.css`). Static file serving rooted at the configured +`static_dir` (default `web`, resolved **relative to the working directory**, not +to the config file; startup fails if it holds no `index.html`). + +`/` is routed to `index.html` explicitly and **every unmatched path 404s**. This +is one page, not a client-routed SPA, so a catch-all HTML fallback only hid +mistakes: a typo'd asset path came back `200 text/html` and surfaced as a JS +parse error, and a wrong API path answered `200` instead of telling the caller of +a frozen contract that it had the path wrong. ### `GET /stream` (SSE, `text/event-stream`) -The merged live stream. Two SSE **event names**: +The merged live stream. Three SSE **event names**: - `event: chain` — `data:` is one NormalizedEvent (§3). - `event: status` — `data:` is a node status object: @@ -164,7 +191,18 @@ The merged live stream. Two SSE **event names**: { "node": "node-2", "state": "connected", "events_per_sec": 4.2 } ``` `state` ∈ `"connected" | "reconnecting" | "down"`. Emitted on every state - change and at least every few seconds as a heartbeat with a refreshed rate. + *change* and at least every few seconds as a heartbeat with a refreshed rate. + A state identical to the last one published is not re-announced, so a node + that stays down holds one colour instead of flipping once per retry cycle. +- `event: geometry` — `data:` is the new slot geometry, the same three fields + `/api/meta` publishes: + ```json + { "genesis_time": 1770407233, "ms_per_slot": 4000, "intervals_per_slot": 5 } + ``` + Emitted only when a refresh re-resolves geometry and it **changed** (§2). It + tells an already-loaded tab that its geometry and slot watermark are stale; + the frontend responds with a reload banner. Not retained in history: a tab + opening afterwards reads current geometry from `/api/meta`. Keep-alive comments are sent to hold idle connections open. Best-effort: a slow browser may miss events (same contract as upstream). @@ -189,7 +227,11 @@ One-shot bootstrap the frontend fetches on load: ### `GET /api/history` (JSON) Startup backfill so a freshly-opened dashboard isn't blank. Returns the collector's bounded in-memory ring of recent events (retained for -`history_slots` slots, hard-capped) plus the latest status per node: +`history_slots` slots, hard-capped) plus the latest status per node. The response +is additionally capped per `(node, topic)` at the frontend's own per-node point +cap, since anything past that is megabytes the panels discard on arrival; the cap +is per topic as well as per node so an attestation flood can't crowd the +block/aggregate events out of the propagation panel's backfill. ```json { @@ -278,6 +320,14 @@ saturates at the right edge. The slot boundary is drawn in the overflow accent (`--prop-over`) over the gridlines, and only the far end of the band is labelled (`+ms_per_slot`). +A **negative** `offset_ms` is legitimate (§2: clock skew, or an event stamped +just before its slot boundary) and there is no room left of `LEFT_MARGIN` to plot +it in, so it clamps to `x = 0`. The clamp must not be silent: at the 0.1–0.2s +resolution this panel exists for, an event 30ms before its boundary and one +landing exactly on it are different observations. Such a dot keeps its topic fill +and is **outlined in the overflow accent** (`--prop-over`) — the same accent the +right edge uses for the mirror-image case. + The propagation panel keeps its single fixed `0 … ms_per_slot` scale: it already distinguishes over-one-slot deltas by color (`--prop-over`), so it needs no second region. @@ -297,6 +347,10 @@ the three colors; empty topics show a "Waiting for … events" note. **Status bar**: one chip per node showing `state` (green/amber/red) and `events_per_sec`, driven by `status` events. +**Geometry banner**: on `event: geometry` (§4), a banner states that the view is +stale and offers a reload. Without it the panels simply empty, which reads as +"the chain died" — the one conclusion this tool must not fabricate. + Design language: match the calm, technical look of the approved mockup (thin gridlines, muted labels, the three topic colors above). Must be readable in both light and dark; degrade gracefully if a node is `down` (empty lane, red chip). diff --git a/tooling/event-monitor/README.md b/tooling/event-monitor/README.md index 93bf0807..a4f2cc7c 100644 --- a/tooling/event-monitor/README.md +++ b/tooling/event-monitor/README.md @@ -25,8 +25,13 @@ Two caveats worth knowing before drawing conclusions from a panel: paths are comparable. - The collector re-resolves slot geometry every 60s and drops its retained history if `genesis_time` changes, so a regenerated genesis no longer corrupts - offsets silently. An already-open dashboard still needs a **page reload** after - such a change to pick up the new geometry and reset its own slot watermark. + offsets silently. An already-open dashboard still needs a **page reload** to + pick up the new geometry and reset its own slot watermark, and it now says so: + the change is pushed on the live stream and the tab raises a reload banner + rather than quietly emptying every panel. +- A dot outlined in magenta arrived *before* its slot boundary (negative offset, + i.e. the collector's clock is ahead of the node's). There is no room to plot it + left of the lane labels, so it sits at 0 with the outline as the tell. ## Run diff --git a/tooling/event-monitor/src/collector.rs b/tooling/event-monitor/src/collector.rs index bb6b9aec..a8bbf683 100644 --- a/tooling/event-monitor/src/collector.rs +++ b/tooling/event-monitor/src/collector.rs @@ -18,6 +18,48 @@ const INITIAL_BACKOFF: Duration = Duration::from_millis(250); const MAX_BACKOFF: Duration = Duration::from_secs(10); const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); +/// Bounds the TCP/TLS handshake. Without it, a node whose packets are dropped +/// rather than refused leaves the collector parked in `send()` indefinitely, so +/// it reports `reconnecting` forever and can never reach `down`. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +/// Bounds the wait for *each* read: first the response headers, then every +/// subsequent chunk of the stream. Two failure classes need it, and neither is +/// reachable through the reconnect loop without it: +/// +/// - a socket that completes the handshake and then never answers (a blackhole, +/// or a proxy that accepts and drops) hangs in `send()`, so the collector +/// sits at `reconnecting` and never logs a retry; +/// - a half-open connection where the node stops sending leaves the heartbeat +/// publishing `connected` at 0.0 events/sec indefinitely. +/// +/// Upstream sends an SSE keep-alive comment every 15s (axum's `KeepAlive` +/// default), so silence past three of those is a genuine stall rather than an +/// idle chain. Tolerating a loaded node is worth more here than reaching `down` +/// quickly: a blackholed node costs this timeout per attempt, so it reports +/// `reconnecting` for a few minutes before the ramp settles into `down`. Slow to +/// converge, but no longer wrong forever. +const READ_TIMEOUT: Duration = Duration::from_secs(45); + +/// The shared HTTP client for every collector *and* the slot-geometry fetches. +/// +/// Both timeouts belong on the client rather than on a `RequestBuilder`: +/// `RequestBuilder::timeout` is a *total* deadline covering the response body, +/// which for an SSE subscription means tearing down a perfectly healthy +/// long-lived stream on a timer. `read_timeout` bounds each read instead, which +/// is what "stalled" actually means here. Geometry fetches layer their own +/// stricter total timeout on top (see [`crate::timing`]). +pub fn build_client() -> reqwest::Result { + client_with_timeouts(CONNECT_TIMEOUT, READ_TIMEOUT) +} + +fn client_with_timeouts(connect: Duration, read: Duration) -> reqwest::Result { + reqwest::Client::builder() + .connect_timeout(connect) + .read_timeout(read) + .build() +} + #[derive(Debug, thiserror::Error)] enum CollectorError { #[error("http error: {0}")] @@ -126,11 +168,45 @@ fn now_ms() -> i64 { i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) } -fn node_status(name: &str, state: NodeState, events_per_sec: f64) -> NodeStatus { - NodeStatus { - node: name.to_string(), - state, - events_per_sec, +/// Publishes this node's [`NodeStatus`] on the hub, remembering the last state +/// it sent so a state that hasn't changed isn't re-announced. +struct StatusPublisher { + node: String, + hub: Hub, + last_state: Option, +} + +impl StatusPublisher { + fn new(node: String, hub: Hub) -> Self { + Self { + node, + hub, + last_state: None, + } + } + + /// Publishes unconditionally, carrying `events_per_sec`. For the heartbeat, + /// whose whole point is a refreshed rate even when the state is unchanged. + fn publish(&mut self, state: NodeState, events_per_sec: f64) { + self.last_state = Some(state); + self.hub.publish_status(NodeStatus { + node: self.node.clone(), + state, + events_per_sec, + }); + } + + /// Publishes only when the state differs from the last one sent, at a rate + /// of zero (no state but `Connected` has a meaningful rate). + /// + /// The dedupe is what keeps a permanently-down node's chip still: the + /// reconnect loop passes through `Reconnecting` on its way into every + /// attempt, so without it the chip flipped red → amber → red once per + /// cycle for a node that never came back. + fn publish_change(&mut self, state: NodeState) { + if self.last_state != Some(state) { + self.publish(state, 0.0); + } } } @@ -145,10 +221,11 @@ pub async fn run_collector( client: reqwest::Client, ) { let mut backoff = Backoff::new(); - loop { - hub.publish_status(node_status(&node.name, NodeState::Reconnecting, 0.0)); + let mut status = StatusPublisher::new(node.name.clone(), hub); + status.publish_change(NodeState::Reconnecting); - let attempt = connect_and_stream(&node, &topics, &timing, &hub, &client).await; + loop { + let attempt = connect_and_stream(&node, &topics, &timing, &mut status, &client).await; match &attempt.error { None => tracing::info!(node = %node.name, "SSE stream ended; reconnecting"), Some(err) => { @@ -163,7 +240,7 @@ pub async fn run_collector( } else { NodeState::Reconnecting }; - hub.publish_status(node_status(&node.name, state, 0.0)); + status.publish_change(state); tokio::time::sleep(delay).await; } } @@ -175,7 +252,7 @@ async fn connect_and_stream( node: &NodeConfig, topics: &[String], timing: &watch::Receiver, - hub: &Hub, + status: &mut StatusPublisher, client: &reqwest::Client, ) -> Attempt { let url = format!( @@ -195,9 +272,12 @@ async fn connect_and_stream( }; let mut stream = response.bytes_stream().eventsource(); - hub.publish_status(node_status(&node.name, NodeState::Connected, 0.0)); + status.publish(NodeState::Connected, 0.0); tracing::info!(node = %node.name, %url, "connected to SSE stream"); + // Chain events go straight to the hub; only status flows through the + // publisher, which the heartbeat arm borrows mutably. + let hub = status.hub.clone(); let mut rate = RateTracker::new(); let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); heartbeat.tick().await; // the first tick fires immediately; consume it @@ -213,7 +293,7 @@ async fn connect_and_stream( // Re-read per frame so a geometry refresh takes effect // without tearing down the connection. let geometry = *timing.borrow(); - handle_frame(node, &event, geometry, hub, &mut warned_implausible_slot); + handle_frame(node, &event, geometry, &hub, &mut warned_implausible_slot); } Some(Err(err)) => { let error = Some(CollectorError::Stream(err.to_string())); @@ -226,11 +306,7 @@ async fn connect_and_stream( // Surviving a whole heartbeat interval while streaming is what // marks the session healthy for backoff purposes. healthy = true; - hub.publish_status(node_status( - &node.name, - NodeState::Connected, - rate.rate_and_reset(), - )); + status.publish(NodeState::Connected, rate.rate_and_reset()); } } } @@ -281,6 +357,7 @@ fn handle_frame( #[cfg(test)] mod tests { use super::*; + use crate::hub::HubMessage; #[test] fn backoff_doubles_until_capped() { @@ -351,6 +428,112 @@ mod tests { } } + #[tokio::test] + async fn a_socket_that_accepts_and_never_answers_times_out() { + // The failure class the read timeout exists for: the handshake succeeds, + // so `connect_timeout` never fires, and no byte of the response ever + // arrives. Before the read timeout this parked in `send()` forever, so + // the reconnect loop was never reached and `Down` was unreachable for + // every blackholed node. + // + // Uses a short read timeout so the test is fast; the ramp from a failed + // attempt to `Down` is covered by the backoff tests above. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind blackhole listener"); + let addr = listener.local_addr().expect("blackhole has no local addr"); + // Accept connections and hold them open without writing a single byte. + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((socket, _)) = listener.accept().await { + held.push(socket); + } + }); + + let client = client_with_timeouts(CONNECT_TIMEOUT, Duration::from_millis(200)) + .expect("client should build"); + let err = client + .get(format!("http://{addr}/lean/v0/events?topics=block")) + .send() + .await + .expect_err("a blackholed socket must not resolve to a response"); + assert!(err.is_timeout(), "expected a timeout error, got: {err}"); + } + + #[test] + fn the_shared_client_carries_a_read_timeout() { + // Guards the read timeout against being dropped from the builder, which + // would silently reintroduce the hang above. Only the read timeout is + // checkable this way: reqwest hands `connect_timeout` to the connector + // and never reports it on `Client`'s `Debug`, and a behavioural test for + // it needs an address that blackholes SYNs, which no CI network + // guarantees. + let client = build_client().expect("client should build"); + let debug = format!("{client:?}"); + assert!( + debug.contains(&format!("read_timeout: {READ_TIMEOUT:?}")), + "read timeout missing from the shared client: {debug}" + ); + } + + #[test] + fn the_read_timeout_tolerates_the_upstream_keep_alive_interval() { + // Upstream holds an idle stream open with a keep-alive comment every 15s + // (axum's `KeepAlive` default). A read timeout at or under that would + // tear down healthy connections on an idle chain, which is the failure + // this fix must not trade for the one it removes. + assert!( + READ_TIMEOUT >= Duration::from_secs(45), + "read timeout must leave room for missed keep-alives" + ); + } + + #[test] + fn a_state_that_has_not_changed_is_not_republished() { + // Regression: the reconnect loop announced Reconnecting on its way into + // every attempt, so a node that stayed down flipped its chip + // red -> amber -> red once per cycle. + let hub = Hub::new(64); + let mut rx = hub.subscribe(); + let mut status = StatusPublisher::new("node-2".to_string(), hub); + + status.publish_change(NodeState::Reconnecting); + status.publish_change(NodeState::Reconnecting); + status.publish_change(NodeState::Down); + status.publish_change(NodeState::Reconnecting); + + let mut states = Vec::new(); + while let Ok(HubMessage::Status(status)) = rx.try_recv() { + states.push(status.state); + } + assert_eq!( + states, + vec![ + NodeState::Reconnecting, + NodeState::Down, + NodeState::Reconnecting + ] + ); + } + + #[test] + fn heartbeats_always_publish_so_the_rate_stays_fresh() { + // The dedupe must not swallow heartbeats: they carry a refreshed + // events/sec even though the state is unchanged. + let hub = Hub::new(64); + let mut rx = hub.subscribe(); + let mut status = StatusPublisher::new("node-2".to_string(), hub); + + status.publish(NodeState::Connected, 4.0); + status.publish(NodeState::Connected, 2.0); + + let mut rates = Vec::new(); + while let Ok(HubMessage::Status(status)) = rx.try_recv() { + rates.push(status.events_per_sec); + } + assert_eq!(rates, vec![4.0, 2.0]); + } + #[test] fn rate_tracker_counts_ticks_since_last_reset() { let mut rate = RateTracker::new(); diff --git a/tooling/event-monitor/src/config.rs b/tooling/event-monitor/src/config.rs index c6e07ea2..afb79b76 100644 --- a/tooling/event-monitor/src/config.rs +++ b/tooling/event-monitor/src/config.rs @@ -121,6 +121,20 @@ impl Config { "`nodes` is empty; add at least one [[nodes]] entry", )); } + // `static_dir` is relative to the *working directory*, not to this config + // file, and it defaults to `web` — so every invocation from outside + // `tooling/event-monitor/` hits this. Without the check the collector + // starts, logs that the dashboard is ready, and answers `GET /` with a + // 404 no one is watching for. + let index_html = Path::new(&self.static_dir).join("index.html"); + if !index_html.is_file() { + return Err(invalid(&format!( + "`static_dir` is {:?} but {} does not exist; \ + the path is relative to the working directory, not to this file", + self.static_dir, + index_html.display() + ))); + } Ok(()) } @@ -155,8 +169,18 @@ mod tests { assert!(cfg.ms_per_slot.is_none()); } + /// The real `web/` directory, by absolute path. `validate` checks + /// `static_dir` against the filesystem, so fixtures must not depend on the + /// test process's working directory. + fn web_dir() -> String { + format!("{}/web", env!("CARGO_MANIFEST_DIR")) + } + + /// Parses `toml_str` with a valid absolute `static_dir` prepended, for the + /// fixtures that go on to call [`Config::validate`]. fn parse(toml_str: &str) -> Config { - toml::from_str(toml_str).expect("fixture should parse") + let with_static_dir = format!("static_dir = {:?}\n{toml_str}", web_dir()); + toml::from_str(&with_static_dir).expect("fixture should parse") } #[test] @@ -210,6 +234,30 @@ mod tests { assert!(cfg.validate(Path::new("config.toml")).is_ok()); } + #[test] + fn a_static_dir_without_an_index_html_is_rejected() { + // Regression: this used to start cleanly, log "dashboard ready", and + // answer GET / with a 404. It fires on any invocation from outside + // `tooling/event-monitor/`, since `static_dir` defaults to `web`. + let cfg: Config = toml::from_str( + r#" + listen = "127.0.0.1:8080" + static_dir = "definitely-not-a-real-directory" + + [[nodes]] + name = "node-2" + url = "http://127.0.0.1:5052" + "#, + ) + .expect("fixture should parse"); + let err = cfg.validate(Path::new("config.toml")).unwrap_err(); + assert!(matches!(err, ConfigError::Invalid { .. })); + assert!( + err.to_string().contains("index.html"), + "the error should name the file it looked for: {err}" + ); + } + #[test] fn overrides_and_multiple_nodes_parse() { let toml_str = r#" diff --git a/tooling/event-monitor/src/hub.rs b/tooling/event-monitor/src/hub.rs index 79ef096d..16480878 100644 --- a/tooling/event-monitor/src/hub.rs +++ b/tooling/event-monitor/src/hub.rs @@ -9,6 +9,7 @@ use serde::Serialize; use tokio::sync::broadcast; use crate::model::{NodeStatus, NormalizedEvent}; +use crate::timing::Timing; /// Capacity of the broadcast channel. A slow subscriber (browser) that falls /// behind by more than this many messages will observe a `Lagged` error on @@ -21,8 +22,20 @@ const HUB_CAPACITY: usize = 4096; /// without bound before the slot-age prune catches up. const HISTORY_MAX_EVENTS: usize = 50_000; -/// One message on the hub: either a normalized chain event (`event: chain`) -/// or a node status update (`event: status`) per CONTRACT.md §4. +/// Per `(node, topic)` cap on what one `GET /api/history` response carries. +/// Matches `MAX_POINTS_PER_NODE` in `web/beeswarm.js`, the tightest cap on the +/// consuming side: shipping more than a panel can hold is megabytes the frontend +/// discards on arrival. +/// +/// Keyed per topic as well as per node so an attestation flood can't crowd out +/// the block/aggregate events the propagation panel groups by `id` — there are +/// an order of magnitude more attestations than blocks, so a per-node-only cap +/// would starve exactly the series that panel needs. +const SNAPSHOT_MAX_EVENTS_PER_SERIES: usize = 2_000; + +/// One message on the hub: a normalized chain event (`event: chain`), a node +/// status update (`event: status`), or a slot-geometry change +/// (`event: geometry`) per CONTRACT.md §4. /// /// Chain events are `Arc`-wrapped because every one is fanned out to each /// `/stream` subscriber *and* retained in the history ring; sharing one @@ -31,6 +44,7 @@ const HISTORY_MAX_EVENTS: usize = 50_000; pub enum HubMessage { Chain(Arc), Status(NodeStatus), + Geometry(Timing), } /// Point-in-time backfill payload served by `GET /api/history`: the retained @@ -101,9 +115,26 @@ impl History { /// Cheap because the events are `Arc`s: this copies pointers, not the /// events themselves, which matters because the caller holds the mutex /// across it while every collector is trying to publish. + /// + /// Walks newest-first so the per-series cap keeps the *most recent* + /// [`SNAPSHOT_MAX_EVENTS_PER_SERIES`] events of each `(node, topic)`, then + /// restores the oldest-first order the contract promises. fn snapshot(&self) -> HistorySnapshot { + let mut kept: BTreeMap<(&str, &str), usize> = BTreeMap::new(); + let mut events = Vec::new(); + for event in self.events.iter().rev() { + let count = kept + .entry((event.node.as_str(), event.topic.as_str())) + .or_default(); + if *count >= SNAPSHOT_MAX_EVENTS_PER_SERIES { + continue; + } + *count += 1; + events.push(Arc::clone(event)); + } + events.reverse(); HistorySnapshot { - events: self.events.iter().map(Arc::clone).collect(), + events, status: self.status.values().cloned().collect(), } } @@ -150,6 +181,15 @@ impl Hub { let _ = self.tx.send(HubMessage::Status(status)); } + /// Publishes a slot-geometry change so already-loaded dashboards learn their + /// `ms_per_slot` / `intervals_per_slot` and slot watermark are stale + /// (CONTRACT.md §4). Deliberately *not* retained in history: a tab opening + /// afterwards fetches current geometry from `/api/meta`, so replaying this + /// would only tell it to reload into the state it already has. + pub fn publish_geometry(&self, timing: Timing) { + let _ = self.tx.send(HubMessage::Geometry(timing)); + } + pub fn subscribe(&self) -> broadcast::Receiver { self.tx.subscribe() } @@ -181,9 +221,13 @@ mod tests { use crate::model::NodeState; fn chain_event(node: &str, slot: u64) -> NormalizedEvent { + topic_event(node, "block", slot) + } + + fn topic_event(node: &str, topic: &str, slot: u64) -> NormalizedEvent { NormalizedEvent { node: node.to_string(), - topic: "block".to_string(), + topic: topic.to_string(), slot, arrival_ms: slot as i64 * 4_000, offset_ms: 0, @@ -207,8 +251,64 @@ mod tests { let msg = rx.recv().await.unwrap(); match msg { HubMessage::Status(status) => assert_eq!(status.node, "node-2"), - HubMessage::Chain(_) => panic!("expected a Status message"), + other => panic!("expected a Status message, got {other:?}"), + } + } + + #[tokio::test] + async fn geometry_is_broadcast_but_not_retained_in_history() { + let hub = Hub::new(64); + let mut rx = hub.subscribe(); + let timing = Timing { + genesis_time: 1_770_407_233, + ms_per_slot: 4_000, + intervals_per_slot: 5, + }; + + hub.publish_geometry(timing); + + match rx.recv().await.unwrap() { + HubMessage::Geometry(published) => assert_eq!(published, timing), + other => panic!("expected a Geometry message, got {other:?}"), } + // A tab opening after the change reads current geometry from /api/meta, + // so there is nothing for the backfill to say. + assert!(hub.history_snapshot().events.is_empty()); + } + + #[test] + fn the_history_snapshot_caps_each_node_and_topic_independently() { + // A flood of one topic must not crowd another out of the backfill: the + // propagation panel groups block/aggregate by id, and there are an order + // of magnitude more attestations than blocks. + let hub = Hub::new(u64::MAX); // no slot-age pruning, so the cap is what bites + let flood = SNAPSHOT_MAX_EVENTS_PER_SERIES + 500; + for i in 0..flood { + hub.publish_chain(topic_event("node-0", "attestation", i as u64)); + } + for i in 0..10 { + hub.publish_chain(topic_event("node-0", "block", i as u64)); + } + + let snap = hub.history_snapshot(); + let count = |topic: &str| { + snap.events + .iter() + .filter(|event| event.topic == topic) + .count() + }; + assert_eq!(count("attestation"), SNAPSHOT_MAX_EVENTS_PER_SERIES); + assert_eq!(count("block"), 10); + // Newest kept, oldest dropped, and the contract's oldest-first order + // survives the newest-first walk. + let attestations: Vec = snap + .events + .iter() + .filter(|event| event.topic == "attestation") + .map(|event| event.slot) + .collect(); + assert_eq!(attestations.first(), Some(&500)); + assert_eq!(attestations.last(), Some(&(flood as u64 - 1))); } #[test] diff --git a/tooling/event-monitor/src/main.rs b/tooling/event-monitor/src/main.rs index 117e62a0..455b8f8f 100644 --- a/tooling/event-monitor/src/main.rs +++ b/tooling/event-monitor/src/main.rs @@ -32,7 +32,9 @@ async fn main() -> anyhow::Result<()> { tracing::error!(%err, config = %args.config.display(), "failed to load config"); })?; - let client = reqwest::Client::new(); + // Carries the connect/read timeouts every SSE subscription depends on to + // fail rather than hang; see `collector::build_client`. + let client = collector::build_client()?; let timing = timing::bootstrap(&config.nodes, config.timing_overrides(), &client) .await .inspect_err(|err| tracing::error!(%err, "failed to bootstrap slot timing"))?; diff --git a/tooling/event-monitor/src/server.rs b/tooling/event-monitor/src/server.rs index 0c93deba..fe5141b2 100644 --- a/tooling/event-monitor/src/server.rs +++ b/tooling/event-monitor/src/server.rs @@ -74,8 +74,15 @@ struct AppState { timing: watch::Receiver, } -/// Builds the full axum app: `/stream`, `/api/meta`, `/api/history`, and -/// static file serving (with an `index.html` fallback) rooted at `static_dir`. +/// Builds the full axum app: `/stream`, `/api/meta`, `/api/history`, and static +/// file serving rooted at `static_dir`. +/// +/// `/` is routed to `index.html` explicitly and everything unmatched 404s. The +/// dashboard is one page, not a client-routed SPA, so a catch-all `index.html` +/// fallback bought nothing and cost the two signals worth having: a typo'd asset +/// path came back as `200 text/html` and surfaced as a mystery JS parse error, +/// and a wrong API path answered `200` instead of telling a frozen contract's +/// caller it had the path wrong. pub fn build_router( hub: Hub, meta: MetaConfig, @@ -84,15 +91,13 @@ pub fn build_router( ) -> Router { let state = AppState { hub, meta, timing }; - let index_html = static_dir.join("index.html"); - let serve_dir = ServeDir::new(static_dir).fallback(ServeFile::new(index_html)); - Router::new() .route("/stream", get(stream_handler)) .route("/api/meta", get(meta_handler)) .route("/api/history", get(history_handler)) + .route_service("/", ServeFile::new(static_dir.join("index.html"))) .with_state(state) - .fallback_service(serve_dir) + .fallback_service(ServeDir::new(static_dir)) } async fn meta_handler(State(state): State) -> Json { @@ -117,6 +122,7 @@ async fn stream_handler( match message { Ok(HubMessage::Chain(event)) => Some(Ok(sse_event("chain", &event))), Ok(HubMessage::Status(status)) => Some(Ok(sse_event("status", &status))), + Ok(HubMessage::Geometry(timing)) => Some(Ok(sse_event("geometry", &timing))), // Best-effort stream: a slow browser subscriber that falls // behind simply skips the messages it missed, same contract as // the upstream ethlambda SSE endpoint (CONTRACT.md §4). diff --git a/tooling/event-monitor/src/timing.rs b/tooling/event-monitor/src/timing.rs index d2bda6d3..66904710 100644 --- a/tooling/event-monitor/src/timing.rs +++ b/tooling/event-monitor/src/timing.rs @@ -31,8 +31,26 @@ const FETCH_TIMEOUT: Duration = Duration::from_secs(3); /// has piled up against one edge. const REFRESH_INTERVAL: Duration = Duration::from_secs(60); +/// Upper bound on `MILLISECONDS_PER_SLOT`. One hour per slot is already absurd +/// for a consensus client; anything past it is a malformed spec response, not a +/// configuration choice. +const MAX_MS_PER_SLOT: u64 = 3_600_000; + +/// Upper bound on `genesis_time` (seconds): 2100-01-01. Its only job is to keep +/// `genesis_time * 1000` far inside `i64`, since [`Timing::offset_ms`] does that +/// multiply and an absurd value from a malformed response would otherwise +/// overflow — a panic in a debug build. +/// +/// Note this bounds only the *absurd* side. A `genesis_time` in the near future +/// is legitimate: a devnet is routinely configured before its genesis fires. +const MAX_GENESIS_TIME: u64 = 4_102_444_800; + /// Resolved slot geometry used to compute `offset_ms` for incoming events. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// +/// `Serialize` so a change can be pushed to open dashboards on `/stream` +/// (CONTRACT.md §4 `event: geometry`); the field names are the same three +/// `/api/meta` publishes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] pub struct Timing { pub genesis_time: u64, pub ms_per_slot: u64, @@ -40,6 +58,34 @@ pub struct Timing { } impl Timing { + /// Rejects geometry that would make every downstream number meaningless. + /// + /// Nothing here crashes if it slips through — `slot_at` floors the divisor, + /// the frontend defaults a zero `ms_per_slot` — but the result is every dot + /// piled against one edge with no explanation, which for a monitoring tool + /// is worse than refusing to start. Checked at the one place geometry + /// enters the process, so both the network fetch and the config overrides + /// go through it. + fn validate(&self) -> Result<(), BootstrapError> { + let reject = |reason: String| Err(BootstrapError::ImplausibleGeometry { reason }); + if self.ms_per_slot == 0 || self.ms_per_slot > MAX_MS_PER_SLOT { + return reject(format!( + "ms_per_slot is {}, expected 1..={MAX_MS_PER_SLOT}", + self.ms_per_slot + )); + } + if self.intervals_per_slot == 0 { + return reject("intervals_per_slot is 0; the slot axis divides by it".to_string()); + } + if self.genesis_time > MAX_GENESIS_TIME { + return reject(format!( + "genesis_time is {}, expected 0..={MAX_GENESIS_TIME}", + self.genesis_time + )); + } + Ok(()) + } + /// `offset_ms = arrival_ms - (genesis_time*1000 + slot*ms_per_slot)`. /// /// May be negative: an event can arrive before its nominal slot start @@ -91,6 +137,8 @@ pub enum BootstrapError { "no reachable node provided slot geometry and config did not override genesis_time/ms_per_slot" )] NoTimingSource, + #[error("implausible slot geometry: {reason}")] + ImplausibleGeometry { reason: String }, } struct Fetched { @@ -169,11 +217,13 @@ pub async fn bootstrap( .map(|f| f.intervals_per_slot) .unwrap_or(DEFAULT_INTERVALS_PER_SLOT); - Ok(Timing { + let timing = Timing { genesis_time, ms_per_slot, intervals_per_slot, - }) + }; + timing.validate()?; + Ok(timing) } /// Re-resolves slot geometry every [`REFRESH_INTERVAL`] and republishes it on @@ -191,9 +241,14 @@ pub async fn bootstrap( /// none, and an unreachable node is expected during a rolling restart. /// /// Already-loaded browser tabs keep the `ms_per_slot` / `intervals_per_slot` -/// they fetched from `/api/meta`, and their own client-side slot watermark, so -/// they need a reload after a geometry change. Server-computed `offset_ms` -/// values are correct immediately. +/// they fetched from `/api/meta`, and their own client-side slot watermark, +/// which only ever moves up. A restarted chain's low slots therefore sit below +/// it and every event ages out of the window on arrival, so the tab goes blank — +/// indistinguishable from the chain having died, which is the question this tool +/// exists to answer. The change is broadcast on `/stream` (CONTRACT.md §4 +/// `event: geometry`) so the tab can say so and offer a reload instead of +/// silently emptying. Server-computed `offset_ms` values are correct +/// immediately. pub async fn run_refresher( nodes: Vec, overrides: TimingOverrides, @@ -228,6 +283,9 @@ pub async fn run_refresher( "slot geometry changed; dropping retained history and re-basing offsets" ); hub.reset_history(); + // Announce before switching over, so no subscriber can receive an event + // stamped with the new geometry before being told the geometry moved. + hub.publish_geometry(fresh); // A send error means every receiver is gone, i.e. we are shutting down. let _ = timing_tx.send(fresh); } @@ -293,6 +351,82 @@ mod tests { assert_eq!(t.slot_at(0), 0); } + #[test] + fn valid_geometry_passes_validation() { + assert!(timing().validate().is_ok()); + // Genesis in the future is legitimate: a devnet is configured before it + // fires, and until then every offset is simply negative. + let future = Timing { + genesis_time: MAX_GENESIS_TIME, + ..timing() + }; + assert!(future.validate().is_ok()); + } + + #[test] + fn zero_or_absurd_ms_per_slot_is_rejected() { + // A malformed spec response used to pass straight through: nothing + // crashed, but every dot piled against one edge with no explanation. + for ms_per_slot in [0, MAX_MS_PER_SLOT + 1] { + let t = Timing { + ms_per_slot, + ..timing() + }; + assert!( + matches!( + t.validate(), + Err(BootstrapError::ImplausibleGeometry { .. }) + ), + "ms_per_slot {ms_per_slot} should be rejected" + ); + } + } + + #[test] + fn zero_intervals_per_slot_is_rejected() { + // The beeswarm's axis split divides by it. + let t = Timing { + intervals_per_slot: 0, + ..timing() + }; + assert!(matches!( + t.validate(), + Err(BootstrapError::ImplausibleGeometry { .. }) + )); + } + + #[test] + fn a_genesis_time_that_would_overflow_the_offset_math_is_rejected() { + // `offset_ms` multiplies genesis_time by 1000 as an i64, which panics in + // a debug build on an absurd value off the wire. + let t = Timing { + genesis_time: u64::MAX, + ..timing() + }; + assert!(matches!( + t.validate(), + Err(BootstrapError::ImplausibleGeometry { .. }) + )); + } + + #[test] + fn the_largest_accepted_genesis_time_does_not_overflow_offset_ms() { + // Pins the bound to what it exists for: the accepted maximum, at the + // largest accepted slot duration, must stay inside i64. + let t = Timing { + genesis_time: MAX_GENESIS_TIME, + ms_per_slot: MAX_MS_PER_SLOT, + intervals_per_slot: 5, + }; + assert!(t.validate().is_ok()); + assert!( + MAX_GENESIS_TIME.checked_mul(1000).unwrap() < i64::MAX as u64, + "MAX_GENESIS_TIME must keep genesis_time * 1000 inside i64" + ); + // Non-panicking in a debug build is the assertion. + let _ = t.offset_ms(0, 0); + } + #[test] fn slot_at_survives_a_bogus_zero_ms_per_slot() { // `ms_per_slot` comes off the wire, so a malformed spec response must diff --git a/tooling/event-monitor/tests/http_routes.rs b/tooling/event-monitor/tests/http_routes.rs new file mode 100644 index 00000000..2344b510 --- /dev/null +++ b/tooling/event-monitor/tests/http_routes.rs @@ -0,0 +1,135 @@ +//! Integration test: the routing table built by [`server::build_router`]. +//! +//! Asserts the two things a single-page dashboard needs from it — `/` serves +//! `index.html`, and anything unmatched 404s rather than being answered with +//! `index.html`. A catch-all HTML fallback turns a typo'd asset path into a +//! mystery JS parse error and a typo'd API path into a silent `200`. + +use std::path::Path; + +use tokio::net::TcpListener; +use tokio::sync::watch; + +use event_monitor::config::{Config, NodeConfig}; +use event_monitor::hub::Hub; +use event_monitor::server::{self, MetaConfig}; +use event_monitor::timing::Timing; + +/// The real `web/` directory, by absolute path, so the test does not depend on +/// the test process's working directory. +fn web_dir() -> String { + format!("{}/web", env!("CARGO_MANIFEST_DIR")) +} + +fn test_config() -> Config { + let toml_str = format!( + r#" + listen = "127.0.0.1:0" + static_dir = {:?} + + [[nodes]] + name = "node-2" + url = "http://127.0.0.1:5052" + "#, + web_dir() + ); + toml::from_str(&toml_str).expect("fixture should parse") +} + +/// Serves the real router on an ephemeral port. Returns its base URL plus the +/// geometry sender, which the caller holds so the channel stays open for the +/// lifetime of the test. +async fn spawn_server() -> (String, watch::Sender) { + let config = test_config(); + let (timing_tx, timing_rx) = watch::channel(Timing { + genesis_time: 1_770_407_233, + ms_per_slot: 4_000, + intervals_per_slot: 5, + }); + + let app = server::build_router( + Hub::new(64), + MetaConfig::new(&config), + timing_rx, + Path::new(&web_dir()), + ); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind test server"); + let addr = listener.local_addr().expect("server has no local addr"); + tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("test server crashed"); + }); + (format!("http://{addr}"), timing_tx) +} + +#[tokio::test] +async fn known_routes_answer_and_unknown_ones_404() { + let (base, _timing_tx) = spawn_server().await; + let client = reqwest::Client::new(); + + let get = |path: &str| { + let url = format!("{base}{path}"); + let client = client.clone(); + async move { client.get(url).send().await.expect("request failed") } + }; + + // `/` is the dashboard. + let index = get("/").await; + assert_eq!(index.status(), 200); + let content_type = index + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + assert!( + content_type.starts_with("text/html"), + "unexpected content-type: {content_type}" + ); + assert!(index.text().await.unwrap().contains("event-monitor")); + + // Real assets are served from `static_dir`. + assert_eq!(get("/app.js").await.status(), 200); + assert_eq!(get("/style.css").await.status(), 200); + + // The JSON endpoints answer on their exact paths. + assert_eq!(get("/api/meta").await.status(), 200); + assert_eq!(get("/api/history").await.status(), 200); + + // Regression: each of these used to come back `200 text/html`, hiding the + // typo behind a JS parse error or a JSON decode failure. + for wrong in ["/api/typo", "/nonexistent.js", "/api/meta/extra", "/nope"] { + assert_eq!(get(wrong).await.status(), 404, "{wrong} should 404"); + } +} + +#[tokio::test] +async fn meta_reports_the_live_slot_geometry() { + let (base, _timing_tx) = spawn_server().await; + let meta: serde_json::Value = reqwest::get(format!("{base}/api/meta")) + .await + .expect("request failed") + .json() + .await + .expect("meta should be JSON"); + + assert_eq!(meta["genesis_time"], 1_770_407_233_u64); + assert_eq!(meta["ms_per_slot"], 4_000); + assert_eq!(meta["intervals_per_slot"], 5); + assert_eq!(meta["nodes"][0]["name"], "node-2"); +} + +#[tokio::test] +async fn node_config_endpoint_joins_paths_without_doubling_slashes() { + let node = NodeConfig { + name: "node-2".to_string(), + url: "http://127.0.0.1:5052/".to_string(), + }; + assert_eq!( + node.endpoint("/lean/v0/events"), + "http://127.0.0.1:5052/lean/v0/events" + ); +} diff --git a/tooling/event-monitor/tests/sse_integration.rs b/tooling/event-monitor/tests/sse_integration.rs index f5637dc3..82c89922 100644 --- a/tooling/event-monitor/tests/sse_integration.rs +++ b/tooling/event-monitor/tests/sse_integration.rs @@ -48,7 +48,7 @@ async fn collect_chain_events( while collected.len() < want { match rx.recv().await.expect("hub sender dropped unexpectedly") { HubMessage::Chain(event) => collected.push(event), - HubMessage::Status(_) => continue, + HubMessage::Status(_) | HubMessage::Geometry(_) => continue, } } }) @@ -83,7 +83,7 @@ async fn collector_normalizes_frames_from_a_live_sse_server() { }); let hub = Hub::new(64); let mut rx = hub.subscribe(); - let client = reqwest::Client::new(); + let client = event_monitor::collector::build_client().expect("client should build"); let topics = vec![ "block".to_string(), @@ -143,7 +143,7 @@ async fn collector_publishes_connected_status_on_success() { }); let hub = Hub::new(64); let mut rx = hub.subscribe(); - let client = reqwest::Client::new(); + let client = event_monitor::collector::build_client().expect("client should build"); let collector_handle = tokio::spawn(run_collector( node, diff --git a/tooling/event-monitor/web/app.js b/tooling/event-monitor/web/app.js index 1032ed84..54cc6b81 100644 --- a/tooling/event-monitor/web/app.js +++ b/tooling/event-monitor/web/app.js @@ -61,6 +61,22 @@ function createLiveSource() { return es; } +// A geometry change (CONTRACT.md §4 `event: geometry`) invalidates this tab +// wholesale: `ms_per_slot` / `intervals_per_slot` came from the /api/meta fetch +// at load, and the panels' slot watermark only ever moves up, so a restarted +// chain's low slots age out on arrival and every panel empties. That is exactly +// indistinguishable from the chain having died, so say so rather than letting +// the view go quietly blank. +function wireGeometryBanner(source) { + const banner = document.getElementById("geometry-banner"); + if (!banner) return; + const reload = document.getElementById("geometry-reload"); + if (reload) reload.addEventListener("click", () => location.reload()); + source.addEventListener("geometry", () => { + banner.hidden = false; + }); +} + async function fetchMeta() { const res = await fetch("/api/meta"); if (!res.ok) throw new Error(`GET /api/meta failed: ${res.status}`); @@ -127,6 +143,7 @@ async function boot() { }); wireWindowControl(meta, [beeswarm, propagation]); + wireGeometryBanner(source); const ingest = (ev) => { beeswarm.addEvent(ev); diff --git a/tooling/event-monitor/web/beeswarm.js b/tooling/event-monitor/web/beeswarm.js index 199f2e89..7c24f594 100644 --- a/tooling/event-monitor/web/beeswarm.js +++ b/tooling/event-monitor/web/beeswarm.js @@ -21,6 +21,9 @@ const LEFT_MARGIN = 92; const RIGHT_MARGIN = 16; const JITTER_RANGE = 11; // px, +/- around the lane center const DOT_RADIUS = 2.6; +// Outline width for a dot whose real offset was negative and so had to clamp to +// x=0; see addEvent(). +const EARLY_RING_WIDTH = 1.4; // Fade older slots non-linearly: the newest slot is fully opaque and each // older slot drops geometrically toward a faint floor (front-loaded, so @@ -117,10 +120,18 @@ export function createBeeswarm({ canvas, legendEl, noteEl, meta }) { // Clamp to the full two-slot span, not to one slot: the overflow band is // what makes a late arrival distinguishable from an on-boundary one. const offsetMs = Math.min(Math.max(ev.offset_ms, 0), msPerSlot * 2); + // Negative offsets are real (CONTRACT.md §2: clock skew, or an event + // stamped just before its slot boundary) and there is no room left of + // LEFT_MARGIN to plot them in, so they clamp to x=0 like any other early + // arrival. Flagging them keeps the clamp honest: at the 0.1-0.2s resolution + // this tool exists for, an event 30ms *before* its boundary and one landing + // exactly on it must not be the same dot. Marked in the overflow accent, the + // same treatment the right edge already gets. const seed = `${ev.node}|${ev.topic}|${ev.slot}|${ev.id ?? ""}|${ev.validator_id ?? ""}|${ev.arrival_ms}`; arr.push({ topic: ev.topic, offsetMs, + early: ev.offset_ms < 0, slot: ev.slot, jitter: hashJitter(seed) * JITTER_RANGE, }); @@ -242,6 +253,13 @@ export function createBeeswarm({ canvas, legendEl, noteEl, meta }) { ctx.beginPath(); ctx.arc(x, y + pt.jitter, DOT_RADIUS, 0, Math.PI * 2); ctx.fill(); + // Ring, not a recolor: a clamped-negative offset still needs to say + // which topic it was. + if (pt.early) { + ctx.strokeStyle = theme.overflowEdge; + ctx.lineWidth = EARLY_RING_WIDTH; + ctx.stroke(); + } } }); ctx.globalAlpha = 1; @@ -294,7 +312,9 @@ export function createBeeswarm({ canvas, legendEl, noteEl, meta }) { `Showing the last ${windowSlots} slots. Older slots fade out, then drop. ` + `Past the slot boundary the axis compresses the next full slot into the ` + `tinted band, so late arrivals stay distinguishable; beyond that they ` + - `saturate at the right edge.`; + `saturate at the right edge. A dot outlined in the same accent arrived ` + + `before its slot boundary (negative offset, i.e. clock skew) and is ` + + `clamped to 0.`; } } diff --git a/tooling/event-monitor/web/demo.js b/tooling/event-monitor/web/demo.js index 8f22cdb7..09e3e848 100644 --- a/tooling/event-monitor/web/demo.js +++ b/tooling/event-monitor/web/demo.js @@ -12,6 +12,12 @@ const INTERVALS_PER_SLOT = 5; const WINDOW_SLOTS = 30; const VALIDATOR_COUNT = 16; const SLOW_NODE = "node-5"; // consistently later, so the panels have something to show +// A node whose clock runs ahead of the collector's, so some of its blocks are +// stamped before their own slot boundary (CONTRACT.md §2: offset_ms may be +// negative). Rare but real, and the only way ?demo=1 exercises the clamped-early +// rendering in beeswarm.js. +const SKEWED_NODE = "node-3"; +const SKEWED_BLOCK_FRACTION = 0.25; export function createDemoMeta() { return { @@ -71,9 +77,10 @@ export function createDemoSource(meta) { const blockRoot = fakeRoot(`block:${currentSlot}`); const aggId = fakeRoot(`aggregate:${currentSlot}`); - // blocks: ~0.6s into the slot + // blocks: ~0.6s into the slot, except for the skewed node's early ones for (const node of meta.nodes) { - const offset = 600 + nodeDelayMs(node.name); + const early = node.name === SKEWED_NODE && Math.random() < SKEWED_BLOCK_FRACTION; + const offset = early ? -(20 + Math.random() * 120) : 600 + nodeDelayMs(node.name); schedule( () => emitChain({ @@ -86,7 +93,9 @@ export function createDemoSource(meta) { validator_id: null, participants: null, }), - offset + // A negative offset means "before the boundary"; there is no earlier + // time to fire at, so emit immediately. + Math.max(offset, 0) ); } diff --git a/tooling/event-monitor/web/index.html b/tooling/event-monitor/web/index.html index b976457c..413a6e40 100644 --- a/tooling/event-monitor/web/index.html +++ b/tooling/event-monitor/web/index.html @@ -13,6 +13,10 @@

event-monitor

+
diff --git a/tooling/event-monitor/web/style.css b/tooling/event-monitor/web/style.css index e9edd37b..f84af170 100644 --- a/tooling/event-monitor/web/style.css +++ b/tooling/event-monitor/web/style.css @@ -149,6 +149,41 @@ h2 { padding: 3px 10px; } +/* Shown when the collector reports a slot-geometry change: this tab's geometry + and slot watermark are stale from that point on and every panel will empty + (see wireGeometryBanner in app.js). Louder than .conn-banner because a + reconnect resolves itself and this needs a click. */ +/* A class-level `display` outranks the UA stylesheet's `[hidden] {display:none}`, + so the attribute has to be honoured explicitly or the banner is always on. */ +.geometry-banner[hidden] { + display: none; +} + +.geometry-banner { + display: inline-flex; + align-items: center; + gap: 8px; + font-size: 12px; + color: var(--status-down); + border: 1px solid var(--status-down); + border-radius: 999px; + padding: 3px 4px 3px 10px; +} + +.geometry-reload { + font: inherit; + color: var(--panel-bg); + background: var(--status-down); + border: none; + border-radius: 999px; + padding: 2px 10px; + cursor: pointer; +} + +.geometry-reload:hover { + filter: brightness(1.1); +} + .window-control { display: inline-flex; align-items: center;