Skip to content
This repository was archived by the owner on Nov 21, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions benchmarks/experiments/export_web.py
Original file line number Diff line number Diff line change
Expand Up @@ -5353,6 +5353,44 @@ def _censored_entries(table):
return out, marks


def _phase_split_notes(table):
"""Arms on a split table that print no `index s`, and why (DECISIONS #132).

Read from fairness_check's PHASE_SPLIT_DECLARED and PHASE_SPLIT_DISCLOSED,
the maps the F14c gate reads, so the page and the gate cannot come to
disagree. Declared arms have no separate index phase to time; disclosed
arms have one and time it inside the total, which the sentence says so a
reader does not read their blank `index s` as a fast build.
"""
lane_wl = _TABLE_LANE.get(table.get("id"))
cols = set(table.get("columns") or [])
if not lane_wl or "index s" not in cols:
return []
try:
import fairness_check
declared = fairness_check.PHASE_SPLIT_DECLARED
disclosed = fairness_check.PHASE_SPLIT_DISCLOSED
except Exception: # noqa: BLE001 - the page must still build without it
return []
lane = lane_wl[0]
shown = {str(e.get("backend")) for e in table.get("entries", [])}
out = []
dec = sorted({display_name(b) for (ln, b) in declared if ln == lane} & shown)
if dec:
out.append(f"{_join_and(dec)} print{'' if len(dec) > 1 else 's'} no index time "
"because there is no separate index phase to time: each builds its "
"index during or before the load, or has none.")
dis = sorted({display_name(b) for (ln, b) in disclosed if ln == lane} & shown)
if dis:
out.append(f"{_join_and(dis)} build{'' if len(dis) > 1 else 's'} {'their' if len(dis) > 1 else 'its'} "
"index after the load and time it inside the total, so "
f"{'their' if len(dis) > 1 else 'its'} total includes the index build "
"and the table has no separate index figure for "
f"{'them' if len(dis) > 1 else 'it'}; the next measurement splits "
f"{'them' if len(dis) > 1 else 'it'}.")
return out


def _split_note(table):
"""The two phases are INSIDE the total, and do not always fill it.

Expand Down Expand Up @@ -6134,6 +6172,7 @@ def _finish_table(table: dict) -> dict:
+ _counts_note(table.get("id"), table.get("entries", []))
+ _index_note(table.get("id"))
+ _split_note(table)
+ _phase_split_notes(table)
+ _censored_notes(table.get("id"))
+ _query_budget_notes(table.get("id"))
+ _zero_growth_notes(table.get("id"))
Expand Down
33 changes: 30 additions & 3 deletions benchmarks/experiments/fairness_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -1012,6 +1012,28 @@ def check_index_decisions(rows=None):
return bad


# DECISIONS #132. F14c's dense lane, read per adapter rather than per row: the
# rows were measured before the split reached every arm, and a mid-campaign
# change would split the lane's rows, so the October answer is a declaration
# for the arms whose index work genuinely has no separate phase, and a
# disclosure for the two that do have one and time it inside one timer. Both
# maps are read by export_web as well, so the page and this gate state the
# same thing. The disclosed arms gain the split at the re-pin (CAMPAIGN
# section 7), after which their entries here go.
PHASE_SPLIT_DECLARED = {
("l3d", "chroma_dense"): "builds its HNSW index during the load (`add`)",
("l3d", "qdrant_dense"): "builds its HNSW index in the background as points arrive, and the wait for it is inside the build timer",
("l3d", "qdrant_dense_int8"): "builds its HNSW index in the background as points arrive, and the wait for it is inside the build timer",
("l3d", "sqlite_vec_dense"): "has no separate index: `vec0` is a brute-force table",
("l3d", "sqlite_vec_dense_int8"): "has no separate index: `vec0` is a brute-force table",
("l3d", "surrealdb_dense"): "defines its index before the load, which builds it as rows arrive",
}
PHASE_SPLIT_DISCLOSED = {
("l3d", "duckdb_vss_dense"): "builds its HNSW index after the load, inside the same timer",
("l3d", "arangodb_dense"): "builds its vector index after the load, inside the same timer",
}


def check_phase_split(rows):
"""F14c: if one arm on a lane splits ingest from index, every arm must.

Expand Down Expand Up @@ -1047,7 +1069,7 @@ def check_phase_split(rows):
st[0] += 1
if r.get("index_s") not in (None, "", "None"):
st[1] += 1
if r.get("index_before_load"):
if r.get("index_before_load") or (lane, r.get("backend")) in PHASE_SPLIT_DECLARED:
st[2] += 1
bad = 0
for lane in sorted(lanes, key=str):
Expand All @@ -1057,15 +1079,20 @@ def check_phase_split(rows):
print(f" n/a {lane}: no arm splits, so there is nothing to be "
f"inconsistent about ({len(arms)} arm(s))")
continue
missing = sorted(b for b, st in arms.items() if not st[1] and not st[2])
disclosed = sorted(b for b, st in arms.items()
if not st[1] and not st[2] and (lane, b) in PHASE_SPLIT_DISCLOSED)
missing = sorted(b for b, st in arms.items()
if not st[1] and not st[2] and (lane, b) not in PHASE_SPLIT_DISCLOSED)
if missing:
print(f" FAIL {lane}: {len(split)} of {len(arms)} arms split ingest "
f"from index; these do not, and declare no reason: "
f"{', '.join(missing)}")
bad += len(missing)
else:
declared = [b for b, st in arms.items() if not st[1] and st[2]]
extra = f", {len(declared)} declared index-before-load" if declared else ""
extra = f", {len(declared)} declared with no separate index phase" if declared else ""
if disclosed:
extra += f", {len(disclosed)} disclosed on the page ({', '.join(disclosed)})"
print(f" ok {lane}: all {len(arms)} arm(s) accounted for "
f"({len(split)} split{extra})")
return bad
Expand Down
Loading